diff options
6 files changed, 296 insertions, 40 deletions
diff --git a/core/java/android/hardware/soundtrigger/IRecognitionStatusCallback.aidl b/core/java/android/hardware/soundtrigger/IRecognitionStatusCallback.aidl index 0bf4f25314e1..b1a02c1d62fb 100644 --- a/core/java/android/hardware/soundtrigger/IRecognitionStatusCallback.aidl +++ b/core/java/android/hardware/soundtrigger/IRecognitionStatusCallback.aidl @@ -26,10 +26,8 @@ oneway interface IRecognitionStatusCallback { * Called when the keyphrase is spoken. * * @param data Optional trigger audio data, if it was requested and is available. - * TODO: See if the data being passed in works well, if not use shared memory. - * This *MUST* not exceed 100K. */ - void onDetected(in SoundTrigger.RecognitionEvent recognitionEvent); + void onDetected(in SoundTrigger.KeyphraseRecognitionEvent recognitionEvent); /** * Called when the detection for the associated keyphrase stops. */ diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.aidl b/core/java/android/hardware/soundtrigger/SoundTrigger.aidl index 9adc6bc6e88b..e16ea71d9b65 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.aidl +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.aidl @@ -18,8 +18,8 @@ package android.hardware.soundtrigger; parcelable SoundTrigger.ConfidenceLevel; parcelable SoundTrigger.Keyphrase; +parcelable SoundTrigger.KeyphraseRecognitionEvent; parcelable SoundTrigger.KeyphraseRecognitionExtra; parcelable SoundTrigger.KeyphraseSoundModel; parcelable SoundTrigger.ModuleProperties; parcelable SoundTrigger.RecognitionConfig; -parcelable SoundTrigger.RecognitionEvent;
\ No newline at end of file diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java index 3e8436851638..4498789cd69c 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.java +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java @@ -361,13 +361,13 @@ public class SoundTrigger { public void writeToParcel(Parcel dest, int flags) { dest.writeString(uuid.toString()); dest.writeBlob(data); - dest.writeTypedArray(keyphrases, 0); + dest.writeTypedArray(keyphrases, flags); } @Override public String toString() { return "KeyphraseSoundModel [keyphrases=" + Arrays.toString(keyphrases) + ", uuid=" - + uuid + ", type=" + type + ", data? " + (data != null) + "]"; + + uuid + ", type=" + type + ", data=" + (data == null ? 0 : data.length) + "]"; } } @@ -504,6 +504,15 @@ public class SoundTrigger { return false; return true; } + + @Override + public String toString() { + return "RecognitionEvent [status=" + status + ", soundModelHandle=" + soundModelHandle + + ", captureAvailable=" + captureAvailable + ", captureSession=" + + captureSession + ", captureDelayMs=" + captureDelayMs + + ", capturePreambleMs=" + capturePreambleMs + + ", data=" + (data == null ? 0 : data.length) + "]"; + } } /** @@ -551,7 +560,7 @@ public class SoundTrigger { @Override public void writeToParcel(Parcel dest, int flags) { dest.writeByte((byte) (captureRequested ? 1 : 0)); - dest.writeTypedArray(keyphrases, 0); + dest.writeTypedArray(keyphrases, flags); dest.writeBlob(data); } @@ -563,7 +572,8 @@ public class SoundTrigger { @Override public String toString() { return "RecognitionConfig [captureRequested=" + captureRequested + ", keyphrases=" - + Arrays.toString(keyphrases) + ", data? " + (data != null) + "]"; + + Arrays.toString(keyphrases) + + ", data=" + (data == null ? 0 : data.length) + "]"; } } @@ -611,6 +621,37 @@ public class SoundTrigger { public int describeContents() { return 0; } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + confidenceLevel; + result = prime * result + userId; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ConfidenceLevel other = (ConfidenceLevel) obj; + if (confidenceLevel != other.confidenceLevel) + return false; + if (userId != other.userId) + return false; + return true; + } + + @Override + public String toString() { + return "ConfidenceLevel [userId=" + userId + + ", confidenceLevel=" + confidenceLevel + "]"; + } } /** @@ -657,13 +698,47 @@ public class SoundTrigger { public void writeToParcel(Parcel dest, int flags) { dest.writeInt(id); dest.writeInt(recognitionModes); - dest.writeTypedArray(confidenceLevels, 0); + dest.writeTypedArray(confidenceLevels, flags); } @Override public int describeContents() { return 0; } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(confidenceLevels); + result = prime * result + id; + result = prime * result + recognitionModes; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + KeyphraseRecognitionExtra other = (KeyphraseRecognitionExtra) obj; + if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) + return false; + if (id != other.id) + return false; + if (recognitionModes != other.recognitionModes) + return false; + return true; + } + + @Override + public String toString() { + return "KeyphraseRecognitionExtra [id=" + id + ", recognitionModes=" + recognitionModes + + ", confidenceLevels=" + Arrays.toString(confidenceLevels) + "]"; + } } /** @@ -676,7 +751,7 @@ public class SoundTrigger { /** Additional data available for each recognized key phrases in the model */ public final boolean keyphraseInCapture; - KeyphraseRecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, + public KeyphraseRecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, byte[] data, boolean keyphraseInCapture, KeyphraseRecognitionExtra[] keyphraseExtras) { super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, @@ -684,6 +759,86 @@ public class SoundTrigger { this.keyphraseInCapture = keyphraseInCapture; this.keyphraseExtras = keyphraseExtras; } + + public static final Parcelable.Creator<KeyphraseRecognitionEvent> CREATOR + = new Parcelable.Creator<KeyphraseRecognitionEvent>() { + public KeyphraseRecognitionEvent createFromParcel(Parcel in) { + return KeyphraseRecognitionEvent.fromParcel(in); + } + + public KeyphraseRecognitionEvent[] newArray(int size) { + return new KeyphraseRecognitionEvent[size]; + } + }; + + private static KeyphraseRecognitionEvent fromParcel(Parcel in) { + int status = in.readInt(); + int soundModelHandle = in.readInt(); + boolean captureAvailable = in.readByte() == 1; + int captureSession = in.readInt(); + int captureDelayMs = in.readInt(); + int capturePreambleMs = in.readInt(); + byte[] data = in.readBlob(); + boolean keyphraseInCapture = in.readByte() == 1; + KeyphraseRecognitionExtra[] keyphraseExtras = + in.createTypedArray(KeyphraseRecognitionExtra.CREATOR); + return new KeyphraseRecognitionEvent(status, soundModelHandle, captureAvailable, + captureSession, captureDelayMs, capturePreambleMs, data, keyphraseInCapture, + keyphraseExtras); + } + + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeInt(status); + dest.writeInt(soundModelHandle); + dest.writeByte((byte) (captureAvailable ? 1 : 0)); + dest.writeInt(captureSession); + dest.writeInt(captureDelayMs); + dest.writeInt(capturePreambleMs); + dest.writeBlob(data); + dest.writeByte((byte) (keyphraseInCapture ? 1 : 0)); + dest.writeTypedArray(keyphraseExtras, flags); + } + + @Override + public int describeContents() { + return 0; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + Arrays.hashCode(keyphraseExtras); + result = prime * result + (keyphraseInCapture ? 1231 : 1237); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; + KeyphraseRecognitionEvent other = (KeyphraseRecognitionEvent) obj; + if (!Arrays.equals(keyphraseExtras, other.keyphraseExtras)) + return false; + if (keyphraseInCapture != other.keyphraseInCapture) + return false; + return true; + } + + @Override + public String toString() { + return "KeyphraseRecognitionEvent [keyphraseExtras=" + Arrays.toString(keyphraseExtras) + + ", keyphraseInCapture=" + keyphraseInCapture + ", status=" + status + + ", soundModelHandle=" + soundModelHandle + ", captureAvailable=" + + captureAvailable + ", captureSession=" + captureSession + ", captureDelayMs=" + + captureDelayMs + ", capturePreambleMs=" + capturePreambleMs + + ", data=" + (data == null ? 0 : data.length) + "]"; + } } /** diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java index a8c08d55c003..e40ece3955fd 100644 --- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java +++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java @@ -23,6 +23,7 @@ import android.hardware.soundtrigger.KeyphraseMetadata; import android.hardware.soundtrigger.SoundTrigger; import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; import android.hardware.soundtrigger.SoundTrigger.Keyphrase; +import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; @@ -133,11 +134,6 @@ public class AlwaysOnHotwordDetector { private final Object mLock = new Object(); private final Handler mHandler; - /** - * Indicates if there is a sound model enrolled for the keyphrase, - * derived from the model management service (IVoiceInteractionManagerService). - */ - private boolean mIsEnrolledForDetection; private int mAvailability = STATE_NOT_READY; /** @@ -381,10 +377,10 @@ public class AlwaysOnHotwordDetector { } @Override - public void onDetected(RecognitionEvent recognitionEvent) { + public void onDetected(KeyphraseRecognitionEvent event) { Slog.i(TAG, "onDetected"); Message message = Message.obtain(mHandler, MSG_HOTWORD_DETECTED); - message.obj = recognitionEvent.data; + message.obj = event.data; message.sendToTarget(); } @@ -436,7 +432,6 @@ public class AlwaysOnHotwordDetector { Slog.d(TAG, "Hotword availability changed from " + mAvailability + " -> " + availability); } - mIsEnrolledForDetection = enrolled; mAvailability = availability; notifyStateChangedLocked(); } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoundTriggerHelper.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoundTriggerHelper.java index 1e4a518c1c16..cae7ca51d67e 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoundTriggerHelper.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoundTriggerHelper.java @@ -19,6 +19,8 @@ package com.android.server.voiceinteraction; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; import android.hardware.soundtrigger.SoundTrigger.Keyphrase; +import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; +import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; @@ -40,8 +42,6 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { static final String TAG = "SoundTriggerHelper"; // TODO: Set to false. static final boolean DBG = true; - // TODO: Remove this. - static final int TEMP_KEYPHRASE_ID = 100; /** * Return codes for {@link #startRecognition(int, KeyphraseSoundModel, @@ -117,7 +117,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { } if (mCurrentSoundModelHandle != INVALID_SOUND_MODEL_HANDLE) { - Slog.w(TAG, "Canceling previous recognition"); + Slog.w(TAG, "Unloading previous sound model"); // TODO: Inspect the return codes here. mModule.unloadSoundModel(mCurrentSoundModelHandle); mCurrentSoundModelHandle = INVALID_SOUND_MODEL_HANDLE; @@ -127,6 +127,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { // Notify them that it was stopped. IRecognitionStatusCallback oldListener = mActiveListeners.get(keyphraseId); if (oldListener != null && oldListener.asBinder() != listener.asBinder()) { + Slog.w(TAG, "Canceling previous recognition"); try { oldListener.onDetectionStopped(); } catch (RemoteException e) { @@ -221,33 +222,52 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { //---- SoundTrigger.StatusListener methods @Override public void onRecognition(RecognitionEvent event) { - // Check which keyphrase triggered, and fire the appropriate event. - // TODO: Get the keyphrase out of the event and fire events on it. - // For now, as a nasty workaround, we fire all events to the listener for - // keyphrase with TEMP_KEYPHRASE_ID. - IRecognitionStatusCallback listener = null; - synchronized(this) { - // TODO: The keyphrase should come from the recognition event - // as it may be for a different keyphrase than the current one. - listener = mActiveListeners.get(TEMP_KEYPHRASE_ID); - } - if (listener == null) { - Slog.w(TAG, "received onRecognition event without any listener for it"); + if (event == null) { + Slog.w(TAG, "Invalid recognition event!"); return; } + if (DBG) Slog.d(TAG, "onRecognition: " + event); switch (event.status) { - case SoundTrigger.RECOGNITION_STATUS_SUCCESS: + // Fire aborts/failures to all listeners since it's not tied to a keyphrase. + case SoundTrigger.RECOGNITION_STATUS_ABORT: // fall-through + case SoundTrigger.RECOGNITION_STATUS_FAILURE: try { - listener.onDetected(event); + synchronized (this) { + for (int i = 0; i < mActiveListeners.size(); i++) { + mActiveListeners.valueAt(i).onDetectionStopped(); + } + } } catch (RemoteException e) { - Slog.w(TAG, "RemoteException in onDetected"); + Slog.w(TAG, "RemoteException in onDetectionStopped"); } break; - case SoundTrigger.RECOGNITION_STATUS_ABORT: // fall-through - case SoundTrigger.RECOGNITION_STATUS_FAILURE: + case SoundTrigger.RECOGNITION_STATUS_SUCCESS: + if (!(event instanceof KeyphraseRecognitionEvent)) { + Slog.w(TAG, "Invalid recognition event!"); + return; + } + + KeyphraseRecognitionExtra[] keyphraseExtras = + ((KeyphraseRecognitionEvent) event).keyphraseExtras; + if (keyphraseExtras == null || keyphraseExtras.length == 0) { + Slog.w(TAG, "Invalid keyphrase recognition event!"); + return; + } + // TODO: Handle more than one keyphrase extras. + // TODO: Use keyphraseExtras[0].id here instead of 100. + int keyphraseId = 100; try { - listener.onDetectionStopped(); + synchronized(this) { + // Check which keyphrase triggered, and fire the appropriate event. + IRecognitionStatusCallback listener = mActiveListeners.get(keyphraseId); + if (listener != null) { + listener.onDetected((KeyphraseRecognitionEvent) event); + } else { + Slog.w(TAG, "received onRecognition event without any listener for it"); + return; + } + } } catch (RemoteException e) { Slog.w(TAG, "RemoteException in onDetectionStopped"); } @@ -257,6 +277,17 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { @Override public void onServiceDied() { - // TODO: Figure out how to restart the recognition here. + synchronized (this) { + try { + for (int i = 0; i < mActiveListeners.size(); i++) { + mActiveListeners.valueAt(i).onDetectionStopped(); + } + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException in onDetectionStopped"); + } + mCurrentSoundModelHandle = INVALID_SOUND_MODEL_HANDLE; + // Remove all listeners. + mActiveListeners.clear(); + } } } diff --git a/tests/SoundTriggerTests/src/android/hardware/soundtrigger/SoundTriggerTest.java b/tests/SoundTriggerTests/src/android/hardware/soundtrigger/SoundTriggerTest.java index 5d32c66e42df..4372ff98824c 100644 --- a/tests/SoundTriggerTests/src/android/hardware/soundtrigger/SoundTriggerTest.java +++ b/tests/SoundTriggerTests/src/android/hardware/soundtrigger/SoundTriggerTest.java @@ -17,7 +17,10 @@ package android.hardware.soundtrigger; import android.hardware.soundtrigger.SoundTrigger; +import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; import android.hardware.soundtrigger.SoundTrigger.Keyphrase; +import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; +import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; import android.os.Parcel; @@ -253,4 +256,78 @@ public class SoundTriggerTest extends InstrumentationTestCase { // Verify that they are the same assertEquals(re, unparceled); } + + @SmallTest + public void testKeyphraseRecognitionEventParcelUnparcel_noKeyphrases() throws Exception { + KeyphraseRecognitionEvent re = new KeyphraseRecognitionEvent( + SoundTrigger.RECOGNITION_STATUS_SUCCESS, 1, true, 2, 3, 4, null, false, null); + + // Write to a parcel + Parcel parcel = Parcel.obtain(); + re.writeToParcel(parcel, 0); + + // Read from it + parcel.setDataPosition(0); + KeyphraseRecognitionEvent unparceled = + KeyphraseRecognitionEvent.CREATOR.createFromParcel(parcel); + + // Verify that they are the same + assertEquals(re, unparceled); + } + + @SmallTest + public void testKeyphraseRecognitionEventParcelUnparcel_zeroData() throws Exception { + KeyphraseRecognitionExtra[] kpExtra = new KeyphraseRecognitionExtra[0]; + KeyphraseRecognitionEvent re = new KeyphraseRecognitionEvent( + SoundTrigger.RECOGNITION_STATUS_FAILURE, 2, true, 2, 3, 4, new byte[1], + true, kpExtra); + + // Write to a parcel + Parcel parcel = Parcel.obtain(); + re.writeToParcel(parcel, 0); + + // Read from it + parcel.setDataPosition(0); + KeyphraseRecognitionEvent unparceled = + KeyphraseRecognitionEvent.CREATOR.createFromParcel(parcel); + + // Verify that they are the same + assertEquals(re, unparceled); + } + + @LargeTest + public void testKeyphraseRecognitionEventParcelUnparcel_largeData() throws Exception { + byte[] data = new byte[200 * 1024]; + mRandom.nextBytes(data); + KeyphraseRecognitionExtra[] kpExtra = new KeyphraseRecognitionExtra[4]; + ConfidenceLevel cl1 = new ConfidenceLevel(1, 90); + ConfidenceLevel cl2 = new ConfidenceLevel(2, 30); + kpExtra[0] = new KeyphraseRecognitionExtra(1, + SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION, + new ConfidenceLevel[] {cl1, cl2}); + kpExtra[1] = new KeyphraseRecognitionExtra(1, + SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, + new ConfidenceLevel[] {cl2}); + kpExtra[2] = new KeyphraseRecognitionExtra(1, + SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, null); + kpExtra[3] = new KeyphraseRecognitionExtra(1, + SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, + new ConfidenceLevel[0]); + + KeyphraseRecognitionEvent re = new KeyphraseRecognitionEvent( + SoundTrigger.RECOGNITION_STATUS_FAILURE, 1, true, 2, 3, 4, data, + false, kpExtra); + + // Write to a parcel + Parcel parcel = Parcel.obtain(); + re.writeToParcel(parcel, 0); + + // Read from it + parcel.setDataPosition(0); + KeyphraseRecognitionEvent unparceled = + KeyphraseRecognitionEvent.CREATOR.createFromParcel(parcel); + + // Verify that they are the same + assertEquals(re, unparceled); + } } |