diff options
14 files changed, 302 insertions, 81 deletions
diff --git a/api/current.txt b/api/current.txt index 53738dcd8887..ae35e19cd728 100644 --- a/api/current.txt +++ b/api/current.txt @@ -34941,6 +34941,7 @@ package android.service.voice { method public void onDestroy(); method public boolean[] onGetSupportedCommands(java.lang.String[]); method public void onHandleAssist(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent); + method public void onHandleAssistSecondary(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent, int, int); method public void onHandleScreenshot(android.graphics.Bitmap); method public void onHide(); method public boolean onKeyDown(int, android.view.KeyEvent); diff --git a/api/system-current.txt b/api/system-current.txt index ea83257beac4..89ae0ddc5b72 100644 --- a/api/system-current.txt +++ b/api/system-current.txt @@ -37515,6 +37515,7 @@ package android.service.voice { method public void onDestroy(); method public boolean[] onGetSupportedCommands(java.lang.String[]); method public void onHandleAssist(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent); + method public void onHandleAssistSecondary(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent, int, int); method public void onHandleScreenshot(android.graphics.Bitmap); method public void onHide(); method public boolean onKeyDown(int, android.view.KeyEvent); diff --git a/api/test-current.txt b/api/test-current.txt index caf3231fd22a..3acbc2e11587 100644 --- a/api/test-current.txt +++ b/api/test-current.txt @@ -35014,6 +35014,7 @@ package android.service.voice { method public void onDestroy(); method public boolean[] onGetSupportedCommands(java.lang.String[]); method public void onHandleAssist(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent); + method public void onHandleAssistSecondary(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent, int, int); method public void onHandleScreenshot(android.graphics.Bitmap); method public void onHide(); method public boolean onKeyDown(int, android.view.KeyEvent); diff --git a/core/java/android/app/ActivityManagerInternal.java b/core/java/android/app/ActivityManagerInternal.java index 511663444053..374c4f64b9ce 100644 --- a/core/java/android/app/ActivityManagerInternal.java +++ b/core/java/android/app/ActivityManagerInternal.java @@ -23,6 +23,8 @@ import android.service.voice.IVoiceInteractionSession; import com.android.internal.app.IVoiceInteractor; +import java.util.List; + /** * Activity manager local system service interface. * @@ -125,4 +127,10 @@ public abstract class ActivityManagerInternal { * Callback for window manager to let activity manager know that the app transition is finished. */ public abstract void notifyAppTransitionFinished(); + + /** + * Returns the top activity from each of the currently visible stacks. The first entry will be + * the focused activity. + */ + public abstract List<IBinder> getTopVisibleActivities(); } diff --git a/core/java/android/app/ActivityManagerNative.java b/core/java/android/app/ActivityManagerNative.java index 65d48e609618..7bf03deee669 100644 --- a/core/java/android/app/ActivityManagerNative.java +++ b/core/java/android/app/ActivityManagerNative.java @@ -2411,8 +2411,11 @@ public abstract class ActivityManagerNative extends Binder implements IActivityM data.enforceInterface(IActivityManager.descriptor); int requestType = data.readInt(); IResultReceiver receiver = IResultReceiver.Stub.asInterface(data.readStrongBinder()); + Bundle receiverExtras = data.readBundle(); IBinder activityToken = data.readStrongBinder(); - boolean res = requestAssistContextExtras(requestType, receiver, activityToken); + boolean focused = data.readInt() == 1; + boolean res = requestAssistContextExtras(requestType, receiver, receiverExtras, + activityToken, focused); reply.writeNoException(); reply.writeInt(res ? 1 : 0); return true; @@ -6080,13 +6083,16 @@ class ActivityManagerProxy implements IActivityManager } public boolean requestAssistContextExtras(int requestType, IResultReceiver receiver, - IBinder activityToken) throws RemoteException { + Bundle receiverExtras, + IBinder activityToken, boolean focused) throws RemoteException { Parcel data = Parcel.obtain(); Parcel reply = Parcel.obtain(); data.writeInterfaceToken(IActivityManager.descriptor); data.writeInt(requestType); data.writeStrongBinder(receiver.asBinder()); + data.writeBundle(receiverExtras); data.writeStrongBinder(activityToken); + data.writeInt(focused ? 1 : 0); mRemote.transact(REQUEST_ASSIST_CONTEXT_EXTRAS_TRANSACTION, data, reply, 0); reply.readException(); boolean res = reply.readInt() != 0; diff --git a/core/java/android/app/IActivityManager.java b/core/java/android/app/IActivityManager.java index 8ee6fd0a9472..6975116c6be3 100644 --- a/core/java/android/app/IActivityManager.java +++ b/core/java/android/app/IActivityManager.java @@ -523,7 +523,8 @@ public interface IActivityManager extends IInterface { public Bundle getAssistContextExtras(int requestType) throws RemoteException; public boolean requestAssistContextExtras(int requestType, IResultReceiver receiver, - IBinder activityToken) throws RemoteException; + Bundle receiverExtras, + IBinder activityToken, boolean focused) throws RemoteException; public void reportAssistContextExtras(IBinder token, Bundle extras, AssistStructure structure, AssistContent content, Uri referrer) throws RemoteException; diff --git a/core/java/android/service/voice/IVoiceInteractionSession.aidl b/core/java/android/service/voice/IVoiceInteractionSession.aidl index dbc28f72ca98..78e6bc362648 100644 --- a/core/java/android/service/voice/IVoiceInteractionSession.aidl +++ b/core/java/android/service/voice/IVoiceInteractionSession.aidl @@ -30,7 +30,8 @@ import com.android.internal.app.IVoiceInteractionSessionShowCallback; oneway interface IVoiceInteractionSession { void show(in Bundle sessionArgs, int flags, IVoiceInteractionSessionShowCallback showCallback); void hide(); - void handleAssist(in Bundle assistData, in AssistStructure structure, in AssistContent content); + void handleAssist(in Bundle assistData, in AssistStructure structure, in AssistContent content, + int index, int count); void handleScreenshot(in Bitmap screenshot); void taskStarted(in Intent intent, int taskId); void taskFinished(in Intent intent, int taskId); diff --git a/core/java/android/service/voice/VoiceInteractionSession.java b/core/java/android/service/voice/VoiceInteractionSession.java index 6ff9fe74859d..e354ab35eaf1 100644 --- a/core/java/android/service/voice/VoiceInteractionSession.java +++ b/core/java/android/service/voice/VoiceInteractionSession.java @@ -77,7 +77,7 @@ import static android.view.ViewGroup.LayoutParams.MATCH_PARENT; */ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCallbacks2 { static final String TAG = "VoiceInteractionSession"; - static final boolean DEBUG = true; + static final boolean DEBUG = false; /** * Flag received in {@link #onShow}: originator requested that the session be started with @@ -110,6 +110,16 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall */ public static final int SHOW_SOURCE_ACTIVITY = 1<<4; + // Keys for Bundle values + /** @hide */ + public static final String KEY_DATA = "data"; + /** @hide */ + public static final String KEY_STRUCTURE = "structure"; + /** @hide */ + public static final String KEY_CONTENT = "content"; + /** @hide */ + public static final String KEY_RECEIVER_EXTRAS = "receiverExtras"; + final Context mContext; final HandlerCaller mHandlerCaller; @@ -229,7 +239,7 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall @Override public void handleAssist(final Bundle data, final AssistStructure structure, - final AssistContent content) { + final AssistContent content, final int index, final int count) { // We want to pre-warm the AssistStructure before handing it off to the main // thread. We also want to do this on a separate thread, so that if the app // is for some reason slow (due to slow filling in of async children in the @@ -247,8 +257,9 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall failure = e; } } - mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOOOO(MSG_HANDLE_ASSIST, - data, failure == null ? structure : null, failure, content)); + mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOOOOII(MSG_HANDLE_ASSIST, + data, failure == null ? structure : null, failure, content, + index, count)); } }; retriever.start(); @@ -831,9 +842,16 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall case MSG_HANDLE_ASSIST: args = (SomeArgs)msg.obj; if (DEBUG) Log.d(TAG, "onHandleAssist: data=" + args.arg1 - + " structure=" + args.arg2 + " content=" + args.arg3); - doOnHandleAssist((Bundle) args.arg1, (AssistStructure) args.arg2, - (Throwable) args.arg3, (AssistContent) args.arg4); + + " structure=" + args.arg2 + " content=" + args.arg3 + + " activityIndex=" + args.argi5 + " activityCount=" + args.argi6); + if (args.argi5 == 0) { + doOnHandleAssist((Bundle) args.arg1, (AssistStructure) args.arg2, + (Throwable) args.arg3, (AssistContent) args.arg4); + } else { + doOnHandleAssistSecondary((Bundle) args.arg1, (AssistStructure) args.arg2, + (Throwable) args.arg3, (AssistContent) args.arg4, + args.argi5, args.argi6); + } break; case MSG_HANDLE_SCREENSHOT: if (DEBUG) Log.d(TAG, "onHandleScreenshot: " + msg.obj); @@ -1320,6 +1338,14 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall onHandleAssist(data, structure, content); } + void doOnHandleAssistSecondary(Bundle data, AssistStructure structure, Throwable failure, + AssistContent content, int index, int count) { + if (failure != null) { + onAssistStructureFailure(failure); + } + onHandleAssistSecondary(data, structure, content, index, count); + } + /** * Called when there has been a failure transferring the {@link AssistStructure} to * the assistant. This may happen, for example, if the data is too large and results @@ -1356,6 +1382,45 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall } /** + * Called to receive data from other applications that the user was or is interacting with, + * that are currently on the screen in a multi-window display environment, not including the + * currently focused activity. This could be + * a free-form window, a picture-in-picture window, or another window in a split-screen display. + * <p> + * This method is very similar to + * {@link #onHandleAssist} except that it is called + * for additional non-focused activities along with an index and count that indicates + * which additional activity the data is for. {@code index} will be between 1 and + * {@code count}-1 and this method is called once for each additional window, in no particular + * order. The {@code count} indicates how many windows to expect assist data for, including the + * top focused activity, which continues to be returned via {@link #onHandleAssist}. + * <p> + * To be responsive to assist requests, process assist data as soon as it is received, + * without waiting for all queued activities to return assist data. + * + * @param data Arbitrary data supplied by the app through + * {@link android.app.Activity#onProvideAssistData Activity.onProvideAssistData}. + * May be null if assist data has been disabled by the user or device policy. + * @param structure If available, the structure definition of all windows currently + * displayed by the app. May be null if assist data has been disabled by the user + * or device policy; will be an empty stub if the application has disabled assist + * by marking its window as secure. + * @param content Additional content data supplied by the app through + * {@link android.app.Activity#onProvideAssistContent Activity.onProvideAssistContent}. + * May be null if assist data has been disabled by the user or device policy; will + * not be automatically filled in with data from the app if the app has marked its + * window as secure. + * @param index the index of the additional activity that this data + * is for. + * @param count the total number of additional activities for which the assist data is being + * returned, including the focused activity that is returned via + * {@link #onHandleAssist}. + */ + public void onHandleAssistSecondary(@Nullable Bundle data, @Nullable AssistStructure structure, + @Nullable AssistContent content, int index, int count) { + } + + /** * Called to receive a screenshot of what the user was currently viewing when an assist * session is started. May be null if screenshots are disabled by the user, policy, * or application. If the original show request did not specify diff --git a/core/java/com/android/internal/os/HandlerCaller.java b/core/java/com/android/internal/os/HandlerCaller.java index 113768ecd4eb..c26fc3a0a713 100644 --- a/core/java/com/android/internal/os/HandlerCaller.java +++ b/core/java/com/android/internal/os/HandlerCaller.java @@ -209,6 +209,18 @@ public class HandlerCaller { return mH.obtainMessage(what, 0, 0, args); } + public Message obtainMessageOOOOII(int what, Object arg1, Object arg2, + Object arg3, Object arg4, int arg5, int arg6) { + SomeArgs args = SomeArgs.obtain(); + args.arg1 = arg1; + args.arg2 = arg2; + args.arg3 = arg3; + args.arg4 = arg4; + args.argi5 = arg5; + args.argi6 = arg6; + return mH.obtainMessage(what, 0, 0, args); + } + public Message obtainMessageIIII(int what, int arg1, int arg2, int arg3, int arg4) { SomeArgs args = SomeArgs.obtain(); @@ -218,7 +230,7 @@ public class HandlerCaller { args.argi4 = arg4; return mH.obtainMessage(what, 0, 0, args); } - + public Message obtainMessageIIIIII(int what, int arg1, int arg2, int arg3, int arg4, int arg5, int arg6) { SomeArgs args = SomeArgs.obtain(); @@ -230,7 +242,7 @@ public class HandlerCaller { args.argi6 = arg6; return mH.obtainMessage(what, 0, 0, args); } - + public Message obtainMessageIIIIO(int what, int arg1, int arg2, int arg3, int arg4, Object arg5) { SomeArgs args = SomeArgs.obtain(); diff --git a/services/core/java/com/android/server/am/ActivityManagerService.java b/services/core/java/com/android/server/am/ActivityManagerService.java index 0f48d218b9d4..5b5e1752f619 100644 --- a/services/core/java/com/android/server/am/ActivityManagerService.java +++ b/services/core/java/com/android/server/am/ActivityManagerService.java @@ -628,13 +628,16 @@ public final class ActivityManagerService extends ActivityManagerNative public Bundle result = null; public AssistStructure structure = null; public AssistContent content = null; + public Bundle receiverExtras; + public PendingAssistExtras(ActivityRecord _activity, Bundle _extras, Intent _intent, - String _hint, IResultReceiver _receiver, int _userHandle) { + String _hint, IResultReceiver _receiver, Bundle _receiverExtras, int _userHandle) { activity = _activity; extras = _extras; intent = _intent; hint = _hint; receiver = _receiver; + receiverExtras = _receiverExtras; userHandle = _userHandle; } @Override @@ -11804,7 +11807,7 @@ public final class ActivityManagerService extends ActivityManagerNative @Override public Bundle getAssistContextExtras(int requestType) { PendingAssistExtras pae = enqueueAssistContext(requestType, null, null, null, - null, UserHandle.getCallingUserId(), null, PENDING_ASSIST_EXTRAS_TIMEOUT); + null, null, true, UserHandle.getCallingUserId(), null, PENDING_ASSIST_EXTRAS_TIMEOUT); if (pae == null) { return null; } @@ -11868,14 +11871,17 @@ public final class ActivityManagerService extends ActivityManagerNative @Override public boolean requestAssistContextExtras(int requestType, IResultReceiver receiver, - IBinder activityToken) { - return enqueueAssistContext(requestType, null, null, receiver, activityToken, - UserHandle.getCallingUserId(), null, PENDING_ASSIST_EXTRAS_LONG_TIMEOUT) != null; + Bundle receiverExtras, + IBinder activityToken, boolean focused) { + return enqueueAssistContext(requestType, null, null, receiver, receiverExtras, + activityToken, focused, + UserHandle.getCallingUserId(), null, PENDING_ASSIST_EXTRAS_LONG_TIMEOUT) + != null; } private PendingAssistExtras enqueueAssistContext(int requestType, Intent intent, String hint, - IResultReceiver receiver, IBinder activityToken, int userHandle, Bundle args, - long timeout) { + IResultReceiver receiver, Bundle receiverExtras, IBinder activityToken, boolean focused, + int userHandle, Bundle args, long timeout) { enforceCallingPermission(android.Manifest.permission.GET_TOP_ACTIVITY_INFO, "enqueueAssistContext()"); synchronized (this) { @@ -11888,14 +11894,24 @@ public final class ActivityManagerService extends ActivityManagerNative Slog.w(TAG, "getAssistContextExtras failed: no process for " + activity); return null; } - if (activityToken != null) { - ActivityRecord caller = ActivityRecord.forTokenLocked(activityToken); - if (activity != caller) { - Slog.w(TAG, "enqueueAssistContext failed: caller " + caller - + " is not current top " + activity); + if (focused) { + if (activityToken != null) { + ActivityRecord caller = ActivityRecord.forTokenLocked(activityToken); + if (activity != caller) { + Slog.w(TAG, "enqueueAssistContext failed: caller " + caller + + " is not current top " + activity); + return null; + } + } + } else { + activity = ActivityRecord.forTokenLocked(activityToken); + if (activity == null) { + Slog.w(TAG, "enqueueAssistContext failed: activity for token=" + activityToken + + " couldn't be found"); return null; } } + PendingAssistExtras pae; Bundle extras = new Bundle(); if (args != null) { @@ -11903,7 +11919,8 @@ public final class ActivityManagerService extends ActivityManagerNative } extras.putString(Intent.EXTRA_ASSIST_PACKAGE, activity.packageName); extras.putInt(Intent.EXTRA_ASSIST_UID, activity.app.uid); - pae = new PendingAssistExtras(activity, extras, intent, hint, receiver, userHandle); + pae = new PendingAssistExtras(activity, extras, intent, hint, receiver, receiverExtras, + userHandle); try { activity.app.thread.requestAssistContextExtras(activity.appToken, pae, requestType); @@ -11973,9 +11990,11 @@ public final class ActivityManagerService extends ActivityManagerNative if ((sendReceiver=pae.receiver) != null) { // Caller wants result sent back to them. sendBundle = new Bundle(); - sendBundle.putBundle("data", pae.extras); - sendBundle.putParcelable("structure", pae.structure); - sendBundle.putParcelable("content", pae.content); + sendBundle.putBundle(VoiceInteractionSession.KEY_DATA, pae.extras); + sendBundle.putParcelable(VoiceInteractionSession.KEY_STRUCTURE, pae.structure); + sendBundle.putParcelable(VoiceInteractionSession.KEY_CONTENT, pae.content); + sendBundle.putBundle(VoiceInteractionSession.KEY_RECEIVER_EXTRAS, + pae.receiverExtras); } } if (sendReceiver != null) { @@ -12005,8 +12024,8 @@ public final class ActivityManagerService extends ActivityManagerNative public boolean launchAssistIntent(Intent intent, int requestType, String hint, int userHandle, Bundle args) { - return enqueueAssistContext(requestType, intent, hint, null, null, userHandle, args, - PENDING_ASSIST_EXTRAS_TIMEOUT) != null; + return enqueueAssistContext(requestType, intent, hint, null, null, null, true, + userHandle, args, PENDING_ASSIST_EXTRAS_TIMEOUT) != null; } public void registerProcessObserver(IProcessObserver observer) { @@ -20947,6 +20966,13 @@ public final class ActivityManagerService extends ActivityManagerNative mStackSupervisor.notifyAppTransitionDone(); } } + + @Override + public List<IBinder> getTopVisibleActivities() { + synchronized (ActivityManagerService.this) { + return mStackSupervisor.getTopVisibleActivities(); + } + } } private final class SleepTokenImpl extends SleepToken { diff --git a/services/core/java/com/android/server/am/ActivityStackSupervisor.java b/services/core/java/com/android/server/am/ActivityStackSupervisor.java index d34e8fc461ea..7e4ffae7d767 100644 --- a/services/core/java/com/android/server/am/ActivityStackSupervisor.java +++ b/services/core/java/com/android/server/am/ActivityStackSupervisor.java @@ -98,6 +98,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; @@ -4270,4 +4271,31 @@ public final class ActivityStackSupervisor implements DisplayListener { } return result; } + + /** + * @return a list of activities which are the top ones in each visible stack. The first + * entry will be the focused activity. + */ + public List<IBinder> getTopVisibleActivities() { + final ActivityDisplay display = mActivityDisplays.get(Display.DEFAULT_DISPLAY); + if (display == null) { + return Collections.EMPTY_LIST; + } + ArrayList<IBinder> topActivityTokens = new ArrayList<>(); + final ArrayList<ActivityStack> stacks = display.mStacks; + for (int i = stacks.size() - 1; i >= 0; i--) { + ActivityStack stack = stacks.get(i); + if (stack.getStackVisibilityLocked(null) == ActivityStack.STACK_VISIBLE) { + ActivityRecord top = stack.topActivity(); + if (top != null) { + if (stack == mFocusedStack) { + topActivityTokens.add(0, top.appToken); + } else { + topActivityTokens.add(top.appToken); + } + } + } + } + return topActivityTokens; + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java index 154472336fd2..1e9db18aa6a3 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java @@ -17,6 +17,7 @@ package com.android.server.voiceinteraction; import android.app.ActivityManager; +import android.app.ActivityManagerInternal; import android.app.ActivityManagerNative; import android.app.IActivityManager; import android.content.BroadcastReceiver; @@ -42,9 +43,11 @@ import android.view.IWindowManager; import com.android.internal.app.IVoiceInteractionSessionShowCallback; import com.android.internal.app.IVoiceInteractor; +import com.android.server.LocalServices; import java.io.FileDescriptor; import java.io.PrintWriter; +import java.util.List; class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConnection.Callback { final static String TAG = "VoiceInteractionServiceManager"; @@ -148,8 +151,14 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne mActiveSession = new VoiceInteractionSessionConnection(mLock, mSessionComponentName, mUser, mContext, this, mInfo.getServiceInfo().applicationInfo.uid, mHandler); } + List<IBinder> activityTokens = null; + if (activityToken == null) { + // Let's get top activities from all visible stacks + activityTokens = LocalServices.getService(ActivityManagerInternal.class) + .getTopVisibleActivities(); + } return mActiveSession.showLocked(args, flags, mDisabledShowContext, showCallback, - activityToken); + activityToken, activityTokens); } public boolean hideSessionLocked() { diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java index e04f312ad7b4..0922a12ee0b3 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java @@ -45,6 +45,7 @@ import android.service.voice.VoiceInteractionSession; import android.util.Slog; import android.view.IWindowManager; import android.view.WindowManager; + import com.android.internal.app.IAssistScreenshotReceiver; import com.android.internal.app.IVoiceInteractionSessionShowCallback; import com.android.internal.app.IVoiceInteractor; @@ -55,10 +56,15 @@ import com.android.server.statusbar.StatusBarManagerInternal; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.List; final class VoiceInteractionSessionConnection implements ServiceConnection { + final static String TAG = "VoiceInteractionServiceManager"; + private static final String KEY_RECEIVER_EXTRA_COUNT = "count"; + private static final String KEY_RECEIVER_EXTRA_INDEX = "index"; + final IBinder mToken = new Binder(); final Object mLock; final ComponentName mSessionComponentName; @@ -82,11 +88,27 @@ final class VoiceInteractionSessionConnection implements ServiceConnection { IVoiceInteractionSession mSession; IVoiceInteractor mInteractor; boolean mHaveAssistData; - Bundle mAssistData; + int mPendingAssistDataCount; + ArrayList<AssistDataForActivity> mAssistData = new ArrayList<>(); boolean mHaveScreenshot; Bitmap mScreenshot; ArrayList<IVoiceInteractionSessionShowCallback> mPendingShowCallbacks = new ArrayList<>(); + static class AssistDataForActivity { + int activityIndex; + int activityCount; + Bundle data; + + public AssistDataForActivity(Bundle data) { + this.data = data; + Bundle receiverExtras = data.getBundle(VoiceInteractionSession.KEY_RECEIVER_EXTRAS); + if (receiverExtras != null) { + activityIndex = receiverExtras.getInt(KEY_RECEIVER_EXTRA_INDEX); + activityCount = receiverExtras.getInt(KEY_RECEIVER_EXTRA_COUNT); + } + } + } + IVoiceInteractionSessionShowCallback mShowCallback = new IVoiceInteractionSessionShowCallback.Stub() { @Override @@ -125,7 +147,7 @@ final class VoiceInteractionSessionConnection implements ServiceConnection { synchronized (mLock) { if (mShown) { mHaveAssistData = true; - mAssistData = resultData; + mAssistData.add(new AssistDataForActivity(resultData)); deliverSessionDataLocked(); } } @@ -198,7 +220,8 @@ final class VoiceInteractionSessionConnection implements ServiceConnection { } public boolean showLocked(Bundle args, int flags, int disabledContext, - IVoiceInteractionSessionShowCallback showCallback, IBinder activityToken) { + IVoiceInteractionSessionShowCallback showCallback, IBinder activityToken, + List<IBinder> topActivities) { if (mBound) { if (!mFullyBound) { mFullyBound = mContext.bindServiceAsUser(mBindIntent, mFullConnection, @@ -220,30 +243,41 @@ final class VoiceInteractionSessionConnection implements ServiceConnection { mShowArgs = args; mShowFlags = flags; mHaveAssistData = false; + mPendingAssistDataCount = 0; boolean needDisclosure = false; if ((flags&VoiceInteractionSession.SHOW_WITH_ASSIST) != 0) { if (mAppOps.noteOpNoThrow(AppOpsManager.OP_ASSIST_STRUCTURE, mCallingUid, mSessionComponentName.getPackageName()) == AppOpsManager.MODE_ALLOWED && structureEnabled) { - try { - MetricsLogger.count(mContext, "assist_with_context", 1); - if (mAm.requestAssistContextExtras(ActivityManager.ASSIST_CONTEXT_FULL, - mAssistReceiver, activityToken)) { - needDisclosure = true; - } else { - // Wasn't allowed... given that, let's not do the screenshot either. - mHaveAssistData = true; - mAssistData = null; - screenshotEnabled = false; + mAssistData.clear(); + final int count = activityToken != null ? 1 : topActivities.size(); + for (int i = 0; i < count; i++) { + IBinder topActivity = count == 1 ? activityToken : topActivities.get(i); + try { + MetricsLogger.count(mContext, "assist_with_context", 1); + Bundle receiverExtras = new Bundle(); + receiverExtras.putInt(KEY_RECEIVER_EXTRA_INDEX, i); + receiverExtras.putInt(KEY_RECEIVER_EXTRA_COUNT, count); + if (mAm.requestAssistContextExtras(ActivityManager.ASSIST_CONTEXT_FULL, + mAssistReceiver, receiverExtras, topActivity, i == 0)) { + needDisclosure = true; + mPendingAssistDataCount++; + } else if (i == 0) { + // Wasn't allowed... given that, let's not do the screenshot either. + mHaveAssistData = true; + mAssistData.clear(); + screenshotEnabled = false; + break; + } + } catch (RemoteException e) { } - } catch (RemoteException e) { } } else { mHaveAssistData = true; - mAssistData = null; + mAssistData.clear(); } } else { - mAssistData = null; + mAssistData.clear(); } mHaveScreenshot = false; if ((flags&VoiceInteractionSession.SHOW_WITH_SCREENSHOT) != 0) { @@ -335,41 +369,26 @@ final class VoiceInteractionSessionConnection implements ServiceConnection { return; } if (mHaveAssistData) { - Bundle assistData; - AssistStructure structure; - AssistContent content; - if (mAssistData != null) { - assistData = mAssistData.getBundle("data"); - structure = mAssistData.getParcelable("structure"); - content = mAssistData.getParcelable("content"); - int uid = mAssistData.getInt(Intent.EXTRA_ASSIST_UID, -1); - if (uid >= 0 && content != null) { - Intent intent = content.getIntent(); - if (intent != null) { - ClipData data = intent.getClipData(); - if (data != null && Intent.isAccessUriMode(intent.getFlags())) { - grantClipDataPermissions(data, intent.getFlags(), uid, - mCallingUid, mSessionComponentName.getPackageName()); - } - } - ClipData data = content.getClipData(); - if (data != null) { - grantClipDataPermissions(data, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - uid, mCallingUid, mSessionComponentName.getPackageName()); + AssistDataForActivity assistData; + while (!mAssistData.isEmpty()) { + if (mPendingAssistDataCount <= 0) { + Slog.e(TAG, "mPendingAssistDataCount is " + mPendingAssistDataCount); + } + mPendingAssistDataCount--; + assistData = mAssistData.remove(0); + if (assistData.data == null) { + try { + mSession.handleAssist(null, null, null, assistData.activityIndex, + assistData.activityCount); + } catch (RemoteException e) { } + } else { + deliverSessionDataLocked(assistData); } - } else { - assistData = null; - structure = null; - content = null; - } - try { - mSession.handleAssist(assistData, structure, content); - } catch (RemoteException e) { } - mAssistData = null; - mHaveAssistData = false; + if (mPendingAssistDataCount <= 0) { + mHaveAssistData = false; + } // else, more to come } if (mHaveScreenshot) { try { @@ -381,6 +400,37 @@ final class VoiceInteractionSessionConnection implements ServiceConnection { } } + private void deliverSessionDataLocked(AssistDataForActivity assistDataForActivity) { + Bundle assistData = assistDataForActivity.data.getBundle( + VoiceInteractionSession.KEY_DATA); + AssistStructure structure = assistDataForActivity.data.getParcelable( + VoiceInteractionSession.KEY_STRUCTURE); + AssistContent content = assistDataForActivity.data.getParcelable( + VoiceInteractionSession.KEY_CONTENT); + int uid = assistDataForActivity.data.getInt(Intent.EXTRA_ASSIST_UID, -1); + if (uid >= 0 && content != null) { + Intent intent = content.getIntent(); + if (intent != null) { + ClipData data = intent.getClipData(); + if (data != null && Intent.isAccessUriMode(intent.getFlags())) { + grantClipDataPermissions(data, intent.getFlags(), uid, + mCallingUid, mSessionComponentName.getPackageName()); + } + } + ClipData data = content.getClipData(); + if (data != null) { + grantClipDataPermissions(data, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + uid, mCallingUid, mSessionComponentName.getPackageName()); + } + } + try { + mSession.handleAssist(assistData, structure, content, + assistDataForActivity.activityIndex, assistDataForActivity.activityCount); + } catch (RemoteException e) { + } + } + public boolean hideLocked() { if (mBound) { if (mShown) { @@ -388,7 +438,7 @@ final class VoiceInteractionSessionConnection implements ServiceConnection { mShowArgs = null; mShowFlags = 0; mHaveAssistData = false; - mAssistData = null; + mAssistData.clear(); if (mSession != null) { try { mSession.hide(); diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java index 450334cb2d86..5767f110f0a4 100644 --- a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java +++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java @@ -194,6 +194,18 @@ public class MainInteractionSession extends VoiceInteractionSession } @Override + public void onHandleAssistSecondary(final Bundle data, final AssistStructure structure, + final AssistContent content, int index, int count) { + Log.i(TAG, "Got secondary activity assist data " + index + " of " + count); + Log.i(TAG, "Showing assist structure after a few seconds..."); + mContentView.postDelayed(new Runnable() { + public void run() { + onHandleAssist(data, structure, content); + } + }, 2000 * index); + } + + @Override public void onHandleScreenshot(Bitmap screenshot) { if (screenshot != null) { mScreenshot.setImageBitmap(screenshot); |