diff options
author | Xin Li <delphij@google.com> | 2020-08-31 21:21:38 -0700 |
---|---|---|
committer | Xin Li <delphij@google.com> | 2020-08-31 21:21:38 -0700 |
commit | 628590d7ec80e10a3fc24b1c18a1afb55cca10a8 (patch) | |
tree | 4b1c3f52d86d7fb53afbe9e9438468588fa489f8 /services/accessibility | |
parent | b11b8ec3aec8bb42f2c07e1c5ac7942da293baa8 (diff) | |
parent | d2d3a20624d968199353ccf6ddbae6f3ac39c9af (diff) |
Merge Android R (rvc-dev-plus-aosp-without-vendor@6692709)
Bug: 166295507
Merged-In: I3d92a6de21a938f6b352ec26dc23420c0fe02b27
Change-Id: Ifdb80563ef042738778ebb8a7581a97c4e3d96e2
Diffstat (limited to 'services/accessibility')
35 files changed, 10383 insertions, 5031 deletions
diff --git a/services/accessibility/Android.bp b/services/accessibility/Android.bp index 284a2f2626a4..21a0c7489827 100644 --- a/services/accessibility/Android.bp +++ b/services/accessibility/Android.bp @@ -7,6 +7,7 @@ filegroup { java_library_static { name: "services.accessibility", + defaults: ["services_defaults"], srcs: [":services.accessibility-sources"], libs: ["services.core"], } diff --git a/services/accessibility/TEST_MAPPING b/services/accessibility/TEST_MAPPING index d90c3bd9b4c2..2b8fee3b54c8 100644 --- a/services/accessibility/TEST_MAPPING +++ b/services/accessibility/TEST_MAPPING @@ -69,6 +69,9 @@ ], "postsubmit": [ { + "name": "CtsAccessibilityServiceSdk29TestCases" + }, + { "name": "CtsAccessibilityServiceTestCases" }, { diff --git a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java index fc43882b4ffd..6b852adce0f1 100644 --- a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java @@ -16,10 +16,19 @@ package com.android.server.accessibility; +import static android.accessibilityservice.AccessibilityService.KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE; +import static android.accessibilityservice.AccessibilityService.KEY_ACCESSIBILITY_SCREENSHOT_HARDWAREBUFFER; +import static android.accessibilityservice.AccessibilityService.KEY_ACCESSIBILITY_SCREENSHOT_STATUS; +import static android.accessibilityservice.AccessibilityService.KEY_ACCESSIBILITY_SCREENSHOT_TIMESTAMP; import static android.accessibilityservice.AccessibilityServiceInfo.DEFAULT; -import static android.view.Display.DEFAULT_DISPLAY; import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY; +import static android.view.accessibility.AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS; +import static android.view.accessibility.AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS; +import static android.view.accessibility.AccessibilityNodeInfo.ACTION_CLICK; +import static android.view.accessibility.AccessibilityNodeInfo.ACTION_LONG_CLICK; +import android.accessibilityservice.AccessibilityGestureEvent; +import android.accessibilityservice.AccessibilityService; import android.accessibilityservice.AccessibilityServiceInfo; import android.accessibilityservice.IAccessibilityServiceClient; import android.accessibilityservice.IAccessibilityServiceConnection; @@ -32,7 +41,12 @@ import android.content.Intent; import android.content.ServiceConnection; import android.content.pm.PackageManager; import android.content.pm.ParceledListSlice; +import android.graphics.GraphicBuffer; +import android.graphics.ParcelableColorSpace; import android.graphics.Region; +import android.hardware.HardwareBuffer; +import android.hardware.display.DisplayManager; +import android.hardware.display.DisplayManagerInternal; import android.os.Binder; import android.os.Build; import android.os.Bundle; @@ -40,30 +54,40 @@ import android.os.Handler; import android.os.IBinder; import android.os.Looper; import android.os.Message; +import android.os.PowerManager; +import android.os.RemoteCallback; import android.os.RemoteException; +import android.os.ServiceManager; +import android.os.SystemClock; import android.util.Slog; import android.util.SparseArray; +import android.view.Display; import android.view.KeyEvent; import android.view.MagnificationSpec; +import android.view.SurfaceControl.ScreenshotGraphicBuffer; import android.view.View; +import android.view.WindowInfo; import android.view.accessibility.AccessibilityCache; import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; -import android.view.accessibility.IAccessibilityInteractionConnection; import android.view.accessibility.IAccessibilityInteractionConnectionCallback; import com.android.internal.annotations.GuardedBy; +import com.android.internal.compat.IPlatformCompat; import com.android.internal.os.SomeArgs; import com.android.internal.util.DumpUtils; -import com.android.server.accessibility.AccessibilityManagerService.RemoteAccessibilityConnection; -import com.android.server.accessibility.AccessibilityManagerService.SecurityPolicy; +import com.android.internal.util.function.pooled.PooledLambda; +import com.android.server.LocalServices; +import com.android.server.accessibility.AccessibilityWindowManager.RemoteAccessibilityConnection; +import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.WindowManagerInternal; import java.io.FileDescriptor; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -77,11 +101,19 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ FingerprintGestureDispatcher.FingerprintGestureClient { private static final boolean DEBUG = false; private static final String LOG_TAG = "AbstractAccessibilityServiceConnection"; + private static final int WAIT_WINDOWS_TIMEOUT_MILLIS = 5000; + protected static final String TAKE_SCREENSHOT = "takeScreenshot"; protected final Context mContext; protected final SystemSupport mSystemSupport; - private final WindowManagerInternal mWindowManagerService; - private final GlobalActionPerformer mGlobalActionPerformer; + protected final WindowManagerInternal mWindowManagerService; + private final SystemActionPerformer mSystemActionPerformer; + private final AccessibilityWindowManager mA11yWindowManager; + private final DisplayManager mDisplayManager; + private final PowerManager mPowerManager; + private final IPlatformCompat mIPlatformCompat; + + private final Handler mMainHandler; // Handler for scheduling method invocations on the main thread. public final InvocationHandler mInvocationHandler; @@ -93,7 +125,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ // Lock must match the one used by AccessibilityManagerService protected final Object mLock; - protected final SecurityPolicy mSecurityPolicy; + protected final AccessibilitySecurityPolicy mSecurityPolicy; // The service that's bound to this instance. Whenever this value is non-null, this // object is registered as a death recipient @@ -111,6 +143,10 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ boolean mRequestTouchExplorationMode; + private boolean mServiceHandlesDoubleTap; + + private boolean mRequestMultiFingerGestures; + boolean mRequestFilterKeyEvents; boolean mRetrieveInteractiveWindows; @@ -139,8 +175,10 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ // types as message types allowing us to remove messages per event type. public Handler mEventDispatchHandler; - final IBinder mOverlayWindowToken = new Binder(); + final SparseArray<IBinder> mOverlayWindowTokens = new SparseArray(); + /** The timestamp of requesting to take screenshot in milliseconds */ + private long mRequestTakeScreenshotTimestampMs; public interface SystemSupport { /** @@ -155,9 +193,10 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ @Nullable MagnificationSpec getCompatibleMagnificationSpecLocked(int windowId); /** - * @return The current injector of motion events, if one exists + * @param displayId The display id. + * @return The current injector of motion events used on the display, if one exists. */ - @Nullable MotionEventInjector getMotionEventInjectorLocked(); + @Nullable MotionEventInjector getMotionEventInjectorForDisplayLocked(int displayId); /** * @return The current dispatcher for fingerprint gestures, if one exists @@ -170,50 +209,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ @NonNull MagnificationController getMagnificationController(); /** - * Resolve a connection wrapper for a window id - * - * @param windowId The id of the window of interest - * - * @return a connection to the window - */ - RemoteAccessibilityConnection getConnectionLocked(int windowId); - - /** - * Perform the specified accessibility action - * - * @param resolvedWindowId The window ID - * [Other parameters match the method on IAccessibilityServiceConnection] - * - * @return Whether or not the action could be sent to the app process - */ - boolean performAccessibilityAction(int resolvedWindowId, - long accessibilityNodeId, int action, Bundle arguments, int interactionId, - IAccessibilityInteractionConnectionCallback callback, int fetchFlags, - long interrogatingTid); - - /** - * Replace the interaction callback if needed, for example if the window is in picture- - * in-picture mode and needs its nodes replaced. - * - * @param originalCallback The callback we were planning to use - * @param resolvedWindowId The ID of the window we're calling - * @param interactionId The id for the original callback - * @param interrogatingPid Process ID of requester - * @param interrogatingTid Thread ID of requester - * - * @return The callback to use, which may be the original one. - */ - @NonNull IAccessibilityInteractionConnectionCallback replaceCallbackIfNeeded( - IAccessibilityInteractionConnectionCallback originalCallback, - int resolvedWindowId, int interactionId, int interrogatingPid, - long interrogatingTid); - - /** - * Request that the system make sure windows are available to interrogate - */ - void ensureWindowsAvailableTimed(); - - /** * Called back to notify system that the client has changed * @param serviceInfoChanged True if the service's AccessibilityServiceInfo changed. */ @@ -237,13 +232,18 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ /* This is exactly PendingIntent.getActivity, separated out for testability */ PendingIntent getPendingIntentActivity(Context context, int requestCode, Intent intent, int flags); + + void setGestureDetectionPassthroughRegion(int displayId, Region region); + + void setTouchExplorationPassthroughRegion(int displayId, Region region); } public AbstractAccessibilityServiceConnection(Context context, ComponentName componentName, AccessibilityServiceInfo accessibilityServiceInfo, int id, Handler mainHandler, - Object lock, SecurityPolicy securityPolicy, SystemSupport systemSupport, + Object lock, AccessibilitySecurityPolicy securityPolicy, SystemSupport systemSupport, WindowManagerInternal windowManagerInternal, - GlobalActionPerformer globalActionPerfomer) { + SystemActionPerformer systemActionPerfomer, + AccessibilityWindowManager a11yWindowManager) { mContext = context; mWindowManagerService = windowManagerInternal; mId = id; @@ -251,9 +251,15 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mAccessibilityServiceInfo = accessibilityServiceInfo; mLock = lock; mSecurityPolicy = securityPolicy; - mGlobalActionPerformer = globalActionPerfomer; + mSystemActionPerformer = systemActionPerfomer; mSystemSupport = systemSupport; + mMainHandler = mainHandler; mInvocationHandler = new InvocationHandler(mainHandler.getLooper()); + mA11yWindowManager = a11yWindowManager; + mDisplayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + mPowerManager = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE); + mIPlatformCompat = IPlatformCompat.Stub.asInterface( + ServiceManager.getService(Context.PLATFORM_COMPAT_SERVICE)); mEventDispatchHandler = new Handler(mainHandler.getLooper()) { @Override public void handleMessage(Message message) { @@ -312,6 +318,10 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mRequestTouchExplorationMode = (info.flags & AccessibilityServiceInfo.FLAG_REQUEST_TOUCH_EXPLORATION_MODE) != 0; + mServiceHandlesDoubleTap = (info.flags + & AccessibilityServiceInfo.FLAG_SERVICE_HANDLES_DOUBLE_TAP) != 0; + mRequestMultiFingerGestures = (info.flags + & AccessibilityServiceInfo.FLAG_REQUEST_MULTI_FINGER_GESTURES) != 0; mRequestFilterKeyEvents = (info.flags & AccessibilityServiceInfo.FLAG_REQUEST_FILTER_KEY_EVENTS) != 0; mRetrieveInteractiveWindows = (info.flags @@ -328,7 +338,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } public boolean canReceiveEventsLocked() { - return (mEventTypes != 0 && mFeedbackType != 0 && mService != null); + return (mEventTypes != 0 && mService != null); } @Override @@ -362,7 +372,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ // configurable properties. AccessibilityServiceInfo oldInfo = mAccessibilityServiceInfo; if (oldInfo != null) { - oldInfo.updateDynamicallyConfigurableProperties(info); + oldInfo.updateDynamicallyConfigurableProperties(mIPlatformCompat, info); setDynamicallyConfigurableProperties(oldInfo); } else { setDynamicallyConfigurableProperties(info); @@ -374,13 +384,13 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } } - protected abstract boolean isCalledForCurrentUserLocked(); + protected abstract boolean hasRightsToCurrentUserLocked(); + @Nullable @Override - public List<AccessibilityWindowInfo> getWindows() { - mSystemSupport.ensureWindowsAvailableTimed(); + public AccessibilityWindowInfo.WindowListSparseArray getWindows() { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return null; } final boolean permissionGranted = @@ -388,30 +398,40 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ if (!permissionGranted) { return null; } - if (mSecurityPolicy.mWindows == null) { - return null; - } if (!mSecurityPolicy.checkAccessibilityAccess(this)) { return null; } - List<AccessibilityWindowInfo> windows = new ArrayList<>(); - final int windowCount = mSecurityPolicy.mWindows.size(); - for (int i = 0; i < windowCount; i++) { - AccessibilityWindowInfo window = mSecurityPolicy.mWindows.get(i); - AccessibilityWindowInfo windowClone = - AccessibilityWindowInfo.obtain(window); - windowClone.setConnectionId(mId); - windows.add(windowClone); + final AccessibilityWindowInfo.WindowListSparseArray allWindows = + new AccessibilityWindowInfo.WindowListSparseArray(); + final ArrayList<Integer> displayList = mA11yWindowManager.getDisplayListLocked(); + final int displayListCounts = displayList.size(); + if (displayListCounts > 0) { + for (int i = 0; i < displayListCounts; i++) { + final int displayId = displayList.get(i); + ensureWindowsAvailableTimedLocked(displayId); + + final List<AccessibilityWindowInfo> windowList = getWindowsByDisplayLocked( + displayId); + if (windowList != null) { + allWindows.put(displayId, windowList); + } + } } - return windows; + return allWindows; } } @Override public AccessibilityWindowInfo getWindow(int windowId) { - mSystemSupport.ensureWindowsAvailableTimed(); synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + int displayId = Display.INVALID_DISPLAY; + if (windowId != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID) { + displayId = mA11yWindowManager.getDisplayIdByUserIdAndWindowIdLocked( + mSystemSupport.getCurrentUserIdLocked(), windowId); + } + ensureWindowsAvailableTimedLocked(displayId); + + if (!hasRightsToCurrentUserLocked()) { return null; } final boolean permissionGranted = @@ -422,7 +442,8 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ if (!mSecurityPolicy.checkAccessibilityAccess(this)) { return null; } - AccessibilityWindowInfo window = mSecurityPolicy.findA11yWindowInfoById(windowId); + AccessibilityWindowInfo window = + mA11yWindowManager.findA11yWindowInfoByIdLocked(windowId); if (window != null) { AccessibilityWindowInfo windowClone = AccessibilityWindowInfo.obtain(window); windowClone.setConnectionId(mId); @@ -443,21 +464,23 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ MagnificationSpec spec; synchronized (mLock) { mUsesAccessibilityCache = true; - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return null; } resolvedWindowId = resolveAccessibilityWindowIdLocked(accessibilityWindowId); final boolean permissionGranted = - mSecurityPolicy.canGetAccessibilityNodeInfoLocked(this, resolvedWindowId); + mSecurityPolicy.canGetAccessibilityNodeInfoLocked( + mSystemSupport.getCurrentUserIdLocked(), this, resolvedWindowId); if (!permissionGranted) { return null; } else { - connection = mSystemSupport.getConnectionLocked(resolvedWindowId); + connection = mA11yWindowManager.getConnectionLocked( + mSystemSupport.getCurrentUserIdLocked(), resolvedWindowId); if (connection == null) { return null; } } - if (!mSecurityPolicy.computePartialInteractiveRegionForWindowLocked( + if (!mA11yWindowManager.computePartialInteractiveRegionForWindowLocked( resolvedWindowId, partialInteractiveRegion)) { partialInteractiveRegion.recycle(); partialInteractiveRegion = null; @@ -468,7 +491,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ return null; } final int interrogatingPid = Binder.getCallingPid(); - callback = mSystemSupport.replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, + callback = replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, interrogatingPid, interrogatingTid); final long identityToken = Binder.clearCallingIdentity(); try { @@ -502,21 +525,23 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ MagnificationSpec spec; synchronized (mLock) { mUsesAccessibilityCache = true; - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return null; } resolvedWindowId = resolveAccessibilityWindowIdLocked(accessibilityWindowId); final boolean permissionGranted = - mSecurityPolicy.canGetAccessibilityNodeInfoLocked(this, resolvedWindowId); + mSecurityPolicy.canGetAccessibilityNodeInfoLocked( + mSystemSupport.getCurrentUserIdLocked(), this, resolvedWindowId); if (!permissionGranted) { return null; } else { - connection = mSystemSupport.getConnectionLocked(resolvedWindowId); + connection = mA11yWindowManager.getConnectionLocked( + mSystemSupport.getCurrentUserIdLocked(), resolvedWindowId); if (connection == null) { return null; } } - if (!mSecurityPolicy.computePartialInteractiveRegionForWindowLocked( + if (!mA11yWindowManager.computePartialInteractiveRegionForWindowLocked( resolvedWindowId, partialInteractiveRegion)) { partialInteractiveRegion.recycle(); partialInteractiveRegion = null; @@ -527,7 +552,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ return null; } final int interrogatingPid = Binder.getCallingPid(); - callback = mSystemSupport.replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, + callback = replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, interrogatingPid, interrogatingTid); final long identityToken = Binder.clearCallingIdentity(); try { @@ -561,21 +586,23 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ MagnificationSpec spec; synchronized (mLock) { mUsesAccessibilityCache = true; - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return null; } resolvedWindowId = resolveAccessibilityWindowIdLocked(accessibilityWindowId); final boolean permissionGranted = - mSecurityPolicy.canGetAccessibilityNodeInfoLocked(this, resolvedWindowId); + mSecurityPolicy.canGetAccessibilityNodeInfoLocked( + mSystemSupport.getCurrentUserIdLocked(), this, resolvedWindowId); if (!permissionGranted) { return null; } else { - connection = mSystemSupport.getConnectionLocked(resolvedWindowId); + connection = mA11yWindowManager.getConnectionLocked( + mSystemSupport.getCurrentUserIdLocked(), resolvedWindowId); if (connection == null) { return null; } } - if (!mSecurityPolicy.computePartialInteractiveRegionForWindowLocked( + if (!mA11yWindowManager.computePartialInteractiveRegionForWindowLocked( resolvedWindowId, partialInteractiveRegion)) { partialInteractiveRegion.recycle(); partialInteractiveRegion = null; @@ -586,7 +613,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ return null; } final int interrogatingPid = Binder.getCallingPid(); - callback = mSystemSupport.replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, + callback = replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, interrogatingPid, interrogatingTid); final long identityToken = Binder.clearCallingIdentity(); try { @@ -619,22 +646,24 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ Region partialInteractiveRegion = Region.obtain(); MagnificationSpec spec; synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return null; } resolvedWindowId = resolveAccessibilityWindowIdForFindFocusLocked( accessibilityWindowId, focusType); final boolean permissionGranted = - mSecurityPolicy.canGetAccessibilityNodeInfoLocked(this, resolvedWindowId); + mSecurityPolicy.canGetAccessibilityNodeInfoLocked( + mSystemSupport.getCurrentUserIdLocked(), this, resolvedWindowId); if (!permissionGranted) { return null; } else { - connection = mSystemSupport.getConnectionLocked(resolvedWindowId); + connection = mA11yWindowManager.getConnectionLocked( + mSystemSupport.getCurrentUserIdLocked(), resolvedWindowId); if (connection == null) { return null; } } - if (!mSecurityPolicy.computePartialInteractiveRegionForWindowLocked( + if (!mA11yWindowManager.computePartialInteractiveRegionForWindowLocked( resolvedWindowId, partialInteractiveRegion)) { partialInteractiveRegion.recycle(); partialInteractiveRegion = null; @@ -645,7 +674,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ return null; } final int interrogatingPid = Binder.getCallingPid(); - callback = mSystemSupport.replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, + callback = replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, interrogatingPid, interrogatingTid); final long identityToken = Binder.clearCallingIdentity(); try { @@ -678,21 +707,23 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ Region partialInteractiveRegion = Region.obtain(); MagnificationSpec spec; synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return null; } resolvedWindowId = resolveAccessibilityWindowIdLocked(accessibilityWindowId); final boolean permissionGranted = - mSecurityPolicy.canGetAccessibilityNodeInfoLocked(this, resolvedWindowId); + mSecurityPolicy.canGetAccessibilityNodeInfoLocked( + mSystemSupport.getCurrentUserIdLocked(), this, resolvedWindowId); if (!permissionGranted) { return null; } else { - connection = mSystemSupport.getConnectionLocked(resolvedWindowId); + connection = mA11yWindowManager.getConnectionLocked( + mSystemSupport.getCurrentUserIdLocked(), resolvedWindowId); if (connection == null) { return null; } } - if (!mSecurityPolicy.computePartialInteractiveRegionForWindowLocked( + if (!mA11yWindowManager.computePartialInteractiveRegionForWindowLocked( resolvedWindowId, partialInteractiveRegion)) { partialInteractiveRegion.recycle(); partialInteractiveRegion = null; @@ -703,7 +734,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ return null; } final int interrogatingPid = Binder.getCallingPid(); - callback = mSystemSupport.replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, + callback = replaceCallbackIfNeeded(callback, resolvedWindowId, interactionId, interrogatingPid, interrogatingTid); final long identityToken = Binder.clearCallingIdentity(); try { @@ -731,38 +762,51 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } @Override + public void dispatchGesture(int sequence, ParceledListSlice gestureSteps, int displayId) { + } + + @Override public boolean performAccessibilityAction(int accessibilityWindowId, long accessibilityNodeId, int action, Bundle arguments, int interactionId, IAccessibilityInteractionConnectionCallback callback, long interrogatingTid) throws RemoteException { final int resolvedWindowId; - IAccessibilityInteractionConnection connection = null; synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return false; } resolvedWindowId = resolveAccessibilityWindowIdLocked(accessibilityWindowId); - if (!mSecurityPolicy.canGetAccessibilityNodeInfoLocked(this, resolvedWindowId)) { + if (!mSecurityPolicy.canGetAccessibilityNodeInfoLocked( + mSystemSupport.getCurrentUserIdLocked(), this, resolvedWindowId)) { return false; } } if (!mSecurityPolicy.checkAccessibilityAccess(this)) { return false; } - boolean returnValue = - mSystemSupport.performAccessibilityAction(resolvedWindowId, accessibilityNodeId, + return performAccessibilityActionInternal( + mSystemSupport.getCurrentUserIdLocked(), resolvedWindowId, accessibilityNodeId, action, arguments, interactionId, callback, mFetchFlags, interrogatingTid); - return returnValue; } @Override public boolean performGlobalAction(int action) { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return false; } } - return mGlobalActionPerformer.performGlobalAction(action); + return mSystemActionPerformer.performSystemAction(action); + } + + @Override + public @NonNull List<AccessibilityNodeInfo.AccessibilityAction> getSystemActions() { + synchronized (mLock) { + if (!hasRightsToCurrentUserLocked()) { + return Collections.emptyList(); + } + } + return mSystemActionPerformer.getSystemActions(); } @Override @@ -781,7 +825,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ @Override public float getMagnificationScale(int displayId) { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return 1.0f; } } @@ -797,7 +841,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ public Region getMagnificationRegion(int displayId) { synchronized (mLock) { final Region region = Region.obtain(); - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return region; } MagnificationController magnificationController = @@ -820,7 +864,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ @Override public float getMagnificationCenterX(int displayId) { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return 0.0f; } MagnificationController magnificationController = @@ -842,7 +886,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ @Override public float getMagnificationCenterY(int displayId) { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return 0.0f; } MagnificationController magnificationController = @@ -874,7 +918,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ @Override public boolean resetMagnification(int displayId, boolean animate) { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return false; } if (!mSecurityPolicy.canControlMagnification(this)) { @@ -896,7 +940,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ public boolean setMagnificationScaleAndCenter(int displayId, float scale, float centerX, float centerY, boolean animate) { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return false; } if (!mSecurityPolicy.canControlMagnification(this)) { @@ -932,6 +976,100 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } @Override + public void takeScreenshot(int displayId, RemoteCallback callback) { + final long currentTimestamp = SystemClock.uptimeMillis(); + if (mRequestTakeScreenshotTimestampMs != 0 + && (currentTimestamp - mRequestTakeScreenshotTimestampMs) + <= AccessibilityService.ACCESSIBILITY_TAKE_SCREENSHOT_REQUEST_INTERVAL_TIMES_MS) { + sendScreenshotFailure(AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT, + callback); + return; + } + mRequestTakeScreenshotTimestampMs = currentTimestamp; + + synchronized (mLock) { + if (!hasRightsToCurrentUserLocked()) { + sendScreenshotFailure(AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR, + callback); + return; + } + + if (!mSecurityPolicy.canTakeScreenshotLocked(this)) { + throw new SecurityException("Services don't have the capability of taking" + + " the screenshot."); + } + } + + if (!mSecurityPolicy.checkAccessibilityAccess(this)) { + sendScreenshotFailure( + AccessibilityService.ERROR_TAKE_SCREENSHOT_NO_ACCESSIBILITY_ACCESS, + callback); + return; + } + + // Private virtual displays are created by the ap and is not allowed to access by other + // aps. We assume the contents on this display should not be captured. + final DisplayManager displayManager = + (DisplayManager) mContext.getSystemService(Context.DISPLAY_SERVICE); + final Display display = displayManager.getDisplay(displayId); + if ((display == null) || (display.getType() == Display.TYPE_VIRTUAL + && (display.getFlags() & Display.FLAG_PRIVATE) != 0)) { + sendScreenshotFailure( + AccessibilityService.ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY, callback); + return; + } + + final long identity = Binder.clearCallingIdentity(); + try { + mMainHandler.post(PooledLambda.obtainRunnable((nonArg) -> { + final ScreenshotGraphicBuffer screenshotBuffer = LocalServices + .getService(DisplayManagerInternal.class).userScreenshot(displayId); + if (screenshotBuffer != null) { + sendScreenshotSuccess(screenshotBuffer, callback); + } else { + sendScreenshotFailure( + AccessibilityService.ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY, callback); + } + }, null).recycleOnUse()); + } finally { + Binder.restoreCallingIdentity(identity); + } + } + + private void sendScreenshotSuccess(ScreenshotGraphicBuffer screenshotBuffer, + RemoteCallback callback) { + final GraphicBuffer graphicBuffer = screenshotBuffer.getGraphicBuffer(); + try (HardwareBuffer hardwareBuffer = + HardwareBuffer.createFromGraphicBuffer(graphicBuffer)) { + final ParcelableColorSpace colorSpace = + new ParcelableColorSpace(screenshotBuffer.getColorSpace()); + + final Bundle payload = new Bundle(); + payload.putInt(KEY_ACCESSIBILITY_SCREENSHOT_STATUS, + AccessibilityService.TAKE_SCREENSHOT_SUCCESS); + payload.putParcelable(KEY_ACCESSIBILITY_SCREENSHOT_HARDWAREBUFFER, + hardwareBuffer); + payload.putParcelable(KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE, colorSpace); + payload.putLong(KEY_ACCESSIBILITY_SCREENSHOT_TIMESTAMP, + SystemClock.uptimeMillis()); + + // Send back the result. + callback.sendResult(payload); + hardwareBuffer.close(); + } + } + + private void sendScreenshotFailure(@AccessibilityService.ScreenshotErrorCode int errorCode, + RemoteCallback callback) { + mMainHandler.post(PooledLambda.obtainRunnable((nonArg) -> { + final Bundle payload = new Bundle(); + payload.putInt(KEY_ACCESSIBILITY_SCREENSHOT_STATUS, errorCode); + // Send back the result. + callback.sendResult(payload); + }, null).recycleOnUse()); + } + + @Override public void dump(FileDescriptor fd, final PrintWriter pw, String[] args) { if (!DumpUtils.checkDumpPermission(mContext, LOG_TAG, pw)) return; synchronized (mLock) { @@ -943,29 +1081,92 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ pw.append(", eventTypes=" + AccessibilityEvent.eventTypeToString(mEventTypes)); pw.append(", notificationTimeout=" + mNotificationTimeout); + pw.append(", requestA11yBtn=" + mRequestAccessibilityButton); pw.append("]"); } } public void onAdded() { + final Display[] displays = mDisplayManager.getDisplays(); + for (int i = 0; i < displays.length; i++) { + final int displayId = displays[i].getDisplayId(); + onDisplayAdded(displayId); + } + } + + /** + * Called whenever a logical display has been added to the system. Add a window token for adding + * an accessibility overlay. + * + * @param displayId The id of the logical display that was added. + */ + public void onDisplayAdded(int displayId) { final long identity = Binder.clearCallingIdentity(); try { - mWindowManagerService.addWindowToken(mOverlayWindowToken, - TYPE_ACCESSIBILITY_OVERLAY, DEFAULT_DISPLAY); + final IBinder overlayWindowToken = new Binder(); + mWindowManagerService.addWindowToken(overlayWindowToken, TYPE_ACCESSIBILITY_OVERLAY, + displayId); + synchronized (mLock) { + mOverlayWindowTokens.put(displayId, overlayWindowToken); + } } finally { Binder.restoreCallingIdentity(identity); } } public void onRemoved() { + final Display[] displays = mDisplayManager.getDisplays(); + for (int i = 0; i < displays.length; i++) { + final int displayId = displays[i].getDisplayId(); + onDisplayRemoved(displayId); + } + } + + /** + * Called whenever a logical display has been removed from the system. Remove a window token for + * removing an accessibility overlay. + * + * @param displayId The id of the logical display that was added. + */ + public void onDisplayRemoved(int displayId) { final long identity = Binder.clearCallingIdentity(); try { - mWindowManagerService.removeWindowToken(mOverlayWindowToken, true, DEFAULT_DISPLAY); + mWindowManagerService.removeWindowToken(mOverlayWindowTokens.get(displayId), true, + displayId); + synchronized (mLock) { + mOverlayWindowTokens.remove(displayId); + } } finally { Binder.restoreCallingIdentity(identity); } } + /** + * Gets overlay window token by the display Id. + * + * @param displayId The id of the logical display that was added. + * @return window token. + */ + @Override + public IBinder getOverlayWindowToken(int displayId) { + synchronized (mLock) { + return mOverlayWindowTokens.get(displayId); + } + } + + /** + * Gets windowId of given token. + * + * @param token The token + * @return window id + */ + @Override + public int getWindowIdForLeashToken(@NonNull IBinder token) { + synchronized (mLock) { + return mA11yWindowManager.getWindowIdLocked(token); + } + } + public void resetLocked() { mSystemSupport.getKeyEventDispatcher().flush(this); try { @@ -1127,9 +1328,14 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } } - public void notifyGesture(int gestureId) { + public void notifyGesture(AccessibilityGestureEvent gestureEvent) { mInvocationHandler.obtainMessage(InvocationHandler.MSG_ON_GESTURE, - gestureId, 0).sendToTarget(); + gestureEvent).sendToTarget(); + } + + public void notifySystemActionsChangedLocked() { + mInvocationHandler.sendEmptyMessage( + InvocationHandler.MSG_ON_SYSTEM_ACTIONS_CHANGED); } public void notifyClearAccessibilityNodeInfoCache() { @@ -1147,8 +1353,8 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mInvocationHandler.notifySoftKeyboardShowModeChangedLocked(showState); } - public void notifyAccessibilityButtonClickedLocked() { - mInvocationHandler.notifyAccessibilityButtonClickedLocked(); + public void notifyAccessibilityButtonClickedLocked(int displayId) { + mInvocationHandler.notifyAccessibilityButtonClickedLocked(displayId); } public void notifyAccessibilityButtonAvailabilityChangedLocked(boolean available) { @@ -1187,11 +1393,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } } - private void notifyAccessibilityButtonClickedInternal() { + private void notifyAccessibilityButtonClickedInternal(int displayId) { final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); if (listener != null) { try { - listener.onAccessibilityButtonClicked(); + listener.onAccessibilityButtonClicked(displayId); } catch (RemoteException re) { Slog.e(LOG_TAG, "Error sending accessibility button click to " + mService, re); } @@ -1218,18 +1424,30 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } } - private void notifyGestureInternal(int gestureId) { + private void notifyGestureInternal(AccessibilityGestureEvent gestureInfo) { final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); if (listener != null) { try { - listener.onGesture(gestureId); + listener.onGesture(gestureInfo); } catch (RemoteException re) { - Slog.e(LOG_TAG, "Error during sending gesture " + gestureId + Slog.e(LOG_TAG, "Error during sending gesture " + gestureInfo + " to " + mService, re); } } } + private void notifySystemActionsChangedInternal() { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + listener.onSystemActionsChanged(); + } catch (RemoteException re) { + Slog.e(LOG_TAG, "Error sending system actions change to " + mService, + re); + } + } + } + private void notifyClearAccessibilityCacheInternal() { final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); if (listener != null) { @@ -1250,25 +1468,168 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ private int resolveAccessibilityWindowIdLocked(int accessibilityWindowId) { if (accessibilityWindowId == AccessibilityWindowInfo.ACTIVE_WINDOW_ID) { - return mSecurityPolicy.getActiveWindowId(); + return mA11yWindowManager.getActiveWindowId(mSystemSupport.getCurrentUserIdLocked()); } return accessibilityWindowId; } private int resolveAccessibilityWindowIdForFindFocusLocked(int windowId, int focusType) { if (windowId == AccessibilityWindowInfo.ACTIVE_WINDOW_ID) { - return mSecurityPolicy.mActiveWindowId; + return mA11yWindowManager.getActiveWindowId(mSystemSupport.getCurrentUserIdLocked()); } if (windowId == AccessibilityWindowInfo.ANY_WINDOW_ID) { - if (focusType == AccessibilityNodeInfo.FOCUS_INPUT) { - return mSecurityPolicy.mFocusedWindowId; - } else if (focusType == AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) { - return mSecurityPolicy.mAccessibilityFocusedWindowId; - } + return mA11yWindowManager.getFocusedWindowId(focusType); } return windowId; } + /** + * Request that the system make sure windows are available to interrogate. + * + * @param displayId The logical display id. + */ + private void ensureWindowsAvailableTimedLocked(int displayId) { + if (mA11yWindowManager.getWindowListLocked(displayId) != null) { + return; + } + // If we have no registered callback, update the state we + // we may have to register one but it didn't happen yet. + if (!mA11yWindowManager.isTrackingWindowsLocked(displayId)) { + // Invokes client change to make sure tracking window enabled. + mSystemSupport.onClientChangeLocked(false); + } + // We have no windows but do not care about them, done. + if (!mA11yWindowManager.isTrackingWindowsLocked(displayId)) { + return; + } + + // Wait for the windows with a timeout. + final long startMillis = SystemClock.uptimeMillis(); + while (mA11yWindowManager.getWindowListLocked(displayId) == null) { + final long elapsedMillis = SystemClock.uptimeMillis() - startMillis; + final long remainMillis = WAIT_WINDOWS_TIMEOUT_MILLIS - elapsedMillis; + if (remainMillis <= 0) { + return; + } + try { + mLock.wait(remainMillis); + } catch (InterruptedException ie) { + /* ignore */ + } + } + } + + /** + * Perform the specified accessibility action + * + * @param resolvedWindowId The window ID + * [Other parameters match the method on IAccessibilityServiceConnection] + * + * @return Whether or not the action could be sent to the app process + */ + private boolean performAccessibilityActionInternal(int userId, int resolvedWindowId, + long accessibilityNodeId, int action, Bundle arguments, int interactionId, + IAccessibilityInteractionConnectionCallback callback, int fetchFlags, + long interrogatingTid) { + RemoteAccessibilityConnection connection; + IBinder activityToken = null; + synchronized (mLock) { + connection = mA11yWindowManager.getConnectionLocked(userId, resolvedWindowId); + if (connection == null) { + return false; + } + final boolean isA11yFocusAction = (action == ACTION_ACCESSIBILITY_FOCUS) + || (action == ACTION_CLEAR_ACCESSIBILITY_FOCUS); + if (!isA11yFocusAction) { + final WindowInfo windowInfo = + mA11yWindowManager.findWindowInfoByIdLocked(resolvedWindowId); + if (windowInfo != null) activityToken = windowInfo.activityToken; + } + final AccessibilityWindowInfo a11yWindowInfo = + mA11yWindowManager.findA11yWindowInfoByIdLocked(resolvedWindowId); + if (a11yWindowInfo != null && a11yWindowInfo.isInPictureInPictureMode() + && mA11yWindowManager.getPictureInPictureActionReplacingConnection() != null + && !isA11yFocusAction) { + connection = mA11yWindowManager.getPictureInPictureActionReplacingConnection(); + } + } + final int interrogatingPid = Binder.getCallingPid(); + final long identityToken = Binder.clearCallingIdentity(); + try { + // Regardless of whether or not the action succeeds, it was generated by an + // accessibility service that is driven by user actions, so note user activity. + mPowerManager.userActivity(SystemClock.uptimeMillis(), + PowerManager.USER_ACTIVITY_EVENT_ACCESSIBILITY, 0); + + if (action == ACTION_CLICK || action == ACTION_LONG_CLICK) { + mA11yWindowManager.notifyOutsideTouch(userId, resolvedWindowId); + } + if (activityToken != null) { + LocalServices.getService(ActivityTaskManagerInternal.class) + .setFocusedActivity(activityToken); + } + connection.getRemote().performAccessibilityAction(accessibilityNodeId, action, + arguments, interactionId, callback, fetchFlags, interrogatingPid, + interrogatingTid); + } catch (RemoteException re) { + if (DEBUG) { + Slog.e(LOG_TAG, "Error calling performAccessibilityAction: " + re); + } + return false; + } finally { + Binder.restoreCallingIdentity(identityToken); + } + return true; + } + + /** + * Replace the interaction callback if needed, for example if the window is in picture- + * in-picture mode and needs its nodes replaced. + * + * @param originalCallback The callback we were planning to use + * @param resolvedWindowId The ID of the window we're calling + * @param interactionId The id for the original callback + * @param interrogatingPid Process ID of requester + * @param interrogatingTid Thread ID of requester + * + * @return The callback to use, which may be the original one. + */ + private IAccessibilityInteractionConnectionCallback replaceCallbackIfNeeded( + IAccessibilityInteractionConnectionCallback originalCallback, int resolvedWindowId, + int interactionId, int interrogatingPid, long interrogatingTid) { + final RemoteAccessibilityConnection pipActionReplacingConnection = + mA11yWindowManager.getPictureInPictureActionReplacingConnection(); + synchronized (mLock) { + final AccessibilityWindowInfo windowInfo = + mA11yWindowManager.findA11yWindowInfoByIdLocked(resolvedWindowId); + if ((windowInfo == null) || !windowInfo.isInPictureInPictureMode() + || (pipActionReplacingConnection == null)) { + return originalCallback; + } + } + return new ActionReplacingCallback(originalCallback, + pipActionReplacingConnection.getRemote(), interactionId, + interrogatingPid, interrogatingTid); + } + + private List<AccessibilityWindowInfo> getWindowsByDisplayLocked(int displayId) { + final List<AccessibilityWindowInfo> internalWindowList = + mA11yWindowManager.getWindowListLocked(displayId); + if (internalWindowList == null) { + return null; + } + final List<AccessibilityWindowInfo> returnedWindowList = new ArrayList<>(); + final int windowCount = internalWindowList.size(); + for (int i = 0; i < windowCount; i++) { + AccessibilityWindowInfo window = internalWindowList.get(i); + AccessibilityWindowInfo windowClone = + AccessibilityWindowInfo.obtain(window); + windowClone.setConnectionId(mId); + returnedWindowList.add(windowClone); + } + return returnedWindowList; + } + public ComponentName getComponentName() { return mComponentName; } @@ -1281,6 +1642,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ private static final int MSG_ON_SOFT_KEYBOARD_STATE_CHANGED = 6; private static final int MSG_ON_ACCESSIBILITY_BUTTON_CLICKED = 7; private static final int MSG_ON_ACCESSIBILITY_BUTTON_AVAILABILITY_CHANGED = 8; + private static final int MSG_ON_SYSTEM_ACTIONS_CHANGED = 9; /** List of magnification callback states, mapping from displayId -> Boolean */ @GuardedBy("mlock") @@ -1296,8 +1658,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ final int type = message.what; switch (type) { case MSG_ON_GESTURE: { - final int gestureId = message.arg1; - notifyGestureInternal(gestureId); + notifyGestureInternal((AccessibilityGestureEvent) message.obj); } break; case MSG_CLEAR_ACCESSIBILITY_CACHE: { @@ -1321,14 +1682,18 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } break; case MSG_ON_ACCESSIBILITY_BUTTON_CLICKED: { - notifyAccessibilityButtonClickedInternal(); + final int displayId = (int) message.arg1; + notifyAccessibilityButtonClickedInternal(displayId); } break; case MSG_ON_ACCESSIBILITY_BUTTON_AVAILABILITY_CHANGED: { final boolean available = (message.arg1 != 0); notifyAccessibilityButtonAvailabilityChangedInternal(available); } break; - + case MSG_ON_SYSTEM_ACTIONS_CHANGED: { + notifySystemActionsChangedInternal(); + break; + } default: { throw new IllegalArgumentException("Unknown message: " + type); } @@ -1383,8 +1748,8 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mIsSoftKeyboardCallbackEnabled = enabled; } - public void notifyAccessibilityButtonClickedLocked() { - final Message msg = obtainMessage(MSG_ON_ACCESSIBILITY_BUTTON_CLICKED); + public void notifyAccessibilityButtonClickedLocked(int displayId) { + final Message msg = obtainMessage(MSG_ON_ACCESSIBILITY_BUTTON_CLICKED, displayId, 0); msg.sendToTarget(); } @@ -1394,4 +1759,22 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ msg.sendToTarget(); } } + + public boolean isServiceHandlesDoubleTapEnabled() { + return mServiceHandlesDoubleTap; + } + + public boolean isMultiFingerGesturesEnabled() { + return mRequestMultiFingerGestures; + } + + @Override + public void setGestureDetectionPassthroughRegion(int displayId, Region region) { + mSystemSupport.setGestureDetectionPassthroughRegion(displayId, region); + } + + @Override + public void setTouchExplorationPassthroughRegion(int displayId, Region region) { + mSystemSupport.setTouchExplorationPassthroughRegion(displayId, region); + } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityGestureDetector.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityGestureDetector.java deleted file mode 100644 index d7670112d55c..000000000000 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityGestureDetector.java +++ /dev/null @@ -1,639 +0,0 @@ -/* - ** Copyright 2015, The Android Open Source Project - ** - ** Licensed under the Apache License, Version 2.0 (the "License"); - ** you may not use this file except in compliance with the License. - ** You may obtain a copy of the License at - ** - ** http://www.apache.org/licenses/LICENSE-2.0 - ** - ** Unless required by applicable law or agreed to in writing, software - ** distributed under the License is distributed on an "AS IS" BASIS, - ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ** See the License for the specific language governing permissions and - ** limitations under the License. - */ - -package com.android.server.accessibility; - -import android.accessibilityservice.AccessibilityService; -import android.content.Context; -import android.gesture.Gesture; -import android.gesture.GesturePoint; -import android.gesture.GestureStore; -import android.gesture.GestureStroke; -import android.gesture.Prediction; -import android.graphics.PointF; -import android.util.Slog; -import android.util.TypedValue; -import android.view.GestureDetector; -import android.view.MotionEvent; -import android.view.ViewConfiguration; - -import com.android.internal.R; - -import java.util.ArrayList; - -/** - * This class handles gesture detection for the Touch Explorer. It collects - * touch events and determines when they match a gesture, as well as when they - * won't match a gesture. These state changes are then surfaced to mListener. - */ -class AccessibilityGestureDetector extends GestureDetector.SimpleOnGestureListener { - - private static final boolean DEBUG = false; - - // Tag for logging received events. - private static final String LOG_TAG = "AccessibilityGestureDetector"; - - // Constants for sampling motion event points. - // We sample based on a minimum distance between points, primarily to improve accuracy by - // reducing noisy minor changes in direction. - private static final float MIN_INCHES_BETWEEN_SAMPLES = 0.1f; - private final float mMinPixelsBetweenSamplesX; - private final float mMinPixelsBetweenSamplesY; - - // Constants for separating gesture segments - private static final float ANGLE_THRESHOLD = 0.0f; - - // Constants for line segment directions - private static final int LEFT = 0; - private static final int RIGHT = 1; - private static final int UP = 2; - private static final int DOWN = 3; - private static final int[][] DIRECTIONS_TO_GESTURE_ID = { - { - AccessibilityService.GESTURE_SWIPE_LEFT, - AccessibilityService.GESTURE_SWIPE_LEFT_AND_RIGHT, - AccessibilityService.GESTURE_SWIPE_LEFT_AND_UP, - AccessibilityService.GESTURE_SWIPE_LEFT_AND_DOWN - }, - { - AccessibilityService.GESTURE_SWIPE_RIGHT_AND_LEFT, - AccessibilityService.GESTURE_SWIPE_RIGHT, - AccessibilityService.GESTURE_SWIPE_RIGHT_AND_UP, - AccessibilityService.GESTURE_SWIPE_RIGHT_AND_DOWN - }, - { - AccessibilityService.GESTURE_SWIPE_UP_AND_LEFT, - AccessibilityService.GESTURE_SWIPE_UP_AND_RIGHT, - AccessibilityService.GESTURE_SWIPE_UP, - AccessibilityService.GESTURE_SWIPE_UP_AND_DOWN - }, - { - AccessibilityService.GESTURE_SWIPE_DOWN_AND_LEFT, - AccessibilityService.GESTURE_SWIPE_DOWN_AND_RIGHT, - AccessibilityService.GESTURE_SWIPE_DOWN_AND_UP, - AccessibilityService.GESTURE_SWIPE_DOWN - } - }; - - - /** - * Listener functions are called as a result of onMoveEvent(). The current - * MotionEvent in the context of these functions is the event passed into - * onMotionEvent. - */ - public interface Listener { - /** - * Called when the user has performed a double tap and then held down - * the second tap. - * - * @param event The most recent MotionEvent received. - * @param policyFlags The policy flags of the most recent event. - */ - void onDoubleTapAndHold(MotionEvent event, int policyFlags); - - /** - * Called when the user lifts their finger on the second tap of a double - * tap. - * - * @param event The most recent MotionEvent received. - * @param policyFlags The policy flags of the most recent event. - * - * @return true if the event is consumed, else false - */ - boolean onDoubleTap(MotionEvent event, int policyFlags); - - /** - * Called when the system has decided the event stream is a gesture. - * - * @return true if the event is consumed, else false - */ - boolean onGestureStarted(); - - /** - * Called when an event stream is recognized as a gesture. - * - * @param gestureId ID of the gesture that was recognized. - * - * @return true if the event is consumed, else false - */ - boolean onGestureCompleted(int gestureId); - - /** - * Called when the system has decided an event stream doesn't match any - * known gesture. - * - * @param event The most recent MotionEvent received. - * @param policyFlags The policy flags of the most recent event. - * - * @return true if the event is consumed, else false - */ - public boolean onGestureCancelled(MotionEvent event, int policyFlags); - } - - private final Listener mListener; - private final Context mContext; // Retained for on-demand construction of GestureDetector. - private final GestureDetector mGestureDetector; // Double-tap detector. - - // Indicates that a single tap has occurred. - private boolean mFirstTapDetected; - - // Indicates that the down event of a double tap has occured. - private boolean mDoubleTapDetected; - - // Indicates that motion events are being collected to match a gesture. - private boolean mRecognizingGesture; - - // Indicates that we've collected enough data to be sure it could be a - // gesture. - private boolean mGestureStarted; - - // Indicates that motion events from the second pointer are being checked - // for a double tap. - private boolean mSecondFingerDoubleTap; - - // Tracks the most recent time where ACTION_POINTER_DOWN was sent for the - // second pointer. - private long mSecondPointerDownTime; - - // Policy flags of the previous event. - private int mPolicyFlags; - - // These values track the previous point that was saved to use for gesture - // detection. They are only updated when the user moves more than the - // recognition threshold. - private float mPreviousGestureX; - private float mPreviousGestureY; - - // These values track the previous point that was used to determine if there - // was a transition into or out of gesture detection. They are updated when - // the user moves more than the detection threshold. - private float mBaseX; - private float mBaseY; - private long mBaseTime; - - // This is the calculated movement threshold used track if the user is still - // moving their finger. - private final float mGestureDetectionThreshold; - - // Buffer for storing points for gesture detection. - private final ArrayList<GesturePoint> mStrokeBuffer = new ArrayList<GesturePoint>(100); - - // The minimal delta between moves to add a gesture point. - private static final int TOUCH_TOLERANCE = 3; - - // The minimal score for accepting a predicted gesture. - private static final float MIN_PREDICTION_SCORE = 2.0f; - - // Distance a finger must travel before we decide if it is a gesture or not. - private static final int GESTURE_CONFIRM_MM = 10; - - // Time threshold used to determine if an interaction is a gesture or not. - // If the first movement of 1cm takes longer than this value, we assume it's - // a slow movement, and therefore not a gesture. - // - // This value was determined by measuring the time for the first 1cm - // movement when gesturing, and touch exploring. Based on user testing, - // all gestures started with the initial movement taking less than 100ms. - // When touch exploring, the first movement almost always takes longer than - // 200ms. - private static final long CANCEL_ON_PAUSE_THRESHOLD_NOT_STARTED_MS = 150; - - // Time threshold used to determine if a gesture should be cancelled. If - // the finger takes more than this time to move 1cm, the ongoing gesture is - // cancelled. - private static final long CANCEL_ON_PAUSE_THRESHOLD_STARTED_MS = 300; - - /** - * Construct the gesture detector for {@link TouchExplorer}. - * - * @see #AccessibilityGestureDetector(Context, Listener, GestureDetector) - */ - AccessibilityGestureDetector(Context context, Listener listener) { - this(context, listener, null); - } - - /** - * Construct the gesture detector for {@link TouchExplorer}. - * - * @param context A context handle for accessing resources. - * @param listener A listener to callback with gesture state or information. - * @param detector The gesture detector to handle touch event. If null the default one created - * in place, or for testing purpose. - */ - AccessibilityGestureDetector(Context context, Listener listener, GestureDetector detector) { - mListener = listener; - mContext = context; - - // Break the circular dependency between constructors and let the class to be testable - if (detector == null) { - mGestureDetector = new GestureDetector(context, this); - } else { - mGestureDetector = detector; - } - mGestureDetector.setOnDoubleTapListener(this); - mGestureDetectionThreshold = TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_MM, 1, - context.getResources().getDisplayMetrics()) * GESTURE_CONFIRM_MM; - - // Calculate minimum gesture velocity - final float pixelsPerInchX = context.getResources().getDisplayMetrics().xdpi; - final float pixelsPerInchY = context.getResources().getDisplayMetrics().ydpi; - mMinPixelsBetweenSamplesX = MIN_INCHES_BETWEEN_SAMPLES * pixelsPerInchX; - mMinPixelsBetweenSamplesY = MIN_INCHES_BETWEEN_SAMPLES * pixelsPerInchY; - } - - /** - * Handle a motion event. If an action is completed, the appropriate - * callback on mListener is called, and the return value of the callback is - * passed to the caller. - * - * @param event The transformed motion event to be handled. - * @param rawEvent The raw motion event. It's important that this be the raw - * event, before any transformations have been applied, so that measurements - * can be made in physical units. - * @param policyFlags Policy flags for the event. - * - * @return true if the event is consumed, else false - */ - public boolean onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { - // The accessibility gesture detector is interested in the movements in physical space, - // so it uses the rawEvent to ignore magnification and other transformations. - final float x = rawEvent.getX(); - final float y = rawEvent.getY(); - final long time = rawEvent.getEventTime(); - - mPolicyFlags = policyFlags; - switch (rawEvent.getActionMasked()) { - case MotionEvent.ACTION_DOWN: - mDoubleTapDetected = false; - mSecondFingerDoubleTap = false; - mRecognizingGesture = true; - mGestureStarted = false; - mPreviousGestureX = x; - mPreviousGestureY = y; - mStrokeBuffer.clear(); - mStrokeBuffer.add(new GesturePoint(x, y, time)); - - mBaseX = x; - mBaseY = y; - mBaseTime = time; - break; - - case MotionEvent.ACTION_MOVE: - if (mRecognizingGesture) { - final float deltaX = mBaseX - x; - final float deltaY = mBaseY - y; - final double moveDelta = Math.hypot(deltaX, deltaY); - if (moveDelta > mGestureDetectionThreshold) { - // If the pointer has moved more than the threshold, - // update the stored values. - mBaseX = x; - mBaseY = y; - mBaseTime = time; - - // Since the pointer has moved, this is not a double - // tap. - mFirstTapDetected = false; - mDoubleTapDetected = false; - - // If this hasn't been confirmed as a gesture yet, send - // the event. - if (!mGestureStarted) { - mGestureStarted = true; - return mListener.onGestureStarted(); - } - } else if (!mFirstTapDetected) { - // The finger may not move if they are double tapping. - // In that case, we shouldn't cancel the gesture. - final long timeDelta = time - mBaseTime; - final long threshold = mGestureStarted ? - CANCEL_ON_PAUSE_THRESHOLD_STARTED_MS : - CANCEL_ON_PAUSE_THRESHOLD_NOT_STARTED_MS; - - // If the pointer hasn't moved for longer than the - // timeout, cancel gesture detection. - if (timeDelta > threshold) { - cancelGesture(); - return mListener.onGestureCancelled(rawEvent, policyFlags); - } - } - - final float dX = Math.abs(x - mPreviousGestureX); - final float dY = Math.abs(y - mPreviousGestureY); - if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { - mPreviousGestureX = x; - mPreviousGestureY = y; - mStrokeBuffer.add(new GesturePoint(x, y, time)); - } - } - break; - - case MotionEvent.ACTION_UP: - if (mDoubleTapDetected) { - return finishDoubleTap(rawEvent, policyFlags); - } - if (mGestureStarted) { - final float dX = Math.abs(x - mPreviousGestureX); - final float dY = Math.abs(y - mPreviousGestureY); - if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { - mStrokeBuffer.add(new GesturePoint(x, y, time)); - } - return recognizeGesture(rawEvent, policyFlags); - } - break; - - case MotionEvent.ACTION_POINTER_DOWN: - // Once a second finger is used, we're definitely not - // recognizing a gesture. - cancelGesture(); - - if (rawEvent.getPointerCount() == 2) { - // If this was the second finger, attempt to recognize double - // taps on it. - mSecondFingerDoubleTap = true; - mSecondPointerDownTime = time; - } else { - // If there are more than two fingers down, stop watching - // for a double tap. - mSecondFingerDoubleTap = false; - } - break; - - case MotionEvent.ACTION_POINTER_UP: - // If we're detecting taps on the second finger, see if we - // should finish the double tap. - if (mSecondFingerDoubleTap && mDoubleTapDetected) { - return finishDoubleTap(rawEvent, policyFlags); - } - break; - - case MotionEvent.ACTION_CANCEL: - clear(); - break; - } - - // If we're detecting taps on the second finger, map events from the - // finger to the first finger. - if (mSecondFingerDoubleTap) { - MotionEvent newEvent = mapSecondPointerToFirstPointer(rawEvent); - if (newEvent == null) { - return false; - } - boolean handled = mGestureDetector.onTouchEvent(newEvent); - newEvent.recycle(); - return handled; - } - - if (!mRecognizingGesture) { - return false; - } - - // Pass the transformed event on to the standard gesture detector. - return mGestureDetector.onTouchEvent(event); - } - - public void clear() { - mFirstTapDetected = false; - mDoubleTapDetected = false; - mSecondFingerDoubleTap = false; - mGestureStarted = false; - mGestureDetector.onTouchEvent(MotionEvent.obtain(0L, 0L, MotionEvent.ACTION_CANCEL, - 0.0f, 0.0f, 0)); - cancelGesture(); - } - - public boolean firstTapDetected() { - return mFirstTapDetected; - } - - @Override - public void onLongPress(MotionEvent e) { - maybeSendLongPress(e, mPolicyFlags); - } - - @Override - public boolean onSingleTapUp(MotionEvent event) { - mFirstTapDetected = true; - return false; - } - - @Override - public boolean onSingleTapConfirmed(MotionEvent event) { - clear(); - return false; - } - - @Override - public boolean onDoubleTap(MotionEvent event) { - // The processing of the double tap is deferred until the finger is - // lifted, so that we can detect a long press on the second tap. - mDoubleTapDetected = true; - return false; - } - - private void maybeSendLongPress(MotionEvent event, int policyFlags) { - if (!mDoubleTapDetected) { - return; - } - - clear(); - - mListener.onDoubleTapAndHold(event, policyFlags); - } - - private boolean finishDoubleTap(MotionEvent event, int policyFlags) { - clear(); - - return mListener.onDoubleTap(event, policyFlags); - } - - private void cancelGesture() { - mRecognizingGesture = false; - mGestureStarted = false; - mStrokeBuffer.clear(); - } - - /** - * Looks at the sequence of motions in mStrokeBuffer, classifies the gesture, then calls - * Listener callbacks for success or failure. - * - * @param event The raw motion event to pass to the listener callbacks. - * @param policyFlags Policy flags for the event. - * - * @return true if the event is consumed, else false - */ - private boolean recognizeGesture(MotionEvent event, int policyFlags) { - if (mStrokeBuffer.size() < 2) { - return mListener.onGestureCancelled(event, policyFlags); - } - - // Look at mStrokeBuffer and extract 2 line segments, delimited by near-perpendicular - // direction change. - // Method: for each sampled motion event, check the angle of the most recent motion vector - // versus the preceding motion vector, and segment the line if the angle is about - // 90 degrees. - - ArrayList<PointF> path = new ArrayList<>(); - PointF lastDelimiter = new PointF(mStrokeBuffer.get(0).x, mStrokeBuffer.get(0).y); - path.add(lastDelimiter); - - float dX = 0; // Sum of unit vectors from last delimiter to each following point - float dY = 0; - int count = 0; // Number of points since last delimiter - float length = 0; // Vector length from delimiter to most recent point - - PointF next = new PointF(); - for (int i = 1; i < mStrokeBuffer.size(); ++i) { - next = new PointF(mStrokeBuffer.get(i).x, mStrokeBuffer.get(i).y); - if (count > 0) { - // Average of unit vectors from delimiter to following points - float currentDX = dX / count; - float currentDY = dY / count; - - // newDelimiter is a possible new delimiter, based on a vector with length from - // the last delimiter to the previous point, but in the direction of the average - // unit vector from delimiter to previous points. - // Using the averaged vector has the effect of "squaring off the curve", - // creating a sharper angle between the last motion and the preceding motion from - // the delimiter. In turn, this sharper angle achieves the splitting threshold - // even in a gentle curve. - PointF newDelimiter = new PointF(length * currentDX + lastDelimiter.x, - length * currentDY + lastDelimiter.y); - - // Unit vector from newDelimiter to the most recent point - float nextDX = next.x - newDelimiter.x; - float nextDY = next.y - newDelimiter.y; - float nextLength = (float) Math.sqrt(nextDX * nextDX + nextDY * nextDY); - nextDX = nextDX / nextLength; - nextDY = nextDY / nextLength; - - // Compare the initial motion direction to the most recent motion direction, - // and segment the line if direction has changed by about 90 degrees. - float dot = currentDX * nextDX + currentDY * nextDY; - if (dot < ANGLE_THRESHOLD) { - path.add(newDelimiter); - lastDelimiter = newDelimiter; - dX = 0; - dY = 0; - count = 0; - } - } - - // Vector from last delimiter to most recent point - float currentDX = next.x - lastDelimiter.x; - float currentDY = next.y - lastDelimiter.y; - length = (float) Math.sqrt(currentDX * currentDX + currentDY * currentDY); - - // Increment sum of unit vectors from delimiter to each following point - count = count + 1; - dX = dX + currentDX / length; - dY = dY + currentDY / length; - } - - path.add(next); - Slog.i(LOG_TAG, "path=" + path.toString()); - - // Classify line segments, and call Listener callbacks. - return recognizeGesturePath(event, policyFlags, path); - } - - /** - * Classifies a pair of line segments, by direction. - * Calls Listener callbacks for success or failure. - * - * @param event The raw motion event to pass to the listener's onGestureCanceled method. - * @param policyFlags Policy flags for the event. - * @param path A sequence of motion line segments derived from motion points in mStrokeBuffer. - * - * @return true if the event is consumed, else false - */ - private boolean recognizeGesturePath(MotionEvent event, int policyFlags, - ArrayList<PointF> path) { - - if (path.size() == 2) { - PointF start = path.get(0); - PointF end = path.get(1); - - float dX = end.x - start.x; - float dY = end.y - start.y; - int direction = toDirection(dX, dY); - switch (direction) { - case LEFT: - return mListener.onGestureCompleted(AccessibilityService.GESTURE_SWIPE_LEFT); - case RIGHT: - return mListener.onGestureCompleted(AccessibilityService.GESTURE_SWIPE_RIGHT); - case UP: - return mListener.onGestureCompleted(AccessibilityService.GESTURE_SWIPE_UP); - case DOWN: - return mListener.onGestureCompleted(AccessibilityService.GESTURE_SWIPE_DOWN); - default: - // Do nothing. - } - - } else if (path.size() == 3) { - PointF start = path.get(0); - PointF mid = path.get(1); - PointF end = path.get(2); - - float dX0 = mid.x - start.x; - float dY0 = mid.y - start.y; - - float dX1 = end.x - mid.x; - float dY1 = end.y - mid.y; - - int segmentDirection0 = toDirection(dX0, dY0); - int segmentDirection1 = toDirection(dX1, dY1); - int gestureId = DIRECTIONS_TO_GESTURE_ID[segmentDirection0][segmentDirection1]; - return mListener.onGestureCompleted(gestureId); - } - // else if (path.size() < 2 || 3 < path.size()) then no gesture recognized. - return mListener.onGestureCancelled(event, policyFlags); - } - - /** Maps a vector to a dominant direction in set {LEFT, RIGHT, UP, DOWN}. */ - private static int toDirection(float dX, float dY) { - if (Math.abs(dX) > Math.abs(dY)) { - // Horizontal - return (dX < 0) ? LEFT : RIGHT; - } else { - // Vertical - return (dY < 0) ? UP : DOWN; - } - } - - private MotionEvent mapSecondPointerToFirstPointer(MotionEvent event) { - // Only map basic events when two fingers are down. - if (event.getPointerCount() != 2 || - (event.getActionMasked() != MotionEvent.ACTION_POINTER_DOWN && - event.getActionMasked() != MotionEvent.ACTION_POINTER_UP && - event.getActionMasked() != MotionEvent.ACTION_MOVE)) { - return null; - } - - int action = event.getActionMasked(); - - if (action == MotionEvent.ACTION_POINTER_DOWN) { - action = MotionEvent.ACTION_DOWN; - } else if (action == MotionEvent.ACTION_POINTER_UP) { - action = MotionEvent.ACTION_UP; - } - - // Map the information from the second pointer to the first. - return MotionEvent.obtain(mSecondPointerDownTime, event.getEventTime(), action, - event.getX(1), event.getY(1), event.getPressure(1), event.getSize(1), - event.getMetaState(), event.getXPrecision(), event.getYPrecision(), - event.getDeviceId(), event.getEdgeFlags()); - } -} diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java index 303230b00c6f..020f2253743d 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java @@ -17,6 +17,7 @@ package com.android.server.accessibility; import android.content.Context; +import android.graphics.Region; import android.os.PowerManager; import android.util.Slog; import android.util.SparseArray; @@ -30,6 +31,8 @@ import android.view.MotionEvent; import android.view.accessibility.AccessibilityEvent; import com.android.server.LocalServices; +import com.android.server.accessibility.gestures.TouchExplorer; +import com.android.server.accessibility.magnification.MagnificationGestureHandler; import com.android.server.policy.WindowManagerPolicy; import java.util.ArrayList; @@ -97,9 +100,28 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo */ static final int FLAG_FEATURE_TRIGGERED_SCREEN_MAGNIFIER = 0x00000040; - static final int FEATURES_AFFECTING_MOTION_EVENTS = FLAG_FEATURE_INJECT_MOTION_EVENTS - | FLAG_FEATURE_AUTOCLICK | FLAG_FEATURE_TOUCH_EXPLORATION - | FLAG_FEATURE_SCREEN_MAGNIFIER | FLAG_FEATURE_TRIGGERED_SCREEN_MAGNIFIER; + /** + * Flag for dispatching double tap and double tap and hold to the service. + * + * @see #setUserAndEnabledFeatures(int, int) + */ + static final int FLAG_SERVICE_HANDLES_DOUBLE_TAP = 0x00000080; + +/** + * Flag for enabling multi-finger gestures. + * + * @see #setUserAndEnabledFeatures(int, int) + */ + static final int FLAG_REQUEST_MULTI_FINGER_GESTURES = 0x00000100; + + static final int FEATURES_AFFECTING_MOTION_EVENTS = + FLAG_FEATURE_INJECT_MOTION_EVENTS + | FLAG_FEATURE_AUTOCLICK + | FLAG_FEATURE_TOUCH_EXPLORATION + | FLAG_FEATURE_SCREEN_MAGNIFIER + | FLAG_FEATURE_TRIGGERED_SCREEN_MAGNIFIER + | FLAG_SERVICE_HANDLES_DOUBLE_TAP + | FLAG_REQUEST_MULTI_FINGER_GESTURES; private final Context mContext; @@ -114,7 +136,7 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo private final SparseArray<MagnificationGestureHandler> mMagnificationGestureHandler = new SparseArray<>(0); - private final SparseArray<MotionEventInjector> mMotionEventInjector = new SparseArray<>(0); + private final SparseArray<MotionEventInjector> mMotionEventInjectors = new SparseArray<>(0); private AutoclickController mAutoclickController; @@ -385,9 +407,16 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo for (int i = displaysList.size() - 1; i >= 0; i--) { final int displayId = displaysList.get(i).getDisplayId(); + final Context displayContext = mContext.createDisplayContext(displaysList.get(i)); if ((mEnabledFeatures & FLAG_FEATURE_TOUCH_EXPLORATION) != 0) { - TouchExplorer explorer = new TouchExplorer(mContext, mAms); + TouchExplorer explorer = new TouchExplorer(displayContext, mAms); + if ((mEnabledFeatures & FLAG_SERVICE_HANDLES_DOUBLE_TAP) != 0) { + explorer.setServiceHandlesDoubleTap(true); + } + if ((mEnabledFeatures & FLAG_REQUEST_MULTI_FINGER_GESTURES) != 0) { + explorer.setMultiFingerGesturesEnabled(true); + } addFirstEventHandler(displayId, explorer); mTouchExplorer.put(displayId, explorer); } @@ -400,7 +429,7 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo final boolean triggerable = (mEnabledFeatures & FLAG_FEATURE_TRIGGERED_SCREEN_MAGNIFIER) != 0; MagnificationGestureHandler magnificationGestureHandler = - new MagnificationGestureHandler(mContext, + new FullScreenMagnificationGestureHandler(displayContext, mAms.getMagnificationController(), detectControlGestures, triggerable, displayId); addFirstEventHandler(displayId, magnificationGestureHandler); @@ -411,12 +440,14 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo MotionEventInjector injector = new MotionEventInjector( mContext.getMainLooper()); addFirstEventHandler(displayId, injector); - // TODO: Need to set MotionEventInjector per display. - mAms.setMotionEventInjector(injector); - mMotionEventInjector.put(displayId, injector); + mMotionEventInjectors.put(displayId, injector); } } + if ((mEnabledFeatures & FLAG_FEATURE_INJECT_MOTION_EVENTS) != 0) { + mAms.setMotionEventInjectors(mMotionEventInjectors); + } + if ((mEnabledFeatures & FLAG_FEATURE_FILTER_KEY_EVENTS) != 0) { mKeyboardInterceptor = new KeyboardInterceptor(mAms, LocalServices.getService(WindowManagerPolicy.class)); @@ -461,15 +492,14 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo } private void disableFeatures() { - for (int i = mMotionEventInjector.size() - 1; i >= 0; i--) { - final MotionEventInjector injector = mMotionEventInjector.valueAt(i); - // TODO: Need to set MotionEventInjector per display. - mAms.setMotionEventInjector(null); + for (int i = mMotionEventInjectors.size() - 1; i >= 0; i--) { + final MotionEventInjector injector = mMotionEventInjectors.valueAt(i); if (injector != null) { injector.onDestroy(); } } - mMotionEventInjector.clear(); + mAms.setMotionEventInjectors(null); + mMotionEventInjectors.clear(); if (mAutoclickController != null) { mAutoclickController.onDestroy(); mAutoclickController = null; @@ -697,4 +727,16 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo return shouldProcess; } } + + public void setGestureDetectionPassthroughRegion(int displayId, Region region) { + if (region != null && mTouchExplorer.contains(displayId)) { + mTouchExplorer.get(displayId).setGestureDetectionPassthroughRegion(region); + } + } + + public void setTouchExplorationPassthroughRegion(int displayId, Region region) { + if (region != null && mTouchExplorer.contains(displayId)) { + mTouchExplorer.get(displayId).setTouchExplorationPassthroughRegion(region); + } + } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 3c0621128322..fcf270b4ef35 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -16,33 +16,33 @@ package com.android.server.accessibility; -import static android.accessibilityservice.AccessibilityService.SHOW_MODE_AUTO; -import static android.accessibilityservice.AccessibilityService.SHOW_MODE_HARD_KEYBOARD_ORIGINAL_VALUE; -import static android.accessibilityservice.AccessibilityService.SHOW_MODE_HARD_KEYBOARD_OVERRIDDEN; -import static android.accessibilityservice.AccessibilityService.SHOW_MODE_HIDDEN; -import static android.accessibilityservice.AccessibilityService.SHOW_MODE_IGNORE_HARD_KEYBOARD; -import static android.accessibilityservice.AccessibilityService.SHOW_MODE_MASK; -import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY; -import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED; -import static android.view.accessibility.AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS; -import static android.view.accessibility.AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS; -import static android.view.accessibility.AccessibilityNodeInfo.ACTION_CLICK; -import static android.view.accessibility.AccessibilityNodeInfo.ACTION_LONG_CLICK; - +import static android.provider.Settings.Secure.ACCESSIBILITY_DISPLAY_MAGNIFICATION_NAVBAR_ENABLED; +import static android.view.accessibility.AccessibilityManager.ACCESSIBILITY_BUTTON; +import static android.view.accessibility.AccessibilityManager.ACCESSIBILITY_SHORTCUT_KEY; +import static android.view.accessibility.AccessibilityManager.ShortcutType; + +import static com.android.internal.accessibility.AccessibilityShortcutController.MAGNIFICATION_COMPONENT_NAME; +import static com.android.internal.accessibility.AccessibilityShortcutController.MAGNIFICATION_CONTROLLER_NAME; +import static com.android.internal.accessibility.common.ShortcutConstants.CHOOSER_PACKAGE_NAME; +import static com.android.internal.accessibility.util.AccessibilityStatsLogUtils.logAccessibilityShortcutActivated; import static com.android.internal.util.FunctionalUtils.ignoreRemoteException; import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; +import static com.android.server.accessibility.AccessibilityUserState.doesShortcutTargetsStringContain; import android.Manifest; +import android.accessibilityservice.AccessibilityGestureEvent; import android.accessibilityservice.AccessibilityService; import android.accessibilityservice.AccessibilityServiceInfo; +import android.accessibilityservice.AccessibilityShortcutInfo; import android.accessibilityservice.IAccessibilityServiceClient; import android.annotation.NonNull; import android.annotation.Nullable; import android.app.ActivityOptions; import android.app.AlertDialog; -import android.app.AppOpsManager; import android.app.PendingIntent; +import android.app.RemoteAction; import android.appwidget.AppWidgetManagerInternal; +import android.content.ActivityNotFoundException; import android.content.BroadcastReceiver; import android.content.ComponentName; import android.content.ContentResolver; @@ -52,9 +52,9 @@ import android.content.DialogInterface.OnClickListener; import android.content.Intent; import android.content.IntentFilter; import android.content.pm.PackageManager; +import android.content.pm.PackageManagerInternal; import android.content.pm.ResolveInfo; import android.content.pm.ServiceInfo; -import android.content.pm.UserInfo; import android.database.ContentObserver; import android.graphics.Point; import android.graphics.Rect; @@ -93,8 +93,6 @@ import android.view.Display; import android.view.IWindow; import android.view.KeyEvent; import android.view.MagnificationSpec; -import android.view.View; -import android.view.WindowInfo; import android.view.WindowManager; import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityInteractionClient; @@ -102,27 +100,27 @@ import android.view.accessibility.AccessibilityManager; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; import android.view.accessibility.IAccessibilityInteractionConnection; -import android.view.accessibility.IAccessibilityInteractionConnectionCallback; import android.view.accessibility.IAccessibilityManager; import android.view.accessibility.IAccessibilityManagerClient; +import android.view.accessibility.IWindowMagnificationConnection; import com.android.internal.R; import com.android.internal.accessibility.AccessibilityShortcutController; import com.android.internal.accessibility.AccessibilityShortcutController.ToggleableFrameworkFeatureInfo; +import com.android.internal.accessibility.dialog.AccessibilityButtonChooserActivity; +import com.android.internal.accessibility.dialog.AccessibilityShortcutChooserActivity; import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; import com.android.internal.content.PackageMonitor; import com.android.internal.util.ArrayUtils; import com.android.internal.util.DumpUtils; import com.android.internal.util.IntPair; -import com.android.internal.util.function.pooled.PooledLambda; import com.android.server.LocalServices; import com.android.server.SystemService; +import com.android.server.accessibility.magnification.WindowMagnificationManager; import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.WindowManagerInternal; -import libcore.util.EmptyArray; - import org.xmlpull.v1.XmlPullParserException; import java.io.FileDescriptor; @@ -131,15 +129,14 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.function.Consumer; -import java.util.function.IntSupplier; +import java.util.function.Function; +import java.util.function.Predicate; /** * This class is instantiated by the system as a system level service and can be @@ -148,7 +145,11 @@ import java.util.function.IntSupplier; * on the device. Events are dispatched to {@link AccessibilityService}s. */ public class AccessibilityManagerService extends IAccessibilityManager.Stub - implements AbstractAccessibilityServiceConnection.SystemSupport { + implements AbstractAccessibilityServiceConnection.SystemSupport, + AccessibilityUserState.ServiceInfoChangeListener, + AccessibilityWindowManager.AccessibilityEventSender, + AccessibilitySecurityPolicy.AccessibilityUserManager, + SystemActionPerformer.SystemActionsChangedListener { private static final boolean DEBUG = false; @@ -158,12 +159,12 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub // when that accessibility services are bound. private static final int WAIT_FOR_USER_STATE_FULLY_INITIALIZED_MILLIS = 3000; - private static final int WAIT_WINDOWS_TIMEOUT_MILLIS = 5000; - // TODO: Restructure service initialization so services aren't connected before all of // their capabilities are ready. private static final int WAIT_MOTION_INJECTOR_TIMEOUT_MILLIS = 1000; + static final String FUNCTION_REGISTER_SYSTEM_ACTION = "registerSystemAction"; + static final String FUNCTION_UNREGISTER_SYSTEM_ACTION = "unregisterSystemAction"; private static final String FUNCTION_REGISTER_UI_TEST_AUTOMATION_SERVICE = "registerUiTestAutomationService"; @@ -175,8 +176,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private static final String SET_PIP_ACTION_REPLACEMENT = "setPictureInPictureActionReplacingConnection"; - private static final String FUNCTION_DUMP = "dump"; - private static final char COMPONENT_NAME_SEPARATOR = ':'; private static final int OWN_PROCESS_ID = android.os.Process.myPid(); @@ -186,8 +185,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private static int sIdCounter = MAGNIFICATION_GESTURE_HANDLER_ID + 1; - private static int sNextWindowId; - private final Context mContext; private final Object mLock = new Object(); @@ -196,30 +193,26 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub new SimpleStringSplitter(COMPONENT_NAME_SEPARATOR); private final Rect mTempRect = new Rect(); - private final Rect mTempRect1 = new Rect(); - private final Point mTempPoint = new Point(); - private final PackageManager mPackageManager; private final PowerManager mPowerManager; private final WindowManagerInternal mWindowManagerService; - private AppWidgetManagerInternal mAppWidgetService; + private final AccessibilitySecurityPolicy mSecurityPolicy; - private final SecurityPolicy mSecurityPolicy; + private final AccessibilityWindowManager mA11yWindowManager; private final AccessibilityDisplayListener mA11yDisplayListener; - private final AppOpsManager mAppOpsManager; - private final ActivityTaskManagerInternal mActivityTaskManagerService; private final MainHandler mMainHandler; - private final GlobalActionPerformer mGlobalActionPerformer; + // Lazily initialized - access through getSystemActionPerfomer() + private SystemActionPerformer mSystemActionPerformer; private MagnificationController mMagnificationController; @@ -229,11 +222,13 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private AccessibilityInputFilter mInputFilter; + private WindowMagnificationManager mWindowMagnificationMgr; + private boolean mHasInputFilter; private KeyEventDispatcher mKeyEventDispatcher; - private MotionEventInjector mMotionEventInjector; + private SparseArray<MotionEventInjector> mMotionEventInjectors; private FingerprintGestureDispatcher mFingerprintGestureDispatcher; @@ -247,16 +242,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private final RemoteCallbackList<IAccessibilityManagerClient> mGlobalClients = new RemoteCallbackList<>(); - private final SparseArray<RemoteAccessibilityConnection> mGlobalInteractionConnections = - new SparseArray<>(); - - private RemoteAccessibilityConnection mPictureInPictureActionReplacingConnection; - - private final SparseArray<IBinder> mGlobalWindowTokens = new SparseArray<>(); - - private final SparseArray<UserState> mUserStates = new SparseArray<>(); - - private final UserManager mUserManager; + private final SparseArray<AccessibilityUserState> mUserStates = new SparseArray<>(); private final UiAutomationManager mUiAutomationManager = new UiAutomationManager(mLock); @@ -265,11 +251,10 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub //TODO: Remove this hack private boolean mInitialized; - private WindowsForAccessibilityCallback mWindowsForAccessibilityCallback; - + private Point mTempPoint = new Point(); private boolean mIsAccessibilityButtonShown; - private UserState getCurrentUserStateLocked() { + private AccessibilityUserState getCurrentUserStateLocked() { return getUserStateLocked(mCurrentUserId); } @@ -292,6 +277,27 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } + @VisibleForTesting + AccessibilityManagerService( + Context context, + PackageManager packageManager, + AccessibilitySecurityPolicy securityPolicy, + SystemActionPerformer systemActionPerformer, + AccessibilityWindowManager a11yWindowManager, + AccessibilityDisplayListener a11yDisplayListener) { + mContext = context; + mPowerManager = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE); + mWindowManagerService = LocalServices.getService(WindowManagerInternal.class); + mMainHandler = new MainHandler(mContext.getMainLooper()); + mActivityTaskManagerService = LocalServices.getService(ActivityTaskManagerInternal.class); + mPackageManager = packageManager; + mSecurityPolicy = securityPolicy; + mSystemActionPerformer = systemActionPerformer; + mA11yWindowManager = a11yWindowManager; + mA11yDisplayListener = a11yDisplayListener; + init(); + } + /** * Creates a new instance. * @@ -299,20 +305,23 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub */ public AccessibilityManagerService(Context context) { mContext = context; - mPackageManager = mContext.getPackageManager(); - mPowerManager = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE); + mPowerManager = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE); mWindowManagerService = LocalServices.getService(WindowManagerInternal.class); - mUserManager = (UserManager) context.getSystemService(Context.USER_SERVICE); - mSecurityPolicy = new SecurityPolicy(); - mAppOpsManager = (AppOpsManager) context.getSystemService(Context.APP_OPS_SERVICE); mMainHandler = new MainHandler(mContext.getMainLooper()); - mGlobalActionPerformer = new GlobalActionPerformer(mContext, mWindowManagerService); - mA11yDisplayListener = new AccessibilityDisplayListener(mContext, mMainHandler); mActivityTaskManagerService = LocalServices.getService(ActivityTaskManagerInternal.class); + mPackageManager = mContext.getPackageManager(); + mSecurityPolicy = new AccessibilitySecurityPolicy(mContext, this); + mA11yWindowManager = new AccessibilityWindowManager(mLock, mMainHandler, + mWindowManagerService, this, mSecurityPolicy, this); + mA11yDisplayListener = new AccessibilityDisplayListener(mContext, mMainHandler); + init(); + } + private void init() { + mSecurityPolicy.setAccessibilityWindowManager(mA11yWindowManager); registerBroadcastReceivers(); new AccessibilityContentObserver(mMainHandler).register( - context.getContentResolver()); + mContext.getContentResolver()); } @Override @@ -325,6 +334,11 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return mIsAccessibilityButtonShown; } + @Override + public void onServiceInfoChangedLocked(AccessibilityUserState userState) { + scheduleNotifyClientsOfServicesStateChangeLocked(userState); + } + @Nullable public FingerprintGestureDispatcher getFingerprintGestureDispatcher() { return mFingerprintGestureDispatcher; @@ -333,45 +347,46 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private void onBootPhase(int phase) { if (phase == SystemService.PHASE_SYSTEM_SERVICES_READY) { if (mPackageManager.hasSystemFeature(PackageManager.FEATURE_APP_WIDGETS)) { - mAppWidgetService = LocalServices.getService(AppWidgetManagerInternal.class); + mSecurityPolicy.setAppWidgetManager( + LocalServices.getService(AppWidgetManagerInternal.class)); } } } - private UserState getUserState(int userId) { + private AccessibilityUserState getUserState(int userId) { synchronized (mLock) { return getUserStateLocked(userId); } } - private UserState getUserStateLocked(int userId) { - UserState state = mUserStates.get(userId); + @NonNull + private AccessibilityUserState getUserStateLocked(int userId) { + AccessibilityUserState state = mUserStates.get(userId); if (state == null) { - state = new UserState(userId); + state = new AccessibilityUserState(userId, mContext, this); mUserStates.put(userId, state); } return state; } boolean getBindInstantServiceAllowed(int userId) { - final UserState userState = getUserState(userId); - if (userState == null) return false; - return userState.getBindInstantServiceAllowed(); + synchronized (mLock) { + final AccessibilityUserState userState = getUserStateLocked(userId); + return userState.getBindInstantServiceAllowedLocked(); + } } void setBindInstantServiceAllowed(int userId, boolean allowed) { - UserState userState; + mContext.enforceCallingOrSelfPermission( + Manifest.permission.MANAGE_BIND_INSTANT_SERVICE, + "setBindInstantServiceAllowed"); synchronized (mLock) { - userState = getUserState(userId); - if (userState == null) { - if (!allowed) { - return; - } - userState = new UserState(userId); - mUserStates.put(userId, userState); + final AccessibilityUserState userState = getUserStateLocked(userId); + if (allowed != userState.getBindInstantServiceAllowedLocked()) { + userState.setBindInstantServiceAllowedLocked(allowed); + onUserStateChangedLocked(userState); } } - userState.setBindInstantServiceAllowed(allowed); } private void registerBroadcastReceivers() { @@ -385,7 +400,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return; } // We will update when the automation service dies. - UserState userState = getCurrentUserStateLocked(); + final AccessibilityUserState userState = getCurrentUserStateLocked(); // We have to reload the installed services since some services may // have different attributes, resolve info (does not support equals), // etc. Remove them then to force reload. @@ -407,13 +422,21 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (userId != mCurrentUserId) { return; } - UserState userState = getUserStateLocked(userId); - boolean reboundAService = userState.mBindingServices.removeIf( + final AccessibilityUserState userState = getUserStateLocked(userId); + final boolean reboundAService = userState.getBindingServicesLocked().removeIf( component -> component != null + && component.getPackageName().equals(packageName)) + || userState.mCrashedServices.removeIf(component -> component != null && component.getPackageName().equals(packageName)); - if (reboundAService) { + // Reloads the installed services info to make sure the rebound service could + // get a new one. + userState.mInstalledServices.clear(); + final boolean configurationChanged = + readConfigurationForUserStateLocked(userState); + if (reboundAService || configurationChanged) { onUserStateChangedLocked(userState); } + migrateAccessibilityButtonSettingsIfNecessaryLocked(userState, packageName); } } @@ -426,14 +449,18 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (userId != mCurrentUserId) { return; } - UserState userState = getUserStateLocked(userId); - Iterator<ComponentName> it = userState.mEnabledServices.iterator(); + final AccessibilityUserState userState = getUserStateLocked(userId); + final Predicate<ComponentName> filter = + component -> component != null && component.getPackageName().equals( + packageName); + userState.mBindingServices.removeIf(filter); + userState.mCrashedServices.removeIf(filter); + final Iterator<ComponentName> it = userState.mEnabledServices.iterator(); while (it.hasNext()) { - ComponentName comp = it.next(); - String compPkg = comp.getPackageName(); + final ComponentName comp = it.next(); + final String compPkg = comp.getPackageName(); if (compPkg.equals(packageName)) { it.remove(); - userState.mBindingServices.remove(comp); // Update the enabled services setting. persistComponentNamesToSettingLocked( Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES, @@ -461,18 +488,18 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (userId != mCurrentUserId) { return false; } - UserState userState = getUserStateLocked(userId); - Iterator<ComponentName> it = userState.mEnabledServices.iterator(); + final AccessibilityUserState userState = getUserStateLocked(userId); + final Iterator<ComponentName> it = userState.mEnabledServices.iterator(); while (it.hasNext()) { - ComponentName comp = it.next(); - String compPkg = comp.getPackageName(); + final ComponentName comp = it.next(); + final String compPkg = comp.getPackageName(); for (String pkg : packages) { if (compPkg.equals(pkg)) { if (!doit) { return true; } it.remove(); - userState.mBindingServices.remove(comp); + userState.getBindingServicesLocked().remove(comp); persistComponentNamesToSettingLocked( Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES, userState.mEnabledServices, userId); @@ -509,7 +536,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } else if (Intent.ACTION_USER_PRESENT.equals(action)) { // We will update when the automation service dies. synchronized (mLock) { - UserState userState = getCurrentUserStateLocked(); + AccessibilityUserState userState = getCurrentUserStateLocked(); if (readConfigurationForUserStateLocked(userState)) { onUserStateChangedLocked(userState); } @@ -522,12 +549,56 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub intent.getStringExtra(Intent.EXTRA_SETTING_PREVIOUS_VALUE), intent.getStringExtra(Intent.EXTRA_SETTING_NEW_VALUE)); } + } else if (ACCESSIBILITY_DISPLAY_MAGNIFICATION_NAVBAR_ENABLED.equals(which)) { + synchronized (mLock) { + restoreLegacyDisplayMagnificationNavBarIfNeededLocked( + intent.getStringExtra(Intent.EXTRA_SETTING_NEW_VALUE), + intent.getIntExtra(Intent.EXTRA_SETTING_RESTORED_FROM_SDK_INT, + 0)); + } } } } }, UserHandle.ALL, intentFilter, null, null); } + // Called only during settings restore; currently supports only the owner user + // TODO: b/22388012 + private void restoreLegacyDisplayMagnificationNavBarIfNeededLocked(String newSetting, + int restoreFromSdkInt) { + if (restoreFromSdkInt >= Build.VERSION_CODES.R) { + return; + } + + boolean displayMagnificationNavBarEnabled; + try { + displayMagnificationNavBarEnabled = Integer.parseInt(newSetting) == 1; + } catch (NumberFormatException e) { + Slog.w(LOG_TAG, "number format is incorrect" + e); + return; + } + + final AccessibilityUserState userState = getUserStateLocked(UserHandle.USER_SYSTEM); + final Set<String> targetsFromSetting = new ArraySet<>(); + readColonDelimitedSettingToSet(Settings.Secure.ACCESSIBILITY_BUTTON_TARGETS, + userState.mUserId, targetsFromSetting, str -> str); + final boolean targetsContainMagnification = targetsFromSetting.contains( + MAGNIFICATION_CONTROLLER_NAME); + if (targetsContainMagnification == displayMagnificationNavBarEnabled) { + return; + } + + if (displayMagnificationNavBarEnabled) { + targetsFromSetting.add(MAGNIFICATION_CONTROLLER_NAME); + } else { + targetsFromSetting.remove(MAGNIFICATION_CONTROLLER_NAME); + } + persistColonDelimitedSetToSettingLocked(Settings.Secure.ACCESSIBILITY_BUTTON_TARGETS, + userState.mUserId, targetsFromSetting, str -> str); + readAccessibilityButtonTargetsLocked(userState); + onUserStateChangedLocked(userState); + } + @Override public long addClient(IAccessibilityManagerClient callback, int userId) { synchronized (mLock) { @@ -540,7 +611,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub // If the client is from a process that runs across users such as // the system UI or the system we add it to the global state that // is shared across users. - UserState userState = getUserStateLocked(resolvedUserId); + AccessibilityUserState userState = getUserStateLocked(resolvedUserId); Client client = new Client(callback, Binder.getCallingUid(), userState); if (mSecurityPolicy.isCallerInteractingAcrossUsers(userId)) { mGlobalClients.register(callback, client); @@ -548,7 +619,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub Slog.i(LOG_TAG, "Added global client for pid:" + Binder.getCallingPid()); } return IntPair.of( - userState.getClientState(), + getClientStateLocked(userState), client.mLastSentRelevantEventTypes); } else { userState.mUserClients.register(callback, client); @@ -560,7 +631,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub + " and userId:" + mCurrentUserId); } return IntPair.of( - (resolvedUserId == mCurrentUserId) ? userState.getClientState() : 0, + (resolvedUserId == mCurrentUserId) ? getClientStateLocked(userState) : 0, client.mLastSentRelevantEventTypes); } } @@ -574,7 +645,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (event.getWindowId() == AccessibilityWindowInfo.PICTURE_IN_PICTURE_ACTION_REPLACER_WINDOW_ID) { // The replacer window isn't shown to services. Move its events into the pip. - AccessibilityWindowInfo pip = mSecurityPolicy.getPictureInPictureWindow(); + AccessibilityWindowInfo pip = mA11yWindowManager.getPictureInPictureWindowLocked(); if (pip != null) { int pipId = pip.getId(); event.setWindowId(pipId); @@ -589,13 +660,14 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub // Make sure the reported package is one the caller has access to. event.setPackageName(mSecurityPolicy.resolveValidReportedPackageLocked( - event.getPackageName(), UserHandle.getCallingAppId(), resolvedUserId)); + event.getPackageName(), UserHandle.getCallingAppId(), resolvedUserId, + getCallingPid())); // This method does nothing for a background user. if (resolvedUserId == mCurrentUserId) { - if (mSecurityPolicy.canDispatchAccessibilityEventLocked(event)) { - mSecurityPolicy.updateActiveAndAccessibilityFocusedWindowLocked( - event.getWindowId(), event.getSourceNodeId(), + if (mSecurityPolicy.canDispatchAccessibilityEventLocked(mCurrentUserId, event)) { + mA11yWindowManager.updateActiveAndAccessibilityFocusedWindowLocked( + mCurrentUserId, event.getWindowId(), event.getSourceNodeId(), event.getEventType(), event.getAction()); mSecurityPolicy.updateEventSourceLocked(event); dispatchEvent = true; @@ -612,10 +684,24 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub // Make sure clients receiving this event will be able to get the // current state of the windows as the window manager may be delaying // the computation for performance reasons. - if (event.getEventType() == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED - && mWindowsForAccessibilityCallback != null) { - WindowManagerInternal wm = LocalServices.getService(WindowManagerInternal.class); - wm.computeWindowsForAccessibility(); + boolean shouldComputeWindows = false; + int displayId = Display.INVALID_DISPLAY; + synchronized (mLock) { + final int windowId = event.getWindowId(); + if (event.getEventType() == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED + && windowId != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID) { + displayId = mA11yWindowManager.getDisplayIdByUserIdAndWindowIdLocked( + mCurrentUserId, windowId); + } + if (displayId != Display.INVALID_DISPLAY + && mA11yWindowManager.isTrackingWindowsLocked(displayId)) { + shouldComputeWindows = true; + } + } + if (shouldComputeWindows) { + final WindowManagerInternal wm = LocalServices.getService( + WindowManagerInternal.class); + wm.computeWindowsForAccessibility(displayId); } synchronized (mLock) { notifyAccessibilityServicesDelayedLocked(event, false); @@ -638,6 +724,40 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub event.recycle(); } + /** + * This is the implementation of AccessibilityManager system API. + * System UI calls into this method through AccessibilityManager system API to register a + * system action. + */ + @Override + public void registerSystemAction(RemoteAction action, int actionId) { + mSecurityPolicy.enforceCallerIsRecentsOrHasPermission( + Manifest.permission.MANAGE_ACCESSIBILITY, + FUNCTION_REGISTER_SYSTEM_ACTION); + getSystemActionPerformer().registerSystemAction(actionId, action); + } + + /** + * This is the implementation of AccessibilityManager system API. + * System UI calls into this method through AccessibilityManager system API to unregister a + * system action. + */ + @Override + public void unregisterSystemAction(int actionId) { + mSecurityPolicy.enforceCallerIsRecentsOrHasPermission( + Manifest.permission.MANAGE_ACCESSIBILITY, + FUNCTION_UNREGISTER_SYSTEM_ACTION); + getSystemActionPerformer().unregisterSystemAction(actionId); + } + + private SystemActionPerformer getSystemActionPerformer() { + if (mSystemActionPerformer == null) { + mSystemActionPerformer = + new SystemActionPerformer(mContext, mWindowManagerService, null, this); + } + return mSystemActionPerformer; + } + @Override public List<AccessibilityServiceInfo> getInstalledAccessibilityServiceList(int userId) { synchronized (mLock) { @@ -661,7 +781,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub .resolveCallingUserIdEnforcingPermissionsLocked(userId); // The automation service can suppress other services. - final UserState userState = getUserStateLocked(resolvedUserId); + final AccessibilityUserState userState = getUserStateLocked(resolvedUserId); if (mUiAutomationManager.suppressingAccessibilityServicesLocked()) { return Collections.emptyList(); } @@ -716,109 +836,16 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } @Override - public int addAccessibilityInteractionConnection(IWindow windowToken, + public int addAccessibilityInteractionConnection(IWindow windowToken, IBinder leashToken, IAccessibilityInteractionConnection connection, String packageName, int userId) throws RemoteException { - final int windowId; - synchronized (mLock) { - // We treat calls from a profile as if made by its parent as profiles - // share the accessibility state of the parent. The call below - // performs the current profile parent resolution. - final int resolvedUserId = mSecurityPolicy - .resolveCallingUserIdEnforcingPermissionsLocked(userId); - final int resolvedUid = UserHandle.getUid(resolvedUserId, UserHandle.getCallingAppId()); - - // Make sure the reported package is one the caller has access to. - packageName = mSecurityPolicy.resolveValidReportedPackageLocked( - packageName, UserHandle.getCallingAppId(), resolvedUserId); - - windowId = sNextWindowId++; - // If the window is from a process that runs across users such as - // the system UI or the system we add it to the global state that - // is shared across users. - if (mSecurityPolicy.isCallerInteractingAcrossUsers(userId)) { - RemoteAccessibilityConnection wrapper = new RemoteAccessibilityConnection( - windowId, connection, packageName, resolvedUid, UserHandle.USER_ALL); - wrapper.linkToDeath(); - mGlobalInteractionConnections.put(windowId, wrapper); - mGlobalWindowTokens.put(windowId, windowToken.asBinder()); - if (DEBUG) { - Slog.i(LOG_TAG, "Added global connection for pid:" + Binder.getCallingPid() - + " with windowId: " + windowId + " and token: " - + windowToken.asBinder()); - } - } else { - RemoteAccessibilityConnection wrapper = new RemoteAccessibilityConnection( - windowId, connection, packageName, resolvedUid, resolvedUserId); - wrapper.linkToDeath(); - UserState userState = getUserStateLocked(resolvedUserId); - userState.mInteractionConnections.put(windowId, wrapper); - userState.mWindowTokens.put(windowId, windowToken.asBinder()); - if (DEBUG) { - Slog.i(LOG_TAG, "Added user connection for pid:" + Binder.getCallingPid() - + " with windowId: " + windowId + " and userId:" + mCurrentUserId - + " and token: " + windowToken.asBinder()); - } - } - } - WindowManagerInternal wm = LocalServices.getService(WindowManagerInternal.class); - wm.computeWindowsForAccessibility(); - return windowId; + return mA11yWindowManager.addAccessibilityInteractionConnection( + windowToken, leashToken, connection, packageName, userId); } @Override public void removeAccessibilityInteractionConnection(IWindow window) { - synchronized (mLock) { - // We treat calls from a profile as if made by its parent as profiles - // share the accessibility state of the parent. The call below - // performs the current profile parent resolution. - mSecurityPolicy.resolveCallingUserIdEnforcingPermissionsLocked( - UserHandle.getCallingUserId()); - IBinder token = window.asBinder(); - final int removedWindowId = removeAccessibilityInteractionConnectionInternalLocked( - token, mGlobalWindowTokens, mGlobalInteractionConnections); - if (removedWindowId >= 0) { - mSecurityPolicy.onAccessibilityClientRemovedLocked(removedWindowId); - if (DEBUG) { - Slog.i(LOG_TAG, "Removed global connection for pid:" + Binder.getCallingPid() - + " with windowId: " + removedWindowId + " and token: " + window.asBinder()); - } - return; - } - final int userCount = mUserStates.size(); - for (int i = 0; i < userCount; i++) { - UserState userState = mUserStates.valueAt(i); - final int removedWindowIdForUser = - removeAccessibilityInteractionConnectionInternalLocked( - token, userState.mWindowTokens, userState.mInteractionConnections); - if (removedWindowIdForUser >= 0) { - mSecurityPolicy.onAccessibilityClientRemovedLocked(removedWindowIdForUser); - if (DEBUG) { - Slog.i(LOG_TAG, "Removed user connection for pid:" + Binder.getCallingPid() - + " with windowId: " + removedWindowIdForUser + " and userId:" - + mUserStates.keyAt(i) + " and token: " + window.asBinder()); - } - return; - } - } - } - } - - private int removeAccessibilityInteractionConnectionInternalLocked(IBinder windowToken, - SparseArray<IBinder> windowTokens, - SparseArray<RemoteAccessibilityConnection> interactionConnections) { - final int count = windowTokens.size(); - for (int i = 0; i < count; i++) { - if (windowTokens.valueAt(i) == windowToken) { - final int windowId = windowTokens.keyAt(i); - windowTokens.removeAt(i); - RemoteAccessibilityConnection wrapper = interactionConnections.get(windowId); - wrapper.unlinkToDeath(); - interactionConnections.remove(windowId); - return windowId; - } - } - return -1; + mA11yWindowManager.removeAccessibilityInteractionConnection(window); } @Override @@ -826,19 +853,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub IAccessibilityInteractionConnection connection) throws RemoteException { mSecurityPolicy.enforceCallingPermission(Manifest.permission.MODIFY_ACCESSIBILITY_DATA, SET_PIP_ACTION_REPLACEMENT); - synchronized (mLock) { - if (mPictureInPictureActionReplacingConnection != null) { - mPictureInPictureActionReplacingConnection.unlinkToDeath(); - mPictureInPictureActionReplacingConnection = null; - } - if (connection != null) { - RemoteAccessibilityConnection wrapper = new RemoteAccessibilityConnection( - AccessibilityWindowInfo.PICTURE_IN_PICTURE_ACTION_REPLACER_WINDOW_ID, - connection, "foo.bar.baz", Process.SYSTEM_UID, UserHandle.USER_ALL); - mPictureInPictureActionReplacingConnection = wrapper; - wrapper.linkToDeath(); - } - } + mA11yWindowManager.setPictureInPictureActionReplacingConnection(connection); } @Override @@ -852,7 +867,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub synchronized (mLock) { mUiAutomationManager.registerUiTestAutomationServiceLocked(owner, serviceClient, mContext, accessibilityServiceInfo, sIdCounter++, mMainHandler, - mSecurityPolicy, this, mWindowManagerService, mGlobalActionPerformer, flags); + mSecurityPolicy, this, mWindowManagerService, getSystemActionPerformer(), + mA11yWindowManager, flags); onUserStateChangedLocked(getCurrentUserStateLocked()); } } @@ -875,15 +891,16 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } synchronized (mLock) { // Set the temporary state. - UserState userState = getCurrentUserStateLocked(); + AccessibilityUserState userState = getCurrentUserStateLocked(); - userState.mIsTouchExplorationEnabled = touchExplorationEnabled; - userState.mIsDisplayMagnificationEnabled = false; - userState.mIsNavBarMagnificationEnabled = false; - userState.mIsAutoclickEnabled = false; + userState.setTouchExplorationEnabledLocked(touchExplorationEnabled); + userState.setDisplayMagnificationEnabledLocked(false); + userState.disableShortcutMagnificationLocked(); + userState.setAutoclickEnabledLocked(false); userState.mEnabledServices.clear(); userState.mEnabledServices.add(service); - userState.mBindingServices.clear(); + userState.getBindingServicesLocked().clear(); + userState.getCrashedServicesLocked().clear(); userState.mTouchExplorationGrantedServices.clear(); userState.mTouchExplorationGrantedServices.add(service); @@ -906,10 +923,10 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (resolvedUserId != mCurrentUserId) { return null; } - if (mSecurityPolicy.findA11yWindowInfoById(windowId) == null) { + if (mA11yWindowManager.findA11yWindowInfoByIdLocked(windowId) == null) { return null; } - return findWindowTokenLocked(windowId); + return mA11yWindowManager.getWindowTokenForUserAndWindowIdLocked(userId, windowId); } } @@ -918,17 +935,26 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub * navigation area has been clicked. * * @param displayId The logical display id. + * @param targetName The flattened {@link ComponentName} string or the class name of a system + * class implementing a supported accessibility feature, or {@code null} if there's no + * specified target. */ @Override - public void notifyAccessibilityButtonClicked(int displayId) { + public void notifyAccessibilityButtonClicked(int displayId, String targetName) { if (mContext.checkCallingOrSelfPermission(android.Manifest.permission.STATUS_BAR_SERVICE) != PackageManager.PERMISSION_GRANTED) { throw new SecurityException("Caller does not hold permission " + android.Manifest.permission.STATUS_BAR_SERVICE); } - synchronized (mLock) { - notifyAccessibilityButtonClickedLocked(displayId); + if (targetName == null) { + synchronized (mLock) { + final AccessibilityUserState userState = getCurrentUserStateLocked(); + targetName = userState.getTargetAssignedToAccessibilityButton(); + } } + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::performAccessibilityShortcutInternal, this, + displayId, ACCESSIBILITY_BUTTON, targetName)); } /** @@ -940,27 +966,48 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub */ @Override public void notifyAccessibilityButtonVisibilityChanged(boolean shown) { - if (mContext.checkCallingOrSelfPermission(android.Manifest.permission.STATUS_BAR_SERVICE) - != PackageManager.PERMISSION_GRANTED) { - throw new SecurityException("Caller does not hold permission " - + android.Manifest.permission.STATUS_BAR_SERVICE); - } + mSecurityPolicy.enforceCallingOrSelfPermission( + android.Manifest.permission.STATUS_BAR_SERVICE); synchronized (mLock) { notifyAccessibilityButtonVisibilityChangedLocked(shown); } } - - boolean onGesture(int gestureId) { + /** + * Called when a gesture is detected on a display. + * + * @param gestureEvent the detail of the gesture. + * @return true if the event is handled. + */ + public boolean onGesture(AccessibilityGestureEvent gestureEvent) { synchronized (mLock) { - boolean handled = notifyGestureLocked(gestureId, false); + boolean handled = notifyGestureLocked(gestureEvent, false); if (!handled) { - handled = notifyGestureLocked(gestureId, true); + handled = notifyGestureLocked(gestureEvent, true); } return handled; } } + /** + * Called when the system action list is changed. + */ + @Override + public void onSystemActionsChanged() { + synchronized (mLock) { + AccessibilityUserState state = getCurrentUserStateLocked(); + notifySystemActionsChangedLocked(state); + } + } + + @VisibleForTesting + void notifySystemActionsChangedLocked(AccessibilityUserState userState) { + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + AccessibilityServiceConnection service = userState.mBoundServices.get(i); + service.notifySystemActionsChangedLocked(); + } + } + @VisibleForTesting public boolean notifyKeyEvent(KeyEvent event, int policyFlags) { synchronized (mLock) { @@ -996,30 +1043,34 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub * Called by AccessibilityInputFilter when it creates or destroys the motionEventInjector. * Not using a getter because the AccessibilityInputFilter isn't thread-safe * - * @param motionEventInjector The new value of the motionEventInjector. May be null. + * @param motionEventInjectors The array of motionEventInjectors. May be null. + * */ - void setMotionEventInjector(MotionEventInjector motionEventInjector) { + void setMotionEventInjectors(SparseArray<MotionEventInjector> motionEventInjectors) { synchronized (mLock) { - mMotionEventInjector = motionEventInjector; + mMotionEventInjectors = motionEventInjectors; // We may be waiting on this object being set mLock.notifyAll(); } } @Override - public MotionEventInjector getMotionEventInjectorLocked() { + public @Nullable MotionEventInjector getMotionEventInjectorForDisplayLocked(int displayId) { final long endMillis = SystemClock.uptimeMillis() + WAIT_MOTION_INJECTOR_TIMEOUT_MILLIS; - while ((mMotionEventInjector == null) && (SystemClock.uptimeMillis() < endMillis)) { + MotionEventInjector motionEventInjector = null; + while ((mMotionEventInjectors == null) && (SystemClock.uptimeMillis() < endMillis)) { try { mLock.wait(endMillis - SystemClock.uptimeMillis()); } catch (InterruptedException ie) { /* ignore */ } } - if (mMotionEventInjector == null) { + if (mMotionEventInjectors == null) { Slog.e(LOG_TAG, "MotionEventInjector installation timed out"); + } else { + motionEventInjector = mMotionEventInjectors.get(displayId); } - return mMotionEventInjector; + return motionEventInjector; } /** @@ -1030,7 +1081,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub * @return Whether accessibility a click point was found and set. */ // TODO: (multi-display) Make sure this works for multiple displays. - boolean getAccessibilityFocusClickPointInScreen(Point outPoint) { + public boolean getAccessibilityFocusClickPointInScreen(Point outPoint) { return getInteractionBridge().getAccessibilityFocusClickPointInScreenNotLocked(outPoint); } @@ -1049,6 +1100,15 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } /** + * Returns true if accessibility focus is confined to the active window. + */ + public boolean accessibilityFocusOnlyInActiveWindow() { + synchronized (mLock) { + return mA11yWindowManager.isTrackingWindowsLocked(); + } + } + + /** * Gets the bounds of a window. * * @param outBounds The output to which to write the bounds. @@ -1056,10 +1116,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub boolean getWindowBounds(int windowId, Rect outBounds) { IBinder token; synchronized (mLock) { - token = mGlobalWindowTokens.get(windowId); - if (token == null) { - token = getCurrentUserStateLocked().mWindowTokens.get(windowId); - } + token = getWindowToken(windowId, mCurrentUserId); } mWindowManagerService.getWindowFrame(token, outBounds); if (!outBounds.isEmpty()) { @@ -1068,22 +1125,16 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return false; } - boolean accessibilityFocusOnlyInActiveWindow() { - synchronized (mLock) { - return mWindowsForAccessibilityCallback == null; - } - } - - int getActiveWindowId() { - return mSecurityPolicy.getActiveWindowId(); + public int getActiveWindowId() { + return mA11yWindowManager.getActiveWindowId(mCurrentUserId); } - void onTouchInteractionStart() { - mSecurityPolicy.onTouchInteractionStart(); + public void onTouchInteractionStart() { + mA11yWindowManager.onTouchInteractionStart(); } - void onTouchInteractionEnd() { - mSecurityPolicy.onTouchInteractionEnd(); + public void onTouchInteractionEnd() { + mA11yWindowManager.onTouchInteractionEnd(); } private void switchUser(int userId) { @@ -1093,7 +1144,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } // Disconnect from services for the old user. - UserState oldUserState = getCurrentUserStateLocked(); + AccessibilityUserState oldUserState = getCurrentUserStateLocked(); oldUserState.onSwitchToAnotherUserLocked(); // Disable the local managers for the old user. @@ -1110,13 +1161,14 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub // The user changed. mCurrentUserId = userId; - UserState userState = getCurrentUserStateLocked(); + AccessibilityUserState userState = getCurrentUserStateLocked(); readConfigurationForUserStateLocked(userState); // Even if reading did not yield change, we have to update // the state since the context in which the current user // state was used has changed since it was inactive. onUserStateChangedLocked(userState); + migrateAccessibilityButtonSettingsIfNecessaryLocked(userState, null); if (announceNewUser) { // Schedule announcement of the current user if needed. @@ -1129,8 +1181,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private void announceNewUserIfNeeded() { synchronized (mLock) { - UserState userState = getCurrentUserStateLocked(); - if (userState.isHandlingAccessibilityEvents()) { + AccessibilityUserState userState = getCurrentUserStateLocked(); + if (userState.isHandlingAccessibilityEventsLocked()) { UserManager userManager = (UserManager) mContext.getSystemService( Context.USER_SERVICE); String message = mContext.getString(R.string.user_switched, @@ -1147,7 +1199,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub synchronized (mLock) { int parentUserId = mSecurityPolicy.resolveProfileParentLocked(userId); if (parentUserId == mCurrentUserId) { - UserState userState = getUserStateLocked(mCurrentUserId); + AccessibilityUserState userState = getUserStateLocked(mCurrentUserId); onUserStateChangedLocked(userState); } } @@ -1165,7 +1217,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub readComponentNamesFromStringLocked(oldSetting, mTempComponentNameSet, false); readComponentNamesFromStringLocked(newSetting, mTempComponentNameSet, true); - UserState userState = getUserStateLocked(UserHandle.USER_SYSTEM); + AccessibilityUserState userState = getUserStateLocked(UserHandle.USER_SYSTEM); userState.mEnabledServices.clear(); userState.mEnabledServices.addAll(mTempComponentNameSet); persistComponentNamesToSettingLocked( @@ -1173,6 +1225,11 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub userState.mEnabledServices, UserHandle.USER_SYSTEM); onUserStateChangedLocked(userState); + migrateAccessibilityButtonSettingsIfNecessaryLocked(userState, null); + } + + private int getClientStateLocked(AccessibilityUserState userState) { + return userState.getClientStateLocked(mUiAutomationManager.isUiAutomationRunningLocked()); } private InteractionBridge getInteractionBridge() { @@ -1184,7 +1241,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - private boolean notifyGestureLocked(int gestureId, boolean isDefault) { + private boolean notifyGestureLocked(AccessibilityGestureEvent gestureEvent, boolean isDefault) { // TODO: Now we are giving the gestures to the last enabled // service that can handle them which is the last one // in our list since we write the last enabled as the @@ -1194,11 +1251,11 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub // gestures to avoid user frustration when different // behavior is observed from different combinations of // enabled accessibility services. - UserState state = getCurrentUserStateLocked(); + AccessibilityUserState state = getCurrentUserStateLocked(); for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { AccessibilityServiceConnection service = state.mBoundServices.get(i); if (service.mRequestTouchExplorationMode && service.mIsDefault == isDefault) { - service.notifyGesture(gestureId); + service.notifyGesture(gestureEvent); return true; } } @@ -1206,7 +1263,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } private void notifyClearAccessibilityCacheLocked() { - UserState state = getCurrentUserStateLocked(); + AccessibilityUserState state = getCurrentUserStateLocked(); for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { AccessibilityServiceConnection service = state.mBoundServices.get(i); service.notifyClearAccessibilityNodeInfoCache(); @@ -1215,83 +1272,13 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private void notifyMagnificationChangedLocked(int displayId, @NonNull Region region, float scale, float centerX, float centerY) { - final UserState state = getCurrentUserStateLocked(); + final AccessibilityUserState state = getCurrentUserStateLocked(); for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = state.mBoundServices.get(i); service.notifyMagnificationChangedLocked(displayId, region, scale, centerX, centerY); } } - private void notifySoftKeyboardShowModeChangedLocked(int showMode) { - final UserState state = getCurrentUserStateLocked(); - for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { - final AccessibilityServiceConnection service = state.mBoundServices.get(i); - service.notifySoftKeyboardShowModeChangedLocked(showMode); - } - } - - private void notifyAccessibilityButtonClickedLocked(int displayId) { - final UserState state = getCurrentUserStateLocked(); - - int potentialTargets = state.mIsNavBarMagnificationEnabled ? 1 : 0; - for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { - final AccessibilityServiceConnection service = state.mBoundServices.get(i); - if (service.mRequestAccessibilityButton) { - potentialTargets++; - } - } - - if (potentialTargets == 0) { - return; - } - if (potentialTargets == 1) { - if (state.mIsNavBarMagnificationEnabled) { - mMainHandler.sendMessage(obtainMessage( - AccessibilityManagerService::sendAccessibilityButtonToInputFilter, this, - displayId)); - return; - } else { - for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { - final AccessibilityServiceConnection service = state.mBoundServices.get(i); - if (service.mRequestAccessibilityButton) { - // TODO(b/120762691): Need to notify each accessibility service if - // accessibility button is clicked per display. - service.notifyAccessibilityButtonClickedLocked(); - return; - } - } - } - } else { - if (state.mServiceAssignedToAccessibilityButton == null - && !state.mIsNavBarMagnificationAssignedToAccessibilityButton) { - mMainHandler.sendMessage(obtainMessage( - AccessibilityManagerService::showAccessibilityButtonTargetSelection, this, - displayId)); - } else if (state.mIsNavBarMagnificationEnabled - && state.mIsNavBarMagnificationAssignedToAccessibilityButton) { - mMainHandler.sendMessage(obtainMessage( - AccessibilityManagerService::sendAccessibilityButtonToInputFilter, this, - displayId)); - return; - } else { - for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { - final AccessibilityServiceConnection service = state.mBoundServices.get(i); - if (service.mRequestAccessibilityButton && (service.mComponentName.equals( - state.mServiceAssignedToAccessibilityButton))) { - // TODO(b/120762691): Need to notify each accessibility service if - // accessibility button is clicked per display. - service.notifyAccessibilityButtonClickedLocked(); - return; - } - } - } - // The user may have turned off the assigned service or feature - mMainHandler.sendMessage(obtainMessage( - AccessibilityManagerService::showAccessibilityButtonTargetSelection, this, - displayId)); - } - } - private void sendAccessibilityButtonToInputFilter(int displayId) { synchronized (mLock) { if (mHasInputFilter && mInputFilter != null) { @@ -1300,15 +1287,32 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - private void showAccessibilityButtonTargetSelection(int displayId) { - Intent intent = new Intent(AccessibilityManager.ACTION_CHOOSE_ACCESSIBILITY_BUTTON); + private void showAccessibilityTargetsSelection(int displayId, + @ShortcutType int shortcutType) { + final Intent intent = new Intent(AccessibilityManager.ACTION_CHOOSE_ACCESSIBILITY_BUTTON); + final String chooserClassName = (shortcutType == ACCESSIBILITY_SHORTCUT_KEY) + ? AccessibilityShortcutChooserActivity.class.getName() + : AccessibilityButtonChooserActivity.class.getName(); + intent.setClassName(CHOOSER_PACKAGE_NAME, chooserClassName); intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK | Intent.FLAG_ACTIVITY_CLEAR_TASK); final Bundle bundle = ActivityOptions.makeBasic().setLaunchDisplayId(displayId).toBundle(); mContext.startActivityAsUser(intent, bundle, UserHandle.of(mCurrentUserId)); } + private void launchShortcutTargetActivity(int displayId, ComponentName name) { + final Intent intent = new Intent(); + final Bundle bundle = ActivityOptions.makeBasic().setLaunchDisplayId(displayId).toBundle(); + intent.setComponent(name); + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + try { + mContext.startActivityAsUser(intent, bundle, UserHandle.of(mCurrentUserId)); + } catch (ActivityNotFoundException ignore) { + // ignore the exception + } + } + private void notifyAccessibilityButtonVisibilityChangedLocked(boolean available) { - final UserState state = getCurrentUserStateLocked(); + final AccessibilityUserState state = getCurrentUserStateLocked(); mIsAccessibilityButtonShown = available; for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection clientConnection = state.mBoundServices.get(i); @@ -1319,29 +1323,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - /** - * Removes an AccessibilityInteractionConnection. - * - * @param windowId The id of the window to which the connection is targeted. - * @param userId The id of the user owning the connection. UserHandle.USER_ALL - * if global. - */ - private void removeAccessibilityInteractionConnectionLocked(int windowId, int userId) { - if (userId == UserHandle.USER_ALL) { - mGlobalWindowTokens.remove(windowId); - mGlobalInteractionConnections.remove(windowId); - } else { - UserState userState = getCurrentUserStateLocked(); - userState.mWindowTokens.remove(windowId); - userState.mInteractionConnections.remove(windowId); - } - mSecurityPolicy.onAccessibilityClientRemovedLocked(windowId); - if (DEBUG) { - Slog.i(LOG_TAG, "Removing interaction connection to windowId: " + windowId); - } - } - - private boolean readInstalledAccessibilityServiceLocked(UserState userState) { + private boolean readInstalledAccessibilityServiceLocked(AccessibilityUserState userState) { mTempAccessibilityServiceInfoList.clear(); int flags = PackageManager.GET_SERVICES @@ -1350,7 +1332,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub | PackageManager.MATCH_DIRECT_BOOT_AWARE | PackageManager.MATCH_DIRECT_BOOT_UNAWARE; - if (userState.getBindInstantServiceAllowed()) { + if (userState.getBindInstantServiceAllowedLocked()) { flags |= PackageManager.MATCH_INSTANT; } @@ -1361,13 +1343,17 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub ResolveInfo resolveInfo = installedServices.get(i); ServiceInfo serviceInfo = resolveInfo.serviceInfo; - if (!canRegisterService(serviceInfo)) { + if (!mSecurityPolicy.canRegisterService(serviceInfo)) { continue; } AccessibilityServiceInfo accessibilityServiceInfo; try { accessibilityServiceInfo = new AccessibilityServiceInfo(resolveInfo, mContext); + if (userState.mCrashedServices.contains(serviceInfo.getComponentName())) { + // Restore the crashed attribute. + accessibilityServiceInfo.crashed = true; + } mTempAccessibilityServiceInfoList.add(accessibilityServiceInfo); } catch (XmlPullParserException | IOException xppe) { Slog.e(LOG_TAG, "Error while initializing AccessibilityServiceInfo", xppe); @@ -1385,29 +1371,19 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return false; } - private boolean canRegisterService(ServiceInfo serviceInfo) { - if (!android.Manifest.permission.BIND_ACCESSIBILITY_SERVICE.equals( - serviceInfo.permission)) { - Slog.w(LOG_TAG, "Skipping accessibility service " + new ComponentName( - serviceInfo.packageName, serviceInfo.name).flattenToShortString() - + ": it does not require the permission " - + android.Manifest.permission.BIND_ACCESSIBILITY_SERVICE); - return false; - } - - int servicePackageUid = serviceInfo.applicationInfo.uid; - if (mAppOpsManager.noteOpNoThrow(AppOpsManager.OPSTR_BIND_ACCESSIBILITY_SERVICE, - servicePackageUid, serviceInfo.packageName) != AppOpsManager.MODE_ALLOWED) { - Slog.w(LOG_TAG, "Skipping accessibility service " + new ComponentName( - serviceInfo.packageName, serviceInfo.name).flattenToShortString() - + ": disallowed by AppOps"); - return false; + private boolean readInstalledAccessibilityShortcutLocked(AccessibilityUserState userState) { + final List<AccessibilityShortcutInfo> shortcutInfos = AccessibilityManager + .getInstance(mContext).getInstalledAccessibilityShortcutListAsUser( + mContext, mCurrentUserId); + if (!shortcutInfos.equals(userState.mInstalledShortcuts)) { + userState.mInstalledShortcuts.clear(); + userState.mInstalledShortcuts.addAll(shortcutInfos); + return true; } - - return true; + return false; } - private boolean readEnabledAccessibilityServicesLocked(UserState userState) { + private boolean readEnabledAccessibilityServicesLocked(AccessibilityUserState userState) { mTempComponentNameSet.clear(); readComponentNamesFromSettingLocked(Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES, userState.mUserId, mTempComponentNameSet); @@ -1422,7 +1398,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } private boolean readTouchExplorationGrantedAccessibilityServicesLocked( - UserState userState) { + AccessibilityUserState userState) { mTempComponentNameSet.clear(); readComponentNamesFromSettingLocked( Settings.Secure.TOUCH_EXPLORATION_GRANTED_ACCESSIBILITY_SERVICES, @@ -1447,7 +1423,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private void notifyAccessibilityServicesDelayedLocked(AccessibilityEvent event, boolean isDefault) { try { - UserState state = getCurrentUserStateLocked(); + AccessibilityUserState state = getCurrentUserStateLocked(); for (int i = 0, count = state.mBoundServices.size(); i < count; i++) { AccessibilityServiceConnection service = state.mBoundServices.get(i); @@ -1462,7 +1438,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - private void updateRelevantEventsLocked(UserState userState) { + private void updateRelevantEventsLocked(AccessibilityUserState userState) { mMainHandler.post(() -> { broadcastToClients(userState, ignoreRemoteException(client -> { int relevantEventTypes; @@ -1482,7 +1458,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub }); } - private int computeRelevantEventTypesLocked(UserState userState, Client client) { + private int computeRelevantEventTypesLocked(AccessibilityUserState userState, Client client) { int relevantEventTypes = 0; int serviceCount = userState.mBoundServices.size(); @@ -1528,20 +1504,11 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } private void broadcastToClients( - UserState userState, Consumer<Client> clientAction) { + AccessibilityUserState userState, Consumer<Client> clientAction) { mGlobalClients.broadcastForEachCookie(clientAction); userState.mUserClients.broadcastForEachCookie(clientAction); } - private void unbindAllServicesLocked(UserState userState) { - List<AccessibilityServiceConnection> services = userState.mBoundServices; - for (int count = services.size(); count > 0; count--) { - // When the service is unbound, it disappears from the list, so there's no need to - // keep track of the index - services.get(0).unbindLocked(); - } - } - /** * Populates a set with the {@link ComponentName}s stored in a colon * separated value setting for a given user. @@ -1552,9 +1519,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub */ private void readComponentNamesFromSettingLocked(String settingName, int userId, Set<ComponentName> outComponentNames) { - String settingValue = Settings.Secure.getStringForUser(mContext.getContentResolver(), - settingName, userId); - readComponentNamesFromStringLocked(settingValue, outComponentNames, false); + readColonDelimitedSettingToSet(settingName, userId, outComponentNames, + str -> ComponentName.unflattenFromString(str)); } /** @@ -1569,34 +1535,57 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private void readComponentNamesFromStringLocked(String names, Set<ComponentName> outComponentNames, boolean doMerge) { + readColonDelimitedStringToSet(names, outComponentNames, doMerge, + str -> ComponentName.unflattenFromString(str)); + } + + @Override + public void persistComponentNamesToSettingLocked(String settingName, + Set<ComponentName> componentNames, int userId) { + persistColonDelimitedSetToSettingLocked(settingName, userId, componentNames, + componentName -> componentName.flattenToShortString()); + } + + private <T> void readColonDelimitedSettingToSet(String settingName, int userId, Set<T> outSet, + Function<String, T> toItem) { + final String settingValue = Settings.Secure.getStringForUser(mContext.getContentResolver(), + settingName, userId); + readColonDelimitedStringToSet(settingValue, outSet, false, toItem); + } + + private <T> void readColonDelimitedStringToSet(String names, Set<T> outSet, boolean doMerge, + Function<String, T> toItem) { if (!doMerge) { - outComponentNames.clear(); + outSet.clear(); } - if (names != null) { - TextUtils.SimpleStringSplitter splitter = mStringColonSplitter; + if (!TextUtils.isEmpty(names)) { + final TextUtils.SimpleStringSplitter splitter = mStringColonSplitter; splitter.setString(names); while (splitter.hasNext()) { - String str = splitter.next(); - if (str == null || str.length() <= 0) { + final String str = splitter.next(); + if (TextUtils.isEmpty(str)) { continue; } - ComponentName enabledService = ComponentName.unflattenFromString(str); - if (enabledService != null) { - outComponentNames.add(enabledService); + final T item = toItem.apply(str); + if (item != null) { + outSet.add(item); } } } } - @Override - public void persistComponentNamesToSettingLocked(String settingName, - Set<ComponentName> componentNames, int userId) { - StringBuilder builder = new StringBuilder(); - for (ComponentName componentName : componentNames) { + private <T> void persistColonDelimitedSetToSettingLocked(String settingName, int userId, + Set<T> set, Function<T, String> toString) { + final StringBuilder builder = new StringBuilder(); + for (T item : set) { + final String str = (item != null ? toString.apply(item) : null); + if (TextUtils.isEmpty(str)) { + continue; + } if (builder.length() > 0) { builder.append(COMPONENT_NAME_SEPARATOR); } - builder.append(componentName.flattenToShortString()); + builder.append(str); } final long identity = Binder.clearCallingIdentity(); try { @@ -1608,7 +1597,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - private void updateServicesLocked(UserState userState) { + private void updateServicesLocked(AccessibilityUserState userState) { Map<ComponentName, AccessibilityServiceConnection> componentNameToServiceMap = userState.mComponentNameToServiceMap; boolean isUnlockingOrUnlocked = LocalServices.getService(UserManagerInternal.class) @@ -1627,8 +1616,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub continue; } - // Wait for the binding if it is in process. - if (userState.mBindingServices.contains(componentName)) { + // Skip the component since it may be in process or crashed. + if (userState.getBindingServicesLocked().contains(componentName) + || userState.getCrashedServicesLocked().contains(componentName)) { continue; } if (userState.mEnabledServices.contains(componentName) @@ -1636,8 +1626,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (service == null) { service = new AccessibilityServiceConnection(userState, mContext, componentName, installedService, sIdCounter++, mMainHandler, mLock, mSecurityPolicy, - this, mWindowManagerService, mGlobalActionPerformer, - mActivityTaskManagerService); + this, mWindowManagerService, getSystemActionPerformer(), + mA11yWindowManager, mActivityTaskManagerService); } else if (userState.mBoundServices.contains(service)) { continue; } @@ -1645,6 +1635,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } else { if (service != null) { service.unbindLocked(); + removeShortcutTargetForUnboundServiceLocked(userState, service); } } } @@ -1664,15 +1655,15 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (audioManager != null) { audioManager.setAccessibilityServiceUids(mTempIntArray); } - updateAccessibilityEnabledSetting(userState); + updateAccessibilityEnabledSettingLocked(userState); } - private void scheduleUpdateClientsIfNeededLocked(UserState userState) { - final int clientState = userState.getClientState(); - if (userState.mLastSentClientState != clientState + private void scheduleUpdateClientsIfNeededLocked(AccessibilityUserState userState) { + final int clientState = getClientStateLocked(userState); + if (userState.getLastSentClientStateLocked() != clientState && (mGlobalClients.getRegisteredCallbackCount() > 0 || userState.mUserClients.getRegisteredCallbackCount() > 0)) { - userState.mLastSentClientState = clientState; + userState.setLastSentClientStateLocked(clientState); mMainHandler.sendMessage(obtainMessage( AccessibilityManagerService::sendStateToAllClients, this, clientState, userState.mUserId)); @@ -1694,7 +1685,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub client -> client.setState(clientState))); } - private void scheduleNotifyClientsOfServicesStateChangeLocked(UserState userState) { + private void scheduleNotifyClientsOfServicesStateChangeLocked( + AccessibilityUserState userState) { updateRecommendedUiTimeoutLocked(userState); mMainHandler.sendMessage(obtainMessage( AccessibilityManagerService::sendServicesStateChanged, @@ -1713,44 +1705,51 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub client -> client.notifyServicesStateChanged(uiTimeout))); } - private void scheduleUpdateInputFilter(UserState userState) { + private void scheduleUpdateInputFilter(AccessibilityUserState userState) { mMainHandler.sendMessage(obtainMessage( AccessibilityManagerService::updateInputFilter, this, userState)); } - private void scheduleUpdateFingerprintGestureHandling(UserState userState) { + private void scheduleUpdateFingerprintGestureHandling(AccessibilityUserState userState) { mMainHandler.sendMessage(obtainMessage( AccessibilityManagerService::updateFingerprintGestureHandling, this, userState)); } - private void updateInputFilter(UserState userState) { + private void updateInputFilter(AccessibilityUserState userState) { if (mUiAutomationManager.suppressingAccessibilityServicesLocked()) return; boolean setInputFilter = false; AccessibilityInputFilter inputFilter = null; synchronized (mLock) { int flags = 0; - if (userState.mIsDisplayMagnificationEnabled) { + if (userState.isDisplayMagnificationEnabledLocked()) { flags |= AccessibilityInputFilter.FLAG_FEATURE_SCREEN_MAGNIFIER; } - if (userState.mIsNavBarMagnificationEnabled) { + if (userState.isShortcutMagnificationEnabledLocked()) { flags |= AccessibilityInputFilter.FLAG_FEATURE_TRIGGERED_SCREEN_MAGNIFIER; } if (userHasMagnificationServicesLocked(userState)) { flags |= AccessibilityInputFilter.FLAG_FEATURE_CONTROL_SCREEN_MAGNIFIER; } // Touch exploration without accessibility makes no sense. - if (userState.isHandlingAccessibilityEvents() && userState.mIsTouchExplorationEnabled) { + if (userState.isHandlingAccessibilityEventsLocked() + && userState.isTouchExplorationEnabledLocked()) { flags |= AccessibilityInputFilter.FLAG_FEATURE_TOUCH_EXPLORATION; + if (userState.isServiceHandlesDoubleTapEnabledLocked()) { + flags |= AccessibilityInputFilter.FLAG_SERVICE_HANDLES_DOUBLE_TAP; + } + if (userState.isMultiFingerGesturesEnabledLocked()) { + flags |= AccessibilityInputFilter.FLAG_REQUEST_MULTI_FINGER_GESTURES; + } } - if (userState.mIsFilterKeyEventsEnabled) { + if (userState.isFilterKeyEventsEnabledLocked()) { flags |= AccessibilityInputFilter.FLAG_FEATURE_FILTER_KEY_EVENTS; } - if (userState.mIsAutoclickEnabled) { + if (userState.isAutoclickEnabledLocked()) { flags |= AccessibilityInputFilter.FLAG_FEATURE_AUTOCLICK; } - if (userState.mIsPerformGesturesEnabled) { + if (userState.isPerformGesturesEnabledLocked()) { flags |= AccessibilityInputFilter.FLAG_FEATURE_INJECT_MOTION_EVENTS; } if (flags != 0) { @@ -1783,8 +1782,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub String label = service.getServiceInfo().getResolveInfo() .loadLabel(mContext.getPackageManager()).toString(); - final UserState userState = getCurrentUserStateLocked(); - if (userState.mIsTouchExplorationEnabled) { + final AccessibilityUserState userState = getCurrentUserStateLocked(); + if (userState.isTouchExplorationEnabledLocked()) { return; } if (mEnableTouchExplorationDialog != null @@ -1794,40 +1793,40 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub mEnableTouchExplorationDialog = new AlertDialog.Builder(mContext) .setIconAttribute(android.R.attr.alertDialogIcon) .setPositiveButton(android.R.string.ok, new OnClickListener() { - @Override - public void onClick(DialogInterface dialog, int which) { - // The user allowed the service to toggle touch exploration. - userState.mTouchExplorationGrantedServices.add(service.mComponentName); - persistComponentNamesToSettingLocked( - Settings.Secure.TOUCH_EXPLORATION_GRANTED_ACCESSIBILITY_SERVICES, - userState.mTouchExplorationGrantedServices, userState.mUserId); - // Enable touch exploration. - userState.mIsTouchExplorationEnabled = true; - final long identity = Binder.clearCallingIdentity(); - try { - Settings.Secure.putIntForUser(mContext.getContentResolver(), - Settings.Secure.TOUCH_EXPLORATION_ENABLED, 1, - userState.mUserId); - } finally { - Binder.restoreCallingIdentity(identity); - } - onUserStateChangedLocked(userState); - } - }) - .setNegativeButton(android.R.string.cancel, new OnClickListener() { - @Override - public void onClick(DialogInterface dialog, int which) { - dialog.dismiss(); - } - }) - .setTitle(R.string.enable_explore_by_touch_warning_title) - .setMessage(mContext.getString( - R.string.enable_explore_by_touch_warning_message, label)) - .create(); + @Override + public void onClick(DialogInterface dialog, int which) { + // The user allowed the service to toggle touch exploration. + userState.mTouchExplorationGrantedServices.add(service.mComponentName); + persistComponentNamesToSettingLocked( + Settings.Secure.TOUCH_EXPLORATION_GRANTED_ACCESSIBILITY_SERVICES, + userState.mTouchExplorationGrantedServices, userState.mUserId); + // Enable touch exploration. + userState.setTouchExplorationEnabledLocked(true); + final long identity = Binder.clearCallingIdentity(); + try { + Settings.Secure.putIntForUser(mContext.getContentResolver(), + Settings.Secure.TOUCH_EXPLORATION_ENABLED, 1, + userState.mUserId); + } finally { + Binder.restoreCallingIdentity(identity); + } + onUserStateChangedLocked(userState); + } + }) + .setNegativeButton(android.R.string.cancel, new OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + dialog.dismiss(); + } + }) + .setTitle(R.string.enable_explore_by_touch_warning_title) + .setMessage(mContext.getString( + R.string.enable_explore_by_touch_warning_message, label)) + .create(); mEnableTouchExplorationDialog.getWindow().setType( WindowManager.LayoutParams.TYPE_SYSTEM_ALERT); mEnableTouchExplorationDialog.getWindow().getAttributes().privateFlags - |= WindowManager.LayoutParams.PRIVATE_FLAG_SHOW_FOR_ALL_USERS; + |= WindowManager.LayoutParams.SYSTEM_FLAG_SHOW_FOR_ALL_USERS; mEnableTouchExplorationDialog.setCanceledOnTouchOutside(true); mEnableTouchExplorationDialog.show(); } @@ -1838,14 +1837,12 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub * * @param userState the new user state */ - private void onUserStateChangedLocked(UserState userState) { + private void onUserStateChangedLocked(AccessibilityUserState userState) { // TODO: Remove this hack mInitialized = true; updateLegacyCapabilitiesLocked(userState); updateServicesLocked(userState); - updateAccessibilityShortcutLocked(userState); updateWindowsForAccessibilityCallbackLocked(userState); - updateAccessibilityFocusBehaviorLocked(userState); updateFilterKeyEventsLocked(userState); updateTouchExplorationLocked(userState); updatePerformGesturesLocked(userState); @@ -1854,32 +1851,11 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub scheduleUpdateInputFilter(userState); updateRelevantEventsLocked(userState); scheduleUpdateClientsIfNeededLocked(userState); + updateAccessibilityShortcutKeyTargetsLocked(userState); updateAccessibilityButtonTargetsLocked(userState); } - private void updateAccessibilityFocusBehaviorLocked(UserState userState) { - // If there is no service that can operate with interactive windows - // then we keep the old behavior where a window loses accessibility - // focus if it is no longer active. This still changes the behavior - // for services that do not operate with interactive windows and run - // at the same time as the one(s) which does. In practice however, - // there is only one service that uses accessibility focus and it - // is typically the one that operates with interactive windows, So, - // this is fine. Note that to allow a service to work across windows - // we have to allow accessibility focus stay in any of them. Sigh... - List<AccessibilityServiceConnection> boundServices = userState.mBoundServices; - final int boundServiceCount = boundServices.size(); - for (int i = 0; i < boundServiceCount; i++) { - AccessibilityServiceConnection boundService = boundServices.get(i); - if (boundService.canRetrieveInteractiveWindowsLocked()) { - userState.mAccessibilityFocusOnlyInActiveWindow = false; - return; - } - } - userState.mAccessibilityFocusOnlyInActiveWindow = true; - } - - private void updateWindowsForAccessibilityCallbackLocked(UserState userState) { + private void updateWindowsForAccessibilityCallbackLocked(AccessibilityUserState userState) { // We observe windows for accessibility only if there is at least // one bound service that can retrieve window content that specified // it is interested in accessing such windows. For services that are @@ -1892,28 +1868,28 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub for (int i = 0; !observingWindows && (i < boundServiceCount); i++) { AccessibilityServiceConnection boundService = boundServices.get(i); if (boundService.canRetrieveInteractiveWindowsLocked()) { + userState.setAccessibilityFocusOnlyInActiveWindow(false); observingWindows = true; } } + userState.setAccessibilityFocusOnlyInActiveWindow(true); - if (observingWindows) { - if (mWindowsForAccessibilityCallback == null) { - mWindowsForAccessibilityCallback = new WindowsForAccessibilityCallback(); - mWindowManagerService.setWindowsForAccessibilityCallback( - mWindowsForAccessibilityCallback); + // Gets all valid displays and start tracking windows of each display if there is at least + // one bound service that can retrieve window content. + final ArrayList<Display> displays = getValidDisplayList(); + for (int i = 0; i < displays.size(); i++) { + final Display display = displays.get(i); + if (display != null) { + if (observingWindows) { + mA11yWindowManager.startTrackingWindows(display.getDisplayId()); + } else { + mA11yWindowManager.stopTrackingWindows(display.getDisplayId()); + } } - return; - } - - if (mWindowsForAccessibilityCallback != null) { - mWindowsForAccessibilityCallback = null; - mWindowManagerService.setWindowsForAccessibilityCallback(null); - // Drop all windows we know about. - mSecurityPolicy.clearWindowsLocked(); } } - private void updateLegacyCapabilitiesLocked(UserState userState) { + private void updateLegacyCapabilitiesLocked(AccessibilityUserState userState) { // Up to JB-MR1 we had a allowlist with services that can enable touch // exploration. When a service is first started we show a dialog to the // use to get a permission to allowlist the service. @@ -1935,20 +1911,20 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - private void updatePerformGesturesLocked(UserState userState) { + private void updatePerformGesturesLocked(AccessibilityUserState userState) { final int serviceCount = userState.mBoundServices.size(); for (int i = 0; i < serviceCount; i++) { AccessibilityServiceConnection service = userState.mBoundServices.get(i); if ((service.getCapabilities() & AccessibilityServiceInfo.CAPABILITY_CAN_PERFORM_GESTURES) != 0) { - userState.mIsPerformGesturesEnabled = true; + userState.setPerformGesturesEnabledLocked(true); return; } } - userState.mIsPerformGesturesEnabled = false; + userState.setPerformGesturesEnabledLocked(false); } - private void updateFilterKeyEventsLocked(UserState userState) { + private void updateFilterKeyEventsLocked(AccessibilityUserState userState) { final int serviceCount = userState.mBoundServices.size(); for (int i = 0; i < serviceCount; i++) { AccessibilityServiceConnection service = userState.mBoundServices.get(i); @@ -1956,31 +1932,33 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub && (service.getCapabilities() & AccessibilityServiceInfo .CAPABILITY_CAN_REQUEST_FILTER_KEY_EVENTS) != 0) { - userState.mIsFilterKeyEventsEnabled = true; + userState.setFilterKeyEventsEnabledLocked(true); return; } } - userState.mIsFilterKeyEventsEnabled = false; + userState.setFilterKeyEventsEnabledLocked(false); } - private boolean readConfigurationForUserStateLocked(UserState userState) { + private boolean readConfigurationForUserStateLocked(AccessibilityUserState userState) { boolean somethingChanged = readInstalledAccessibilityServiceLocked(userState); + somethingChanged |= readInstalledAccessibilityShortcutLocked(userState); somethingChanged |= readEnabledAccessibilityServicesLocked(userState); somethingChanged |= readTouchExplorationGrantedAccessibilityServicesLocked(userState); somethingChanged |= readTouchExplorationEnabledSettingLocked(userState); somethingChanged |= readHighTextContrastEnabledSettingLocked(userState); somethingChanged |= readMagnificationEnabledSettingsLocked(userState); somethingChanged |= readAutoclickEnabledSettingLocked(userState); - somethingChanged |= readAccessibilityShortcutSettingLocked(userState); - somethingChanged |= readAccessibilityButtonSettingsLocked(userState); + somethingChanged |= readAccessibilityShortcutKeySettingLocked(userState); + somethingChanged |= readAccessibilityButtonTargetsLocked(userState); + somethingChanged |= readAccessibilityButtonTargetComponentLocked(userState); somethingChanged |= readUserRecommendedUiTimeoutSettingsLocked(userState); return somethingChanged; } - private void updateAccessibilityEnabledSetting(UserState userState) { + private void updateAccessibilityEnabledSettingLocked(AccessibilityUserState userState) { final long identity = Binder.clearCallingIdentity(); final boolean isA11yEnabled = mUiAutomationManager.isUiAutomationRunningLocked() - || userState.isHandlingAccessibilityEvents(); + || userState.isHandlingAccessibilityEventsLocked(); try { Settings.Secure.putIntForUser(mContext.getContentResolver(), Settings.Secure.ACCESSIBILITY_ENABLED, @@ -1991,136 +1969,141 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - private boolean readTouchExplorationEnabledSettingLocked(UserState userState) { + private boolean readTouchExplorationEnabledSettingLocked(AccessibilityUserState userState) { final boolean touchExplorationEnabled = Settings.Secure.getIntForUser( mContext.getContentResolver(), Settings.Secure.TOUCH_EXPLORATION_ENABLED, 0, userState.mUserId) == 1; - if (touchExplorationEnabled != userState.mIsTouchExplorationEnabled) { - userState.mIsTouchExplorationEnabled = touchExplorationEnabled; + if (touchExplorationEnabled != userState.isTouchExplorationEnabledLocked()) { + userState.setTouchExplorationEnabledLocked(touchExplorationEnabled); return true; } return false; } - private boolean readMagnificationEnabledSettingsLocked(UserState userState) { + private boolean readMagnificationEnabledSettingsLocked(AccessibilityUserState userState) { final boolean displayMagnificationEnabled = Settings.Secure.getIntForUser( mContext.getContentResolver(), Settings.Secure.ACCESSIBILITY_DISPLAY_MAGNIFICATION_ENABLED, 0, userState.mUserId) == 1; - final boolean navBarMagnificationEnabled = Settings.Secure.getIntForUser( - mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_DISPLAY_MAGNIFICATION_NAVBAR_ENABLED, - 0, userState.mUserId) == 1; - if ((displayMagnificationEnabled != userState.mIsDisplayMagnificationEnabled) - || (navBarMagnificationEnabled != userState.mIsNavBarMagnificationEnabled)) { - userState.mIsDisplayMagnificationEnabled = displayMagnificationEnabled; - userState.mIsNavBarMagnificationEnabled = navBarMagnificationEnabled; + if ((displayMagnificationEnabled != userState.isDisplayMagnificationEnabledLocked())) { + userState.setDisplayMagnificationEnabledLocked(displayMagnificationEnabled); return true; } return false; } - private boolean readAutoclickEnabledSettingLocked(UserState userState) { + private boolean readAutoclickEnabledSettingLocked(AccessibilityUserState userState) { final boolean autoclickEnabled = Settings.Secure.getIntForUser( mContext.getContentResolver(), Settings.Secure.ACCESSIBILITY_AUTOCLICK_ENABLED, 0, userState.mUserId) == 1; - if (autoclickEnabled != userState.mIsAutoclickEnabled) { - userState.mIsAutoclickEnabled = autoclickEnabled; + if (autoclickEnabled != userState.isAutoclickEnabledLocked()) { + userState.setAutoclickEnabledLocked(autoclickEnabled); return true; } return false; } - private boolean readHighTextContrastEnabledSettingLocked(UserState userState) { + private boolean readHighTextContrastEnabledSettingLocked(AccessibilityUserState userState) { final boolean highTextContrastEnabled = Settings.Secure.getIntForUser( mContext.getContentResolver(), Settings.Secure.ACCESSIBILITY_HIGH_TEXT_CONTRAST_ENABLED, 0, userState.mUserId) == 1; - if (highTextContrastEnabled != userState.mIsTextHighContrastEnabled) { - userState.mIsTextHighContrastEnabled = highTextContrastEnabled; + if (highTextContrastEnabled != userState.isTextHighContrastEnabledLocked()) { + userState.setTextHighContrastEnabledLocked(highTextContrastEnabled); return true; } return false; } - private void updateTouchExplorationLocked(UserState userState) { - boolean enabled = mUiAutomationManager.isTouchExplorationEnabledLocked(); + private void updateTouchExplorationLocked(AccessibilityUserState userState) { + boolean touchExplorationEnabled = mUiAutomationManager.isTouchExplorationEnabledLocked(); + boolean serviceHandlesDoubleTapEnabled = false; + boolean requestMultiFingerGestures = false; final int serviceCount = userState.mBoundServices.size(); for (int i = 0; i < serviceCount; i++) { AccessibilityServiceConnection service = userState.mBoundServices.get(i); if (canRequestAndRequestsTouchExplorationLocked(service, userState)) { - enabled = true; + touchExplorationEnabled = true; + serviceHandlesDoubleTapEnabled = service.isServiceHandlesDoubleTapEnabled(); + requestMultiFingerGestures = service.isMultiFingerGesturesEnabled(); break; } } - if (enabled != userState.mIsTouchExplorationEnabled) { - userState.mIsTouchExplorationEnabled = enabled; + if (touchExplorationEnabled != userState.isTouchExplorationEnabledLocked()) { + userState.setTouchExplorationEnabledLocked(touchExplorationEnabled); final long identity = Binder.clearCallingIdentity(); try { Settings.Secure.putIntForUser(mContext.getContentResolver(), - Settings.Secure.TOUCH_EXPLORATION_ENABLED, enabled ? 1 : 0, + Settings.Secure.TOUCH_EXPLORATION_ENABLED, touchExplorationEnabled ? 1 : 0, userState.mUserId); } finally { Binder.restoreCallingIdentity(identity); } } + userState.setServiceHandlesDoubleTapLocked(serviceHandlesDoubleTapEnabled); + userState.setMultiFingerGesturesLocked(requestMultiFingerGestures); } - private boolean readAccessibilityShortcutSettingLocked(UserState userState) { - String componentNameToEnableString = AccessibilityShortcutController - .getTargetServiceComponentNameString(mContext, userState.mUserId); - if ((componentNameToEnableString == null) || componentNameToEnableString.isEmpty()) { - if (userState.mServiceToEnableWithShortcut == null) { - return false; + private boolean readAccessibilityShortcutKeySettingLocked(AccessibilityUserState userState) { + final String settingValue = Settings.Secure.getStringForUser(mContext.getContentResolver(), + Settings.Secure.ACCESSIBILITY_SHORTCUT_TARGET_SERVICE, userState.mUserId); + final Set<String> targetsFromSetting = new ArraySet<>(); + readColonDelimitedStringToSet(settingValue, targetsFromSetting, false, str -> str); + // Fall back to device's default a11y service, only when setting is never updated. + if (settingValue == null) { + final String defaultService = mContext.getString( + R.string.config_defaultAccessibilityService); + if (!TextUtils.isEmpty(defaultService)) { + targetsFromSetting.add(defaultService); } - userState.mServiceToEnableWithShortcut = null; - return true; } - ComponentName componentNameToEnable = - ComponentName.unflattenFromString(componentNameToEnableString); - if ((componentNameToEnable != null) - && componentNameToEnable.equals(userState.mServiceToEnableWithShortcut)) { + + final Set<String> currentTargets = + userState.getShortcutTargetsLocked(ACCESSIBILITY_SHORTCUT_KEY); + if (targetsFromSetting.equals(currentTargets)) { return false; } + currentTargets.clear(); + currentTargets.addAll(targetsFromSetting); + scheduleNotifyClientsOfServicesStateChangeLocked(userState); + return true; + } + + private boolean readAccessibilityButtonTargetsLocked(AccessibilityUserState userState) { + final Set<String> targetsFromSetting = new ArraySet<>(); + readColonDelimitedSettingToSet(Settings.Secure.ACCESSIBILITY_BUTTON_TARGETS, + userState.mUserId, targetsFromSetting, str -> str); - userState.mServiceToEnableWithShortcut = componentNameToEnable; + final Set<String> currentTargets = + userState.getShortcutTargetsLocked(ACCESSIBILITY_BUTTON); + if (targetsFromSetting.equals(currentTargets)) { + return false; + } + currentTargets.clear(); + currentTargets.addAll(targetsFromSetting); scheduleNotifyClientsOfServicesStateChangeLocked(userState); return true; } - private boolean readAccessibilityButtonSettingsLocked(UserState userState) { - String componentId = Settings.Secure.getStringForUser(mContext.getContentResolver(), + private boolean readAccessibilityButtonTargetComponentLocked(AccessibilityUserState userState) { + final String componentId = Settings.Secure.getStringForUser(mContext.getContentResolver(), Settings.Secure.ACCESSIBILITY_BUTTON_TARGET_COMPONENT, userState.mUserId); if (TextUtils.isEmpty(componentId)) { - if ((userState.mServiceAssignedToAccessibilityButton == null) - && !userState.mIsNavBarMagnificationAssignedToAccessibilityButton) { - return false; - } - userState.mServiceAssignedToAccessibilityButton = null; - userState.mIsNavBarMagnificationAssignedToAccessibilityButton = false; - return true; - } - - if (componentId.equals(MagnificationController.class.getName())) { - if (userState.mIsNavBarMagnificationAssignedToAccessibilityButton) { + if (userState.getTargetAssignedToAccessibilityButton() == null) { return false; } - userState.mServiceAssignedToAccessibilityButton = null; - userState.mIsNavBarMagnificationAssignedToAccessibilityButton = true; + userState.setTargetAssignedToAccessibilityButton(null); return true; } - - ComponentName componentName = ComponentName.unflattenFromString(componentId); - if (Objects.equals(componentName, userState.mServiceAssignedToAccessibilityButton)) { + if (componentId.equals(userState.getTargetAssignedToAccessibilityButton())) { return false; } - userState.mServiceAssignedToAccessibilityButton = componentName; - userState.mIsNavBarMagnificationAssignedToAccessibilityButton = false; + userState.setTargetAssignedToAccessibilityButton(componentId); return true; } - private boolean readUserRecommendedUiTimeoutSettingsLocked(UserState userState) { + private boolean readUserRecommendedUiTimeoutSettingsLocked(AccessibilityUserState userState) { final int nonInteractiveUiTimeout = Settings.Secure.getIntForUser( mContext.getContentResolver(), Settings.Secure.ACCESSIBILITY_NON_INTERACTIVE_UI_TIMEOUT_MS, 0, @@ -2129,10 +2112,10 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub mContext.getContentResolver(), Settings.Secure.ACCESSIBILITY_INTERACTIVE_UI_TIMEOUT_MS, 0, userState.mUserId); - if (nonInteractiveUiTimeout != userState.mUserNonInteractiveUiTimeout - || interactiveUiTimeout != userState.mUserInteractiveUiTimeout) { - userState.mUserNonInteractiveUiTimeout = nonInteractiveUiTimeout; - userState.mUserInteractiveUiTimeout = interactiveUiTimeout; + if (nonInteractiveUiTimeout != userState.getUserNonInteractiveUiTimeoutLocked() + || interactiveUiTimeout != userState.getUserInteractiveUiTimeoutLocked()) { + userState.setUserNonInteractiveUiTimeoutLocked(nonInteractiveUiTimeout); + userState.setUserInteractiveUiTimeoutLocked(interactiveUiTimeout); scheduleNotifyClientsOfServicesStateChangeLocked(userState); return true; } @@ -2140,44 +2123,32 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } /** - * Check if the service that will be enabled by the shortcut is installed. If it isn't, - * clear the value and the associated setting so a sideloaded service can't spoof the - * package name of the default service. - * - * @param userState + * Check if the target that will be enabled by the accessibility shortcut key is installed. + * If it isn't, remove it from the list and associated setting so a side loaded service can't + * spoof the package name of the default service. */ - private void updateAccessibilityShortcutLocked(UserState userState) { - if (userState.mServiceToEnableWithShortcut == null) { + private void updateAccessibilityShortcutKeyTargetsLocked(AccessibilityUserState userState) { + final Set<String> currentTargets = + userState.getShortcutTargetsLocked(ACCESSIBILITY_SHORTCUT_KEY); + final int lastSize = currentTargets.size(); + if (lastSize == 0) { return; } - boolean shortcutServiceIsInstalled = - AccessibilityShortcutController.getFrameworkShortcutFeaturesMap() - .containsKey(userState.mServiceToEnableWithShortcut); - for (int i = 0; !shortcutServiceIsInstalled && (i < userState.mInstalledServices.size()); - i++) { - if (userState.mInstalledServices.get(i).getComponentName() - .equals(userState.mServiceToEnableWithShortcut)) { - shortcutServiceIsInstalled = true; - } + currentTargets.removeIf( + name -> !userState.isShortcutTargetInstalledLocked(name)); + if (lastSize == currentTargets.size()) { + return; } - if (!shortcutServiceIsInstalled) { - userState.mServiceToEnableWithShortcut = null; - final long identity = Binder.clearCallingIdentity(); - try { - Settings.Secure.putStringForUser(mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_SHORTCUT_TARGET_SERVICE, null, - userState.mUserId); - Settings.Secure.putIntForUser(mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_SHORTCUT_ENABLED, 0, userState.mUserId); - } finally { - Binder.restoreCallingIdentity(identity); - } - } + // Update setting key with new value. + persistColonDelimitedSetToSettingLocked( + Settings.Secure.ACCESSIBILITY_SHORTCUT_TARGET_SERVICE, + userState.mUserId, currentTargets, str -> str); + scheduleNotifyClientsOfServicesStateChangeLocked(userState); } private boolean canRequestAndRequestsTouchExplorationLocked( - AccessibilityServiceConnection service, UserState userState) { + AccessibilityServiceConnection service, AccessibilityUserState userState) { // Service not ready or cannot request the feature - well nothing to do. if (!service.canReceiveEventsLocked() || !service.mRequestTouchExplorationMode) { return false; @@ -2207,11 +2178,15 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return false; } - private void updateMagnificationLocked(UserState userState) { + private void updateMagnificationLocked(AccessibilityUserState userState) { if (userState.mUserId != mCurrentUserId) { return; } + if (mMagnificationController != null) { + mMagnificationController.setUserId(userState.mUserId); + } + if (mUiAutomationManager.suppressingAccessibilityServicesLocked() && mMagnificationController != null) { mMagnificationController.unregisterAll(); @@ -2222,8 +2197,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub // We would skip overlay display because it uses overlay window to simulate secondary // displays in one display. It's not a real display and there's no input events for it. final ArrayList<Display> displays = getValidDisplayList(); - if (userState.mIsDisplayMagnificationEnabled - || userState.mIsNavBarMagnificationEnabled) { + if (userState.isDisplayMagnificationEnabledLocked() + || userState.isShortcutMagnificationEnabledLocked()) { for (int i = 0; i < displays.size(); i++) { final Display display = displays.get(i); getMagnificationController().register(display.getDisplayId()); @@ -2247,7 +2222,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub * Returns whether the specified user has any services that are capable of * controlling magnification. */ - private boolean userHasMagnificationServicesLocked(UserState userState) { + private boolean userHasMagnificationServicesLocked(AccessibilityUserState userState) { final List<AccessibilityServiceConnection> services = userState.mBoundServices; for (int i = 0, count = services.size(); i < count; i++) { final AccessibilityServiceConnection service = services.get(i); @@ -2262,7 +2237,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub * Returns whether the specified user has any services that are capable of * controlling magnification and are actively listening for magnification updates. */ - private boolean userHasListeningMagnificationServicesLocked(UserState userState, + private boolean userHasListeningMagnificationServicesLocked(AccessibilityUserState userState, int displayId) { final List<AccessibilityServiceConnection> services = userState.mBoundServices; for (int i = 0, count = services.size(); i < count; i++) { @@ -2275,7 +2250,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return false; } - private void updateFingerprintGestureHandling(UserState userState) { + private void updateFingerprintGestureHandling(AccessibilityUserState userState) { final List<AccessibilityServiceConnection> services; synchronized (mLock) { services = userState.mBoundServices; @@ -2307,7 +2282,14 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - private void updateAccessibilityButtonTargetsLocked(UserState userState) { + /** + * 1) Update accessibility button availability to accessibility services. + * 2) Check if the target that will be enabled by the accessibility button is installed. + * If it isn't, remove it from the list and associated setting so a side loaded service can't + * spoof the package name of the default service. + */ + private void updateAccessibilityButtonTargetsLocked(AccessibilityUserState userState) { + // Update accessibility button availability. for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = userState.mBoundServices.get(i); if (service.mRequestAccessibilityButton) { @@ -2315,11 +2297,154 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub service.isAccessibilityButtonAvailableLocked(userState)); } } + + final Set<String> currentTargets = + userState.getShortcutTargetsLocked(ACCESSIBILITY_BUTTON); + final int lastSize = currentTargets.size(); + if (lastSize == 0) { + return; + } + currentTargets.removeIf( + name -> !userState.isShortcutTargetInstalledLocked(name)); + if (lastSize == currentTargets.size()) { + return; + } + + // Update setting key with new value. + persistColonDelimitedSetToSettingLocked(Settings.Secure.ACCESSIBILITY_BUTTON_TARGETS, + userState.mUserId, currentTargets, str -> str); + scheduleNotifyClientsOfServicesStateChangeLocked(userState); } - private void updateRecommendedUiTimeoutLocked(UserState userState) { - int newNonInteractiveUiTimeout = userState.mUserNonInteractiveUiTimeout; - int newInteractiveUiTimeout = userState.mUserInteractiveUiTimeout; + /** + * 1) Check if the service assigned to accessibility button target sdk version > Q. + * If it isn't, remove it from the list and associated setting. + * (It happens when an accessibility service package is downgraded.) + * 2) For a service targeting sdk version > Q and requesting a11y button, it should be in the + * enabled list if's assigned to a11y button. + * (It happens when an accessibility service package is same graded, and updated requesting + * a11y button flag) + * 3) Check if an enabled service targeting sdk version > Q and requesting a11y button is + * assigned to a shortcut. If it isn't, assigns it to the accessibility button. + * (It happens when an enabled accessibility service package is upgraded.) + * + * @param packageName The package name to check, or {@code null} to check all services. + */ + private void migrateAccessibilityButtonSettingsIfNecessaryLocked( + AccessibilityUserState userState, @Nullable String packageName) { + final Set<String> buttonTargets = + userState.getShortcutTargetsLocked(ACCESSIBILITY_BUTTON); + int lastSize = buttonTargets.size(); + buttonTargets.removeIf(name -> { + if (packageName != null && name != null && !name.contains(packageName)) { + return false; + } + final ComponentName componentName = ComponentName.unflattenFromString(name); + if (componentName == null) { + return false; + } + final AccessibilityServiceInfo serviceInfo = + userState.getInstalledServiceInfoLocked(componentName); + if (serviceInfo == null) { + return false; + } + if (serviceInfo.getResolveInfo().serviceInfo.applicationInfo + .targetSdkVersion <= Build.VERSION_CODES.Q) { + // A11y services targeting sdk version <= Q should not be in the list. + Slog.v(LOG_TAG, "Legacy service " + componentName + + " should not in the button"); + return true; + } + final boolean requestA11yButton = (serviceInfo.flags + & AccessibilityServiceInfo.FLAG_REQUEST_ACCESSIBILITY_BUTTON) != 0; + if (requestA11yButton && !userState.mEnabledServices.contains(componentName)) { + // An a11y service targeting sdk version > Q and request A11y button and is assigned + // to a11y btn should be in the enabled list. + Slog.v(LOG_TAG, "Service requesting a11y button and be assigned to the button" + + componentName + " should be enabled state"); + return true; + } + return false; + }); + boolean changed = (lastSize != buttonTargets.size()); + lastSize = buttonTargets.size(); + + final Set<String> shortcutKeyTargets = + userState.getShortcutTargetsLocked(ACCESSIBILITY_SHORTCUT_KEY); + userState.mEnabledServices.forEach(componentName -> { + if (packageName != null && componentName != null + && !packageName.equals(componentName.getPackageName())) { + return; + } + final AccessibilityServiceInfo serviceInfo = + userState.getInstalledServiceInfoLocked(componentName); + if (serviceInfo == null) { + return; + } + final boolean requestA11yButton = (serviceInfo.flags + & AccessibilityServiceInfo.FLAG_REQUEST_ACCESSIBILITY_BUTTON) != 0; + if (!(serviceInfo.getResolveInfo().serviceInfo.applicationInfo + .targetSdkVersion > Build.VERSION_CODES.Q && requestA11yButton)) { + return; + } + final String serviceName = componentName.flattenToString(); + if (TextUtils.isEmpty(serviceName)) { + return; + } + if (doesShortcutTargetsStringContain(buttonTargets, serviceName) + || doesShortcutTargetsStringContain(shortcutKeyTargets, serviceName)) { + return; + } + // For enabled a11y services targeting sdk version > Q and requesting a11y button should + // be assigned to a shortcut. + Slog.v(LOG_TAG, "A enabled service requesting a11y button " + componentName + + " should be assign to the button or shortcut."); + buttonTargets.add(serviceName); + }); + changed |= (lastSize != buttonTargets.size()); + if (!changed) { + return; + } + + // Update setting key with new value. + persistColonDelimitedSetToSettingLocked(Settings.Secure.ACCESSIBILITY_BUTTON_TARGETS, + userState.mUserId, buttonTargets, str -> str); + scheduleNotifyClientsOfServicesStateChangeLocked(userState); + } + + /** + * Remove the shortcut target for the unbound service which is requesting accessibility button + * and targeting sdk > Q from the accessibility button and shortcut. + * + * @param userState The accessibility user state. + * @param service The unbound service. + */ + private void removeShortcutTargetForUnboundServiceLocked(AccessibilityUserState userState, + AccessibilityServiceConnection service) { + if (!service.mRequestAccessibilityButton + || service.getServiceInfo().getResolveInfo().serviceInfo.applicationInfo + .targetSdkVersion <= Build.VERSION_CODES.Q) { + return; + } + final ComponentName serviceName = service.getComponentName(); + if (userState.removeShortcutTargetLocked(ACCESSIBILITY_SHORTCUT_KEY, serviceName)) { + final Set<String> currentTargets = userState.getShortcutTargetsLocked( + ACCESSIBILITY_SHORTCUT_KEY); + persistColonDelimitedSetToSettingLocked( + Settings.Secure.ACCESSIBILITY_SHORTCUT_TARGET_SERVICE, + userState.mUserId, currentTargets, str -> str); + } + if (userState.removeShortcutTargetLocked(ACCESSIBILITY_BUTTON, serviceName)) { + final Set<String> currentTargets = userState.getShortcutTargetsLocked( + ACCESSIBILITY_BUTTON); + persistColonDelimitedSetToSettingLocked(Settings.Secure.ACCESSIBILITY_BUTTON_TARGETS, + userState.mUserId, currentTargets, str -> str); + } + } + + private void updateRecommendedUiTimeoutLocked(AccessibilityUserState userState) { + int newNonInteractiveUiTimeout = userState.getUserNonInteractiveUiTimeoutLocked(); + int newInteractiveUiTimeout = userState.getUserInteractiveUiTimeoutLocked(); // read from a11y services if user does not specify value if (newNonInteractiveUiTimeout == 0 || newInteractiveUiTimeout == 0) { int serviceNonInteractiveUiTimeout = 0; @@ -2342,17 +2467,15 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub newInteractiveUiTimeout = serviceInteractiveUiTimeout; } } - userState.mNonInteractiveUiTimeout = newNonInteractiveUiTimeout; - userState.mInteractiveUiTimeout = newInteractiveUiTimeout; + userState.setNonInteractiveUiTimeoutLocked(newNonInteractiveUiTimeout); + userState.setInteractiveUiTimeoutLocked(newInteractiveUiTimeout); } @GuardedBy("mLock") @Override public MagnificationSpec getCompatibleMagnificationSpecLocked(int windowId) { - IBinder windowToken = mGlobalWindowTokens.get(windowId); - if (windowToken == null) { - windowToken = getCurrentUserStateLocked().mWindowTokens.get(windowId); - } + IBinder windowToken = mA11yWindowManager.getWindowTokenForUserAndWindowIdLocked( + mCurrentUserId, windowId); if (windowToken != null) { return mWindowManagerService.getCompatibleMagnificationSpecForWindow( windowToken); @@ -2375,62 +2498,232 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub int flags) { return PendingIntent.getActivity(context, requestCode, intent, flags); } + /** - * AIDL-exposed method to be called when the accessibility shortcut is enabled. Requires + * AIDL-exposed method to be called when the accessibility shortcut key is enabled. Requires * permission to write secure settings, since someone with that permission can enable * accessibility services themselves. + * + * @param targetName The flattened {@link ComponentName} string or the class name of a system + * class implementing a supported accessibility feature, or {@code null} if there's no + * specified target. */ @Override - public void performAccessibilityShortcut() { + public void performAccessibilityShortcut(String targetName) { if ((UserHandle.getAppId(Binder.getCallingUid()) != Process.SYSTEM_UID) && (mContext.checkCallingPermission(Manifest.permission.MANAGE_ACCESSIBILITY) != PackageManager.PERMISSION_GRANTED)) { throw new SecurityException( "performAccessibilityShortcut requires the MANAGE_ACCESSIBILITY permission"); } + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::performAccessibilityShortcutInternal, this, + Display.DEFAULT_DISPLAY, ACCESSIBILITY_SHORTCUT_KEY, targetName)); + } + + /** + * Perform the accessibility shortcut action. + * + * @param shortcutType The shortcut type. + * @param displayId The display id of the accessibility button. + * @param targetName The flattened {@link ComponentName} string or the class name of a system + * class implementing a supported accessibility feature, or {@code null} if there's no + * specified target. + */ + private void performAccessibilityShortcutInternal(int displayId, + @ShortcutType int shortcutType, @Nullable String targetName) { + final List<String> shortcutTargets = getAccessibilityShortcutTargetsInternal(shortcutType); + if (shortcutTargets.isEmpty()) { + Slog.d(LOG_TAG, "No target to perform shortcut, shortcutType=" + shortcutType); + return; + } + // In case the caller specified a target name + if (targetName != null && !doesShortcutTargetsStringContain(shortcutTargets, targetName)) { + Slog.v(LOG_TAG, "Perform shortcut failed, invalid target name:" + targetName); + targetName = null; + } + if (targetName == null) { + // In case there are many targets assigned to the given shortcut. + if (shortcutTargets.size() > 1) { + showAccessibilityTargetsSelection(displayId, shortcutType); + return; + } + targetName = shortcutTargets.get(0); + } + // In case user assigned magnification to the given shortcut. + if (targetName.equals(MAGNIFICATION_CONTROLLER_NAME)) { + final boolean enabled = !getMagnificationController().isMagnifying(displayId); + logAccessibilityShortcutActivated(MAGNIFICATION_COMPONENT_NAME, shortcutType, enabled); + sendAccessibilityButtonToInputFilter(displayId); + return; + } + final ComponentName targetComponentName = ComponentName.unflattenFromString(targetName); + if (targetComponentName == null) { + Slog.d(LOG_TAG, "Perform shortcut failed, invalid target name:" + targetName); + return; + } + // In case user assigned an accessibility framework feature to the given shortcut. + if (performAccessibilityFrameworkFeature(targetComponentName, shortcutType)) { + return; + } + // In case user assigned an accessibility shortcut target to the given shortcut. + if (performAccessibilityShortcutTargetActivity(displayId, targetComponentName)) { + logAccessibilityShortcutActivated(targetComponentName, shortcutType); + return; + } + // in case user assigned an accessibility service to the given shortcut. + if (performAccessibilityShortcutTargetService( + displayId, shortcutType, targetComponentName)) { + return; + } + } + + private boolean performAccessibilityFrameworkFeature(ComponentName assignedTarget, + @ShortcutType int shortcutType) { final Map<ComponentName, ToggleableFrameworkFeatureInfo> frameworkFeatureMap = AccessibilityShortcutController.getFrameworkShortcutFeaturesMap(); - synchronized(mLock) { - final UserState userState = getUserStateLocked(mCurrentUserId); - final ComponentName serviceName = userState.mServiceToEnableWithShortcut; - if (serviceName == null) { - return; + if (!frameworkFeatureMap.containsKey(assignedTarget)) { + return false; + } + // Toggle the requested framework feature + final ToggleableFrameworkFeatureInfo featureInfo = frameworkFeatureMap.get(assignedTarget); + final SettingStringHelper setting = new SettingStringHelper(mContext.getContentResolver(), + featureInfo.getSettingKey(), mCurrentUserId); + // Assuming that the default state will be to have the feature off + if (!TextUtils.equals(featureInfo.getSettingOnValue(), setting.read())) { + logAccessibilityShortcutActivated(assignedTarget, shortcutType, /* serviceEnabled= */ + true); + setting.write(featureInfo.getSettingOnValue()); + } else { + logAccessibilityShortcutActivated(assignedTarget, shortcutType, /* serviceEnabled= */ + false); + setting.write(featureInfo.getSettingOffValue()); + } + return true; + } + + private boolean performAccessibilityShortcutTargetActivity(int displayId, + ComponentName assignedTarget) { + synchronized (mLock) { + final AccessibilityUserState userState = getCurrentUserStateLocked(); + for (int i = 0; i < userState.mInstalledShortcuts.size(); i++) { + final AccessibilityShortcutInfo shortcutInfo = userState.mInstalledShortcuts.get(i); + if (!shortcutInfo.getComponentName().equals(assignedTarget)) { + continue; + } + launchShortcutTargetActivity(displayId, assignedTarget); + return true; + } + } + return false; + } + + /** + * Perform accessibility service shortcut action. + * + * 1) For {@link AccessibilityManager#ACCESSIBILITY_BUTTON} type and services targeting sdk + * version <= Q: callbacks to accessibility service if service is bounded and requests + * accessibility button. + * 2) For {@link AccessibilityManager#ACCESSIBILITY_SHORTCUT_KEY} type and service targeting sdk + * version <= Q: turns on / off the accessibility service. + * 3) For {@link AccessibilityManager#ACCESSIBILITY_SHORTCUT_KEY} type and service targeting sdk + * version > Q and request accessibility button: turn on the accessibility service if it's + * not in the enabled state. + * (It'll happen when a service is disabled and assigned to shortcut then upgraded.) + * 4) For services targeting sdk version > Q: + * a) Turns on / off the accessibility service, if service does not request accessibility + * button. + * b) Callbacks to accessibility service if service is bounded and requests accessibility + * button. + */ + private boolean performAccessibilityShortcutTargetService(int displayId, + @ShortcutType int shortcutType, ComponentName assignedTarget) { + synchronized (mLock) { + final AccessibilityUserState userState = getCurrentUserStateLocked(); + final AccessibilityServiceInfo installedServiceInfo = + userState.getInstalledServiceInfoLocked(assignedTarget); + if (installedServiceInfo == null) { + Slog.d(LOG_TAG, "Perform shortcut failed, invalid component name:" + + assignedTarget); + return false; } - if (frameworkFeatureMap.containsKey(serviceName)) { - // Toggle the requested framework feature - ToggleableFrameworkFeatureInfo featureInfo = frameworkFeatureMap.get(serviceName); - SettingStringHelper setting = new SettingStringHelper(mContext.getContentResolver(), - featureInfo.getSettingKey(), mCurrentUserId); - // Assuming that the default state will be to have the feature off - if (!TextUtils.equals(featureInfo.getSettingOnValue(), setting.read())) { - setting.write(featureInfo.getSettingOnValue()); + + final AccessibilityServiceConnection serviceConnection = + userState.getServiceConnectionLocked(assignedTarget); + final int targetSdk = installedServiceInfo.getResolveInfo() + .serviceInfo.applicationInfo.targetSdkVersion; + final boolean requestA11yButton = (installedServiceInfo.flags + & AccessibilityServiceInfo.FLAG_REQUEST_ACCESSIBILITY_BUTTON) != 0; + // Turns on / off the accessibility service + if ((targetSdk <= Build.VERSION_CODES.Q && shortcutType == ACCESSIBILITY_SHORTCUT_KEY) + || (targetSdk > Build.VERSION_CODES.Q && !requestA11yButton)) { + if (serviceConnection == null) { + logAccessibilityShortcutActivated(assignedTarget, + shortcutType, /* serviceEnabled= */ true); + enableAccessibilityServiceLocked(assignedTarget, mCurrentUserId); + } else { - setting.write(featureInfo.getSettingOffValue()); + logAccessibilityShortcutActivated(assignedTarget, + shortcutType, /* serviceEnabled= */ false); + disableAccessibilityServiceLocked(assignedTarget, mCurrentUserId); } + return true; } - final long identity = Binder.clearCallingIdentity(); - try { - if (userState.mComponentNameToServiceMap.get(serviceName) == null) { - enableAccessibilityServiceLocked(serviceName, mCurrentUserId); - } else { - disableAccessibilityServiceLocked(serviceName, mCurrentUserId); + if (shortcutType == ACCESSIBILITY_SHORTCUT_KEY && targetSdk > Build.VERSION_CODES.Q + && requestA11yButton) { + if (!userState.getEnabledServicesLocked().contains(assignedTarget)) { + enableAccessibilityServiceLocked(assignedTarget, mCurrentUserId); + return true; } - } finally { - Binder.restoreCallingIdentity(identity); } + // Callbacks to a11y service if it's bounded and requests a11y button. + if (serviceConnection == null + || !userState.mBoundServices.contains(serviceConnection) + || !serviceConnection.mRequestAccessibilityButton) { + Slog.d(LOG_TAG, "Perform shortcut failed, service is not ready:" + + assignedTarget); + return false; + } + // ServiceConnection means service enabled. + logAccessibilityShortcutActivated(assignedTarget, shortcutType, /* serviceEnabled= */ + true); + serviceConnection.notifyAccessibilityButtonClickedLocked(displayId); + return true; } - }; + } @Override - public String getAccessibilityShortcutService() { - if (mContext.checkCallingPermission(Manifest.permission.MANAGE_ACCESSIBILITY) + public List<String> getAccessibilityShortcutTargets(@ShortcutType int shortcutType) { + if (mContext.checkCallingOrSelfPermission(Manifest.permission.MANAGE_ACCESSIBILITY) != PackageManager.PERMISSION_GRANTED) { throw new SecurityException( "getAccessibilityShortcutService requires the MANAGE_ACCESSIBILITY permission"); } - synchronized(mLock) { - final UserState userState = getUserStateLocked(mCurrentUserId); - return userState.mServiceToEnableWithShortcut.flattenToString(); + return getAccessibilityShortcutTargetsInternal(shortcutType); + } + + private List<String> getAccessibilityShortcutTargetsInternal(@ShortcutType int shortcutType) { + synchronized (mLock) { + final AccessibilityUserState userState = getCurrentUserStateLocked(); + final ArrayList<String> shortcutTargets = new ArrayList<>( + userState.getShortcutTargetsLocked(shortcutType)); + if (shortcutType != ACCESSIBILITY_BUTTON) { + return shortcutTargets; + } + // Adds legacy a11y services requesting a11y button into the list. + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + if (!service.mRequestAccessibilityButton + || service.getServiceInfo().getResolveInfo().serviceInfo.applicationInfo + .targetSdkVersion > Build.VERSION_CODES.Q) { + continue; + } + final String serviceName = service.getComponentName().flattenToString(); + if (!TextUtils.isEmpty(serviceName)) { + shortcutTargets.add(serviceName); + } + } + return shortcutTargets; } } @@ -2445,7 +2738,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub persistComponentNamesToSettingLocked(Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES, mTempComponentNameSet, userId); - UserState userState = getUserStateLocked(userId); + AccessibilityUserState userState = getUserStateLocked(userId); if (userState.mEnabledServices.add(componentName)) { onUserStateChangedLocked(userState); } @@ -2462,12 +2755,17 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub persistComponentNamesToSettingLocked(Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES, mTempComponentNameSet, userId); - UserState userState = getUserStateLocked(userId); + AccessibilityUserState userState = getUserStateLocked(userId); if (userState.mEnabledServices.remove(componentName)) { onUserStateChangedLocked(userState); } } + @Override + public void sendAccessibilityEventForCurrentUserLocked(AccessibilityEvent event) { + sendAccessibilityEventLocked(event, mCurrentUserId); + } + private void sendAccessibilityEventLocked(AccessibilityEvent event, int userId) { // Resync to avoid calling out with the lock held event.setEventTime(SystemClock.uptimeMillis()); @@ -2512,7 +2810,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub throw new SecurityException("Only SYSTEM can call getAccessibilityWindowId"); } - return findWindowIdLocked(windowToken); + return mA11yWindowManager.findWindowIdLocked(mCurrentUserId, windowToken); } } @@ -2525,150 +2823,65 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub @Override public long getRecommendedTimeoutMillis() { synchronized(mLock) { - final UserState userState = getCurrentUserStateLocked(); + final AccessibilityUserState userState = getCurrentUserStateLocked(); return getRecommendedTimeoutMillisLocked(userState); } } - private long getRecommendedTimeoutMillisLocked(UserState userState) { - return IntPair.of(userState.mInteractiveUiTimeout, - userState.mNonInteractiveUiTimeout); + private long getRecommendedTimeoutMillisLocked(AccessibilityUserState userState) { + return IntPair.of(userState.getInteractiveUiTimeoutLocked(), + userState.getNonInteractiveUiTimeoutLocked()); } @Override - public void dump(FileDescriptor fd, final PrintWriter pw, String[] args) { - if (!DumpUtils.checkDumpPermission(mContext, LOG_TAG, pw)) return; + public void setWindowMagnificationConnection( + IWindowMagnificationConnection connection) throws RemoteException { + mSecurityPolicy.enforceCallingOrSelfPermission( + android.Manifest.permission.STATUS_BAR_SERVICE); + + getWindowMagnificationMgr().setConnection(connection); + } + + WindowMagnificationManager getWindowMagnificationMgr() { synchronized (mLock) { - pw.println("ACCESSIBILITY MANAGER (dumpsys accessibility)"); - pw.println(); - final int userCount = mUserStates.size(); - for (int i = 0; i < userCount; i++) { - UserState userState = mUserStates.valueAt(i); - pw.append("User state[attributes:{id=" + userState.mUserId); - pw.append(", currentUser=" + (userState.mUserId == mCurrentUserId)); - pw.append(", touchExplorationEnabled=" + userState.mIsTouchExplorationEnabled); - pw.append(", displayMagnificationEnabled=" - + userState.mIsDisplayMagnificationEnabled); - pw.append(", navBarMagnificationEnabled=" - + userState.mIsNavBarMagnificationEnabled); - pw.append(", autoclickEnabled=" + userState.mIsAutoclickEnabled); - pw.append(", nonInteractiveUiTimeout=" + userState.mNonInteractiveUiTimeout); - pw.append(", interactiveUiTimeout=" + userState.mInteractiveUiTimeout); - pw.append(", installedServiceCount=" + userState.mInstalledServices.size()); - if (mUiAutomationManager.isUiAutomationRunningLocked()) { - pw.append(", "); - mUiAutomationManager.dumpUiAutomationService(fd, pw, args); - pw.println(); - } - pw.append("}"); - pw.println(); - pw.append(" Bound services:{"); - final int serviceCount = userState.mBoundServices.size(); - for (int j = 0; j < serviceCount; j++) { - if (j > 0) { - pw.append(", "); - pw.println(); - pw.append(" "); - } - AccessibilityServiceConnection service = userState.mBoundServices.get(j); - service.dump(fd, pw, args); - } - pw.println("}"); - pw.append(" Enabled services:{"); - Iterator<ComponentName> it = userState.mEnabledServices.iterator(); - if (it.hasNext()) { - ComponentName componentName = it.next(); - pw.append(componentName.toShortString()); - while (it.hasNext()) { - componentName = it.next(); - pw.append(", "); - pw.append(componentName.toShortString()); - } - } - pw.println("}"); - pw.append(" Binding services:{"); - it = userState.mBindingServices.iterator(); - if (it.hasNext()) { - ComponentName componentName = it.next(); - pw.append(componentName.toShortString()); - while (it.hasNext()) { - componentName = it.next(); - pw.append(", "); - pw.append(componentName.toShortString()); - } - } - pw.println("}]"); - pw.println(); - } - if (mSecurityPolicy.mWindows != null) { - final int windowCount = mSecurityPolicy.mWindows.size(); - for (int j = 0; j < windowCount; j++) { - if (j > 0) { - pw.append(','); - pw.println(); - } - pw.append("Window["); - AccessibilityWindowInfo window = mSecurityPolicy.mWindows.get(j); - pw.append(window.toString()); - pw.append(']'); - } - pw.println(); + if (mWindowMagnificationMgr == null) { + mWindowMagnificationMgr = new WindowMagnificationManager(); } + return mWindowMagnificationMgr; } } - private void putSecureIntForUser(String key, int value, int userid) { - final long identity = Binder.clearCallingIdentity(); - try { - Settings.Secure.putIntForUser(mContext.getContentResolver(), key, value, userid); - } finally { - Binder.restoreCallingIdentity(identity); + @Override + public void associateEmbeddedHierarchy(@NonNull IBinder host, @NonNull IBinder embedded) { + synchronized (mLock) { + mA11yWindowManager.associateEmbeddedHierarchyLocked(host, embedded); } } - class RemoteAccessibilityConnection implements DeathRecipient { - private final int mUid; - private final String mPackageName; - private final int mWindowId; - private final int mUserId; - private final IAccessibilityInteractionConnection mConnection; - - RemoteAccessibilityConnection(int windowId, - IAccessibilityInteractionConnection connection, - String packageName, int uid, int userId) { - mWindowId = windowId; - mPackageName = packageName; - mUid = uid; - mUserId = userId; - mConnection = connection; - } - - public int getUid() { - return mUid; - } - - public String getPackageName() { - return mPackageName; - } - - public IAccessibilityInteractionConnection getRemote() { - return mConnection; - } - - public void linkToDeath() throws RemoteException { - mConnection.asBinder().linkToDeath(this, 0); - } - - public void unlinkToDeath() { - mConnection.asBinder().unlinkToDeath(this, 0); + @Override + public void disassociateEmbeddedHierarchy(@NonNull IBinder token) { + synchronized (mLock) { + mA11yWindowManager.disassociateEmbeddedHierarchyLocked(token); } + } - @Override - public void binderDied() { - unlinkToDeath(); - synchronized (mLock) { - removeAccessibilityInteractionConnectionLocked(mWindowId, mUserId); + @Override + public void dump(FileDescriptor fd, final PrintWriter pw, String[] args) { + if (!DumpUtils.checkDumpPermission(mContext, LOG_TAG, pw)) return; + synchronized (mLock) { + pw.println("ACCESSIBILITY MANAGER (dumpsys accessibility)"); + pw.println(); + pw.append("currentUserId=").append(String.valueOf(mCurrentUserId)); + pw.println(); + final int userCount = mUserStates.size(); + for (int i = 0; i < userCount; i++) { + mUserStates.valueAt(i).dump(fd, pw, args); } + if (mUiAutomationManager.isUiAutomationRunningLocked()) { + mUiAutomationManager.dumpUiAutomationService(fd, pw, args); + pw.println(); + } + mA11yWindowManager.dump(fd, pw, args); } } @@ -2695,96 +2908,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - void clearAccessibilityFocus(IntSupplier windowId) { - clearAccessibilityFocus(windowId.getAsInt()); - } - - void clearAccessibilityFocus(int windowId) { - getInteractionBridge().clearAccessibilityFocusNotLocked(windowId); - } - - private IBinder findWindowTokenLocked(int windowId) { - IBinder token = mGlobalWindowTokens.get(windowId); - if (token != null) { - return token; - } - return getCurrentUserStateLocked().mWindowTokens.get(windowId); - } - - private int findWindowIdLocked(IBinder token) { - final int globalIndex = mGlobalWindowTokens.indexOfValue(token); - if (globalIndex >= 0) { - return mGlobalWindowTokens.keyAt(globalIndex); - } - UserState userState = getCurrentUserStateLocked(); - final int userIndex = userState.mWindowTokens.indexOfValue(token); - if (userIndex >= 0) { - return userState.mWindowTokens.keyAt(userIndex); - } - return -1; - } - - private void notifyOutsideTouchIfNeeded(int targetWindowId, int action) { - if (action != ACTION_CLICK && action != ACTION_LONG_CLICK) { - return; - } - - final List<Integer> outsideWindowsIds; - final List<RemoteAccessibilityConnection> connectionList = new ArrayList<>(); - synchronized (mLock) { - outsideWindowsIds = mSecurityPolicy.getWatchOutsideTouchWindowIdLocked(targetWindowId); - for (int i = 0; i < outsideWindowsIds.size(); i++) { - connectionList.add(getConnectionLocked(outsideWindowsIds.get(i))); - } - } - for (int i = 0; i < connectionList.size(); i++) { - final RemoteAccessibilityConnection connection = connectionList.get(i); - if (connection != null) { - try { - connection.getRemote().notifyOutsideTouch(); - } catch (RemoteException re) { - if (DEBUG) { - Slog.e(LOG_TAG, "Error calling notifyOutsideTouch()"); - } - } - } - } - } - - @Override - public void ensureWindowsAvailableTimed() { - synchronized (mLock) { - if (mSecurityPolicy.mWindows != null) { - return; - } - // If we have no registered callback, update the state we - // we may have to register one but it didn't happen yet. - if (mWindowsForAccessibilityCallback == null) { - UserState userState = getCurrentUserStateLocked(); - onUserStateChangedLocked(userState); - } - // We have no windows but do not care about them, done. - if (mWindowsForAccessibilityCallback == null) { - return; - } - - // Wait for the windows with a timeout. - final long startMillis = SystemClock.uptimeMillis(); - while (mSecurityPolicy.mWindows == null) { - final long elapsedMillis = SystemClock.uptimeMillis() - startMillis; - final long remainMillis = WAIT_WINDOWS_TIMEOUT_MILLIS - elapsedMillis; - if (remainMillis <= 0) { - return; - } - try { - mLock.wait(remainMillis); - } catch (InterruptedException ie) { - /* ignore */ - } - } - } - } - @Override public MagnificationController getMagnificationController() { synchronized (mLock) { @@ -2797,96 +2920,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } @Override - public boolean performAccessibilityAction(int resolvedWindowId, - long accessibilityNodeId, int action, Bundle arguments, int interactionId, - IAccessibilityInteractionConnectionCallback callback, int fetchFlags, - long interrogatingTid) { - RemoteAccessibilityConnection connection; - IBinder activityToken = null; - synchronized (mLock) { - connection = getConnectionLocked(resolvedWindowId); - if (connection == null) { - return false; - } - final boolean isA11yFocusAction = (action == ACTION_ACCESSIBILITY_FOCUS) - || (action == ACTION_CLEAR_ACCESSIBILITY_FOCUS); - final AccessibilityWindowInfo a11yWindowInfo = - mSecurityPolicy.findA11yWindowInfoById(resolvedWindowId); - if (!isA11yFocusAction) { - final WindowInfo windowInfo = - mSecurityPolicy.findWindowInfoById(resolvedWindowId); - if (windowInfo != null) activityToken = windowInfo.activityToken; - } - if ((a11yWindowInfo != null) && a11yWindowInfo.isInPictureInPictureMode() - && (mPictureInPictureActionReplacingConnection != null) && !isA11yFocusAction) { - connection = mPictureInPictureActionReplacingConnection; - } - } - final int interrogatingPid = Binder.getCallingPid(); - final long identityToken = Binder.clearCallingIdentity(); - try { - // Regardless of whether or not the action succeeds, it was generated by an - // accessibility service that is driven by user actions, so note user activity. - mPowerManager.userActivity(SystemClock.uptimeMillis(), - PowerManager.USER_ACTIVITY_EVENT_ACCESSIBILITY, 0); - - notifyOutsideTouchIfNeeded(resolvedWindowId, action); - if (activityToken != null) { - LocalServices.getService(ActivityTaskManagerInternal.class) - .setFocusedActivity(activityToken); - } - connection.mConnection.performAccessibilityAction(accessibilityNodeId, action, - arguments, interactionId, callback, fetchFlags, interrogatingPid, - interrogatingTid); - } catch (RemoteException re) { - if (DEBUG) { - Slog.e(LOG_TAG, "Error calling performAccessibilityAction: " + re); - } - return false; - } finally { - Binder.restoreCallingIdentity(identityToken); - } - return true; - } - - @Override - public RemoteAccessibilityConnection getConnectionLocked(int windowId) { - if (DEBUG) { - Slog.i(LOG_TAG, "Trying to get interaction connection to windowId: " + windowId); - } - RemoteAccessibilityConnection connection = - mGlobalInteractionConnections.get(windowId); - if (connection == null) { - connection = getCurrentUserStateLocked().mInteractionConnections.get(windowId); - } - if (connection != null && connection.mConnection != null) { - return connection; - } - if (DEBUG) { - Slog.e(LOG_TAG, "No interaction connection to window: " + windowId); - } - return null; - } - - @Override - public IAccessibilityInteractionConnectionCallback replaceCallbackIfNeeded( - IAccessibilityInteractionConnectionCallback originalCallback, - int resolvedWindowId, int interactionId, int interrogatingPid, - long interrogatingTid) { - AccessibilityWindowInfo windowInfo = - mSecurityPolicy.findA11yWindowInfoById(resolvedWindowId); - if ((windowInfo == null) || !windowInfo.isInPictureInPictureMode() - || (mPictureInPictureActionReplacingConnection == null)) { - return originalCallback; - } - return new ActionReplacingCallback(originalCallback, - mPictureInPictureActionReplacingConnection.mConnection, interactionId, - interrogatingPid, interrogatingTid); - } - - @Override public void onClientChangeLocked(boolean serviceInfoChanged) { - AccessibilityManagerService.UserState userState = getUserStateLocked(mCurrentUserId); + AccessibilityUserState userState = getUserStateLocked(mCurrentUserId); onUserStateChangedLocked(userState); if (serviceInfoChanged) { scheduleNotifyClientsOfServicesStateChangeLocked(userState); @@ -2897,118 +2932,10 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub public void onShellCommand(FileDescriptor in, FileDescriptor out, FileDescriptor err, String[] args, ShellCallback callback, ResultReceiver resultReceiver) { - new AccessibilityShellCommand(this).exec(this, in, out, err, args, + new AccessibilityShellCommand(this, mSystemActionPerformer).exec(this, in, out, err, args, callback, resultReceiver); } - final class WindowsForAccessibilityCallback implements - WindowManagerInternal.WindowsForAccessibilityCallback { - - @Override - public void onWindowsForAccessibilityChanged(List<WindowInfo> windows) { - synchronized (mLock) { - if (DEBUG) { - Slog.i(LOG_TAG, "Windows changed: " + windows); - } - - // Let the policy update the focused and active windows. - mSecurityPolicy.updateWindowsLocked(windows); - - // Someone may be waiting for the windows - advertise it. - mLock.notifyAll(); - } - } - - private AccessibilityWindowInfo populateReportedWindowLocked(WindowInfo window) { - final int windowId = findWindowIdLocked(window.token); - if (windowId < 0) { - return null; - } - - AccessibilityWindowInfo reportedWindow = AccessibilityWindowInfo.obtain(); - - reportedWindow.setId(windowId); - reportedWindow.setType(getTypeForWindowManagerWindowType(window.type)); - reportedWindow.setLayer(window.layer); - reportedWindow.setFocused(window.focused); - reportedWindow.setBoundsInScreen(window.boundsInScreen); - reportedWindow.setTitle(window.title); - reportedWindow.setAnchorId(window.accessibilityIdOfAnchor); - reportedWindow.setPictureInPicture(window.inPictureInPicture); - - final int parentId = findWindowIdLocked(window.parentToken); - if (parentId >= 0) { - reportedWindow.setParentId(parentId); - } - - if (window.childTokens != null) { - final int childCount = window.childTokens.size(); - for (int i = 0; i < childCount; i++) { - IBinder childToken = window.childTokens.get(i); - final int childId = findWindowIdLocked(childToken); - if (childId >= 0) { - reportedWindow.addChild(childId); - } - } - } - - return reportedWindow; - } - - private int getTypeForWindowManagerWindowType(int windowType) { - switch (windowType) { - case WindowManager.LayoutParams.TYPE_APPLICATION: - case WindowManager.LayoutParams.TYPE_APPLICATION_MEDIA: - case WindowManager.LayoutParams.TYPE_APPLICATION_PANEL: - case WindowManager.LayoutParams.TYPE_APPLICATION_STARTING: - case WindowManager.LayoutParams.TYPE_APPLICATION_SUB_PANEL: - case WindowManager.LayoutParams.TYPE_APPLICATION_ABOVE_SUB_PANEL: - case WindowManager.LayoutParams.TYPE_BASE_APPLICATION: - case WindowManager.LayoutParams.TYPE_DRAWN_APPLICATION: - case WindowManager.LayoutParams.TYPE_PHONE: - case WindowManager.LayoutParams.TYPE_PRIORITY_PHONE: - case WindowManager.LayoutParams.TYPE_TOAST: - case WindowManager.LayoutParams.TYPE_APPLICATION_ATTACHED_DIALOG: { - return AccessibilityWindowInfo.TYPE_APPLICATION; - } - - case WindowManager.LayoutParams.TYPE_INPUT_METHOD: - case WindowManager.LayoutParams.TYPE_INPUT_METHOD_DIALOG: { - return AccessibilityWindowInfo.TYPE_INPUT_METHOD; - } - - case WindowManager.LayoutParams.TYPE_KEYGUARD_DIALOG: - case WindowManager.LayoutParams.TYPE_NAVIGATION_BAR: - case WindowManager.LayoutParams.TYPE_NAVIGATION_BAR_PANEL: - case WindowManager.LayoutParams.TYPE_SEARCH_BAR: - case WindowManager.LayoutParams.TYPE_STATUS_BAR: - case WindowManager.LayoutParams.TYPE_STATUS_BAR_PANEL: - case WindowManager.LayoutParams.TYPE_STATUS_BAR_SUB_PANEL: - case WindowManager.LayoutParams.TYPE_VOLUME_OVERLAY: - case WindowManager.LayoutParams.TYPE_SYSTEM_ALERT: - case WindowManager.LayoutParams.TYPE_SYSTEM_DIALOG: - case WindowManager.LayoutParams.TYPE_SYSTEM_ERROR: - case WindowManager.LayoutParams.TYPE_SYSTEM_OVERLAY: - case WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY: - case WindowManager.LayoutParams.TYPE_SCREENSHOT: { - return AccessibilityWindowInfo.TYPE_SYSTEM; - } - - case WindowManager.LayoutParams.TYPE_DOCK_DIVIDER: { - return AccessibilityWindowInfo.TYPE_SPLIT_SCREEN_DIVIDER; - } - - case TYPE_ACCESSIBILITY_OVERLAY: { - return AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY; - } - - default: { - return -1; - } - } - } - } - private final class InteractionBridge { private final ComponentName COMPONENT_NAME = new ComponentName("com.android.server.accessibility", "InteractionBridge"); @@ -3022,7 +2949,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub info.setCapabilities(AccessibilityServiceInfo.CAPABILITY_CAN_RETRIEVE_WINDOW_CONTENT); info.flags |= AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS; info.flags |= AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS; - final UserState userState; + final AccessibilityUserState userState; synchronized (mLock) { userState = getCurrentUserStateLocked(); } @@ -3030,7 +2957,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub userState, mContext, COMPONENT_NAME, info, sIdCounter++, mMainHandler, mLock, mSecurityPolicy, AccessibilityManagerService.this, mWindowManagerService, - mGlobalActionPerformer, mActivityTaskManagerService) { + getSystemActionPerformer(), mA11yWindowManager, mActivityTaskManagerService) { @Override public boolean supportsFlagForNotImportantViews(AccessibilityServiceInfo info) { return true; @@ -3048,24 +2975,20 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub mDefaultDisplay = displayManager.getDisplay(Display.DEFAULT_DISPLAY); } - public void clearAccessibilityFocusNotLocked(int windowId) { - RemoteAccessibilityConnection connection; - synchronized (mLock) { - connection = getConnectionLocked(windowId); - if (connection == null) { - return; - } - } - try { - connection.getRemote().clearAccessibilityFocus(); - } catch (RemoteException re) { - if (DEBUG) { - Slog.e(LOG_TAG, "Error calling clearAccessibilityFocus()"); - } - } + /** + * Gets a point within the accessibility focused node where we can send down and up events + * to perform a click. + * + * @param outPoint The click point to populate. + * @return Whether accessibility a click point was found and set. + */ + // TODO: (multi-display) Make sure this works for multiple displays. + boolean getAccessibilityFocusClickPointInScreen(Point outPoint) { + return getInteractionBridge() + .getAccessibilityFocusClickPointInScreenNotLocked(outPoint); } - /** + /** * Perform an accessibility action on the view that currently has accessibility focus. * Has no effect if no item has accessibility focus, if the item with accessibility * focus does not expose the specified action, or if the action fails. @@ -3123,8 +3046,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private AccessibilityNodeInfo getAccessibilityFocusNotLocked() { final int focusedWindowId; synchronized (mLock) { - focusedWindowId = mSecurityPolicy.mAccessibilityFocusedWindowId; - if (focusedWindowId == SecurityPolicy.INVALID_WINDOW_ID) { + focusedWindowId = mA11yWindowManager.getFocusedWindowId( + AccessibilityNodeInfo.FOCUS_ACCESSIBILITY); + if (focusedWindowId == AccessibilityWindowInfo.UNDEFINED_WINDOW_ID) { return null; } } @@ -3138,729 +3062,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - public class SecurityPolicy { - public static final int INVALID_WINDOW_ID = -1; - - private static final int KEEP_SOURCE_EVENT_TYPES = AccessibilityEvent.TYPE_VIEW_CLICKED - | AccessibilityEvent.TYPE_VIEW_FOCUSED - | AccessibilityEvent.TYPE_VIEW_HOVER_ENTER - | AccessibilityEvent.TYPE_VIEW_HOVER_EXIT - | AccessibilityEvent.TYPE_VIEW_LONG_CLICKED - | AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED - | AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED - | AccessibilityEvent.TYPE_WINDOWS_CHANGED - | AccessibilityEvent.TYPE_VIEW_SELECTED - | AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED - | AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED - | AccessibilityEvent.TYPE_VIEW_SCROLLED - | AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED - | AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED - | AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY; - - // In Z order top to bottom - public List<AccessibilityWindowInfo> mWindows; - public SparseArray<AccessibilityWindowInfo> mA11yWindowInfoById = new SparseArray<>(); - public SparseArray<WindowInfo> mWindowInfoById = new SparseArray<>(); - - public int mActiveWindowId = INVALID_WINDOW_ID; - public int mFocusedWindowId = INVALID_WINDOW_ID; - public int mAccessibilityFocusedWindowId = INVALID_WINDOW_ID; - public long mAccessibilityFocusNodeId = AccessibilityNodeInfo.UNDEFINED_ITEM_ID; - - private boolean mTouchInteractionInProgress; - private boolean mHasWatchOutsideTouchWindow; - - private boolean canDispatchAccessibilityEventLocked(AccessibilityEvent event) { - final int eventType = event.getEventType(); - switch (eventType) { - // All events that are for changes in a global window - // state should *always* be dispatched. - case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: - case AccessibilityEvent.TYPE_NOTIFICATION_STATE_CHANGED: - case AccessibilityEvent.TYPE_ANNOUNCEMENT: - // All events generated by the user touching the - // screen should *always* be dispatched. - case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START: - case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END: - case AccessibilityEvent.TYPE_GESTURE_DETECTION_START: - case AccessibilityEvent.TYPE_GESTURE_DETECTION_END: - case AccessibilityEvent.TYPE_TOUCH_INTERACTION_START: - case AccessibilityEvent.TYPE_TOUCH_INTERACTION_END: - case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER: - case AccessibilityEvent.TYPE_VIEW_HOVER_EXIT: - // Also always dispatch the event that assist is reading context. - case AccessibilityEvent.TYPE_ASSIST_READING_CONTEXT: - // Also windows changing should always be anounced. - case AccessibilityEvent.TYPE_WINDOWS_CHANGED: { - return true; - } - // All events for changes in window content should be - // dispatched *only* if this window is one of the windows - // the accessibility layer reports which are windows - // that a sighted user can touch. - default: { - return isRetrievalAllowingWindowLocked(event.getWindowId()); - } - } - } - - private boolean isValidPackageForUid(String packageName, int uid) { - final long token = Binder.clearCallingIdentity(); - try { - return uid == mPackageManager.getPackageUidAsUser( - packageName, UserHandle.getUserId(uid)); - } catch (PackageManager.NameNotFoundException e) { - return false; - } finally { - Binder.restoreCallingIdentity(token); - } - } - - String resolveValidReportedPackageLocked(CharSequence packageName, int appId, int userId) { - // Okay to pass no package - if (packageName == null) { - return null; - } - // The system gets to pass any package - if (appId == Process.SYSTEM_UID) { - return packageName.toString(); - } - // Passing a package in your UID is fine - final String packageNameStr = packageName.toString(); - final int resolvedUid = UserHandle.getUid(userId, appId); - if (isValidPackageForUid(packageNameStr, resolvedUid)) { - return packageName.toString(); - } - // Appwidget hosts get to pass packages for widgets they host - if (mAppWidgetService != null && ArrayUtils.contains(mAppWidgetService - .getHostedWidgetPackages(resolvedUid), packageNameStr)) { - return packageName.toString(); - } - // Otherwise, set the package to the first one in the UID - final String[] packageNames = mPackageManager.getPackagesForUid(resolvedUid); - if (ArrayUtils.isEmpty(packageNames)) { - return null; - } - // Okay, the caller reported a package it does not have access to. - // Instead of crashing the caller for better backwards compatibility - // we report the first package in the UID. Since most of the time apps - // don't use shared user id, this will yield correct results and for - // the edge case of using a shared user id we may report the wrong - // package but this is fine since first, this is a cheating app and - // second there is no way to get the correct package anyway. - return packageNames[0]; - } - - /** - * Get a list of package names an app may report, including any widget packages it owns. - * - * @param targetPackage The known valid target package - * @param targetUid The uid of the target app - * @return - */ - String[] computeValidReportedPackages(String targetPackage, int targetUid) { - if (UserHandle.getAppId(targetUid) == Process.SYSTEM_UID) { - // Empty array means any package is Okay - return EmptyArray.STRING; - } - // IMPORTANT: The target package is already vetted to be in the target UID - String[] uidPackages = new String[]{targetPackage}; - // Appwidget hosts get to pass packages for widgets they host - if (mAppWidgetService != null) { - final ArraySet<String> widgetPackages = mAppWidgetService - .getHostedWidgetPackages(targetUid); - if (widgetPackages != null && !widgetPackages.isEmpty()) { - final String[] validPackages = new String[uidPackages.length - + widgetPackages.size()]; - System.arraycopy(uidPackages, 0, validPackages, 0, uidPackages.length); - final int widgetPackageCount = widgetPackages.size(); - for (int i = 0; i < widgetPackageCount; i++) { - validPackages[uidPackages.length + i] = widgetPackages.valueAt(i); - } - return validPackages; - } - } - return uidPackages; - } - - public void clearWindowsLocked() { - List<WindowInfo> windows = Collections.emptyList(); - final int activeWindowId = mActiveWindowId; - updateWindowsLocked(windows); - mActiveWindowId = activeWindowId; - mWindows = null; - } - - /** - * A callback when accessibility interaction client is removed. - */ - public void onAccessibilityClientRemovedLocked(int windowId) { - // Active window cannot update immediately, if windows callback is unregistered. - // Update active window to invalid, when its a11y interaction client is removed. - if (mWindowsForAccessibilityCallback == null && windowId >= 0 - && mActiveWindowId == windowId) { - mActiveWindowId = INVALID_WINDOW_ID; - } - } - - public void updateWindowsLocked(List<WindowInfo> windows) { - if (mWindows == null) { - mWindows = new ArrayList<>(); - } - - List<AccessibilityWindowInfo> oldWindowList = new ArrayList<>(mWindows); - SparseArray<AccessibilityWindowInfo> oldWindowsById = mA11yWindowInfoById.clone(); - - mWindows.clear(); - mA11yWindowInfoById.clear(); - - for (int i = 0; i < mWindowInfoById.size(); i++) { - mWindowInfoById.valueAt(i).recycle(); - } - mWindowInfoById.clear(); - mHasWatchOutsideTouchWindow = false; - - mFocusedWindowId = INVALID_WINDOW_ID; - if (!mTouchInteractionInProgress) { - mActiveWindowId = INVALID_WINDOW_ID; - } - - // If the active window goes away while the user is touch exploring we - // reset the active window id and wait for the next hover event from - // under the user's finger to determine which one is the new one. It - // is possible that the finger is not moving and the input system - // filters out such events. - boolean activeWindowGone = true; - - final int windowCount = windows.size(); - - // We'll clear accessibility focus if the window with focus is no longer visible to - // accessibility services - boolean shouldClearAccessibilityFocus = - mAccessibilityFocusedWindowId != INVALID_WINDOW_ID; - if (windowCount > 0) { - for (int i = 0; i < windowCount; i++) { - final WindowInfo windowInfo = windows.get(i); - final AccessibilityWindowInfo window; - if (mWindowsForAccessibilityCallback != null) { - window = mWindowsForAccessibilityCallback - .populateReportedWindowLocked(windowInfo); - } else { - window = null; - } - if (window != null) { - - // Flip layers in list to be consistent with AccessibilityService#getWindows - window.setLayer(windowCount - 1 - window.getLayer()); - - final int windowId = window.getId(); - if (window.isFocused()) { - mFocusedWindowId = windowId; - if (!mTouchInteractionInProgress) { - mActiveWindowId = windowId; - window.setActive(true); - } else if (windowId == mActiveWindowId) { - activeWindowGone = false; - } - } - if (!mHasWatchOutsideTouchWindow && windowInfo.hasFlagWatchOutsideTouch) { - mHasWatchOutsideTouchWindow = true; - } - mWindows.add(window); - mA11yWindowInfoById.put(windowId, window); - mWindowInfoById.put(windowId, WindowInfo.obtain(windowInfo)); - } - } - - if (mTouchInteractionInProgress && activeWindowGone) { - mActiveWindowId = mFocusedWindowId; - } - - // Focused window may change the active one, so set the - // active window once we decided which it is. - final int accessibilityWindowCount = mWindows.size(); - for (int i = 0; i < accessibilityWindowCount; i++) { - final AccessibilityWindowInfo window = mWindows.get(i); - if (window.getId() == mActiveWindowId) { - window.setActive(true); - } - if (window.getId() == mAccessibilityFocusedWindowId) { - window.setAccessibilityFocused(true); - shouldClearAccessibilityFocus = false; - } - } - } - - sendEventsForChangedWindowsLocked(oldWindowList, oldWindowsById); - - final int oldWindowCount = oldWindowList.size(); - for (int i = oldWindowCount - 1; i >= 0; i--) { - oldWindowList.remove(i).recycle(); - } - - if (shouldClearAccessibilityFocus) { - mMainHandler.sendMessage(obtainMessage( - AccessibilityManagerService::clearAccessibilityFocus, - AccessibilityManagerService.this, - box(mAccessibilityFocusedWindowId))); - } - } - - private void sendEventsForChangedWindowsLocked(List<AccessibilityWindowInfo> oldWindows, - SparseArray<AccessibilityWindowInfo> oldWindowsById) { - List<AccessibilityEvent> events = new ArrayList<>(); - // Send events for all removed windows - final int oldWindowsCount = oldWindows.size(); - for (int i = 0; i < oldWindowsCount; i++) { - final AccessibilityWindowInfo window = oldWindows.get(i); - if (mA11yWindowInfoById.get(window.getId()) == null) { - events.add(AccessibilityEvent.obtainWindowsChangedEvent( - window.getId(), AccessibilityEvent.WINDOWS_CHANGE_REMOVED)); - } - } - - // Look for other changes - int oldWindowIndex = 0; - final int newWindowCount = mWindows.size(); - for (int i = 0; i < newWindowCount; i++) { - final AccessibilityWindowInfo newWindow = mWindows.get(i); - final AccessibilityWindowInfo oldWindow = oldWindowsById.get(newWindow.getId()); - if (oldWindow == null) { - events.add(AccessibilityEvent.obtainWindowsChangedEvent( - newWindow.getId(), AccessibilityEvent.WINDOWS_CHANGE_ADDED)); - } else { - int changes = newWindow.differenceFrom(oldWindow); - if (changes != 0) { - events.add(AccessibilityEvent.obtainWindowsChangedEvent( - newWindow.getId(), changes)); - } - } - } - - final int numEvents = events.size(); - for (int i = 0; i < numEvents; i++) { - sendAccessibilityEventLocked(events.get(i), mCurrentUserId); - } - } - - public boolean computePartialInteractiveRegionForWindowLocked(int windowId, - Region outRegion) { - if (mWindows == null) { - return false; - } - - // Windows are ordered in z order so start from the bottom and find - // the window of interest. After that all windows that cover it should - // be subtracted from the resulting region. Note that for accessibility - // we are returning only interactive windows. - Region windowInteractiveRegion = null; - boolean windowInteractiveRegionChanged = false; - - final int windowCount = mWindows.size(); - for (int i = windowCount - 1; i >= 0; i--) { - AccessibilityWindowInfo currentWindow = mWindows.get(i); - if (windowInteractiveRegion == null) { - if (currentWindow.getId() == windowId) { - Rect currentWindowBounds = mTempRect; - currentWindow.getBoundsInScreen(currentWindowBounds); - outRegion.set(currentWindowBounds); - windowInteractiveRegion = outRegion; - continue; - } - } else if (currentWindow.getType() - != AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) { - Rect currentWindowBounds = mTempRect; - currentWindow.getBoundsInScreen(currentWindowBounds); - if (windowInteractiveRegion.op(currentWindowBounds, Region.Op.DIFFERENCE)) { - windowInteractiveRegionChanged = true; - } - } - } - - return windowInteractiveRegionChanged; - } - - public void updateEventSourceLocked(AccessibilityEvent event) { - if ((event.getEventType() & KEEP_SOURCE_EVENT_TYPES) == 0) { - event.setSource((View) null); - } - } - - public void updateActiveAndAccessibilityFocusedWindowLocked(int windowId, long nodeId, - int eventType, int eventAction) { - // The active window is either the window that has input focus or - // the window that the user is currently touching. If the user is - // touching a window that does not have input focus as soon as the - // the user stops touching that window the focused window becomes - // the active one. Here we detect the touched window and make it - // active. In updateWindowsLocked() we update the focused window - // and if the user is not touching the screen, we make the focused - // window the active one. - switch (eventType) { - case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: { - // If no service has the capability to introspect screen, - // we do not register callback in the window manager for - // window changes, so we have to ask the window manager - // what the focused window is to update the active one. - // The active window also determined events from which - // windows are delivered. - synchronized (mLock) { - if (mWindowsForAccessibilityCallback == null) { - mFocusedWindowId = getFocusedWindowId(); - if (windowId == mFocusedWindowId) { - mActiveWindowId = windowId; - } - } - } - } break; - - case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER: { - // Do not allow delayed hover events to confuse us - // which the active window is. - synchronized (mLock) { - if (mTouchInteractionInProgress && mActiveWindowId != windowId) { - setActiveWindowLocked(windowId); - } - } - } break; - - case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED: { - synchronized (mLock) { - if (mAccessibilityFocusedWindowId != windowId) { - mMainHandler.sendMessage(obtainMessage( - AccessibilityManagerService::clearAccessibilityFocus, - AccessibilityManagerService.this, - box(mAccessibilityFocusedWindowId))); - mSecurityPolicy.setAccessibilityFocusedWindowLocked(windowId); - mAccessibilityFocusNodeId = nodeId; - } - } - } break; - - case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED: { - synchronized (mLock) { - if (mAccessibilityFocusNodeId == nodeId) { - mAccessibilityFocusNodeId = AccessibilityNodeInfo.UNDEFINED_ITEM_ID; - } - // Clear the window with focus if it no longer has focus and we aren't - // just moving focus from one view to the other in the same window - if ((mAccessibilityFocusNodeId == AccessibilityNodeInfo.UNDEFINED_ITEM_ID) - && (mAccessibilityFocusedWindowId == windowId) - && (eventAction != AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS) - ) { - mAccessibilityFocusedWindowId = INVALID_WINDOW_ID; - } - } - } break; - } - } - - public void onTouchInteractionStart() { - synchronized (mLock) { - mTouchInteractionInProgress = true; - } - } - - public void onTouchInteractionEnd() { - synchronized (mLock) { - mTouchInteractionInProgress = false; - // We want to set the active window to be current immediately - // after the user has stopped touching the screen since if the - // user types with the IME he should get a feedback for the - // letter typed in the text view which is in the input focused - // window. Note that we always deliver hover accessibility events - // (they are a result of user touching the screen) so change of - // the active window before all hover accessibility events from - // the touched window are delivered is fine. - final int oldActiveWindow = mSecurityPolicy.mActiveWindowId; - setActiveWindowLocked(mFocusedWindowId); - - // If there is no service that can operate with active windows - // we keep accessibility focus behavior to constrain it only in - // the active window. Look at updateAccessibilityFocusBehaviorLocked - // for details. - if (oldActiveWindow != mSecurityPolicy.mActiveWindowId - && mAccessibilityFocusedWindowId == oldActiveWindow - && getCurrentUserStateLocked().mAccessibilityFocusOnlyInActiveWindow) { - mMainHandler.sendMessage(obtainMessage( - AccessibilityManagerService::clearAccessibilityFocus, - AccessibilityManagerService.this, box(oldActiveWindow))); - } - } - } - - private IntSupplier box(int value) { - return PooledLambda.obtainSupplier(value).recycleOnUse(); - } - - public int getActiveWindowId() { - if (mActiveWindowId == INVALID_WINDOW_ID && !mTouchInteractionInProgress) { - mActiveWindowId = getFocusedWindowId(); - } - return mActiveWindowId; - } - - private void setActiveWindowLocked(int windowId) { - if (mActiveWindowId != windowId) { - sendAccessibilityEventLocked( - AccessibilityEvent.obtainWindowsChangedEvent( - mActiveWindowId, AccessibilityEvent.WINDOWS_CHANGE_ACTIVE), - mCurrentUserId); - - mActiveWindowId = windowId; - if (mWindows != null) { - final int windowCount = mWindows.size(); - for (int i = 0; i < windowCount; i++) { - AccessibilityWindowInfo window = mWindows.get(i); - if (window.getId() == windowId) { - window.setActive(true); - sendAccessibilityEventLocked( - AccessibilityEvent.obtainWindowsChangedEvent(windowId, - AccessibilityEvent.WINDOWS_CHANGE_ACTIVE), - mCurrentUserId); - } else { - window.setActive(false); - } - } - } - } - } - - private void setAccessibilityFocusedWindowLocked(int windowId) { - if (mAccessibilityFocusedWindowId != windowId) { - sendAccessibilityEventLocked( - AccessibilityEvent.obtainWindowsChangedEvent( - mAccessibilityFocusedWindowId, - WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED), - mCurrentUserId); - - mAccessibilityFocusedWindowId = windowId; - if (mWindows != null) { - final int windowCount = mWindows.size(); - for (int i = 0; i < windowCount; i++) { - AccessibilityWindowInfo window = mWindows.get(i); - if (window.getId() == windowId) { - window.setAccessibilityFocused(true); - sendAccessibilityEventLocked( - AccessibilityEvent.obtainWindowsChangedEvent( - windowId, WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED), - mCurrentUserId); - - } else { - window.setAccessibilityFocused(false); - } - } - } - } - } - - public boolean canGetAccessibilityNodeInfoLocked( - AbstractAccessibilityServiceConnection service, int windowId) { - return canRetrieveWindowContentLocked(service) - && isRetrievalAllowingWindowLocked(windowId); - } - - public boolean canRetrieveWindowsLocked(AbstractAccessibilityServiceConnection service) { - return canRetrieveWindowContentLocked(service) && service.mRetrieveInteractiveWindows; - } - - public boolean canRetrieveWindowContentLocked(AbstractAccessibilityServiceConnection service) { - return (service.getCapabilities() - & AccessibilityServiceInfo.CAPABILITY_CAN_RETRIEVE_WINDOW_CONTENT) != 0; - } - - public boolean canControlMagnification(AbstractAccessibilityServiceConnection service) { - return (service.getCapabilities() - & AccessibilityServiceInfo.CAPABILITY_CAN_CONTROL_MAGNIFICATION) != 0; - } - - public boolean canPerformGestures(AccessibilityServiceConnection service) { - return (service.getCapabilities() - & AccessibilityServiceInfo.CAPABILITY_CAN_PERFORM_GESTURES) != 0; - } - - public boolean canCaptureFingerprintGestures(AccessibilityServiceConnection service) { - return (service.getCapabilities() - & AccessibilityServiceInfo.CAPABILITY_CAN_REQUEST_FINGERPRINT_GESTURES) != 0; - } - - private int resolveProfileParentLocked(int userId) { - if (userId != mCurrentUserId) { - final long identity = Binder.clearCallingIdentity(); - try { - UserInfo parent = mUserManager.getProfileParent(userId); - if (parent != null) { - return parent.getUserHandle().getIdentifier(); - } - } finally { - Binder.restoreCallingIdentity(identity); - } - } - return userId; - } - - public int resolveCallingUserIdEnforcingPermissionsLocked(int userId) { - final int callingUid = Binder.getCallingUid(); - if (callingUid == 0 - || callingUid == Process.SYSTEM_UID - || callingUid == Process.SHELL_UID) { - if (userId == UserHandle.USER_CURRENT - || userId == UserHandle.USER_CURRENT_OR_SELF) { - return mCurrentUserId; - } - return resolveProfileParentLocked(userId); - } - final int callingUserId = UserHandle.getUserId(callingUid); - if (callingUserId == userId) { - return resolveProfileParentLocked(userId); - } - final int callingUserParentId = resolveProfileParentLocked(callingUserId); - if (callingUserParentId == mCurrentUserId && - (userId == UserHandle.USER_CURRENT - || userId == UserHandle.USER_CURRENT_OR_SELF)) { - return mCurrentUserId; - } - if (!hasPermission(Manifest.permission.INTERACT_ACROSS_USERS) - && !hasPermission(Manifest.permission.INTERACT_ACROSS_USERS_FULL)) { - throw new SecurityException("Call from user " + callingUserId + " as user " - + userId + " without permission INTERACT_ACROSS_USERS or " - + "INTERACT_ACROSS_USERS_FULL not allowed."); - } - if (userId == UserHandle.USER_CURRENT - || userId == UserHandle.USER_CURRENT_OR_SELF) { - return mCurrentUserId; - } - throw new IllegalArgumentException("Calling user can be changed to only " - + "UserHandle.USER_CURRENT or UserHandle.USER_CURRENT_OR_SELF."); - } - - public boolean isCallerInteractingAcrossUsers(int userId) { - final int callingUid = Binder.getCallingUid(); - return (Binder.getCallingPid() == android.os.Process.myPid() - || callingUid == Process.SHELL_UID - || userId == UserHandle.USER_CURRENT - || userId == UserHandle.USER_CURRENT_OR_SELF); - } - - private boolean isRetrievalAllowingWindowLocked(int windowId) { - // The system gets to interact with any window it wants. - if (Binder.getCallingUid() == Process.SYSTEM_UID) { - return true; - } - if (Binder.getCallingUid() == Process.SHELL_UID) { - if (!isShellAllowedToRetrieveWindowLocked(windowId)) { - return false; - } - } - if (windowId == mActiveWindowId) { - return true; - } - return findA11yWindowInfoById(windowId) != null; - } - - private boolean isShellAllowedToRetrieveWindowLocked(int windowId) { - long token = Binder.clearCallingIdentity(); - try { - IBinder windowToken = findWindowTokenLocked(windowId); - if (windowToken == null) { - return false; - } - int userId = mWindowManagerService.getWindowOwnerUserId(windowToken); - if (userId == UserHandle.USER_NULL) { - return false; - } - return !mUserManager.hasUserRestriction( - UserManager.DISALLOW_DEBUGGING_FEATURES, UserHandle.of(userId)); - } finally { - Binder.restoreCallingIdentity(token); - } - } - - public AccessibilityWindowInfo findA11yWindowInfoById(int windowId) { - return mA11yWindowInfoById.get(windowId); - } - - private WindowInfo findWindowInfoById(int windowId) { - return mWindowInfoById.get(windowId); - } - - private List<Integer> getWatchOutsideTouchWindowIdLocked(int targetWindowId) { - final WindowInfo targetWindow = mWindowInfoById.get(targetWindowId); - if (targetWindow != null && mHasWatchOutsideTouchWindow) { - final List<Integer> outsideWindowsId = new ArrayList<>(); - for (int i = 0; i < mWindowInfoById.size(); i++) { - WindowInfo window = mWindowInfoById.valueAt(i); - if (window != null && window.layer < targetWindow.layer - && window.hasFlagWatchOutsideTouch) { - outsideWindowsId.add(mWindowInfoById.keyAt(i)); - } - } - return outsideWindowsId; - } - return Collections.emptyList(); - } - - private AccessibilityWindowInfo getPictureInPictureWindow() { - if (mWindows != null) { - final int windowCount = mWindows.size(); - for (int i = 0; i < windowCount; i++) { - AccessibilityWindowInfo window = mWindows.get(i); - if (window.isInPictureInPictureMode()) { - return window; - } - } - } - return null; - } - - private void enforceCallingPermission(String permission, String function) { - if (OWN_PROCESS_ID == Binder.getCallingPid()) { - return; - } - if (!hasPermission(permission)) { - throw new SecurityException("You do not have " + permission - + " required to call " + function + " from pid=" - + Binder.getCallingPid() + ", uid=" + Binder.getCallingUid()); - } - } - - private boolean hasPermission(String permission) { - return mContext.checkCallingPermission(permission) == PackageManager.PERMISSION_GRANTED; - } - - private int getFocusedWindowId() { - IBinder token = mWindowManagerService.getFocusedWindowToken(); - synchronized (mLock) { - return findWindowIdLocked(token); - } - } - - public boolean checkAccessibilityAccess(AbstractAccessibilityServiceConnection service) { - final String packageName = service.getComponentName().getPackageName(); - final ResolveInfo resolveInfo = service.getServiceInfo().getResolveInfo(); - - if (resolveInfo == null) { - // For InteractionBridge and UiAutomation - return true; - } - - final int uid = resolveInfo.serviceInfo.applicationInfo.uid; - final long identityToken = Binder.clearCallingIdentity(); - try { - // For the caller is system, just block the data to a11y services. - if (OWN_PROCESS_ID == Binder.getCallingPid()) { - return mAppOpsManager.noteOpNoThrow(AppOpsManager.OPSTR_ACCESS_ACCESSIBILITY, - uid, packageName) == AppOpsManager.MODE_ALLOWED; - } - - return mAppOpsManager.noteOp(AppOpsManager.OPSTR_ACCESS_ACCESSIBILITY, - uid, packageName) == AppOpsManager.MODE_ALLOWED; - } finally { - Binder.restoreCallingIdentity(identityToken); - } - } - } - /** * Gets all currently valid logical displays. * @@ -3876,11 +3077,19 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub public class AccessibilityDisplayListener implements DisplayManager.DisplayListener { private final DisplayManager mDisplayManager; private final ArrayList<Display> mDisplaysList = new ArrayList<>(); + private int mSystemUiUid = 0; AccessibilityDisplayListener(Context context, MainHandler handler) { mDisplayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); mDisplayManager.registerDisplayListener(this, handler); initializeDisplayList(); + + final PackageManagerInternal pm = + LocalServices.getService(PackageManagerInternal.class); + if (pm != null) { + mSystemUiUid = pm.getPackageUid(pm.getSystemUiServiceComponent().getPackageName(), + PackageManager.MATCH_SYSTEM_ONLY, mCurrentUserId); + } } ArrayList<Display> getValidDisplayList() { @@ -3898,10 +3107,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub // to create event handler per display. The events should be handled by the // display which is overlaid by it. final Display display = displays[i]; - if (display.getType() == Display.TYPE_OVERLAY) { - continue; + if (isValidDisplay(display)) { + mDisplaysList.add(display); } - mDisplaysList.add(display); } } } @@ -3909,7 +3117,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub @Override public void onDisplayAdded(int displayId) { final Display display = mDisplayManager.getDisplay(displayId); - if (display == null || display.getType() == Display.TYPE_OVERLAY) { + if (!isValidDisplay(display)) { return; } @@ -3918,33 +3126,76 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (mInputFilter != null) { mInputFilter.onDisplayChanged(); } - UserState userState = getCurrentUserStateLocked(); + AccessibilityUserState userState = getCurrentUserStateLocked(); + if (displayId != Display.DEFAULT_DISPLAY) { + final List<AccessibilityServiceConnection> services = userState.mBoundServices; + for (int i = 0; i < services.size(); i++) { + AccessibilityServiceConnection boundClient = services.get(i); + boundClient.onDisplayAdded(displayId); + } + } updateMagnificationLocked(userState); + updateWindowsForAccessibilityCallbackLocked(userState); } } @Override public void onDisplayRemoved(int displayId) { synchronized (mLock) { - for (int i = 0; i < mDisplaysList.size(); i++) { - if (mDisplaysList.get(i).getDisplayId() == displayId) { - mDisplaysList.remove(i); - break; - } + if (!removeDisplayFromList(displayId)) { + return; } if (mInputFilter != null) { mInputFilter.onDisplayChanged(); } + AccessibilityUserState userState = getCurrentUserStateLocked(); + if (displayId != Display.DEFAULT_DISPLAY) { + final List<AccessibilityServiceConnection> services = userState.mBoundServices; + for (int i = 0; i < services.size(); i++) { + AccessibilityServiceConnection boundClient = services.get(i); + boundClient.onDisplayRemoved(displayId); + } + } } if (mMagnificationController != null) { mMagnificationController.onDisplayRemoved(displayId); } + mA11yWindowManager.stopTrackingWindows(displayId); + } + + @GuardedBy("mLock") + private boolean removeDisplayFromList(int displayId) { + for (int i = 0; i < mDisplaysList.size(); i++) { + if (mDisplaysList.get(i).getDisplayId() == displayId) { + mDisplaysList.remove(i); + return true; + } + } + return false; } @Override public void onDisplayChanged(int displayId) { /* do nothing */ } + + private boolean isValidDisplay(@Nullable Display display) { + if (display == null || display.getType() == Display.TYPE_OVERLAY) { + return false; + } + // Private virtual displays are created by the ap and is not allowed to access by other + // aps. We assume we could ignore them. + // The exceptional case is for bubbles. Because the bubbles use the activityView, and + // the virtual display of the activityView is private, so if the owner UID of the + // private virtual display is the one of system ui which creates the virtual display of + // bubbles, then this private virtual display should track the windows. + if (display.getType() == Display.TYPE_VIRTUAL + && (display.getFlags() & Display.FLAG_PRIVATE) != 0 + && display.getOwnerUid() != mSystemUiUid) { + return false; + } + return true; + } } /** Represents an {@link AccessibilityManager} */ @@ -3953,7 +3204,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub final String[] mPackageNames; int mLastSentRelevantEventTypes; - private Client(IAccessibilityManagerClient callback, int clientUid, UserState userState) { + private Client(IAccessibilityManagerClient callback, int clientUid, + AccessibilityUserState userState) { mCallback = callback; mPackageNames = mPackageManager.getPackagesForUid(clientUid); synchronized (mLock) { @@ -3962,320 +3214,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } - public class UserState { - public final int mUserId; - - // Non-transient state. - - public final RemoteCallbackList<IAccessibilityManagerClient> mUserClients = - new RemoteCallbackList<>(); - - public final SparseArray<RemoteAccessibilityConnection> mInteractionConnections = - new SparseArray<>(); - - public final SparseArray<IBinder> mWindowTokens = new SparseArray<>(); - - // Transient state. - - public final ArrayList<AccessibilityServiceConnection> mBoundServices = new ArrayList<>(); - - public final Map<ComponentName, AccessibilityServiceConnection> mComponentNameToServiceMap = - new HashMap<>(); - - public final List<AccessibilityServiceInfo> mInstalledServices = - new ArrayList<>(); - - private final Set<ComponentName> mBindingServices = new HashSet<>(); - - public final Set<ComponentName> mEnabledServices = new HashSet<>(); - - public final Set<ComponentName> mTouchExplorationGrantedServices = - new HashSet<>(); - - public ComponentName mServiceChangingSoftKeyboardMode; - - public ComponentName mServiceToEnableWithShortcut; - - public int mLastSentClientState = -1; - public int mNonInteractiveUiTimeout = 0; - public int mInteractiveUiTimeout = 0; - - private int mSoftKeyboardShowMode = 0; - - public boolean mIsNavBarMagnificationAssignedToAccessibilityButton; - public ComponentName mServiceAssignedToAccessibilityButton; - - public boolean mIsTouchExplorationEnabled; - public boolean mIsTextHighContrastEnabled; - public boolean mIsDisplayMagnificationEnabled; - public boolean mIsNavBarMagnificationEnabled; - public boolean mIsAutoclickEnabled; - public boolean mIsPerformGesturesEnabled; - public boolean mIsFilterKeyEventsEnabled; - public boolean mAccessibilityFocusOnlyInActiveWindow; - public int mUserNonInteractiveUiTimeout; - public int mUserInteractiveUiTimeout; - - private boolean mBindInstantServiceAllowed; - - public UserState(int userId) { - mUserId = userId; - } - - public int getClientState() { - int clientState = 0; - final boolean a11yEnabled = (mUiAutomationManager.isUiAutomationRunningLocked() - || isHandlingAccessibilityEvents()); - if (a11yEnabled) { - clientState |= AccessibilityManager.STATE_FLAG_ACCESSIBILITY_ENABLED; - } - // Touch exploration relies on enabled accessibility. - if (a11yEnabled && mIsTouchExplorationEnabled) { - clientState |= AccessibilityManager.STATE_FLAG_TOUCH_EXPLORATION_ENABLED; - } - if (mIsTextHighContrastEnabled) { - clientState |= AccessibilityManager.STATE_FLAG_HIGH_TEXT_CONTRAST_ENABLED; - } - return clientState; - } - - public boolean isHandlingAccessibilityEvents() { - return !mBoundServices.isEmpty() || !mBindingServices.isEmpty(); - } - - public void onSwitchToAnotherUserLocked() { - // Unbind all services. - unbindAllServicesLocked(this); - - // Clear service management state. - mBoundServices.clear(); - mBindingServices.clear(); - - // Clear event management state. - mLastSentClientState = -1; - - // clear UI timeout - mNonInteractiveUiTimeout = 0; - mInteractiveUiTimeout = 0; - - // Clear state persisted in settings. - mEnabledServices.clear(); - mTouchExplorationGrantedServices.clear(); - mIsTouchExplorationEnabled = false; - mIsDisplayMagnificationEnabled = false; - mIsNavBarMagnificationEnabled = false; - mServiceAssignedToAccessibilityButton = null; - mIsNavBarMagnificationAssignedToAccessibilityButton = false; - mIsAutoclickEnabled = false; - mUserNonInteractiveUiTimeout = 0; - mUserInteractiveUiTimeout = 0; - } - - public void addServiceLocked(AccessibilityServiceConnection serviceConnection) { - if (!mBoundServices.contains(serviceConnection)) { - serviceConnection.onAdded(); - mBoundServices.add(serviceConnection); - mComponentNameToServiceMap.put(serviceConnection.mComponentName, serviceConnection); - scheduleNotifyClientsOfServicesStateChangeLocked(this); - } - } - - /** - * Removes a service. - * There are three states to a service here: off, bound, and binding. - * This stops tracking the service as bound. - * - * @param serviceConnection The service. - */ - public void removeServiceLocked(AccessibilityServiceConnection serviceConnection) { - mBoundServices.remove(serviceConnection); - serviceConnection.onRemoved(); - if ((mServiceChangingSoftKeyboardMode != null) - && (mServiceChangingSoftKeyboardMode.equals( - serviceConnection.getServiceInfo().getComponentName()))) { - setSoftKeyboardModeLocked(SHOW_MODE_AUTO, null); - } - // It may be possible to bind a service twice, which confuses the map. Rebuild the map - // to make sure we can still reach a service - mComponentNameToServiceMap.clear(); - for (int i = 0; i < mBoundServices.size(); i++) { - AccessibilityServiceConnection boundClient = mBoundServices.get(i); - mComponentNameToServiceMap.put(boundClient.mComponentName, boundClient); - } - scheduleNotifyClientsOfServicesStateChangeLocked(this); - } - - /** - * Make sure a services disconnected but still 'on' state is reflected in UserState - * There are three states to a service here: off, bound, and binding. - * This drops a service from a bound state, to the binding state. - * The binding state describes the situation where a service is on, but not bound. - * - * @param serviceConnection The service. - */ - public void serviceDisconnectedLocked(AccessibilityServiceConnection serviceConnection) { - removeServiceLocked(serviceConnection); - mBindingServices.add(serviceConnection.getComponentName()); - } - - public Set<ComponentName> getBindingServicesLocked() { - return mBindingServices; - } - - /** - * Returns enabled service list. - */ - public Set<ComponentName> getEnabledServicesLocked() { - return mEnabledServices; - } - - public int getSoftKeyboardShowMode() { - return mSoftKeyboardShowMode; - } - - /** - * Set the soft keyboard mode. This mode is a bit odd, as it spans multiple settings. - * The ACCESSIBILITY_SOFT_KEYBOARD_MODE setting can be checked by the rest of the system - * to see if it should suppress showing the IME. The SHOW_IME_WITH_HARD_KEYBOARD setting - * setting can be changed by the user, and prevents the system from suppressing the soft - * keyboard when the hard keyboard is connected. The hard keyboard setting needs to defer - * to the user's preference, if they have supplied one. - * - * @param newMode The new mode - * @param requester The service requesting the change, so we can undo it when the - * service stops. Set to null if something other than a service is forcing - * the change. - * - * @return Whether or not the soft keyboard mode equals the new mode after the call - */ - public boolean setSoftKeyboardModeLocked(int newMode, @Nullable ComponentName requester) { - if ((newMode != SHOW_MODE_AUTO) && (newMode != SHOW_MODE_HIDDEN) - && (newMode != SHOW_MODE_IGNORE_HARD_KEYBOARD)) - { - Slog.w(LOG_TAG, "Invalid soft keyboard mode"); - return false; - } - if (mSoftKeyboardShowMode == newMode) return true; - - if (newMode == SHOW_MODE_IGNORE_HARD_KEYBOARD) { - if (hasUserOverriddenHardKeyboardSettingLocked()) { - // The user has specified a default for this setting - return false; - } - // Save the original value. But don't do this if the value in settings is already - // the new mode. That happens when we start up after a reboot, and we don't want - // to overwrite the value we had from when we first started controlling the setting. - if (getSoftKeyboardValueFromSettings() != SHOW_MODE_IGNORE_HARD_KEYBOARD) { - setOriginalHardKeyboardValue( - Settings.Secure.getInt(mContext.getContentResolver(), - Settings.Secure.SHOW_IME_WITH_HARD_KEYBOARD, 0) != 0); - } - putSecureIntForUser(Settings.Secure.SHOW_IME_WITH_HARD_KEYBOARD, 1, mUserId); - } else if (mSoftKeyboardShowMode == SHOW_MODE_IGNORE_HARD_KEYBOARD) { - putSecureIntForUser(Settings.Secure.SHOW_IME_WITH_HARD_KEYBOARD, - getOriginalHardKeyboardValue() ? 1 : 0, mUserId); - } - - saveSoftKeyboardValueToSettings(newMode); - mSoftKeyboardShowMode = newMode; - mServiceChangingSoftKeyboardMode = requester; - notifySoftKeyboardShowModeChangedLocked(mSoftKeyboardShowMode); - return true; - } - - /** - * If the settings are inconsistent with the internal state, make the internal state - * match the settings. - */ - public void reconcileSoftKeyboardModeWithSettingsLocked() { - final ContentResolver cr = mContext.getContentResolver(); - final boolean showWithHardKeyboardSettings = - Settings.Secure.getInt(cr, Settings.Secure.SHOW_IME_WITH_HARD_KEYBOARD, 0) != 0; - if (mSoftKeyboardShowMode == SHOW_MODE_IGNORE_HARD_KEYBOARD) { - if (!showWithHardKeyboardSettings) { - // The user has overridden the setting. Respect that and prevent further changes - // to this behavior. - setSoftKeyboardModeLocked(SHOW_MODE_AUTO, null); - setUserOverridesHardKeyboardSettingLocked(); - } - } - - // If the setting and the internal state are out of sync, set both to default - if (getSoftKeyboardValueFromSettings() != mSoftKeyboardShowMode) - { - Slog.e(LOG_TAG, - "Show IME setting inconsistent with internal state. Overwriting"); - setSoftKeyboardModeLocked(SHOW_MODE_AUTO, null); - putSecureIntForUser(Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, - SHOW_MODE_AUTO, mUserId); - } - } - - private void setUserOverridesHardKeyboardSettingLocked() { - final int softKeyboardSetting = Settings.Secure.getInt(mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, 0); - putSecureIntForUser(Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, - softKeyboardSetting | SHOW_MODE_HARD_KEYBOARD_OVERRIDDEN, - mUserId); - } - - private boolean hasUserOverriddenHardKeyboardSettingLocked() { - final int softKeyboardSetting = Settings.Secure.getInt(mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, 0); - return (softKeyboardSetting & SHOW_MODE_HARD_KEYBOARD_OVERRIDDEN) - != 0; - } - - private void setOriginalHardKeyboardValue(boolean originalHardKeyboardValue) { - final int oldSoftKeyboardSetting = Settings.Secure.getInt(mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, 0); - final int newSoftKeyboardSetting = oldSoftKeyboardSetting - & (~SHOW_MODE_HARD_KEYBOARD_ORIGINAL_VALUE) - | ((originalHardKeyboardValue) ? SHOW_MODE_HARD_KEYBOARD_ORIGINAL_VALUE : 0); - putSecureIntForUser(Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, - newSoftKeyboardSetting, mUserId); - } - - private void saveSoftKeyboardValueToSettings(int softKeyboardShowMode) { - final int oldSoftKeyboardSetting = Settings.Secure.getInt(mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, 0); - final int newSoftKeyboardSetting = oldSoftKeyboardSetting & (~SHOW_MODE_MASK) - | softKeyboardShowMode; - putSecureIntForUser(Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, - newSoftKeyboardSetting, mUserId); - } - - private int getSoftKeyboardValueFromSettings() { - return Settings.Secure.getInt(mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, - SHOW_MODE_AUTO) & SHOW_MODE_MASK; - } - - private boolean getOriginalHardKeyboardValue() { - return (Settings.Secure.getInt(mContext.getContentResolver(), - Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, 0) - & SHOW_MODE_HARD_KEYBOARD_ORIGINAL_VALUE) != 0; - } - - public boolean getBindInstantServiceAllowed() { - synchronized (mLock) { - return mBindInstantServiceAllowed; - } - } - - public void setBindInstantServiceAllowed(boolean allowed) { - synchronized (mLock) { - mContext.enforceCallingOrSelfPermission( - Manifest.permission.MANAGE_BIND_INSTANT_SERVICE, - "setBindInstantServiceAllowed"); - if (allowed) { - mBindInstantServiceAllowed = allowed; - onUserStateChangedLocked(this); - } - } - } - } - private final class AccessibilityContentObserver extends ContentObserver { private final Uri mTouchExplorationEnabledUri = Settings.Secure.getUriFor( @@ -4284,9 +3222,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private final Uri mDisplayMagnificationEnabledUri = Settings.Secure.getUriFor( Settings.Secure.ACCESSIBILITY_DISPLAY_MAGNIFICATION_ENABLED); - private final Uri mNavBarMagnificationEnabledUri = Settings.Secure.getUriFor( - Settings.Secure.ACCESSIBILITY_DISPLAY_MAGNIFICATION_NAVBAR_ENABLED); - private final Uri mAutoclickEnabledUri = Settings.Secure.getUriFor( Settings.Secure.ACCESSIBILITY_AUTOCLICK_ENABLED); @@ -4311,6 +3246,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private final Uri mAccessibilityButtonComponentIdUri = Settings.Secure.getUriFor( Settings.Secure.ACCESSIBILITY_BUTTON_TARGET_COMPONENT); + private final Uri mAccessibilityButtonTargetsUri = Settings.Secure.getUriFor( + Settings.Secure.ACCESSIBILITY_BUTTON_TARGETS); + private final Uri mUserNonInteractiveUiTimeoutUri = Settings.Secure.getUriFor( Settings.Secure.ACCESSIBILITY_NON_INTERACTIVE_UI_TIMEOUT_MS); @@ -4326,8 +3264,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub false, this, UserHandle.USER_ALL); contentResolver.registerContentObserver(mDisplayMagnificationEnabledUri, false, this, UserHandle.USER_ALL); - contentResolver.registerContentObserver(mNavBarMagnificationEnabledUri, - false, this, UserHandle.USER_ALL); contentResolver.registerContentObserver(mAutoclickEnabledUri, false, this, UserHandle.USER_ALL); contentResolver.registerContentObserver(mEnabledAccessibilityServicesUri, @@ -4346,6 +3282,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub contentResolver.registerContentObserver( mAccessibilityButtonComponentIdUri, false, this, UserHandle.USER_ALL); contentResolver.registerContentObserver( + mAccessibilityButtonTargetsUri, false, this, UserHandle.USER_ALL); + contentResolver.registerContentObserver( mUserNonInteractiveUiTimeoutUri, false, this, UserHandle.USER_ALL); contentResolver.registerContentObserver( mUserInteractiveUiTimeoutUri, false, this, UserHandle.USER_ALL); @@ -4356,14 +3294,13 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub synchronized (mLock) { // Profiles share the accessibility state of the parent. Therefore, // we are checking for changes only the parent settings. - UserState userState = getCurrentUserStateLocked(); + AccessibilityUserState userState = getCurrentUserStateLocked(); if (mTouchExplorationEnabledUri.equals(uri)) { if (readTouchExplorationEnabledSettingLocked(userState)) { onUserStateChangedLocked(userState); } - } else if (mDisplayMagnificationEnabledUri.equals(uri) - || mNavBarMagnificationEnabledUri.equals(uri)) { + } else if (mDisplayMagnificationEnabledUri.equals(uri)) { if (readMagnificationEnabledSettingsLocked(userState)) { onUserStateChangedLocked(userState); } @@ -4373,6 +3310,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } else if (mEnabledAccessibilityServicesUri.equals(uri)) { if (readEnabledAccessibilityServicesLocked(userState)) { + userState.updateCrashedServicesIfNeededLocked(); onUserStateChangedLocked(userState); } } else if (mTouchExplorationGrantedAccessibilityServicesUri.equals(uri)) { @@ -4387,11 +3325,15 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub || mShowImeWithHardKeyboardUri.equals(uri)) { userState.reconcileSoftKeyboardModeWithSettingsLocked(); } else if (mAccessibilityShortcutServiceIdUri.equals(uri)) { - if (readAccessibilityShortcutSettingLocked(userState)) { + if (readAccessibilityShortcutKeySettingLocked(userState)) { onUserStateChangedLocked(userState); } } else if (mAccessibilityButtonComponentIdUri.equals(uri)) { - if (readAccessibilityButtonSettingsLocked(userState)) { + if (readAccessibilityButtonTargetComponentLocked(userState)) { + onUserStateChangedLocked(userState); + } + } else if (mAccessibilityButtonTargetsUri.equals(uri)) { + if (readAccessibilityButtonTargetsLocked(userState)) { onUserStateChangedLocked(userState); } } else if (mUserNonInteractiveUiTimeoutUri.equals(uri) @@ -4401,4 +3343,40 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } } + + @Override + public void setGestureDetectionPassthroughRegion(int displayId, Region region) { + mMainHandler.sendMessage( + obtainMessage( + AccessibilityManagerService::setGestureDetectionPassthroughRegionInternal, + this, + displayId, + region)); + } + + @Override + public void setTouchExplorationPassthroughRegion(int displayId, Region region) { + mMainHandler.sendMessage( + obtainMessage( + AccessibilityManagerService::setTouchExplorationPassthroughRegionInternal, + this, + displayId, + region)); + } + + private void setTouchExplorationPassthroughRegionInternal(int displayId, Region region) { + synchronized (mLock) { + if (mHasInputFilter && mInputFilter != null) { + mInputFilter.setTouchExplorationPassthroughRegion(displayId, region); + } + } + } + + private void setGestureDetectionPassthroughRegionInternal(int displayId, Region region) { + synchronized (mLock) { + if (mHasInputFilter && mInputFilter != null) { + mInputFilter.setGestureDetectionPassthroughRegion(displayId, region); + } + } + } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilitySecurityPolicy.java b/services/accessibility/java/com/android/server/accessibility/AccessibilitySecurityPolicy.java new file mode 100644 index 000000000000..41f32075fb77 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilitySecurityPolicy.java @@ -0,0 +1,584 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility; + +import android.Manifest; +import android.accessibilityservice.AccessibilityServiceInfo; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.app.AppOpsManager; +import android.appwidget.AppWidgetManagerInternal; +import android.content.ComponentName; +import android.content.Context; +import android.content.pm.PackageManager; +import android.content.pm.ResolveInfo; +import android.content.pm.ServiceInfo; +import android.content.pm.UserInfo; +import android.os.Binder; +import android.os.IBinder; +import android.os.Process; +import android.os.UserHandle; +import android.os.UserManager; +import android.util.ArraySet; +import android.util.Slog; +import android.view.accessibility.AccessibilityEvent; + +import com.android.internal.util.ArrayUtils; +import com.android.server.LocalServices; +import com.android.server.wm.ActivityTaskManagerInternal; + +import libcore.util.EmptyArray; + +/** + * This class provides APIs of accessibility security policies for accessibility manager + * to grant accessibility capabilities or events access right to accessibility service. + */ +public class AccessibilitySecurityPolicy { + private static final int OWN_PROCESS_ID = android.os.Process.myPid(); + private static final String LOG_TAG = "AccessibilitySecurityPolicy"; + + private final Context mContext; + private final PackageManager mPackageManager; + private final UserManager mUserManager; + private final AppOpsManager mAppOpsManager; + + private AppWidgetManagerInternal mAppWidgetService; + + private static final int KEEP_SOURCE_EVENT_TYPES = AccessibilityEvent.TYPE_VIEW_CLICKED + | AccessibilityEvent.TYPE_VIEW_FOCUSED + | AccessibilityEvent.TYPE_VIEW_HOVER_ENTER + | AccessibilityEvent.TYPE_VIEW_HOVER_EXIT + | AccessibilityEvent.TYPE_VIEW_LONG_CLICKED + | AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED + | AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED + | AccessibilityEvent.TYPE_WINDOWS_CHANGED + | AccessibilityEvent.TYPE_VIEW_SELECTED + | AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED + | AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED + | AccessibilityEvent.TYPE_VIEW_SCROLLED + | AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED + | AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED + | AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY; + + /** + * Methods that should find their way into separate modules, but are still in AMS + * TODO (b/111889696): Refactoring UserState to AccessibilityUserManager. + */ + public interface AccessibilityUserManager { + /** + * Returns current userId maintained in accessibility manager service + */ + int getCurrentUserIdLocked(); + // TODO: Should include resolveProfileParentLocked, but that was already in SecurityPolicy + } + + private final AccessibilityUserManager mAccessibilityUserManager; + private AccessibilityWindowManager mAccessibilityWindowManager; + private final ActivityTaskManagerInternal mAtmInternal; + + /** + * Constructor for AccessibilityManagerService. + */ + public AccessibilitySecurityPolicy(@NonNull Context context, + @NonNull AccessibilityUserManager a11yUserManager) { + mContext = context; + mAccessibilityUserManager = a11yUserManager; + mPackageManager = mContext.getPackageManager(); + mUserManager = (UserManager) mContext.getSystemService(Context.USER_SERVICE); + mAppOpsManager = (AppOpsManager) context.getSystemService(Context.APP_OPS_SERVICE); + mAtmInternal = LocalServices.getService(ActivityTaskManagerInternal.class); + } + + /** + * Setup AccessibilityWindowManager. This isn't part of the constructor because the + * window manager and security policy both call each other. + */ + public void setAccessibilityWindowManager(@NonNull AccessibilityWindowManager awm) { + mAccessibilityWindowManager = awm; + } + + /** + * Setup AppWidgetManger during boot phase. + */ + public void setAppWidgetManager(@NonNull AppWidgetManagerInternal appWidgetManager) { + mAppWidgetService = appWidgetManager; + } + + /** + * Check if an accessibility event can be dispatched. Events should be dispatched only if they + * are dispatched from items that services can see. + * + * @param userId The userId to check + * @param event The event to check + * @return {@code true} if the event can be dispatched + */ + public boolean canDispatchAccessibilityEventLocked(int userId, + @NonNull AccessibilityEvent event) { + final int eventType = event.getEventType(); + switch (eventType) { + // All events that are for changes in a global window + // state should *always* be dispatched. + case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: + case AccessibilityEvent.TYPE_NOTIFICATION_STATE_CHANGED: + case AccessibilityEvent.TYPE_ANNOUNCEMENT: + // All events generated by the user touching the + // screen should *always* be dispatched. + case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START: + case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END: + case AccessibilityEvent.TYPE_GESTURE_DETECTION_START: + case AccessibilityEvent.TYPE_GESTURE_DETECTION_END: + case AccessibilityEvent.TYPE_TOUCH_INTERACTION_START: + case AccessibilityEvent.TYPE_TOUCH_INTERACTION_END: + case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER: + case AccessibilityEvent.TYPE_VIEW_HOVER_EXIT: + // Also always dispatch the event that assist is reading context. + case AccessibilityEvent.TYPE_ASSIST_READING_CONTEXT: + // Also windows changing should always be dispatched. + case AccessibilityEvent.TYPE_WINDOWS_CHANGED: { + return true; + } + // All events for changes in window content should be + // dispatched *only* if this window is one of the windows + // the accessibility layer reports which are windows + // that a sighted user can touch. + default: { + return isRetrievalAllowingWindowLocked(userId, event.getWindowId()); + } + } + } + + /** + * Find a valid package name for an app to expose to accessibility + * + * @param packageName The package name the app wants to expose + * @param appId The app's id + * @param userId The app's user id + * @param pid The app's process pid that requested this + * @return A package name that is valid to report + */ + @Nullable + public String resolveValidReportedPackageLocked( + @Nullable CharSequence packageName, int appId, int userId, int pid) { + // Okay to pass no package + if (packageName == null) { + return null; + } + // The system gets to pass any package + if (appId == Process.SYSTEM_UID) { + return packageName.toString(); + } + // Passing a package in your UID is fine + final String packageNameStr = packageName.toString(); + final int resolvedUid = UserHandle.getUid(userId, appId); + if (isValidPackageForUid(packageNameStr, resolvedUid)) { + return packageName.toString(); + } + // Appwidget hosts get to pass packages for widgets they host + if (mAppWidgetService != null && ArrayUtils.contains(mAppWidgetService + .getHostedWidgetPackages(resolvedUid), packageNameStr)) { + return packageName.toString(); + } + // If app has the targeted permission to act as another package + if (mContext.checkPermission(Manifest.permission.ACT_AS_PACKAGE_FOR_ACCESSIBILITY, + pid, resolvedUid) == PackageManager.PERMISSION_GRANTED) { + return packageName.toString(); + } + // Otherwise, set the package to the first one in the UID + final String[] packageNames = mPackageManager.getPackagesForUid(resolvedUid); + if (ArrayUtils.isEmpty(packageNames)) { + return null; + } + // Okay, the caller reported a package it does not have access to. + // Instead of crashing the caller for better backwards compatibility + // we report the first package in the UID. Since most of the time apps + // don't use shared user id, this will yield correct results and for + // the edge case of using a shared user id we may report the wrong + // package but this is fine since first, this is a cheating app and + // second there is no way to get the correct package anyway. + return packageNames[0]; + } + + /** + * Get the packages that are valid for a uid. In some situations, like app widgets, there + * could be several valid packages + * + * @param targetPackage A package that is known to be valid for this id + * @param targetUid The whose packages should be checked + * @return An array of all valid package names. An empty array means any package is OK + */ + @NonNull + public String[] computeValidReportedPackages( + @NonNull String targetPackage, int targetUid) { + if (UserHandle.getAppId(targetUid) == Process.SYSTEM_UID) { + // Empty array means any package is Okay + return EmptyArray.STRING; + } + // IMPORTANT: The target package is already vetted to be in the target UID + String[] uidPackages = new String[]{targetPackage}; + // Appwidget hosts get to pass packages for widgets they host + if (mAppWidgetService != null) { + final ArraySet<String> widgetPackages = mAppWidgetService + .getHostedWidgetPackages(targetUid); + if (widgetPackages != null && !widgetPackages.isEmpty()) { + final String[] validPackages = new String[uidPackages.length + + widgetPackages.size()]; + System.arraycopy(uidPackages, 0, validPackages, 0, uidPackages.length); + final int widgetPackageCount = widgetPackages.size(); + for (int i = 0; i < widgetPackageCount; i++) { + validPackages[uidPackages.length + i] = widgetPackages.valueAt(i); + } + return validPackages; + } + } + return uidPackages; + } + + /** + * Reset the event source for events that should not carry one + * + * @param event The event potentially to modify + */ + public void updateEventSourceLocked(@NonNull AccessibilityEvent event) { + if ((event.getEventType() & KEEP_SOURCE_EVENT_TYPES) == 0) { + event.setSource(null); + } + } + + /** + * Check if a service can have access to a window + * + * @param userId The id of the user running the service + * @param service The service requesting access + * @param windowId The window it wants access to + * + * @return Whether ot not the service may retrieve info from the window + */ + public boolean canGetAccessibilityNodeInfoLocked(int userId, + @NonNull AbstractAccessibilityServiceConnection service, int windowId) { + return canRetrieveWindowContentLocked(service) + && isRetrievalAllowingWindowLocked(userId, windowId); + } + + /** + * Check if a service can have access the list of windows + * + * @param service The service requesting access + * + * @return Whether ot not the service may retrieve the window list + */ + public boolean canRetrieveWindowsLocked( + @NonNull AbstractAccessibilityServiceConnection service) { + return canRetrieveWindowContentLocked(service) && service.mRetrieveInteractiveWindows; + } + + /** + * Check if a service can have access the content of windows on the screen + * + * @param service The service requesting access + * + * @return Whether ot not the service may retrieve the content + */ + public boolean canRetrieveWindowContentLocked( + @NonNull AbstractAccessibilityServiceConnection service) { + return (service.getCapabilities() + & AccessibilityServiceInfo.CAPABILITY_CAN_RETRIEVE_WINDOW_CONTENT) != 0; + } + + /** + * Check if a service can control magnification + * + * @param service The service requesting access + * + * @return Whether ot not the service may control magnification + */ + public boolean canControlMagnification( + @NonNull AbstractAccessibilityServiceConnection service) { + return (service.getCapabilities() + & AccessibilityServiceInfo.CAPABILITY_CAN_CONTROL_MAGNIFICATION) != 0; + } + + /** + * Check if a service can perform gestures + * + * @param service The service requesting access + * + * @return Whether ot not the service may perform gestures + */ + public boolean canPerformGestures(@NonNull AccessibilityServiceConnection service) { + return (service.getCapabilities() + & AccessibilityServiceInfo.CAPABILITY_CAN_PERFORM_GESTURES) != 0; + } + + /** + * Check if a service can capture gestures from the fingerprint sensor + * + * @param service The service requesting access + * + * @return Whether ot not the service may capture gestures from the fingerprint sensor + */ + public boolean canCaptureFingerprintGestures(@NonNull AccessibilityServiceConnection service) { + return (service.getCapabilities() + & AccessibilityServiceInfo.CAPABILITY_CAN_REQUEST_FINGERPRINT_GESTURES) != 0; + } + + /** + * Checks if a service can take screenshot. + * + * @param service The service requesting access + * + * @return Whether ot not the service may take screenshot + */ + public boolean canTakeScreenshotLocked( + @NonNull AbstractAccessibilityServiceConnection service) { + return (service.getCapabilities() + & AccessibilityServiceInfo.CAPABILITY_CAN_TAKE_SCREENSHOT) != 0; + } + + /** + * Returns the parent userId of the profile according to the specified userId. + * + * @param userId The userId to check + * @return the parent userId of the profile, or self if no parent exist + */ + public int resolveProfileParentLocked(int userId) { + if (userId != mAccessibilityUserManager.getCurrentUserIdLocked()) { + final long identity = Binder.clearCallingIdentity(); + try { + UserInfo parent = mUserManager.getProfileParent(userId); + if (parent != null) { + return parent.getUserHandle().getIdentifier(); + } + } finally { + Binder.restoreCallingIdentity(identity); + } + } + return userId; + } + + /** + * Returns the parent userId of the profile according to the specified userId. Enforcing + * permissions check if specified userId is not caller's userId. + * + * @param userId The userId to check + * @return the parent userId of the profile, or self if no parent exist + * @throws SecurityException if caller cannot interact across users + * @throws IllegalArgumentException if specified invalid userId + */ + public int resolveCallingUserIdEnforcingPermissionsLocked(int userId) { + final int callingUid = Binder.getCallingUid(); + final int currentUserId = mAccessibilityUserManager.getCurrentUserIdLocked(); + if (callingUid == 0 + || callingUid == Process.SYSTEM_UID + || callingUid == Process.SHELL_UID) { + if (userId == UserHandle.USER_CURRENT + || userId == UserHandle.USER_CURRENT_OR_SELF) { + return currentUserId; + } + return resolveProfileParentLocked(userId); + } + final int callingUserId = UserHandle.getUserId(callingUid); + if (callingUserId == userId) { + return resolveProfileParentLocked(userId); + } + final int callingUserParentId = resolveProfileParentLocked(callingUserId); + if (callingUserParentId == currentUserId && (userId == UserHandle.USER_CURRENT + || userId == UserHandle.USER_CURRENT_OR_SELF)) { + return currentUserId; + } + if (!hasPermission(Manifest.permission.INTERACT_ACROSS_USERS) + && !hasPermission(Manifest.permission.INTERACT_ACROSS_USERS_FULL)) { + throw new SecurityException("Call from user " + callingUserId + " as user " + + userId + " without permission INTERACT_ACROSS_USERS or " + + "INTERACT_ACROSS_USERS_FULL not allowed."); + } + if (userId == UserHandle.USER_CURRENT + || userId == UserHandle.USER_CURRENT_OR_SELF) { + return currentUserId; + } + return resolveProfileParentLocked(userId); + } + + /** + * Returns false if caller is not SYSTEM and SHELL, and tried to interact across users. + * + * @param userId The userId to interact. + * @return false if caller cannot interact across users. + */ + public boolean isCallerInteractingAcrossUsers(int userId) { + final int callingUid = Binder.getCallingUid(); + return (Binder.getCallingPid() == android.os.Process.myPid() + || callingUid == Process.SHELL_UID + || userId == UserHandle.USER_CURRENT + || userId == UserHandle.USER_CURRENT_OR_SELF); + } + + private boolean isValidPackageForUid(String packageName, int uid) { + final long token = Binder.clearCallingIdentity(); + try { + return uid == mPackageManager.getPackageUidAsUser( + packageName, UserHandle.getUserId(uid)); + } catch (PackageManager.NameNotFoundException e) { + return false; + } finally { + Binder.restoreCallingIdentity(token); + } + } + + private boolean isRetrievalAllowingWindowLocked(int userId, int windowId) { + // The system gets to interact with any window it wants. + if (Binder.getCallingUid() == Process.SYSTEM_UID) { + return true; + } + if (Binder.getCallingUid() == Process.SHELL_UID) { + if (!isShellAllowedToRetrieveWindowLocked(userId, windowId)) { + return false; + } + } + if (mAccessibilityWindowManager.resolveParentWindowIdLocked(windowId) + == mAccessibilityWindowManager.getActiveWindowId(userId)) { + return true; + } + return mAccessibilityWindowManager.findA11yWindowInfoByIdLocked(windowId) != null; + } + + private boolean isShellAllowedToRetrieveWindowLocked(int userId, int windowId) { + long token = Binder.clearCallingIdentity(); + try { + IBinder windowToken = mAccessibilityWindowManager + .getWindowTokenForUserAndWindowIdLocked(userId, windowId); + if (windowToken == null) { + return false; + } + int windowOwnerUserId = mAccessibilityWindowManager.getWindowOwnerUserId(windowToken); + if (windowOwnerUserId == UserHandle.USER_NULL) { + return false; + } + return !mUserManager.hasUserRestriction( + UserManager.DISALLOW_DEBUGGING_FEATURES, UserHandle.of(windowOwnerUserId)); + } finally { + Binder.restoreCallingIdentity(token); + } + } + + /** + * Enforcing permission check to caller. + * + * @param permission The permission to check + * @param function The function name to check + */ + public void enforceCallingPermission(@NonNull String permission, @Nullable String function) { + if (OWN_PROCESS_ID == Binder.getCallingPid()) { + return; + } + if (!hasPermission(permission)) { + throw new SecurityException("You do not have " + permission + + " required to call " + function + " from pid=" + + Binder.getCallingPid() + ", uid=" + Binder.getCallingUid()); + } + } + + /** + * Permission check to caller. + * + * @param permission The permission to check + * @return true if caller has permission + */ + public boolean hasPermission(@NonNull String permission) { + return mContext.checkCallingPermission(permission) == PackageManager.PERMISSION_GRANTED; + } + + /** + * Checks if accessibility service could register into the system. + * + * @param serviceInfo The ServiceInfo + * @return True if it could register into the system + */ + public boolean canRegisterService(@NonNull ServiceInfo serviceInfo) { + if (!android.Manifest.permission.BIND_ACCESSIBILITY_SERVICE.equals( + serviceInfo.permission)) { + Slog.w(LOG_TAG, "Skipping accessibility service " + new ComponentName( + serviceInfo.packageName, serviceInfo.name).flattenToShortString() + + ": it does not require the permission " + + android.Manifest.permission.BIND_ACCESSIBILITY_SERVICE); + return false; + } + + int servicePackageUid = serviceInfo.applicationInfo.uid; + if (mAppOpsManager.noteOpNoThrow(AppOpsManager.OPSTR_BIND_ACCESSIBILITY_SERVICE, + servicePackageUid, serviceInfo.packageName) != AppOpsManager.MODE_ALLOWED) { + Slog.w(LOG_TAG, "Skipping accessibility service " + new ComponentName( + serviceInfo.packageName, serviceInfo.name).flattenToShortString() + + ": disallowed by AppOps"); + return false; + } + + return true; + } + + /** + * Checks if accessibility service could execute accessibility operations. + * + * @param service The accessibility service connection + * @return True if it could execute accessibility operations + */ + public boolean checkAccessibilityAccess(AbstractAccessibilityServiceConnection service) { + final String packageName = service.getComponentName().getPackageName(); + final ResolveInfo resolveInfo = service.getServiceInfo().getResolveInfo(); + + if (resolveInfo == null) { + // For InteractionBridge and UiAutomation + return true; + } + + final int uid = resolveInfo.serviceInfo.applicationInfo.uid; + final long identityToken = Binder.clearCallingIdentity(); + try { + // For the caller is system, just block the data to a11y services. + if (OWN_PROCESS_ID == Binder.getCallingPid()) { + return mAppOpsManager.noteOpNoThrow(AppOpsManager.OPSTR_ACCESS_ACCESSIBILITY, + uid, packageName) == AppOpsManager.MODE_ALLOWED; + } + + return mAppOpsManager.noteOp(AppOpsManager.OPSTR_ACCESS_ACCESSIBILITY, + uid, packageName) == AppOpsManager.MODE_ALLOWED; + } finally { + Binder.restoreCallingIdentity(identityToken); + } + } + + /** + * Enforcing permission check to IPC caller or grant it if it's not through IPC. + * + * @param permission The permission to check + */ + public void enforceCallingOrSelfPermission(@NonNull String permission) { + if (mContext.checkCallingOrSelfPermission(permission) + != PackageManager.PERMISSION_GRANTED) { + throw new SecurityException("Caller does not hold permission " + + permission); + } + } + + /** + * Enforcing permission check to IPC caller or grant it if it's recents. + * + * @param permission The permission to check + */ + public void enforceCallerIsRecentsOrHasPermission(@NonNull String permission, String func) { + mAtmInternal.enforceCallerIsRecentsOrHasPermission(permission, func); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java index 91031e602824..fea2e7b841e0 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java @@ -18,8 +18,10 @@ package com.android.server.accessibility; import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; +import android.Manifest; import android.accessibilityservice.AccessibilityServiceInfo; import android.accessibilityservice.IAccessibilityServiceClient; +import android.app.PendingIntent; import android.content.ComponentName; import android.content.Context; import android.content.Intent; @@ -27,13 +29,14 @@ import android.content.pm.ParceledListSlice; import android.os.Binder; import android.os.Handler; import android.os.IBinder; +import android.os.Process; import android.os.RemoteException; import android.os.UserHandle; import android.provider.Settings; import android.util.Slog; +import android.view.Display; -import com.android.server.accessibility.AccessibilityManagerService.SecurityPolicy; -import com.android.server.accessibility.AccessibilityManagerService.UserState; +import com.android.server.inputmethod.InputMethodManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.WindowManagerInternal; @@ -51,31 +54,28 @@ import java.util.Set; class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnection { private static final String LOG_TAG = "AccessibilityServiceConnection"; /* - Holding a weak reference so there isn't a loop of references. UserState keeps lists of bound - and binding services. These are freed on user changes, but just in case it somehow gets lost - the weak reference will let the memory get GCed. + Holding a weak reference so there isn't a loop of references. AccessibilityUserState keeps + lists of bound and binding services. These are freed on user changes, but just in case it + somehow gets lost the weak reference will let the memory get GCed. Having the reference be null when being called is a very bad sign, but we check the condition. */ - final WeakReference<UserState> mUserStateWeakReference; + final WeakReference<AccessibilityUserState> mUserStateWeakReference; final Intent mIntent; final ActivityTaskManagerInternal mActivityTaskManagerService; private final Handler mMainHandler; - private boolean mWasConnectedAndDied; - - - public AccessibilityServiceConnection(UserState userState, Context context, + AccessibilityServiceConnection(AccessibilityUserState userState, Context context, ComponentName componentName, AccessibilityServiceInfo accessibilityServiceInfo, int id, Handler mainHandler, - Object lock, SecurityPolicy securityPolicy, SystemSupport systemSupport, + Object lock, AccessibilitySecurityPolicy securityPolicy, SystemSupport systemSupport, WindowManagerInternal windowManagerInternal, - GlobalActionPerformer globalActionPerfomer, + SystemActionPerformer systemActionPerfomer, AccessibilityWindowManager awm, ActivityTaskManagerInternal activityTaskManagerService) { super(context, componentName, accessibilityServiceInfo, id, mainHandler, lock, - securityPolicy, systemSupport, windowManagerInternal, globalActionPerfomer); - mUserStateWeakReference = new WeakReference<UserState>(userState); + securityPolicy, systemSupport, windowManagerInternal, systemActionPerfomer, awm); + mUserStateWeakReference = new WeakReference<AccessibilityUserState>(userState); mIntent = new Intent().setComponent(mComponentName); mMainHandler = mainHandler; mIntent.putExtra(Intent.EXTRA_CLIENT_LABEL, @@ -84,20 +84,23 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect final long identity = Binder.clearCallingIdentity(); try { mIntent.putExtra(Intent.EXTRA_CLIENT_INTENT, mSystemSupport.getPendingIntentActivity( - mContext, 0, new Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS), 0)); + mContext, 0, new Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS), + PendingIntent.FLAG_IMMUTABLE)); } finally { Binder.restoreCallingIdentity(identity); } } public void bindLocked() { - UserState userState = mUserStateWeakReference.get(); + AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return; final long identity = Binder.clearCallingIdentity(); try { - int flags = Context.BIND_AUTO_CREATE | Context.BIND_FOREGROUND_SERVICE_WHILE_AWAKE - | Context.BIND_ALLOW_BACKGROUND_ACTIVITY_STARTS; - if (userState.getBindInstantServiceAllowed()) { + int flags = Context.BIND_AUTO_CREATE + | Context.BIND_FOREGROUND_SERVICE_WHILE_AWAKE + | Context.BIND_ALLOW_BACKGROUND_ACTIVITY_STARTS + | Context.BIND_INCLUDE_CAPABILITIES; + if (userState.getBindInstantServiceAllowedLocked()) { flags |= Context.BIND_ALLOW_INSTANT; } if (mService == null && mContext.bindServiceAsUser( @@ -114,13 +117,12 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect public void unbindLocked() { mContext.unbindService(this); - UserState userState = mUserStateWeakReference.get(); + AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return; userState.removeServiceLocked(this); mSystemSupport.getMagnificationController().resetAllIfNeeded(mId); - // Set uid to -1 to clear allowing app switches. - mActivityTaskManagerService.setAllowAppSwitches(mComponentName.flattenToString(), - /* uid= */ -1, userState.mUserId); + mActivityTaskManagerService.setAllowAppSwitches(mComponentName.flattenToString(), -1, + userState.mUserId); resetLocked(); } @@ -131,7 +133,7 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect @Override public void disableSelf() { synchronized (mLock) { - UserState userState = mUserStateWeakReference.get(); + AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return; if (userState.getEnabledServicesLocked().remove(mComponentName)) { final long identity = Binder.clearCallingIdentity(); @@ -164,7 +166,7 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect } } mServiceInterface = IAccessibilityServiceClient.Stub.asInterface(service); - UserState userState = mUserStateWeakReference.get(); + AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return; userState.addServiceLocked(this); mSystemSupport.onClientChangeLocked(false); @@ -177,20 +179,21 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect @Override public AccessibilityServiceInfo getServiceInfo() { - // Update crashed data - mAccessibilityServiceInfo.crashed = mWasConnectedAndDied; return mAccessibilityServiceInfo; } private void initializeService() { IAccessibilityServiceClient serviceInterface = null; synchronized (mLock) { - UserState userState = mUserStateWeakReference.get(); + AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return; - Set<ComponentName> bindingServices = userState.getBindingServicesLocked(); - if (bindingServices.contains(mComponentName) || mWasConnectedAndDied) { + final Set<ComponentName> bindingServices = userState.getBindingServicesLocked(); + final Set<ComponentName> crashedServices = userState.getCrashedServicesLocked(); + if (bindingServices.contains(mComponentName) + || crashedServices.contains(mComponentName)) { bindingServices.remove(mComponentName); - mWasConnectedAndDied = false; + crashedServices.remove(mComponentName); + mAccessibilityServiceInfo.crashed = false; serviceInterface = mServiceInterface; } // There's a chance that service is removed from enabled_accessibility_services setting @@ -207,7 +210,7 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect return; } try { - serviceInterface.init(this, mId, mOverlayWindowToken); + serviceInterface.init(this, mId, mOverlayWindowTokens.get(Display.DEFAULT_DISPLAY)); } catch (RemoteException re) { Slog.w(LOG_TAG, "Error while setting connection for service: " + serviceInterface, re); @@ -218,31 +221,42 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect @Override public void onServiceDisconnected(ComponentName componentName) { binderDied(); - UserState userState = mUserStateWeakReference.get(); + AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState != null) { - // Set uid to -1 to clear allowing app switches. - mActivityTaskManagerService.setAllowAppSwitches(mComponentName.flattenToString(), - /* uid= */ -1, userState.mUserId); + mActivityTaskManagerService.setAllowAppSwitches(mComponentName.flattenToString(), -1, + userState.mUserId); } } @Override - protected boolean isCalledForCurrentUserLocked() { + protected boolean hasRightsToCurrentUserLocked() { // We treat calls from a profile as if made by its parent as profiles // share the accessibility state of the parent. The call below // performs the current profile parent resolution. - final int resolvedUserId = mSecurityPolicy - .resolveCallingUserIdEnforcingPermissionsLocked(UserHandle.USER_CURRENT); - return resolvedUserId == mSystemSupport.getCurrentUserIdLocked(); + final int callingUid = Binder.getCallingUid(); + if (callingUid == Process.ROOT_UID + || callingUid == Process.SYSTEM_UID + || callingUid == Process.SHELL_UID) { + return true; + } + if (mSecurityPolicy.resolveProfileParentLocked(UserHandle.getUserId(callingUid)) + == mSystemSupport.getCurrentUserIdLocked()) { + return true; + } + if (mSecurityPolicy.hasPermission(Manifest.permission.INTERACT_ACROSS_USERS) + || mSecurityPolicy.hasPermission(Manifest.permission.INTERACT_ACROSS_USERS_FULL)) { + return true; + } + return false; } @Override public boolean setSoftKeyboardShowMode(int showMode) { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return false; } - final UserState userState = mUserStateWeakReference.get(); + final AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return false; return userState.setSoftKeyboardModeLocked(showMode, mComponentName); } @@ -250,18 +264,35 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect @Override public int getSoftKeyboardShowMode() { - final UserState userState = mUserStateWeakReference.get(); - return (userState != null) ? userState.getSoftKeyboardShowMode() : 0; + final AccessibilityUserState userState = mUserStateWeakReference.get(); + return (userState != null) ? userState.getSoftKeyboardShowModeLocked() : 0; } + @Override + public boolean switchToInputMethod(String imeId) { + synchronized (mLock) { + if (!hasRightsToCurrentUserLocked()) { + return false; + } + } + final boolean result; + final int callingUserId = UserHandle.getCallingUserId(); + final long identity = Binder.clearCallingIdentity(); + try { + result = InputMethodManagerInternal.get().switchToInputMethod(imeId, callingUserId); + } finally { + Binder.restoreCallingIdentity(identity); + } + return result; + } @Override public boolean isAccessibilityButtonAvailable() { synchronized (mLock) { - if (!isCalledForCurrentUserLocked()) { + if (!hasRightsToCurrentUserLocked()) { return false; } - UserState userState = mUserStateWeakReference.get(); + AccessibilityUserState userState = mUserStateWeakReference.get(); return (userState != null) && isAccessibilityButtonAvailableLocked(userState); } } @@ -275,8 +306,8 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect if (!isConnectedLocked()) { return; } - mWasConnectedAndDied = true; - UserState userState = mUserStateWeakReference.get(); + mAccessibilityServiceInfo.crashed = true; + AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState != null) { userState.serviceDisconnectedLocked(this); } @@ -286,46 +317,16 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect } } - public boolean isAccessibilityButtonAvailableLocked(UserState userState) { + public boolean isAccessibilityButtonAvailableLocked(AccessibilityUserState userState) { // If the service does not request the accessibility button, it isn't available if (!mRequestAccessibilityButton) { return false; } - // If the accessibility button isn't currently shown, it cannot be available to services if (!mSystemSupport.isAccessibilityButtonShown()) { return false; } - - // If magnification is on and assigned to the accessibility button, services cannot be - if (userState.mIsNavBarMagnificationEnabled - && userState.mIsNavBarMagnificationAssignedToAccessibilityButton) { - return false; - } - - int requestingServices = 0; - for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { - final AccessibilityServiceConnection service = userState.mBoundServices.get(i); - if (service.mRequestAccessibilityButton) { - requestingServices++; - } - } - - if (requestingServices == 1) { - // If only a single service is requesting, it must be this service, and the - // accessibility button is available to it - return true; - } else { - // With more than one active service, we derive the target from the user's settings - if (userState.mServiceAssignedToAccessibilityButton == null) { - // If the user has not made an assignment, we treat the button as available to - // all services until the user interacts with the button to make an assignment - return true; - } else { - // If an assignment was made, it defines availability - return mComponentName.equals(userState.mServiceAssignedToAccessibilityButton); - } - } + return true; } @Override @@ -370,14 +371,15 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect } @Override - public void sendGesture(int sequence, ParceledListSlice gestureSteps) { + public void dispatchGesture(int sequence, ParceledListSlice gestureSteps, int displayId) { + final boolean isTouchableDisplay = mWindowManagerService.isTouchableDisplay(displayId); synchronized (mLock) { if (mSecurityPolicy.canPerformGestures(this)) { MotionEventInjector motionEventInjector = - mSystemSupport.getMotionEventInjectorLocked(); - if (motionEventInjector != null) { + mSystemSupport.getMotionEventInjectorForDisplayLocked(displayId); + if (motionEventInjector != null && isTouchableDisplay) { motionEventInjector.injectEvents( - gestureSteps.getList(), mServiceInterface, sequence); + gestureSteps.getList(), mServiceInterface, sequence, displayId); } else { try { mServiceInterface.onPerformGestureResult(sequence, false); diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityShellCommand.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityShellCommand.java index ff59c24a7ca2..b36626f9d736 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityShellCommand.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityShellCommand.java @@ -17,6 +17,9 @@ package com.android.server.accessibility; import android.annotation.NonNull; +import android.app.ActivityManager; +import android.os.Binder; +import android.os.Process; import android.os.ShellCommand; import android.os.UserHandle; @@ -27,9 +30,12 @@ import java.io.PrintWriter; */ final class AccessibilityShellCommand extends ShellCommand { final @NonNull AccessibilityManagerService mService; + final @NonNull SystemActionPerformer mSystemActionPerformer; - AccessibilityShellCommand(@NonNull AccessibilityManagerService service) { + AccessibilityShellCommand(@NonNull AccessibilityManagerService service, + @NonNull SystemActionPerformer systemActionPerformer) { mService = service; + mSystemActionPerformer = systemActionPerformer; } @Override @@ -44,6 +50,9 @@ final class AccessibilityShellCommand extends ShellCommand { case "set-bind-instant-service-allowed": { return runSetBindInstantServiceAllowed(); } + case "call-system-action": { + return runCallSystemAction(); + } } return -1; } @@ -73,6 +82,22 @@ final class AccessibilityShellCommand extends ShellCommand { return 0; } + private int runCallSystemAction() { + final int callingUid = Binder.getCallingUid(); + if (callingUid != Process.ROOT_UID + && callingUid != Process.SYSTEM_UID + && callingUid != Process.SHELL_UID) { + return -1; + } + final String option = getNextArg(); + if (option != null) { + int actionId = Integer.parseInt(option); + mSystemActionPerformer.performSystemAction(actionId); + return 0; + } + return -1; + } + private Integer parseUserId() { final String option = getNextOption(); if (option != null) { @@ -83,7 +108,7 @@ final class AccessibilityShellCommand extends ShellCommand { return null; } } - return UserHandle.USER_SYSTEM; + return ActivityManager.getCurrentUser(); } @Override @@ -96,5 +121,7 @@ final class AccessibilityShellCommand extends ShellCommand { pw.println(" Set whether binding to services provided by instant apps is allowed."); pw.println(" get-bind-instant-service-allowed [--user <USER_ID>]"); pw.println(" Get whether binding to services provided by instant apps is allowed."); + pw.println(" call-system-action <ACTION_ID>"); + pw.println(" Calls the system action with the given action id."); } }
\ No newline at end of file diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityUserState.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityUserState.java new file mode 100644 index 000000000000..0845d019c060 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityUserState.java @@ -0,0 +1,804 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility; + +import static android.accessibilityservice.AccessibilityService.SHOW_MODE_AUTO; +import static android.accessibilityservice.AccessibilityService.SHOW_MODE_HARD_KEYBOARD_ORIGINAL_VALUE; +import static android.accessibilityservice.AccessibilityService.SHOW_MODE_HARD_KEYBOARD_OVERRIDDEN; +import static android.accessibilityservice.AccessibilityService.SHOW_MODE_HIDDEN; +import static android.accessibilityservice.AccessibilityService.SHOW_MODE_IGNORE_HARD_KEYBOARD; +import static android.accessibilityservice.AccessibilityService.SHOW_MODE_MASK; +import static android.view.accessibility.AccessibilityManager.ACCESSIBILITY_BUTTON; +import static android.view.accessibility.AccessibilityManager.ACCESSIBILITY_SHORTCUT_KEY; +import static android.view.accessibility.AccessibilityManager.ShortcutType; + +import static com.android.internal.accessibility.AccessibilityShortcutController.MAGNIFICATION_CONTROLLER_NAME; + +import android.accessibilityservice.AccessibilityService.SoftKeyboardShowMode; +import android.accessibilityservice.AccessibilityServiceInfo; +import android.accessibilityservice.AccessibilityShortcutInfo; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.ComponentName; +import android.content.Context; +import android.os.Binder; +import android.os.RemoteCallbackList; +import android.provider.Settings; +import android.text.TextUtils; +import android.util.ArraySet; +import android.util.Slog; +import android.view.accessibility.AccessibilityManager; +import android.view.accessibility.IAccessibilityManagerClient; + +import com.android.internal.accessibility.AccessibilityShortcutController; + +import java.io.FileDescriptor; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Class that hold states and settings per user and share between + * {@link AccessibilityManagerService} and {@link AccessibilityServiceConnection}. + */ +class AccessibilityUserState { + private static final String LOG_TAG = AccessibilityUserState.class.getSimpleName(); + + final int mUserId; + + // Non-transient state. + + final RemoteCallbackList<IAccessibilityManagerClient> mUserClients = new RemoteCallbackList<>(); + + // Transient state. + + final ArrayList<AccessibilityServiceConnection> mBoundServices = new ArrayList<>(); + + final Map<ComponentName, AccessibilityServiceConnection> mComponentNameToServiceMap = + new HashMap<>(); + + final List<AccessibilityServiceInfo> mInstalledServices = new ArrayList<>(); + + final List<AccessibilityShortcutInfo> mInstalledShortcuts = new ArrayList<>(); + + final Set<ComponentName> mBindingServices = new HashSet<>(); + + final Set<ComponentName> mCrashedServices = new HashSet<>(); + + final Set<ComponentName> mEnabledServices = new HashSet<>(); + + final Set<ComponentName> mTouchExplorationGrantedServices = new HashSet<>(); + + final ArraySet<String> mAccessibilityShortcutKeyTargets = new ArraySet<>(); + + final ArraySet<String> mAccessibilityButtonTargets = new ArraySet<>(); + + private final ServiceInfoChangeListener mServiceInfoChangeListener; + + private ComponentName mServiceChangingSoftKeyboardMode; + + private String mTargetAssignedToAccessibilityButton; + + private boolean mBindInstantServiceAllowed; + private boolean mIsAutoclickEnabled; + private boolean mIsDisplayMagnificationEnabled; + private boolean mIsFilterKeyEventsEnabled; + private boolean mIsPerformGesturesEnabled; + private boolean mAccessibilityFocusOnlyInActiveWindow; + private boolean mIsTextHighContrastEnabled; + private boolean mIsTouchExplorationEnabled; + private boolean mServiceHandlesDoubleTap; + private boolean mRequestMultiFingerGestures; + private int mUserInteractiveUiTimeout; + private int mUserNonInteractiveUiTimeout; + private int mNonInteractiveUiTimeout = 0; + private int mInteractiveUiTimeout = 0; + private int mLastSentClientState = -1; + + private Context mContext; + + @SoftKeyboardShowMode + private int mSoftKeyboardShowMode = SHOW_MODE_AUTO; + + interface ServiceInfoChangeListener { + void onServiceInfoChangedLocked(AccessibilityUserState userState); + } + + AccessibilityUserState(int userId, @NonNull Context context, + @NonNull ServiceInfoChangeListener serviceInfoChangeListener) { + mUserId = userId; + mContext = context; + mServiceInfoChangeListener = serviceInfoChangeListener; + } + + boolean isHandlingAccessibilityEventsLocked() { + return !mBoundServices.isEmpty() || !mBindingServices.isEmpty(); + } + + void onSwitchToAnotherUserLocked() { + // Unbind all services. + unbindAllServicesLocked(); + + // Clear service management state. + mBoundServices.clear(); + mBindingServices.clear(); + mCrashedServices.clear(); + + // Clear event management state. + mLastSentClientState = -1; + + // clear UI timeout + mNonInteractiveUiTimeout = 0; + mInteractiveUiTimeout = 0; + + // Clear state persisted in settings. + mEnabledServices.clear(); + mTouchExplorationGrantedServices.clear(); + mAccessibilityShortcutKeyTargets.clear(); + mAccessibilityButtonTargets.clear(); + mTargetAssignedToAccessibilityButton = null; + mIsTouchExplorationEnabled = false; + mServiceHandlesDoubleTap = false; + mRequestMultiFingerGestures = false; + mIsDisplayMagnificationEnabled = false; + mIsAutoclickEnabled = false; + mUserNonInteractiveUiTimeout = 0; + mUserInteractiveUiTimeout = 0; + } + + void addServiceLocked(AccessibilityServiceConnection serviceConnection) { + if (!mBoundServices.contains(serviceConnection)) { + serviceConnection.onAdded(); + mBoundServices.add(serviceConnection); + mComponentNameToServiceMap.put(serviceConnection.getComponentName(), serviceConnection); + mServiceInfoChangeListener.onServiceInfoChangedLocked(this); + } + } + + /** + * Removes a service. + * There are three states to a service here: off, bound, and binding. + * This stops tracking the service as bound. + * + * @param serviceConnection The service. + */ + void removeServiceLocked(AccessibilityServiceConnection serviceConnection) { + mBoundServices.remove(serviceConnection); + serviceConnection.onRemoved(); + if ((mServiceChangingSoftKeyboardMode != null) + && (mServiceChangingSoftKeyboardMode.equals( + serviceConnection.getServiceInfo().getComponentName()))) { + setSoftKeyboardModeLocked(SHOW_MODE_AUTO, null); + } + // It may be possible to bind a service twice, which confuses the map. Rebuild the map + // to make sure we can still reach a service + mComponentNameToServiceMap.clear(); + for (int i = 0; i < mBoundServices.size(); i++) { + AccessibilityServiceConnection boundClient = mBoundServices.get(i); + mComponentNameToServiceMap.put(boundClient.getComponentName(), boundClient); + } + mServiceInfoChangeListener.onServiceInfoChangedLocked(this); + } + + /** + * Make sure a services disconnected but still 'on' state is reflected in AccessibilityUserState + * There are four states to a service here: off, bound, and binding, and crashed. + * This drops a service from a bound state, to the crashed state. + * The crashed state describes the situation where a service used to be bound, but no longer is + * despite still being enabled. + * + * @param serviceConnection The service. + */ + void serviceDisconnectedLocked(AccessibilityServiceConnection serviceConnection) { + removeServiceLocked(serviceConnection); + mCrashedServices.add(serviceConnection.getComponentName()); + } + + /** + * Set the soft keyboard mode. This mode is a bit odd, as it spans multiple settings. + * The ACCESSIBILITY_SOFT_KEYBOARD_MODE setting can be checked by the rest of the system + * to see if it should suppress showing the IME. The SHOW_IME_WITH_HARD_KEYBOARD setting + * setting can be changed by the user, and prevents the system from suppressing the soft + * keyboard when the hard keyboard is connected. The hard keyboard setting needs to defer + * to the user's preference, if they have supplied one. + * + * @param newMode The new mode + * @param requester The service requesting the change, so we can undo it when the + * service stops. Set to null if something other than a service is forcing + * the change. + * + * @return Whether or not the soft keyboard mode equals the new mode after the call + */ + boolean setSoftKeyboardModeLocked(@SoftKeyboardShowMode int newMode, + @Nullable ComponentName requester) { + if ((newMode != SHOW_MODE_AUTO) + && (newMode != SHOW_MODE_HIDDEN) + && (newMode != SHOW_MODE_IGNORE_HARD_KEYBOARD)) { + Slog.w(LOG_TAG, "Invalid soft keyboard mode"); + return false; + } + if (mSoftKeyboardShowMode == newMode) { + return true; + } + + if (newMode == SHOW_MODE_IGNORE_HARD_KEYBOARD) { + if (hasUserOverriddenHardKeyboardSetting()) { + // The user has specified a default for this setting + return false; + } + // Save the original value. But don't do this if the value in settings is already + // the new mode. That happens when we start up after a reboot, and we don't want + // to overwrite the value we had from when we first started controlling the setting. + if (getSoftKeyboardValueFromSettings() != SHOW_MODE_IGNORE_HARD_KEYBOARD) { + setOriginalHardKeyboardValue(getSecureIntForUser( + Settings.Secure.SHOW_IME_WITH_HARD_KEYBOARD, 0, mUserId) != 0); + } + putSecureIntForUser(Settings.Secure.SHOW_IME_WITH_HARD_KEYBOARD, 1, mUserId); + } else if (mSoftKeyboardShowMode == SHOW_MODE_IGNORE_HARD_KEYBOARD) { + putSecureIntForUser(Settings.Secure.SHOW_IME_WITH_HARD_KEYBOARD, + getOriginalHardKeyboardValue() ? 1 : 0, mUserId); + } + + saveSoftKeyboardValueToSettings(newMode); + mSoftKeyboardShowMode = newMode; + mServiceChangingSoftKeyboardMode = requester; + for (int i = mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = mBoundServices.get(i); + service.notifySoftKeyboardShowModeChangedLocked(mSoftKeyboardShowMode); + } + return true; + } + + @SoftKeyboardShowMode + int getSoftKeyboardShowModeLocked() { + return mSoftKeyboardShowMode; + } + + /** + * If the settings are inconsistent with the internal state, make the internal state + * match the settings. + */ + void reconcileSoftKeyboardModeWithSettingsLocked() { + final boolean showWithHardKeyboardSettings = + getSecureIntForUser(Settings.Secure.SHOW_IME_WITH_HARD_KEYBOARD, 0, mUserId) != 0; + if (mSoftKeyboardShowMode == SHOW_MODE_IGNORE_HARD_KEYBOARD) { + if (!showWithHardKeyboardSettings) { + // The user has overridden the setting. Respect that and prevent further changes + // to this behavior. + setSoftKeyboardModeLocked(SHOW_MODE_AUTO, null); + setUserOverridesHardKeyboardSetting(); + } + } + + // If the setting and the internal state are out of sync, set both to default + if (getSoftKeyboardValueFromSettings() != mSoftKeyboardShowMode) { + Slog.e(LOG_TAG, "Show IME setting inconsistent with internal state. Overwriting"); + setSoftKeyboardModeLocked(SHOW_MODE_AUTO, null); + putSecureIntForUser(Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, + SHOW_MODE_AUTO, mUserId); + } + } + + boolean getBindInstantServiceAllowedLocked() { + return mBindInstantServiceAllowed; + } + + /* Need to have a permission check on callee */ + void setBindInstantServiceAllowedLocked(boolean allowed) { + mBindInstantServiceAllowed = allowed; + } + + /** + * Returns binding service list. + */ + Set<ComponentName> getBindingServicesLocked() { + return mBindingServices; + } + + /** + * Returns crashed service list. + */ + Set<ComponentName> getCrashedServicesLocked() { + return mCrashedServices; + } + + /** + * Returns enabled service list. + */ + Set<ComponentName> getEnabledServicesLocked() { + return mEnabledServices; + } + + /** + * Remove service from crashed service list if users disable it. + */ + void updateCrashedServicesIfNeededLocked() { + for (int i = 0, count = mInstalledServices.size(); i < count; i++) { + final AccessibilityServiceInfo installedService = mInstalledServices.get(i); + final ComponentName componentName = ComponentName.unflattenFromString( + installedService.getId()); + + if (mCrashedServices.contains(componentName) + && !mEnabledServices.contains(componentName)) { + // Remove it from mCrashedServices since users toggle the switch bar to retry. + mCrashedServices.remove(componentName); + } + } + } + + List<AccessibilityServiceConnection> getBoundServicesLocked() { + return mBoundServices; + } + + int getClientStateLocked(boolean isUiAutomationRunning) { + int clientState = 0; + final boolean a11yEnabled = isUiAutomationRunning + || isHandlingAccessibilityEventsLocked(); + if (a11yEnabled) { + clientState |= AccessibilityManager.STATE_FLAG_ACCESSIBILITY_ENABLED; + } + // Touch exploration relies on enabled accessibility. + if (a11yEnabled && mIsTouchExplorationEnabled) { + clientState |= AccessibilityManager.STATE_FLAG_TOUCH_EXPLORATION_ENABLED; + clientState |= AccessibilityManager.STATE_FLAG_DISPATCH_DOUBLE_TAP; + clientState |= AccessibilityManager.STATE_FLAG_REQUEST_MULTI_FINGER_GESTURES; + } + if (mIsTextHighContrastEnabled) { + clientState |= AccessibilityManager.STATE_FLAG_HIGH_TEXT_CONTRAST_ENABLED; + } + return clientState; + } + + private void setUserOverridesHardKeyboardSetting() { + final int softKeyboardSetting = getSecureIntForUser( + Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, SHOW_MODE_AUTO, mUserId); + putSecureIntForUser(Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, + softKeyboardSetting | SHOW_MODE_HARD_KEYBOARD_OVERRIDDEN, + mUserId); + } + + private boolean hasUserOverriddenHardKeyboardSetting() { + final int softKeyboardSetting = getSecureIntForUser( + Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, SHOW_MODE_AUTO, mUserId); + return (softKeyboardSetting & SHOW_MODE_HARD_KEYBOARD_OVERRIDDEN) + != 0; + } + + private void setOriginalHardKeyboardValue(boolean originalHardKeyboardValue) { + final int oldSoftKeyboardSetting = getSecureIntForUser( + Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, SHOW_MODE_AUTO, mUserId); + final int newSoftKeyboardSetting = oldSoftKeyboardSetting + & (~SHOW_MODE_HARD_KEYBOARD_ORIGINAL_VALUE) + | ((originalHardKeyboardValue) ? SHOW_MODE_HARD_KEYBOARD_ORIGINAL_VALUE : 0); + putSecureIntForUser(Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, + newSoftKeyboardSetting, mUserId); + } + + private void saveSoftKeyboardValueToSettings(int softKeyboardShowMode) { + final int oldSoftKeyboardSetting = getSecureIntForUser( + Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, SHOW_MODE_AUTO, mUserId); + final int newSoftKeyboardSetting = oldSoftKeyboardSetting & (~SHOW_MODE_MASK) + | softKeyboardShowMode; + putSecureIntForUser(Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, + newSoftKeyboardSetting, mUserId); + } + + private int getSoftKeyboardValueFromSettings() { + return getSecureIntForUser( + Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, SHOW_MODE_AUTO, mUserId) + & SHOW_MODE_MASK; + } + + private boolean getOriginalHardKeyboardValue() { + return (getSecureIntForUser( + Settings.Secure.ACCESSIBILITY_SOFT_KEYBOARD_MODE, SHOW_MODE_AUTO, mUserId) + & SHOW_MODE_HARD_KEYBOARD_ORIGINAL_VALUE) != 0; + } + + private void unbindAllServicesLocked() { + final List<AccessibilityServiceConnection> services = mBoundServices; + for (int count = services.size(); count > 0; count--) { + // When the service is unbound, it disappears from the list, so there's no need to + // keep track of the index + services.get(0).unbindLocked(); + } + } + + private int getSecureIntForUser(String key, int def, int userId) { + return Settings.Secure.getIntForUser(mContext.getContentResolver(), key, def, userId); + } + + private void putSecureIntForUser(String key, int value, int userId) { + final long identity = Binder.clearCallingIdentity(); + try { + Settings.Secure.putIntForUser(mContext.getContentResolver(), key, value, userId); + } finally { + Binder.restoreCallingIdentity(identity); + } + } + + void dump(FileDescriptor fd, PrintWriter pw, String[] args) { + pw.append("User state["); + pw.println(); + pw.append(" attributes:{id=").append(String.valueOf(mUserId)); + pw.append(", touchExplorationEnabled=").append(String.valueOf(mIsTouchExplorationEnabled)); + pw.append(", serviceHandlesDoubleTap=") + .append(String.valueOf(mServiceHandlesDoubleTap)); + pw.append(", requestMultiFingerGestures=") + .append(String.valueOf(mRequestMultiFingerGestures)); + pw.append(", displayMagnificationEnabled=").append(String.valueOf( + mIsDisplayMagnificationEnabled)); + pw.append(", autoclickEnabled=").append(String.valueOf(mIsAutoclickEnabled)); + pw.append(", nonInteractiveUiTimeout=").append(String.valueOf(mNonInteractiveUiTimeout)); + pw.append(", interactiveUiTimeout=").append(String.valueOf(mInteractiveUiTimeout)); + pw.append(", installedServiceCount=").append(String.valueOf(mInstalledServices.size())); + pw.append("}"); + pw.println(); + pw.append(" shortcut key:{"); + int size = mAccessibilityShortcutKeyTargets.size(); + for (int i = 0; i < size; i++) { + final String componentId = mAccessibilityShortcutKeyTargets.valueAt(i); + pw.append(componentId); + if (i + 1 < size) { + pw.append(", "); + } + } + pw.println("}"); + pw.append(" button:{"); + size = mAccessibilityButtonTargets.size(); + for (int i = 0; i < size; i++) { + final String componentId = mAccessibilityButtonTargets.valueAt(i); + pw.append(componentId); + if (i + 1 < size) { + pw.append(", "); + } + } + pw.println("}"); + pw.append(" button target:{").append(mTargetAssignedToAccessibilityButton); + pw.println("}"); + pw.append(" Bound services:{"); + final int serviceCount = mBoundServices.size(); + for (int j = 0; j < serviceCount; j++) { + if (j > 0) { + pw.append(", "); + pw.println(); + pw.append(" "); + } + AccessibilityServiceConnection service = mBoundServices.get(j); + service.dump(fd, pw, args); + } + pw.println("}"); + pw.append(" Enabled services:{"); + Iterator<ComponentName> it = mEnabledServices.iterator(); + if (it.hasNext()) { + ComponentName componentName = it.next(); + pw.append(componentName.toShortString()); + while (it.hasNext()) { + componentName = it.next(); + pw.append(", "); + pw.append(componentName.toShortString()); + } + } + pw.println("}"); + pw.append(" Binding services:{"); + it = mBindingServices.iterator(); + if (it.hasNext()) { + ComponentName componentName = it.next(); + pw.append(componentName.toShortString()); + while (it.hasNext()) { + componentName = it.next(); + pw.append(", "); + pw.append(componentName.toShortString()); + } + } + pw.println("}"); + pw.append(" Crashed services:{"); + it = mCrashedServices.iterator(); + if (it.hasNext()) { + ComponentName componentName = it.next(); + pw.append(componentName.toShortString()); + while (it.hasNext()) { + componentName = it.next(); + pw.append(", "); + pw.append(componentName.toShortString()); + } + } + pw.println("}]"); + } + + public boolean isAutoclickEnabledLocked() { + return mIsAutoclickEnabled; + } + + public void setAutoclickEnabledLocked(boolean enabled) { + mIsAutoclickEnabled = enabled; + } + + public boolean isDisplayMagnificationEnabledLocked() { + return mIsDisplayMagnificationEnabled; + } + + public void setDisplayMagnificationEnabledLocked(boolean enabled) { + mIsDisplayMagnificationEnabled = enabled; + } + + public boolean isFilterKeyEventsEnabledLocked() { + return mIsFilterKeyEventsEnabled; + } + + public void setFilterKeyEventsEnabledLocked(boolean enabled) { + mIsFilterKeyEventsEnabled = enabled; + } + + public int getInteractiveUiTimeoutLocked() { + return mInteractiveUiTimeout; + } + + public void setInteractiveUiTimeoutLocked(int timeout) { + mInteractiveUiTimeout = timeout; + } + + public int getLastSentClientStateLocked() { + return mLastSentClientState; + } + + public void setLastSentClientStateLocked(int state) { + mLastSentClientState = state; + } + + /** + * Returns true if navibar magnification or shortcut key magnification is enabled. + */ + public boolean isShortcutMagnificationEnabledLocked() { + return mAccessibilityShortcutKeyTargets.contains(MAGNIFICATION_CONTROLLER_NAME) + || mAccessibilityButtonTargets.contains(MAGNIFICATION_CONTROLLER_NAME); + } + + /** + * Disable both shortcuts' magnification function. + */ + public void disableShortcutMagnificationLocked() { + mAccessibilityShortcutKeyTargets.remove(MAGNIFICATION_CONTROLLER_NAME); + mAccessibilityButtonTargets.remove(MAGNIFICATION_CONTROLLER_NAME); + } + + /** + * Returns a set which contains the flattened component names and the system class names + * assigned to the given shortcut. + * + * @param shortcutType The shortcut type. + * @return The array set of the strings + */ + public ArraySet<String> getShortcutTargetsLocked(@ShortcutType int shortcutType) { + if (shortcutType == ACCESSIBILITY_SHORTCUT_KEY) { + return mAccessibilityShortcutKeyTargets; + } else if (shortcutType == ACCESSIBILITY_BUTTON) { + return mAccessibilityButtonTargets; + } + return null; + } + + /** + * Whether or not the given shortcut target is installed in device. + * + * @param name The shortcut target name + * @return true if the shortcut target is installed. + */ + public boolean isShortcutTargetInstalledLocked(String name) { + if (TextUtils.isEmpty(name)) { + return false; + } + if (MAGNIFICATION_CONTROLLER_NAME.equals(name)) { + return true; + } + + final ComponentName componentName = ComponentName.unflattenFromString(name); + if (componentName == null) { + return false; + } + if (AccessibilityShortcutController.getFrameworkShortcutFeaturesMap() + .containsKey(componentName)) { + return true; + } + if (getInstalledServiceInfoLocked(componentName) != null) { + return true; + } + for (int i = 0; i < mInstalledShortcuts.size(); i++) { + if (mInstalledShortcuts.get(i).getComponentName().equals(componentName)) { + return true; + } + } + return false; + } + + /** + * Removes given shortcut target in the list. + * + * @param shortcutType The shortcut type. + * @param target The component name of the shortcut target. + * @return true if the shortcut target is removed. + */ + public boolean removeShortcutTargetLocked(@ShortcutType int shortcutType, + ComponentName target) { + return getShortcutTargetsLocked(shortcutType).removeIf(name -> { + ComponentName componentName; + if (name == null + || (componentName = ComponentName.unflattenFromString(name)) == null) { + return false; + } + return componentName.equals(target); + }); + } + + /** + * Returns installed accessibility service info by the given service component name. + */ + public AccessibilityServiceInfo getInstalledServiceInfoLocked(ComponentName componentName) { + for (int i = 0; i < mInstalledServices.size(); i++) { + final AccessibilityServiceInfo serviceInfo = mInstalledServices.get(i); + if (serviceInfo.getComponentName().equals(componentName)) { + return serviceInfo; + } + } + return null; + } + + /** + * Returns accessibility service connection by the given service component name. + */ + public AccessibilityServiceConnection getServiceConnectionLocked(ComponentName componentName) { + return mComponentNameToServiceMap.get(componentName); + } + + public int getNonInteractiveUiTimeoutLocked() { + return mNonInteractiveUiTimeout; + } + + public void setNonInteractiveUiTimeoutLocked(int timeout) { + mNonInteractiveUiTimeout = timeout; + } + + public boolean isPerformGesturesEnabledLocked() { + return mIsPerformGesturesEnabled; + } + + public void setPerformGesturesEnabledLocked(boolean enabled) { + mIsPerformGesturesEnabled = enabled; + } + + public boolean isAccessibilityFocusOnlyInActiveWindow() { + return mAccessibilityFocusOnlyInActiveWindow; + } + + public void setAccessibilityFocusOnlyInActiveWindow(boolean enabled) { + mAccessibilityFocusOnlyInActiveWindow = enabled; + } + public ComponentName getServiceChangingSoftKeyboardModeLocked() { + return mServiceChangingSoftKeyboardMode; + } + + public void setServiceChangingSoftKeyboardModeLocked( + ComponentName serviceChangingSoftKeyboardMode) { + mServiceChangingSoftKeyboardMode = serviceChangingSoftKeyboardMode; + } + + public boolean isTextHighContrastEnabledLocked() { + return mIsTextHighContrastEnabled; + } + + public void setTextHighContrastEnabledLocked(boolean enabled) { + mIsTextHighContrastEnabled = enabled; + } + + public boolean isTouchExplorationEnabledLocked() { + return mIsTouchExplorationEnabled; + } + + public void setTouchExplorationEnabledLocked(boolean enabled) { + mIsTouchExplorationEnabled = enabled; + } + + public boolean isServiceHandlesDoubleTapEnabledLocked() { + return mServiceHandlesDoubleTap; + } + + public void setServiceHandlesDoubleTapLocked(boolean enabled) { + mServiceHandlesDoubleTap = enabled; + } + + public boolean isMultiFingerGesturesEnabledLocked() { + return mRequestMultiFingerGestures; + } + + public void setMultiFingerGesturesLocked(boolean enabled) { + mRequestMultiFingerGestures = enabled; + } + + public int getUserInteractiveUiTimeoutLocked() { + return mUserInteractiveUiTimeout; + } + + public void setUserInteractiveUiTimeoutLocked(int timeout) { + mUserInteractiveUiTimeout = timeout; + } + + public int getUserNonInteractiveUiTimeoutLocked() { + return mUserNonInteractiveUiTimeout; + } + + public void setUserNonInteractiveUiTimeoutLocked(int timeout) { + mUserNonInteractiveUiTimeout = timeout; + } + + /** + * Gets a shortcut target which is assigned to the accessibility button by the chooser + * activity. + * + * @return The flattened component name or the system class name of the shortcut target. + */ + public String getTargetAssignedToAccessibilityButton() { + return mTargetAssignedToAccessibilityButton; + } + + /** + * Sets a shortcut target which is assigned to the accessibility button by the chooser + * activity. + * + * @param target The flattened component name or the system class name of the shortcut target. + */ + public void setTargetAssignedToAccessibilityButton(String target) { + mTargetAssignedToAccessibilityButton = target; + } + + /** + * Whether or not the given target name is contained in the shortcut collection. Since the + * component name string format could be short or long, this function un-flatten the component + * name from the string in {@code shortcutTargets} and compared with the given target name. + * + * @param shortcutTargets The shortcut type. + * @param targetName The target name. + * @return {@code true} if the target is in the shortcut collection. + */ + public static boolean doesShortcutTargetsStringContain(Collection<String> shortcutTargets, + String targetName) { + if (shortcutTargets == null || targetName == null) { + return false; + } + // Some system features, such as magnification, don't have component name. Using string + // compare first. + if (shortcutTargets.contains(targetName)) { + return true; + } + final ComponentName targetComponentName = ComponentName.unflattenFromString(targetName); + if (targetComponentName == null) { + return false; + } + for (String stringName : shortcutTargets) { + if (!TextUtils.isEmpty(stringName) + && targetComponentName.equals(ComponentName.unflattenFromString(stringName))) { + return true; + } + } + return false; + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java new file mode 100644 index 000000000000..d15c60b9501d --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java @@ -0,0 +1,1748 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility; + +import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY; +import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED; + +import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.graphics.Region; +import android.os.Binder; +import android.os.Handler; +import android.os.IBinder; +import android.os.Process; +import android.os.RemoteException; +import android.os.UserHandle; +import android.text.TextUtils; +import android.util.ArrayMap; +import android.util.Slog; +import android.util.SparseArray; +import android.view.Display; +import android.view.IWindow; +import android.view.WindowInfo; +import android.view.WindowManager; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityWindowInfo; +import android.view.accessibility.IAccessibilityInteractionConnection; + +import com.android.internal.annotations.VisibleForTesting; +import com.android.server.accessibility.AccessibilitySecurityPolicy.AccessibilityUserManager; +import com.android.server.wm.WindowManagerInternal; + +import java.io.FileDescriptor; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * This class provides APIs for accessibility manager to manage {@link AccessibilityWindowInfo}s and + * {@link WindowInfo}s. + */ +public class AccessibilityWindowManager { + private static final String LOG_TAG = "AccessibilityWindowManager"; + private static final boolean DEBUG = false; + + private static int sNextWindowId; + + private final Object mLock; + private final Handler mHandler; + private final WindowManagerInternal mWindowManagerInternal; + private final AccessibilityEventSender mAccessibilityEventSender; + private final AccessibilitySecurityPolicy mSecurityPolicy; + private final AccessibilityUserManager mAccessibilityUserManager; + + // Connections and window tokens for cross-user windows + private final SparseArray<RemoteAccessibilityConnection> + mGlobalInteractionConnections = new SparseArray<>(); + private final SparseArray<IBinder> mGlobalWindowTokens = new SparseArray<>(); + + // Connections and window tokens for per-user windows, indexed as one sparse array per user + private final SparseArray<SparseArray<RemoteAccessibilityConnection>> + mInteractionConnections = new SparseArray<>(); + private final SparseArray<SparseArray<IBinder>> mWindowTokens = new SparseArray<>(); + + private RemoteAccessibilityConnection mPictureInPictureActionReplacingConnection; + // There is only one active window in the system. It is updated when the top focused window + // of the top focused display changes and when we receive a TYPE_WINDOW_STATE_CHANGED event. + private int mActiveWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + // There is only one top focused window in the system. It is updated when the window manager + // updates the window lists. + private int mTopFocusedWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + private int mAccessibilityFocusedWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + private long mAccessibilityFocusNodeId = AccessibilityNodeInfo.UNDEFINED_ITEM_ID; + // The top focused display and window token updated with the callback of window lists change. + private int mTopFocusedDisplayId; + private IBinder mTopFocusedWindowToken; + // The display has the accessibility focused window currently. + private int mAccessibilityFocusedDisplayId = Display.INVALID_DISPLAY; + + private boolean mTouchInteractionInProgress; + + /** List of Display Windows Observer, mapping from displayId -> DisplayWindowsObserver. */ + private final SparseArray<DisplayWindowsObserver> mDisplayWindowsObservers = + new SparseArray<>(); + + /** + * Map of host view and embedded hierarchy, mapping from leash token of its ViewRootImpl. + * The key is the token from embedded hierarchy, and the value is the token from its host. + */ + private final ArrayMap<IBinder, IBinder> mHostEmbeddedMap = new ArrayMap<>(); + + /** + * Map of window id and view hierarchy. + * The key is the window id when the ViewRootImpl register to accessibility, and the value is + * its leash token. + */ + private final SparseArray<IBinder> mWindowIdMap = new SparseArray<>(); + + /** + * This class implements {@link WindowManagerInternal.WindowsForAccessibilityCallback} to + * receive {@link WindowInfo}s from window manager when there's an accessibility change in + * window and holds window lists information per display. + */ + private final class DisplayWindowsObserver implements + WindowManagerInternal.WindowsForAccessibilityCallback { + + private final int mDisplayId; + private final SparseArray<AccessibilityWindowInfo> mA11yWindowInfoById = + new SparseArray<>(); + private final SparseArray<WindowInfo> mWindowInfoById = new SparseArray<>(); + private final List<WindowInfo> mCachedWindowInfos = new ArrayList<>(); + private List<AccessibilityWindowInfo> mWindows; + private boolean mTrackingWindows = false; + private boolean mHasWatchOutsideTouchWindow; + + /** + * Constructor for DisplayWindowsObserver. + */ + DisplayWindowsObserver(int displayId) { + mDisplayId = displayId; + } + + /** + * Starts tracking windows changes from window manager by registering callback. + * + * @return true if callback registers successful. + */ + boolean startTrackingWindowsLocked() { + boolean result = true; + + if (!mTrackingWindows) { + // Turns on the flag before setup the callback. + // In some cases, onWindowsForAccessibilityChanged will be called immediately in + // setWindowsForAccessibilityCallback. We'll lost windows if flag is false. + mTrackingWindows = true; + result = mWindowManagerInternal.setWindowsForAccessibilityCallback( + mDisplayId, this); + if (!result) { + mTrackingWindows = false; + Slog.w(LOG_TAG, "set windowsObserver callbacks fail, displayId:" + + mDisplayId); + } + } + return result; + } + + /** + * Stops tracking windows changes from window manager, and clear all windows info. + */ + void stopTrackingWindowsLocked() { + if (mTrackingWindows) { + mWindowManagerInternal.setWindowsForAccessibilityCallback( + mDisplayId, null); + mTrackingWindows = false; + clearWindowsLocked(); + } + } + + /** + * Returns true if windows changes tracking. + * + * @return true if windows changes tracking + */ + boolean isTrackingWindowsLocked() { + return mTrackingWindows; + } + + /** + * Returns accessibility windows. + * @return accessibility windows. + */ + @Nullable + List<AccessibilityWindowInfo> getWindowListLocked() { + return mWindows; + } + + /** + * Returns accessibility window info according to given windowId. + * + * @param windowId The windowId + * @return The accessibility window info + */ + @Nullable + AccessibilityWindowInfo findA11yWindowInfoByIdLocked(int windowId) { + return mA11yWindowInfoById.get(windowId); + } + + /** + * Returns the window info according to given windowId. + * + * @param windowId The windowId + * @return The window info + */ + @Nullable + WindowInfo findWindowInfoByIdLocked(int windowId) { + return mWindowInfoById.get(windowId); + } + + /** + * Returns {@link AccessibilityWindowInfo} of PIP window. + * + * @return PIP accessibility window info + */ + @Nullable + AccessibilityWindowInfo getPictureInPictureWindowLocked() { + if (mWindows != null) { + final int windowCount = mWindows.size(); + for (int i = 0; i < windowCount; i++) { + final AccessibilityWindowInfo window = mWindows.get(i); + if (window.isInPictureInPictureMode()) { + return window; + } + } + } + return null; + } + + /** + * Sets the active flag of the window according to given windowId, others set to inactive. + * + * @param windowId The windowId + */ + void setActiveWindowLocked(int windowId) { + if (mWindows != null) { + final int windowCount = mWindows.size(); + for (int i = 0; i < windowCount; i++) { + AccessibilityWindowInfo window = mWindows.get(i); + if (window.getId() == windowId) { + window.setActive(true); + mAccessibilityEventSender.sendAccessibilityEventForCurrentUserLocked( + AccessibilityEvent.obtainWindowsChangedEvent(windowId, + AccessibilityEvent.WINDOWS_CHANGE_ACTIVE)); + } else { + window.setActive(false); + } + } + } + } + + /** + * Sets the window accessibility focused according to given windowId, others set + * unfocused. + * + * @param windowId The windowId + */ + void setAccessibilityFocusedWindowLocked(int windowId) { + if (mWindows != null) { + final int windowCount = mWindows.size(); + for (int i = 0; i < windowCount; i++) { + AccessibilityWindowInfo window = mWindows.get(i); + if (window.getId() == windowId) { + mAccessibilityFocusedDisplayId = mDisplayId; + window.setAccessibilityFocused(true); + mAccessibilityEventSender.sendAccessibilityEventForCurrentUserLocked( + AccessibilityEvent.obtainWindowsChangedEvent( + windowId, WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED)); + + } else { + window.setAccessibilityFocused(false); + } + } + } + } + + /** + * Computes partial interactive region of given windowId. + * + * @param windowId The windowId + * @param outRegion The output to which to write the bounds. + * @return true if outRegion is not empty. + */ + boolean computePartialInteractiveRegionForWindowLocked(int windowId, + @NonNull Region outRegion) { + if (mWindows == null) { + return false; + } + + // Windows are ordered in z order so start from the bottom and find + // the window of interest. After that all windows that cover it should + // be subtracted from the resulting region. Note that for accessibility + // we are returning only interactive windows. + Region windowInteractiveRegion = null; + boolean windowInteractiveRegionChanged = false; + + final int windowCount = mWindows.size(); + final Region currentWindowRegions = new Region(); + for (int i = windowCount - 1; i >= 0; i--) { + AccessibilityWindowInfo currentWindow = mWindows.get(i); + if (windowInteractiveRegion == null) { + if (currentWindow.getId() == windowId) { + currentWindow.getRegionInScreen(currentWindowRegions); + outRegion.set(currentWindowRegions); + windowInteractiveRegion = outRegion; + continue; + } + } else if (currentWindow.getType() + != AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) { + currentWindow.getRegionInScreen(currentWindowRegions); + if (windowInteractiveRegion.op(currentWindowRegions, Region.Op.DIFFERENCE)) { + windowInteractiveRegionChanged = true; + } + } + } + + return windowInteractiveRegionChanged; + } + + List<Integer> getWatchOutsideTouchWindowIdLocked(int targetWindowId) { + final WindowInfo targetWindow = mWindowInfoById.get(targetWindowId); + if (targetWindow != null && mHasWatchOutsideTouchWindow) { + final List<Integer> outsideWindowsId = new ArrayList<>(); + for (int i = 0; i < mWindowInfoById.size(); i++) { + final WindowInfo window = mWindowInfoById.valueAt(i); + if (window != null && window.layer < targetWindow.layer + && window.hasFlagWatchOutsideTouch) { + outsideWindowsId.add(mWindowInfoById.keyAt(i)); + } + } + return outsideWindowsId; + } + return Collections.emptyList(); + } + + /** + * Callbacks from window manager when there's an accessibility change in windows. + * + * @param forceSend Send the windows for accessibility even if they haven't changed. + * @param topFocusedDisplayId The display Id which has the top focused window. + * @param topFocusedWindowToken The window token of top focused window. + * @param windows The windows for accessibility. + */ + @Override + public void onWindowsForAccessibilityChanged(boolean forceSend, int topFocusedDisplayId, + IBinder topFocusedWindowToken, @NonNull List<WindowInfo> windows) { + synchronized (mLock) { + if (DEBUG) { + Slog.i(LOG_TAG, "Display Id = " + mDisplayId); + Slog.i(LOG_TAG, "Windows changed: " + windows); + } + if (shouldUpdateWindowsLocked(forceSend, windows)) { + mTopFocusedDisplayId = topFocusedDisplayId; + mTopFocusedWindowToken = topFocusedWindowToken; + cacheWindows(windows); + // Lets the policy update the focused and active windows. + updateWindowsLocked(mAccessibilityUserManager.getCurrentUserIdLocked(), + windows); + // Someone may be waiting for the windows - advertise it. + mLock.notifyAll(); + } + } + } + + private boolean shouldUpdateWindowsLocked(boolean forceSend, + @NonNull List<WindowInfo> windows) { + if (forceSend) { + return true; + } + + final int windowCount = windows.size(); + // We computed the windows and if they changed notify the client. + if (mCachedWindowInfos.size() != windowCount) { + // Different size means something changed. + return true; + } else if (!mCachedWindowInfos.isEmpty() || !windows.isEmpty()) { + // Since we always traverse windows from high to low layer + // the old and new windows at the same index should be the + // same, otherwise something changed. + for (int i = 0; i < windowCount; i++) { + WindowInfo oldWindow = mCachedWindowInfos.get(i); + WindowInfo newWindow = windows.get(i); + // We do not care for layer changes given the window + // order does not change. This brings no new information + // to the clients. + if (windowChangedNoLayer(oldWindow, newWindow)) { + return true; + } + } + } + + return false; + } + + private void cacheWindows(List<WindowInfo> windows) { + final int oldWindowCount = mCachedWindowInfos.size(); + for (int i = oldWindowCount - 1; i >= 0; i--) { + mCachedWindowInfos.remove(i).recycle(); + } + final int newWindowCount = windows.size(); + for (int i = 0; i < newWindowCount; i++) { + WindowInfo newWindow = windows.get(i); + mCachedWindowInfos.add(WindowInfo.obtain(newWindow)); + } + } + + private boolean windowChangedNoLayer(WindowInfo oldWindow, WindowInfo newWindow) { + if (oldWindow == newWindow) { + return false; + } + if (oldWindow == null) { + return true; + } + if (newWindow == null) { + return true; + } + if (oldWindow.type != newWindow.type) { + return true; + } + if (oldWindow.focused != newWindow.focused) { + return true; + } + if (oldWindow.token == null) { + if (newWindow.token != null) { + return true; + } + } else if (!oldWindow.token.equals(newWindow.token)) { + return true; + } + if (oldWindow.parentToken == null) { + if (newWindow.parentToken != null) { + return true; + } + } else if (!oldWindow.parentToken.equals(newWindow.parentToken)) { + return true; + } + if (oldWindow.activityToken == null) { + if (newWindow.activityToken != null) { + return true; + } + } else if (!oldWindow.activityToken.equals(newWindow.activityToken)) { + return true; + } + if (!oldWindow.regionInScreen.equals(newWindow.regionInScreen)) { + return true; + } + if (oldWindow.childTokens != null && newWindow.childTokens != null + && !oldWindow.childTokens.equals(newWindow.childTokens)) { + return true; + } + if (!TextUtils.equals(oldWindow.title, newWindow.title)) { + return true; + } + if (oldWindow.accessibilityIdOfAnchor != newWindow.accessibilityIdOfAnchor) { + return true; + } + if (oldWindow.inPictureInPicture != newWindow.inPictureInPicture) { + return true; + } + if (oldWindow.hasFlagWatchOutsideTouch != newWindow.hasFlagWatchOutsideTouch) { + return true; + } + if (oldWindow.displayId != newWindow.displayId) { + return true; + } + return false; + } + + /** + * Clears all {@link AccessibilityWindowInfo}s and {@link WindowInfo}s. + */ + private void clearWindowsLocked() { + final List<WindowInfo> windows = Collections.emptyList(); + final int activeWindowId = mActiveWindowId; + // UserId is useless in updateWindowsLocked, when we update a empty window list. + // Just pass current userId here. + updateWindowsLocked(mAccessibilityUserManager.getCurrentUserIdLocked(), windows); + // Do not reset mActiveWindowId here. mActiveWindowId will be clear after accessibility + // interaction connection removed. + mActiveWindowId = activeWindowId; + mWindows = null; + } + + /** + * Updates windows info according to specified userId and windows. + * + * @param userId The userId to update + * @param windows The windows to update + */ + private void updateWindowsLocked(int userId, @NonNull List<WindowInfo> windows) { + if (mWindows == null) { + mWindows = new ArrayList<>(); + } + + final List<AccessibilityWindowInfo> oldWindowList = new ArrayList<>(mWindows); + final SparseArray<AccessibilityWindowInfo> oldWindowsById = mA11yWindowInfoById.clone(); + boolean shouldClearAccessibilityFocus = false; + + mWindows.clear(); + mA11yWindowInfoById.clear(); + + for (int i = 0; i < mWindowInfoById.size(); i++) { + mWindowInfoById.valueAt(i).recycle(); + } + mWindowInfoById.clear(); + mHasWatchOutsideTouchWindow = false; + + final int windowCount = windows.size(); + final boolean isTopFocusedDisplay = mDisplayId == mTopFocusedDisplayId; + final boolean isAccessibilityFocusedDisplay = + mDisplayId == mAccessibilityFocusedDisplayId; + // Modifies the value of top focused window, active window and a11y focused window + // only if this display is top focused display which has the top focused window. + if (isTopFocusedDisplay) { + if (windowCount > 0) { + // Sets the top focus window by top focused window token. + mTopFocusedWindowId = findWindowIdLocked(userId, mTopFocusedWindowToken); + } else { + // Resets the top focus window when stopping tracking window of this display. + mTopFocusedWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + } + // The active window doesn't need to be reset if the touch operation is progressing. + if (!mTouchInteractionInProgress) { + mActiveWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + } + } + + // If the active window goes away while the user is touch exploring we + // reset the active window id and wait for the next hover event from + // under the user's finger to determine which one is the new one. It + // is possible that the finger is not moving and the input system + // filters out such events. + boolean activeWindowGone = true; + + // We'll clear accessibility focus if the window with focus is no longer visible to + // accessibility services. + if (isAccessibilityFocusedDisplay) { + shouldClearAccessibilityFocus = mAccessibilityFocusedWindowId + != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + } + if (windowCount > 0) { + for (int i = 0; i < windowCount; i++) { + final WindowInfo windowInfo = windows.get(i); + final AccessibilityWindowInfo window; + if (mTrackingWindows) { + window = populateReportedWindowLocked(userId, windowInfo); + } else { + window = null; + } + if (window != null) { + + // Flip layers in list to be consistent with AccessibilityService#getWindows + window.setLayer(windowCount - 1 - window.getLayer()); + + final int windowId = window.getId(); + if (window.isFocused() && isTopFocusedDisplay) { + if (!mTouchInteractionInProgress) { + // This display is top one, and sets the focus window + // as active window. + mActiveWindowId = windowId; + window.setActive(true); + } else if (windowId == mActiveWindowId) { + activeWindowGone = false; + } + } + if (!mHasWatchOutsideTouchWindow && windowInfo.hasFlagWatchOutsideTouch) { + mHasWatchOutsideTouchWindow = true; + } + mWindows.add(window); + mA11yWindowInfoById.put(windowId, window); + mWindowInfoById.put(windowId, WindowInfo.obtain(windowInfo)); + } + } + final int accessibilityWindowCount = mWindows.size(); + if (isTopFocusedDisplay) { + if (mTouchInteractionInProgress && activeWindowGone) { + mActiveWindowId = mTopFocusedWindowId; + } + // Focused window may change the active one, so set the + // active window once we decided which it is. + for (int i = 0; i < accessibilityWindowCount; i++) { + final AccessibilityWindowInfo window = mWindows.get(i); + if (window.getId() == mActiveWindowId) { + window.setActive(true); + } + } + } + if (isAccessibilityFocusedDisplay) { + for (int i = 0; i < accessibilityWindowCount; i++) { + final AccessibilityWindowInfo window = mWindows.get(i); + if (window.getId() == mAccessibilityFocusedWindowId) { + window.setAccessibilityFocused(true); + shouldClearAccessibilityFocus = false; + break; + } + } + } + } + + sendEventsForChangedWindowsLocked(oldWindowList, oldWindowsById); + + final int oldWindowCount = oldWindowList.size(); + for (int i = oldWindowCount - 1; i >= 0; i--) { + oldWindowList.remove(i).recycle(); + } + + if (shouldClearAccessibilityFocus) { + clearAccessibilityFocusLocked(mAccessibilityFocusedWindowId); + } + } + + private void sendEventsForChangedWindowsLocked(List<AccessibilityWindowInfo> oldWindows, + SparseArray<AccessibilityWindowInfo> oldWindowsById) { + List<AccessibilityEvent> events = new ArrayList<>(); + // Sends events for all removed windows. + final int oldWindowsCount = oldWindows.size(); + for (int i = 0; i < oldWindowsCount; i++) { + final AccessibilityWindowInfo window = oldWindows.get(i); + if (mA11yWindowInfoById.get(window.getId()) == null) { + events.add(AccessibilityEvent.obtainWindowsChangedEvent( + window.getId(), AccessibilityEvent.WINDOWS_CHANGE_REMOVED)); + } + } + + // Looks for other changes. + final int newWindowCount = mWindows.size(); + for (int i = 0; i < newWindowCount; i++) { + final AccessibilityWindowInfo newWindow = mWindows.get(i); + final AccessibilityWindowInfo oldWindow = oldWindowsById.get(newWindow.getId()); + if (oldWindow == null) { + events.add(AccessibilityEvent.obtainWindowsChangedEvent( + newWindow.getId(), AccessibilityEvent.WINDOWS_CHANGE_ADDED)); + } else { + int changes = newWindow.differenceFrom(oldWindow); + if (changes != 0) { + events.add(AccessibilityEvent.obtainWindowsChangedEvent( + newWindow.getId(), changes)); + } + } + } + + final int numEvents = events.size(); + for (int i = 0; i < numEvents; i++) { + mAccessibilityEventSender.sendAccessibilityEventForCurrentUserLocked(events.get(i)); + } + } + + private AccessibilityWindowInfo populateReportedWindowLocked(int userId, + WindowInfo window) { + final int windowId = findWindowIdLocked(userId, window.token); + if (windowId < 0) { + return null; + } + + final AccessibilityWindowInfo reportedWindow = AccessibilityWindowInfo.obtain(); + + reportedWindow.setId(windowId); + reportedWindow.setType(getTypeForWindowManagerWindowType(window.type)); + reportedWindow.setLayer(window.layer); + reportedWindow.setFocused(window.focused); + reportedWindow.setRegionInScreen(window.regionInScreen); + reportedWindow.setTitle(window.title); + reportedWindow.setAnchorId(window.accessibilityIdOfAnchor); + reportedWindow.setPictureInPicture(window.inPictureInPicture); + reportedWindow.setDisplayId(window.displayId); + + final int parentId = findWindowIdLocked(userId, window.parentToken); + if (parentId >= 0) { + reportedWindow.setParentId(parentId); + } + + if (window.childTokens != null) { + final int childCount = window.childTokens.size(); + for (int i = 0; i < childCount; i++) { + final IBinder childToken = window.childTokens.get(i); + final int childId = findWindowIdLocked(userId, childToken); + if (childId >= 0) { + reportedWindow.addChild(childId); + } + } + } + + return reportedWindow; + } + + private int getTypeForWindowManagerWindowType(int windowType) { + switch (windowType) { + case WindowManager.LayoutParams.TYPE_APPLICATION: + case WindowManager.LayoutParams.TYPE_APPLICATION_MEDIA: + case WindowManager.LayoutParams.TYPE_APPLICATION_PANEL: + case WindowManager.LayoutParams.TYPE_APPLICATION_STARTING: + case WindowManager.LayoutParams.TYPE_APPLICATION_SUB_PANEL: + case WindowManager.LayoutParams.TYPE_APPLICATION_ABOVE_SUB_PANEL: + case WindowManager.LayoutParams.TYPE_BASE_APPLICATION: + case WindowManager.LayoutParams.TYPE_DRAWN_APPLICATION: + case WindowManager.LayoutParams.TYPE_PHONE: + case WindowManager.LayoutParams.TYPE_PRIORITY_PHONE: + case WindowManager.LayoutParams.TYPE_TOAST: + case WindowManager.LayoutParams.TYPE_APPLICATION_ATTACHED_DIALOG: { + return AccessibilityWindowInfo.TYPE_APPLICATION; + } + + case WindowManager.LayoutParams.TYPE_INPUT_METHOD: + case WindowManager.LayoutParams.TYPE_INPUT_METHOD_DIALOG: { + return AccessibilityWindowInfo.TYPE_INPUT_METHOD; + } + + case WindowManager.LayoutParams.TYPE_KEYGUARD_DIALOG: + case WindowManager.LayoutParams.TYPE_NAVIGATION_BAR: + case WindowManager.LayoutParams.TYPE_NAVIGATION_BAR_PANEL: + case WindowManager.LayoutParams.TYPE_SEARCH_BAR: + case WindowManager.LayoutParams.TYPE_STATUS_BAR: + case WindowManager.LayoutParams.TYPE_NOTIFICATION_SHADE: + case WindowManager.LayoutParams.TYPE_STATUS_BAR_ADDITIONAL: + case WindowManager.LayoutParams.TYPE_STATUS_BAR_SUB_PANEL: + case WindowManager.LayoutParams.TYPE_VOLUME_OVERLAY: + case WindowManager.LayoutParams.TYPE_SYSTEM_ALERT: + case WindowManager.LayoutParams.TYPE_SYSTEM_DIALOG: + case WindowManager.LayoutParams.TYPE_SYSTEM_ERROR: + case WindowManager.LayoutParams.TYPE_SYSTEM_OVERLAY: + case WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY: + case WindowManager.LayoutParams.TYPE_SCREENSHOT: + case WindowManager.LayoutParams.TYPE_TRUSTED_APPLICATION_OVERLAY: { + return AccessibilityWindowInfo.TYPE_SYSTEM; + } + + case WindowManager.LayoutParams.TYPE_DOCK_DIVIDER: { + return AccessibilityWindowInfo.TYPE_SPLIT_SCREEN_DIVIDER; + } + + case TYPE_ACCESSIBILITY_OVERLAY: { + return AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY; + } + + default: { + return -1; + } + } + } + + /** + * Dumps all {@link AccessibilityWindowInfo}s here. + */ + void dumpLocked(FileDescriptor fd, final PrintWriter pw, String[] args) { + if (mWindows != null) { + final int windowCount = mWindows.size(); + for (int j = 0; j < windowCount; j++) { + if (j == 0) { + pw.append("Display["); + pw.append(Integer.toString(mDisplayId)); + pw.append("] : "); + pw.println(); + } + if (j > 0) { + pw.append(','); + pw.println(); + } + pw.append("Window["); + AccessibilityWindowInfo window = mWindows.get(j); + pw.append(window.toString()); + pw.append(']'); + } + pw.println(); + } + } + } + /** + * Interface to send {@link AccessibilityEvent}. + */ + public interface AccessibilityEventSender { + /** + * Sends {@link AccessibilityEvent} for current user. + */ + void sendAccessibilityEventForCurrentUserLocked(AccessibilityEvent event); + } + + /** + * Wrapper of accessibility interaction connection for window. + */ + // In order to avoid using DexmakerShareClassLoaderRule, make this class visible for testing. + @VisibleForTesting(visibility = VisibleForTesting.Visibility.PACKAGE) + public final class RemoteAccessibilityConnection implements IBinder.DeathRecipient { + private final int mUid; + private final String mPackageName; + private final int mWindowId; + private final int mUserId; + private final IAccessibilityInteractionConnection mConnection; + + RemoteAccessibilityConnection(int windowId, + IAccessibilityInteractionConnection connection, + String packageName, int uid, int userId) { + mWindowId = windowId; + mPackageName = packageName; + mUid = uid; + mUserId = userId; + mConnection = connection; + } + + int getUid() { + return mUid; + } + + String getPackageName() { + return mPackageName; + } + + IAccessibilityInteractionConnection getRemote() { + return mConnection; + } + + void linkToDeath() throws RemoteException { + mConnection.asBinder().linkToDeath(this, 0); + } + + void unlinkToDeath() { + mConnection.asBinder().unlinkToDeath(this, 0); + } + + @Override + public void binderDied() { + unlinkToDeath(); + synchronized (mLock) { + removeAccessibilityInteractionConnectionLocked(mWindowId, mUserId); + } + } + } + + /** + * Constructor for AccessibilityManagerService. + */ + public AccessibilityWindowManager(@NonNull Object lock, @NonNull Handler handler, + @NonNull WindowManagerInternal windowManagerInternal, + @NonNull AccessibilityEventSender accessibilityEventSender, + @NonNull AccessibilitySecurityPolicy securityPolicy, + @NonNull AccessibilityUserManager accessibilityUserManager) { + mLock = lock; + mHandler = handler; + mWindowManagerInternal = windowManagerInternal; + mAccessibilityEventSender = accessibilityEventSender; + mSecurityPolicy = securityPolicy; + mAccessibilityUserManager = accessibilityUserManager; + } + + /** + * Starts tracking windows changes from window manager for specified display. + * + * @param displayId The logical display id. + */ + public void startTrackingWindows(int displayId) { + synchronized (mLock) { + DisplayWindowsObserver observer = mDisplayWindowsObservers.get(displayId); + if (observer == null) { + observer = new DisplayWindowsObserver(displayId); + } + if (observer.isTrackingWindowsLocked()) { + return; + } + if (observer.startTrackingWindowsLocked()) { + mDisplayWindowsObservers.put(displayId, observer); + } + } + } + + /** + * Stops tracking windows changes from window manager, and clear all windows info for specified + * display. + * + * @param displayId The logical display id. + */ + public void stopTrackingWindows(int displayId) { + synchronized (mLock) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.get(displayId); + if (observer != null) { + observer.stopTrackingWindowsLocked(); + mDisplayWindowsObservers.remove(displayId); + } + } + } + + /** + * Checks if we are tracking windows on any display. + * + * @return {@code true} if the observer is tracking windows on any display, + * {@code false} otherwise. + */ + public boolean isTrackingWindowsLocked() { + final int count = mDisplayWindowsObservers.size(); + if (count > 0) { + return true; + } + return false; + } + + /** + * Checks if we are tracking windows on specified display. + * + * @param displayId The logical display id. + * @return {@code true} if the observer is tracking windows on specified display, + * {@code false} otherwise. + */ + public boolean isTrackingWindowsLocked(int displayId) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.get(displayId); + if (observer != null) { + return observer.isTrackingWindowsLocked(); + } + return false; + } + + /** + * Returns accessibility windows for specified display. + * + * @param displayId The logical display id. + * @return accessibility windows for specified display. + */ + @Nullable + public List<AccessibilityWindowInfo> getWindowListLocked(int displayId) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.get(displayId); + if (observer != null) { + return observer.getWindowListLocked(); + } + return null; + } + + /** + * Adds accessibility interaction connection according to given window token, package name and + * window token. + * + * @param window The window token of accessibility interaction connection + * @param leashToken The leash token of accessibility interaction connection + * @param connection The accessibility interaction connection + * @param packageName The package name + * @param userId The userId + * @return The windowId of added connection + * @throws RemoteException + */ + public int addAccessibilityInteractionConnection(@NonNull IWindow window, + @NonNull IBinder leashToken, @NonNull IAccessibilityInteractionConnection connection, + @NonNull String packageName, int userId) throws RemoteException { + final int windowId; + boolean shouldComputeWindows = false; + final IBinder token = window.asBinder(); + final int displayId = mWindowManagerInternal.getDisplayIdForWindow(token); + synchronized (mLock) { + // We treat calls from a profile as if made by its parent as profiles + // share the accessibility state of the parent. The call below + // performs the current profile parent resolution. + final int resolvedUserId = mSecurityPolicy + .resolveCallingUserIdEnforcingPermissionsLocked(userId); + final int resolvedUid = UserHandle.getUid(resolvedUserId, UserHandle.getCallingAppId()); + + // Makes sure the reported package is one the caller has access to. + packageName = mSecurityPolicy.resolveValidReportedPackageLocked( + packageName, UserHandle.getCallingAppId(), resolvedUserId, + Binder.getCallingPid()); + + windowId = sNextWindowId++; + // If the window is from a process that runs across users such as + // the system UI or the system we add it to the global state that + // is shared across users. + if (mSecurityPolicy.isCallerInteractingAcrossUsers(userId)) { + RemoteAccessibilityConnection wrapper = new RemoteAccessibilityConnection( + windowId, connection, packageName, resolvedUid, UserHandle.USER_ALL); + wrapper.linkToDeath(); + mGlobalInteractionConnections.put(windowId, wrapper); + mGlobalWindowTokens.put(windowId, token); + if (DEBUG) { + Slog.i(LOG_TAG, "Added global connection for pid:" + Binder.getCallingPid() + + " with windowId: " + windowId + " and token: " + token); + } + } else { + RemoteAccessibilityConnection wrapper = new RemoteAccessibilityConnection( + windowId, connection, packageName, resolvedUid, resolvedUserId); + wrapper.linkToDeath(); + getInteractionConnectionsForUserLocked(resolvedUserId).put(windowId, wrapper); + getWindowTokensForUserLocked(resolvedUserId).put(windowId, token); + if (DEBUG) { + Slog.i(LOG_TAG, "Added user connection for pid:" + Binder.getCallingPid() + + " with windowId: " + windowId + " and token: " + token); + } + } + + if (isTrackingWindowsLocked(displayId)) { + shouldComputeWindows = true; + } + registerIdLocked(leashToken, windowId); + } + if (shouldComputeWindows) { + mWindowManagerInternal.computeWindowsForAccessibility(displayId); + } + + mWindowManagerInternal.setAccessibilityIdToSurfaceMetadata(token, windowId); + return windowId; + } + + /** + * Removes accessibility interaction connection according to given window token. + * + * @param window The window token of accessibility interaction connection + */ + public void removeAccessibilityInteractionConnection(@NonNull IWindow window) { + synchronized (mLock) { + // We treat calls from a profile as if made by its parent as profiles + // share the accessibility state of the parent. The call below + // performs the current profile parent resolution. + mSecurityPolicy.resolveCallingUserIdEnforcingPermissionsLocked( + UserHandle.getCallingUserId()); + IBinder token = window.asBinder(); + final int removedWindowId = removeAccessibilityInteractionConnectionInternalLocked( + token, mGlobalWindowTokens, mGlobalInteractionConnections); + if (removedWindowId >= 0) { + onAccessibilityInteractionConnectionRemovedLocked(removedWindowId, token); + if (DEBUG) { + Slog.i(LOG_TAG, "Removed global connection for pid:" + Binder.getCallingPid() + + " with windowId: " + removedWindowId + " and token: " + + window.asBinder()); + } + return; + } + final int userCount = mWindowTokens.size(); + for (int i = 0; i < userCount; i++) { + final int userId = mWindowTokens.keyAt(i); + final int removedWindowIdForUser = + removeAccessibilityInteractionConnectionInternalLocked(token, + getWindowTokensForUserLocked(userId), + getInteractionConnectionsForUserLocked(userId)); + if (removedWindowIdForUser >= 0) { + onAccessibilityInteractionConnectionRemovedLocked( + removedWindowIdForUser, token); + if (DEBUG) { + Slog.i(LOG_TAG, "Removed user connection for pid:" + Binder.getCallingPid() + + " with windowId: " + removedWindowIdForUser + " and userId:" + + userId + " and token: " + window.asBinder()); + } + return; + } + } + } + } + + /** + * Resolves a connection wrapper for a window id. + * + * @param userId The user id for any user-specific windows + * @param windowId The id of the window of interest + * + * @return a connection to the window + */ + @Nullable + public RemoteAccessibilityConnection getConnectionLocked(int userId, int windowId) { + if (DEBUG) { + Slog.i(LOG_TAG, "Trying to get interaction connection to windowId: " + windowId); + } + RemoteAccessibilityConnection connection = mGlobalInteractionConnections.get(windowId); + if (connection == null && isValidUserForInteractionConnectionsLocked(userId)) { + connection = getInteractionConnectionsForUserLocked(userId).get(windowId); + } + if (connection != null && connection.getRemote() != null) { + return connection; + } + if (DEBUG) { + Slog.e(LOG_TAG, "No interaction connection to window: " + windowId); + } + return null; + } + + private int removeAccessibilityInteractionConnectionInternalLocked(IBinder windowToken, + SparseArray<IBinder> windowTokens, SparseArray<RemoteAccessibilityConnection> + interactionConnections) { + final int count = windowTokens.size(); + for (int i = 0; i < count; i++) { + if (windowTokens.valueAt(i) == windowToken) { + final int windowId = windowTokens.keyAt(i); + windowTokens.removeAt(i); + RemoteAccessibilityConnection wrapper = interactionConnections.get(windowId); + wrapper.unlinkToDeath(); + interactionConnections.remove(windowId); + return windowId; + } + } + return -1; + } + + /** + * Removes accessibility interaction connection according to given windowId and userId. + * + * @param windowId The windowId of accessibility interaction connection + * @param userId The userId to remove + */ + private void removeAccessibilityInteractionConnectionLocked(int windowId, int userId) { + IBinder window = null; + if (userId == UserHandle.USER_ALL) { + window = mGlobalWindowTokens.get(windowId); + mGlobalWindowTokens.remove(windowId); + mGlobalInteractionConnections.remove(windowId); + } else { + if (isValidUserForWindowTokensLocked(userId)) { + window = getWindowTokensForUserLocked(userId).get(windowId); + getWindowTokensForUserLocked(userId).remove(windowId); + } + if (isValidUserForInteractionConnectionsLocked(userId)) { + getInteractionConnectionsForUserLocked(userId).remove(windowId); + } + } + onAccessibilityInteractionConnectionRemovedLocked(windowId, window); + if (DEBUG) { + Slog.i(LOG_TAG, "Removing interaction connection to windowId: " + windowId); + } + } + + /** + * Invoked when accessibility interaction connection of window is removed. + * + * @param windowId Removed windowId + * @param binder Removed window token + */ + private void onAccessibilityInteractionConnectionRemovedLocked( + int windowId, @Nullable IBinder binder) { + // Active window will not update, if windows callback is unregistered. + // Update active window to invalid, when its a11y interaction connection is removed. + if (!isTrackingWindowsLocked() && windowId >= 0 && mActiveWindowId == windowId) { + mActiveWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + } + if (binder != null) { + mWindowManagerInternal.setAccessibilityIdToSurfaceMetadata( + binder, AccessibilityWindowInfo.UNDEFINED_WINDOW_ID); + } + unregisterIdLocked(windowId); + } + + /** + * Gets window token according to given userId and windowId. + * + * @param userId The userId + * @param windowId The windowId + * @return The window token + */ + @Nullable + public IBinder getWindowTokenForUserAndWindowIdLocked(int userId, int windowId) { + IBinder windowToken = mGlobalWindowTokens.get(windowId); + if (windowToken == null && isValidUserForWindowTokensLocked(userId)) { + windowToken = getWindowTokensForUserLocked(userId).get(windowId); + } + return windowToken; + } + + /** + * Returns the userId that owns the given window token, {@link UserHandle#USER_NULL} + * if not found. + * + * @param windowToken The window token + * @return The userId + */ + public int getWindowOwnerUserId(@NonNull IBinder windowToken) { + return mWindowManagerInternal.getWindowOwnerUserId(windowToken); + } + + /** + * Returns windowId of given userId and window token. + * + * @param userId The userId + * @param token The window token + * @return The windowId + */ + public int findWindowIdLocked(int userId, @NonNull IBinder token) { + final int globalIndex = mGlobalWindowTokens.indexOfValue(token); + if (globalIndex >= 0) { + return mGlobalWindowTokens.keyAt(globalIndex); + } + if (isValidUserForWindowTokensLocked(userId)) { + final int userIndex = getWindowTokensForUserLocked(userId).indexOfValue(token); + if (userIndex >= 0) { + return getWindowTokensForUserLocked(userId).keyAt(userIndex); + } + } + return -1; + } + + /** + * Establish the relationship between the host and the embedded view hierarchy. + * + * @param host The token of host hierarchy + * @param embedded The token of the embedded hierarchy + */ + public void associateEmbeddedHierarchyLocked(@NonNull IBinder host, @NonNull IBinder embedded) { + // Use embedded window as key, since one host window may have multiple embedded windows. + associateLocked(embedded, host); + } + + /** + * Clear the relationship by given token. + * + * @param token The token + */ + public void disassociateEmbeddedHierarchyLocked(@NonNull IBinder token) { + disassociateLocked(token); + } + + /** + * Gets the parent windowId of the window according to the specified windowId. + * + * @param windowId The windowId to check + * @return The windowId of the parent window, or self if no parent exists + */ + public int resolveParentWindowIdLocked(int windowId) { + final IBinder token = getTokenLocked(windowId); + if (token == null) { + return windowId; + } + final IBinder resolvedToken = resolveTopParentTokenLocked(token); + final int resolvedWindowId = getWindowIdLocked(resolvedToken); + return resolvedWindowId != -1 ? resolvedWindowId : windowId; + } + + private IBinder resolveTopParentTokenLocked(IBinder token) { + final IBinder hostToken = getHostTokenLocked(token); + if (hostToken == null) { + return token; + } + return resolveTopParentTokenLocked(hostToken); + } + + /** + * Computes partial interactive region of given windowId. + * + * @param windowId The windowId + * @param outRegion The output to which to write the bounds. + * @return true if outRegion is not empty. + */ + public boolean computePartialInteractiveRegionForWindowLocked(int windowId, + @NonNull Region outRegion) { + windowId = resolveParentWindowIdLocked(windowId); + final DisplayWindowsObserver observer = getDisplayWindowObserverByWindowIdLocked(windowId); + if (observer != null) { + return observer.computePartialInteractiveRegionForWindowLocked(windowId, outRegion); + } + + return false; + } + + /** + * Updates active windowId and accessibility focused windowId according to given accessibility + * event and action. + * + * @param userId The userId + * @param windowId The windowId of accessibility event + * @param nodeId The accessibility node id of accessibility event + * @param eventType The accessibility event type + * @param eventAction The accessibility event action + */ + public void updateActiveAndAccessibilityFocusedWindowLocked(int userId, int windowId, + long nodeId, int eventType, int eventAction) { + // The active window is either the window that has input focus or + // the window that the user is currently touching. If the user is + // touching a window that does not have input focus as soon as the + // the user stops touching that window the focused window becomes + // the active one. Here we detect the touched window and make it + // active. In updateWindowsLocked() we update the focused window + // and if the user is not touching the screen, we make the focused + // window the active one. + switch (eventType) { + case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: { + // If no service has the capability to introspect screen, + // we do not register callback in the window manager for + // window changes, so we have to ask the window manager + // what the focused window is to update the active one. + // The active window also determined events from which + // windows are delivered. + synchronized (mLock) { + if (!isTrackingWindowsLocked()) { + mTopFocusedWindowId = findFocusedWindowId(userId); + if (windowId == mTopFocusedWindowId) { + mActiveWindowId = windowId; + } + } + } + } break; + + case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER: { + // Do not allow delayed hover events to confuse us + // which the active window is. + synchronized (mLock) { + if (mTouchInteractionInProgress && mActiveWindowId != windowId) { + setActiveWindowLocked(windowId); + } + } + } break; + + case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED: { + synchronized (mLock) { + if (mAccessibilityFocusedWindowId != windowId) { + clearAccessibilityFocusLocked(mAccessibilityFocusedWindowId); + setAccessibilityFocusedWindowLocked(windowId); + } + mAccessibilityFocusNodeId = nodeId; + } + } break; + + case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED: { + synchronized (mLock) { + if (mAccessibilityFocusNodeId == nodeId) { + mAccessibilityFocusNodeId = AccessibilityNodeInfo.UNDEFINED_ITEM_ID; + } + // Clear the window with focus if it no longer has focus and we aren't + // just moving focus from one view to the other in the same window. + if ((mAccessibilityFocusNodeId == AccessibilityNodeInfo.UNDEFINED_ITEM_ID) + && (mAccessibilityFocusedWindowId == windowId) + && (eventAction != AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS)) { + mAccessibilityFocusedWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + mAccessibilityFocusedDisplayId = Display.INVALID_DISPLAY; + } + } + } break; + } + } + + /** + * Callbacks from AccessibilityManagerService when touch explorer turn on and + * motion down detected. + */ + public void onTouchInteractionStart() { + synchronized (mLock) { + mTouchInteractionInProgress = true; + } + } + + /** + * Callbacks from AccessibilityManagerService when touch explorer turn on and + * gesture or motion up detected. + */ + public void onTouchInteractionEnd() { + synchronized (mLock) { + mTouchInteractionInProgress = false; + // We want to set the active window to be current immediately + // after the user has stopped touching the screen since if the + // user types with the IME he should get a feedback for the + // letter typed in the text view which is in the input focused + // window. Note that we always deliver hover accessibility events + // (they are a result of user touching the screen) so change of + // the active window before all hover accessibility events from + // the touched window are delivered is fine. + final int oldActiveWindow = mActiveWindowId; + setActiveWindowLocked(mTopFocusedWindowId); + + // If there is no service that can operate with interactive windows + // then we keep the old behavior where a window loses accessibility + // focus if it is no longer active. This still changes the behavior + // for services that do not operate with interactive windows and run + // at the same time as the one(s) which does. In practice however, + // there is only one service that uses accessibility focus and it + // is typically the one that operates with interactive windows, So, + // this is fine. Note that to allow a service to work across windows + // we have to allow accessibility focus stay in any of them. Sigh... + final boolean accessibilityFocusOnlyInActiveWindow = !isTrackingWindowsLocked(); + if (oldActiveWindow != mActiveWindowId + && mAccessibilityFocusedWindowId == oldActiveWindow + && accessibilityFocusOnlyInActiveWindow) { + clearAccessibilityFocusLocked(oldActiveWindow); + } + } + } + + /** + * Gets the id of the current active window. + * + * @return The userId + */ + public int getActiveWindowId(int userId) { + if (mActiveWindowId == AccessibilityWindowInfo.UNDEFINED_WINDOW_ID + && !mTouchInteractionInProgress) { + mActiveWindowId = findFocusedWindowId(userId); + } + return mActiveWindowId; + } + + private void setActiveWindowLocked(int windowId) { + if (mActiveWindowId != windowId) { + mAccessibilityEventSender.sendAccessibilityEventForCurrentUserLocked( + AccessibilityEvent.obtainWindowsChangedEvent( + mActiveWindowId, AccessibilityEvent.WINDOWS_CHANGE_ACTIVE)); + + mActiveWindowId = windowId; + // Goes through all windows for each display. + final int count = mDisplayWindowsObservers.size(); + for (int i = 0; i < count; i++) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i); + if (observer != null) { + observer.setActiveWindowLocked(windowId); + } + } + } + } + + private void setAccessibilityFocusedWindowLocked(int windowId) { + if (mAccessibilityFocusedWindowId != windowId) { + mAccessibilityEventSender.sendAccessibilityEventForCurrentUserLocked( + AccessibilityEvent.obtainWindowsChangedEvent( + mAccessibilityFocusedWindowId, + WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED)); + + mAccessibilityFocusedWindowId = windowId; + // Goes through all windows for each display. + final int count = mDisplayWindowsObservers.size(); + for (int i = 0; i < count; i++) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i); + if (observer != null) { + observer.setAccessibilityFocusedWindowLocked(windowId); + } + } + } + } + + /** + * Returns accessibility window info according to given windowId. + * + * @param windowId The windowId + * @return The accessibility window info + */ + @Nullable + public AccessibilityWindowInfo findA11yWindowInfoByIdLocked(int windowId) { + windowId = resolveParentWindowIdLocked(windowId); + final DisplayWindowsObserver observer = getDisplayWindowObserverByWindowIdLocked(windowId); + if (observer != null) { + return observer.findA11yWindowInfoByIdLocked(windowId); + } + return null; + } + + /** + * Returns the window info according to given windowId. + * + * @param windowId The windowId + * @return The window info + */ + @Nullable + public WindowInfo findWindowInfoByIdLocked(int windowId) { + windowId = resolveParentWindowIdLocked(windowId); + final DisplayWindowsObserver observer = getDisplayWindowObserverByWindowIdLocked(windowId); + if (observer != null) { + return observer.findWindowInfoByIdLocked(windowId); + } + return null; + } + + /** + * Returns focused windowId or accessibility focused windowId according to given focusType. + * + * @param focusType {@link AccessibilityNodeInfo#FOCUS_INPUT} or + * {@link AccessibilityNodeInfo#FOCUS_ACCESSIBILITY} + * @return The focused windowId + */ + public int getFocusedWindowId(int focusType) { + if (focusType == AccessibilityNodeInfo.FOCUS_INPUT) { + return mTopFocusedWindowId; + } else if (focusType == AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) { + return mAccessibilityFocusedWindowId; + } + return AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + } + + /** + * Returns {@link AccessibilityWindowInfo} of PIP window. + * + * @return PIP accessibility window info + */ + @Nullable + public AccessibilityWindowInfo getPictureInPictureWindowLocked() { + AccessibilityWindowInfo windowInfo = null; + final int count = mDisplayWindowsObservers.size(); + for (int i = 0; i < count; i++) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i); + if (observer != null) { + if ((windowInfo = observer.getPictureInPictureWindowLocked()) != null) { + break; + } + } + } + return windowInfo; + } + + /** + * Sets an IAccessibilityInteractionConnection to replace the actions of a picture-in-picture + * window. + */ + public void setPictureInPictureActionReplacingConnection( + @Nullable IAccessibilityInteractionConnection connection) throws RemoteException { + synchronized (mLock) { + if (mPictureInPictureActionReplacingConnection != null) { + mPictureInPictureActionReplacingConnection.unlinkToDeath(); + mPictureInPictureActionReplacingConnection = null; + } + if (connection != null) { + RemoteAccessibilityConnection wrapper = new RemoteAccessibilityConnection( + AccessibilityWindowInfo.PICTURE_IN_PICTURE_ACTION_REPLACER_WINDOW_ID, + connection, "foo.bar.baz", Process.SYSTEM_UID, UserHandle.USER_ALL); + mPictureInPictureActionReplacingConnection = wrapper; + wrapper.linkToDeath(); + } + } + } + + /** + * Returns accessibility interaction connection for picture-in-picture window. + */ + @Nullable + public RemoteAccessibilityConnection getPictureInPictureActionReplacingConnection() { + return mPictureInPictureActionReplacingConnection; + } + + /** + * Invokes {@link IAccessibilityInteractionConnection#notifyOutsideTouch()} for windows that + * have watch outside touch flag and its layer is upper than target window. + */ + public void notifyOutsideTouch(int userId, int targetWindowId) { + final List<Integer> outsideWindowsIds; + final List<RemoteAccessibilityConnection> connectionList = new ArrayList<>(); + synchronized (mLock) { + final DisplayWindowsObserver observer = + getDisplayWindowObserverByWindowIdLocked(targetWindowId); + if (observer != null) { + outsideWindowsIds = observer.getWatchOutsideTouchWindowIdLocked(targetWindowId); + for (int i = 0; i < outsideWindowsIds.size(); i++) { + connectionList.add(getConnectionLocked(userId, outsideWindowsIds.get(i))); + } + } + } + for (int i = 0; i < connectionList.size(); i++) { + final RemoteAccessibilityConnection connection = connectionList.get(i); + if (connection != null) { + try { + connection.getRemote().notifyOutsideTouch(); + } catch (RemoteException re) { + if (DEBUG) { + Slog.e(LOG_TAG, "Error calling notifyOutsideTouch()"); + } + } + } + } + } + + /** + * Returns the display ID according to given userId and windowId. + * + * @param userId The userId + * @param windowId The windowId + * @return The display ID + */ + public int getDisplayIdByUserIdAndWindowIdLocked(int userId, int windowId) { + final IBinder windowToken = getWindowTokenForUserAndWindowIdLocked(userId, windowId); + final int displayId = mWindowManagerInternal.getDisplayIdForWindow(windowToken); + return displayId; + } + + /** + * Returns the display list including all displays which are tracking windows. + * + * @return The display list. + */ + public ArrayList<Integer> getDisplayListLocked() { + final ArrayList<Integer> displayList = new ArrayList<>(); + final int count = mDisplayWindowsObservers.size(); + for (int i = 0; i < count; i++) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i); + if (observer != null) { + displayList.add(observer.mDisplayId); + } + } + return displayList; + } + + /** + * Gets current input focused window token from window manager, and returns its windowId. + * + * @param userId The userId + * @return The input focused windowId, or -1 if not found + */ + private int findFocusedWindowId(int userId) { + final IBinder token = mWindowManagerInternal.getFocusedWindowToken(); + synchronized (mLock) { + return findWindowIdLocked(userId, token); + } + } + + private boolean isValidUserForInteractionConnectionsLocked(int userId) { + return mInteractionConnections.indexOfKey(userId) >= 0; + } + + private boolean isValidUserForWindowTokensLocked(int userId) { + return mWindowTokens.indexOfKey(userId) >= 0; + } + + private SparseArray<RemoteAccessibilityConnection> getInteractionConnectionsForUserLocked( + int userId) { + SparseArray<RemoteAccessibilityConnection> connection = mInteractionConnections.get( + userId); + if (connection == null) { + connection = new SparseArray<>(); + mInteractionConnections.put(userId, connection); + } + return connection; + } + + private SparseArray<IBinder> getWindowTokensForUserLocked(int userId) { + SparseArray<IBinder> windowTokens = mWindowTokens.get(userId); + if (windowTokens == null) { + windowTokens = new SparseArray<>(); + mWindowTokens.put(userId, windowTokens); + } + return windowTokens; + } + + private void clearAccessibilityFocusLocked(int windowId) { + mHandler.sendMessage(obtainMessage( + AccessibilityWindowManager::clearAccessibilityFocusMainThread, + AccessibilityWindowManager.this, + mAccessibilityUserManager.getCurrentUserIdLocked(), windowId)); + } + + private void clearAccessibilityFocusMainThread(int userId, int windowId) { + final RemoteAccessibilityConnection connection; + synchronized (mLock) { + connection = getConnectionLocked(userId, windowId); + if (connection == null) { + return; + } + } + try { + connection.getRemote().clearAccessibilityFocus(); + } catch (RemoteException re) { + if (DEBUG) { + Slog.e(LOG_TAG, "Error calling clearAccessibilityFocus()"); + } + } + } + + private DisplayWindowsObserver getDisplayWindowObserverByWindowIdLocked(int windowId) { + final int count = mDisplayWindowsObservers.size(); + for (int i = 0; i < count; i++) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i); + if (observer != null) { + if (observer.findWindowInfoByIdLocked(windowId) != null) { + return mDisplayWindowsObservers.get(observer.mDisplayId); + } + } + } + return null; + } + + /** + * Associate the token of the embedded view hierarchy to the host view hierarchy. + * + * @param embedded The leash token from the view root of embedded hierarchy + * @param host The leash token from the view root of host hierarchy + */ + void associateLocked(IBinder embedded, IBinder host) { + mHostEmbeddedMap.put(embedded, host); + } + + /** + * Clear the relationship of given token. + * + * @param token The leash token + */ + void disassociateLocked(IBinder token) { + mHostEmbeddedMap.remove(token); + for (int i = mHostEmbeddedMap.size() - 1; i >= 0; i--) { + if (mHostEmbeddedMap.valueAt(i).equals(token)) { + mHostEmbeddedMap.removeAt(i); + } + } + } + + /** + * Register the leash token with its windowId. + * + * @param token The token. + * @param windowId The windowID. + */ + void registerIdLocked(IBinder token, int windowId) { + mWindowIdMap.put(windowId, token); + } + + /** + * Unregister the windowId and also disassociate its token. + * + * @param windowId The windowID + */ + void unregisterIdLocked(int windowId) { + final IBinder token = mWindowIdMap.get(windowId); + if (token == null) { + return; + } + disassociateLocked(token); + mWindowIdMap.remove(windowId); + } + + /** + * Get the leash token by given windowID. + * + * @param windowId The windowID. + * @return The token, or {@code NULL} if this windowID doesn't exist + */ + IBinder getTokenLocked(int windowId) { + return mWindowIdMap.get(windowId); + } + + /** + * Get the windowId by given leash token. + * + * @param token The token + * @return The windowID, or -1 if the token doesn't exist + */ + int getWindowIdLocked(IBinder token) { + final int index = mWindowIdMap.indexOfValue(token); + if (index == -1) { + return index; + } + return mWindowIdMap.keyAt(index); + } + + /** + * Get the leash token of the host hierarchy by given token. + * + * @param token The token + * @return The token of host hierarchy, or {@code NULL} if no host exists + */ + IBinder getHostTokenLocked(IBinder token) { + return mHostEmbeddedMap.get(token); + } + + /** + * Dumps all {@link AccessibilityWindowInfo}s here. + */ + public void dump(FileDescriptor fd, final PrintWriter pw, String[] args) { + final int count = mDisplayWindowsObservers.size(); + for (int i = 0; i < count; i++) { + final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i); + if (observer != null) { + observer.dumpLocked(fd, pw, args); + } + } + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/BaseEventStreamTransformation.java b/services/accessibility/java/com/android/server/accessibility/BaseEventStreamTransformation.java index ce54586c52ae..16457216801b 100644 --- a/services/accessibility/java/com/android/server/accessibility/BaseEventStreamTransformation.java +++ b/services/accessibility/java/com/android/server/accessibility/BaseEventStreamTransformation.java @@ -16,7 +16,7 @@ package com.android.server.accessibility; -abstract class BaseEventStreamTransformation implements EventStreamTransformation { +public abstract class BaseEventStreamTransformation implements EventStreamTransformation { private EventStreamTransformation mNext; @Override diff --git a/services/accessibility/java/com/android/server/accessibility/EventStreamTransformation.java b/services/accessibility/java/com/android/server/accessibility/EventStreamTransformation.java index 7982996e7a4a..61aff9a360ba 100644 --- a/services/accessibility/java/com/android/server/accessibility/EventStreamTransformation.java +++ b/services/accessibility/java/com/android/server/accessibility/EventStreamTransformation.java @@ -54,7 +54,7 @@ import android.view.accessibility.AccessibilityEvent; * For example, if it received a down motion event followed by a cancel motion * event, it should not handle subsequent move and up events until it gets a down. */ -interface EventStreamTransformation { +public interface EventStreamTransformation { /** * Receives a motion event. Passed are the event transformed by previous diff --git a/services/accessibility/java/com/android/server/accessibility/MagnificationGestureHandler.java b/services/accessibility/java/com/android/server/accessibility/FullScreenMagnificationGestureHandler.java index 2fbaee65864a..b7f8e674f3ba 100644 --- a/services/accessibility/java/com/android/server/accessibility/MagnificationGestureHandler.java +++ b/services/accessibility/java/com/android/server/accessibility/FullScreenMagnificationGestureHandler.java @@ -24,7 +24,9 @@ import static android.view.MotionEvent.ACTION_POINTER_DOWN; import static android.view.MotionEvent.ACTION_POINTER_UP; import static android.view.MotionEvent.ACTION_UP; -import static com.android.server.accessibility.GestureUtils.distance; +import static com.android.internal.accessibility.util.AccessibilityStatsLogUtils.logMagnificationTripleTap; +import static com.android.server.accessibility.gestures.GestureUtils.distance; +import static com.android.server.accessibility.gestures.GestureUtils.distanceClosestPointerToPoint; import static java.lang.Math.abs; import static java.util.Arrays.asList; @@ -36,9 +38,11 @@ import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; import android.content.IntentFilter; +import android.graphics.PointF; import android.os.Handler; import android.os.Looper; import android.os.Message; +import android.os.SystemClock; import android.util.Log; import android.util.MathUtils; import android.util.Slog; @@ -52,13 +56,16 @@ import android.view.ScaleGestureDetector; import android.view.ScaleGestureDetector.OnScaleGestureListener; import android.view.ViewConfiguration; +import com.android.internal.R; import com.android.internal.annotations.VisibleForTesting; +import com.android.server.accessibility.gestures.GestureUtils; +import com.android.server.accessibility.magnification.MagnificationGestureHandler; import java.util.ArrayDeque; import java.util.Queue; /** - * This class handles magnification in response to touch events. + * This class handles full screen magnification in response to touch events. * * The behavior is as follows: * @@ -106,14 +113,14 @@ import java.util.Queue; * 7. The magnification scale will be persisted in settings and in the cloud. */ @SuppressWarnings("WeakerAccess") -class MagnificationGestureHandler extends BaseEventStreamTransformation { - private static final String LOG_TAG = "MagnificationGestureHandler"; +class FullScreenMagnificationGestureHandler extends MagnificationGestureHandler { + private static final String LOG_TAG = "FullScreenMagnificationGestureHandler"; private static final boolean DEBUG_ALL = false; - private static final boolean DEBUG_STATE_TRANSITIONS = false || DEBUG_ALL; - private static final boolean DEBUG_DETECTING = false || DEBUG_ALL; - private static final boolean DEBUG_PANNING_SCALING = false || DEBUG_ALL; - private static final boolean DEBUG_EVENT_STREAM = false || DEBUG_ALL; + private static final boolean DEBUG_STATE_TRANSITIONS = false | DEBUG_ALL; + private static final boolean DEBUG_DETECTING = false | DEBUG_ALL; + private static final boolean DEBUG_PANNING_SCALING = false | DEBUG_ALL; + private static final boolean DEBUG_EVENT_STREAM = false | DEBUG_ALL; // The MIN_SCALE is different from MagnificationController.MIN_SCALE due // to AccessibilityService.MagnificationController#setScale() has @@ -165,14 +172,14 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { * {@code false} if it should ignore such triggers. * @param displayId The logical display id. */ - public MagnificationGestureHandler(Context context, + FullScreenMagnificationGestureHandler(Context context, MagnificationController magnificationController, boolean detectTripleTap, boolean detectShortcutTrigger, int displayId) { if (DEBUG_ALL) { Log.i(LOG_TAG, - "MagnificationGestureHandler(detectTripleTap = " + detectTripleTap + "FullScreenMagnificationGestureHandler(detectTripleTap = " + detectTripleTap + ", detectShortcutTrigger = " + detectShortcutTrigger + ")"); } @@ -261,7 +268,8 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { clearAndTransitionToStateDetecting(); } - void notifyShortcutTriggered() { + @Override + public void notifyShortcutTriggered() { if (mDetectShortcutTrigger) { boolean wasMagnifying = mMagnificationController.resetIfNeeded(mDisplayId, /* animate */ true); @@ -381,10 +389,10 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { float mInitialScaleFactor = -1; boolean mScaling; - public PanningScalingState(Context context) { + PanningScalingState(Context context) { final TypedValue scaleValue = new TypedValue(); context.getResources().getValue( - com.android.internal.R.dimen.config_screen_magnification_scaling_threshold, + R.dimen.config_screen_magnification_scaling_threshold, scaleValue, false); mScalingThreshold = scaleValue.getFloat(); mScaleGestureDetector = new ScaleGestureDetector(context, this, Handler.getMain()); @@ -405,7 +413,6 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { } else if (action == ACTION_UP || action == ACTION_CANCEL) { persistScaleAndTransitionTo(mDetectingState); - } } @@ -487,10 +494,9 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { @Override public String toString() { - return "PanningScalingState{" + - "mInitialScaleFactor=" + mInitialScaleFactor + - ", mScaling=" + mScaling + - '}'; + return "PanningScalingState{" + "mInitialScaleFactor=" + mInitialScaleFactor + + ", mScaling=" + mScaling + + '}'; } } @@ -542,7 +548,7 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { clear(); transitionTo(mDetectingState); } - break; + break; case ACTION_DOWN: case ACTION_POINTER_UP: { @@ -559,10 +565,10 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { @Override public String toString() { - return "ViewportDraggingState{" + - "mZoomedInBeforeDrag=" + mZoomedInBeforeDrag + - ", mLastMoveOutsideMagnifiedRegion=" + mLastMoveOutsideMagnifiedRegion + - '}'; + return "ViewportDraggingState{" + + "mZoomedInBeforeDrag=" + mZoomedInBeforeDrag + + ", mLastMoveOutsideMagnifiedRegion=" + mLastMoveOutsideMagnifiedRegion + + '}'; } } @@ -575,16 +581,17 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { @Override public void onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { - // Ensure that the state at the end of delegation is consistent with the last delegated + // Ensures that the state at the end of delegation is consistent with the last delegated // UP/DOWN event in queue: still delegating if pointer is down, detecting otherwise switch (event.getActionMasked()) { case ACTION_UP: case ACTION_CANCEL: { transitionTo(mDetectingState); - } break; + } + break; case ACTION_DOWN: { - transitionTo(mDelegatingState); + transitionTo(mDelegatingState); mLastDelegatedDownEventTime = event.getDownTime(); } break; } @@ -610,6 +617,7 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { private static final int MESSAGE_ON_TRIPLE_TAP_AND_HOLD = 1; private static final int MESSAGE_TRANSITION_TO_DELEGATING_STATE = 2; + private static final int MESSAGE_TRANSITION_TO_PANNINGSCALING_STATE = 3; final int mLongTapMinDelay; final int mSwipeMinDistance; @@ -621,16 +629,19 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { private MotionEvent mPreLastDown; private MotionEvent mLastUp; private MotionEvent mPreLastUp; + private PointF mSecondPointerDownLocation = new PointF(Float.NaN, Float.NaN); + + private long mLastDetectingDownEventTime; @VisibleForTesting boolean mShortcutTriggered; @VisibleForTesting Handler mHandler = new Handler(Looper.getMainLooper(), this); - public DetectingState(Context context) { + DetectingState(Context context) { mLongTapMinDelay = ViewConfiguration.getLongPressTimeout(); mMultiTapMaxDelay = ViewConfiguration.getDoubleTapTimeout() + context.getResources().getInteger( - com.android.internal.R.integer.config_screen_magnification_multi_tap_adjustment); + R.integer.config_screen_magnification_multi_tap_adjustment); mSwipeMinDistance = ViewConfiguration.get(context).getScaledTouchSlop(); mMultiTapMaxDistance = ViewConfiguration.get(context).getScaledDoubleTapSlop(); } @@ -649,6 +660,10 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { transitionToDelegatingStateAndClear(); } break; + case MESSAGE_TRANSITION_TO_PANNINGSCALING_STATE: { + transitToPanningScalingStateAndClear(); + } + break; default: { throw new IllegalArgumentException("Unknown message type: " + type); } @@ -662,6 +677,7 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { switch (event.getActionMasked()) { case MotionEvent.ACTION_DOWN: { + mLastDetectingDownEventTime = event.getDownTime(); mHandler.removeMessages(MESSAGE_TRANSITION_TO_DELEGATING_STATE); if (!mMagnificationController.magnificationRegionContains( @@ -694,14 +710,20 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { } break; case ACTION_POINTER_DOWN: { - if (mMagnificationController.isMagnifying(mDisplayId)) { - transitionTo(mPanningScalingState); - clear(); + if (mMagnificationController.isMagnifying(mDisplayId) + && event.getPointerCount() == 2) { + storeSecondPointerDownLocation(event); + mHandler.sendEmptyMessageDelayed(MESSAGE_TRANSITION_TO_PANNINGSCALING_STATE, + ViewConfiguration.getTapTimeout()); } else { transitionToDelegatingStateAndClear(); } } break; + case ACTION_POINTER_UP: { + transitionToDelegatingStateAndClear(); + } + break; case ACTION_MOVE: { if (isFingerDown() && distance(mLastDown, /* move */ event) > mSwipeMinDistance) { @@ -711,11 +733,19 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { // For convenience, viewport dragging takes precedence // over insta-delegating on 3tap&swipe // (which is a rare combo to be used aside from magnification) - if (isMultiTapTriggered(2 /* taps */)) { + if (isMultiTapTriggered(2 /* taps */) && event.getPointerCount() == 1) { transitionToViewportDraggingStateAndClear(event); + } else if (isMagnifying() && event.getPointerCount() == 2) { + //Primary pointer is swiping, so transit to PanningScalingState + transitToPanningScalingStateAndClear(); } else { transitionToDelegatingStateAndClear(); } + } else if (isMagnifying() && secondPointerDownValid() + && distanceClosestPointerToPoint( + mSecondPointerDownLocation, /* move */ event) > mSwipeMinDistance) { + //Second pointer is swiping, so transit to PanningScalingState + transitToPanningScalingStateAndClear(); } } break; @@ -747,15 +777,37 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { } } + private void storeSecondPointerDownLocation(MotionEvent event) { + final int index = event.getActionIndex(); + mSecondPointerDownLocation.set(event.getX(index), event.getY(index)); + } + + private boolean secondPointerDownValid() { + return !(Float.isNaN(mSecondPointerDownLocation.x) && Float.isNaN( + mSecondPointerDownLocation.y)); + } + + private void transitToPanningScalingStateAndClear() { + transitionTo(mPanningScalingState); + clear(); + } + public boolean isMultiTapTriggered(int numTaps) { // Shortcut acts as the 2 initial taps if (mShortcutTriggered) return tapCount() + 2 >= numTaps; - return mDetectTripleTap + final boolean multitapTriggered = mDetectTripleTap && tapCount() >= numTaps && isMultiTap(mPreLastDown, mLastDown) && isMultiTap(mPreLastUp, mLastUp); + + // Only log the triple tap event, use numTaps to filter. + if (multitapTriggered && numTaps > 2) { + final boolean enabled = mMagnificationController.isMagnifying(mDisplayId); + logMagnificationTripleTap(enabled); + } + return multitapTriggered; } private boolean isMultiTap(MotionEvent first, MotionEvent second) { @@ -807,11 +859,13 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { setShortcutTriggered(false); removePendingDelayedMessages(); clearDelayedMotionEvents(); + mSecondPointerDownLocation.set(Float.NaN, Float.NaN); } private void removePendingDelayedMessages() { mHandler.removeMessages(MESSAGE_ON_TRIPLE_TAP_AND_HOLD); mHandler.removeMessages(MESSAGE_TRANSITION_TO_DELEGATING_STATE); + mHandler.removeMessages(MESSAGE_TRANSITION_TO_PANNINGSCALING_STATE); } private void cacheDelayedMotionEvent(MotionEvent event, MotionEvent rawEvent, @@ -838,14 +892,25 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { } private void sendDelayedMotionEvents() { - while (mDelayedEventQueue != null) { + if (mDelayedEventQueue == null) { + return; + } + + // Adjust down time to prevent subsequent modules being misleading, and also limit + // the maximum offset to mMultiTapMaxDelay to prevent the down time of 2nd tap is + // in the future when multi-tap happens. + final long offset = Math.min( + SystemClock.uptimeMillis() - mLastDetectingDownEventTime, mMultiTapMaxDelay); + + do { MotionEventInfo info = mDelayedEventQueue; mDelayedEventQueue = info.mNext; + info.event.setDownTime(info.event.getDownTime() + offset); handleEventWith(mDelegatingState, info.event, info.rawEvent, info.policyFlags); info.recycle(); - } + } while (mDelayedEventQueue != null); } private void clearDelayedMotionEvents() { @@ -864,10 +929,10 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { transitionTo(mDelegatingState); sendDelayedMotionEvents(); removePendingDelayedMessages(); + mSecondPointerDownLocation.set(Float.NaN, Float.NaN); } private void onTripleTap(MotionEvent up) { - if (DEBUG_DETECTING) { Slog.i(LOG_TAG, "onTripleTap(); delayed: " + MotionEventInfo.toString(mDelayedEventQueue)); @@ -882,6 +947,10 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { } } + private boolean isMagnifying() { + return mMagnificationController.isMagnifying(mDisplayId); + } + void transitionToViewportDraggingStateAndClear(MotionEvent down) { if (DEBUG_DETECTING) Slog.i(LOG_TAG, "onTripleTapAndHold()"); @@ -890,6 +959,10 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { mViewportDraggingState.mZoomedInBeforeDrag = mMagnificationController.isMagnifying(mDisplayId); + // Triple tap and hold also belongs to triple tap event. + final boolean enabled = !mViewportDraggingState.mZoomedInBeforeDrag; + logMagnificationTripleTap(enabled); + zoomOn(down.getX(), down.getY()); transitionTo(mViewportDraggingState); @@ -897,11 +970,11 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { @Override public String toString() { - return "DetectingState{" + - "tapCount()=" + tapCount() + - ", mShortcutTriggered=" + mShortcutTriggered + - ", mDelayedEventQueue=" + MotionEventInfo.toString(mDelayedEventQueue) + - '}'; + return "DetectingState{" + + "tapCount()=" + tapCount() + + ", mShortcutTriggered=" + mShortcutTriggered + + ", mDelayedEventQueue=" + MotionEventInfo.toString(mDelayedEventQueue) + + '}'; } void toggleShortcutTriggered() { @@ -970,18 +1043,18 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { @Override public String toString() { - return "MagnificationGesture{" + - "mDetectingState=" + mDetectingState + - ", mDelegatingState=" + mDelegatingState + - ", mMagnifiedInteractionState=" + mPanningScalingState + - ", mViewportDraggingState=" + mViewportDraggingState + - ", mDetectTripleTap=" + mDetectTripleTap + - ", mDetectShortcutTrigger=" + mDetectShortcutTrigger + - ", mCurrentState=" + State.nameOf(mCurrentState) + - ", mPreviousState=" + State.nameOf(mPreviousState) + - ", mMagnificationController=" + mMagnificationController + - ", mDisplayId=" + mDisplayId + - '}'; + return "MagnificationGesture{" + + "mDetectingState=" + mDetectingState + + ", mDelegatingState=" + mDelegatingState + + ", mMagnifiedInteractionState=" + mPanningScalingState + + ", mViewportDraggingState=" + mViewportDraggingState + + ", mDetectTripleTap=" + mDetectTripleTap + + ", mDetectShortcutTrigger=" + mDetectShortcutTrigger + + ", mCurrentState=" + State.nameOf(mCurrentState) + + ", mPreviousState=" + State.nameOf(mPreviousState) + + ", mMagnificationController=" + mMagnificationController + + ", mDisplayId=" + mDisplayId + + '}'; } private static final class MotionEventInfo { @@ -1069,9 +1142,10 @@ class MagnificationGestureHandler extends BaseEventStreamTransformation { */ private static class ScreenStateReceiver extends BroadcastReceiver { private final Context mContext; - private final MagnificationGestureHandler mGestureHandler; + private final FullScreenMagnificationGestureHandler mGestureHandler; - public ScreenStateReceiver(Context context, MagnificationGestureHandler gestureHandler) { + ScreenStateReceiver(Context context, + FullScreenMagnificationGestureHandler gestureHandler) { mContext = context; mGestureHandler = gestureHandler; } diff --git a/services/accessibility/java/com/android/server/accessibility/GlobalActionPerformer.java b/services/accessibility/java/com/android/server/accessibility/GlobalActionPerformer.java deleted file mode 100644 index b9b2654b93cc..000000000000 --- a/services/accessibility/java/com/android/server/accessibility/GlobalActionPerformer.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - ** Copyright 2017, The Android Open Source Project - ** - ** Licensed under the Apache License, Version 2.0 (the "License"); - ** you may not use this file except in compliance with the License. - ** You may obtain a copy of the License at - ** - ** http://www.apache.org/licenses/LICENSE-2.0 - ** - ** Unless required by applicable law or agreed to in writing, software - ** distributed under the License is distributed on an "AS IS" BASIS, - ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ** See the License for the specific language governing permissions and - ** limitations under the License. - */ - -package com.android.server.accessibility; - -import android.accessibilityservice.AccessibilityService; -import android.app.StatusBarManager; -import android.content.Context; -import android.hardware.input.InputManager; -import android.os.Binder; -import android.os.Handler; -import android.os.Looper; -import android.os.PowerManager; -import android.os.SystemClock; -import android.view.InputDevice; -import android.view.KeyCharacterMap; -import android.view.KeyEvent; - -import com.android.internal.annotations.VisibleForTesting; -import com.android.internal.util.ScreenshotHelper; -import com.android.server.LocalServices; -import com.android.server.statusbar.StatusBarManagerInternal; -import com.android.server.wm.WindowManagerInternal; - -import java.util.function.Supplier; - -/** - * Handle the back-end of AccessibilityService#performGlobalAction - */ -public class GlobalActionPerformer { - private final WindowManagerInternal mWindowManagerService; - private final Context mContext; - private Supplier<ScreenshotHelper> mScreenshotHelperSupplier; - - public GlobalActionPerformer(Context context, WindowManagerInternal windowManagerInternal) { - mContext = context; - mWindowManagerService = windowManagerInternal; - mScreenshotHelperSupplier = null; - } - - // Used to mock ScreenshotHelper - @VisibleForTesting - public GlobalActionPerformer(Context context, WindowManagerInternal windowManagerInternal, - Supplier<ScreenshotHelper> screenshotHelperSupplier) { - this(context, windowManagerInternal); - mScreenshotHelperSupplier = screenshotHelperSupplier; - } - - public boolean performGlobalAction(int action) { - final long identity = Binder.clearCallingIdentity(); - try { - switch (action) { - case AccessibilityService.GLOBAL_ACTION_BACK: { - sendDownAndUpKeyEvents(KeyEvent.KEYCODE_BACK); - } - return true; - case AccessibilityService.GLOBAL_ACTION_HOME: { - sendDownAndUpKeyEvents(KeyEvent.KEYCODE_HOME); - } - return true; - case AccessibilityService.GLOBAL_ACTION_RECENTS: { - return openRecents(); - } - case AccessibilityService.GLOBAL_ACTION_NOTIFICATIONS: { - expandNotifications(); - } - return true; - case AccessibilityService.GLOBAL_ACTION_QUICK_SETTINGS: { - expandQuickSettings(); - } - return true; - case AccessibilityService.GLOBAL_ACTION_POWER_DIALOG: { - showGlobalActions(); - } - return true; - case AccessibilityService.GLOBAL_ACTION_TOGGLE_SPLIT_SCREEN: { - return toggleSplitScreen(); - } - case AccessibilityService.GLOBAL_ACTION_LOCK_SCREEN: { - return lockScreen(); - } - case AccessibilityService.GLOBAL_ACTION_TAKE_SCREENSHOT: { - return takeScreenshot(); - } - } - return false; - } finally { - Binder.restoreCallingIdentity(identity); - } - } - - private void sendDownAndUpKeyEvents(int keyCode) { - final long token = Binder.clearCallingIdentity(); - - // Inject down. - final long downTime = SystemClock.uptimeMillis(); - sendKeyEventIdentityCleared(keyCode, KeyEvent.ACTION_DOWN, downTime, downTime); - sendKeyEventIdentityCleared( - keyCode, KeyEvent.ACTION_UP, downTime, SystemClock.uptimeMillis()); - - Binder.restoreCallingIdentity(token); - } - - private void sendKeyEventIdentityCleared(int keyCode, int action, long downTime, long time) { - KeyEvent event = KeyEvent.obtain(downTime, time, action, keyCode, 0, 0, - KeyCharacterMap.VIRTUAL_KEYBOARD, 0, KeyEvent.FLAG_FROM_SYSTEM, - InputDevice.SOURCE_KEYBOARD, null); - InputManager.getInstance() - .injectInputEvent(event, InputManager.INJECT_INPUT_EVENT_MODE_ASYNC); - event.recycle(); - } - - private void expandNotifications() { - final long token = Binder.clearCallingIdentity(); - - StatusBarManager statusBarManager = (StatusBarManager) mContext.getSystemService( - android.app.Service.STATUS_BAR_SERVICE); - statusBarManager.expandNotificationsPanel(); - - Binder.restoreCallingIdentity(token); - } - - private void expandQuickSettings() { - final long token = Binder.clearCallingIdentity(); - - StatusBarManager statusBarManager = (StatusBarManager) mContext.getSystemService( - android.app.Service.STATUS_BAR_SERVICE); - statusBarManager.expandSettingsPanel(); - - Binder.restoreCallingIdentity(token); - } - - private boolean openRecents() { - final long token = Binder.clearCallingIdentity(); - try { - StatusBarManagerInternal statusBarService = LocalServices.getService( - StatusBarManagerInternal.class); - if (statusBarService == null) { - return false; - } - statusBarService.toggleRecentApps(); - } finally { - Binder.restoreCallingIdentity(token); - } - return true; - } - - private void showGlobalActions() { - mWindowManagerService.showGlobalActions(); - } - - private boolean toggleSplitScreen() { - final long token = Binder.clearCallingIdentity(); - try { - StatusBarManagerInternal statusBarService = LocalServices.getService( - StatusBarManagerInternal.class); - if (statusBarService == null) { - return false; - } - statusBarService.toggleSplitScreen(); - } finally { - Binder.restoreCallingIdentity(token); - } - return true; - } - - private boolean lockScreen() { - mContext.getSystemService(PowerManager.class).goToSleep(SystemClock.uptimeMillis(), - PowerManager.GO_TO_SLEEP_REASON_ACCESSIBILITY, 0); - mWindowManagerService.lockNow(); - return true; - } - - private boolean takeScreenshot() { - ScreenshotHelper screenshotHelper = (mScreenshotHelperSupplier != null) - ? mScreenshotHelperSupplier.get() : new ScreenshotHelper(mContext); - screenshotHelper.takeScreenshot(android.view.WindowManager.TAKE_SCREENSHOT_FULLSCREEN, - true, true, new Handler(Looper.getMainLooper()), null); - return true; - } -} diff --git a/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java b/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java index 7b6a12822faa..3310cb4e3e79 100644 --- a/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java +++ b/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java @@ -101,11 +101,12 @@ public class MotionEventInjector extends BaseEventStreamTransformation implement * either complete or cancelled. */ public void injectEvents(List<GestureStep> gestureSteps, - IAccessibilityServiceClient serviceInterface, int sequence) { + IAccessibilityServiceClient serviceInterface, int sequence, int displayId) { SomeArgs args = SomeArgs.obtain(); args.arg1 = gestureSteps; args.arg2 = serviceInterface; args.argi1 = sequence; + args.argi2 = displayId; mHandler.sendMessage(mHandler.obtainMessage(MESSAGE_INJECT_EVENTS, args)); } @@ -146,7 +147,7 @@ public class MotionEventInjector extends BaseEventStreamTransformation implement if (message.what == MESSAGE_INJECT_EVENTS) { SomeArgs args = (SomeArgs) message.obj; injectEventsMainThread((List<GestureStep>) args.arg1, - (IAccessibilityServiceClient) args.arg2, args.argi1); + (IAccessibilityServiceClient) args.arg2, args.argi1, args.argi2); args.recycle(); return true; } @@ -165,7 +166,7 @@ public class MotionEventInjector extends BaseEventStreamTransformation implement } private void injectEventsMainThread(List<GestureStep> gestureSteps, - IAccessibilityServiceClient serviceInterface, int sequence) { + IAccessibilityServiceClient serviceInterface, int sequence, int displayId) { if (mIsDestroyed) { try { serviceInterface.onPerformGestureResult(sequence, false); @@ -209,6 +210,7 @@ public class MotionEventInjector extends BaseEventStreamTransformation implement for (int i = 0; i < events.size(); i++) { MotionEvent event = events.get(i); + event.setDisplayId(displayId); int isEndOfSequence = (i == events.size() - 1) ? 1 : 0; Message message = mHandler.obtainMessage( MESSAGE_SEND_MOTION_EVENT, isEndOfSequence, 0, event); diff --git a/services/accessibility/java/com/android/server/accessibility/SystemActionPerformer.java b/services/accessibility/java/com/android/server/accessibility/SystemActionPerformer.java new file mode 100644 index 000000000000..35054991d3ff --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/SystemActionPerformer.java @@ -0,0 +1,399 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility; + +import static android.view.WindowManager.ScreenshotSource.SCREENSHOT_ACCESSIBILITY_ACTIONS; + +import android.accessibilityservice.AccessibilityService; +import android.app.PendingIntent; +import android.app.RemoteAction; +import android.app.StatusBarManager; +import android.content.Context; +import android.hardware.input.InputManager; +import android.os.Binder; +import android.os.Handler; +import android.os.Looper; +import android.os.PowerManager; +import android.os.SystemClock; +import android.util.ArrayMap; +import android.util.Slog; +import android.view.InputDevice; +import android.view.KeyCharacterMap; +import android.view.KeyEvent; +import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; + +import com.android.internal.R; +import com.android.internal.annotations.GuardedBy; +import com.android.internal.annotations.VisibleForTesting; +import com.android.internal.util.ScreenshotHelper; +import com.android.server.LocalServices; +import com.android.server.statusbar.StatusBarManagerInternal; +import com.android.server.wm.WindowManagerInternal; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Handle the back-end of system AccessibilityAction. + * + * This class should support three use cases with combined usage of new API and legacy API: + * + * Use case 1: SystemUI doesn't use the new system action registration API. Accessibility + * service doesn't use the new system action API to obtain action list. Accessibility + * service uses legacy global action id to perform predefined system actions. + * Use case 2: SystemUI uses the new system action registration API to register available system + * actions. Accessibility service doesn't use the new system action API to obtain action + * list. Accessibility service uses legacy global action id to trigger the system + * actions registered by SystemUI. + * Use case 3: SystemUI doesn't use the new system action registration API.Accessibility service + * obtains the available system actions using new AccessibilityService API and trigger + * the predefined system actions. + */ +public class SystemActionPerformer { + private static final String TAG = "SystemActionPerformer"; + + interface SystemActionsChangedListener { + void onSystemActionsChanged(); + } + private final SystemActionsChangedListener mListener; + + private final Object mSystemActionLock = new Object(); + // Resource id based ActionId -> RemoteAction + @GuardedBy("mSystemActionLock") + private final Map<Integer, RemoteAction> mRegisteredSystemActions = new ArrayMap<>(); + + // Legacy system actions. + private final AccessibilityAction mLegacyHomeAction; + private final AccessibilityAction mLegacyBackAction; + private final AccessibilityAction mLegacyRecentsAction; + private final AccessibilityAction mLegacyNotificationsAction; + private final AccessibilityAction mLegacyQuickSettingsAction; + private final AccessibilityAction mLegacyPowerDialogAction; + private final AccessibilityAction mLegacyLockScreenAction; + private final AccessibilityAction mLegacyTakeScreenshotAction; + + private final WindowManagerInternal mWindowManagerService; + private final Context mContext; + private Supplier<ScreenshotHelper> mScreenshotHelperSupplier; + + public SystemActionPerformer( + Context context, + WindowManagerInternal windowManagerInternal) { + this(context, windowManagerInternal, null, null); + } + + // Used to mock ScreenshotHelper + @VisibleForTesting + public SystemActionPerformer( + Context context, + WindowManagerInternal windowManagerInternal, + Supplier<ScreenshotHelper> screenshotHelperSupplier) { + this(context, windowManagerInternal, screenshotHelperSupplier, null); + } + + public SystemActionPerformer( + Context context, + WindowManagerInternal windowManagerInternal, + Supplier<ScreenshotHelper> screenshotHelperSupplier, + SystemActionsChangedListener listener) { + mContext = context; + mWindowManagerService = windowManagerInternal; + mListener = listener; + mScreenshotHelperSupplier = screenshotHelperSupplier; + + mLegacyHomeAction = new AccessibilityAction( + AccessibilityService.GLOBAL_ACTION_HOME, + mContext.getResources().getString( + R.string.accessibility_system_action_home_label)); + mLegacyBackAction = new AccessibilityAction( + AccessibilityService.GLOBAL_ACTION_BACK, + mContext.getResources().getString( + R.string.accessibility_system_action_back_label)); + mLegacyRecentsAction = new AccessibilityAction( + AccessibilityService.GLOBAL_ACTION_RECENTS, + mContext.getResources().getString( + R.string.accessibility_system_action_recents_label)); + mLegacyNotificationsAction = new AccessibilityAction( + AccessibilityService.GLOBAL_ACTION_NOTIFICATIONS, + mContext.getResources().getString( + R.string.accessibility_system_action_notifications_label)); + mLegacyQuickSettingsAction = new AccessibilityAction( + AccessibilityService.GLOBAL_ACTION_QUICK_SETTINGS, + mContext.getResources().getString( + R.string.accessibility_system_action_quick_settings_label)); + mLegacyPowerDialogAction = new AccessibilityAction( + AccessibilityService.GLOBAL_ACTION_POWER_DIALOG, + mContext.getResources().getString( + R.string.accessibility_system_action_power_dialog_label)); + mLegacyLockScreenAction = new AccessibilityAction( + AccessibilityService.GLOBAL_ACTION_LOCK_SCREEN, + mContext.getResources().getString( + R.string.accessibility_system_action_lock_screen_label)); + mLegacyTakeScreenshotAction = new AccessibilityAction( + AccessibilityService.GLOBAL_ACTION_TAKE_SCREENSHOT, + mContext.getResources().getString( + R.string.accessibility_system_action_screenshot_label)); + } + + /** + * This method is called to register a system action. If a system action is already registered + * with the given id, the existing system action will be overwritten. + * + * This method is supposed to be package internal since this class is meant to be used by + * AccessibilityManagerService only. But Mockito has a bug which requiring this to be public + * to be mocked. + */ + @VisibleForTesting + public void registerSystemAction(int id, RemoteAction action) { + synchronized (mSystemActionLock) { + mRegisteredSystemActions.put(id, action); + } + if (mListener != null) { + mListener.onSystemActionsChanged(); + } + } + + /** + * This method is called to unregister a system action previously registered through + * registerSystemAction. + * + * This method is supposed to be package internal since this class is meant to be used by + * AccessibilityManagerService only. But Mockito has a bug which requiring this to be public + * to be mocked. + */ + @VisibleForTesting + public void unregisterSystemAction(int id) { + synchronized (mSystemActionLock) { + mRegisteredSystemActions.remove(id); + } + if (mListener != null) { + mListener.onSystemActionsChanged(); + } + } + + /** + * This method returns the list of available system actions. + */ + @VisibleForTesting(visibility = VisibleForTesting.Visibility.PACKAGE) + public List<AccessibilityAction> getSystemActions() { + List<AccessibilityAction> systemActions = new ArrayList<>(); + synchronized (mSystemActionLock) { + for (Map.Entry<Integer, RemoteAction> entry : mRegisteredSystemActions.entrySet()) { + AccessibilityAction systemAction = new AccessibilityAction( + entry.getKey(), + entry.getValue().getTitle()); + systemActions.add(systemAction); + } + + // add AccessibilitySystemAction entry for legacy system actions if not overwritten + addLegacySystemActions(systemActions); + } + return systemActions; + } + + private void addLegacySystemActions(List<AccessibilityAction> systemActions) { + if (!mRegisteredSystemActions.containsKey(AccessibilityService.GLOBAL_ACTION_BACK)) { + systemActions.add(mLegacyBackAction); + } + if (!mRegisteredSystemActions.containsKey(AccessibilityService.GLOBAL_ACTION_HOME)) { + systemActions.add(mLegacyHomeAction); + } + if (!mRegisteredSystemActions.containsKey(AccessibilityService.GLOBAL_ACTION_RECENTS)) { + systemActions.add(mLegacyRecentsAction); + } + if (!mRegisteredSystemActions.containsKey( + AccessibilityService.GLOBAL_ACTION_NOTIFICATIONS)) { + systemActions.add(mLegacyNotificationsAction); + } + if (!mRegisteredSystemActions.containsKey( + AccessibilityService.GLOBAL_ACTION_QUICK_SETTINGS)) { + systemActions.add(mLegacyQuickSettingsAction); + } + if (!mRegisteredSystemActions.containsKey( + AccessibilityService.GLOBAL_ACTION_POWER_DIALOG)) { + systemActions.add(mLegacyPowerDialogAction); + } + if (!mRegisteredSystemActions.containsKey( + AccessibilityService.GLOBAL_ACTION_LOCK_SCREEN)) { + systemActions.add(mLegacyLockScreenAction); + } + if (!mRegisteredSystemActions.containsKey( + AccessibilityService.GLOBAL_ACTION_TAKE_SCREENSHOT)) { + systemActions.add(mLegacyTakeScreenshotAction); + } + } + + /** + * Trigger the registered action by the matching action id. + */ + public boolean performSystemAction(int actionId) { + final long identity = Binder.clearCallingIdentity(); + try { + synchronized (mSystemActionLock) { + // If a system action is registered with the given actionId, call the corresponding + // RemoteAction. + RemoteAction registeredAction = mRegisteredSystemActions.get(actionId); + if (registeredAction != null) { + try { + registeredAction.getActionIntent().send(); + return true; + } catch (PendingIntent.CanceledException ex) { + Slog.e(TAG, + "canceled PendingIntent for global action " + + registeredAction.getTitle(), + ex); + } + return false; + } + } + + // No RemoteAction registered with the given actionId, try the default legacy system + // actions. + switch (actionId) { + case AccessibilityService.GLOBAL_ACTION_BACK: { + sendDownAndUpKeyEvents(KeyEvent.KEYCODE_BACK); + return true; + } + case AccessibilityService.GLOBAL_ACTION_HOME: { + sendDownAndUpKeyEvents(KeyEvent.KEYCODE_HOME); + return true; + } + case AccessibilityService.GLOBAL_ACTION_RECENTS: + return openRecents(); + case AccessibilityService.GLOBAL_ACTION_NOTIFICATIONS: { + expandNotifications(); + return true; + } + case AccessibilityService.GLOBAL_ACTION_QUICK_SETTINGS: { + expandQuickSettings(); + return true; + } + case AccessibilityService.GLOBAL_ACTION_POWER_DIALOG: { + showGlobalActions(); + return true; + } + case AccessibilityService.GLOBAL_ACTION_TOGGLE_SPLIT_SCREEN: + return toggleSplitScreen(); + case AccessibilityService.GLOBAL_ACTION_LOCK_SCREEN: + return lockScreen(); + case AccessibilityService.GLOBAL_ACTION_TAKE_SCREENSHOT: + return takeScreenshot(); + case AccessibilityService.GLOBAL_ACTION_KEYCODE_HEADSETHOOK : + sendDownAndUpKeyEvents(KeyEvent.KEYCODE_HEADSETHOOK); + return true; + default: + Slog.e(TAG, "Invalid action id: " + actionId); + return false; + } + } finally { + Binder.restoreCallingIdentity(identity); + } + } + + private void sendDownAndUpKeyEvents(int keyCode) { + final long token = Binder.clearCallingIdentity(); + + // Inject down. + final long downTime = SystemClock.uptimeMillis(); + sendKeyEventIdentityCleared(keyCode, KeyEvent.ACTION_DOWN, downTime, downTime); + sendKeyEventIdentityCleared( + keyCode, KeyEvent.ACTION_UP, downTime, SystemClock.uptimeMillis()); + + Binder.restoreCallingIdentity(token); + } + + private void sendKeyEventIdentityCleared(int keyCode, int action, long downTime, long time) { + KeyEvent event = KeyEvent.obtain(downTime, time, action, keyCode, 0, 0, + KeyCharacterMap.VIRTUAL_KEYBOARD, 0, KeyEvent.FLAG_FROM_SYSTEM, + InputDevice.SOURCE_KEYBOARD, null); + InputManager.getInstance() + .injectInputEvent(event, InputManager.INJECT_INPUT_EVENT_MODE_ASYNC); + event.recycle(); + } + + private void expandNotifications() { + final long token = Binder.clearCallingIdentity(); + + StatusBarManager statusBarManager = (StatusBarManager) mContext.getSystemService( + android.app.Service.STATUS_BAR_SERVICE); + statusBarManager.expandNotificationsPanel(); + + Binder.restoreCallingIdentity(token); + } + + private void expandQuickSettings() { + final long token = Binder.clearCallingIdentity(); + + StatusBarManager statusBarManager = (StatusBarManager) mContext.getSystemService( + android.app.Service.STATUS_BAR_SERVICE); + statusBarManager.expandSettingsPanel(); + + Binder.restoreCallingIdentity(token); + } + + private boolean openRecents() { + final long token = Binder.clearCallingIdentity(); + try { + StatusBarManagerInternal statusBarService = LocalServices.getService( + StatusBarManagerInternal.class); + if (statusBarService == null) { + return false; + } + statusBarService.toggleRecentApps(); + } finally { + Binder.restoreCallingIdentity(token); + } + return true; + } + + private void showGlobalActions() { + mWindowManagerService.showGlobalActions(); + } + + private boolean toggleSplitScreen() { + final long token = Binder.clearCallingIdentity(); + try { + StatusBarManagerInternal statusBarService = LocalServices.getService( + StatusBarManagerInternal.class); + if (statusBarService == null) { + return false; + } + statusBarService.toggleSplitScreen(); + } finally { + Binder.restoreCallingIdentity(token); + } + return true; + } + + private boolean lockScreen() { + mContext.getSystemService(PowerManager.class).goToSleep(SystemClock.uptimeMillis(), + PowerManager.GO_TO_SLEEP_REASON_ACCESSIBILITY, 0); + mWindowManagerService.lockNow(); + return true; + } + + private boolean takeScreenshot() { + ScreenshotHelper screenshotHelper = (mScreenshotHelperSupplier != null) + ? mScreenshotHelperSupplier.get() : new ScreenshotHelper(mContext); + screenshotHelper.takeScreenshot(android.view.WindowManager.TAKE_SCREENSHOT_FULLSCREEN, + true, true, SCREENSHOT_ACCESSIBILITY_ACTIONS, + new Handler(Looper.getMainLooper()), null); + return true; + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/TouchExplorer.java b/services/accessibility/java/com/android/server/accessibility/TouchExplorer.java deleted file mode 100644 index 294cc5aad132..000000000000 --- a/services/accessibility/java/com/android/server/accessibility/TouchExplorer.java +++ /dev/null @@ -1,1728 +0,0 @@ -/* - ** Copyright 2011, The Android Open Source Project - ** - ** Licensed under the Apache License, Version 2.0 (the "License"); - ** you may not use this file except in compliance with the License. - ** You may obtain a copy of the License at - ** - ** http://www.apache.org/licenses/LICENSE-2.0 - ** - ** Unless required by applicable law or agreed to in writing, software - ** distributed under the License is distributed on an "AS IS" BASIS, - ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ** See the License for the specific language governing permissions and - ** limitations under the License. - */ - -package com.android.server.accessibility; - -import android.content.Context; -import android.graphics.Point; -import android.os.Handler; -import android.util.Slog; -import android.view.InputDevice; -import android.view.MotionEvent; -import android.view.MotionEvent.PointerCoords; -import android.view.MotionEvent.PointerProperties; -import android.view.ViewConfiguration; -import android.view.accessibility.AccessibilityEvent; -import android.view.accessibility.AccessibilityManager; -import android.view.accessibility.AccessibilityNodeInfo; - -import com.android.server.policy.WindowManagerPolicy; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * This class is a strategy for performing touch exploration. It - * transforms the motion event stream by modifying, adding, replacing, - * and consuming certain events. The interaction model is: - * - * <ol> - * <li>1. One finger moving slow around performs touch exploration.</li> - * <li>2. One finger moving fast around performs gestures.</li> - * <li>3. Two close fingers moving in the same direction perform a drag.</li> - * <li>4. Multi-finger gestures are delivered to view hierarchy.</li> - * <li>5. Two fingers moving in different directions are considered a multi-finger gesture.</li> - * <li>7. Double tapping clicks on the on the last touch explored location if it was in - * a window that does not take focus, otherwise the click is within the accessibility - * focused rectangle.</li> - * <li>7. Tapping and holding for a while performs a long press in a similar fashion - * as the click above.</li> - * <ol> - * - * @hide - */ -class TouchExplorer extends BaseEventStreamTransformation - implements AccessibilityGestureDetector.Listener { - - private static final boolean DEBUG = false; - - // Tag for logging received events. - private static final String LOG_TAG = "TouchExplorer"; - - // States this explorer can be in. - private static final int STATE_TOUCH_EXPLORING = 0x00000001; - private static final int STATE_DRAGGING = 0x00000002; - private static final int STATE_DELEGATING = 0x00000004; - private static final int STATE_GESTURE_DETECTING = 0x00000005; - - private static final int CLICK_LOCATION_NONE = 0; - private static final int CLICK_LOCATION_ACCESSIBILITY_FOCUS = 1; - private static final int CLICK_LOCATION_LAST_TOUCH_EXPLORED = 2; - - // The maximum of the cosine between the vectors of two moving - // pointers so they can be considered moving in the same direction. - private static final float MAX_DRAGGING_ANGLE_COS = 0.525321989f; // cos(pi/4) - - // Constant referring to the ids bits of all pointers. - private static final int ALL_POINTER_ID_BITS = 0xFFFFFFFF; - - // This constant captures the current implementation detail that - // pointer IDs are between 0 and 31 inclusive (subject to change). - // (See MAX_POINTER_ID in frameworks/base/include/ui/Input.h) - private static final int MAX_POINTER_COUNT = 32; - - // Invalid pointer ID. - private static final int INVALID_POINTER_ID = -1; - - // The minimal distance before we take the middle of the distance between - // the two dragging pointers as opposed to use the location of the primary one. - private static final int MIN_POINTER_DISTANCE_TO_USE_MIDDLE_LOCATION_DIP = 200; - - // The timeout after which we are no longer trying to detect a gesture. - private static final int EXIT_GESTURE_DETECTION_TIMEOUT = 2000; - - // Timeout before trying to decide what the user is trying to do. - private final int mDetermineUserIntentTimeout; - - // Slop between the first and second tap to be a double tap. - private final int mDoubleTapSlop; - - // The current state of the touch explorer. - private int mCurrentState = STATE_TOUCH_EXPLORING; - - // The ID of the pointer used for dragging. - private int mDraggingPointerId; - - // Handler for performing asynchronous operations. - private final Handler mHandler; - - // Command for delayed sending of a hover enter and move event. - private final SendHoverEnterAndMoveDelayed mSendHoverEnterAndMoveDelayed; - - // Command for delayed sending of a hover exit event. - private final SendHoverExitDelayed mSendHoverExitDelayed; - - // Command for delayed sending of touch exploration end events. - private final SendAccessibilityEventDelayed mSendTouchExplorationEndDelayed; - - // Command for delayed sending of touch interaction end events. - private final SendAccessibilityEventDelayed mSendTouchInteractionEndDelayed; - - // Command for exiting gesture detection mode after a timeout. - private final ExitGestureDetectionModeDelayed mExitGestureDetectionModeDelayed; - - // Helper to detect gestures. - private final AccessibilityGestureDetector mGestureDetector; - - // The scaled minimal distance before we take the middle of the distance between - // the two dragging pointers as opposed to use the location of the primary one. - private final int mScaledMinPointerDistanceToUseMiddleLocation; - - // Helper class to track received pointers. - private final ReceivedPointerTracker mReceivedPointerTracker; - - // Helper class to track injected pointers. - private final InjectedPointerTracker mInjectedPointerTracker; - - // Handle to the accessibility manager service. - private final AccessibilityManagerService mAms; - - // Temporary point to avoid instantiation. - private final Point mTempPoint = new Point(); - - // Context in which this explorer operates. - private final Context mContext; - - // The long pressing pointer id if coordinate remapping is needed. - private int mLongPressingPointerId = -1; - - // The long pressing pointer X if coordinate remapping is needed. - private int mLongPressingPointerDeltaX; - - // The long pressing pointer Y if coordinate remapping is needed. - private int mLongPressingPointerDeltaY; - - // The id of the last touch explored window. - private int mLastTouchedWindowId; - - // Whether touch exploration is in progress. - private boolean mTouchExplorationInProgress; - - /** - * Creates a new instance. - * - * @param context A context handle for accessing resources. - * @param service The service to notify touch interaction and gesture completed and to perform - * action. - */ - public TouchExplorer(Context context, AccessibilityManagerService service) { - this(context, service, null); - } - - /** - * Creates a new instance. - * - * @param context A context handle for accessing resources. - * @param service The service to notify touch interaction and gesture completed and to perform - * action. - * @param detector The gesture detector to handle accessibility touch event. If null the default - * one created in place, or for testing purpose. - */ - public TouchExplorer(Context context, AccessibilityManagerService service, - AccessibilityGestureDetector detector) { - mContext = context; - mAms = service; - mReceivedPointerTracker = new ReceivedPointerTracker(); - mInjectedPointerTracker = new InjectedPointerTracker(); - mDetermineUserIntentTimeout = ViewConfiguration.getDoubleTapTimeout(); - mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop(); - mHandler = new Handler(context.getMainLooper()); - mExitGestureDetectionModeDelayed = new ExitGestureDetectionModeDelayed(); - mSendHoverEnterAndMoveDelayed = new SendHoverEnterAndMoveDelayed(); - mSendHoverExitDelayed = new SendHoverExitDelayed(); - mSendTouchExplorationEndDelayed = new SendAccessibilityEventDelayed( - AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END, - mDetermineUserIntentTimeout); - mSendTouchInteractionEndDelayed = new SendAccessibilityEventDelayed( - AccessibilityEvent.TYPE_TOUCH_INTERACTION_END, - mDetermineUserIntentTimeout); - if (detector == null) { - mGestureDetector = new AccessibilityGestureDetector(context, this); - } else { - mGestureDetector = detector; - } - final float density = context.getResources().getDisplayMetrics().density; - mScaledMinPointerDistanceToUseMiddleLocation = - (int) (MIN_POINTER_DISTANCE_TO_USE_MIDDLE_LOCATION_DIP * density); - } - - @Override - public void clearEvents(int inputSource) { - if (inputSource == InputDevice.SOURCE_TOUCHSCREEN) { - clear(); - } - super.clearEvents(inputSource); - } - - @Override - public void onDestroy() { - clear(); - } - - private void clear() { - // If we have not received an event then we are in initial - // state. Therefore, there is not need to clean anything. - MotionEvent event = mReceivedPointerTracker.getLastReceivedEvent(); - if (event != null) { - clear(mReceivedPointerTracker.getLastReceivedEvent(), WindowManagerPolicy.FLAG_TRUSTED); - } - } - - private void clear(MotionEvent event, int policyFlags) { - switch (mCurrentState) { - case STATE_TOUCH_EXPLORING: { - // If a touch exploration gesture is in progress send events for its end. - sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); - } break; - case STATE_DRAGGING: { - mDraggingPointerId = INVALID_POINTER_ID; - // Send exit to all pointers that we have delivered. - sendUpForInjectedDownPointers(event, policyFlags); - } break; - case STATE_DELEGATING: { - // Send exit to all pointers that we have delivered. - sendUpForInjectedDownPointers(event, policyFlags); - } break; - case STATE_GESTURE_DETECTING: { - // No state specific cleanup required. - } break; - } - // Remove all pending callbacks. - mSendHoverEnterAndMoveDelayed.cancel(); - mSendHoverExitDelayed.cancel(); - mExitGestureDetectionModeDelayed.cancel(); - mSendTouchExplorationEndDelayed.cancel(); - mSendTouchInteractionEndDelayed.cancel(); - // Reset the pointer trackers. - mReceivedPointerTracker.clear(); - mInjectedPointerTracker.clear(); - // Clear the gesture detector - mGestureDetector.clear(); - // Go to initial state. - // Clear the long pressing pointer remap data. - mLongPressingPointerId = -1; - mLongPressingPointerDeltaX = 0; - mLongPressingPointerDeltaY = 0; - mCurrentState = STATE_TOUCH_EXPLORING; - mTouchExplorationInProgress = false; - mAms.onTouchInteractionEnd(); - } - - @Override - public void onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { - if (!event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)) { - super.onMotionEvent(event, rawEvent, policyFlags); - return; - } - - if (DEBUG) { - Slog.d(LOG_TAG, "Received event: " + event + ", policyFlags=0x" - + Integer.toHexString(policyFlags)); - Slog.d(LOG_TAG, getStateSymbolicName(mCurrentState)); - } - - mReceivedPointerTracker.onMotionEvent(rawEvent); - - if (mGestureDetector.onMotionEvent(event, rawEvent, policyFlags)) { - // Event was handled by the gesture detector. - return; - } - - if (event.getActionMasked() == MotionEvent.ACTION_CANCEL) { - clear(event, policyFlags); - return; - } - - switch(mCurrentState) { - case STATE_TOUCH_EXPLORING: { - handleMotionEventStateTouchExploring(event, rawEvent, policyFlags); - } break; - case STATE_DRAGGING: { - handleMotionEventStateDragging(event, policyFlags); - } break; - case STATE_DELEGATING: { - handleMotionEventStateDelegating(event, policyFlags); - } break; - case STATE_GESTURE_DETECTING: { - // Already handled. - } break; - default: - Slog.e(LOG_TAG, "Illegal state: " + mCurrentState); - clear(event, policyFlags); - } - } - - @Override - public void onAccessibilityEvent(AccessibilityEvent event) { - final int eventType = event.getEventType(); - - // The event for gesture end should be strictly after the - // last hover exit event. - if (mSendTouchExplorationEndDelayed.isPending() - && eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT) { - mSendTouchExplorationEndDelayed.cancel(); - sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END); - } - - // The event for touch interaction end should be strictly after the - // last hover exit and the touch exploration gesture end events. - if (mSendTouchInteractionEndDelayed.isPending() - && eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT) { - mSendTouchInteractionEndDelayed.cancel(); - sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); - } - - // If a new window opens or the accessibility focus moves we no longer - // want to click/long press on the last touch explored location. - switch (eventType) { - case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: - case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED: { - if (mInjectedPointerTracker.mLastInjectedHoverEventForClick != null) { - mInjectedPointerTracker.mLastInjectedHoverEventForClick.recycle(); - mInjectedPointerTracker.mLastInjectedHoverEventForClick = null; - } - mLastTouchedWindowId = -1; - } break; - case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER: - case AccessibilityEvent.TYPE_VIEW_HOVER_EXIT: { - mLastTouchedWindowId = event.getWindowId(); - } break; - } - super.onAccessibilityEvent(event); - } - - @Override - public void onDoubleTapAndHold(MotionEvent event, int policyFlags) { - // Ignore the event if we aren't touch exploring. - if (mCurrentState != STATE_TOUCH_EXPLORING) { - return; - } - - // Pointers should not be zero when running this command. - if (mReceivedPointerTracker.getLastReceivedEvent().getPointerCount() == 0) { - return; - } - - final int pointerIndex = event.getActionIndex(); - final int pointerId = event.getPointerId(pointerIndex); - - Point clickLocation = mTempPoint; - final int result = computeClickLocation(clickLocation); - - if (result == CLICK_LOCATION_NONE) { - return; - } - - mLongPressingPointerId = pointerId; - mLongPressingPointerDeltaX = (int) event.getX(pointerIndex) - clickLocation.x; - mLongPressingPointerDeltaY = (int) event.getY(pointerIndex) - clickLocation.y; - - sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); - - mCurrentState = STATE_DELEGATING; - sendDownForAllNotInjectedPointers(event, policyFlags); - } - - @Override - public boolean onDoubleTap(MotionEvent event, int policyFlags) { - // Ignore the event if we aren't touch exploring. - if (mCurrentState != STATE_TOUCH_EXPLORING) { - return false; - } - - mAms.onTouchInteractionEnd(); - // Remove pending event deliveries. - mSendHoverEnterAndMoveDelayed.cancel(); - mSendHoverExitDelayed.cancel(); - - if (mSendTouchExplorationEndDelayed.isPending()) { - mSendTouchExplorationEndDelayed.forceSendAndRemove(); - } - - // Announce the end of a new touch interaction. - sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); - - // Try to use the standard accessibility API to click - if (mAms.performActionOnAccessibilityFocusedItem( - AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK)) { - return true; - } - Slog.e(LOG_TAG, "ACTION_CLICK failed. Dispatching motion events to simulate click."); - - final int pointerIndex = event.getActionIndex(); - final int pointerId = event.getPointerId(pointerIndex); - - Point clickLocation = mTempPoint; - final int result = computeClickLocation(clickLocation); - if (result == CLICK_LOCATION_NONE) { - // We can't send a click to no location, but the gesture was still - // consumed. - return true; - } - - // Do the click. - PointerProperties[] properties = new PointerProperties[1]; - properties[0] = new PointerProperties(); - event.getPointerProperties(pointerIndex, properties[0]); - PointerCoords[] coords = new PointerCoords[1]; - coords[0] = new PointerCoords(); - coords[0].x = clickLocation.x; - coords[0].y = clickLocation.y; - MotionEvent click_event = MotionEvent.obtain(event.getDownTime(), - event.getEventTime(), MotionEvent.ACTION_DOWN, 1, properties, - coords, 0, 0, 1.0f, 1.0f, event.getDeviceId(), 0, - event.getSource(), event.getDisplayId(), event.getFlags()); - final boolean targetAccessibilityFocus = (result == CLICK_LOCATION_ACCESSIBILITY_FOCUS); - sendActionDownAndUp(click_event, policyFlags, targetAccessibilityFocus); - click_event.recycle(); - return true; - } - - @Override - public boolean onGestureStarted() { - // We have to perform gesture detection, so - // clear the current state and try to detect. - mCurrentState = STATE_GESTURE_DETECTING; - mSendHoverEnterAndMoveDelayed.cancel(); - mSendHoverExitDelayed.cancel(); - mExitGestureDetectionModeDelayed.post(); - // Send accessibility event to announce the start - // of gesture recognition. - sendAccessibilityEvent(AccessibilityEvent.TYPE_GESTURE_DETECTION_START); - return false; - } - - @Override - public boolean onGestureCompleted(int gestureId) { - if (mCurrentState != STATE_GESTURE_DETECTING) { - return false; - } - - endGestureDetection(true); - - mAms.onGesture(gestureId); - - return true; - } - - @Override - public boolean onGestureCancelled(MotionEvent event, int policyFlags) { - if (mCurrentState == STATE_GESTURE_DETECTING) { - endGestureDetection(event.getActionMasked() == MotionEvent.ACTION_UP); - return true; - } else if (mCurrentState == STATE_TOUCH_EXPLORING) { - // If the finger is still moving, pass the event on. - if (event.getActionMasked() == MotionEvent.ACTION_MOVE) { - final int pointerId = mReceivedPointerTracker.getPrimaryPointerId(); - final int pointerIdBits = (1 << pointerId); - - // We have just decided that the user is touch, - // exploring so start sending events. - mSendHoverEnterAndMoveDelayed.addEvent(event); - mSendHoverEnterAndMoveDelayed.forceSendAndRemove(); - mSendHoverExitDelayed.cancel(); - sendMotionEvent(event, MotionEvent.ACTION_HOVER_MOVE, pointerIdBits, policyFlags); - return true; - } - } - return false; - } - - /** - * Handles a motion event in touch exploring state. - * - * @param event The event to be handled. - * @param rawEvent The raw (unmodified) motion event. - * @param policyFlags The policy flags associated with the event. - */ - private void handleMotionEventStateTouchExploring(MotionEvent event, MotionEvent rawEvent, - int policyFlags) { - ReceivedPointerTracker receivedTracker = mReceivedPointerTracker; - - switch (event.getActionMasked()) { - case MotionEvent.ACTION_DOWN: { - mAms.onTouchInteractionStart(); - - // If we still have not notified the user for the last - // touch, we figure out what to do. If were waiting - // we resent the delayed callback and wait again. - mSendHoverEnterAndMoveDelayed.cancel(); - mSendHoverExitDelayed.cancel(); - - // If a touch exploration gesture is in progress send events for its end. - if(mTouchExplorationInProgress) { - sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); - } - - // Avoid duplicated TYPE_TOUCH_INTERACTION_START event when 2nd tap of double tap. - if (!mGestureDetector.firstTapDetected()) { - mSendTouchExplorationEndDelayed.forceSendAndRemove(); - mSendTouchInteractionEndDelayed.forceSendAndRemove(); - sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_START); - } else { - // Let gesture to handle to avoid duplicated TYPE_TOUCH_INTERACTION_END event. - mSendTouchInteractionEndDelayed.cancel(); - } - - if (!mGestureDetector.firstTapDetected() && !mTouchExplorationInProgress) { - if (!mSendHoverEnterAndMoveDelayed.isPending()) { - // Deliver hover enter with a delay to have a chance - // to detect what the user is trying to do. - final int pointerId = receivedTracker.getPrimaryPointerId(); - final int pointerIdBits = (1 << pointerId); - mSendHoverEnterAndMoveDelayed.post(event, true, pointerIdBits, - policyFlags); - } else { - // Cache the event until we discern exploration from gesturing. - mSendHoverEnterAndMoveDelayed.addEvent(event); - } - } - } break; - case MotionEvent.ACTION_POINTER_DOWN: { - // Another finger down means that if we have not started to deliver - // hover events, we will not have to. The code for ACTION_MOVE will - // decide what we will actually do next. - mSendHoverEnterAndMoveDelayed.cancel(); - mSendHoverExitDelayed.cancel(); - } break; - case MotionEvent.ACTION_MOVE: { - final int pointerId = receivedTracker.getPrimaryPointerId(); - final int pointerIndex = event.findPointerIndex(pointerId); - final int pointerIdBits = (1 << pointerId); - switch (event.getPointerCount()) { - case 1: { - // We have not started sending events since we try to - // figure out what the user is doing. - if (mSendHoverEnterAndMoveDelayed.isPending()) { - // Cache the event until we discern exploration from gesturing. - mSendHoverEnterAndMoveDelayed.addEvent(event); - } else { - if (mTouchExplorationInProgress) { - sendTouchExplorationGestureStartAndHoverEnterIfNeeded(policyFlags); - sendMotionEvent(event, MotionEvent.ACTION_HOVER_MOVE, pointerIdBits, - policyFlags); - } - } - } break; - case 2: { - // More than one pointer so the user is not touch exploring - // and now we have to decide whether to delegate or drag. - if (mSendHoverEnterAndMoveDelayed.isPending()) { - // We have not started sending events so cancel - // scheduled sending events. - mSendHoverEnterAndMoveDelayed.cancel(); - mSendHoverExitDelayed.cancel(); - } else { - if (mTouchExplorationInProgress) { - // If the user is touch exploring the second pointer may be - // performing a double tap to activate an item without need - // for the user to lift his exploring finger. - // It is *important* to use the distance traveled by the pointers - // on the screen which may or may not be magnified. - final float deltaX = receivedTracker.getReceivedPointerDownX( - pointerId) - rawEvent.getX(pointerIndex); - final float deltaY = receivedTracker.getReceivedPointerDownY( - pointerId) - rawEvent.getY(pointerIndex); - final double moveDelta = Math.hypot(deltaX, deltaY); - if (moveDelta < mDoubleTapSlop) { - break; - } - // We are sending events so send exit and gesture - // end since we transition to another state. - sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); - } - } - - // Remove move history before send injected non-move events - event = MotionEvent.obtainNoHistory(event); - if (isDraggingGesture(event)) { - // Two pointers moving in the same direction within - // a given distance perform a drag. - mCurrentState = STATE_DRAGGING; - mDraggingPointerId = pointerId; - event.setEdgeFlags(receivedTracker.getLastReceivedDownEdgeFlags()); - sendMotionEvent(event, MotionEvent.ACTION_DOWN, pointerIdBits, - policyFlags); - } else { - // Two pointers moving arbitrary are delegated to the view hierarchy. - mCurrentState = STATE_DELEGATING; - sendDownForAllNotInjectedPointers(event, policyFlags); - } - } break; - default: { - // More than one pointer so the user is not touch exploring - // and now we have to decide whether to delegate or drag. - if (mSendHoverEnterAndMoveDelayed.isPending()) { - // We have not started sending events so cancel - // scheduled sending events. - mSendHoverEnterAndMoveDelayed.cancel(); - mSendHoverExitDelayed.cancel(); - } else { - // We are sending events so send exit and gesture - // end since we transition to another state. - sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); - } - - // More than two pointers are delegated to the view hierarchy. - mCurrentState = STATE_DELEGATING; - event = MotionEvent.obtainNoHistory(event); - sendDownForAllNotInjectedPointers(event, policyFlags); - } - } - } break; - case MotionEvent.ACTION_UP: { - mAms.onTouchInteractionEnd(); - final int pointerId = event.getPointerId(event.getActionIndex()); - final int pointerIdBits = (1 << pointerId); - - if (mSendHoverEnterAndMoveDelayed.isPending()) { - // If we have not delivered the enter schedule an exit. - mSendHoverExitDelayed.post(event, pointerIdBits, policyFlags); - } else { - // The user is touch exploring so we send events for end. - sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); - } - - if (!mSendTouchInteractionEndDelayed.isPending()) { - mSendTouchInteractionEndDelayed.post(); - } - - } break; - } - } - - /** - * Handles a motion event in dragging state. - * - * @param event The event to be handled. - * @param policyFlags The policy flags associated with the event. - */ - private void handleMotionEventStateDragging(MotionEvent event, int policyFlags) { - int pointerIdBits = 0; - // Clear the dragging pointer id if it's no longer valid. - if (event.findPointerIndex(mDraggingPointerId) == -1) { - Slog.e(LOG_TAG, "mDraggingPointerId doesn't match any pointers on current event. " + - "mDraggingPointerId: " + Integer.toString(mDraggingPointerId) + - ", Event: " + event); - mDraggingPointerId = INVALID_POINTER_ID; - } else { - pointerIdBits = (1 << mDraggingPointerId); - } - switch (event.getActionMasked()) { - case MotionEvent.ACTION_DOWN: { - Slog.e(LOG_TAG, "Dragging state can be reached only if two " - + "pointers are already down"); - clear(event, policyFlags); - return; - } - case MotionEvent.ACTION_POINTER_DOWN: { - // We are in dragging state so we have two pointers and another one - // goes down => delegate the three pointers to the view hierarchy - mCurrentState = STATE_DELEGATING; - if (mDraggingPointerId != INVALID_POINTER_ID) { - sendMotionEvent(event, MotionEvent.ACTION_UP, pointerIdBits, policyFlags); - } - sendDownForAllNotInjectedPointers(event, policyFlags); - } break; - case MotionEvent.ACTION_MOVE: { - if (mDraggingPointerId == INVALID_POINTER_ID) { - break; - } - switch (event.getPointerCount()) { - case 1: { - // do nothing - } break; - case 2: { - if (isDraggingGesture(event)) { - final float firstPtrX = event.getX(0); - final float firstPtrY = event.getY(0); - final float secondPtrX = event.getX(1); - final float secondPtrY = event.getY(1); - - final int pointerIndex = event.findPointerIndex(mDraggingPointerId); - final float deltaX = - (pointerIndex == 0) ? (secondPtrX - firstPtrX) - : (firstPtrX - secondPtrX); - final float deltaY = - (pointerIndex == 0) ? (secondPtrY - firstPtrY) - : (firstPtrY - secondPtrY); - final double distance = Math.hypot(deltaX, deltaY); - - if (distance > mScaledMinPointerDistanceToUseMiddleLocation) { - event.offsetLocation(deltaX / 2, deltaY / 2); - } - - // If still dragging send a drag event. - sendMotionEvent(event, MotionEvent.ACTION_MOVE, pointerIdBits, - policyFlags); - } else { - // The two pointers are moving either in different directions or - // no close enough => delegate the gesture to the view hierarchy. - mCurrentState = STATE_DELEGATING; - // Remove move history before send injected non-move events - event = MotionEvent.obtainNoHistory(event); - // Send an event to the end of the drag gesture. - sendMotionEvent(event, MotionEvent.ACTION_UP, pointerIdBits, - policyFlags); - // Deliver all pointers to the view hierarchy. - sendDownForAllNotInjectedPointers(event, policyFlags); - } - } break; - default: { - mCurrentState = STATE_DELEGATING; - event = MotionEvent.obtainNoHistory(event); - // Send an event to the end of the drag gesture. - sendMotionEvent(event, MotionEvent.ACTION_UP, pointerIdBits, - policyFlags); - // Deliver all pointers to the view hierarchy. - sendDownForAllNotInjectedPointers(event, policyFlags); - } - } - } break; - case MotionEvent.ACTION_POINTER_UP: { - final int pointerId = event.getPointerId(event.getActionIndex()); - if (pointerId == mDraggingPointerId) { - mDraggingPointerId = INVALID_POINTER_ID; - // Send an event to the end of the drag gesture. - sendMotionEvent(event, MotionEvent.ACTION_UP, pointerIdBits, policyFlags); - } - } break; - case MotionEvent.ACTION_UP: { - mAms.onTouchInteractionEnd(); - // Announce the end of a new touch interaction. - sendAccessibilityEvent( - AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); - final int pointerId = event.getPointerId(event.getActionIndex()); - if (pointerId == mDraggingPointerId) { - mDraggingPointerId = INVALID_POINTER_ID; - // Send an event to the end of the drag gesture. - sendMotionEvent(event, MotionEvent.ACTION_UP, pointerIdBits, policyFlags); - } - mCurrentState = STATE_TOUCH_EXPLORING; - } break; - } - } - - /** - * Handles a motion event in delegating state. - * - * @param event The event to be handled. - * @param policyFlags The policy flags associated with the event. - */ - private void handleMotionEventStateDelegating(MotionEvent event, int policyFlags) { - switch (event.getActionMasked()) { - case MotionEvent.ACTION_DOWN: { - Slog.e(LOG_TAG, "Delegating state can only be reached if " - + "there is at least one pointer down!"); - clear(event, policyFlags); - return; - } - case MotionEvent.ACTION_UP: { - // Offset the event if we are doing a long press as the - // target is not necessarily under the user's finger. - if (mLongPressingPointerId >= 0) { - event = offsetEvent(event, - mLongPressingPointerDeltaX, - - mLongPressingPointerDeltaY); - // Clear the long press state. - mLongPressingPointerId = -1; - mLongPressingPointerDeltaX = 0; - mLongPressingPointerDeltaY = 0; - } - - // Deliver the event. - sendMotionEvent(event, event.getAction(), ALL_POINTER_ID_BITS, policyFlags); - - // Announce the end of a the touch interaction. - mAms.onTouchInteractionEnd(); - sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); - - mCurrentState = STATE_TOUCH_EXPLORING; - } break; - default: { - // Deliver the event. - sendMotionEvent(event, event.getAction(), ALL_POINTER_ID_BITS, policyFlags); - } - } - } - - private void endGestureDetection(boolean interactionEnd) { - mAms.onTouchInteractionEnd(); - - // Announce the end of the gesture recognition. - sendAccessibilityEvent(AccessibilityEvent.TYPE_GESTURE_DETECTION_END); - // Don't announce the end of a the touch interaction if users didn't lift their fingers. - if (interactionEnd) { - sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); - } - - mExitGestureDetectionModeDelayed.cancel(); - mCurrentState = STATE_TOUCH_EXPLORING; - } - - /** - * Sends an accessibility event of the given type. - * - * @param type The event type. - */ - private void sendAccessibilityEvent(int type) { - AccessibilityManager accessibilityManager = AccessibilityManager.getInstance(mContext); - if (accessibilityManager.isEnabled()) { - AccessibilityEvent event = AccessibilityEvent.obtain(type); - event.setWindowId(mAms.getActiveWindowId()); - accessibilityManager.sendAccessibilityEvent(event); - switch (type) { - case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START: { - mTouchExplorationInProgress = true; - } break; - case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END: { - mTouchExplorationInProgress = false; - } break; - } - } - } - - /** - * Sends down events to the view hierarchy for all pointers which are - * not already being delivered i.e. pointers that are not yet injected. - * - * @param prototype The prototype from which to create the injected events. - * @param policyFlags The policy flags associated with the event. - */ - private void sendDownForAllNotInjectedPointers(MotionEvent prototype, int policyFlags) { - InjectedPointerTracker injectedPointers = mInjectedPointerTracker; - - // Inject the injected pointers. - int pointerIdBits = 0; - final int pointerCount = prototype.getPointerCount(); - for (int i = 0; i < pointerCount; i++) { - final int pointerId = prototype.getPointerId(i); - // Do not send event for already delivered pointers. - if (!injectedPointers.isInjectedPointerDown(pointerId)) { - pointerIdBits |= (1 << pointerId); - final int action = computeInjectionAction(MotionEvent.ACTION_DOWN, i); - sendMotionEvent(prototype, action, pointerIdBits, policyFlags); - } - } - } - - /** - * Sends the exit events if needed. Such events are hover exit and touch explore - * gesture end. - * - * @param policyFlags The policy flags associated with the event. - */ - private void sendHoverExitAndTouchExplorationGestureEndIfNeeded(int policyFlags) { - MotionEvent event = mInjectedPointerTracker.getLastInjectedHoverEvent(); - if (event != null && event.getActionMasked() != MotionEvent.ACTION_HOVER_EXIT) { - final int pointerIdBits = event.getPointerIdBits(); - if (!mSendTouchExplorationEndDelayed.isPending()) { - mSendTouchExplorationEndDelayed.post(); - } - sendMotionEvent(event, MotionEvent.ACTION_HOVER_EXIT, pointerIdBits, policyFlags); - } - } - - /** - * Sends the enter events if needed. Such events are hover enter and touch explore - * gesture start. - * - * @param policyFlags The policy flags associated with the event. - */ - private void sendTouchExplorationGestureStartAndHoverEnterIfNeeded(int policyFlags) { - MotionEvent event = mInjectedPointerTracker.getLastInjectedHoverEvent(); - if (event != null && event.getActionMasked() == MotionEvent.ACTION_HOVER_EXIT) { - final int pointerIdBits = event.getPointerIdBits(); - sendMotionEvent(event, MotionEvent.ACTION_HOVER_ENTER, pointerIdBits, policyFlags); - } - } - - /** - * Sends up events to the view hierarchy for all pointers which are - * already being delivered i.e. pointers that are injected. - * - * @param prototype The prototype from which to create the injected events. - * @param policyFlags The policy flags associated with the event. - */ - private void sendUpForInjectedDownPointers(MotionEvent prototype, int policyFlags) { - final InjectedPointerTracker injectedTracked = mInjectedPointerTracker; - int pointerIdBits = 0; - final int pointerCount = prototype.getPointerCount(); - for (int i = 0; i < pointerCount; i++) { - final int pointerId = prototype.getPointerId(i); - // Skip non injected down pointers. - if (!injectedTracked.isInjectedPointerDown(pointerId)) { - continue; - } - pointerIdBits |= (1 << pointerId); - final int action = computeInjectionAction(MotionEvent.ACTION_UP, i); - sendMotionEvent(prototype, action, pointerIdBits, policyFlags); - } - } - - /** - * Sends an up and down events. - * - * @param prototype The prototype from which to create the injected events. - * @param policyFlags The policy flags associated with the event. - * @param targetAccessibilityFocus Whether the event targets the accessibility focus. - */ - private void sendActionDownAndUp(MotionEvent prototype, int policyFlags, - boolean targetAccessibilityFocus) { - // Tap with the pointer that last explored. - final int pointerId = prototype.getPointerId(prototype.getActionIndex()); - final int pointerIdBits = (1 << pointerId); - prototype.setTargetAccessibilityFocus(targetAccessibilityFocus); - sendMotionEvent(prototype, MotionEvent.ACTION_DOWN, pointerIdBits, policyFlags); - prototype.setTargetAccessibilityFocus(targetAccessibilityFocus); - sendMotionEvent(prototype, MotionEvent.ACTION_UP, pointerIdBits, policyFlags); - } - - /** - * Sends an event. - * - * @param prototype The prototype from which to create the injected events. - * @param action The action of the event. - * @param pointerIdBits The bits of the pointers to send. - * @param policyFlags The policy flags associated with the event. - */ - private void sendMotionEvent(MotionEvent prototype, int action, int pointerIdBits, - int policyFlags) { - prototype.setAction(action); - - MotionEvent event = null; - if (pointerIdBits == ALL_POINTER_ID_BITS) { - event = prototype; - } else { - try { - event = prototype.split(pointerIdBits); - } catch (IllegalArgumentException e) { - Slog.e(LOG_TAG, "sendMotionEvent: Failed to split motion event: " + e); - return; - } - } - if (action == MotionEvent.ACTION_DOWN) { - event.setDownTime(event.getEventTime()); - } else { - event.setDownTime(mInjectedPointerTracker.getLastInjectedDownEventTime()); - } - - // If the user is long pressing but the long pressing pointer - // was not exactly over the accessibility focused item we need - // to remap the location of that pointer so the user does not - // have to explicitly touch explore something to be able to - // long press it, or even worse to avoid the user long pressing - // on the wrong item since click and long press behave differently. - if (mLongPressingPointerId >= 0) { - event = offsetEvent(event, - mLongPressingPointerDeltaX, - - mLongPressingPointerDeltaY); - } - - if (DEBUG) { - Slog.d(LOG_TAG, "Injecting event: " + event + ", policyFlags=0x" - + Integer.toHexString(policyFlags)); - } - - // Make sure that the user will see the event. - policyFlags |= WindowManagerPolicy.FLAG_PASS_TO_USER; - // TODO: For now pass null for the raw event since the touch - // explorer is the last event transformation and it does - // not care about the raw event. - super.onMotionEvent(event, null, policyFlags); - - mInjectedPointerTracker.onMotionEvent(event); - - if (event != prototype) { - event.recycle(); - } - } - - /** - * Offsets all pointers in the given event by adding the specified X and Y - * offsets. - * - * @param event The event to offset. - * @param offsetX The X offset. - * @param offsetY The Y offset. - * @return An event with the offset pointers or the original event if both - * offsets are zero. - */ - private MotionEvent offsetEvent(MotionEvent event, int offsetX, int offsetY) { - if (offsetX == 0 && offsetY == 0) { - return event; - } - final int remappedIndex = event.findPointerIndex(mLongPressingPointerId); - final int pointerCount = event.getPointerCount(); - PointerProperties[] props = PointerProperties.createArray(pointerCount); - PointerCoords[] coords = PointerCoords.createArray(pointerCount); - for (int i = 0; i < pointerCount; i++) { - event.getPointerProperties(i, props[i]); - event.getPointerCoords(i, coords[i]); - if (i == remappedIndex) { - coords[i].x += offsetX; - coords[i].y += offsetY; - } - } - return MotionEvent.obtain(event.getDownTime(), - event.getEventTime(), event.getAction(), event.getPointerCount(), - props, coords, event.getMetaState(), event.getButtonState(), - 1.0f, 1.0f, event.getDeviceId(), event.getEdgeFlags(), - event.getSource(), event.getDisplayId(), event.getFlags()); - } - - /** - * Computes the action for an injected event based on a masked action - * and a pointer index. - * - * @param actionMasked The masked action. - * @param pointerIndex The index of the pointer which has changed. - * @return The action to be used for injection. - */ - private int computeInjectionAction(int actionMasked, int pointerIndex) { - switch (actionMasked) { - case MotionEvent.ACTION_DOWN: - case MotionEvent.ACTION_POINTER_DOWN: { - InjectedPointerTracker injectedTracker = mInjectedPointerTracker; - // Compute the action based on how many down pointers are injected. - if (injectedTracker.getInjectedPointerDownCount() == 0) { - return MotionEvent.ACTION_DOWN; - } else { - return (pointerIndex << MotionEvent.ACTION_POINTER_INDEX_SHIFT) - | MotionEvent.ACTION_POINTER_DOWN; - } - } - case MotionEvent.ACTION_POINTER_UP: { - InjectedPointerTracker injectedTracker = mInjectedPointerTracker; - // Compute the action based on how many down pointers are injected. - if (injectedTracker.getInjectedPointerDownCount() == 1) { - return MotionEvent.ACTION_UP; - } else { - return (pointerIndex << MotionEvent.ACTION_POINTER_INDEX_SHIFT) - | MotionEvent.ACTION_POINTER_UP; - } - } - default: - return actionMasked; - } - } - - /** - * Determines whether a two pointer gesture is a dragging one. - * - * @param event The event with the pointer data. - * @return True if the gesture is a dragging one. - */ - private boolean isDraggingGesture(MotionEvent event) { - ReceivedPointerTracker receivedTracker = mReceivedPointerTracker; - - final float firstPtrX = event.getX(0); - final float firstPtrY = event.getY(0); - final float secondPtrX = event.getX(1); - final float secondPtrY = event.getY(1); - - final float firstPtrDownX = receivedTracker.getReceivedPointerDownX(0); - final float firstPtrDownY = receivedTracker.getReceivedPointerDownY(0); - final float secondPtrDownX = receivedTracker.getReceivedPointerDownX(1); - final float secondPtrDownY = receivedTracker.getReceivedPointerDownY(1); - - return GestureUtils.isDraggingGesture(firstPtrDownX, firstPtrDownY, secondPtrDownX, - secondPtrDownY, firstPtrX, firstPtrY, secondPtrX, secondPtrY, - MAX_DRAGGING_ANGLE_COS); - } - - private int computeClickLocation(Point outLocation) { - MotionEvent lastExploreEvent = mInjectedPointerTracker.getLastInjectedHoverEventForClick(); - if (lastExploreEvent != null) { - final int lastExplorePointerIndex = lastExploreEvent.getActionIndex(); - outLocation.x = (int) lastExploreEvent.getX(lastExplorePointerIndex); - outLocation.y = (int) lastExploreEvent.getY(lastExplorePointerIndex); - if (!mAms.accessibilityFocusOnlyInActiveWindow() - || mLastTouchedWindowId == mAms.getActiveWindowId()) { - if (mAms.getAccessibilityFocusClickPointInScreen(outLocation)) { - return CLICK_LOCATION_ACCESSIBILITY_FOCUS; - } else { - return CLICK_LOCATION_LAST_TOUCH_EXPLORED; - } - } - } - if (mAms.getAccessibilityFocusClickPointInScreen(outLocation)) { - return CLICK_LOCATION_ACCESSIBILITY_FOCUS; - } - return CLICK_LOCATION_NONE; - } - - /** - * Gets the symbolic name of a state. - * - * @param state A state. - * @return The state symbolic name. - */ - private static String getStateSymbolicName(int state) { - switch (state) { - case STATE_TOUCH_EXPLORING: - return "STATE_TOUCH_EXPLORING"; - case STATE_DRAGGING: - return "STATE_DRAGGING"; - case STATE_DELEGATING: - return "STATE_DELEGATING"; - case STATE_GESTURE_DETECTING: - return "STATE_GESTURE_DETECTING"; - default: - return "Unknown state: " + state; - } - } - - /** - * Class for delayed exiting from gesture detecting mode. - */ - private final class ExitGestureDetectionModeDelayed implements Runnable { - - public void post() { - mHandler.postDelayed(this, EXIT_GESTURE_DETECTION_TIMEOUT); - } - - public void cancel() { - mHandler.removeCallbacks(this); - } - - @Override - public void run() { - // Announce the end of gesture recognition. - sendAccessibilityEvent(AccessibilityEvent.TYPE_GESTURE_DETECTION_END); - clear(); - } - } - - /** - * Class for delayed sending of hover enter and move events. - */ - class SendHoverEnterAndMoveDelayed implements Runnable { - private final String LOG_TAG_SEND_HOVER_DELAYED = "SendHoverEnterAndMoveDelayed"; - - private final List<MotionEvent> mEvents = new ArrayList<MotionEvent>(); - - private int mPointerIdBits; - private int mPolicyFlags; - - public void post(MotionEvent event, boolean touchExplorationInProgress, - int pointerIdBits, int policyFlags) { - cancel(); - addEvent(event); - mPointerIdBits = pointerIdBits; - mPolicyFlags = policyFlags; - mHandler.postDelayed(this, mDetermineUserIntentTimeout); - } - - public void addEvent(MotionEvent event) { - mEvents.add(MotionEvent.obtain(event)); - } - - public void cancel() { - if (isPending()) { - mHandler.removeCallbacks(this); - clear(); - } - } - - private boolean isPending() { - return mHandler.hasCallbacks(this); - } - - private void clear() { - mPointerIdBits = -1; - mPolicyFlags = 0; - final int eventCount = mEvents.size(); - for (int i = eventCount - 1; i >= 0; i--) { - mEvents.remove(i).recycle(); - } - } - - public void forceSendAndRemove() { - if (isPending()) { - run(); - cancel(); - } - } - - public void run() { - // Send an accessibility event to announce the touch exploration start. - sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START); - - if (!mEvents.isEmpty()) { - // Deliver a down event. - sendMotionEvent(mEvents.get(0), MotionEvent.ACTION_HOVER_ENTER, - mPointerIdBits, mPolicyFlags); - if (DEBUG) { - Slog.d(LOG_TAG_SEND_HOVER_DELAYED, - "Injecting motion event: ACTION_HOVER_ENTER"); - } - - // Deliver move events. - final int eventCount = mEvents.size(); - for (int i = 1; i < eventCount; i++) { - sendMotionEvent(mEvents.get(i), MotionEvent.ACTION_HOVER_MOVE, - mPointerIdBits, mPolicyFlags); - if (DEBUG) { - Slog.d(LOG_TAG_SEND_HOVER_DELAYED, - "Injecting motion event: ACTION_HOVER_MOVE"); - } - } - } - clear(); - } - } - - /** - * Class for delayed sending of hover exit events. - */ - class SendHoverExitDelayed implements Runnable { - private final String LOG_TAG_SEND_HOVER_DELAYED = "SendHoverExitDelayed"; - - private MotionEvent mPrototype; - private int mPointerIdBits; - private int mPolicyFlags; - - public void post(MotionEvent prototype, int pointerIdBits, int policyFlags) { - cancel(); - mPrototype = MotionEvent.obtain(prototype); - mPointerIdBits = pointerIdBits; - mPolicyFlags = policyFlags; - mHandler.postDelayed(this, mDetermineUserIntentTimeout); - } - - public void cancel() { - if (isPending()) { - mHandler.removeCallbacks(this); - clear(); - } - } - - private boolean isPending() { - return mHandler.hasCallbacks(this); - } - - private void clear() { - mPrototype.recycle(); - mPrototype = null; - mPointerIdBits = -1; - mPolicyFlags = 0; - } - - public void forceSendAndRemove() { - if (isPending()) { - run(); - cancel(); - } - } - - public void run() { - if (DEBUG) { - Slog.d(LOG_TAG_SEND_HOVER_DELAYED, "Injecting motion event:" - + " ACTION_HOVER_EXIT"); - } - sendMotionEvent(mPrototype, MotionEvent.ACTION_HOVER_EXIT, - mPointerIdBits, mPolicyFlags); - if (!mSendTouchExplorationEndDelayed.isPending()) { - mSendTouchExplorationEndDelayed.cancel(); - mSendTouchExplorationEndDelayed.post(); - } - if (mSendTouchInteractionEndDelayed.isPending()) { - mSendTouchInteractionEndDelayed.cancel(); - mSendTouchInteractionEndDelayed.post(); - } - clear(); - } - } - - private class SendAccessibilityEventDelayed implements Runnable { - private final int mEventType; - private final int mDelay; - - public SendAccessibilityEventDelayed(int eventType, int delay) { - mEventType = eventType; - mDelay = delay; - } - - public void cancel() { - mHandler.removeCallbacks(this); - } - - public void post() { - mHandler.postDelayed(this, mDelay); - } - - public boolean isPending() { - return mHandler.hasCallbacks(this); - } - - public void forceSendAndRemove() { - if (isPending()) { - run(); - cancel(); - } - } - - @Override - public void run() { - sendAccessibilityEvent(mEventType); - } - } - - @Override - public String toString() { - return "TouchExplorer { " + - "mCurrentState: " + getStateSymbolicName(mCurrentState) + - ", mDetermineUserIntentTimeout: " + mDetermineUserIntentTimeout + - ", mDoubleTapSlop: " + mDoubleTapSlop + - ", mDraggingPointerId: " + mDraggingPointerId + - ", mLongPressingPointerId: " + mLongPressingPointerId + - ", mLongPressingPointerDeltaX: " + mLongPressingPointerDeltaX + - ", mLongPressingPointerDeltaY: " + mLongPressingPointerDeltaY + - ", mLastTouchedWindowId: " + mLastTouchedWindowId + - ", mScaledMinPointerDistanceToUseMiddleLocation: " - + mScaledMinPointerDistanceToUseMiddleLocation + - ", mTempPoint: " + mTempPoint + - ", mTouchExplorationInProgress: " + mTouchExplorationInProgress + - " }"; - } - - class InjectedPointerTracker { - private static final String LOG_TAG_INJECTED_POINTER_TRACKER = "InjectedPointerTracker"; - - // Keep track of which pointers sent to the system are down. - private int mInjectedPointersDown; - - // The time of the last injected down. - private long mLastInjectedDownEventTime; - - // The last injected hover event. - private MotionEvent mLastInjectedHoverEvent; - - // The last injected hover event used for performing clicks. - private MotionEvent mLastInjectedHoverEventForClick; - - /** - * Processes an injected {@link MotionEvent} event. - * - * @param event The event to process. - */ - public void onMotionEvent(MotionEvent event) { - final int action = event.getActionMasked(); - switch (action) { - case MotionEvent.ACTION_DOWN: - case MotionEvent.ACTION_POINTER_DOWN: { - final int pointerId = event.getPointerId(event.getActionIndex()); - final int pointerFlag = (1 << pointerId); - mInjectedPointersDown |= pointerFlag; - mLastInjectedDownEventTime = event.getDownTime(); - } break; - case MotionEvent.ACTION_UP: - case MotionEvent.ACTION_POINTER_UP: { - final int pointerId = event.getPointerId(event.getActionIndex()); - final int pointerFlag = (1 << pointerId); - mInjectedPointersDown &= ~pointerFlag; - if (mInjectedPointersDown == 0) { - mLastInjectedDownEventTime = 0; - } - } break; - case MotionEvent.ACTION_HOVER_ENTER: - case MotionEvent.ACTION_HOVER_MOVE: - case MotionEvent.ACTION_HOVER_EXIT: { - if (mLastInjectedHoverEvent != null) { - mLastInjectedHoverEvent.recycle(); - } - mLastInjectedHoverEvent = MotionEvent.obtain(event); - if (mLastInjectedHoverEventForClick != null) { - mLastInjectedHoverEventForClick.recycle(); - } - mLastInjectedHoverEventForClick = MotionEvent.obtain(event); - } break; - } - if (DEBUG) { - Slog.i(LOG_TAG_INJECTED_POINTER_TRACKER, "Injected pointer:\n" + toString()); - } - } - - /** - * Clears the internals state. - */ - public void clear() { - mInjectedPointersDown = 0; - } - - /** - * @return The time of the last injected down event. - */ - public long getLastInjectedDownEventTime() { - return mLastInjectedDownEventTime; - } - - /** - * @return The number of down pointers injected to the view hierarchy. - */ - public int getInjectedPointerDownCount() { - return Integer.bitCount(mInjectedPointersDown); - } - - /** - * @return The bits of the injected pointers that are down. - */ - public int getInjectedPointersDown() { - return mInjectedPointersDown; - } - - /** - * Whether an injected pointer is down. - * - * @param pointerId The unique pointer id. - * @return True if the pointer is down. - */ - public boolean isInjectedPointerDown(int pointerId) { - final int pointerFlag = (1 << pointerId); - return (mInjectedPointersDown & pointerFlag) != 0; - } - - /** - * @return The the last injected hover event. - */ - public MotionEvent getLastInjectedHoverEvent() { - return mLastInjectedHoverEvent; - } - - /** - * @return The the last injected hover event. - */ - public MotionEvent getLastInjectedHoverEventForClick() { - return mLastInjectedHoverEventForClick; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("========================="); - builder.append("\nDown pointers #"); - builder.append(Integer.bitCount(mInjectedPointersDown)); - builder.append(" [ "); - for (int i = 0; i < MAX_POINTER_COUNT; i++) { - if ((mInjectedPointersDown & i) != 0) { - builder.append(i); - builder.append(" "); - } - } - builder.append("]"); - builder.append("\n========================="); - return builder.toString(); - } - } - - class ReceivedPointerTracker { - private static final String LOG_TAG_RECEIVED_POINTER_TRACKER = "ReceivedPointerTracker"; - - // Keep track of where and when a pointer went down. - private final float[] mReceivedPointerDownX = new float[MAX_POINTER_COUNT]; - private final float[] mReceivedPointerDownY = new float[MAX_POINTER_COUNT]; - private final long[] mReceivedPointerDownTime = new long[MAX_POINTER_COUNT]; - - // Which pointers are down. - private int mReceivedPointersDown; - - // The edge flags of the last received down event. - private int mLastReceivedDownEdgeFlags; - - // Primary pointer which is either the first that went down - // or if it goes up the next one that most recently went down. - private int mPrimaryPointerId; - - // Keep track of the last up pointer data. - private long mLastReceivedUpPointerDownTime; - private float mLastReceivedUpPointerDownX; - private float mLastReceivedUpPointerDownY; - - private MotionEvent mLastReceivedEvent; - - /** - * Clears the internals state. - */ - public void clear() { - Arrays.fill(mReceivedPointerDownX, 0); - Arrays.fill(mReceivedPointerDownY, 0); - Arrays.fill(mReceivedPointerDownTime, 0); - mReceivedPointersDown = 0; - mPrimaryPointerId = 0; - mLastReceivedUpPointerDownTime = 0; - mLastReceivedUpPointerDownX = 0; - mLastReceivedUpPointerDownY = 0; - } - - /** - * Processes a received {@link MotionEvent} event. - * - * @param event The event to process. - */ - public void onMotionEvent(MotionEvent event) { - if (mLastReceivedEvent != null) { - mLastReceivedEvent.recycle(); - } - mLastReceivedEvent = MotionEvent.obtain(event); - - final int action = event.getActionMasked(); - switch (action) { - case MotionEvent.ACTION_DOWN: { - handleReceivedPointerDown(event.getActionIndex(), event); - } break; - case MotionEvent.ACTION_POINTER_DOWN: { - handleReceivedPointerDown(event.getActionIndex(), event); - } break; - case MotionEvent.ACTION_UP: { - handleReceivedPointerUp(event.getActionIndex(), event); - } break; - case MotionEvent.ACTION_POINTER_UP: { - handleReceivedPointerUp(event.getActionIndex(), event); - } break; - } - if (DEBUG) { - Slog.i(LOG_TAG_RECEIVED_POINTER_TRACKER, "Received pointer:\n" + toString()); - } - } - - /** - * @return The last received event. - */ - public MotionEvent getLastReceivedEvent() { - return mLastReceivedEvent; - } - - /** - * @return The number of received pointers that are down. - */ - public int getReceivedPointerDownCount() { - return Integer.bitCount(mReceivedPointersDown); - } - - /** - * Whether an received pointer is down. - * - * @param pointerId The unique pointer id. - * @return True if the pointer is down. - */ - public boolean isReceivedPointerDown(int pointerId) { - final int pointerFlag = (1 << pointerId); - return (mReceivedPointersDown & pointerFlag) != 0; - } - - /** - * @param pointerId The unique pointer id. - * @return The X coordinate where the pointer went down. - */ - public float getReceivedPointerDownX(int pointerId) { - return mReceivedPointerDownX[pointerId]; - } - - /** - * @param pointerId The unique pointer id. - * @return The Y coordinate where the pointer went down. - */ - public float getReceivedPointerDownY(int pointerId) { - return mReceivedPointerDownY[pointerId]; - } - - /** - * @param pointerId The unique pointer id. - * @return The time when the pointer went down. - */ - public long getReceivedPointerDownTime(int pointerId) { - return mReceivedPointerDownTime[pointerId]; - } - - /** - * @return The id of the primary pointer. - */ - public int getPrimaryPointerId() { - if (mPrimaryPointerId == INVALID_POINTER_ID) { - mPrimaryPointerId = findPrimaryPointerId(); - } - return mPrimaryPointerId; - } - - /** - * @return The time when the last up received pointer went down. - */ - public long getLastReceivedUpPointerDownTime() { - return mLastReceivedUpPointerDownTime; - } - - /** - * @return The down X of the last received pointer that went up. - */ - public float getLastReceivedUpPointerDownX() { - return mLastReceivedUpPointerDownX; - } - - /** - * @return The down Y of the last received pointer that went up. - */ - public float getLastReceivedUpPointerDownY() { - return mLastReceivedUpPointerDownY; - } - - /** - * @return The edge flags of the last received down event. - */ - public int getLastReceivedDownEdgeFlags() { - return mLastReceivedDownEdgeFlags; - } - - /** - * Handles a received pointer down event. - * - * @param pointerIndex The index of the pointer that has changed. - * @param event The event to be handled. - */ - private void handleReceivedPointerDown(int pointerIndex, MotionEvent event) { - final int pointerId = event.getPointerId(pointerIndex); - final int pointerFlag = (1 << pointerId); - - mLastReceivedUpPointerDownTime = 0; - mLastReceivedUpPointerDownX = 0; - mLastReceivedUpPointerDownX = 0; - - mLastReceivedDownEdgeFlags = event.getEdgeFlags(); - - mReceivedPointersDown |= pointerFlag; - mReceivedPointerDownX[pointerId] = event.getX(pointerIndex); - mReceivedPointerDownY[pointerId] = event.getY(pointerIndex); - mReceivedPointerDownTime[pointerId] = event.getEventTime(); - - mPrimaryPointerId = pointerId; - } - - /** - * Handles a received pointer up event. - * - * @param pointerIndex The index of the pointer that has changed. - * @param event The event to be handled. - */ - private void handleReceivedPointerUp(int pointerIndex, MotionEvent event) { - final int pointerId = event.getPointerId(pointerIndex); - final int pointerFlag = (1 << pointerId); - - mLastReceivedUpPointerDownTime = getReceivedPointerDownTime(pointerId); - mLastReceivedUpPointerDownX = mReceivedPointerDownX[pointerId]; - mLastReceivedUpPointerDownY = mReceivedPointerDownY[pointerId]; - - mReceivedPointersDown &= ~pointerFlag; - mReceivedPointerDownX[pointerId] = 0; - mReceivedPointerDownY[pointerId] = 0; - mReceivedPointerDownTime[pointerId] = 0; - - if (mPrimaryPointerId == pointerId) { - mPrimaryPointerId = INVALID_POINTER_ID; - } - } - - /** - * @return The primary pointer id. - */ - private int findPrimaryPointerId() { - int primaryPointerId = INVALID_POINTER_ID; - long minDownTime = Long.MAX_VALUE; - - // Find the pointer that went down first. - int pointerIdBits = mReceivedPointersDown; - while (pointerIdBits > 0) { - final int pointerId = Integer.numberOfTrailingZeros(pointerIdBits); - pointerIdBits &= ~(1 << pointerId); - final long downPointerTime = mReceivedPointerDownTime[pointerId]; - if (downPointerTime < minDownTime) { - minDownTime = downPointerTime; - primaryPointerId = pointerId; - } - } - return primaryPointerId; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("========================="); - builder.append("\nDown pointers #"); - builder.append(getReceivedPointerDownCount()); - builder.append(" [ "); - for (int i = 0; i < MAX_POINTER_COUNT; i++) { - if (isReceivedPointerDown(i)) { - builder.append(i); - builder.append(" "); - } - } - builder.append("]"); - builder.append("\nPrimary pointer id [ "); - builder.append(getPrimaryPointerId()); - builder.append(" ]"); - builder.append("\n========================="); - return builder.toString(); - } - } -} diff --git a/services/accessibility/java/com/android/server/accessibility/UiAutomationManager.java b/services/accessibility/java/com/android/server/accessibility/UiAutomationManager.java index 72c84e28f24f..d1c3a02c6761 100644 --- a/services/accessibility/java/com/android/server/accessibility/UiAutomationManager.java +++ b/services/accessibility/java/com/android/server/accessibility/UiAutomationManager.java @@ -25,8 +25,10 @@ import android.content.Context; import android.os.Handler; import android.os.IBinder; import android.os.IBinder.DeathRecipient; +import android.os.RemoteCallback; import android.os.RemoteException; import android.util.Slog; +import android.view.Display; import android.view.accessibility.AccessibilityEvent; import com.android.internal.util.DumpUtils; @@ -65,6 +67,7 @@ class UiAutomationManager { mUiAutomationServiceOwner.unlinkToDeath(this, 0); mUiAutomationServiceOwner = null; destroyUiAutomationService(); + Slog.v(LOG_TAG, "UiAutomation service owner died"); } }; @@ -82,10 +85,11 @@ class UiAutomationManager { IAccessibilityServiceClient serviceClient, Context context, AccessibilityServiceInfo accessibilityServiceInfo, int id, Handler mainHandler, - AccessibilityManagerService.SecurityPolicy securityPolicy, + AccessibilitySecurityPolicy securityPolicy, AbstractAccessibilityServiceConnection.SystemSupport systemSupport, WindowManagerInternal windowManagerInternal, - GlobalActionPerformer globalActionPerfomer, int flags) { + SystemActionPerformer systemActionPerfomer, + AccessibilityWindowManager awm, int flags) { synchronized (mLock) { accessibilityServiceInfo.setComponentName(COMPONENT_NAME); @@ -105,7 +109,7 @@ class UiAutomationManager { mSystemSupport = systemSupport; mUiAutomationService = new UiAutomationService(context, accessibilityServiceInfo, id, mainHandler, mLock, securityPolicy, systemSupport, windowManagerInternal, - globalActionPerfomer); + systemActionPerfomer, awm); mUiAutomationServiceOwner = owner; mUiAutomationFlags = flags; mUiAutomationServiceInfo = accessibilityServiceInfo; @@ -221,11 +225,12 @@ class UiAutomationManager { UiAutomationService(Context context, AccessibilityServiceInfo accessibilityServiceInfo, int id, Handler mainHandler, Object lock, - AccessibilityManagerService.SecurityPolicy securityPolicy, + AccessibilitySecurityPolicy securityPolicy, SystemSupport systemSupport, WindowManagerInternal windowManagerInternal, - GlobalActionPerformer globalActionPerfomer) { + SystemActionPerformer systemActionPerfomer, AccessibilityWindowManager awm) { super(context, COMPONENT_NAME, accessibilityServiceInfo, id, mainHandler, lock, - securityPolicy, systemSupport, windowManagerInternal, globalActionPerfomer); + securityPolicy, systemSupport, windowManagerInternal, systemActionPerfomer, + awm); mMainHandler = mainHandler; } @@ -244,7 +249,8 @@ class UiAutomationManager { // another thread. if (serviceInterface != null) { service.linkToDeath(this, 0); - serviceInterface.init(this, mId, mOverlayWindowToken); + serviceInterface.init(this, mId, + mOverlayWindowTokens.get(Display.DEFAULT_DISPLAY)); } } catch (RemoteException re) { Slog.w(LOG_TAG, "Error initialized connection", re); @@ -259,7 +265,7 @@ class UiAutomationManager { } @Override - protected boolean isCalledForCurrentUserLocked() { + protected boolean hasRightsToCurrentUserLocked() { // Allow UiAutomation to work for any user return true; } @@ -292,6 +298,11 @@ class UiAutomationManager { } @Override + public boolean switchToInputMethod(String imeId) { + return false; + } + + @Override public boolean isAccessibilityButtonAvailable() { return false; } @@ -315,5 +326,8 @@ class UiAutomationManager { @Override public void onFingerprintGesture(int gesture) {} + + @Override + public void takeScreenshot(int displayId, RemoteCallback callback) {} } } diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/EventDispatcher.java b/services/accessibility/java/com/android/server/accessibility/gestures/EventDispatcher.java new file mode 100644 index 000000000000..070626be9f80 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/EventDispatcher.java @@ -0,0 +1,408 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import static com.android.server.accessibility.gestures.TouchExplorer.DEBUG; +import static com.android.server.accessibility.gestures.TouchState.ALL_POINTER_ID_BITS; +import static com.android.server.accessibility.gestures.TouchState.MAX_POINTER_COUNT; + +import android.content.Context; +import android.graphics.Point; +import android.util.Slog; +import android.view.MotionEvent; +import android.view.MotionEvent.PointerCoords; +import android.view.MotionEvent.PointerProperties; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityManager; + +import com.android.server.accessibility.AccessibilityManagerService; +import com.android.server.accessibility.EventStreamTransformation; +import com.android.server.policy.WindowManagerPolicy; + +/** + * This class dispatches motion events and accessibility events relating to touch exploration and + * gesture dispatch. TouchExplorer is responsible for insuring that the receiver of motion events is + * set correctly so that events go to the right place. + */ +class EventDispatcher { + private static final String LOG_TAG = "EventDispatcher"; + private static final int CLICK_LOCATION_NONE = 0; + private static final int CLICK_LOCATION_ACCESSIBILITY_FOCUS = 1; + private static final int CLICK_LOCATION_LAST_TOUCH_EXPLORED = 2; + + private final AccessibilityManagerService mAms; + private Context mContext; + // The receiver of motion events. + private EventStreamTransformation mReceiver; + + // The long pressing pointer id if coordinate remapping is needed for double tap and hold + private int mLongPressingPointerId = -1; + + // The long pressing pointer X if coordinate remapping is needed for double tap and hold. + private int mLongPressingPointerDeltaX; + + // The long pressing pointer Y if coordinate remapping is needed for double tap and hold. + private int mLongPressingPointerDeltaY; + + // Temporary point to avoid instantiation. + private final Point mTempPoint = new Point(); + + private TouchState mState; + + EventDispatcher( + Context context, + AccessibilityManagerService ams, + EventStreamTransformation receiver, + TouchState state) { + mContext = context; + mAms = ams; + mReceiver = receiver; + mState = state; + } + + public void setReceiver(EventStreamTransformation receiver) { + mReceiver = receiver; + } + + /** + * Sends an event. + * + * @param prototype The prototype from which to create the injected events. + * @param action The action of the event. + * @param rawEvent The original event prior to magnification or other transformations. + * @param pointerIdBits The bits of the pointers to send. + * @param policyFlags The policy flags associated with the event. + */ + void sendMotionEvent( + MotionEvent prototype, + int action, + MotionEvent rawEvent, + int pointerIdBits, + int policyFlags) { + prototype.setAction(action); + + MotionEvent event = null; + if (pointerIdBits == ALL_POINTER_ID_BITS) { + event = prototype; + } else { + try { + event = prototype.split(pointerIdBits); + } catch (IllegalArgumentException e) { + Slog.e(LOG_TAG, "sendMotionEvent: Failed to split motion event: " + e); + return; + } + } + if (action == MotionEvent.ACTION_DOWN) { + event.setDownTime(event.getEventTime()); + } else { + event.setDownTime(mState.getLastInjectedDownEventTime()); + } + // If the user is long pressing but the long pressing pointer + // was not exactly over the accessibility focused item we need + // to remap the location of that pointer so the user does not + // have to explicitly touch explore something to be able to + // long press it, or even worse to avoid the user long pressing + // on the wrong item since click and long press behave differently. + if (mLongPressingPointerId >= 0) { + event = offsetEvent(event, -mLongPressingPointerDeltaX, -mLongPressingPointerDeltaY); + } + + if (DEBUG) { + Slog.d( + LOG_TAG, + "Injecting event: " + + event + + ", policyFlags=0x" + + Integer.toHexString(policyFlags)); + } + + // Make sure that the user will see the event. + policyFlags |= WindowManagerPolicy.FLAG_PASS_TO_USER; + if (mReceiver != null) { + mReceiver.onMotionEvent(event, rawEvent, policyFlags); + } else { + Slog.e(LOG_TAG, "Error sending event: no receiver specified."); + } + mState.onInjectedMotionEvent(event); + + if (event != prototype) { + event.recycle(); + } + } + + /** + * Sends an accessibility event of the given type. + * + * @param type The event type. + */ + void sendAccessibilityEvent(int type) { + AccessibilityManager accessibilityManager = AccessibilityManager.getInstance(mContext); + if (accessibilityManager.isEnabled()) { + AccessibilityEvent event = AccessibilityEvent.obtain(type); + event.setWindowId(mAms.getActiveWindowId()); + accessibilityManager.sendAccessibilityEvent(event); + if (DEBUG) { + Slog.d( + LOG_TAG, + "Sending accessibility event" + AccessibilityEvent.eventTypeToString(type)); + } + } + // Todo: get rid of this and have TouchState control the sending of events rather than react + // to it. + mState.onInjectedAccessibilityEvent(type); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("========================="); + builder.append("\nDown pointers #"); + builder.append(Integer.bitCount(mState.getInjectedPointersDown())); + builder.append(" [ "); + for (int i = 0; i < MAX_POINTER_COUNT; i++) { + if (mState.isInjectedPointerDown(i)) { + builder.append(i); + builder.append(" "); + } + } + builder.append("]"); + builder.append("\n========================="); + return builder.toString(); + } + + /** + * /** Offsets all pointers in the given event by adding the specified X and Y offsets. + * + * @param event The event to offset. + * @param offsetX The X offset. + * @param offsetY The Y offset. + * @return An event with the offset pointers or the original event if both offsets are zero. + */ + private MotionEvent offsetEvent(MotionEvent event, int offsetX, int offsetY) { + if (offsetX == 0 && offsetY == 0) { + return event; + } + final int remappedIndex = event.findPointerIndex(mLongPressingPointerId); + final int pointerCount = event.getPointerCount(); + PointerProperties[] props = PointerProperties.createArray(pointerCount); + PointerCoords[] coords = PointerCoords.createArray(pointerCount); + for (int i = 0; i < pointerCount; i++) { + event.getPointerProperties(i, props[i]); + event.getPointerCoords(i, coords[i]); + if (i == remappedIndex) { + coords[i].x += offsetX; + coords[i].y += offsetY; + } + } + return MotionEvent.obtain( + event.getDownTime(), + event.getEventTime(), + event.getAction(), + event.getPointerCount(), + props, + coords, + event.getMetaState(), + event.getButtonState(), + 1.0f, + 1.0f, + event.getDeviceId(), + event.getEdgeFlags(), + event.getSource(), + event.getDisplayId(), + event.getFlags()); + } + + /** + * Computes the action for an injected event based on a masked action and a pointer index. + * + * @param actionMasked The masked action. + * @param pointerIndex The index of the pointer which has changed. + * @return The action to be used for injection. + */ + private int computeInjectionAction(int actionMasked, int pointerIndex) { + switch (actionMasked) { + case MotionEvent.ACTION_DOWN: + case MotionEvent.ACTION_POINTER_DOWN: + // Compute the action based on how many down pointers are injected. + if (mState.getInjectedPointerDownCount() == 0) { + return MotionEvent.ACTION_DOWN; + } else { + return (pointerIndex << MotionEvent.ACTION_POINTER_INDEX_SHIFT) + | MotionEvent.ACTION_POINTER_DOWN; + } + case MotionEvent.ACTION_POINTER_UP: + // Compute the action based on how many down pointers are injected. + if (mState.getInjectedPointerDownCount() == 1) { + return MotionEvent.ACTION_UP; + } else { + return (pointerIndex << MotionEvent.ACTION_POINTER_INDEX_SHIFT) + | MotionEvent.ACTION_POINTER_UP; + } + default: + return actionMasked; + } + } + /** + * Sends down events to the view hierarchy for all pointers which are not already being + * delivered i.e. pointers that are not yet injected. + * + * @param prototype The prototype from which to create the injected events. + * @param policyFlags The policy flags associated with the event. + */ + void sendDownForAllNotInjectedPointers(MotionEvent prototype, int policyFlags) { + + // Inject the injected pointers. + int pointerIdBits = 0; + final int pointerCount = prototype.getPointerCount(); + for (int i = 0; i < pointerCount; i++) { + final int pointerId = prototype.getPointerId(i); + // Do not send event for already delivered pointers. + if (!mState.isInjectedPointerDown(pointerId)) { + pointerIdBits |= (1 << pointerId); + final int action = computeInjectionAction(MotionEvent.ACTION_DOWN, i); + sendMotionEvent( + prototype, + action, + mState.getLastReceivedEvent(), + pointerIdBits, + policyFlags); + } + } + } + + /** + * Sends up events to the view hierarchy for all pointers which are already being delivered i.e. + * pointers that are injected. + * + * @param prototype The prototype from which to create the injected events. + * @param policyFlags The policy flags associated with the event. + */ + void sendUpForInjectedDownPointers(MotionEvent prototype, int policyFlags) { + int pointerIdBits = prototype.getPointerIdBits(); + final int pointerCount = prototype.getPointerCount(); + for (int i = 0; i < pointerCount; i++) { + final int pointerId = prototype.getPointerId(i); + // Skip non injected down pointers. + if (!mState.isInjectedPointerDown(pointerId)) { + continue; + } + final int action = computeInjectionAction(MotionEvent.ACTION_POINTER_UP, i); + sendMotionEvent( + prototype, action, mState.getLastReceivedEvent(), pointerIdBits, policyFlags); + pointerIdBits &= ~(1 << pointerId); + } + } + + public boolean longPressWithTouchEvents(MotionEvent event, int policyFlags) { + final int pointerIndex = event.getActionIndex(); + final int pointerId = event.getPointerId(pointerIndex); + Point clickLocation = mTempPoint; + final int result = computeClickLocation(clickLocation); + if (result == CLICK_LOCATION_NONE) { + return false; + } + mLongPressingPointerId = pointerId; + mLongPressingPointerDeltaX = (int) event.getX(pointerIndex) - clickLocation.x; + mLongPressingPointerDeltaY = (int) event.getY(pointerIndex) - clickLocation.y; + sendDownForAllNotInjectedPointers(event, policyFlags); + return true; + } + + void clear() { + mLongPressingPointerId = -1; + mLongPressingPointerDeltaX = 0; + mLongPressingPointerDeltaY = 0; + } + + public void clickWithTouchEvents(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + final int pointerIndex = event.getActionIndex(); + final int pointerId = event.getPointerId(pointerIndex); + Point clickLocation = mTempPoint; + final int result = computeClickLocation(clickLocation); + if (result == CLICK_LOCATION_NONE) { + Slog.e(LOG_TAG, "Unable to compute click location."); + // We can't send a click to no location, but the gesture was still + // consumed. + return; + } + // Do the click. + PointerProperties[] properties = new PointerProperties[1]; + properties[0] = new PointerProperties(); + event.getPointerProperties(pointerIndex, properties[0]); + PointerCoords[] coords = new PointerCoords[1]; + coords[0] = new PointerCoords(); + coords[0].x = clickLocation.x; + coords[0].y = clickLocation.y; + MotionEvent clickEvent = + MotionEvent.obtain( + event.getDownTime(), + event.getEventTime(), + MotionEvent.ACTION_DOWN, + 1, + properties, + coords, + 0, + 0, + 1.0f, + 1.0f, + event.getDeviceId(), + 0, + event.getSource(), + event.getDisplayId(), + event.getFlags()); + final boolean targetAccessibilityFocus = (result == CLICK_LOCATION_ACCESSIBILITY_FOCUS); + sendActionDownAndUp(clickEvent, rawEvent, policyFlags, targetAccessibilityFocus); + clickEvent.recycle(); + } + + private int computeClickLocation(Point outLocation) { + if (mState.getLastInjectedHoverEventForClick() != null) { + final int lastExplorePointerIndex = + mState.getLastInjectedHoverEventForClick().getActionIndex(); + outLocation.x = + (int) mState.getLastInjectedHoverEventForClick().getX(lastExplorePointerIndex); + outLocation.y = + (int) mState.getLastInjectedHoverEventForClick().getY(lastExplorePointerIndex); + if (!mAms.accessibilityFocusOnlyInActiveWindow() + || mState.getLastTouchedWindowId() == mAms.getActiveWindowId()) { + if (mAms.getAccessibilityFocusClickPointInScreen(outLocation)) { + return CLICK_LOCATION_ACCESSIBILITY_FOCUS; + } else { + return CLICK_LOCATION_LAST_TOUCH_EXPLORED; + } + } + } + if (mAms.getAccessibilityFocusClickPointInScreen(outLocation)) { + return CLICK_LOCATION_ACCESSIBILITY_FOCUS; + } + return CLICK_LOCATION_NONE; + } + + private void sendActionDownAndUp( + MotionEvent prototype, + MotionEvent rawEvent, + int policyFlags, + boolean targetAccessibilityFocus) { + // Tap with the pointer that last explored. + final int pointerId = prototype.getPointerId(prototype.getActionIndex()); + final int pointerIdBits = (1 << pointerId); + prototype.setTargetAccessibilityFocus(targetAccessibilityFocus); + sendMotionEvent(prototype, MotionEvent.ACTION_DOWN, rawEvent, pointerIdBits, policyFlags); + prototype.setTargetAccessibilityFocus(targetAccessibilityFocus); + sendMotionEvent(prototype, MotionEvent.ACTION_UP, rawEvent, pointerIdBits, policyFlags); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/GestureManifold.java b/services/accessibility/java/com/android/server/accessibility/gestures/GestureManifold.java new file mode 100644 index 000000000000..e9c70c60a322 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/GestureManifold.java @@ -0,0 +1,370 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_DOUBLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_DOUBLE_TAP_AND_HOLD; +import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SINGLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SWIPE_DOWN; +import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SWIPE_LEFT; +import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SWIPE_RIGHT; +import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SWIPE_UP; +import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_TRIPLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_DOUBLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_DOUBLE_TAP_AND_HOLD; +import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SINGLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SWIPE_DOWN; +import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SWIPE_LEFT; +import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SWIPE_RIGHT; +import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SWIPE_UP; +import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_TRIPLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_4_FINGER_DOUBLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_4_FINGER_DOUBLE_TAP_AND_HOLD; +import static android.accessibilityservice.AccessibilityService.GESTURE_4_FINGER_SINGLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_4_FINGER_SWIPE_DOWN; +import static android.accessibilityservice.AccessibilityService.GESTURE_4_FINGER_SWIPE_LEFT; +import static android.accessibilityservice.AccessibilityService.GESTURE_4_FINGER_SWIPE_RIGHT; +import static android.accessibilityservice.AccessibilityService.GESTURE_4_FINGER_SWIPE_UP; +import static android.accessibilityservice.AccessibilityService.GESTURE_4_FINGER_TRIPLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP; +import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN_AND_LEFT; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN_AND_RIGHT; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN_AND_UP; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_LEFT; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_LEFT_AND_DOWN; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_LEFT_AND_RIGHT; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_LEFT_AND_UP; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_RIGHT; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_RIGHT_AND_DOWN; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_RIGHT_AND_LEFT; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_RIGHT_AND_UP; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_UP; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_UP_AND_DOWN; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_UP_AND_LEFT; +import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_UP_AND_RIGHT; + +import static com.android.server.accessibility.gestures.Swipe.DOWN; +import static com.android.server.accessibility.gestures.Swipe.LEFT; +import static com.android.server.accessibility.gestures.Swipe.RIGHT; +import static com.android.server.accessibility.gestures.Swipe.UP; +import static com.android.server.accessibility.gestures.TouchExplorer.DEBUG; + +import android.accessibilityservice.AccessibilityGestureEvent; +import android.content.Context; +import android.os.Handler; +import android.util.Slog; +import android.view.MotionEvent; + +import java.util.ArrayList; +import java.util.List; + +/** + * This class coordinates a series of individual gesture matchers to serve as a unified gesture + * detector. Gesture matchers are tied to a single gesture. It calls listener callback functions + * when a gesture starts or completes. + */ +class GestureManifold implements GestureMatcher.StateChangeListener { + + private static final String LOG_TAG = "GestureManifold"; + + private final List<GestureMatcher> mGestures = new ArrayList<>(); + private final Context mContext; + // Handler for performing asynchronous operations. + private final Handler mHandler; + // Listener to be notified of gesture start and end. + private Listener mListener; + // Whether double tap and double tap and hold will be dispatched to the service or handled in + // the framework. + private boolean mServiceHandlesDoubleTap = false; + // Whether multi-finger gestures are enabled. + boolean mMultiFingerGesturesEnabled; + // A list of all the multi-finger gestures, for easy adding and removal. + private final List<GestureMatcher> mMultiFingerGestures = new ArrayList<>(); + // Shared state information. + private TouchState mState; + + GestureManifold(Context context, Listener listener, TouchState state) { + mContext = context; + mHandler = new Handler(context.getMainLooper()); + mListener = listener; + mState = state; + mMultiFingerGesturesEnabled = false; + // Set up gestures. + // Start with double tap. + mGestures.add(new MultiTap(context, 2, GESTURE_DOUBLE_TAP, this)); + mGestures.add(new MultiTapAndHold(context, 2, GESTURE_DOUBLE_TAP_AND_HOLD, this)); + // Second-finger double tap. + mGestures.add(new SecondFingerMultiTap(context, 2, GESTURE_DOUBLE_TAP, this)); + // One-direction swipes. + mGestures.add(new Swipe(context, RIGHT, GESTURE_SWIPE_RIGHT, this)); + mGestures.add(new Swipe(context, LEFT, GESTURE_SWIPE_LEFT, this)); + mGestures.add(new Swipe(context, UP, GESTURE_SWIPE_UP, this)); + mGestures.add(new Swipe(context, DOWN, GESTURE_SWIPE_DOWN, this)); + // Two-direction swipes. + mGestures.add(new Swipe(context, LEFT, RIGHT, GESTURE_SWIPE_LEFT_AND_RIGHT, this)); + mGestures.add(new Swipe(context, LEFT, UP, GESTURE_SWIPE_LEFT_AND_UP, this)); + mGestures.add(new Swipe(context, LEFT, DOWN, GESTURE_SWIPE_LEFT_AND_DOWN, this)); + mGestures.add(new Swipe(context, RIGHT, UP, GESTURE_SWIPE_RIGHT_AND_UP, this)); + mGestures.add(new Swipe(context, RIGHT, DOWN, GESTURE_SWIPE_RIGHT_AND_DOWN, this)); + mGestures.add(new Swipe(context, RIGHT, LEFT, GESTURE_SWIPE_RIGHT_AND_LEFT, this)); + mGestures.add(new Swipe(context, DOWN, UP, GESTURE_SWIPE_DOWN_AND_UP, this)); + mGestures.add(new Swipe(context, DOWN, LEFT, GESTURE_SWIPE_DOWN_AND_LEFT, this)); + mGestures.add(new Swipe(context, DOWN, RIGHT, GESTURE_SWIPE_DOWN_AND_RIGHT, this)); + mGestures.add(new Swipe(context, UP, DOWN, GESTURE_SWIPE_UP_AND_DOWN, this)); + mGestures.add(new Swipe(context, UP, LEFT, GESTURE_SWIPE_UP_AND_LEFT, this)); + mGestures.add(new Swipe(context, UP, RIGHT, GESTURE_SWIPE_UP_AND_RIGHT, this)); + // Set up multi-finger gestures to be enabled later. + // Two-finger taps. + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 2, 1, GESTURE_2_FINGER_SINGLE_TAP, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 2, 2, GESTURE_2_FINGER_DOUBLE_TAP, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTapAndHold( + mContext, 2, 2, GESTURE_2_FINGER_DOUBLE_TAP_AND_HOLD, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 2, 3, GESTURE_2_FINGER_TRIPLE_TAP, this)); + // Three-finger taps. + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 3, 1, GESTURE_3_FINGER_SINGLE_TAP, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 3, 2, GESTURE_3_FINGER_DOUBLE_TAP, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTapAndHold( + mContext, 3, 2, GESTURE_3_FINGER_DOUBLE_TAP_AND_HOLD, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 3, 3, GESTURE_3_FINGER_TRIPLE_TAP, this)); + // Four-finger taps. + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 4, 1, GESTURE_4_FINGER_SINGLE_TAP, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 4, 2, GESTURE_4_FINGER_DOUBLE_TAP, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTapAndHold( + mContext, 4, 2, GESTURE_4_FINGER_DOUBLE_TAP_AND_HOLD, this)); + mMultiFingerGestures.add( + new MultiFingerMultiTap(mContext, 4, 3, GESTURE_4_FINGER_TRIPLE_TAP, this)); + // Two-finger swipes. + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 2, DOWN, GESTURE_2_FINGER_SWIPE_DOWN, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 2, LEFT, GESTURE_2_FINGER_SWIPE_LEFT, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 2, RIGHT, GESTURE_2_FINGER_SWIPE_RIGHT, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 2, UP, GESTURE_2_FINGER_SWIPE_UP, this)); + // Three-finger swipes. + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 3, DOWN, GESTURE_3_FINGER_SWIPE_DOWN, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 3, LEFT, GESTURE_3_FINGER_SWIPE_LEFT, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 3, RIGHT, GESTURE_3_FINGER_SWIPE_RIGHT, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 3, UP, GESTURE_3_FINGER_SWIPE_UP, this)); + // Four-finger swipes. + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 4, DOWN, GESTURE_4_FINGER_SWIPE_DOWN, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 4, LEFT, GESTURE_4_FINGER_SWIPE_LEFT, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 4, RIGHT, GESTURE_4_FINGER_SWIPE_RIGHT, this)); + mMultiFingerGestures.add( + new MultiFingerSwipe(context, 4, UP, GESTURE_4_FINGER_SWIPE_UP, this)); + } + + /** + * Processes a motion event. + * + * @param event The event as received from the previous entry in the event stream. + * @param rawEvent The event without any transformations e.g. magnification. + * @param policyFlags + * @return True if the event has been appropriately handled by the gesture manifold and related + * callback functions, false if it should be handled further by the calling function. + */ + boolean onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mState.isClear()) { + if (event.getActionMasked() == MotionEvent.ACTION_DOWN) { + // Sanity safeguard: if touch state is clear, then matchers should always be clear + // before processing the next down event. + clear(); + } else { + // If for some reason other events come through while in the clear state they could + // compromise the state of particular matchers, so we just ignore them. + return false; + } + } + for (GestureMatcher matcher : mGestures) { + if (matcher.getState() != GestureMatcher.STATE_GESTURE_CANCELED) { + if (DEBUG) { + Slog.d(LOG_TAG, matcher.toString()); + } + matcher.onMotionEvent(event, rawEvent, policyFlags); + if (DEBUG) { + Slog.d(LOG_TAG, matcher.toString()); + } + if (matcher.getState() == GestureMatcher.STATE_GESTURE_COMPLETED) { + // Here we just clear and return. The actual gesture dispatch is done in + // onStateChanged(). + clear(); + // No need to process this event any further. + return true; + } + } + } + return false; + } + + public void clear() { + for (GestureMatcher matcher : mGestures) { + matcher.clear(); + } + } + + /** + * Listener that receives notifications of the state of the gesture detector. Listener functions + * are called as a result of onMotionEvent(). The current MotionEvent in the context of these + * functions is the event passed into onMotionEvent. + */ + public interface Listener { + /** + * When FLAG_SERVICE_HANDLES_DOUBLE_TAP is enabled, this method is not called; double-tap + * and hold is dispatched via onGestureCompleted. Otherwise, this method is called when the + * user has performed a double tap and then held down the second tap. + */ + void onDoubleTapAndHold(MotionEvent event, MotionEvent rawEvent, int policyFlags); + + /** + * When FLAG_SERVICE_HANDLES_DOUBLE_TAP is enabled, this method is not called; double-tap is + * dispatched via onGestureCompleted. Otherwise, this method is called when the user lifts + * their finger on the second tap of a double tap. + * + * @return true if the event is consumed, else false + */ + boolean onDoubleTap(MotionEvent event, MotionEvent rawEvent, int policyFlags); + + /** + * Called when the system has decided the event stream is a potential gesture. + * + * @return true if the event is consumed, else false + */ + boolean onGestureStarted(); + + /** + * Called when an event stream is recognized as a gesture. + * + * @param gestureEvent Information about the gesture. + * @return true if the event is consumed, else false + */ + boolean onGestureCompleted(AccessibilityGestureEvent gestureEvent); + + /** + * Called when the system has decided an event stream doesn't match any known gesture. + * + * @param event The most recent MotionEvent received. + * @param policyFlags The policy flags of the most recent event. + * @return true if the event is consumed, else false + */ + boolean onGestureCancelled(MotionEvent event, MotionEvent rawEvent, int policyFlags); + } + + @Override + public void onStateChanged( + int gestureId, int state, MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (state == GestureMatcher.STATE_GESTURE_STARTED && !mState.isGestureDetecting()) { + if (gestureId == GESTURE_DOUBLE_TAP || gestureId == GESTURE_DOUBLE_TAP_AND_HOLD) { + if (mServiceHandlesDoubleTap) { + mListener.onGestureStarted(); + } + } else { + mListener.onGestureStarted(); + } + } else if (state == GestureMatcher.STATE_GESTURE_COMPLETED) { + onGestureCompleted(gestureId, event, rawEvent, policyFlags); + } else if (state == GestureMatcher.STATE_GESTURE_CANCELED && mState.isGestureDetecting()) { + // We only want to call the cancelation callback if there are no other pending + // detectors. + for (GestureMatcher matcher : mGestures) { + if (matcher.getState() == GestureMatcher.STATE_GESTURE_STARTED) { + return; + } + } + if (DEBUG) { + Slog.d(LOG_TAG, "Cancelling."); + } + mListener.onGestureCancelled(event, rawEvent, policyFlags); + } + } + + private void onGestureCompleted( + int gestureId, MotionEvent event, MotionEvent rawEvent, int policyFlags) { + // Note that gestures that complete immediately call clear() from onMotionEvent. + // Gestures that complete on a delay call clear() here. + switch (gestureId) { + case GESTURE_DOUBLE_TAP: + if (mServiceHandlesDoubleTap) { + AccessibilityGestureEvent gestureEvent = + new AccessibilityGestureEvent(gestureId, event.getDisplayId()); + mListener.onGestureCompleted(gestureEvent); + } else { + mListener.onDoubleTap(event, rawEvent, policyFlags); + } + clear(); + break; + case GESTURE_DOUBLE_TAP_AND_HOLD: + if (mServiceHandlesDoubleTap) { + AccessibilityGestureEvent gestureEvent = + new AccessibilityGestureEvent(gestureId, event.getDisplayId()); + mListener.onGestureCompleted(gestureEvent); + } else { + mListener.onDoubleTapAndHold(event, rawEvent, policyFlags); + } + clear(); + break; + default: + AccessibilityGestureEvent gestureEvent = + new AccessibilityGestureEvent(gestureId, event.getDisplayId()); + mListener.onGestureCompleted(gestureEvent); + break; + } + } + + public boolean isMultiFingerGesturesEnabled() { + return mMultiFingerGesturesEnabled; + } + + public void setMultiFingerGesturesEnabled(boolean mode) { + if (mMultiFingerGesturesEnabled != mode) { + mMultiFingerGesturesEnabled = mode; + if (mode) { + mGestures.addAll(mMultiFingerGestures); + } else { + mGestures.removeAll(mMultiFingerGestures); + } + } + } + + public void setServiceHandlesDoubleTap(boolean mode) { + mServiceHandlesDoubleTap = mode; + } + + public boolean isServiceHandlesDoubleTapEnabled() { + return mServiceHandlesDoubleTap; + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/GestureMatcher.java b/services/accessibility/java/com/android/server/accessibility/gestures/GestureMatcher.java new file mode 100644 index 000000000000..0b30ff57ddde --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/GestureMatcher.java @@ -0,0 +1,371 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import static com.android.server.accessibility.gestures.TouchExplorer.DEBUG; + +import android.annotation.IntDef; +import android.os.Handler; +import android.util.Slog; +import android.view.MotionEvent; +import android.view.ViewConfiguration; + +/** + * This class describes a common base for gesture matchers. A gesture matcher checks a series of + * motion events against a single gesture. Coordinating the individual gesture matchers is done by + * the GestureManifold. To create a new Gesture, extend this class and override the onDown, onMove, + * onUp, etc methods as necessary. If you don't override a method your matcher will do nothing in + * response to that type of event. Finally, be sure to give your gesture a name by overriding + * getGestureName(). + */ +abstract class GestureMatcher { + // Potential states for this individual gesture matcher. + // In STATE_CLEAR, this matcher is accepting new motion events but has not formally signaled + // that there is enough data to judge that a gesture has started. + static final int STATE_CLEAR = 0; + // In STATE_GESTURE_STARTED, this matcher continues to accept motion events and it has signaled + // to the gesture manifold that what looks like the specified gesture has started. + static final int STATE_GESTURE_STARTED = 1; + // In STATE_GESTURE_COMPLETED, this matcher has successfully matched the specified gesture. and + // will not accept motion events until it is cleared. + static final int STATE_GESTURE_COMPLETED = 2; + // In STATE_GESTURE_CANCELED, this matcher will not accept new motion events because it is + // impossible that this set of motion events will match the specified gesture. + static final int STATE_GESTURE_CANCELED = 3; + + @IntDef({STATE_CLEAR, STATE_GESTURE_STARTED, STATE_GESTURE_COMPLETED, STATE_GESTURE_CANCELED}) + public @interface State {} + + @State private int mState = STATE_CLEAR; + // The id number of the gesture that gets passed to accessibility services. + private final int mGestureId; + // handler for asynchronous operations like timeouts + private final Handler mHandler; + + private final StateChangeListener mListener; + + // Use this to transition to new states after a delay. + // e.g. cancel or complete after some timeout. + // Convenience functions for tapTimeout and doubleTapTimeout are already defined here. + protected final DelayedTransition mDelayedTransition; + + GestureMatcher(int gestureId, Handler handler, StateChangeListener listener) { + mGestureId = gestureId; + mHandler = handler; + mDelayedTransition = new DelayedTransition(); + mListener = listener; + } + + /** + * Resets all state information for this matcher. Subclasses that include their own state + * information should override this method to reset their own state information and call + * super.clear(). + */ + protected void clear() { + mState = STATE_CLEAR; + cancelPendingTransitions(); + } + + public int getState() { + return mState; + } + + /** + * Transitions to a new state and notifies any listeners. Note that any pending transitions are + * canceled. + */ + private void setState( + @State int state, MotionEvent event, MotionEvent rawEvent, int policyFlags) { + mState = state; + cancelPendingTransitions(); + mListener.onStateChanged(mGestureId, mState, event, rawEvent, policyFlags); + } + + /** Indicates that there is evidence to suggest that this gesture has started. */ + protected final void startGesture(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + setState(STATE_GESTURE_STARTED, event, rawEvent, policyFlags); + } + + /** Indicates this stream of motion events can no longer match this gesture. */ + protected final void cancelGesture(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + setState(STATE_GESTURE_CANCELED, event, rawEvent, policyFlags); + } + + /** Indicates this gesture is completed. */ + protected final void completeGesture(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + setState(STATE_GESTURE_COMPLETED, event, rawEvent, policyFlags); + } + + public int getGestureId() { + return mGestureId; + } + + /** + * Process a motion event and attempt to match it to this gesture. + * + * @param event the event as passed in from the event stream. + * @param rawEvent the original un-modified event. Useful for calculating movements in physical + * space. + * @param policyFlags the policy flags as passed in from the event stream. + * @return the state of this matcher. + */ + public final int onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mState == STATE_GESTURE_CANCELED || mState == STATE_GESTURE_COMPLETED) { + return mState; + } + switch (event.getActionMasked()) { + case MotionEvent.ACTION_DOWN: + onDown(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_POINTER_DOWN: + onPointerDown(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_MOVE: + onMove(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_POINTER_UP: + onPointerUp(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_UP: + onUp(event, rawEvent, policyFlags); + break; + default: + // Cancel because of invalid event. + setState(STATE_GESTURE_CANCELED, event, rawEvent, policyFlags); + break; + } + return mState; + } + + /** + * Matchers override this method to respond to ACTION_DOWN events. ACTION_DOWN events indicate + * the first finger has touched the screen. If not overridden the default response is to do + * nothing. + */ + protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {} + + /** + * Matchers override this method to respond to ACTION_POINTER_DOWN events. ACTION_POINTER_DOWN + * indicates that more than one finger has touched the screen. If not overridden the default + * response is to do nothing. + * + * @param event the event as passed in from the event stream. + * @param rawEvent the original un-modified event. Useful for calculating movements in physical + * space. + * @param policyFlags the policy flags as passed in from the event stream. + */ + protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {} + + /** + * Matchers override this method to respond to ACTION_MOVE events. ACTION_MOVE indicates that + * one or fingers has moved. If not overridden the default response is to do nothing. + * + * @param event the event as passed in from the event stream. + * @param rawEvent the original un-modified event. Useful for calculating movements in physical + * space. + * @param policyFlags the policy flags as passed in from the event stream. + */ + protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) {} + + /** + * Matchers override this method to respond to ACTION_POINTER_UP events. ACTION_POINTER_UP + * indicates that a finger has lifted from the screen but at least one finger continues to touch + * the screen. If not overridden the default response is to do nothing. + * + * @param event the event as passed in from the event stream. + * @param rawEvent the original un-modified event. Useful for calculating movements in physical + * space. + * @param policyFlags the policy flags as passed in from the event stream. + */ + protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {} + + /** + * Matchers override this method to respond to ACTION_UP events. ACTION_UP indicates that there + * are no more fingers touching the screen. If not overridden the default response is to do + * nothing. + * + * @param event the event as passed in from the event stream. + * @param rawEvent the original un-modified event. Useful for calculating movements in physical + * space. + * @param policyFlags the policy flags as passed in from the event stream. + */ + protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {} + + /** Cancels this matcher after the tap timeout. Any pending state transitions are removed. */ + protected void cancelAfterTapTimeout(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelAfter(ViewConfiguration.getTapTimeout(), event, rawEvent, policyFlags); + } + + /** Cancels this matcher after the double tap timeout. Any pending cancelations are removed. */ + protected final void cancelAfterDoubleTapTimeout( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelAfter(ViewConfiguration.getDoubleTapTimeout(), event, rawEvent, policyFlags); + } + + /** + * Cancels this matcher after the specified timeout. Any pending cancelations are removed. Used + * to prevent this matcher from accepting motion events until it is cleared. + */ + protected final void cancelAfter( + long timeout, MotionEvent event, MotionEvent rawEvent, int policyFlags) { + mDelayedTransition.cancel(); + mDelayedTransition.post(STATE_GESTURE_CANCELED, timeout, event, rawEvent, policyFlags); + } + + /** Cancels any delayed transitions between states scheduled for this matcher. */ + protected final void cancelPendingTransitions() { + mDelayedTransition.cancel(); + } + + /** + * Signals that this gesture has been completed after the tap timeout has expired. Used to + * ensure that there is no conflict with another gesture or for gestures that explicitly require + * a hold. + */ + protected final void completeAfterLongPressTimeout( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + completeAfter(ViewConfiguration.getLongPressTimeout(), event, rawEvent, policyFlags); + } + + /** + * Signals that this gesture has been completed after the tap timeout has expired. Used to + * ensure that there is no conflict with another gesture or for gestures that explicitly require + * a hold. + */ + protected final void completeAfterTapTimeout( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + completeAfter(ViewConfiguration.getTapTimeout(), event, rawEvent, policyFlags); + } + + /** + * Signals that this gesture has been completed after the specified timeout has expired. Used to + * ensure that there is no conflict with another gesture or for gestures that explicitly require + * a hold. + */ + protected final void completeAfter( + long timeout, MotionEvent event, MotionEvent rawEvent, int policyFlags) { + mDelayedTransition.cancel(); + mDelayedTransition.post(STATE_GESTURE_COMPLETED, timeout, event, rawEvent, policyFlags); + } + + /** + * Signals that this gesture has been completed after the double-tap timeout has expired. Used + * to ensure that there is no conflict with another gesture or for gestures that explicitly + * require a hold. + */ + protected final void completeAfterDoubleTapTimeout( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + completeAfter(ViewConfiguration.getDoubleTapTimeout(), event, rawEvent, policyFlags); + } + + public static String getStateSymbolicName(@State int state) { + switch (state) { + case STATE_CLEAR: + return "STATE_CLEAR"; + case STATE_GESTURE_STARTED: + return "STATE_GESTURE_STARTED"; + case STATE_GESTURE_COMPLETED: + return "STATE_GESTURE_COMPLETED"; + case STATE_GESTURE_CANCELED: + return "STATE_GESTURE_CANCELED"; + default: + return "Unknown state: " + state; + } + } + + /** + * Returns a readable name for this matcher that can be displayed to the user and in system + * logs. + */ + abstract String getGestureName(); + + /** + * Returns a String representation of this matcher. Each matcher can override this method to add + * extra state information to the string representation. + */ + public String toString() { + return getGestureName() + ":" + getStateSymbolicName(mState); + } + + /** This class allows matchers to transition between states on a delay. */ + protected final class DelayedTransition implements Runnable { + + private static final String LOG_TAG = "GestureMatcher.DelayedTransition"; + int mTargetState; + MotionEvent mEvent; + MotionEvent mRawEvent; + int mPolicyFlags; + + public void cancel() { + // Avoid meaningless debug messages. + if (DEBUG && isPending()) { + Slog.d( + LOG_TAG, + getGestureName() + + ": canceling delayed transition to " + + getStateSymbolicName(mTargetState)); + } + mHandler.removeCallbacks(this); + } + + public void post( + int state, long delay, MotionEvent event, MotionEvent rawEvent, int policyFlags) { + mTargetState = state; + mEvent = event; + mRawEvent = rawEvent; + mPolicyFlags = policyFlags; + mHandler.postDelayed(this, delay); + if (DEBUG) { + Slog.d( + LOG_TAG, + getGestureName() + + ": posting delayed transition to " + + getStateSymbolicName(mTargetState)); + } + } + + public boolean isPending() { + return mHandler.hasCallbacks(this); + } + + public void forceSendAndRemove() { + if (isPending()) { + run(); + cancel(); + } + } + + @Override + public void run() { + if (DEBUG) { + Slog.d( + LOG_TAG, + getGestureName() + + ": executing delayed transition to " + + getStateSymbolicName(mTargetState)); + } + setState(mTargetState, mEvent, mRawEvent, mPolicyFlags); + } + } + + /** Interface to allow a class to listen for state changes in a specific gesture matcher */ + interface StateChangeListener { + + void onStateChanged( + int gestureId, int state, MotionEvent event, MotionEvent rawEvent, int policyFlags); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/GestureUtils.java b/services/accessibility/java/com/android/server/accessibility/gestures/GestureUtils.java index d5b53bc686da..ec3041848356 100644 --- a/services/accessibility/java/com/android/server/accessibility/GestureUtils.java +++ b/services/accessibility/java/com/android/server/accessibility/gestures/GestureUtils.java @@ -1,12 +1,16 @@ -package com.android.server.accessibility; +package com.android.server.accessibility.gestures; +import android.graphics.PointF; import android.util.MathUtils; import android.view.MotionEvent; /** * Some helper functions for gesture detection. */ -final class GestureUtils { +public final class GestureUtils { + + public static int MM_PER_CM = 10; + public static float CM_PER_INCH = 2.54f; private GestureUtils() { /* cannot be instantiated */ @@ -35,6 +39,27 @@ final class GestureUtils { return MathUtils.dist(first.getX(), first.getY(), second.getX(), second.getY()); } + /** + * Returns the minimum distance between {@code pointerDown} and each pointer of + * {@link MotionEvent}. + * + * @param pointerDown The action pointer location of the {@link MotionEvent} with + * {@link MotionEvent#ACTION_DOWN} or {@link MotionEvent#ACTION_POINTER_DOWN} + * @param moveEvent The {@link MotionEvent} with {@link MotionEvent#ACTION_MOVE} + * @return the movement of the pointer. + */ + public static double distanceClosestPointerToPoint(PointF pointerDown, MotionEvent moveEvent) { + float movement = Float.MAX_VALUE; + for (int i = 0; i < moveEvent.getPointerCount(); i++) { + final float moveDelta = MathUtils.dist(pointerDown.x, pointerDown.y, moveEvent.getX(i), + moveEvent.getY(i)); + if (movement > moveDelta) { + movement = moveDelta; + } + } + return movement; + } + public static boolean isTimedOut(MotionEvent firstUp, MotionEvent secondUp, int timeout) { final long deltaTime = secondUp.getEventTime() - firstUp.getEventTime(); return (deltaTime >= timeout); @@ -85,4 +110,12 @@ final class GestureUtils { return true; } + + /** + * Gets the index of the pointer that went up or down from a motion event. + */ + public static int getActionIndex(MotionEvent event) { + return (event.getAction() + & MotionEvent.ACTION_POINTER_INDEX_MASK) >> MotionEvent.ACTION_POINTER_INDEX_SHIFT; + } } diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerMultiTap.java b/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerMultiTap.java new file mode 100644 index 000000000000..09e060570c93 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerMultiTap.java @@ -0,0 +1,294 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import android.content.Context; +import android.graphics.PointF; +import android.os.Handler; +import android.view.MotionEvent; +import android.view.ViewConfiguration; + +import com.android.internal.util.Preconditions; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * This class matches multi-finger multi-tap gestures. The number of fingers and the number of taps + * for each instance is specified in the constructor. + */ +class MultiFingerMultiTap extends GestureMatcher { + + // The target number of taps. + final int mTargetTapCount; + // The target number of fingers. + final int mTargetFingerCount; + // The acceptable distance between two taps of a finger. + private int mDoubleTapSlop; + // The acceptable distance the pointer can move and still count as a tap. + private int mTouchSlop; + // A tap counts when target number of fingers are down and up once. + protected int mCompletedTapCount; + // A flag set to true when target number of fingers have touched down at once before. + // Used to indicate what next finger action should be. Down when false and lift when true. + protected boolean mIsTargetFingerCountReached = false; + // Store initial down points for slop checking and update when next down if is inside slop. + private PointF[] mBases; + // The points in bases that already have slop checked when onDown or onPointerDown. + // It prevents excluded points matched multiple times by other pointers from next check. + private ArrayList<PointF> mExcludedPointsForDownSlopChecked; + + /** + * @throws IllegalArgumentException if <code>fingers<code/> is less than 2 + * or <code>taps<code/> is not positive. + */ + MultiFingerMultiTap( + Context context, + int fingers, + int taps, + int gestureId, + GestureMatcher.StateChangeListener listener) { + super(gestureId, new Handler(context.getMainLooper()), listener); + Preconditions.checkArgument(fingers >= 2); + Preconditions.checkArgumentPositive(taps, "Tap count must greater than 0."); + mTargetTapCount = taps; + mTargetFingerCount = fingers; + mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop() * fingers; + mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop() * fingers; + + mBases = new PointF[mTargetFingerCount]; + for (int i = 0; i < mBases.length; i++) { + mBases[i] = new PointF(); + } + mExcludedPointsForDownSlopChecked = new ArrayList<>(mTargetFingerCount); + clear(); + } + + @Override + protected void clear() { + mCompletedTapCount = 0; + mIsTargetFingerCountReached = false; + for (int i = 0; i < mBases.length; i++) { + mBases[i].set(Float.NaN, Float.NaN); + } + mExcludedPointsForDownSlopChecked.clear(); + super.clear(); + } + + @Override + protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + // Before the matcher state transit to completed, + // Cancel when an additional down arrived after reaching the target number of taps. + if (mCompletedTapCount == mTargetTapCount) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + cancelAfterTapTimeout(event, rawEvent, policyFlags); + + if (mCompletedTapCount == 0) { + initBaseLocation(rawEvent); + return; + } + // As fingers go up and down, their pointer ids will not be the same. + // Therefore we require that a given finger be in slop range of any one + // of the fingers from the previous tap. + final PointF nearest = findNearestPoint(rawEvent, mDoubleTapSlop, true); + if (nearest != null) { + // Update pointer location to nearest one as a new base for next slop check. + final int index = event.getActionIndex(); + nearest.set(event.getX(index), event.getY(index)); + } else { + cancelGesture(event, rawEvent, policyFlags); + } + } + + @Override + protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags); + + final PointF nearest = findNearestPoint(rawEvent, mTouchSlop, false); + if ((getState() == STATE_GESTURE_STARTED || getState() == STATE_CLEAR) && null != nearest) { + // Increase current tap count when the user have all fingers lifted + // within the tap timeout since the target number of fingers are down. + if (mIsTargetFingerCountReached) { + mCompletedTapCount++; + mIsTargetFingerCountReached = false; + mExcludedPointsForDownSlopChecked.clear(); + } + + // Start gesture detection here to avoid the conflict to 2nd finger double tap + // that never actually started gesture detection. + if (mCompletedTapCount == 1) { + startGesture(event, rawEvent, policyFlags); + } + if (mCompletedTapCount == mTargetTapCount) { + // Done. + completeAfterDoubleTapTimeout(event, rawEvent, policyFlags); + } + } else { + // Either too many taps or nonsensical event stream. + cancelGesture(event, rawEvent, policyFlags); + } + } + + @Override + protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + // Outside the touch slop + if (null == findNearestPoint(rawEvent, mTouchSlop, false)) { + cancelGesture(event, rawEvent, policyFlags); + } + } + + @Override + protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + // Reset timeout to ease the use for some people + // with certain impairments to get all their fingers down. + cancelAfterTapTimeout(event, rawEvent, policyFlags); + final int currentFingerCount = event.getPointerCount(); + // Accept down only before target number of fingers are down + // or the finger count is not more than target. + if ((currentFingerCount > mTargetFingerCount) || mIsTargetFingerCountReached) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + + final PointF nearest; + if (mCompletedTapCount == 0) { + nearest = initBaseLocation(rawEvent); + } else { + nearest = findNearestPoint(rawEvent, mDoubleTapSlop, true); + } + if ((getState() == STATE_GESTURE_STARTED || getState() == STATE_CLEAR) && nearest != null) { + // The user have all fingers down within the tap timeout since first finger down, + // setting the timeout for fingers to be lifted. + if (currentFingerCount == mTargetFingerCount) { + mIsTargetFingerCountReached = true; + } + // Update pointer location to nearest one as a new base for next slop check. + final int index = event.getActionIndex(); + nearest.set(event.getX(index), event.getY(index)); + } else { + cancelGesture(event, rawEvent, policyFlags); + } + } + + @Override + protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + // Accept up only after target number of fingers are down. + if (!mIsTargetFingerCountReached) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + + if (getState() == STATE_GESTURE_STARTED || getState() == STATE_CLEAR) { + // Needs more fingers lifted within the tap timeout + // after reaching the target number of fingers are down. + cancelAfterTapTimeout(event, rawEvent, policyFlags); + } else { + cancelGesture(event, rawEvent, policyFlags); + } + } + + @Override + public String getGestureName() { + final StringBuilder builder = new StringBuilder(); + builder.append(mTargetFingerCount).append("-Finger "); + if (mTargetTapCount == 1) { + builder.append("Single"); + } else if (mTargetTapCount == 2) { + builder.append("Double"); + } else if (mTargetTapCount == 3) { + builder.append("Triple"); + } else if (mTargetTapCount > 3) { + builder.append(mTargetTapCount); + } + return builder.append(" Tap").toString(); + } + + private PointF initBaseLocation(MotionEvent event) { + final int index = event.getActionIndex(); + final int baseIndex = event.getPointerCount() - 1; + final PointF p = mBases[baseIndex]; + if (Float.isNaN(p.x) && Float.isNaN(p.y)) { + p.set(event.getX(index), event.getY(index)); + } + return p; + } + + /** + * Find the nearest location to the given event in the bases. If no one found, it could be not + * inside {@code slop}, filtered or empty bases. When {@code filterMatched} is true, if the + * location of given event matches one of the points in {@link + * #mExcludedPointsForDownSlopChecked} it would be ignored. Otherwise, the location will be + * added to {@link #mExcludedPointsForDownSlopChecked}. + * + * @param event to find nearest point in bases. + * @param slop to check to the given location of the event. + * @param filterMatched true to exclude points already matched other pointers. + * @return the point in bases closed to the location of the given event. + */ + private PointF findNearestPoint(MotionEvent event, float slop, boolean filterMatched) { + float moveDelta = Float.MAX_VALUE; + PointF nearest = null; + for (int i = 0; i < mBases.length; i++) { + final PointF p = mBases[i]; + if (Float.isNaN(p.x) && Float.isNaN(p.y)) { + continue; + } + if (filterMatched && mExcludedPointsForDownSlopChecked.contains(p)) { + continue; + } + final int index = event.getActionIndex(); + final float dX = p.x - event.getX(index); + final float dY = p.y - event.getY(index); + if (dX == 0 && dY == 0) { + if (filterMatched) { + mExcludedPointsForDownSlopChecked.add(p); + } + return p; + } + final float delta = (float) Math.hypot(dX, dY); + if (moveDelta > delta) { + moveDelta = delta; + nearest = p; + } + } + if (moveDelta < slop) { + if (filterMatched) { + mExcludedPointsForDownSlopChecked.add(nearest); + } + return nearest; + } + return null; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(super.toString()); + if (getState() != STATE_GESTURE_CANCELED) { + builder.append(", CompletedTapCount: "); + builder.append(mCompletedTapCount); + builder.append(", IsTargetFingerCountReached: "); + builder.append(mIsTargetFingerCountReached); + builder.append(", Bases: "); + builder.append(Arrays.toString(mBases)); + builder.append(", ExcludedPointsForDownSlopChecked: "); + builder.append(mExcludedPointsForDownSlopChecked.toString()); + } + return builder.toString(); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerMultiTapAndHold.java b/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerMultiTapAndHold.java new file mode 100644 index 000000000000..7824fd902c9b --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerMultiTapAndHold.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import android.content.Context; +import android.view.MotionEvent; + +/** + * This class matches gestures of the form multi-finger multi-tap and hold. The number of fingers + * and taps for each instance is specified in the constructor. + */ +class MultiFingerMultiTapAndHold extends MultiFingerMultiTap { + + MultiFingerMultiTapAndHold( + Context context, + int fingers, + int taps, + int gestureId, + GestureMatcher.StateChangeListener listener) { + super(context, fingers, taps, gestureId, listener); + } + + @Override + protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + super.onPointerDown(event, rawEvent, policyFlags); + if (mIsTargetFingerCountReached && mCompletedTapCount + 1 == mTargetTapCount) { + completeAfterLongPressTimeout(event, rawEvent, policyFlags); + } + } + + @Override + protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mCompletedTapCount + 1 == mTargetFingerCount) { + // Calling super.onUp would complete the multi-tap version of this. + cancelGesture(event, rawEvent, policyFlags); + } else { + super.onUp(event, rawEvent, policyFlags); + cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags); + } + } + + @Override + public String getGestureName() { + final StringBuilder builder = new StringBuilder(); + builder.append(mTargetFingerCount).append("-Finger "); + if (mTargetTapCount == 1) { + builder.append("Single"); + } else if (mTargetTapCount == 2) { + builder.append("Double"); + } else if (mTargetTapCount == 3) { + builder.append("Triple"); + } else if (mTargetTapCount > 3) { + builder.append(mTargetTapCount); + } + return builder.append(" Tap and hold").toString(); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerSwipe.java b/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerSwipe.java new file mode 100644 index 000000000000..4b89731b75b6 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerSwipe.java @@ -0,0 +1,514 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import static android.view.MotionEvent.INVALID_POINTER_ID; + +import static com.android.server.accessibility.gestures.GestureUtils.MM_PER_CM; +import static com.android.server.accessibility.gestures.GestureUtils.getActionIndex; +import static com.android.server.accessibility.gestures.Swipe.GESTURE_CONFIRM_CM; +import static com.android.server.accessibility.gestures.Swipe.MAX_TIME_TO_CONTINUE_SWIPE_MS; +import static com.android.server.accessibility.gestures.Swipe.MAX_TIME_TO_START_SWIPE_MS; +import static com.android.server.accessibility.gestures.TouchExplorer.DEBUG; + +import android.content.Context; +import android.graphics.PointF; +import android.os.Handler; +import android.util.DisplayMetrics; +import android.util.Slog; +import android.util.TypedValue; +import android.view.MotionEvent; +import android.view.ViewConfiguration; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * This class is responsible for matching one-finger swipe gestures. Each instance matches one swipe + * gesture. A swipe is specified as a series of one or more directions e.g. left, left and up, etc. + * At this time swipes with more than two directions are not supported. + */ +class MultiFingerSwipe extends GestureMatcher { + + // Direction constants. + public static final int LEFT = 0; + public static final int RIGHT = 1; + public static final int UP = 2; + public static final int DOWN = 3; + // This is the calculated movement threshold used track if the user is still + // moving their finger. + private final float mGestureDetectionThresholdPixels; + + // Buffer for storing points for gesture detection. + private final ArrayList<PointF>[] mStrokeBuffers; + + // The swipe direction for this matcher. + private int mDirection; + private int[] mPointerIds; + // The starting point of each finger's path in the gesture. + private PointF[] mBase; + // The most recent entry in each finger's gesture path. + private PointF[] mPreviousGesturePoint; + private int mTargetFingerCount; + private int mCurrentFingerCount; + // Whether the appropriate number of fingers have gone down at some point. This is reset only on + // clear. + private boolean mTargetFingerCountReached = false; + // Constants for sampling motion event points. + // We sample based on a minimum distance between points, primarily to improve accuracy by + // reducing noisy minor changes in direction. + private static final float MIN_CM_BETWEEN_SAMPLES = 0.25f; + private final float mMinPixelsBetweenSamplesX; + private final float mMinPixelsBetweenSamplesY; + // The minmimum distance the finger must travel before we evaluate the initial direction of the + // swipe. + // Anything less is still considered a touch. + private int mTouchSlop; + + MultiFingerSwipe( + Context context, + int fingerCount, + int direction, + int gesture, + GestureMatcher.StateChangeListener listener) { + super(gesture, new Handler(context.getMainLooper()), listener); + mTargetFingerCount = fingerCount; + mPointerIds = new int[mTargetFingerCount]; + mBase = new PointF[mTargetFingerCount]; + mPreviousGesturePoint = new PointF[mTargetFingerCount]; + mStrokeBuffers = new ArrayList[mTargetFingerCount]; + mDirection = direction; + DisplayMetrics displayMetrics = context.getResources().getDisplayMetrics(); + mGestureDetectionThresholdPixels = + TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_MM, MM_PER_CM, displayMetrics) + * GESTURE_CONFIRM_CM; + // Calculate minimum gesture velocity + final float pixelsPerCmX = displayMetrics.xdpi / GestureUtils.CM_PER_INCH; + final float pixelsPerCmY = displayMetrics.ydpi / GestureUtils.CM_PER_INCH; + mMinPixelsBetweenSamplesX = MIN_CM_BETWEEN_SAMPLES * pixelsPerCmX; + mMinPixelsBetweenSamplesY = MIN_CM_BETWEEN_SAMPLES * pixelsPerCmY; + mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop(); + clear(); + } + + @Override + protected void clear() { + mTargetFingerCountReached = false; + mCurrentFingerCount = 0; + for (int i = 0; i < mTargetFingerCount; ++i) { + mPointerIds[i] = INVALID_POINTER_ID; + if (mBase[i] == null) { + mBase[i] = new PointF(); + } + mBase[i].x = Float.NaN; + mBase[i].y = Float.NaN; + if (mPreviousGesturePoint[i] == null) { + mPreviousGesturePoint[i] = new PointF(); + } + mPreviousGesturePoint[i].x = Float.NaN; + mPreviousGesturePoint[i].y = Float.NaN; + if (mStrokeBuffers[i] == null) { + mStrokeBuffers[i] = new ArrayList<>(100); + } + mStrokeBuffers[i].clear(); + } + super.clear(); + } + + @Override + protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mCurrentFingerCount > 0) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + mCurrentFingerCount = 1; + final int actionIndex = getActionIndex(rawEvent); + final int pointerId = rawEvent.getPointerId(actionIndex); + int pointerIndex = rawEvent.getPointerCount() - 1; + if (pointerId < 0) { + // Nonsensical pointer id. + cancelGesture(event, rawEvent, policyFlags); + return; + } + if (mPointerIds[pointerIndex] != INVALID_POINTER_ID) { + // Inconsistent event stream. + cancelGesture(event, rawEvent, policyFlags); + return; + } + mPointerIds[pointerIndex] = pointerId; + cancelAfterPauseThreshold(event, rawEvent, policyFlags); + if (Float.isNaN(mBase[pointerIndex].x) && Float.isNaN(mBase[pointerIndex].y)) { + final float x = rawEvent.getX(actionIndex); + final float y = rawEvent.getY(actionIndex); + if (x < 0f || y < 0f) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + mBase[pointerIndex].x = x; + mBase[pointerIndex].y = y; + mPreviousGesturePoint[pointerIndex].x = x; + mPreviousGesturePoint[pointerIndex].y = y; + } else { + // This event doesn't make sense in the middle of a gesture. + cancelGesture(event, rawEvent, policyFlags); + return; + } + } + + @Override + protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (event.getPointerCount() > mTargetFingerCount) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + mCurrentFingerCount += 1; + if (mCurrentFingerCount != rawEvent.getPointerCount()) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + if (mCurrentFingerCount == mTargetFingerCount) { + mTargetFingerCountReached = true; + } + final int actionIndex = getActionIndex(rawEvent); + final int pointerId = rawEvent.getPointerId(actionIndex); + if (pointerId < 0) { + // Nonsensical pointer id. + cancelGesture(event, rawEvent, policyFlags); + return; + } + int pointerIndex = mCurrentFingerCount - 1; + if (mPointerIds[pointerIndex] != INVALID_POINTER_ID) { + // Inconsistent event stream. + cancelGesture(event, rawEvent, policyFlags); + return; + } + mPointerIds[pointerIndex] = pointerId; + cancelAfterPauseThreshold(event, rawEvent, policyFlags); + if (Float.isNaN(mBase[pointerIndex].x) && Float.isNaN(mBase[pointerIndex].y)) { + final float x = rawEvent.getX(actionIndex); + final float y = rawEvent.getY(actionIndex); + if (x < 0f || y < 0f) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + mBase[pointerIndex].x = x; + mBase[pointerIndex].y = y; + mPreviousGesturePoint[pointerIndex].x = x; + mPreviousGesturePoint[pointerIndex].y = y; + } else { + cancelGesture(event, rawEvent, policyFlags); + return; + } + } + + @Override + protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (!mTargetFingerCountReached) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + mCurrentFingerCount -= 1; + final int actionIndex = getActionIndex(event); + final int pointerId = event.getPointerId(actionIndex); + if (pointerId < 0) { + // Nonsensical pointer id. + cancelGesture(event, rawEvent, policyFlags); + return; + } + final int pointerIndex = Arrays.binarySearch(mPointerIds, pointerId); + if (pointerIndex < 0) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + final float x = rawEvent.getX(actionIndex); + final float y = rawEvent.getY(actionIndex); + if (x < 0f || y < 0f) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + final float dX = Math.abs(x - mPreviousGesturePoint[pointerIndex].x); + final float dY = Math.abs(y - mPreviousGesturePoint[pointerIndex].y); + if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { + mStrokeBuffers[pointerIndex].add(new PointF(x, y)); + } + // We will evaluate all the paths on ACTION_UP. + } + + @Override + protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + for (int pointerIndex = 0; pointerIndex < mTargetFingerCount; ++pointerIndex) { + if (mPointerIds[pointerIndex] == INVALID_POINTER_ID) { + // Fingers have started to move before the required number of fingers are down. + // However, they can still move less than the touch slop and still be considered + // touching, not moving. + // So we just ignore fingers that haven't been assigned a pointer id and process + // those who have. + continue; + } + if (DEBUG) { + Slog.d(getGestureName(), "Processing move on finger " + pointerIndex); + } + int index = rawEvent.findPointerIndex(mPointerIds[pointerIndex]); + if (index < 0) { + // This finger is not present in this event. It could have gone up just before this + // movement. + if (DEBUG) { + Slog.d( + getGestureName(), + "Finger " + pointerIndex + " not found in this event. skipping."); + } + continue; + } + final float x = rawEvent.getX(index); + final float y = rawEvent.getY(index); + if (x < 0f || y < 0f) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + final float dX = Math.abs(x - mPreviousGesturePoint[pointerIndex].x); + final float dY = Math.abs(y - mPreviousGesturePoint[pointerIndex].y); + final double moveDelta = + Math.hypot( + Math.abs(x - mBase[pointerIndex].x), + Math.abs(y - mBase[pointerIndex].y)); + if (DEBUG) { + Slog.d( + getGestureName(), + "moveDelta:" + + Double.toString(moveDelta) + + " mGestureDetectionThreshold: " + + Float.toString(mGestureDetectionThresholdPixels)); + } + if (getState() == STATE_CLEAR) { + if (moveDelta < mTouchSlop) { + // This still counts as a touch not a swipe. + continue; + } else if (mStrokeBuffers[pointerIndex].size() == 0) { + // First, make sure we have the right number of fingers down. + if (mCurrentFingerCount != mTargetFingerCount) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + // Then, make sure the pointer is going in the right direction. + int direction = + toDirection(x - mBase[pointerIndex].x, y - mBase[pointerIndex].y); + if (direction != mDirection) { + cancelGesture(event, rawEvent, policyFlags); + return; + } else { + // This is confirmed to be some kind of swipe so start tracking points. + cancelAfterPauseThreshold(event, rawEvent, policyFlags); + for (int i = 0; i < mTargetFingerCount; ++i) { + mStrokeBuffers[i].add(new PointF(mBase[i])); + } + } + } + if (moveDelta > mGestureDetectionThresholdPixels) { + // Try to cancel if the finger starts to go the wrong way. + // Note that this only works because this matcher assumes one direction. + int direction = + toDirection(x - mBase[pointerIndex].x, y - mBase[pointerIndex].y); + if (direction != mDirection) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + // If the pointer has moved more than the threshold, + // update the stored values. + mBase[pointerIndex].x = x; + mBase[pointerIndex].y = y; + mPreviousGesturePoint[pointerIndex].x = x; + mPreviousGesturePoint[pointerIndex].y = y; + if (getState() == STATE_CLEAR) { + startGesture(event, rawEvent, policyFlags); + cancelAfterPauseThreshold(event, rawEvent, policyFlags); + } + } + } + if (getState() == STATE_GESTURE_STARTED) { + if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { + // Sample every 2.5 MM in order to guard against minor variations in path. + mPreviousGesturePoint[pointerIndex].x = x; + mPreviousGesturePoint[pointerIndex].y = y; + mStrokeBuffers[pointerIndex].add(new PointF(x, y)); + cancelAfterPauseThreshold(event, rawEvent, policyFlags); + } + } + } + } + + @Override + protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (getState() != STATE_GESTURE_STARTED) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + mCurrentFingerCount = 0; + final int actionIndex = getActionIndex(event); + final int pointerId = event.getPointerId(actionIndex); + final int pointerIndex = Arrays.binarySearch(mPointerIds, pointerId); + if (pointerIndex < 0) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + final float x = rawEvent.getX(actionIndex); + final float y = rawEvent.getY(actionIndex); + if (x < 0f || y < 0f) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + final float dX = Math.abs(x - mPreviousGesturePoint[pointerIndex].x); + final float dY = Math.abs(y - mPreviousGesturePoint[pointerIndex].y); + if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { + mStrokeBuffers[pointerIndex].add(new PointF(x, y)); + } + recognizeGesture(event, rawEvent, policyFlags); + } + + /** + * queues a transition to STATE_GESTURE_CANCEL based on the current state. If we have + * transitioned to STATE_GESTURE_STARTED the delay is longer. + */ + private void cancelAfterPauseThreshold( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelPendingTransitions(); + switch (getState()) { + case STATE_CLEAR: + cancelAfter(MAX_TIME_TO_START_SWIPE_MS, event, rawEvent, policyFlags); + break; + case STATE_GESTURE_STARTED: + cancelAfter(MAX_TIME_TO_CONTINUE_SWIPE_MS, event, rawEvent, policyFlags); + break; + default: + break; + } + } + /** + * Looks at the sequence of motions in mStrokeBuffer, classifies the gesture, then transitions + * to the complete or cancel state depending on the result. + */ + private void recognizeGesture(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + // Check the path of each finger against the specified direction. + // Note that we sample every 2.5 MMm, and the direction matching is extremely tolerant (each + // direction has a 90-degree arch of tolerance) meaning that minor perpendicular movements + // should not create false negatives. + for (int i = 0; i < mTargetFingerCount; ++i) { + if (DEBUG) { + Slog.d(getGestureName(), "Recognizing finger: " + i); + } + if (mStrokeBuffers[i].size() < 2) { + Slog.d(getGestureName(), "Too few points."); + cancelGesture(event, rawEvent, policyFlags); + return; + } + ArrayList<PointF> path = mStrokeBuffers[i]; + + if (DEBUG) { + Slog.d(getGestureName(), "path=" + path.toString()); + } + // Classify line segments, and call Listener callbacks. + if (!recognizeGesturePath(event, rawEvent, policyFlags, path)) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + } + // If we reach this point then all paths match. + completeGesture(event, rawEvent, policyFlags); + } + + /** + * Tests the path of a given finger against the direction specified in this matcher. + * + * @return True if the path matches the specified direction for this matcher, otherwise false. + */ + private boolean recognizeGesturePath( + MotionEvent event, MotionEvent rawEvent, int policyFlags, ArrayList<PointF> path) { + + final int displayId = event.getDisplayId(); + for (int i = 0; i < path.size() - 1; ++i) { + PointF start = path.get(i); + PointF end = path.get(i + 1); + + float dX = end.x - start.x; + float dY = end.y - start.y; + int direction = toDirection(dX, dY); + if (direction != mDirection) { + if (DEBUG) { + Slog.d( + getGestureName(), + "Found direction " + + directionToString(direction) + + " when expecting " + + directionToString(mDirection)); + } + return false; + } + } + if (DEBUG) { + Slog.d(getGestureName(), "Completed."); + } + return true; + } + + private static int toDirection(float dX, float dY) { + if (Math.abs(dX) > Math.abs(dY)) { + // Horizontal + return (dX < 0) ? LEFT : RIGHT; + } else { + // Vertical + return (dY < 0) ? UP : DOWN; + } + } + + public static String directionToString(int direction) { + switch (direction) { + case LEFT: + return "left"; + case RIGHT: + return "right"; + case UP: + return "up"; + case DOWN: + return "down"; + default: + return "Unknown Direction"; + } + } + + @Override + String getGestureName() { + StringBuilder builder = new StringBuilder(); + builder.append(mTargetFingerCount).append("-finger "); + builder.append("Swipe ").append(directionToString(mDirection)); + return builder.toString(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(super.toString()); + if (getState() != STATE_GESTURE_CANCELED) { + builder.append(", mBase: ") + .append(mBase.toString()) + .append(", mGestureDetectionThreshold:") + .append(mGestureDetectionThresholdPixels) + .append(", mMinPixelsBetweenSamplesX:") + .append(mMinPixelsBetweenSamplesX) + .append(", mMinPixelsBetweenSamplesY:") + .append(mMinPixelsBetweenSamplesY); + } + return builder.toString(); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/MultiTap.java b/services/accessibility/java/com/android/server/accessibility/gestures/MultiTap.java new file mode 100644 index 000000000000..386cb0636cc2 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/MultiTap.java @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import android.content.Context; +import android.os.Handler; +import android.view.MotionEvent; +import android.view.ViewConfiguration; + +/** + * This class matches multi-tap gestures. The number of taps for each instance is specified in the + * constructor. + */ +class MultiTap extends GestureMatcher { + + // Maximum reasonable number of taps. + public static final int MAX_TAPS = 10; + final int mTargetTaps; + // The acceptable distance between two taps + int mDoubleTapSlop; + // The acceptable distance the pointer can move and still count as a tap. + int mTouchSlop; + int mTapTimeout; + int mDoubleTapTimeout; + int mCurrentTaps; + float mBaseX; + float mBaseY; + + MultiTap(Context context, int taps, int gesture, GestureMatcher.StateChangeListener listener) { + super(gesture, new Handler(context.getMainLooper()), listener); + mTargetTaps = taps; + mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop(); + mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop(); + mTapTimeout = ViewConfiguration.getTapTimeout(); + mDoubleTapTimeout = ViewConfiguration.getDoubleTapTimeout(); + clear(); + } + + @Override + protected void clear() { + mCurrentTaps = 0; + mBaseX = Float.NaN; + mBaseY = Float.NaN; + super.clear(); + } + + @Override + protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelAfterTapTimeout(event, rawEvent, policyFlags); + if (Float.isNaN(mBaseX) && Float.isNaN(mBaseY)) { + mBaseX = event.getX(); + mBaseY = event.getY(); + } + if (!isInsideSlop(rawEvent, mDoubleTapSlop)) { + cancelGesture(event, rawEvent, policyFlags); + } + mBaseX = event.getX(); + mBaseY = event.getY(); + if (mCurrentTaps + 1 == mTargetTaps) { + // Start gesture detecting on down of final tap. + // Note that if this instance is matching double tap, + // and the service is not requesting to handle double tap, GestureManifold will + // ignore this. + startGesture(event, rawEvent, policyFlags); + } + } + + @Override + protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags); + if (!isInsideSlop(rawEvent, mTouchSlop)) { + cancelGesture(event, rawEvent, policyFlags); + } + if (getState() == STATE_GESTURE_STARTED || getState() == STATE_CLEAR) { + mCurrentTaps++; + if (mCurrentTaps == mTargetTaps) { + // Done. + completeGesture(event, rawEvent, policyFlags); + return; + } + // Needs more taps. + cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags); + } else { + // Either too many taps or nonsensical event stream. + cancelGesture(event, rawEvent, policyFlags); + } + } + + @Override + protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (!isInsideSlop(rawEvent, mTouchSlop)) { + cancelGesture(event, rawEvent, policyFlags); + } + } + + @Override + protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelGesture(event, rawEvent, policyFlags); + } + + @Override + protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelGesture(event, rawEvent, policyFlags); + } + + @Override + public String getGestureName() { + switch (mTargetTaps) { + case 2: + return "Double Tap"; + case 3: + return "Triple Tap"; + default: + return Integer.toString(mTargetTaps) + " Taps"; + } + } + + private boolean isInsideSlop(MotionEvent rawEvent, int slop) { + final float deltaX = mBaseX - rawEvent.getX(); + final float deltaY = mBaseY - rawEvent.getY(); + if (deltaX == 0 && deltaY == 0) { + return true; + } + final double moveDelta = Math.hypot(deltaX, deltaY); + return moveDelta <= slop; + } + + @Override + public String toString() { + return super.toString() + + ", Taps:" + + mCurrentTaps + + ", mBaseX: " + + Float.toString(mBaseX) + + ", mBaseY: " + + Float.toString(mBaseY); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/MultiTapAndHold.java b/services/accessibility/java/com/android/server/accessibility/gestures/MultiTapAndHold.java new file mode 100644 index 000000000000..a0428a5f9a6a --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/MultiTapAndHold.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import android.content.Context; +import android.view.MotionEvent; + +/** + * This class matches gestures of the form multi-tap and hold. The number of taps for each instance + * is specified in the constructor. + */ +class MultiTapAndHold extends MultiTap { + MultiTapAndHold( + Context context, int taps, int gesture, GestureMatcher.StateChangeListener listener) { + super(context, taps, gesture, listener); + } + + @Override + protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + super.onDown(event, rawEvent, policyFlags); + if (mCurrentTaps + 1 == mTargetTaps) { + completeAfterLongPressTimeout(event, rawEvent, policyFlags); + } + } + + @Override + protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + super.onUp(event, rawEvent, policyFlags); + cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags); + } + + @Override + public String getGestureName() { + switch (mTargetTaps) { + case 2: + return "Double Tap and Hold"; + case 3: + return "Triple Tap and Hold"; + default: + return Integer.toString(mTargetTaps) + " Taps and Hold"; + } + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/SecondFingerMultiTap.java b/services/accessibility/java/com/android/server/accessibility/gestures/SecondFingerMultiTap.java new file mode 100644 index 000000000000..ada251f2363c --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/SecondFingerMultiTap.java @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import static android.view.MotionEvent.INVALID_POINTER_ID; + +import static com.android.server.accessibility.gestures.GestureUtils.getActionIndex; + +import android.content.Context; +import android.os.Handler; +import android.view.MotionEvent; +import android.view.ViewConfiguration; + +/** + * This class matches second-finger multi-tap gestures. A second-finger multi-tap gesture is where + * one finger is held down and a second finger executes the taps. The number of taps for each + * instance is specified in the constructor. + */ +class SecondFingerMultiTap extends GestureMatcher { + final int mTargetTaps; + int mDoubleTapSlop; + int mTouchSlop; + int mTapTimeout; + int mDoubleTapTimeout; + int mCurrentTaps; + int mSecondFingerPointerId; + float mBaseX; + float mBaseY; + + SecondFingerMultiTap( + Context context, int taps, int gesture, GestureMatcher.StateChangeListener listener) { + super(gesture, new Handler(context.getMainLooper()), listener); + mTargetTaps = taps; + mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop(); + mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop(); + mTapTimeout = ViewConfiguration.getTapTimeout(); + mDoubleTapTimeout = ViewConfiguration.getDoubleTapTimeout(); + clear(); + } + + @Override + protected void clear() { + mCurrentTaps = 0; + mBaseX = Float.NaN; + mBaseY = Float.NaN; + mSecondFingerPointerId = INVALID_POINTER_ID; + super.clear(); + } + + @Override + protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (event.getPointerCount() > 2) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + // Second finger has gone down. + int index = getActionIndex(event); + mSecondFingerPointerId = event.getPointerId(index); + cancelAfterTapTimeout(event, rawEvent, policyFlags); + if (Float.isNaN(mBaseX) && Float.isNaN(mBaseY)) { + mBaseX = event.getX(); + mBaseY = event.getY(); + } + if (!isSecondFingerInsideSlop(rawEvent, mDoubleTapSlop)) { + cancelGesture(event, rawEvent, policyFlags); + } + mBaseX = event.getX(); + mBaseY = event.getY(); + } + + @Override + protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (event.getPointerCount() > 2) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags); + if (!isSecondFingerInsideSlop(rawEvent, mTouchSlop)) { + cancelGesture(event, rawEvent, policyFlags); + } + if (getState() == STATE_GESTURE_STARTED || getState() == STATE_CLEAR) { + mCurrentTaps++; + if (mCurrentTaps == mTargetTaps) { + // Done. + completeGesture(event, rawEvent, policyFlags); + return; + } + // Needs more taps. + cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags); + } else { + // Nonsensical event stream. + cancelGesture(event, rawEvent, policyFlags); + } + } + + @Override + protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + switch (event.getPointerCount()) { + case 1: + // We don't need to track anything about one-finger movements. + break; + case 2: + if (!isSecondFingerInsideSlop(rawEvent, mTouchSlop)) { + cancelGesture(event, rawEvent, policyFlags); + } + break; + default: + // More than two fingers means we stop tracking. + cancelGesture(event, rawEvent, policyFlags); + break; + } + } + + @Override + protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + // Cancel early when possible, or it will take precedence over two-finger double tap. + cancelGesture(event, rawEvent, policyFlags); + } + + @Override + public String getGestureName() { + switch (mTargetTaps) { + case 2: + return "Second Finger Double Tap"; + case 3: + return "Second Finger Triple Tap"; + default: + return "Second Finger " + Integer.toString(mTargetTaps) + " Taps"; + } + } + + private boolean isSecondFingerInsideSlop(MotionEvent rawEvent, int slop) { + int pointerIndex = rawEvent.findPointerIndex(mSecondFingerPointerId); + if (pointerIndex == -1) { + return false; + } + final float deltaX = mBaseX - rawEvent.getX(pointerIndex); + final float deltaY = mBaseY - rawEvent.getY(pointerIndex); + if (deltaX == 0 && deltaY == 0) { + return true; + } + final double moveDelta = Math.hypot(deltaX, deltaY); + return moveDelta <= slop; + } + + @Override + public String toString() { + return super.toString() + + ", Taps:" + + mCurrentTaps + + ", mBaseX: " + + Float.toString(mBaseX) + + ", mBaseY: " + + Float.toString(mBaseY); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/Swipe.java b/services/accessibility/java/com/android/server/accessibility/gestures/Swipe.java new file mode 100644 index 000000000000..041b4243293e --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/Swipe.java @@ -0,0 +1,419 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import static com.android.server.accessibility.gestures.GestureUtils.MM_PER_CM; +import static com.android.server.accessibility.gestures.TouchExplorer.DEBUG; + +import android.content.Context; +import android.graphics.PointF; +import android.os.Handler; +import android.util.DisplayMetrics; +import android.util.Slog; +import android.util.TypedValue; +import android.view.MotionEvent; +import android.view.ViewConfiguration; + +import java.util.ArrayList; + +/** + * This class is responsible for matching one-finger swipe gestures. Each instance matches one swipe + * gesture. A swipe is specified as a series of one or more directions e.g. left, left and up, etc. + * At this time swipes with more than two directions are not supported. + */ +class Swipe extends GestureMatcher { + + // Direction constants. + public static final int LEFT = 0; + public static final int RIGHT = 1; + public static final int UP = 2; + public static final int DOWN = 3; + // This is the calculated movement threshold used track if the user is still + // moving their finger. + private final float mGestureDetectionThresholdPixels; + + // Buffer for storing points for gesture detection. + private final ArrayList<PointF> mStrokeBuffer = new ArrayList<>(100); + + // Constants for sampling motion event points. + // We sample based on a minimum distance between points, primarily to improve accuracy by + // reducing noisy minor changes in direction. + private static final float MIN_CM_BETWEEN_SAMPLES = 0.25f; + + // Distance a finger must travel before we decide if it is a gesture or not. + public static final int GESTURE_CONFIRM_CM = 1; + + // Time threshold used to determine if an interaction is a gesture or not. + // If the first movement of 1cm takes longer than this value, we assume it's + // a slow movement, and therefore not a gesture. + // + // This value was determined by measuring the time for the first 1cm + // movement when gesturing, and touch exploring. Based on user testing, + // all gestures started with the initial movement taking less than 100ms. + // When touch exploring, the first movement almost always takes longer than + // 200ms. + public static final long MAX_TIME_TO_START_SWIPE_MS = 150 * GESTURE_CONFIRM_CM; + + // Time threshold used to determine if a gesture should be cancelled. If + // the finger takes more than this time to move to the next sample point, the ongoing gesture + // is cancelled. + public static final long MAX_TIME_TO_CONTINUE_SWIPE_MS = 350 * GESTURE_CONFIRM_CM; + + private int[] mDirections; + private float mBaseX; + private float mBaseY; + private long mBaseTime; + private float mPreviousGestureX; + private float mPreviousGestureY; + private final float mMinPixelsBetweenSamplesX; + private final float mMinPixelsBetweenSamplesY; + // The minmimum distance the finger must travel before we evaluate the initial direction of the + // swipe. + // Anything less is still considered a touch. + private int mTouchSlop; + + // Constants for separating gesture segments + private static final float ANGLE_THRESHOLD = 0.0f; + + Swipe( + Context context, + int direction, + int gesture, + GestureMatcher.StateChangeListener listener) { + this(context, new int[] {direction}, gesture, listener); + } + + Swipe( + Context context, + int direction1, + int direction2, + int gesture, + GestureMatcher.StateChangeListener listener) { + this(context, new int[] {direction1, direction2}, gesture, listener); + } + + private Swipe( + Context context, + int[] directions, + int gesture, + GestureMatcher.StateChangeListener listener) { + super(gesture, new Handler(context.getMainLooper()), listener); + mDirections = directions; + DisplayMetrics displayMetrics = context.getResources().getDisplayMetrics(); + mGestureDetectionThresholdPixels = + TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_MM, MM_PER_CM, displayMetrics) + * GESTURE_CONFIRM_CM; + // Calculate minimum gesture velocity + final float pixelsPerCmX = displayMetrics.xdpi / 2.54f; + final float pixelsPerCmY = displayMetrics.ydpi / 2.54f; + mMinPixelsBetweenSamplesX = MIN_CM_BETWEEN_SAMPLES * pixelsPerCmX; + mMinPixelsBetweenSamplesY = MIN_CM_BETWEEN_SAMPLES * pixelsPerCmY; + mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop(); + clear(); + } + + @Override + protected void clear() { + mBaseX = Float.NaN; + mBaseY = Float.NaN; + mBaseTime = 0; + mPreviousGestureX = Float.NaN; + mPreviousGestureY = Float.NaN; + mStrokeBuffer.clear(); + super.clear(); + } + + @Override + protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (Float.isNaN(mBaseX) && Float.isNaN(mBaseY)) { + mBaseX = rawEvent.getX(); + mBaseY = rawEvent.getY(); + mBaseTime = rawEvent.getEventTime(); + mPreviousGestureX = mBaseX; + mPreviousGestureY = mBaseY; + } + // Otherwise do nothing because this event doesn't make sense in the middle of a gesture. + } + + @Override + protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + final float x = rawEvent.getX(); + final float y = rawEvent.getY(); + final long time = rawEvent.getEventTime(); + final float dX = Math.abs(x - mPreviousGestureX); + final float dY = Math.abs(y - mPreviousGestureY); + final double moveDelta = Math.hypot(Math.abs(x - mBaseX), Math.abs(y - mBaseY)); + final long timeDelta = time - mBaseTime; + if (DEBUG) { + Slog.d( + getGestureName(), + "moveDelta:" + + Double.toString(moveDelta) + + " mGestureDetectionThreshold: " + + Float.toString(mGestureDetectionThresholdPixels)); + } + if (getState() == STATE_CLEAR) { + if (moveDelta < mTouchSlop) { + // This still counts as a touch not a swipe. + return; + } else if (mStrokeBuffer.size() == 0) { + // First, make sure the pointer is going in the right direction. + int direction = toDirection(x - mBaseX, y - mBaseY); + if (direction != mDirections[0]) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + // This is confirmed to be some kind of swipe so start tracking points. + mStrokeBuffer.add(new PointF(mBaseX, mBaseY)); + } + } + if (moveDelta > mGestureDetectionThresholdPixels) { + // This is a gesture, not touch exploration. + mBaseX = x; + mBaseY = y; + mBaseTime = time; + startGesture(event, rawEvent, policyFlags); + } else if (getState() == STATE_CLEAR) { + if (timeDelta > MAX_TIME_TO_START_SWIPE_MS) { + // The user isn't moving fast enough. + cancelGesture(event, rawEvent, policyFlags); + return; + } + } else if (getState() == STATE_GESTURE_STARTED) { + if (timeDelta > MAX_TIME_TO_CONTINUE_SWIPE_MS) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + } + if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { + // At this point gesture detection has started and we are sampling points. + mPreviousGestureX = x; + mPreviousGestureY = y; + mStrokeBuffer.add(new PointF(x, y)); + } + } + + @Override + protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (getState() != STATE_GESTURE_STARTED) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + + final float x = rawEvent.getX(); + final float y = rawEvent.getY(); + final float dX = Math.abs(x - mPreviousGestureX); + final float dY = Math.abs(y - mPreviousGestureY); + if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { + mStrokeBuffer.add(new PointF(x, y)); + } + recognizeGesture(event, rawEvent, policyFlags); + } + + @Override + protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelGesture(event, rawEvent, policyFlags); + } + + @Override + protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelGesture(event, rawEvent, policyFlags); + } + + /** + * Looks at the sequence of motions in mStrokeBuffer, classifies the gesture, then calls + * Listener callbacks for success or failure. + * + * @param event The raw motion event to pass to the listener callbacks. + * @param policyFlags Policy flags for the event. + * @return true if the event is consumed, else false + */ + private void recognizeGesture(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mStrokeBuffer.size() < 2) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + + // Look at mStrokeBuffer and extract 2 line segments, delimited by near-perpendicular + // direction change. + // Method: for each sampled motion event, check the angle of the most recent motion vector + // versus the preceding motion vector, and segment the line if the angle is about + // 90 degrees. + + ArrayList<PointF> path = new ArrayList<>(); + PointF lastDelimiter = mStrokeBuffer.get(0); + path.add(lastDelimiter); + + float dX = 0; // Sum of unit vectors from last delimiter to each following point + float dY = 0; + int count = 0; // Number of points since last delimiter + float length = 0; // Vector length from delimiter to most recent point + + PointF next = null; + for (int i = 1; i < mStrokeBuffer.size(); ++i) { + next = mStrokeBuffer.get(i); + if (count > 0) { + // Average of unit vectors from delimiter to following points + float currentDX = dX / count; + float currentDY = dY / count; + + // newDelimiter is a possible new delimiter, based on a vector with length from + // the last delimiter to the previous point, but in the direction of the average + // unit vector from delimiter to previous points. + // Using the averaged vector has the effect of "squaring off the curve", + // creating a sharper angle between the last motion and the preceding motion from + // the delimiter. In turn, this sharper angle achieves the splitting threshold + // even in a gentle curve. + PointF newDelimiter = + new PointF( + length * currentDX + lastDelimiter.x, + length * currentDY + lastDelimiter.y); + + // Unit vector from newDelimiter to the most recent point + float nextDX = next.x - newDelimiter.x; + float nextDY = next.y - newDelimiter.y; + float nextLength = (float) Math.sqrt(nextDX * nextDX + nextDY * nextDY); + nextDX = nextDX / nextLength; + nextDY = nextDY / nextLength; + + // Compare the initial motion direction to the most recent motion direction, + // and segment the line if direction has changed by about 90 degrees. + float dot = currentDX * nextDX + currentDY * nextDY; + if (dot < ANGLE_THRESHOLD) { + path.add(newDelimiter); + lastDelimiter = newDelimiter; + dX = 0; + dY = 0; + count = 0; + } + } + + // Vector from last delimiter to most recent point + float currentDX = next.x - lastDelimiter.x; + float currentDY = next.y - lastDelimiter.y; + length = (float) Math.sqrt(currentDX * currentDX + currentDY * currentDY); + + // Increment sum of unit vectors from delimiter to each following point + count = count + 1; + dX = dX + currentDX / length; + dY = dY + currentDY / length; + } + + path.add(next); + if (DEBUG) { + Slog.d(getGestureName(), "path=" + path.toString()); + } + // Classify line segments, and call Listener callbacks. + recognizeGesturePath(event, rawEvent, policyFlags, path); + } + + /** + * Classifies a pair of line segments, by direction. Calls Listener callbacks for success or + * failure. + * + * @param event The raw motion event to pass to the listener's onGestureCanceled method. + * @param policyFlags Policy flags for the event. + * @param path A sequence of motion line segments derived from motion points in mStrokeBuffer. + * @return true if the event is consumed, else false + */ + private void recognizeGesturePath( + MotionEvent event, MotionEvent rawEvent, int policyFlags, ArrayList<PointF> path) { + + final int displayId = event.getDisplayId(); + if (path.size() != mDirections.length + 1) { + cancelGesture(event, rawEvent, policyFlags); + return; + } + for (int i = 0; i < path.size() - 1; ++i) { + PointF start = path.get(i); + PointF end = path.get(i + 1); + + float dX = end.x - start.x; + float dY = end.y - start.y; + int direction = toDirection(dX, dY); + if (direction != mDirections[i]) { + if (DEBUG) { + Slog.d( + getGestureName(), + "Found direction " + + directionToString(direction) + + " when expecting " + + directionToString(mDirections[i])); + } + cancelGesture(event, rawEvent, policyFlags); + return; + } + } + if (DEBUG) { + Slog.d(getGestureName(), "Completed."); + } + completeGesture(event, rawEvent, policyFlags); + } + + private static int toDirection(float dX, float dY) { + if (Math.abs(dX) > Math.abs(dY)) { + // Horizontal + return (dX < 0) ? LEFT : RIGHT; + } else { + // Vertical + return (dY < 0) ? UP : DOWN; + } + } + + public static String directionToString(int direction) { + switch (direction) { + case LEFT: + return "left"; + case RIGHT: + return "right"; + case UP: + return "up"; + case DOWN: + return "down"; + default: + return "Unknown Direction"; + } + } + + @Override + String getGestureName() { + StringBuilder builder = new StringBuilder(); + builder.append("Swipe ").append(directionToString(mDirections[0])); + for (int i = 1; i < mDirections.length; ++i) { + builder.append(" and ").append(directionToString(mDirections[i])); + } + return builder.toString(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(super.toString()); + if (getState() != STATE_GESTURE_CANCELED) { + builder.append(", mBaseX: ") + .append(mBaseX) + .append(", mBaseY: ") + .append(mBaseY) + .append(", mGestureDetectionThreshold:") + .append(mGestureDetectionThresholdPixels) + .append(", mMinPixelsBetweenSamplesX:") + .append(mMinPixelsBetweenSamplesX) + .append(", mMinPixelsBetweenSamplesY:") + .append(mMinPixelsBetweenSamplesY); + } + return builder.toString(); + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/TouchExplorer.java b/services/accessibility/java/com/android/server/accessibility/gestures/TouchExplorer.java new file mode 100644 index 000000000000..fbc986bdd730 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/TouchExplorer.java @@ -0,0 +1,1195 @@ +/* + ** Copyright 2011, The Android Open Source Project + ** + ** Licensed under the Apache License, Version 2.0 (the "License"); + ** you may not use this file except in compliance with the License. + ** You may obtain a copy of the License at + ** + ** http://www.apache.org/licenses/LICENSE-2.0 + ** + ** Unless required by applicable law or agreed to in writing, software + ** distributed under the License is distributed on an "AS IS" BASIS, + ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ** See the License for the specific language governing permissions and + ** limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import static android.view.MotionEvent.INVALID_POINTER_ID; + +import static com.android.server.accessibility.gestures.TouchState.ALL_POINTER_ID_BITS; + +import android.accessibilityservice.AccessibilityGestureEvent; +import android.content.Context; +import android.graphics.Region; +import android.os.Handler; +import android.util.Slog; +import android.view.InputDevice; +import android.view.MotionEvent; +import android.view.ViewConfiguration; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityNodeInfo; + +import com.android.server.accessibility.AccessibilityManagerService; +import com.android.server.accessibility.BaseEventStreamTransformation; +import com.android.server.accessibility.EventStreamTransformation; +import com.android.server.policy.WindowManagerPolicy; + +import java.util.ArrayList; +import java.util.List; + +/** + * This class is a strategy for performing touch exploration. It + * transforms the motion event stream by modifying, adding, replacing, + * and consuming certain events. The interaction model is: + * + * <ol> + * <li>1. One finger moving slow around performs touch exploration.</li> + * <li>2. One finger moving fast around performs gestures.</li> + * <li>3. Two close fingers moving in the same direction perform a drag.</li> + * <li>4. Multi-finger gestures are delivered to view hierarchy.</li> + * <li>5. Two fingers moving in different directions are considered a multi-finger gesture.</li> + * <li>6. Double tapping performs a click action on the accessibility + * focused rectangle.</li> + * <li>7. Tapping and holding for a while performs a long press in a similar fashion + * as the click above.</li> + * <ol> + * + * @hide + */ +public class TouchExplorer extends BaseEventStreamTransformation + implements GestureManifold.Listener { + + static final boolean DEBUG = false; + + // Tag for logging received events. + private static final String LOG_TAG = "TouchExplorer"; + + // The maximum of the cosine between the vectors of two moving + // pointers so they can be considered moving in the same direction. + private static final float MAX_DRAGGING_ANGLE_COS = 0.525321989f; // cos(pi/4) + + // The timeout after which we are no longer trying to detect a gesture. + private static final int EXIT_GESTURE_DETECTION_TIMEOUT = 2000; + + // Timeout before trying to decide what the user is trying to do. + private final int mDetermineUserIntentTimeout; + + // Slop between the first and second tap to be a double tap. + private final int mDoubleTapSlop; + + // The current state of the touch explorer. + private TouchState mState; + + // The ID of the pointer used for dragging. + private int mDraggingPointerId; + + + // Handler for performing asynchronous operations. + private final Handler mHandler; + + // Command for delayed sending of a hover enter and move event. + private final SendHoverEnterAndMoveDelayed mSendHoverEnterAndMoveDelayed; + + // Command for delayed sending of a hover exit event. + private final SendHoverExitDelayed mSendHoverExitDelayed; + + // Command for delayed sending of touch exploration end events. + private final SendAccessibilityEventDelayed mSendTouchExplorationEndDelayed; + + // Command for delayed sending of touch interaction end events. + private final SendAccessibilityEventDelayed mSendTouchInteractionEndDelayed; + + // Command for exiting gesture detection mode after a timeout. + private final ExitGestureDetectionModeDelayed mExitGestureDetectionModeDelayed; + + // Helper to detect gestures. + private final GestureManifold mGestureDetector; + + // Helper class to track received pointers. + private final TouchState.ReceivedPointerTracker mReceivedPointerTracker; + + private final EventDispatcher mDispatcher; + + // Handle to the accessibility manager service. + private final AccessibilityManagerService mAms; + + + // Context in which this explorer operates. + private final Context mContext; + + private Region mGestureDetectionPassthroughRegion; + private Region mTouchExplorationPassthroughRegion; + +/** + * Creates a new instance. + * + * @param context A context handle for accessing resources. + * @param service The service to notify touch interaction and gesture completed and to perform + * action. + */ + public TouchExplorer(Context context, AccessibilityManagerService service) { + this(context, service, null); + } + + /** + * Creates a new instance. + * + * @param context A context handle for accessing resources. + * @param service The service to notify touch interaction and gesture completed and to perform + * action. + * @param detector The gesture detector to handle accessibility touch event. If null the default + * one created in place, or for testing purpose. + */ + public TouchExplorer(Context context, AccessibilityManagerService service, + GestureManifold detector) { + mContext = context; + mAms = service; + mState = new TouchState(); + mReceivedPointerTracker = mState.getReceivedPointerTracker(); + mDispatcher = new EventDispatcher(context, mAms, super.getNext(), mState); + mDetermineUserIntentTimeout = ViewConfiguration.getDoubleTapTimeout(); + mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop(); + mHandler = new Handler(context.getMainLooper()); + mExitGestureDetectionModeDelayed = new ExitGestureDetectionModeDelayed(); + mSendHoverEnterAndMoveDelayed = new SendHoverEnterAndMoveDelayed(); + mSendHoverExitDelayed = new SendHoverExitDelayed(); + mSendTouchExplorationEndDelayed = new SendAccessibilityEventDelayed( + AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END, + mDetermineUserIntentTimeout); + mSendTouchInteractionEndDelayed = new SendAccessibilityEventDelayed( + AccessibilityEvent.TYPE_TOUCH_INTERACTION_END, + mDetermineUserIntentTimeout); + if (detector == null) { + mGestureDetector = new GestureManifold(context, this, mState); + } else { + mGestureDetector = detector; + } + mGestureDetectionPassthroughRegion = new Region(); + mTouchExplorationPassthroughRegion = new Region(); + } + + @Override + public void clearEvents(int inputSource) { + if (inputSource == InputDevice.SOURCE_TOUCHSCREEN) { + clear(); + } + super.clearEvents(inputSource); + } + + @Override + public void onDestroy() { + clear(); + } + + private void clear() { + // If we have not received an event then we are in initial + // state. Therefore, there is not need to clean anything. + MotionEvent event = mState.getLastReceivedEvent(); + if (event != null) { + clear(event, WindowManagerPolicy.FLAG_TRUSTED); + } + } + + private void clear(MotionEvent event, int policyFlags) { + if (mState.isTouchExploring()) { + // If a touch exploration gesture is in progress send events for its end. + sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); + } else if (mState.isDragging()) { + mDraggingPointerId = INVALID_POINTER_ID; + // Send exit to all pointers that we have delivered. + mDispatcher.sendUpForInjectedDownPointers(event, policyFlags); + } else if (mState.isDelegating()) { + // Send exit to all pointers that we have delivered. + mDispatcher.sendUpForInjectedDownPointers(event, policyFlags); + } else if (mState.isGestureDetecting()) { + // No state specific cleanup required. + } + // Remove all pending callbacks. + mSendHoverEnterAndMoveDelayed.cancel(); + mSendHoverExitDelayed.cancel(); + mExitGestureDetectionModeDelayed.cancel(); + mSendTouchExplorationEndDelayed.cancel(); + mSendTouchInteractionEndDelayed.cancel(); + // Clear the gesture detector + mGestureDetector.clear(); + // Go to initial state. + mState.clear(); + mAms.onTouchInteractionEnd(); + } + + @Override + public void onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (!event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)) { + super.onMotionEvent(event, rawEvent, policyFlags); + return; + } + + if (DEBUG) { + Slog.d(LOG_TAG, "Received event: " + event + ", policyFlags=0x" + + Integer.toHexString(policyFlags)); + Slog.d(LOG_TAG, mState.toString()); + } + + mState.onReceivedMotionEvent(rawEvent); + if (shouldPerformGestureDetection(event)) { + if (mGestureDetector.onMotionEvent(event, rawEvent, policyFlags)) { + // Event was handled by the gesture detector. + return; + } + } + + if (event.getActionMasked() == MotionEvent.ACTION_CANCEL) { + clear(event, policyFlags); + return; + } + + // TODO: extract the below functions into separate handlers for each state. + // Right now the number of functions and number of states make the code messy. + if (mState.isClear()) { + handleMotionEventStateClear(event, rawEvent, policyFlags); + } else if (mState.isTouchInteracting()) { + handleMotionEventStateTouchInteracting(event, rawEvent, policyFlags); + } else if (mState.isTouchExploring()) { + handleMotionEventStateTouchExploring(event, rawEvent, policyFlags); + } else if (mState.isDragging()) { + handleMotionEventStateDragging(event, rawEvent, policyFlags); + } else if (mState.isDelegating()) { + handleMotionEventStateDelegating(event, rawEvent, policyFlags); + } else if (mState.isGestureDetecting()) { + // Make sure we don't prematurely get TOUCH_INTERACTION_END + // It will be delivered on gesture completion or cancelation. + // Note that the delay for sending GESTURE_DETECTION_END remains in place. + mSendTouchInteractionEndDelayed.cancel(); + } else { + Slog.e(LOG_TAG, "Illegal state: " + mState); + clear(event, policyFlags); + } + } + + @Override + public void onAccessibilityEvent(AccessibilityEvent event) { + final int eventType = event.getEventType(); + + if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT) { + sendsPendingA11yEventsIfNeed(); + } + mState.onReceivedAccessibilityEvent(event); + super.onAccessibilityEvent(event); + } + + /* + * Sends pending {@link AccessibilityEvent#TYPE_TOUCH_EXPLORATION_GESTURE_END} or {@{@link + * AccessibilityEvent#TYPE_TOUCH_EXPLORATION_GESTURE_END}} after receiving last hover exit + * event. + */ + private void sendsPendingA11yEventsIfNeed() { + // The last hover exit A11y event should be sent by view after receiving hover exit motion + // event. In some view hierarchy, the ViewGroup transforms hover move motion event to hover + // exit motion event and than dispatch to itself. It causes unexpected A11y exit events. + if (mSendHoverExitDelayed.isPending()) { + return; + } + // The event for gesture end should be strictly after the + // last hover exit event. + if (mSendTouchExplorationEndDelayed.isPending()) { + mSendTouchExplorationEndDelayed.cancel(); + mDispatcher.sendAccessibilityEvent( + AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END); + } + + // The event for touch interaction end should be strictly after the + // last hover exit and the touch exploration gesture end events. + if (mSendTouchInteractionEndDelayed.isPending()) { + mSendTouchInteractionEndDelayed.cancel(); + mDispatcher.sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + } + } + + @Override + public void onDoubleTapAndHold(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mDispatcher.longPressWithTouchEvents(event, policyFlags)) { + sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); + mState.startDelegating(); + } + } + + @Override + public boolean onDoubleTap(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + mAms.onTouchInteractionEnd(); + // Remove pending event deliveries. + mSendHoverEnterAndMoveDelayed.cancel(); + mSendHoverExitDelayed.cancel(); + + if (mSendTouchExplorationEndDelayed.isPending()) { + mSendTouchExplorationEndDelayed.forceSendAndRemove(); + } + + // Announce the end of a new touch interaction. + mDispatcher.sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + mSendTouchInteractionEndDelayed.cancel(); + // Try to use the standard accessibility API to click + if (!mAms.performActionOnAccessibilityFocusedItem( + AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK)) { + Slog.e(LOG_TAG, "ACTION_CLICK failed. Dispatching motion events to simulate click."); + + mDispatcher.clickWithTouchEvents(event, rawEvent, policyFlags); + return true; + } + return true; + } + + @Override + public boolean onGestureStarted() { + // We have to perform gesture detection, so + // clear the current state and try to detect. + mState.startGestureDetecting(); + mSendHoverEnterAndMoveDelayed.cancel(); + mSendHoverExitDelayed.cancel(); + mExitGestureDetectionModeDelayed.post(); + // Send accessibility event to announce the start + // of gesture recognition. + mDispatcher.sendAccessibilityEvent(AccessibilityEvent.TYPE_GESTURE_DETECTION_START); + return false; + } + + @Override + public boolean onGestureCompleted(AccessibilityGestureEvent gestureEvent) { + endGestureDetection(true); + mSendTouchInteractionEndDelayed.cancel(); + mAms.onGesture(gestureEvent); + + return true; + } + + @Override + public boolean onGestureCancelled(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mState.isGestureDetecting()) { + endGestureDetection(event.getActionMasked() == MotionEvent.ACTION_UP); + return true; + } else if (mState.isTouchExploring()) { + // If the finger is still moving, pass the event on. + if (event.getActionMasked() == MotionEvent.ACTION_MOVE) { + final int pointerId = mReceivedPointerTracker.getPrimaryPointerId(); + final int pointerIdBits = (1 << pointerId); + + // We have just decided that the user is touch, + // exploring so start sending events. + mSendHoverEnterAndMoveDelayed.addEvent(event, mState.getLastReceivedEvent()); + mSendHoverEnterAndMoveDelayed.forceSendAndRemove(); + mSendHoverExitDelayed.cancel(); + mDispatcher.sendMotionEvent( + event, + MotionEvent.ACTION_HOVER_MOVE, + mState.getLastReceivedEvent(), + pointerIdBits, + policyFlags); + return true; + } + } + return false; + } + + /** + * Handles a motion event in the clear state i.e. no fingers are touching the screen. + */ + private void handleMotionEventStateClear( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + switch (event.getActionMasked()) { + // The only way to leave the clear state is for a pointer to go down. + case MotionEvent.ACTION_DOWN: + handleActionDown(event, rawEvent, policyFlags); + break; + default: + // Some other nonsensical event. + break; + } + } + + /** + * Handles ACTION_DOWN while in the clear or touch interacting states. This event represents the + * first finger touching the screen. + */ + private void handleActionDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + mAms.onTouchInteractionStart(); + + // If we still have not notified the user for the last + // touch, we figure out what to do. If were waiting + // we resent the delayed callback and wait again. + mSendHoverEnterAndMoveDelayed.cancel(); + mSendHoverExitDelayed.cancel(); + // If a touch exploration gesture is in progress send events for its end. + if (mState.isTouchExploring()) { + sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); + } + + if (mState.isClear()) { + if (!mSendHoverEnterAndMoveDelayed.isPending()) { + // Queue a delayed transition to STATE_TOUCH_EXPLORING. + // If we do not detect that this is a gesture, delegation or drag the transition + // will fire by default. + // The idea is to avoid getting stuck in STATE_TOUCH_INTERACTING + final int pointerId = mReceivedPointerTracker.getPrimaryPointerId(); + final int pointerIdBits = (1 << pointerId); + mSendHoverEnterAndMoveDelayed.post(event, rawEvent, pointerIdBits, policyFlags); + } else { + // Cache the event until we discern exploration from gesturing. + mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent); + } + mSendTouchExplorationEndDelayed.forceSendAndRemove(); + mSendTouchInteractionEndDelayed.forceSendAndRemove(); + mDispatcher.sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_START); + if (mTouchExplorationPassthroughRegion.contains( + (int) event.getX(), (int) event.getY())) { + // The touch exploration passthrough overrides the gesture detection passthrough in + // the event they overlap. + // Pass this entire gesture through to the system as-is. + mState.startDelegating(); + event = MotionEvent.obtainNoHistory(event); + mDispatcher.sendMotionEvent( + event, event.getAction(), rawEvent, ALL_POINTER_ID_BITS, policyFlags); + mSendHoverEnterAndMoveDelayed.cancel(); + } else if (mGestureDetectionPassthroughRegion.contains( + (int) event.getX(), (int) event.getY())) { + // Jump straight to touch exploration. + mSendHoverEnterAndMoveDelayed.forceSendAndRemove(); + } + } else { + // Avoid duplicated TYPE_TOUCH_INTERACTION_START event when 2nd tap of double tap. + mSendTouchInteractionEndDelayed.cancel(); + } + } + + /** + * Handles a motion event in touch interacting state. + * + * @param event The event to be handled. + * @param rawEvent The raw (unmodified) motion event. + * @param policyFlags The policy flags associated with the event. + */ + private void handleMotionEventStateTouchInteracting( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + switch (event.getActionMasked()) { + case MotionEvent.ACTION_DOWN: + // Continue the previous interaction. + mSendTouchInteractionEndDelayed.cancel(); + handleActionDown(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_POINTER_DOWN: + handleActionPointerDown(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_MOVE: + handleActionMoveStateTouchInteracting(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_UP: + handleActionUp(event, rawEvent, policyFlags); + break; + } + } + + /** + * Handles a motion event in touch exploring state. + * + * @param event The event to be handled. + * @param rawEvent The raw (unmodified) motion event. + * @param policyFlags The policy flags associated with the event. + */ + private void handleMotionEventStateTouchExploring( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + switch (event.getActionMasked()) { + case MotionEvent.ACTION_DOWN: + // We should have already received ACTION_DOWN. Ignore. + break; + case MotionEvent.ACTION_POINTER_DOWN: + handleActionPointerDown(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_MOVE: + handleActionMoveStateTouchExploring(event, rawEvent, policyFlags); + break; + case MotionEvent.ACTION_UP: + handleActionUp(event, rawEvent, policyFlags); + break; + default: + break; + } + } + + /** + * Handles ACTION_POINTER_DOWN when in the touch exploring state. This event represents an + * additional finger touching the screen. + */ + private void handleActionPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + // Another finger down means that if we have not started to deliver + // hover events, we will not have to. The code for ACTION_MOVE will + // decide what we will actually do next. + + if (mSendHoverEnterAndMoveDelayed.isPending()) { + mSendHoverEnterAndMoveDelayed.cancel(); + mSendHoverExitDelayed.cancel(); + } else { + // We have already delivered at least one hover event, so send hover exit to keep the + // stream consistent. + sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); + } + } + /** + * Handles ACTION_MOVE while in the touch interacting state. This is where transitions to + * delegating and dragging states are handled. + */ + private void handleActionMoveStateTouchInteracting( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + final int pointerId = mReceivedPointerTracker.getPrimaryPointerId(); + final int pointerIndex = event.findPointerIndex(pointerId); + final int pointerIdBits = (1 << pointerId); + switch (event.getPointerCount()) { + case 1: + // We have not started sending events since we try to + // figure out what the user is doing. + if (mSendHoverEnterAndMoveDelayed.isPending()) { + // Cache the event until we discern exploration from gesturing. + mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent); + } + break; + case 2: + if (mGestureDetector.isMultiFingerGesturesEnabled()) { + return; + } + // Make sure we don't have any pending transitions to touch exploration + mSendHoverEnterAndMoveDelayed.cancel(); + mSendHoverExitDelayed.cancel(); + // More than one pointer so the user is not touch exploring + // and now we have to decide whether to delegate or drag. + // Remove move history before send injected non-move events + event = MotionEvent.obtainNoHistory(event); + if (isDraggingGesture(event)) { + // Two pointers moving in the same direction within + // a given distance perform a drag. + mState.startDragging(); + mDraggingPointerId = pointerId; + adjustEventLocationForDrag(event); + event.setEdgeFlags(mReceivedPointerTracker.getLastReceivedDownEdgeFlags()); + mDispatcher.sendMotionEvent( + event, MotionEvent.ACTION_DOWN, rawEvent, pointerIdBits, policyFlags); + } else { + // Two pointers moving arbitrary are delegated to the view hierarchy. + mState.startDelegating(); + mDispatcher.sendDownForAllNotInjectedPointers(event, policyFlags); + } + break; + default: + if (mGestureDetector.isMultiFingerGesturesEnabled()) { + return; + } + // More than two pointers are delegated to the view hierarchy. + mState.startDelegating(); + event = MotionEvent.obtainNoHistory(event); + mDispatcher.sendDownForAllNotInjectedPointers(event, policyFlags); + break; + } + } + + /** + * Handles ACTION_UP while in the touch interacting state. This event represents all fingers + * being lifted from the screen. + */ + private void handleActionUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + mAms.onTouchInteractionEnd(); + final int pointerId = event.getPointerId(event.getActionIndex()); + final int pointerIdBits = (1 << pointerId); + if (mSendHoverEnterAndMoveDelayed.isPending()) { + // If we have not delivered the enter schedule an exit. + mSendHoverExitDelayed.post(event, rawEvent, pointerIdBits, policyFlags); + } else { + // The user is touch exploring so we send events for end. + sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); + } + if (!mSendTouchInteractionEndDelayed.isPending()) { + mSendTouchInteractionEndDelayed.post(); + } + } + + /** + * Handles move events while touch exploring. this is also where we drag or delegate based on + * the number of fingers moving on the screen. + */ + private void handleActionMoveStateTouchExploring( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + final int pointerId = mReceivedPointerTracker.getPrimaryPointerId(); + final int pointerIdBits = (1 << pointerId); + final int pointerIndex = event.findPointerIndex(pointerId); + switch (event.getPointerCount()) { + case 1: + // Touch exploration. + sendTouchExplorationGestureStartAndHoverEnterIfNeeded(policyFlags); + mDispatcher.sendMotionEvent( + event, MotionEvent.ACTION_HOVER_MOVE, rawEvent, pointerIdBits, policyFlags); + break; + case 2: + if (mGestureDetector.isMultiFingerGesturesEnabled()) { + return; + } + if (mSendHoverEnterAndMoveDelayed.isPending()) { + // We have not started sending events so cancel + // scheduled sending events. + mSendHoverEnterAndMoveDelayed.cancel(); + mSendHoverExitDelayed.cancel(); + } + // If the user is touch exploring the second pointer may be + // performing a double tap to activate an item without need + // for the user to lift his exploring finger. + // It is *important* to use the distance traveled by the pointers + // on the screen which may or may not be magnified. + final float deltaX = + mReceivedPointerTracker.getReceivedPointerDownX(pointerId) + - rawEvent.getX(pointerIndex); + final float deltaY = + mReceivedPointerTracker.getReceivedPointerDownY(pointerId) + - rawEvent.getY(pointerIndex); + final double moveDelta = Math.hypot(deltaX, deltaY); + if (moveDelta > mDoubleTapSlop) { + // The user is trying to either delegate or drag. + handleActionMoveStateTouchInteracting(event, rawEvent, policyFlags); + } else { + // Otherwise the double tap will be handled by the gesture detector. + sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); + } + break; + default: + if (mGestureDetector.isMultiFingerGesturesEnabled()) { + return; + } + // Three or more fingers is something other than touch exploration. + if (mSendHoverEnterAndMoveDelayed.isPending()) { + // We have not started sending events so cancel + // scheduled sending events. + mSendHoverEnterAndMoveDelayed.cancel(); + mSendHoverExitDelayed.cancel(); + } else { + sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); + } + handleActionMoveStateTouchInteracting(event, rawEvent, policyFlags); + break; + } + } + + /** + * Handles a motion event in dragging state. + * + * @param event The event to be handled. + * @param policyFlags The policy flags associated with the event. + */ + private void handleMotionEventStateDragging( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mGestureDetector.isMultiFingerGesturesEnabled()) { + // Multi-finger gestures conflict with this functionality. + return; + } + int pointerIdBits = 0; + // Clear the dragging pointer id if it's no longer valid. + if (event.findPointerIndex(mDraggingPointerId) == -1) { + Slog.e(LOG_TAG, "mDraggingPointerId doesn't match any pointers on current event. " + + "mDraggingPointerId: " + Integer.toString(mDraggingPointerId) + + ", Event: " + event); + mDraggingPointerId = INVALID_POINTER_ID; + } else { + pointerIdBits = (1 << mDraggingPointerId); + } + switch (event.getActionMasked()) { + case MotionEvent.ACTION_DOWN: { + Slog.e(LOG_TAG, "Dragging state can be reached only if two " + + "pointers are already down"); + clear(event, policyFlags); + return; + } + case MotionEvent.ACTION_POINTER_DOWN: { + // We are in dragging state so we have two pointers and another one + // goes down => delegate the three pointers to the view hierarchy + mState.startDelegating(); + if (mDraggingPointerId != INVALID_POINTER_ID) { + mDispatcher.sendMotionEvent( + event, MotionEvent.ACTION_UP, rawEvent, pointerIdBits, policyFlags); + } + mDispatcher.sendDownForAllNotInjectedPointers(event, policyFlags); + } break; + case MotionEvent.ACTION_MOVE: { + if (mDraggingPointerId == INVALID_POINTER_ID) { + break; + } + switch (event.getPointerCount()) { + case 1: { + // do nothing + } break; + case 2: { + if (isDraggingGesture(event)) { + // If still dragging send a drag event. + adjustEventLocationForDrag(event); + mDispatcher.sendMotionEvent( + event, + MotionEvent.ACTION_MOVE, + rawEvent, + pointerIdBits, + policyFlags); + } else { + // The two pointers are moving either in different directions or + // no close enough => delegate the gesture to the view hierarchy. + mState.startDelegating(); + // Remove move history before send injected non-move events + event = MotionEvent.obtainNoHistory(event); + // Send an event to the end of the drag gesture. + mDispatcher.sendMotionEvent( + event, + MotionEvent.ACTION_UP, + rawEvent, + pointerIdBits, + policyFlags); + // Deliver all pointers to the view hierarchy. + mDispatcher.sendDownForAllNotInjectedPointers(event, policyFlags); + } + } break; + default: { + mState.startDelegating(); + event = MotionEvent.obtainNoHistory(event); + // Send an event to the end of the drag gesture. + mDispatcher.sendMotionEvent( + event, + MotionEvent.ACTION_UP, + rawEvent, + pointerIdBits, + policyFlags); + // Deliver all pointers to the view hierarchy. + mDispatcher.sendDownForAllNotInjectedPointers(event, policyFlags); + } + } + } break; + case MotionEvent.ACTION_POINTER_UP: { + final int pointerId = event.getPointerId(event.getActionIndex()); + if (pointerId == mDraggingPointerId) { + mDraggingPointerId = INVALID_POINTER_ID; + // Send an event to the end of the drag gesture. + mDispatcher.sendMotionEvent( + event, MotionEvent.ACTION_UP, rawEvent, pointerIdBits, policyFlags); + } + } break; + case MotionEvent.ACTION_UP: { + mAms.onTouchInteractionEnd(); + // Announce the end of a new touch interaction. + mDispatcher.sendAccessibilityEvent( + AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + final int pointerId = event.getPointerId(event.getActionIndex()); + if (pointerId == mDraggingPointerId) { + mDraggingPointerId = INVALID_POINTER_ID; + // Send an event to the end of the drag gesture. + mDispatcher.sendMotionEvent( + event, MotionEvent.ACTION_UP, rawEvent, pointerIdBits, policyFlags); + } + } break; + } + } + + /** + * Handles a motion event in delegating state. + * + * @param event The event to be handled. + * @param policyFlags The policy flags associated with the event. + */ + private void handleMotionEventStateDelegating( + MotionEvent event, MotionEvent rawEvent, int policyFlags) { + switch (event.getActionMasked()) { + case MotionEvent.ACTION_DOWN: { + Slog.e(LOG_TAG, "Delegating state can only be reached if " + + "there is at least one pointer down!"); + clear(event, policyFlags); + return; + } + case MotionEvent.ACTION_UP: { + // Deliver the event. + mDispatcher.sendMotionEvent( + event, event.getAction(), rawEvent, ALL_POINTER_ID_BITS, policyFlags); + + // Announce the end of a the touch interaction. + mAms.onTouchInteractionEnd(); + mDispatcher.clear(); + mDispatcher.sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + + } break; + default: { + // Deliver the event. + mDispatcher.sendMotionEvent( + event, event.getAction(), rawEvent, ALL_POINTER_ID_BITS, policyFlags); + } + } + } + + private void endGestureDetection(boolean interactionEnd) { + mAms.onTouchInteractionEnd(); + + // Announce the end of the gesture recognition. + mDispatcher.sendAccessibilityEvent(AccessibilityEvent.TYPE_GESTURE_DETECTION_END); + // Don't announce the end of a the touch interaction if users didn't lift their fingers. + if (interactionEnd) { + mDispatcher.sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + } + + mExitGestureDetectionModeDelayed.cancel(); + } + + + /** + * Sends the exit events if needed. Such events are hover exit and touch explore + * gesture end. + * + * @param policyFlags The policy flags associated with the event. + */ + private void sendHoverExitAndTouchExplorationGestureEndIfNeeded(int policyFlags) { + MotionEvent event = mState.getLastInjectedHoverEvent(); + if (event != null && event.getActionMasked() != MotionEvent.ACTION_HOVER_EXIT) { + final int pointerIdBits = event.getPointerIdBits(); + if (!mSendTouchExplorationEndDelayed.isPending()) { + mSendTouchExplorationEndDelayed.post(); + } + mDispatcher.sendMotionEvent( + event, + MotionEvent.ACTION_HOVER_EXIT, + mState.getLastReceivedEvent(), + pointerIdBits, + policyFlags); + } + } + + /** + * Sends the enter events if needed. Such events are hover enter and touch explore + * gesture start. + * + * @param policyFlags The policy flags associated with the event. + */ + private void sendTouchExplorationGestureStartAndHoverEnterIfNeeded(int policyFlags) { + MotionEvent event = mState.getLastInjectedHoverEvent(); + if (event != null && event.getActionMasked() == MotionEvent.ACTION_HOVER_EXIT) { + final int pointerIdBits = event.getPointerIdBits(); + mDispatcher.sendMotionEvent( + event, + MotionEvent.ACTION_HOVER_ENTER, + mState.getLastReceivedEvent(), + pointerIdBits, + policyFlags); + } + } + + + /** + * Determines whether a two pointer gesture is a dragging one. + * + * @param event The event with the pointer data. + * @return True if the gesture is a dragging one. + */ + private boolean isDraggingGesture(MotionEvent event) { + + final float firstPtrX = event.getX(0); + final float firstPtrY = event.getY(0); + final float secondPtrX = event.getX(1); + final float secondPtrY = event.getY(1); + + final float firstPtrDownX = mReceivedPointerTracker.getReceivedPointerDownX(0); + final float firstPtrDownY = mReceivedPointerTracker.getReceivedPointerDownY(0); + final float secondPtrDownX = mReceivedPointerTracker.getReceivedPointerDownX(1); + final float secondPtrDownY = mReceivedPointerTracker.getReceivedPointerDownY(1); + + return GestureUtils.isDraggingGesture(firstPtrDownX, firstPtrDownY, secondPtrDownX, + secondPtrDownY, firstPtrX, firstPtrY, secondPtrX, secondPtrY, + MAX_DRAGGING_ANGLE_COS); + } + + /** + * Adjust the location of an injected event when performing a drag The new location will be in + * between the two fingers touching the screen. + */ + private void adjustEventLocationForDrag(MotionEvent event) { + + final float firstPtrX = event.getX(0); + final float firstPtrY = event.getY(0); + final float secondPtrX = event.getX(1); + final float secondPtrY = event.getY(1); + final int pointerIndex = event.findPointerIndex(mDraggingPointerId); + final float deltaX = + (pointerIndex == 0) ? (secondPtrX - firstPtrX) : (firstPtrX - secondPtrX); + final float deltaY = + (pointerIndex == 0) ? (secondPtrY - firstPtrY) : (firstPtrY - secondPtrY); + event.offsetLocation(deltaX / 2, deltaY / 2); + } + + public TouchState getState() { + return mState; + } + + @Override + public void setNext(EventStreamTransformation next) { + mDispatcher.setReceiver(next); + super.setNext(next); + } + + /** + * Whether to dispatch double tap and double tap and hold to the service rather than handle them + * in the framework. + */ + public void setServiceHandlesDoubleTap(boolean mode) { + mGestureDetector.setServiceHandlesDoubleTap(mode); + } + + /** + * This function turns on and off multi-finger gestures. When enabled, multi-finger gestures + * will disable delegating and dragging functionality. + */ + public void setMultiFingerGesturesEnabled(boolean enabled) { + mGestureDetector.setMultiFingerGesturesEnabled(enabled); + } + + public void setGestureDetectionPassthroughRegion(Region region) { + mGestureDetectionPassthroughRegion = region; + } + + public void setTouchExplorationPassthroughRegion(Region region) { + mTouchExplorationPassthroughRegion = region; + } + + private boolean shouldPerformGestureDetection(MotionEvent event) { + if (mState.isDelegating()) { + return false; + } + if (event.getActionMasked() == MotionEvent.ACTION_DOWN) { + final int x = (int) event.getX(); + final int y = (int) event.getY(); + if (mTouchExplorationPassthroughRegion.contains(x, y) + || mGestureDetectionPassthroughRegion.contains(x, y)) { + return false; + } + } + return true; + } + + /** + * Class for delayed exiting from gesture detecting mode. + */ + private final class ExitGestureDetectionModeDelayed implements Runnable { + + public void post() { + mHandler.postDelayed(this, EXIT_GESTURE_DETECTION_TIMEOUT); + } + + public void cancel() { + mHandler.removeCallbacks(this); + } + + @Override + public void run() { + // Announce the end of gesture recognition. + mDispatcher.sendAccessibilityEvent(AccessibilityEvent.TYPE_GESTURE_DETECTION_END); + clear(); + } + } + + /** + * Class for delayed sending of hover enter and move events. + */ + class SendHoverEnterAndMoveDelayed implements Runnable { + private final String LOG_TAG_SEND_HOVER_DELAYED = "SendHoverEnterAndMoveDelayed"; + + private final List<MotionEvent> mEvents = new ArrayList<MotionEvent>(); + private final List<MotionEvent> mRawEvents = new ArrayList<MotionEvent>(); + + private int mPointerIdBits; + private int mPolicyFlags; + + public void post( + MotionEvent event, MotionEvent rawEvent, int pointerIdBits, int policyFlags) { + cancel(); + addEvent(event, rawEvent); + mPointerIdBits = pointerIdBits; + mPolicyFlags = policyFlags; + mHandler.postDelayed(this, mDetermineUserIntentTimeout); + } + + public void addEvent(MotionEvent event, MotionEvent rawEvent) { + mEvents.add(MotionEvent.obtain(event)); + mRawEvents.add(MotionEvent.obtain(rawEvent)); + } + + public void cancel() { + if (isPending()) { + mHandler.removeCallbacks(this); + clear(); + } + } + + private boolean isPending() { + return mHandler.hasCallbacks(this); + } + + private void clear() { + mPointerIdBits = -1; + mPolicyFlags = 0; + final int eventCount = mEvents.size(); + for (int i = eventCount - 1; i >= 0; i--) { + mEvents.remove(i).recycle(); + } + final int rawEventcount = mRawEvents.size(); + for (int i = rawEventcount - 1; i >= 0; i--) { + mRawEvents.remove(i).recycle(); + } + } + + public void forceSendAndRemove() { + if (isPending()) { + run(); + cancel(); + } + } + + public void run() { + // Send an accessibility event to announce the touch exploration start. + mDispatcher.sendAccessibilityEvent( + AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START); + + if (!mEvents.isEmpty() && !mRawEvents.isEmpty()) { + // Deliver a down event. + mDispatcher.sendMotionEvent(mEvents.get(0), MotionEvent.ACTION_HOVER_ENTER, + mRawEvents.get(0), mPointerIdBits, mPolicyFlags); + if (DEBUG) { + Slog.d(LOG_TAG_SEND_HOVER_DELAYED, + "Injecting motion event: ACTION_HOVER_ENTER"); + } + + // Deliver move events. + final int eventCount = mEvents.size(); + for (int i = 1; i < eventCount; i++) { + mDispatcher.sendMotionEvent(mEvents.get(i), MotionEvent.ACTION_HOVER_MOVE, + mRawEvents.get(i), mPointerIdBits, mPolicyFlags); + if (DEBUG) { + Slog.d(LOG_TAG_SEND_HOVER_DELAYED, + "Injecting motion event: ACTION_HOVER_MOVE"); + } + } + } + clear(); + } + } + + /** + * Class for delayed sending of hover exit events. + */ + class SendHoverExitDelayed implements Runnable { + private final String LOG_TAG_SEND_HOVER_DELAYED = "SendHoverExitDelayed"; + + private MotionEvent mPrototype; + private MotionEvent mRawEvent; + private int mPointerIdBits; + private int mPolicyFlags; + + public void post( + MotionEvent prototype, MotionEvent rawEvent, int pointerIdBits, int policyFlags) { + cancel(); + mPrototype = MotionEvent.obtain(prototype); + mRawEvent = MotionEvent.obtain(rawEvent); + mPointerIdBits = pointerIdBits; + mPolicyFlags = policyFlags; + mHandler.postDelayed(this, mDetermineUserIntentTimeout); + } + + public void cancel() { + if (isPending()) { + mHandler.removeCallbacks(this); + clear(); + } + } + + private boolean isPending() { + return mHandler.hasCallbacks(this); + } + + private void clear() { + if (mPrototype != null) { + mPrototype.recycle(); + } + if (mRawEvent != null) { + mRawEvent.recycle(); + } + mPrototype = null; + mRawEvent = null; + mPointerIdBits = -1; + mPolicyFlags = 0; + } + + public void forceSendAndRemove() { + if (isPending()) { + run(); + cancel(); + } + } + + public void run() { + if (DEBUG) { + Slog.d(LOG_TAG_SEND_HOVER_DELAYED, "Injecting motion event:" + + " ACTION_HOVER_EXIT"); + } + mDispatcher.sendMotionEvent( + mPrototype, + MotionEvent.ACTION_HOVER_EXIT, + mRawEvent, + mPointerIdBits, + mPolicyFlags); + if (!mSendTouchExplorationEndDelayed.isPending()) { + mSendTouchExplorationEndDelayed.cancel(); + mSendTouchExplorationEndDelayed.post(); + } + if (mSendTouchInteractionEndDelayed.isPending()) { + mSendTouchInteractionEndDelayed.cancel(); + mSendTouchInteractionEndDelayed.post(); + } + clear(); + } + } + + private class SendAccessibilityEventDelayed implements Runnable { + private final int mEventType; + private final int mDelay; + + public SendAccessibilityEventDelayed(int eventType, int delay) { + mEventType = eventType; + mDelay = delay; + } + + public void cancel() { + mHandler.removeCallbacks(this); + } + + public void post() { + mHandler.postDelayed(this, mDelay); + } + + public boolean isPending() { + return mHandler.hasCallbacks(this); + } + + public void forceSendAndRemove() { + if (isPending()) { + run(); + cancel(); + } + } + + @Override + public void run() { + mDispatcher.sendAccessibilityEvent(mEventType); + } + } + + @Override + public String toString() { + return "TouchExplorer { " + + "mTouchState: " + mState + + ", mDetermineUserIntentTimeout: " + mDetermineUserIntentTimeout + + ", mDoubleTapSlop: " + mDoubleTapSlop + + ", mDraggingPointerId: " + mDraggingPointerId + + " }"; + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/TouchState.java b/services/accessibility/java/com/android/server/accessibility/gestures/TouchState.java new file mode 100644 index 000000000000..7a39bc29e8e5 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/gestures/TouchState.java @@ -0,0 +1,560 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.gestures; + +import static android.view.MotionEvent.INVALID_POINTER_ID; + +import static com.android.server.accessibility.gestures.TouchExplorer.DEBUG; + +import android.annotation.IntDef; +import android.util.Slog; +import android.view.MotionEvent; +import android.view.accessibility.AccessibilityEvent; + +/** + * This class describes the state of the touch explorer as well as the state of received and + * injected pointers. This data is accessed both for purposes of touch exploration and gesture + * dispatch. + */ +public class TouchState { + private static final String LOG_TAG = "TouchState"; + // Pointer-related constants + // This constant captures the current implementation detail that + // pointer IDs are between 0 and 31 inclusive (subject to change). + // (See MAX_POINTER_ID in frameworks/base/include/ui/Input.h) + static final int MAX_POINTER_COUNT = 32; + // Constant referring to the ids bits of all pointers. + public static final int ALL_POINTER_ID_BITS = 0xFFFFFFFF; + + // States that the touch explorer can be in. + // In the clear state the user is not touching the screen. + public static final int STATE_CLEAR = 0; + // The user is touching the screen and we are trying to figure out their intent. + // This state gets its name from the TYPE_TOUCH_INTERACTION start and end accessibility events. + public static final int STATE_TOUCH_INTERACTING = 1; + // The user is explicitly exploring the screen. + public static final int STATE_TOUCH_EXPLORING = 2; + // the user is dragging with two fingers. + public static final int STATE_DRAGGING = 3; + // The user is performing some other two finger gesture which we pass through to the view + // hierarchy as a one-finger gesture e.g. two-finger scrolling. + public static final int STATE_DELEGATING = 4; + // The user is performing something that might be a gesture. + public static final int STATE_GESTURE_DETECTING = 5; + + @IntDef({ + STATE_CLEAR, + STATE_TOUCH_INTERACTING, + STATE_TOUCH_EXPLORING, + STATE_DRAGGING, + STATE_DELEGATING, + STATE_GESTURE_DETECTING + }) + public @interface State {} + + // The current state of the touch explorer. + private int mState = STATE_CLEAR; + // Helper class to track received pointers. + // Todo: collapse or hide this class so multiple classes don't modify it. + private final ReceivedPointerTracker mReceivedPointerTracker; + // The most recently received motion event. + private MotionEvent mLastReceivedEvent; + // The accompanying raw event without any transformations. + private MotionEvent mLastReceivedRawEvent; + // The id of the last touch explored window. + private int mLastTouchedWindowId; + // The last injected hover event. + private MotionEvent mLastInjectedHoverEvent; + // The last injected hover event used for performing clicks. + private MotionEvent mLastInjectedHoverEventForClick; + // The time of the last injected down. + private long mLastInjectedDownEventTime; + // Keep track of which pointers sent to the system are down. + private int mInjectedPointersDown; + + public TouchState() { + mReceivedPointerTracker = new ReceivedPointerTracker(); + } + + /** Clears the internal shared state. */ + public void clear() { + setState(STATE_CLEAR); + // Reset the pointer trackers. + if (mLastReceivedEvent != null) { + mLastReceivedEvent.recycle(); + mLastReceivedEvent = null; + } + mLastTouchedWindowId = -1; + mReceivedPointerTracker.clear(); + mInjectedPointersDown = 0; + } + + /** + * Updates the state in response to a touch event received by TouchExplorer. + * + * @param rawEvent The raw touch event. + */ + public void onReceivedMotionEvent(MotionEvent rawEvent) { + if (mLastReceivedEvent != null) { + mLastReceivedEvent.recycle(); + } + if (mLastReceivedRawEvent != null) { + mLastReceivedRawEvent.recycle(); + } + mLastReceivedEvent = MotionEvent.obtain(rawEvent); + mReceivedPointerTracker.onMotionEvent(rawEvent); + } + + /** + * Processes an injected {@link MotionEvent} event. + * + * @param event The event to process. + */ + void onInjectedMotionEvent(MotionEvent event) { + final int action = event.getActionMasked(); + final int pointerId = event.getPointerId(event.getActionIndex()); + final int pointerFlag = (1 << pointerId); + switch (action) { + case MotionEvent.ACTION_DOWN: + case MotionEvent.ACTION_POINTER_DOWN: + mInjectedPointersDown |= pointerFlag; + mLastInjectedDownEventTime = event.getDownTime(); + break; + case MotionEvent.ACTION_UP: + case MotionEvent.ACTION_POINTER_UP: + mInjectedPointersDown &= ~pointerFlag; + if (mInjectedPointersDown == 0) { + mLastInjectedDownEventTime = 0; + } + break; + case MotionEvent.ACTION_HOVER_ENTER: + case MotionEvent.ACTION_HOVER_MOVE: + if (mLastInjectedHoverEvent != null) { + mLastInjectedHoverEvent.recycle(); + } + mLastInjectedHoverEvent = MotionEvent.obtain(event); + break; + case MotionEvent.ACTION_HOVER_EXIT: + if (mLastInjectedHoverEvent != null) { + mLastInjectedHoverEvent.recycle(); + } + mLastInjectedHoverEvent = MotionEvent.obtain(event); + if (mLastInjectedHoverEventForClick != null) { + mLastInjectedHoverEventForClick.recycle(); + } + mLastInjectedHoverEventForClick = MotionEvent.obtain(event); + break; + } + if (DEBUG) { + Slog.i(LOG_TAG, "Injected pointer:\n" + toString()); + } + } + + /** Updates state in response to an accessibility event received from the outside. */ + public void onReceivedAccessibilityEvent(AccessibilityEvent event) { + // If a new window opens or the accessibility focus moves we no longer + // want to click/long press on the last touch explored location. + switch (event.getEventType()) { + case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: + case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED: + if (mLastInjectedHoverEventForClick != null) { + mLastInjectedHoverEventForClick.recycle(); + mLastInjectedHoverEventForClick = null; + } + mLastTouchedWindowId = -1; + break; + case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER: + case AccessibilityEvent.TYPE_VIEW_HOVER_EXIT: + mLastTouchedWindowId = event.getWindowId(); + break; + } + } + + public void onInjectedAccessibilityEvent(int type) { + // The below state transitions go here because the related events are often sent on a + // delay. + // This allows state to accurately reflect the state in the moment. + // TODO: replaced the delayed event senders with delayed state transitions + // so that state transitions trigger events rather than events triggering state + // transitions. + switch (type) { + case AccessibilityEvent.TYPE_TOUCH_INTERACTION_START: + startTouchInteracting(); + break; + case AccessibilityEvent.TYPE_TOUCH_INTERACTION_END: + clear(); + break; + case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START: + startTouchExploring(); + break; + case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END: + startTouchInteracting(); + break; + case AccessibilityEvent.TYPE_GESTURE_DETECTION_START: + startGestureDetecting(); + break; + case AccessibilityEvent.TYPE_GESTURE_DETECTION_END: + startTouchInteracting(); + break; + default: + break; + } + } + + @State + public int getState() { + return mState; + } + + /** Transitions to a new state. */ + public void setState(@State int state) { + if (mState == state) return; + if (DEBUG) { + Slog.i(LOG_TAG, getStateSymbolicName(mState) + "->" + getStateSymbolicName(state)); + } + mState = state; + } + + public boolean isTouchExploring() { + return mState == STATE_TOUCH_EXPLORING; + } + + /** Starts touch exploration. */ + public void startTouchExploring() { + setState(STATE_TOUCH_EXPLORING); + } + + public boolean isDelegating() { + return mState == STATE_DELEGATING; + } + + /** Starts delegating gestures to the view hierarchy. */ + public void startDelegating() { + setState(STATE_DELEGATING); + } + + public boolean isGestureDetecting() { + return mState == STATE_GESTURE_DETECTING; + } + + /** Initiates gesture detection. */ + public void startGestureDetecting() { + setState(STATE_GESTURE_DETECTING); + } + + public boolean isDragging() { + return mState == STATE_DRAGGING; + } + + /** Starts a dragging gesture. */ + public void startDragging() { + setState(STATE_DRAGGING); + } + + public boolean isTouchInteracting() { + return mState == STATE_TOUCH_INTERACTING; + } + + /** + * Transitions to the touch interacting state, where we attempt to figure out what the user is + * doing. + */ + public void startTouchInteracting() { + setState(STATE_TOUCH_INTERACTING); + } + + public boolean isClear() { + return mState == STATE_CLEAR; + } + /** Returns a string representation of the current state. */ + public String toString() { + return "TouchState { " + "mState: " + getStateSymbolicName(mState) + " }"; + } + /** Returns a string representation of the specified state. */ + public static String getStateSymbolicName(int state) { + switch (state) { + case STATE_CLEAR: + return "STATE_CLEAR"; + case STATE_TOUCH_INTERACTING: + return "STATE_TOUCH_INTERACTING"; + case STATE_TOUCH_EXPLORING: + return "STATE_TOUCH_EXPLORING"; + case STATE_DRAGGING: + return "STATE_DRAGGING"; + case STATE_DELEGATING: + return "STATE_DELEGATING"; + case STATE_GESTURE_DETECTING: + return "STATE_GESTURE_DETECTING"; + default: + return "Unknown state: " + state; + } + } + + public ReceivedPointerTracker getReceivedPointerTracker() { + return mReceivedPointerTracker; + } + + /** @return The last received event. */ + public MotionEvent getLastReceivedEvent() { + return mLastReceivedEvent; + } + + /** @return The the last injected hover event. */ + public MotionEvent getLastInjectedHoverEvent() { + return mLastInjectedHoverEvent; + } + + /** @return The time of the last injected down event. */ + public long getLastInjectedDownEventTime() { + return mLastInjectedDownEventTime; + } + + public int getLastTouchedWindowId() { + return mLastTouchedWindowId; + } + + /** @return The number of down pointers injected to the view hierarchy. */ + public int getInjectedPointerDownCount() { + return Integer.bitCount(mInjectedPointersDown); + } + + /** @return The bits of the injected pointers that are down. */ + public int getInjectedPointersDown() { + return mInjectedPointersDown; + } + + /** + * Whether an injected pointer is down. + * + * @param pointerId The unique pointer id. + * @return True if the pointer is down. + */ + public boolean isInjectedPointerDown(int pointerId) { + final int pointerFlag = (1 << pointerId); + return (mInjectedPointersDown & pointerFlag) != 0; + } + + /** @return The the last injected hover event used for a click. */ + public MotionEvent getLastInjectedHoverEventForClick() { + return mLastInjectedHoverEventForClick; + } + + /** This class tracks where and when a pointer went down. It does not track its movement. */ + class ReceivedPointerTracker { + private static final String LOG_TAG_RECEIVED_POINTER_TRACKER = "ReceivedPointerTracker"; + + private final PointerDownInfo[] mReceivedPointers = new PointerDownInfo[MAX_POINTER_COUNT]; + + // Which pointers are down. + private int mReceivedPointersDown; + + // The edge flags of the last received down event. + private int mLastReceivedDownEdgeFlags; + + // Primary pointer which is either the first that went down + // or if it goes up the next one that most recently went down. + private int mPrimaryPointerId; + + ReceivedPointerTracker() { + clear(); + } + + /** Clears the internals state. */ + public void clear() { + mReceivedPointersDown = 0; + mPrimaryPointerId = 0; + for (int i = 0; i < MAX_POINTER_COUNT; ++i) { + mReceivedPointers[i] = new PointerDownInfo(); + } + } + + /** + * Processes a received {@link MotionEvent} event. + * + * @param event The event to process. + */ + public void onMotionEvent(MotionEvent event) { + final int action = event.getActionMasked(); + switch (action) { + case MotionEvent.ACTION_DOWN: + handleReceivedPointerDown(event.getActionIndex(), event); + break; + case MotionEvent.ACTION_POINTER_DOWN: + handleReceivedPointerDown(event.getActionIndex(), event); + break; + case MotionEvent.ACTION_UP: + handleReceivedPointerUp(event.getActionIndex(), event); + break; + case MotionEvent.ACTION_POINTER_UP: + handleReceivedPointerUp(event.getActionIndex(), event); + break; + } + if (DEBUG) { + Slog.i(LOG_TAG_RECEIVED_POINTER_TRACKER, "Received pointer:\n" + toString()); + } + } + + /** @return The number of received pointers that are down. */ + public int getReceivedPointerDownCount() { + return Integer.bitCount(mReceivedPointersDown); + } + + /** + * Whether an received pointer is down. + * + * @param pointerId The unique pointer id. + * @return True if the pointer is down. + */ + public boolean isReceivedPointerDown(int pointerId) { + final int pointerFlag = (1 << pointerId); + return (mReceivedPointersDown & pointerFlag) != 0; + } + + /** + * @param pointerId The unique pointer id. + * @return The X coordinate where the pointer went down. + */ + public float getReceivedPointerDownX(int pointerId) { + return mReceivedPointers[pointerId].mX; + } + + /** + * @param pointerId The unique pointer id. + * @return The Y coordinate where the pointer went down. + */ + public float getReceivedPointerDownY(int pointerId) { + return mReceivedPointers[pointerId].mY; + } + + /** + * @param pointerId The unique pointer id. + * @return The time when the pointer went down. + */ + public long getReceivedPointerDownTime(int pointerId) { + return mReceivedPointers[pointerId].mTime; + } + + /** @return The id of the primary pointer. */ + public int getPrimaryPointerId() { + if (mPrimaryPointerId == INVALID_POINTER_ID) { + mPrimaryPointerId = findPrimaryPointerId(); + } + return mPrimaryPointerId; + } + + /** @return The edge flags of the last received down event. */ + public int getLastReceivedDownEdgeFlags() { + return mLastReceivedDownEdgeFlags; + } + + /** + * Handles a received pointer down event. + * + * @param pointerIndex The index of the pointer that has changed. + * @param event The event to be handled. + */ + private void handleReceivedPointerDown(int pointerIndex, MotionEvent event) { + final int pointerId = event.getPointerId(pointerIndex); + final int pointerFlag = (1 << pointerId); + mLastReceivedDownEdgeFlags = event.getEdgeFlags(); + + mReceivedPointersDown |= pointerFlag; + mReceivedPointers[pointerId].set( + event.getX(pointerIndex), event.getY(pointerIndex), event.getEventTime()); + + mPrimaryPointerId = pointerId; + } + + /** + * Handles a received pointer up event. + * + * @param pointerIndex The index of the pointer that has changed. + * @param event The event to be handled. + */ + private void handleReceivedPointerUp(int pointerIndex, MotionEvent event) { + final int pointerId = event.getPointerId(pointerIndex); + final int pointerFlag = (1 << pointerId); + mReceivedPointersDown &= ~pointerFlag; + mReceivedPointers[pointerId].clear(); + if (mPrimaryPointerId == pointerId) { + mPrimaryPointerId = INVALID_POINTER_ID; + } + } + + /** @return The primary pointer id. */ + private int findPrimaryPointerId() { + int primaryPointerId = INVALID_POINTER_ID; + long minDownTime = Long.MAX_VALUE; + + // Find the pointer that went down first. + int pointerIdBits = mReceivedPointersDown; + while (pointerIdBits > 0) { + final int pointerId = Integer.numberOfTrailingZeros(pointerIdBits); + pointerIdBits &= ~(1 << pointerId); + final long downPointerTime = mReceivedPointers[pointerId].mTime; + if (downPointerTime < minDownTime) { + minDownTime = downPointerTime; + primaryPointerId = pointerId; + } + } + return primaryPointerId; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("========================="); + builder.append("\nDown pointers #"); + builder.append(getReceivedPointerDownCount()); + builder.append(" [ "); + for (int i = 0; i < MAX_POINTER_COUNT; i++) { + if (isReceivedPointerDown(i)) { + builder.append(i); + builder.append(" "); + } + } + builder.append("]"); + builder.append("\nPrimary pointer id [ "); + builder.append(getPrimaryPointerId()); + builder.append(" ]"); + builder.append("\n========================="); + return builder.toString(); + } + } + + /** + * This class tracks where and when an individual pointer went down. Note that it does not track + * when it went up. + */ + class PointerDownInfo { + private float mX; + private float mY; + private long mTime; + + public void set(float x, float y, long time) { + mX = x; + mY = y; + mTime = time; + } + + public void clear() { + mX = 0; + mY = 0; + mTime = 0; + } + } +} diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationGestureHandler.java b/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationGestureHandler.java new file mode 100644 index 000000000000..aa500b5b627f --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationGestureHandler.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.magnification; + +import com.android.server.accessibility.BaseEventStreamTransformation; + +/** + * A base class that detects gestures and defines common methods for magnification. + */ +public abstract class MagnificationGestureHandler extends BaseEventStreamTransformation { + + /** + * Called when the shortcut target is magnification. + */ + public abstract void notifyShortcutTriggered(); +} diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationConnectionWrapper.java b/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationConnectionWrapper.java new file mode 100644 index 000000000000..351c9e08b645 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationConnectionWrapper.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.magnification; + +import static android.os.IBinder.DeathRecipient; + +import android.annotation.NonNull; +import android.os.RemoteException; +import android.util.Slog; +import android.view.accessibility.IWindowMagnificationConnection; +import android.view.accessibility.IWindowMagnificationConnectionCallback; + +/** + * A wrapper of {@link IWindowMagnificationConnection}. + */ +class WindowMagnificationConnectionWrapper { + + private static final boolean DBG = false; + private static final String TAG = "WindowMagnificationConnectionWrapper"; + + private final @NonNull IWindowMagnificationConnection mConnection; + + WindowMagnificationConnectionWrapper(@NonNull IWindowMagnificationConnection connection) { + mConnection = connection; + } + + //Should not use this instance anymore after calling it. + void unlinkToDeath(@NonNull DeathRecipient deathRecipient) { + mConnection.asBinder().unlinkToDeath(deathRecipient, 0); + } + + void linkToDeath(@NonNull DeathRecipient deathRecipient) throws RemoteException { + mConnection.asBinder().linkToDeath(deathRecipient, 0); + } + + boolean enableWindowMagnification(int displayId, float scale, float centerX, float centerY) { + try { + mConnection.enableWindowMagnification(displayId, scale, centerX, centerY); + } catch (RemoteException e) { + if (DBG) { + Slog.e(TAG, "Error calling enableWindowMagnification()"); + } + return false; + } + return true; + } + + boolean setScale(int displayId, float scale) { + try { + mConnection.setScale(displayId, scale); + } catch (RemoteException e) { + if (DBG) { + Slog.e(TAG, "Error calling setScale()"); + } + return false; + } + return true; + } + + boolean disableWindowMagnification(int displayId) { + try { + mConnection.disableWindowMagnification(displayId); + } catch (RemoteException e) { + if (DBG) { + Slog.e(TAG, "Error calling disableWindowMagnification()"); + } + return false; + } + return true; + } + + boolean moveWindowMagnifier(int displayId, float offsetX, float offsetY) { + try { + mConnection.moveWindowMagnifier(displayId, offsetX, offsetY); + } catch (RemoteException e) { + if (DBG) { + Slog.e(TAG, "Error calling moveWindowMagnifier()"); + } + return false; + } + return true; + } + + boolean setConnectionCallback(IWindowMagnificationConnectionCallback connectionCallback) { + try { + mConnection.setConnectionCallback(connectionCallback); + } catch (RemoteException e) { + if (DBG) { + Slog.e(TAG, "Error calling setConnectionCallback()"); + } + return false; + } + return true; + } + +} diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationManager.java b/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationManager.java new file mode 100644 index 000000000000..00db3294c9e6 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationManager.java @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility.magnification; + +import android.annotation.Nullable; +import android.graphics.Rect; +import android.os.IBinder; +import android.os.RemoteException; +import android.util.Slog; +import android.view.accessibility.IWindowMagnificationConnection; +import android.view.accessibility.IWindowMagnificationConnectionCallback; + +import com.android.internal.annotations.VisibleForTesting; + +/** + * A class to manipulate window magnification through {@link WindowMagnificationConnectionWrapper}. + */ +public final class WindowMagnificationManager { + + private static final String TAG = "WindowMagnificationMgr"; + private final Object mLock = new Object(); + @VisibleForTesting + @Nullable WindowMagnificationConnectionWrapper mConnectionWrapper; + private ConnectionCallback mConnectionCallback; + + /** + * Sets {@link IWindowMagnificationConnection}. + * @param connection {@link IWindowMagnificationConnection} + */ + public void setConnection(@Nullable IWindowMagnificationConnection connection) { + synchronized (mLock) { + //Reset connectionWrapper. + if (mConnectionWrapper != null) { + mConnectionWrapper.setConnectionCallback(null); + if (mConnectionCallback != null) { + mConnectionCallback.mExpiredDeathRecipient = true; + } + mConnectionWrapper.unlinkToDeath(mConnectionCallback); + mConnectionWrapper = null; + } + if (connection != null) { + mConnectionWrapper = new WindowMagnificationConnectionWrapper(connection); + } + + if (mConnectionWrapper != null) { + try { + mConnectionCallback = new ConnectionCallback(); + mConnectionWrapper.linkToDeath(mConnectionCallback); + mConnectionWrapper.setConnectionCallback(mConnectionCallback); + } catch (RemoteException e) { + Slog.e(TAG, "setConnection failed", e); + mConnectionWrapper = null; + } + } + } + } + + private class ConnectionCallback extends IWindowMagnificationConnectionCallback.Stub implements + IBinder.DeathRecipient { + private boolean mExpiredDeathRecipient = false; + + @Override + public void onWindowMagnifierBoundsChanged(int display, Rect frame) throws RemoteException { + } + + @Override + public void onChangeMagnificationMode(int display, int magnificationMode) + throws RemoteException { + } + + @Override + public void binderDied() { + synchronized (mLock) { + if (mExpiredDeathRecipient) { + Slog.w(TAG, "binderDied DeathRecipient is expired"); + return; + } + mConnectionWrapper.unlinkToDeath(this); + mConnectionWrapper = null; + mConnectionCallback = null; + } + } + } +} |