diff options
author | Xin Li <delphij@google.com> | 2020-08-31 21:21:38 -0700 |
---|---|---|
committer | Xin Li <delphij@google.com> | 2020-08-31 21:21:38 -0700 |
commit | 628590d7ec80e10a3fc24b1c18a1afb55cca10a8 (patch) | |
tree | 4b1c3f52d86d7fb53afbe9e9438468588fa489f8 /services/autofill | |
parent | b11b8ec3aec8bb42f2c07e1c5ac7942da293baa8 (diff) | |
parent | d2d3a20624d968199353ccf6ddbae6f3ac39c9af (diff) |
Merge Android R (rvc-dev-plus-aosp-without-vendor@6692709)
Bug: 166295507
Merged-In: I3d92a6de21a938f6b352ec26dc23420c0fe02b27
Change-Id: Ifdb80563ef042738778ebb8a7581a97c4e3d96e2
Diffstat (limited to 'services/autofill')
20 files changed, 3805 insertions, 751 deletions
diff --git a/services/autofill/Android.bp b/services/autofill/Android.bp index 539eb1a5220e..1e65e8459edf 100644 --- a/services/autofill/Android.bp +++ b/services/autofill/Android.bp @@ -7,6 +7,7 @@ filegroup { java_library_static { name: "services.autofill", + defaults: ["services_defaults"], srcs: [":services.autofill-sources"], libs: ["services.core"], } diff --git a/services/autofill/java/com/android/server/autofill/AutofillInlineSessionController.java b/services/autofill/java/com/android/server/autofill/AutofillInlineSessionController.java new file mode 100644 index 000000000000..c25dd37bc7d9 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/AutofillInlineSessionController.java @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.ComponentName; +import android.os.Bundle; +import android.os.Handler; +import android.view.autofill.AutofillId; +import android.view.inputmethod.InlineSuggestionsRequest; + +import com.android.internal.annotations.GuardedBy; +import com.android.server.autofill.ui.InlineFillUi; +import com.android.server.inputmethod.InputMethodManagerInternal; + +import java.util.Optional; +import java.util.function.Consumer; + +/** + * Controls the interaction with the IME for the {@link AutofillInlineSuggestionsRequestSession}s. + * + * <p>The class maintains the inline suggestion session with the autofill service. There is at most + * one active inline suggestion session at any given corresponding to one focused view. + * New sessions are created only when {@link #onCreateInlineSuggestionsRequestLocked} is called.</p> + * + * <p>The class manages the interaction between the {@link com.android.server.autofill.Session} and + * the inline suggestion session whenever inline suggestions can be provided. All calls to the + * inline suggestion session must be made through this controller.</p> + */ +final class AutofillInlineSessionController { + @NonNull + private final InputMethodManagerInternal mInputMethodManagerInternal; + private final int mUserId; + @NonNull + private final ComponentName mComponentName; + @NonNull + private final Object mLock; + @NonNull + private final Handler mHandler; + @NonNull + private final InlineFillUi.InlineUiEventCallback mUiCallback; + + @Nullable + @GuardedBy("mLock") + private AutofillInlineSuggestionsRequestSession mSession; + @Nullable + @GuardedBy("mLock") + private InlineFillUi mInlineFillUi; + + AutofillInlineSessionController(InputMethodManagerInternal inputMethodManagerInternal, + int userId, ComponentName componentName, Handler handler, Object lock, + InlineFillUi.InlineUiEventCallback callback) { + mInputMethodManagerInternal = inputMethodManagerInternal; + mUserId = userId; + mComponentName = componentName; + mHandler = handler; + mLock = lock; + mUiCallback = callback; + } + + /** + * Requests the IME to create an {@link InlineSuggestionsRequest} for {@code autofillId}. + * + * @param autofillId the Id of the field for which the request is for. + * @param requestConsumer the callback to be invoked when the IME responds. Note that this is + * never invoked if the IME doesn't respond. + */ + @GuardedBy("mLock") + void onCreateInlineSuggestionsRequestLocked(@NonNull AutofillId autofillId, + @NonNull Consumer<InlineSuggestionsRequest> requestConsumer, @NonNull Bundle uiExtras) { + // TODO(b/151123764): rename the method to better reflect what it does. + if (mSession != null) { + // Destroy the existing session. + mSession.destroySessionLocked(); + } + mInlineFillUi = null; + // TODO(b/151123764): consider reusing the same AutofillInlineSession object for the + // same field. + mSession = new AutofillInlineSuggestionsRequestSession(mInputMethodManagerInternal, mUserId, + mComponentName, mHandler, mLock, autofillId, requestConsumer, uiExtras, + mUiCallback); + mSession.onCreateInlineSuggestionsRequestLocked(); + } + + /** + * Destroys the current session. May send an empty response to IME to clear the suggestions if + * the focus didn't change to a different field. + * + * @param autofillId the currently focused view from the autofill session + */ + @GuardedBy("mLock") + void destroyLocked(@NonNull AutofillId autofillId) { + if (mSession != null) { + mSession.onInlineSuggestionsResponseLocked(InlineFillUi.emptyUi(autofillId)); + mSession.destroySessionLocked(); + mSession = null; + } + mInlineFillUi = null; + } + + /** + * Returns the {@link InlineSuggestionsRequest} provided by IME for the last request. + * + * <p> The caller is responsible for making sure Autofill hears back from IME before calling + * this method, using the {@code requestConsumer} provided when calling {@link + * #onCreateInlineSuggestionsRequestLocked(AutofillId, Consumer, Bundle)}. + */ + @GuardedBy("mLock") + Optional<InlineSuggestionsRequest> getInlineSuggestionsRequestLocked() { + if (mSession != null) { + return mSession.getInlineSuggestionsRequestLocked(); + } + return Optional.empty(); + } + + /** + * Requests the IME to hide the current suggestions, if any. Returns true if the message is sent + * to the IME. This only hides the UI temporarily. For example if user starts typing/deleting + * characters, new filterText will kick in and may revive the suggestion UI. + */ + @GuardedBy("mLock") + boolean hideInlineSuggestionsUiLocked(@NonNull AutofillId autofillId) { + if (mSession != null) { + return mSession.onInlineSuggestionsResponseLocked(InlineFillUi.emptyUi(autofillId)); + } + return false; + } + + /** + * Disables prefix/regex based filtering. Other filtering rules (see {@link + * android.service.autofill.Dataset}) still apply. + */ + @GuardedBy("mLock") + void disableFilterMatching(@NonNull AutofillId autofillId) { + if (mInlineFillUi != null && mInlineFillUi.getAutofillId().equals(autofillId)) { + mInlineFillUi.disableFilterMatching(); + } + } + + /** + * Clear the locally cached inline fill UI, but don't clear the suggestion in the IME. + * + * <p>This is called to invalid the locally cached inline suggestions so we don't resend them + * to the IME, while assuming that the IME will clean up suggestion on their own when the input + * connection is finished. We don't send an empty response to IME so that it doesn't cause UI + * flicker on the IME side if it arrives before the input view is finished on the IME. + */ + @GuardedBy("mLock") + void resetInlineFillUiLocked() { + mInlineFillUi = null; + if (mSession != null) { + mSession.resetInlineFillUiLocked(); + } + } + + /** + * Updates the inline fill UI with the filter text. It'll send updated inline suggestions to + * the IME. + */ + @GuardedBy("mLock") + boolean filterInlineFillUiLocked(@NonNull AutofillId autofillId, @Nullable String filterText) { + if (mInlineFillUi != null && mInlineFillUi.getAutofillId().equals(autofillId)) { + mInlineFillUi.setFilterText(filterText); + return requestImeToShowInlineSuggestionsLocked(); + } + return false; + } + + /** + * Set the current inline fill UI. It'll request the IME to show the inline suggestions when + * the IME becomes visible and is focused on the {@code autofillId}. + * + * @return false if the suggestions are not sent to IME because there is no session, or if the + * IME callback is not available in the session. + */ + @GuardedBy("mLock") + boolean setInlineFillUiLocked(@NonNull InlineFillUi inlineFillUi) { + mInlineFillUi = inlineFillUi; + return requestImeToShowInlineSuggestionsLocked(); + } + + /** + * Sends the suggestions from the current inline fill UI to the IME. + * + * @return false if the suggestions are not sent to IME because there is no session, or if the + * IME callback is not available in the session. + */ + @GuardedBy("mLock") + private boolean requestImeToShowInlineSuggestionsLocked() { + if (mSession != null && mInlineFillUi != null) { + return mSession.onInlineSuggestionsResponseLocked(mInlineFillUi); + } + return false; + } +} diff --git a/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java b/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java new file mode 100644 index 000000000000..84fbe9a75a18 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java @@ -0,0 +1,480 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill; + +import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; +import static com.android.server.autofill.Helper.sDebug; +import static com.android.server.autofill.Helper.sVerbose; + +import android.annotation.BinderThread; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.ComponentName; +import android.os.Bundle; +import android.os.Handler; +import android.os.RemoteException; +import android.util.Slog; +import android.view.autofill.AutofillId; +import android.view.inputmethod.InlineSuggestion; +import android.view.inputmethod.InlineSuggestionsRequest; +import android.view.inputmethod.InlineSuggestionsResponse; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.view.IInlineSuggestionsRequestCallback; +import com.android.internal.view.IInlineSuggestionsResponseCallback; +import com.android.internal.view.InlineSuggestionsRequestInfo; +import com.android.server.autofill.ui.InlineFillUi; +import com.android.server.inputmethod.InputMethodManagerInternal; + +import java.lang.ref.WeakReference; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; + +/** + * Maintains an inline suggestion session with the IME. + * + * <p> Each session corresponds to one request from the Autofill manager service to create an + * {@link InlineSuggestionsRequest}. It's responsible for receiving callbacks from the IME and + * sending {@link android.view.inputmethod.InlineSuggestionsResponse} to IME. + */ +final class AutofillInlineSuggestionsRequestSession { + + private static final String TAG = AutofillInlineSuggestionsRequestSession.class.getSimpleName(); + + @NonNull + private final InputMethodManagerInternal mInputMethodManagerInternal; + private final int mUserId; + @NonNull + private final ComponentName mComponentName; + @NonNull + private final Object mLock; + @NonNull + private final Handler mHandler; + @NonNull + private final Bundle mUiExtras; + @NonNull + private final InlineFillUi.InlineUiEventCallback mUiCallback; + + @GuardedBy("mLock") + @NonNull + private AutofillId mAutofillId; + @GuardedBy("mLock") + @Nullable + private Consumer<InlineSuggestionsRequest> mImeRequestConsumer; + + @GuardedBy("mLock") + private boolean mImeRequestReceived; + @GuardedBy("mLock") + @Nullable + private InlineSuggestionsRequest mImeRequest; + @GuardedBy("mLock") + @Nullable + private IInlineSuggestionsResponseCallback mResponseCallback; + + @GuardedBy("mLock") + @Nullable + private AutofillId mImeCurrentFieldId; + @GuardedBy("mLock") + private boolean mImeInputStarted; + @GuardedBy("mLock") + private boolean mImeInputViewStarted; + @GuardedBy("mLock") + @Nullable + private InlineFillUi mInlineFillUi; + @GuardedBy("mLock") + private Boolean mPreviousResponseIsNotEmpty = null; + + @GuardedBy("mLock") + private boolean mDestroyed = false; + @GuardedBy("mLock") + private boolean mPreviousHasNonPinSuggestionShow; + @GuardedBy("mLock") + private boolean mImeSessionInvalidated = false; + + AutofillInlineSuggestionsRequestSession( + @NonNull InputMethodManagerInternal inputMethodManagerInternal, int userId, + @NonNull ComponentName componentName, @NonNull Handler handler, @NonNull Object lock, + @NonNull AutofillId autofillId, + @NonNull Consumer<InlineSuggestionsRequest> requestConsumer, @NonNull Bundle uiExtras, + @NonNull InlineFillUi.InlineUiEventCallback callback) { + mInputMethodManagerInternal = inputMethodManagerInternal; + mUserId = userId; + mComponentName = componentName; + mHandler = handler; + mLock = lock; + mUiExtras = uiExtras; + mUiCallback = callback; + + mAutofillId = autofillId; + mImeRequestConsumer = requestConsumer; + } + + @GuardedBy("mLock") + @NonNull + AutofillId getAutofillIdLocked() { + return mAutofillId; + } + + /** + * Returns the {@link InlineSuggestionsRequest} provided by IME. + * + * <p> The caller is responsible for making sure Autofill hears back from IME before calling + * this method, using the {@link #mImeRequestConsumer}. + */ + @GuardedBy("mLock") + Optional<InlineSuggestionsRequest> getInlineSuggestionsRequestLocked() { + if (mDestroyed) { + return Optional.empty(); + } + return Optional.ofNullable(mImeRequest); + } + + /** + * Requests showing the inline suggestion in the IME when the IME becomes visible and is focused + * on the {@code autofillId}. + * + * @return false if the IME callback is not available. + */ + @GuardedBy("mLock") + boolean onInlineSuggestionsResponseLocked(@NonNull InlineFillUi inlineFillUi) { + if (mDestroyed) { + return false; + } + if (sDebug) { + Slog.d(TAG, + "onInlineSuggestionsResponseLocked called for:" + inlineFillUi.getAutofillId()); + } + if (mImeRequest == null || mResponseCallback == null || mImeSessionInvalidated) { + return false; + } + // TODO(b/151123764): each session should only correspond to one field. + mAutofillId = inlineFillUi.getAutofillId(); + mInlineFillUi = inlineFillUi; + maybeUpdateResponseToImeLocked(); + return true; + } + + /** + * Prevents further interaction with the IME. Must be called before starting a new request + * session to avoid unwanted behavior from two overlapping requests. + */ + @GuardedBy("mLock") + void destroySessionLocked() { + mDestroyed = true; + + if (!mImeRequestReceived) { + Slog.w(TAG, + "Never received an InlineSuggestionsRequest from the IME for " + mAutofillId); + } + } + + /** + * Requests the IME to create an {@link InlineSuggestionsRequest}. + * + * <p> This method should only be called once per session. + */ + @GuardedBy("mLock") + void onCreateInlineSuggestionsRequestLocked() { + if (mDestroyed) { + return; + } + mImeSessionInvalidated = false; + if (sDebug) Slog.d(TAG, "onCreateInlineSuggestionsRequestLocked called: " + mAutofillId); + mInputMethodManagerInternal.onCreateInlineSuggestionsRequest(mUserId, + new InlineSuggestionsRequestInfo(mComponentName, mAutofillId, mUiExtras), + new InlineSuggestionsRequestCallbackImpl(this)); + } + + /** + * Clear the locally cached inline fill UI, but don't clear the suggestion in IME. + * + * See also {@link AutofillInlineSessionController#resetInlineFillUiLocked()} + */ + @GuardedBy("mLock") + void resetInlineFillUiLocked() { + mInlineFillUi = null; + } + + /** + * Optionally sends inline response to the IME, depending on the current state. + */ + @GuardedBy("mLock") + private void maybeUpdateResponseToImeLocked() { + if (sVerbose) Slog.v(TAG, "maybeUpdateResponseToImeLocked called"); + if (mDestroyed || mResponseCallback == null) { + return; + } + if (mImeInputViewStarted && mInlineFillUi != null && match(mAutofillId, + mImeCurrentFieldId)) { + // if IME is visible, and response is not null, send the response + InlineSuggestionsResponse response = mInlineFillUi.getInlineSuggestionsResponse(); + boolean isEmptyResponse = response.getInlineSuggestions().isEmpty(); + if (isEmptyResponse && Boolean.FALSE.equals(mPreviousResponseIsNotEmpty)) { + // No-op if both the previous response and current response are empty. + return; + } + maybeNotifyFillUiEventLocked(response.getInlineSuggestions()); + updateResponseToImeUncheckLocked(response); + mPreviousResponseIsNotEmpty = !isEmptyResponse; + } + } + + /** + * Sends the {@code response} to the IME, assuming all the relevant checks are already done. + */ + @GuardedBy("mLock") + private void updateResponseToImeUncheckLocked(InlineSuggestionsResponse response) { + if (mDestroyed) { + return; + } + if (sDebug) Slog.d(TAG, "Send inline response: " + response.getInlineSuggestions().size()); + try { + mResponseCallback.onInlineSuggestionsResponse(mAutofillId, response); + } catch (RemoteException e) { + Slog.e(TAG, "RemoteException sending InlineSuggestionsResponse to IME"); + } + } + + @GuardedBy("mLock") + private void maybeNotifyFillUiEventLocked(@NonNull List<InlineSuggestion> suggestions) { + if (mDestroyed) { + return; + } + boolean hasSuggestionToShow = false; + for (int i = 0; i < suggestions.size(); i++) { + InlineSuggestion suggestion = suggestions.get(i); + // It is possible we don't have any match result but we still have pinned + // suggestions. Only notify we have non-pinned suggestions to show + if (!suggestion.getInfo().isPinned()) { + hasSuggestionToShow = true; + break; + } + } + if (sDebug) { + Slog.d(TAG, "maybeNotifyFillUiEventLoked(): hasSuggestionToShow=" + hasSuggestionToShow + + ", mPreviousHasNonPinSuggestionShow=" + mPreviousHasNonPinSuggestionShow); + } + // Use mPreviousHasNonPinSuggestionShow to save previous status, if the display status + // change, we can notify the event. + if (hasSuggestionToShow && !mPreviousHasNonPinSuggestionShow) { + // From no suggestion to has suggestions to show + mUiCallback.notifyInlineUiShown(mAutofillId); + } else if (!hasSuggestionToShow && mPreviousHasNonPinSuggestionShow) { + // From has suggestions to no suggestions to show + mUiCallback.notifyInlineUiHidden(mAutofillId); + } + // Update the latest status + mPreviousHasNonPinSuggestionShow = hasSuggestionToShow; + } + + /** + * Handles the {@code request} and {@code callback} received from the IME. + * + * <p> Should only invoked in the {@link #mHandler} thread. + */ + private void handleOnReceiveImeRequest(@Nullable InlineSuggestionsRequest request, + @Nullable IInlineSuggestionsResponseCallback callback) { + synchronized (mLock) { + if (mDestroyed || mImeRequestReceived) { + return; + } + mImeRequestReceived = true; + mImeSessionInvalidated = false; + + if (request != null && callback != null) { + mImeRequest = request; + mResponseCallback = callback; + handleOnReceiveImeStatusUpdated(mAutofillId, true, false); + } + if (mImeRequestConsumer != null) { + // Note that mImeRequest is only set if both request and callback are non-null. + mImeRequestConsumer.accept(mImeRequest); + mImeRequestConsumer = null; + } + } + } + + /** + * Handles the IME status updates received from the IME. + * + * <p> Should only be invoked in the {@link #mHandler} thread. + */ + private void handleOnReceiveImeStatusUpdated(boolean imeInputStarted, + boolean imeInputViewStarted) { + synchronized (mLock) { + if (mDestroyed) { + return; + } + if (mImeCurrentFieldId != null) { + boolean imeInputStartedChanged = (mImeInputStarted != imeInputStarted); + boolean imeInputViewStartedChanged = (mImeInputViewStarted != imeInputViewStarted); + mImeInputStarted = imeInputStarted; + mImeInputViewStarted = imeInputViewStarted; + if (imeInputStartedChanged || imeInputViewStartedChanged) { + maybeUpdateResponseToImeLocked(); + } + } + } + } + + /** + * Handles the IME status updates received from the IME. + * + * <p> Should only be invoked in the {@link #mHandler} thread. + */ + private void handleOnReceiveImeStatusUpdated(@Nullable AutofillId imeFieldId, + boolean imeInputStarted, boolean imeInputViewStarted) { + synchronized (mLock) { + if (mDestroyed) { + return; + } + if (imeFieldId != null) { + mImeCurrentFieldId = imeFieldId; + } + handleOnReceiveImeStatusUpdated(imeInputStarted, imeInputViewStarted); + } + } + + /** + * Handles the IME session status received from the IME. + * + * <p> Should only be invoked in the {@link #mHandler} thread. + */ + private void handleOnReceiveImeSessionInvalidated() { + synchronized (mLock) { + if (mDestroyed) { + return; + } + mImeSessionInvalidated = true; + } + } + + /** + * Internal implementation of {@link IInlineSuggestionsRequestCallback}. + */ + private static final class InlineSuggestionsRequestCallbackImpl extends + IInlineSuggestionsRequestCallback.Stub { + + private final WeakReference<AutofillInlineSuggestionsRequestSession> mSession; + + private InlineSuggestionsRequestCallbackImpl( + AutofillInlineSuggestionsRequestSession session) { + mSession = new WeakReference<>(session); + } + + @BinderThread + @Override + public void onInlineSuggestionsUnsupported() throws RemoteException { + if (sDebug) Slog.d(TAG, "onInlineSuggestionsUnsupported() called."); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeRequest, session, + null, null)); + } + } + + @BinderThread + @Override + public void onInlineSuggestionsRequest(InlineSuggestionsRequest request, + IInlineSuggestionsResponseCallback callback) { + if (sDebug) Slog.d(TAG, "onInlineSuggestionsRequest() received: " + request); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeRequest, session, + request, callback)); + } + } + + @Override + public void onInputMethodStartInput(AutofillId imeFieldId) throws RemoteException { + if (sVerbose) Slog.v(TAG, "onInputMethodStartInput() received on " + imeFieldId); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated, + session, imeFieldId, true, false)); + } + } + + @Override + public void onInputMethodShowInputRequested(boolean requestResult) throws RemoteException { + if (sVerbose) { + Slog.v(TAG, "onInputMethodShowInputRequested() received: " + requestResult); + } + } + + @BinderThread + @Override + public void onInputMethodStartInputView() { + if (sVerbose) Slog.v(TAG, "onInputMethodStartInputView() received"); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated, + session, true, true)); + } + } + + @BinderThread + @Override + public void onInputMethodFinishInputView() { + if (sVerbose) Slog.v(TAG, "onInputMethodFinishInputView() received"); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated, + session, true, false)); + } + } + + @Override + public void onInputMethodFinishInput() throws RemoteException { + if (sVerbose) Slog.v(TAG, "onInputMethodFinishInput() received"); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated, + session, false, false)); + } + } + + @BinderThread + @Override + public void onInlineSuggestionsSessionInvalidated() throws RemoteException { + if (sDebug) Slog.d(TAG, "onInlineSuggestionsSessionInvalidated() called."); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession + ::handleOnReceiveImeSessionInvalidated, session)); + } + } + } + + private static boolean match(@Nullable AutofillId autofillId, + @Nullable AutofillId imeClientFieldId) { + // The IME doesn't have information about the virtual view id for the child views in the + // web view, so we are only comparing the parent view id here. This means that for cases + // where there are two input fields in the web view, they will have the same view id + // (although different virtual child id), and we will not be able to distinguish them. + return autofillId != null && imeClientFieldId != null + && autofillId.getViewId() == imeClientFieldId.getViewId(); + } +} diff --git a/services/autofill/java/com/android/server/autofill/AutofillManagerService.java b/services/autofill/java/com/android/server/autofill/AutofillManagerService.java index a64f4e475b7d..089861bee479 100644 --- a/services/autofill/java/com/android/server/autofill/AutofillManagerService.java +++ b/services/autofill/java/com/android/server/autofill/AutofillManagerService.java @@ -50,6 +50,7 @@ import android.os.RemoteCallback; import android.os.RemoteException; import android.os.ResultReceiver; import android.os.ShellCallback; +import android.os.SystemClock; import android.os.UserHandle; import android.os.UserManager; import android.provider.DeviceConfig; @@ -63,6 +64,7 @@ import android.util.LocalLog; import android.util.Slog; import android.util.SparseArray; import android.util.SparseBooleanArray; +import android.util.TimeUtils; import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; import android.view.autofill.AutofillManager.SmartSuggestionMode; @@ -151,6 +153,7 @@ public final class AutofillManagerService private final LocalLog mWtfHistory = new LocalLog(50); private final AutofillCompatState mAutofillCompatState = new AutofillCompatState(); + private final DisabledInfoCache mDisabledInfoCache = new DisabledInfoCache(); private final LocalService mLocalService = new LocalService(); private final ActivityManagerInternal mAm; @@ -189,7 +192,7 @@ public final class AutofillManagerService public AutofillManagerService(Context context) { super(context, new SecureSettingsServiceNameResolver(context, Settings.Secure.AUTOFILL_SERVICE), - UserManager.DISALLOW_AUTOFILL); + UserManager.DISALLOW_AUTOFILL, PACKAGE_UPDATE_POLICY_REFRESH_EAGER); mUi = new AutoFillUI(ActivityThread.currentActivityThread().getSystemUiContext()); mAm = LocalServices.getService(ActivityManagerInternal.class); @@ -212,8 +215,7 @@ public final class AutofillManagerService (u, s, t) -> onAugmentedServiceNameChanged(u, s, t)); if (mSupportedSmartSuggestionModes != AutofillManager.FLAG_SMART_SUGGESTION_OFF) { - final UserManager um = getContext().getSystemService(UserManager.class); - final List<UserInfo> users = um.getUsers(); + final List<UserInfo> users = getSupportedUsers(); for (int i = 0; i < users.size(); i++) { final int userId = users.get(i).id; // Must eager load the services so they bind to the augmented autofill service @@ -247,6 +249,9 @@ public final class AutofillManagerService resolver.registerContentObserver(Settings.Global.getUriFor( Settings.Global.AUTOFILL_MAX_VISIBLE_DATASETS), false, observer, UserHandle.USER_ALL); + resolver.registerContentObserver(Settings.Secure.getUriFor( + Settings.Secure.SELECTED_INPUT_METHOD_SUBTYPE), false, observer, + UserHandle.USER_ALL); } @Override // from AbstractMasterSystemService @@ -261,6 +266,9 @@ public final class AutofillManagerService case Settings.Global.AUTOFILL_MAX_VISIBLE_DATASETS: setMaxVisibleDatasetsFromSettings(); break; + case Settings.Secure.SELECTED_INPUT_METHOD_SUBTYPE: + handleInputMethodSwitch(userId); + break; default: Slog.w(TAG, "Unexpected property (" + property + "); updating cache instead"); // fall through @@ -271,6 +279,23 @@ public final class AutofillManagerService } } + private void handleInputMethodSwitch(@UserIdInt int userId) { + // TODO(b/156903336): Used the SettingsObserver with a background thread maybe slow to + // respond to the IME switch in certain situations. + // See: services/core/java/com/android/server/FgThread.java + // In particular, the shared background thread could be doing relatively long-running + // operations like saving state to disk (in addition to simply being a background priority), + // which can cause operations scheduled on it to be delayed for a user-noticeable amount + // of time. + + synchronized (mLock) { + final AutofillManagerServiceImpl service = peekServiceForUserLocked(userId); + if (service != null) { + service.onSwitchInputMethod(); + } + } + } + private void onDeviceConfigChange(@NonNull Set<String> keys) { for (String key : keys) { switch (key) { @@ -289,21 +314,29 @@ public final class AutofillManagerService boolean isTemporary) { mAugmentedAutofillState.setServiceInfo(userId, serviceName, isTemporary); synchronized (mLock) { - getServiceForUserLocked(userId).updateRemoteAugmentedAutofillService(); + final AutofillManagerServiceImpl service = peekServiceForUserLocked(userId); + if (service == null) { + // If we cannot get the service from the services cache, it will call + // updateRemoteAugmentedAutofillService() finally. Skip call this update again. + getServiceForUserLocked(userId); + } else { + service.updateRemoteAugmentedAutofillService(); + } } } @Override // from AbstractMasterSystemService protected AutofillManagerServiceImpl newServiceLocked(@UserIdInt int resolvedUserId, boolean disabled) { - return new AutofillManagerServiceImpl(this, mLock, mUiLatencyHistory, - mWtfHistory, resolvedUserId, mUi, mAutofillCompatState, disabled); + return new AutofillManagerServiceImpl(this, mLock, mUiLatencyHistory, mWtfHistory, + resolvedUserId, mUi, mAutofillCompatState, disabled, mDisabledInfoCache); } @Override // AbstractMasterSystemService protected void onServiceRemoved(@NonNull AutofillManagerServiceImpl service, @UserIdInt int userId) { service.destroyLocked(); + mDisabledInfoCache.remove(userId); mAutofillCompatState.removeCompatibilityModeRequests(userId); } @@ -325,6 +358,11 @@ public final class AutofillManagerService } @Override // from SystemService + public boolean isUserSupported(TargetUser user) { + return user.getUserInfo().isFull() || user.getUserInfo().isManagedProfile(); + } + + @Override // from SystemService public void onSwitchUser(int userHandle) { if (sDebug) Slog.d(TAG, "Hiding UI when user switched"); mUi.hideAll(null); @@ -807,6 +845,7 @@ public final class AutofillManagerService packageName, versionCode, userId); final AutofillOptions options = new AutofillOptions(loggingLevel, compatModeEnabled); mAugmentedAutofillState.injectAugmentedAutofillInfo(options, userId, packageName); + injectDisableAppInfo(options, userId, packageName); return options; } @@ -820,6 +859,14 @@ public final class AutofillManagerService } return false; } + + private void injectDisableAppInfo(@NonNull AutofillOptions options, int userId, + String packageName) { + options.appDisabledExpiration = + mDisabledInfoCache.getAppDisabledExpiration(userId, packageName); + options.disabledActivities = + mDisabledInfoCache.getAppDisabledActivities(userId, packageName); + } } /** @@ -842,6 +889,234 @@ public final class AutofillManagerService } /** + * Stores autofill disable information, i.e. {@link AutofillDisabledInfo}, keyed by user id. + * The information is cleaned up when the service is removed. + */ + static final class DisabledInfoCache { + + private final Object mLock = new Object(); + + @GuardedBy("mLock") + private final SparseArray<AutofillDisabledInfo> mCache = new SparseArray<>(); + + void remove(@UserIdInt int userId) { + synchronized (mLock) { + mCache.remove(userId); + } + } + + void addDisabledAppLocked(@UserIdInt int userId, @NonNull String packageName, + long expiration) { + Preconditions.checkNotNull(packageName); + synchronized (mLock) { + AutofillDisabledInfo info = + getOrCreateAutofillDisabledInfoByUserIdLocked(userId); + info.putDisableAppsLocked(packageName, expiration); + } + } + + void addDisabledActivityLocked(@UserIdInt int userId, @NonNull ComponentName componentName, + long expiration) { + Preconditions.checkNotNull(componentName); + synchronized (mLock) { + AutofillDisabledInfo info = + getOrCreateAutofillDisabledInfoByUserIdLocked(userId); + info.putDisableActivityLocked(componentName, expiration); + } + } + + boolean isAutofillDisabledLocked(@UserIdInt int userId, + @NonNull ComponentName componentName) { + Preconditions.checkNotNull(componentName); + final boolean disabled; + synchronized (mLock) { + final AutofillDisabledInfo info = mCache.get(userId); + disabled = info != null ? info.isAutofillDisabledLocked(componentName) : false; + } + return disabled; + } + + long getAppDisabledExpiration(@UserIdInt int userId, @NonNull String packageName) { + Preconditions.checkNotNull(packageName); + final Long expiration; + synchronized (mLock) { + final AutofillDisabledInfo info = mCache.get(userId); + expiration = info != null ? info.getAppDisabledExpirationLocked(packageName) : 0; + } + return expiration; + } + + @Nullable + ArrayMap<String, Long> getAppDisabledActivities(@UserIdInt int userId, + @NonNull String packageName) { + Preconditions.checkNotNull(packageName); + final ArrayMap<String, Long> disabledList; + synchronized (mLock) { + final AutofillDisabledInfo info = mCache.get(userId); + disabledList = + info != null ? info.getAppDisabledActivitiesLocked(packageName) : null; + } + return disabledList; + } + + void dump(@UserIdInt int userId, String prefix, PrintWriter pw) { + synchronized (mLock) { + final AutofillDisabledInfo info = mCache.get(userId); + if (info != null) { + info.dumpLocked(prefix, pw); + } + } + } + + @NonNull + private AutofillDisabledInfo getOrCreateAutofillDisabledInfoByUserIdLocked( + @UserIdInt int userId) { + AutofillDisabledInfo info = mCache.get(userId); + if (info == null) { + info = new AutofillDisabledInfo(); + mCache.put(userId, info); + } + return info; + } + } + + /** + * The autofill disable information. + * <p> + * This contains disable information set by the AutofillService, e.g. disabled application + * expiration, disable activity expiration. + */ + private static final class AutofillDisabledInfo { + /** + * Apps disabled by the service; key is package name, value is when they will be enabled + * again. + */ + private ArrayMap<String, Long> mDisabledApps; + /** + * Activities disabled by the service; key is component name, value is when they will be + * enabled again. + */ + private ArrayMap<ComponentName, Long> mDisabledActivities; + + void putDisableAppsLocked(@NonNull String packageName, long expiration) { + if (mDisabledApps == null) { + mDisabledApps = new ArrayMap<>(1); + } + mDisabledApps.put(packageName, expiration); + } + + void putDisableActivityLocked(@NonNull ComponentName componentName, long expiration) { + if (mDisabledActivities == null) { + mDisabledActivities = new ArrayMap<>(1); + } + mDisabledActivities.put(componentName, expiration); + } + + long getAppDisabledExpirationLocked(@NonNull String packageName) { + if (mDisabledApps == null) { + return 0; + } + final Long expiration = mDisabledApps.get(packageName); + return expiration != null ? expiration : 0; + } + + ArrayMap<String, Long> getAppDisabledActivitiesLocked(@NonNull String packageName) { + if (mDisabledActivities != null) { + final int size = mDisabledActivities.size(); + ArrayMap<String, Long> disabledList = null; + for (int i = 0; i < size; i++) { + final ComponentName component = mDisabledActivities.keyAt(i); + if (packageName.equals(component.getPackageName())) { + if (disabledList == null) { + disabledList = new ArrayMap<>(); + } + final long expiration = mDisabledActivities.valueAt(i); + disabledList.put(component.flattenToShortString(), expiration); + } + } + return disabledList; + } + return null; + } + + boolean isAutofillDisabledLocked(@NonNull ComponentName componentName) { + // Check activities first. + long elapsedTime = 0; + if (mDisabledActivities != null) { + elapsedTime = SystemClock.elapsedRealtime(); + final Long expiration = mDisabledActivities.get(componentName); + if (expiration != null) { + if (expiration >= elapsedTime) return true; + // Restriction expired - clean it up. + if (sVerbose) { + Slog.v(TAG, "Removing " + componentName.toShortString() + + " from disabled list"); + } + mDisabledActivities.remove(componentName); + } + } + + // Then check apps. + final String packageName = componentName.getPackageName(); + if (mDisabledApps == null) return false; + + final Long expiration = mDisabledApps.get(packageName); + if (expiration == null) return false; + + if (elapsedTime == 0) { + elapsedTime = SystemClock.elapsedRealtime(); + } + + if (expiration >= elapsedTime) return true; + + // Restriction expired - clean it up. + if (sVerbose) Slog.v(TAG, "Removing " + packageName + " from disabled list"); + mDisabledApps.remove(packageName); + return false; + } + + void dumpLocked(String prefix, PrintWriter pw) { + pw.print(prefix); pw.print("Disabled apps: "); + if (mDisabledApps == null) { + pw.println("N/A"); + } else { + final int size = mDisabledApps.size(); + pw.println(size); + final StringBuilder builder = new StringBuilder(); + final long now = SystemClock.elapsedRealtime(); + for (int i = 0; i < size; i++) { + final String packageName = mDisabledApps.keyAt(i); + final long expiration = mDisabledApps.valueAt(i); + builder.append(prefix).append(prefix) + .append(i).append(". ").append(packageName).append(": "); + TimeUtils.formatDuration((expiration - now), builder); + builder.append('\n'); + } + pw.println(builder); + } + + pw.print(prefix); pw.print("Disabled activities: "); + if (mDisabledActivities == null) { + pw.println("N/A"); + } else { + final int size = mDisabledActivities.size(); + pw.println(size); + final StringBuilder builder = new StringBuilder(); + final long now = SystemClock.elapsedRealtime(); + for (int i = 0; i < size; i++) { + final ComponentName component = mDisabledActivities.keyAt(i); + final long expiration = mDisabledActivities.valueAt(i); + builder.append(prefix).append(prefix) + .append(i).append(". ").append(component).append(": "); + TimeUtils.formatDuration((expiration - now), builder); + builder.append('\n'); + } + pw.println(builder); + } + } + } + + /** * Compatibility mode metadata associated with all services. * * <p>This object is defined here instead of on each {@link AutofillManagerServiceImpl} because @@ -1387,12 +1662,8 @@ public final class AutofillManagerService @NonNull IResultReceiver receiver) { boolean enabled = false; synchronized (mLock) { - final AutofillManagerServiceImpl service = peekServiceForUserLocked(userId); - if (service != null) { - enabled = Objects.equals(packageName, service.getServicePackageName()); - } else if (sVerbose) { - Slog.v(TAG, "isServiceEnabled(): no service for " + userId); - } + final AutofillManagerServiceImpl service = getServiceForUserLocked(userId); + enabled = Objects.equals(packageName, service.getServicePackageName()); } send(receiver, enabled); } diff --git a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java index 1bd5201f5b26..57ffe0498a88 100644 --- a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java +++ b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java @@ -57,6 +57,7 @@ import android.service.autofill.FillEventHistory; import android.service.autofill.FillEventHistory.Event; import android.service.autofill.FillResponse; import android.service.autofill.IAutoFillService; +import android.service.autofill.InlineSuggestionRenderService; import android.service.autofill.SaveInfo; import android.service.autofill.UserData; import android.util.ArrayMap; @@ -66,7 +67,6 @@ import android.util.LocalLog; import android.util.Pair; import android.util.Slog; import android.util.SparseArray; -import android.util.TimeUtils; import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; import android.view.autofill.AutofillManager.SmartSuggestionMode; @@ -79,15 +79,17 @@ import com.android.internal.logging.MetricsLogger; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; import com.android.server.LocalServices; import com.android.server.autofill.AutofillManagerService.AutofillCompatState; +import com.android.server.autofill.AutofillManagerService.DisabledInfoCache; import com.android.server.autofill.RemoteAugmentedAutofillService.RemoteAugmentedAutofillServiceCallbacks; import com.android.server.autofill.ui.AutoFillUI; +import com.android.server.contentcapture.ContentCaptureManagerInternal; import com.android.server.infra.AbstractPerUserSystemService; +import com.android.server.inputmethod.InputMethodManagerInternal; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; import java.util.Random; - /** * Bridge between the {@code system_server}'s {@link AutofillManagerService} and the * app's {@link IAutoFillService} implementation. @@ -117,18 +119,9 @@ final class AutofillManagerServiceImpl private final LocalLog mWtfHistory; private final FieldClassificationStrategy mFieldClassificationStrategy; - /** - * Apps disabled by the service; key is package name, value is when they will be enabled again. - */ @GuardedBy("mLock") - private ArrayMap<String, Long> mDisabledApps; - - /** - * Activities disabled by the service; key is component name, value is when they will be enabled - * again. - */ - @GuardedBy("mLock") - private ArrayMap<ComponentName, Long> mDisabledActivities; + @Nullable + private RemoteInlineSuggestionRenderService mRemoteInlineSuggestionRenderService; /** * Data used for field classification. @@ -151,6 +144,13 @@ final class AutofillManagerServiceImpl @GuardedBy("mLock") private FillEventHistory mEventHistory; + /** + * The last inline augmented autofill selection. Note that we don't log the selection from the + * dropdown UI since the service owns the UI in that case. + */ + @GuardedBy("mLock") + private FillEventHistory mAugmentedAutofillEventHistory; + /** Shared instance, doesn't need to be logged */ private final AutofillCompatState mAutofillCompatState; @@ -168,10 +168,16 @@ final class AutofillManagerServiceImpl @Nullable private ServiceInfo mRemoteAugmentedAutofillServiceInfo; + private final InputMethodManagerInternal mInputMethodManagerInternal; + + private final ContentCaptureManagerInternal mContentCaptureManagerInternal; + + private final DisabledInfoCache mDisabledInfoCache; + AutofillManagerServiceImpl(AutofillManagerService master, Object lock, LocalLog uiLatencyHistory, LocalLog wtfHistory, int userId, AutoFillUI ui, AutofillCompatState autofillCompatState, - boolean disabled) { + boolean disabled, DisabledInfoCache disableCache) { super(master, lock, userId); mUiLatencyHistory = uiLatencyHistory; @@ -179,10 +185,23 @@ final class AutofillManagerServiceImpl mUi = ui; mFieldClassificationStrategy = new FieldClassificationStrategy(getContext(), userId); mAutofillCompatState = autofillCompatState; - + mInputMethodManagerInternal = LocalServices.getService(InputMethodManagerInternal.class); + mContentCaptureManagerInternal = LocalServices.getService( + ContentCaptureManagerInternal.class); + mDisabledInfoCache = disableCache; updateLocked(disabled); } + boolean sendActivityAssistDataToContentCapture(@NonNull IBinder activityToken, + @NonNull Bundle data) { + if (mContentCaptureManagerInternal != null) { + mContentCaptureManagerInternal.sendActivityAssistData(getUserId(), activityToken, data); + return true; + } + + return false; + } + @GuardedBy("mLock") void onBackKeyPressed() { final RemoteAugmentedAutofillService remoteService = @@ -208,6 +227,8 @@ final class AutofillManagerServiceImpl sendStateToClients(/* resetClient= */ false); } updateRemoteAugmentedAutofillService(); + updateRemoteInlineSuggestionRenderServiceLocked(); + return enabledChanged; } @@ -239,7 +260,7 @@ final class AutofillManagerServiceImpl if (isEnabledLocked()) return FLAG_ADD_CLIENT_ENABLED; // Check if it's enabled for augmented autofill - if (isAugmentedAutofillServiceAvailableLocked() + if (componentName != null && isAugmentedAutofillServiceAvailableLocked() && isWhitelistedForAugmentedAutofillLocked(componentName)) { return FLAG_ADD_CLIENT_ENABLED_FOR_AUGMENTED_AUTOFILL_ONLY; } @@ -493,7 +514,7 @@ final class AutofillManagerServiceImpl sessionId, taskId, uid, activityToken, appCallbackToken, hasCallback, mUiLatencyHistory, mWtfHistory, serviceComponentName, componentName, compatMode, bindInstantServiceAllowed, forAugmentedAutofillOnly, - flags); + flags, mInputMethodManagerInternal); mSessions.put(newSession.id, newSession); return newSession; @@ -690,6 +711,13 @@ final class AutofillManagerServiceImpl } } + void setLastAugmentedAutofillResponse(int sessionId) { + synchronized (mLock) { + mAugmentedAutofillEventHistory = new FillEventHistory(sessionId, /* clientState= */ + null); + } + } + /** * Resets the last fill selection. */ @@ -699,6 +727,12 @@ final class AutofillManagerServiceImpl } } + void resetLastAugmentedAutofillResponse() { + synchronized (mLock) { + mAugmentedAutofillEventHistory = null; + } + } + @GuardedBy("mLock") private boolean isValidEventLocked(String method, int sessionId) { if (mEventHistory == null) { @@ -769,6 +803,58 @@ final class AutofillManagerServiceImpl } /** + * Updates the last fill response when a dataset is shown. + */ + void logDatasetShown(int sessionId, @Nullable Bundle clientState) { + synchronized (mLock) { + if (isValidEventLocked("logDatasetShown", sessionId)) { + mEventHistory.addEvent( + new Event(Event.TYPE_DATASETS_SHOWN, null, clientState, null, null, null, + null, null, null, null, null)); + } + } + } + + void logAugmentedAutofillAuthenticationSelected(int sessionId, @Nullable String selectedDataset, + @Nullable Bundle clientState) { + synchronized (mLock) { + if (mAugmentedAutofillEventHistory == null + || mAugmentedAutofillEventHistory.getSessionId() != sessionId) { + return; + } + mAugmentedAutofillEventHistory.addEvent( + new Event(Event.TYPE_DATASET_AUTHENTICATION_SELECTED, selectedDataset, + clientState, null, null, null, null, null, null, null, null)); + } + } + + void logAugmentedAutofillSelected(int sessionId, @Nullable String suggestionId, + @Nullable Bundle clientState) { + synchronized (mLock) { + if (mAugmentedAutofillEventHistory == null + || mAugmentedAutofillEventHistory.getSessionId() != sessionId) { + return; + } + mAugmentedAutofillEventHistory.addEvent( + new Event(Event.TYPE_DATASET_SELECTED, suggestionId, clientState, null, null, + null, null, null, null, null, null)); + } + } + + void logAugmentedAutofillShown(int sessionId, @Nullable Bundle clientState) { + synchronized (mLock) { + if (mAugmentedAutofillEventHistory == null + || mAugmentedAutofillEventHistory.getSessionId() != sessionId) { + return; + } + mAugmentedAutofillEventHistory.addEvent( + new Event(Event.TYPE_DATASETS_SHOWN, null, clientState, null, null, null, + null, null, null, null, null)); + + } + } + + /** * Updates the last fill response when an autofill context is committed. */ @GuardedBy("mLock") @@ -851,8 +937,8 @@ final class AutofillManagerServiceImpl * Gets the fill event history. * * @param callingUid The calling uid - * - * @return The history or {@code null} if there is none. + * @return The history for the autofill or the augmented autofill events depending on the {@code + * callingUid}, or {@code null} if there is none. */ FillEventHistory getFillEventHistory(int callingUid) { synchronized (mLock) { @@ -860,6 +946,10 @@ final class AutofillManagerServiceImpl && isCalledByServiceLocked("getFillEventHistory", callingUid)) { return mEventHistory; } + if (mAugmentedAutofillEventHistory != null && isCalledByAugmentedAutofillServiceLocked( + "getFillEventHistory", callingUid)) { + return mAugmentedAutofillEventHistory; + } } return null; } @@ -953,47 +1043,11 @@ final class AutofillManagerServiceImpl } else { pw.println(compatPkgs); } + pw.print(prefix); pw.print("Inline Suggestions Enabled: "); + pw.println(isInlineSuggestionsEnabled()); pw.print(prefix); pw.print("Last prune: "); pw.println(mLastPrune); - pw.print(prefix); pw.print("Disabled apps: "); - - if (mDisabledApps == null) { - pw.println("N/A"); - } else { - final int size = mDisabledApps.size(); - pw.println(size); - final StringBuilder builder = new StringBuilder(); - final long now = SystemClock.elapsedRealtime(); - for (int i = 0; i < size; i++) { - final String packageName = mDisabledApps.keyAt(i); - final long expiration = mDisabledApps.valueAt(i); - builder.append(prefix).append(prefix) - .append(i).append(". ").append(packageName).append(": "); - TimeUtils.formatDuration((expiration - now), builder); - builder.append('\n'); - } - pw.println(builder); - } - - pw.print(prefix); pw.print("Disabled activities: "); - - if (mDisabledActivities == null) { - pw.println("N/A"); - } else { - final int size = mDisabledActivities.size(); - pw.println(size); - final StringBuilder builder = new StringBuilder(); - final long now = SystemClock.elapsedRealtime(); - for (int i = 0; i < size; i++) { - final ComponentName component = mDisabledActivities.keyAt(i); - final long expiration = mDisabledActivities.valueAt(i); - builder.append(prefix).append(prefix) - .append(i).append(". ").append(component).append(": "); - TimeUtils.formatDuration((expiration - now), builder); - builder.append('\n'); - } - pw.println(builder); - } + mDisabledInfoCache.dump(mUserId, prefix, pw); final int size = mSessions.size(); if (size == 0) { @@ -1103,6 +1157,14 @@ final class AutofillManagerServiceImpl } @GuardedBy("mLock") + boolean isInlineSuggestionsEnabled() { + if (mInfo != null) { + return mInfo.isInlineSuggestionsEnabled(); + } + return false; + } + + @GuardedBy("mLock") @Nullable RemoteAugmentedAutofillService getRemoteAugmentedAutofillServiceLocked() { if (mRemoteAugmentedAutofillService == null) { final String serviceName = mMaster.mAugmentedAutofillResolver.getServiceName(mUserId); @@ -1123,20 +1185,54 @@ final class AutofillManagerServiceImpl Slog.v(TAG, "getRemoteAugmentedAutofillServiceLocked(): " + componentName); } - mRemoteAugmentedAutofillService = new RemoteAugmentedAutofillService(getContext(), - componentName, mUserId, new RemoteAugmentedAutofillServiceCallbacks() { + final RemoteAugmentedAutofillServiceCallbacks callbacks = + new RemoteAugmentedAutofillServiceCallbacks() { + @Override + public void resetLastResponse() { + AutofillManagerServiceImpl.this.resetLastAugmentedAutofillResponse(); + } + + @Override + public void setLastResponse(int sessionId) { + AutofillManagerServiceImpl.this.setLastAugmentedAutofillResponse( + sessionId); + } + + @Override + public void logAugmentedAutofillShown(int sessionId, Bundle clientState) { + AutofillManagerServiceImpl.this.logAugmentedAutofillShown(sessionId, + clientState); + } + + @Override + public void logAugmentedAutofillSelected(int sessionId, + String suggestionId, Bundle clientState) { + AutofillManagerServiceImpl.this.logAugmentedAutofillSelected(sessionId, + suggestionId, clientState); + } + + @Override + public void logAugmentedAutofillAuthenticationSelected(int sessionId, + String suggestionId, Bundle clientState) { + AutofillManagerServiceImpl.this + .logAugmentedAutofillAuthenticationSelected( + sessionId, suggestionId, clientState); + } + @Override public void onServiceDied(@NonNull RemoteAugmentedAutofillService service) { Slog.w(TAG, "remote augmented autofill service died"); final RemoteAugmentedAutofillService remoteService = mRemoteAugmentedAutofillService; if (remoteService != null) { - remoteService.destroy(); + remoteService.unbind(); } mRemoteAugmentedAutofillService = null; } - }, mMaster.isInstantServiceAllowed(), mMaster.verbose, - mMaster.mAugmentedServiceIdleUnbindTimeoutMs, + }; + mRemoteAugmentedAutofillService = new RemoteAugmentedAutofillService(getContext(), + componentName, mUserId, callbacks, mMaster.isInstantServiceAllowed(), + mMaster.verbose, mMaster.mAugmentedServiceIdleUnbindTimeoutMs, mMaster.mAugmentedServiceRequestTimeoutMs); } @@ -1155,7 +1251,7 @@ final class AutofillManagerServiceImpl + "destroying old remote service"); } destroySessionsForAugmentedAutofillOnlyLocked(); - mRemoteAugmentedAutofillService.destroy(); + mRemoteAugmentedAutofillService.unbind(); mRemoteAugmentedAutofillService = null; mRemoteAugmentedAutofillServiceInfo = null; resetAugmentedAutofillWhitelistLocked(); @@ -1357,15 +1453,13 @@ final class AutofillManagerServiceImpl void disableAutofillForApp(@NonNull String packageName, long duration, int sessionId, boolean compatMode) { synchronized (mLock) { - if (mDisabledApps == null) { - mDisabledApps = new ArrayMap<>(1); - } long expiration = SystemClock.elapsedRealtime() + duration; // Protect it against overflow if (expiration < 0) { expiration = Long.MAX_VALUE; } - mDisabledApps.put(packageName, expiration); + mDisabledInfoCache.addDisabledAppLocked(mUserId, packageName, expiration); + int intDuration = duration > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) duration; mMetricsLogger.write(Helper.newLogMaker(MetricsEvent.AUTOFILL_SERVICE_DISABLED_APP, packageName, getServicePackageName(), sessionId, compatMode) @@ -1379,15 +1473,12 @@ final class AutofillManagerServiceImpl void disableAutofillForActivity(@NonNull ComponentName componentName, long duration, int sessionId, boolean compatMode) { synchronized (mLock) { - if (mDisabledActivities == null) { - mDisabledActivities = new ArrayMap<>(1); - } long expiration = SystemClock.elapsedRealtime() + duration; // Protect it against overflow if (expiration < 0) { expiration = Long.MAX_VALUE; } - mDisabledActivities.put(componentName, expiration); + mDisabledInfoCache.addDisabledActivityLocked(mUserId, componentName, expiration); final int intDuration = duration > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) duration; @@ -1410,39 +1501,7 @@ final class AutofillManagerServiceImpl */ @GuardedBy("mLock") private boolean isAutofillDisabledLocked(@NonNull ComponentName componentName) { - // Check activities first. - long elapsedTime = 0; - if (mDisabledActivities != null) { - elapsedTime = SystemClock.elapsedRealtime(); - final Long expiration = mDisabledActivities.get(componentName); - if (expiration != null) { - if (expiration >= elapsedTime) return true; - // Restriction expired - clean it up. - if (sVerbose) { - Slog.v(TAG, "Removing " + componentName.toShortString() - + " from disabled list"); - } - mDisabledActivities.remove(componentName); - } - } - - // Then check apps. - final String packageName = componentName.getPackageName(); - if (mDisabledApps == null) return false; - - final Long expiration = mDisabledApps.get(packageName); - if (expiration == null) return false; - - if (elapsedTime == 0) { - elapsedTime = SystemClock.elapsedRealtime(); - } - - if (expiration >= elapsedTime) return true; - - // Restriction expired - clean it up. - if (sVerbose) Slog.v(TAG, "Removing " + packageName + " from disabled list"); - mDisabledApps.remove(packageName); - return false; + return mDisabledInfoCache.isAutofillDisabledLocked(mUserId, componentName); } // Called by AutofillManager, checks UID. @@ -1485,6 +1544,57 @@ final class AutofillManagerServiceImpl return mFieldClassificationStrategy.getDefaultAlgorithm(); } + private void updateRemoteInlineSuggestionRenderServiceLocked() { + if (mRemoteInlineSuggestionRenderService != null) { + if (sVerbose) { + Slog.v(TAG, "updateRemoteInlineSuggestionRenderService(): " + + "destroying old remote service"); + } + mRemoteInlineSuggestionRenderService = null; + } + + mRemoteInlineSuggestionRenderService = getRemoteInlineSuggestionRenderServiceLocked(); + } + + @Nullable RemoteInlineSuggestionRenderService getRemoteInlineSuggestionRenderServiceLocked() { + if (mRemoteInlineSuggestionRenderService == null) { + final ComponentName componentName = RemoteInlineSuggestionRenderService + .getServiceComponentName(getContext(), mUserId); + if (componentName == null) { + Slog.w(TAG, "No valid component found for InlineSuggestionRenderService"); + return null; + } + + mRemoteInlineSuggestionRenderService = new RemoteInlineSuggestionRenderService( + getContext(), componentName, InlineSuggestionRenderService.SERVICE_INTERFACE, + mUserId, new InlineSuggestionRenderCallbacksImpl(), + mMaster.isBindInstantServiceAllowed(), mMaster.verbose); + } + + return mRemoteInlineSuggestionRenderService; + } + + private class InlineSuggestionRenderCallbacksImpl implements + RemoteInlineSuggestionRenderService.InlineSuggestionRenderCallbacks { + + @Override // from InlineSuggestionRenderCallbacksImpl + public void onServiceDied(@NonNull RemoteInlineSuggestionRenderService service) { + // Don't do anything; eventually the system will bind to it again... + Slog.w(TAG, "remote service died: " + service); + mRemoteInlineSuggestionRenderService = null; + } + } + + void onSwitchInputMethod() { + synchronized (mLock) { + final int sessionCount = mSessions.size(); + for (int i = 0; i < sessionCount; i++) { + final Session session = mSessions.valueAt(i); + session.onSwitchInputMethodLocked(); + } + } + } + @Override public String toString() { return "AutofillManagerServiceImpl: [userId=" + mUserId diff --git a/services/autofill/java/com/android/server/autofill/FieldClassificationStrategy.java b/services/autofill/java/com/android/server/autofill/FieldClassificationStrategy.java index 9db6254a8baa..36a450920373 100644 --- a/services/autofill/java/com/android/server/autofill/FieldClassificationStrategy.java +++ b/services/autofill/java/com/android/server/autofill/FieldClassificationStrategy.java @@ -121,7 +121,12 @@ final class FieldClassificationStrategy { synchronized (mLock) { if (mServiceConnection != null) { if (sDebug) Slog.d(TAG, "reset(): unbinding service."); - mContext.unbindService(mServiceConnection); + try { + mContext.unbindService(mServiceConnection); + } catch (IllegalArgumentException e) { + // no-op, just log the error message. + Slog.w(TAG, "reset(): " + e.getMessage()); + } mServiceConnection = null; } else { if (sDebug) Slog.d(TAG, "reset(): service is not bound. Do nothing."); diff --git a/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java b/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java index d42943ce76fe..533bbe68e274 100644 --- a/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java +++ b/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java @@ -27,13 +27,17 @@ import android.annotation.UserIdInt; import android.app.AppGlobals; import android.content.ComponentName; import android.content.Context; +import android.content.Intent; +import android.content.IntentSender; import android.content.pm.PackageManager; import android.content.pm.ServiceInfo; import android.os.Bundle; +import android.os.Handler; import android.os.IBinder; import android.os.ICancellationSignal; import android.os.RemoteException; import android.os.SystemClock; +import android.service.autofill.Dataset; import android.service.autofill.augmented.AugmentedAutofillService; import android.service.autofill.augmented.IAugmentedAutofillService; import android.service.autofill.augmented.IFillCallback; @@ -43,32 +47,48 @@ import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; import android.view.autofill.AutofillValue; import android.view.autofill.IAutoFillManagerClient; +import android.view.inputmethod.InlineSuggestionsRequest; -import com.android.internal.infra.AbstractSinglePendingRequestRemoteService; +import com.android.internal.infra.AbstractRemoteService; +import com.android.internal.infra.AndroidFuture; +import com.android.internal.infra.ServiceConnector; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; import com.android.internal.os.IResultReceiver; +import com.android.server.autofill.ui.InlineFillUi; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; final class RemoteAugmentedAutofillService - extends AbstractSinglePendingRequestRemoteService<RemoteAugmentedAutofillService, - IAugmentedAutofillService> { + extends ServiceConnector.Impl<IAugmentedAutofillService> { private static final String TAG = RemoteAugmentedAutofillService.class.getSimpleName(); private final int mIdleUnbindTimeoutMs; private final int mRequestTimeoutMs; + private final ComponentName mComponentName; + private final RemoteAugmentedAutofillServiceCallbacks mCallbacks; RemoteAugmentedAutofillService(Context context, ComponentName serviceName, int userId, RemoteAugmentedAutofillServiceCallbacks callbacks, boolean bindInstantServiceAllowed, boolean verbose, int idleUnbindTimeoutMs, int requestTimeoutMs) { - super(context, AugmentedAutofillService.SERVICE_INTERFACE, serviceName, userId, callbacks, - context.getMainThreadHandler(), - bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0, verbose); + super(context, + new Intent(AugmentedAutofillService.SERVICE_INTERFACE).setComponent(serviceName), + bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0, + userId, IAugmentedAutofillService.Stub::asInterface); mIdleUnbindTimeoutMs = idleUnbindTimeoutMs; mRequestTimeoutMs = requestTimeoutMs; + mComponentName = serviceName; + mCallbacks = callbacks; // Bind right away. - scheduleBind(); + connect(); } @Nullable @@ -96,215 +116,235 @@ final class RemoteAugmentedAutofillService return new Pair<>(serviceInfo, serviceComponent); } - @Override // from RemoteService - protected void handleOnConnectedStateChanged(boolean state) { - if (state && getTimeoutIdleBindMillis() != PERMANENT_BOUND_TIMEOUT_MS) { - scheduleUnbind(); - } + public ComponentName getComponentName() { + return mComponentName; + } + + @Override // from ServiceConnector.Impl + protected void onServiceConnectionStatusChanged( + IAugmentedAutofillService service, boolean connected) { try { - if (state) { - mService.onConnected(sDebug, sVerbose); + if (connected) { + service.onConnected(sDebug, sVerbose); } else { - mService.onDisconnected(); + service.onDisconnected(); } } catch (Exception e) { - Slog.w(mTag, "Exception calling onConnectedStateChanged(" + state + "): " + e); + Slog.w(TAG, + "Exception calling onServiceConnectionStatusChanged(" + connected + "): ", e); } } @Override // from AbstractRemoteService - protected IAugmentedAutofillService getServiceInterface(IBinder service) { - return IAugmentedAutofillService.Stub.asInterface(service); - } - - @Override // from AbstractRemoteService - protected long getTimeoutIdleBindMillis() { + protected long getAutoDisconnectTimeoutMs() { return mIdleUnbindTimeoutMs; } - @Override // from AbstractRemoteService - protected long getRemoteRequestMillis() { - return mRequestTimeoutMs; - } - /** * Called by {@link Session} to request augmented autofill. */ public void onRequestAutofillLocked(int sessionId, @NonNull IAutoFillManagerClient client, int taskId, @NonNull ComponentName activityComponent, @NonNull AutofillId focusedId, - @Nullable AutofillValue focusedValue) { - scheduleRequest(new PendingAutofillRequest(this, sessionId, client, taskId, - activityComponent, focusedId, focusedValue)); - } - - @Override - public String toString() { - return "RemoteAugmentedAutofillService[" - + ComponentName.flattenToShortString(getComponentName()) + "]"; - } - - /** - * Called by {@link Session} when it's time to destroy all augmented autofill requests. - */ - public void onDestroyAutofillWindowsRequest() { - scheduleAsyncRequest((s) -> s.onDestroyAllFillWindowsRequest()); + @Nullable AutofillValue focusedValue, + @Nullable InlineSuggestionsRequest inlineSuggestionsRequest, + @Nullable Function<InlineFillUi, Boolean> inlineSuggestionsCallback, + @NonNull Runnable onErrorCallback, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, int userId) { + long requestTime = SystemClock.elapsedRealtime(); + AtomicReference<ICancellationSignal> cancellationRef = new AtomicReference<>(); + + postAsync(service -> { + AndroidFuture<Void> requestAutofill = new AndroidFuture<>(); + // TODO(b/122728762): set cancellation signal, timeout (from both client and service), + // cache IAugmentedAutofillManagerClient reference, etc... + client.getAugmentedAutofillClient(new IResultReceiver.Stub() { + @Override + public void send(int resultCode, Bundle resultData) throws RemoteException { + final IBinder realClient = resultData + .getBinder(AutofillManager.EXTRA_AUGMENTED_AUTOFILL_CLIENT); + service.onFillRequest(sessionId, realClient, taskId, activityComponent, + focusedId, focusedValue, requestTime, inlineSuggestionsRequest, + new IFillCallback.Stub() { + @Override + public void onSuccess(@Nullable List<Dataset> inlineSuggestionsData, + @Nullable Bundle clientState, boolean showingFillWindow) { + mCallbacks.resetLastResponse(); + maybeRequestShowInlineSuggestions(sessionId, + inlineSuggestionsRequest, inlineSuggestionsData, + clientState, focusedId, focusedValue, + inlineSuggestionsCallback, + client, onErrorCallback, remoteRenderService, userId); + if (!showingFillWindow) { + requestAutofill.complete(null); + } + } + + @Override + public boolean isCompleted() { + return requestAutofill.isDone() + && !requestAutofill.isCancelled(); + } + + @Override + public void onCancellable(ICancellationSignal cancellation) { + if (requestAutofill.isCancelled()) { + dispatchCancellation(cancellation); + } else { + cancellationRef.set(cancellation); + } + } + + @Override + public void cancel() { + requestAutofill.cancel(true); + } + }); + } + }); + return requestAutofill; + }).orTimeout(mRequestTimeoutMs, TimeUnit.MILLISECONDS) + .whenComplete((res, err) -> { + if (err instanceof CancellationException) { + dispatchCancellation(cancellationRef.get()); + } else if (err instanceof TimeoutException) { + Slog.w(TAG, "PendingAutofillRequest timed out (" + mRequestTimeoutMs + + "ms) for " + RemoteAugmentedAutofillService.this); + // NOTE: so far we don't need notify RemoteAugmentedAutofillServiceCallbacks + dispatchCancellation(cancellationRef.get()); + if (mComponentName != null) { + logResponse(MetricsEvent.TYPE_ERROR, mComponentName.getPackageName(), + activityComponent, sessionId, mRequestTimeoutMs); + } + } else if (err != null) { + Slog.e(TAG, "exception handling getAugmentedAutofillClient() for " + + sessionId + ": ", err); + } else { + // NOTE: so far we don't need notify RemoteAugmentedAutofillServiceCallbacks + } + }); } - private void dispatchOnFillTimeout(@NonNull ICancellationSignal cancellation) { - mHandler.post(() -> { + void dispatchCancellation(@Nullable ICancellationSignal cancellation) { + if (cancellation == null) { + return; + } + Handler.getMain().post(() -> { try { cancellation.cancel(); } catch (RemoteException e) { - Slog.w(mTag, "Error calling cancellation signal: " + e); + Slog.e(TAG, "Error requesting a cancellation", e); } }); } - // TODO(b/123100811): inline into PendingAutofillRequest if it doesn't have any other subclass - private abstract static class MyPendingRequest - extends PendingRequest<RemoteAugmentedAutofillService, IAugmentedAutofillService> { - protected final int mSessionId; - - private MyPendingRequest(@NonNull RemoteAugmentedAutofillService service, int sessionId) { - super(service); - mSessionId = sessionId; + private void maybeRequestShowInlineSuggestions(int sessionId, + @Nullable InlineSuggestionsRequest request, + @Nullable List<Dataset> inlineSuggestionsData, @Nullable Bundle clientState, + @NonNull AutofillId focusedId, @Nullable AutofillValue focusedValue, + @Nullable Function<InlineFillUi, Boolean> inlineSuggestionsCallback, + @NonNull IAutoFillManagerClient client, @NonNull Runnable onErrorCallback, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + int userId) { + if (inlineSuggestionsData == null || inlineSuggestionsData.isEmpty() + || inlineSuggestionsCallback == null || request == null + || remoteRenderService == null) { + // If it was an inline request and the response doesn't have any inline suggestions, + // we will send an empty response to IME. + if (inlineSuggestionsCallback != null && request != null) { + inlineSuggestionsCallback.apply(InlineFillUi.emptyUi(focusedId)); + } + return; } - } - - private static final class PendingAutofillRequest extends MyPendingRequest { - private final @NonNull AutofillId mFocusedId; - private final @Nullable AutofillValue mFocusedValue; - private final @NonNull IAutoFillManagerClient mClient; - private final @NonNull ComponentName mActivityComponent; - private final int mSessionId; - private final int mTaskId; - private final long mRequestTime = SystemClock.elapsedRealtime(); - private final @NonNull IFillCallback mCallback; - private ICancellationSignal mCancellation; - - protected PendingAutofillRequest(@NonNull RemoteAugmentedAutofillService service, - int sessionId, @NonNull IAutoFillManagerClient client, int taskId, - @NonNull ComponentName activityComponent, @NonNull AutofillId focusedId, - @Nullable AutofillValue focusedValue) { - super(service, sessionId); - mClient = client; - mSessionId = sessionId; - mTaskId = taskId; - mActivityComponent = activityComponent; - mFocusedId = focusedId; - mFocusedValue = focusedValue; - mCallback = new IFillCallback.Stub() { - @Override - public void onSuccess() { - if (!finish()) return; - // NOTE: so far we don't need notify RemoteAugmentedAutofillServiceCallbacks - } - - @Override - public void onCancellable(ICancellationSignal cancellation) { - synchronized (mLock) { - final boolean cancelled; - synchronized (mLock) { - mCancellation = cancellation; - cancelled = isCancelledLocked(); - } - if (cancelled) { - try { - cancellation.cancel(); - } catch (RemoteException e) { - Slog.e(mTag, "Error requesting a cancellation", e); + mCallbacks.setLastResponse(sessionId); + + final String filterText = + focusedValue != null && focusedValue.isText() + ? focusedValue.getTextValue().toString() : null; + + final InlineFillUi inlineFillUi = + InlineFillUi.forAugmentedAutofill( + request, inlineSuggestionsData, focusedId, filterText, + new InlineFillUi.InlineSuggestionUiCallback() { + @Override + public void autofill(Dataset dataset, int datasetIndex) { + if (dataset.getAuthentication() != null) { + mCallbacks.logAugmentedAutofillAuthenticationSelected(sessionId, + dataset.getId(), clientState); + final IntentSender action = dataset.getAuthentication(); + final int authenticationId = + AutofillManager.makeAuthenticationId( + Session.AUGMENTED_AUTOFILL_REQUEST_ID, + datasetIndex); + final Intent fillInIntent = new Intent(); + fillInIntent.putExtra(AutofillManager.EXTRA_CLIENT_STATE, + clientState); + try { + client.authenticate(sessionId, authenticationId, action, + fillInIntent, false); + } catch (RemoteException e) { + Slog.w(TAG, "Error starting auth flow"); + inlineSuggestionsCallback.apply( + InlineFillUi.emptyUi(focusedId)); + } + return; + } + mCallbacks.logAugmentedAutofillSelected(sessionId, + dataset.getId(), clientState); + try { + final ArrayList<AutofillId> fieldIds = dataset.getFieldIds(); + final int size = fieldIds.size(); + final boolean hideHighlight = size == 1 + && fieldIds.get(0).equals(focusedId); + client.autofill(sessionId, fieldIds, dataset.getFieldValues(), + hideHighlight); + inlineSuggestionsCallback.apply( + InlineFillUi.emptyUi(focusedId)); + } catch (RemoteException e) { + Slog.w(TAG, "Encounter exception autofilling the values"); + } } - } - } - } - @Override - public boolean isCompleted() { - return isRequestCompleted(); - } + @Override + public void startIntentSender(IntentSender intentSender, + Intent intent) { + try { + client.startIntentSender(intentSender, intent); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException starting intent sender"); + } + } + }, onErrorCallback, remoteRenderService, userId, sessionId); - @Override - public void cancel() { - PendingAutofillRequest.this.cancel(); - } - }; + if (inlineSuggestionsCallback.apply(inlineFillUi)) { + mCallbacks.logAugmentedAutofillShown(sessionId, clientState); } + } - @Override - public void run() { - synchronized (mLock) { - if (isCancelledLocked()) { - if (sDebug) Slog.d(mTag, "run() called after canceled"); - return; - } - } - final RemoteAugmentedAutofillService remoteService = getService(); - if (remoteService == null) return; - - final IResultReceiver receiver = new IResultReceiver.Stub() { + @Override + public String toString() { + return "RemoteAugmentedAutofillService[" + + ComponentName.flattenToShortString(mComponentName) + "]"; + } - @Override - public void send(int resultCode, Bundle resultData) throws RemoteException { - final IBinder realClient = resultData - .getBinder(AutofillManager.EXTRA_AUGMENTED_AUTOFILL_CLIENT); - remoteService.mService.onFillRequest(mSessionId, realClient, mTaskId, - mActivityComponent, mFocusedId, mFocusedValue, mRequestTime, mCallback); - } - }; + /** + * Called by {@link Session} when it's time to destroy all augmented autofill requests. + */ + public void onDestroyAutofillWindowsRequest() { + run((s) -> s.onDestroyAllFillWindowsRequest()); + } - // TODO(b/122728762): set cancellation signal, timeout (from both mClient and service), - // cache IAugmentedAutofillManagerClient reference, etc... - try { - mClient.getAugmentedAutofillClient(receiver); - } catch (RemoteException e) { - Slog.e(TAG, "exception handling getAugmentedAutofillClient() for " - + mSessionId + ": " + e); - finish(); - } - } + public interface RemoteAugmentedAutofillServiceCallbacks + extends AbstractRemoteService.VultureCallback<RemoteAugmentedAutofillService> { + void resetLastResponse(); - @Override - protected void onTimeout(RemoteAugmentedAutofillService remoteService) { - // TODO(b/122858578): must update the logged AUTOFILL_AUGMENTED_REQUEST with the - // timeout - Slog.w(TAG, "PendingAutofillRequest timed out (" + remoteService.mRequestTimeoutMs - + "ms) for " + remoteService); - // NOTE: so far we don't need notify RemoteAugmentedAutofillServiceCallbacks - final ICancellationSignal cancellation; - synchronized (mLock) { - cancellation = mCancellation; - } - if (cancellation != null) { - remoteService.dispatchOnFillTimeout(cancellation); - } - finish(); - logResponse(MetricsEvent.TYPE_ERROR, remoteService.getComponentName().getPackageName(), - mActivityComponent, mSessionId, remoteService.mRequestTimeoutMs); - } + void setLastResponse(int sessionId); - @Override - public boolean cancel() { - if (!super.cancel()) return false; + void logAugmentedAutofillShown(int sessionId, @Nullable Bundle clientState); - final ICancellationSignal cancellation; - synchronized (mLock) { - cancellation = mCancellation; - } - if (cancellation != null) { - try { - cancellation.cancel(); - } catch (RemoteException e) { - Slog.e(mTag, "Error cancelling an augmented fill request", e); - } - } - return true; - } - } + void logAugmentedAutofillSelected(int sessionId, @Nullable String suggestionId, + @Nullable Bundle clientState); - public interface RemoteAugmentedAutofillServiceCallbacks - extends VultureCallback<RemoteAugmentedAutofillService> { - // NOTE: so far we don't need to notify the callback implementation (an inner class on - // AutofillManagerServiceImpl) of the request results (success, timeouts, etc..), so this - // callback interface is empty. + void logAugmentedAutofillAuthenticationSelected(int sessionId, + @Nullable String suggestionId, @Nullable Bundle clientState); } } diff --git a/services/autofill/java/com/android/server/autofill/RemoteFillService.java b/services/autofill/java/com/android/server/autofill/RemoteFillService.java index 3143bcb23c3a..5a9320f61b38 100644 --- a/services/autofill/java/com/android/server/autofill/RemoteFillService.java +++ b/services/autofill/java/com/android/server/autofill/RemoteFillService.java @@ -18,15 +18,15 @@ package com.android.server.autofill; import static android.service.autofill.FillRequest.INVALID_REQUEST_ID; -import static com.android.server.autofill.Helper.sDebug; import static com.android.server.autofill.Helper.sVerbose; import android.annotation.NonNull; import android.annotation.Nullable; import android.content.ComponentName; import android.content.Context; +import android.content.Intent; import android.content.IntentSender; -import android.os.IBinder; +import android.os.Handler; import android.os.ICancellationSignal; import android.os.RemoteException; import android.service.autofill.AutofillService; @@ -39,19 +39,30 @@ import android.service.autofill.SaveRequest; import android.text.format.DateUtils; import android.util.Slog; -import com.android.internal.infra.AbstractSinglePendingRequestRemoteService; +import com.android.internal.infra.AbstractRemoteService; +import com.android.internal.infra.ServiceConnector; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; -final class RemoteFillService - extends AbstractSinglePendingRequestRemoteService<RemoteFillService, IAutoFillService> { +final class RemoteFillService extends ServiceConnector.Impl<IAutoFillService> { + + private static final String TAG = "RemoteFillService"; private static final long TIMEOUT_IDLE_BIND_MILLIS = 5 * DateUtils.SECOND_IN_MILLIS; private static final long TIMEOUT_REMOTE_REQUEST_MILLIS = 5 * DateUtils.SECOND_IN_MILLIS; private final FillServiceCallbacks mCallbacks; + private final Object mLock = new Object(); + private CompletableFuture<FillResponse> mPendingFillRequest; + private int mPendingFillRequestId = INVALID_REQUEST_ID; + private final ComponentName mComponentName; - public interface FillServiceCallbacks extends VultureCallback<RemoteFillService> { + public interface FillServiceCallbacks + extends AbstractRemoteService.VultureCallback<RemoteFillService> { void onFillRequestSuccess(int requestId, @Nullable FillResponse response, @NonNull String servicePackageName, int requestFlags); void onFillRequestFailure(int requestId, @Nullable CharSequence message); @@ -65,38 +76,44 @@ final class RemoteFillService RemoteFillService(Context context, ComponentName componentName, int userId, FillServiceCallbacks callbacks, boolean bindInstantServiceAllowed) { - super(context, AutofillService.SERVICE_INTERFACE, componentName, userId, callbacks, - context.getMainThreadHandler(), Context.BIND_ALLOW_BACKGROUND_ACTIVITY_STARTS - | (bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0), sVerbose); + super(context, new Intent(AutofillService.SERVICE_INTERFACE).setComponent(componentName), + Context.BIND_ALLOW_BACKGROUND_ACTIVITY_STARTS + | (bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0), + userId, IAutoFillService.Stub::asInterface); mCallbacks = callbacks; + mComponentName = componentName; } - @Override // from AbstractRemoteService - protected void handleOnConnectedStateChanged(boolean state) { - if (mService == null) { - Slog.w(mTag, "onConnectedStateChanged(): null service"); - return; - } + @Override // from ServiceConnector.Impl + protected void onServiceConnectionStatusChanged(IAutoFillService service, boolean connected) { try { - mService.onConnectedStateChanged(state); + service.onConnectedStateChanged(connected); } catch (Exception e) { - Slog.w(mTag, "Exception calling onConnectedStateChanged(" + state + "): " + e); + Slog.w(TAG, "Exception calling onConnectedStateChanged(" + connected + "): " + e); } } - @Override // from AbstractRemoteService - protected IAutoFillService getServiceInterface(IBinder service) { - return IAutoFillService.Stub.asInterface(service); + private void dispatchCancellationSignal(@Nullable ICancellationSignal signal) { + if (signal == null) { + return; + } + try { + signal.cancel(); + } catch (RemoteException e) { + Slog.e(TAG, "Error requesting a cancellation", e); + } } - @Override // from AbstractRemoteService - protected long getTimeoutIdleBindMillis() { + @Override // from ServiceConnector.Impl + protected long getAutoDisconnectTimeoutMs() { return TIMEOUT_IDLE_BIND_MILLIS; } - @Override // from AbstractRemoteService - protected long getRemoteRequestMillis() { - return TIMEOUT_REMOTE_REQUEST_MILLIS; + @Override // from ServiceConnector.Impl + public void addLast(Job<IAutoFillService, ?> iAutoFillServiceJob) { + // Only maintain single request at a time + cancelPendingJobs(); + super.addLast(iAutoFillServiceJob); } /** @@ -105,261 +122,109 @@ final class RemoteFillService * <p>This can be used when the request is unnecessary or will be superceeded by a request that * will soon be queued. * - * @return the future id of the canceled request, or {@link FillRequest#INVALID_REQUEST_ID} if - * no {@link PendingFillRequest} was canceled. + * @return the id of the canceled request, or {@link FillRequest#INVALID_REQUEST_ID} if no + * {@link FillRequest} was canceled. */ - public CompletableFuture<Integer> cancelCurrentRequest() { - return CompletableFuture.supplyAsync(() -> { - if (isDestroyed()) { - return INVALID_REQUEST_ID; - } - - BasePendingRequest<RemoteFillService, IAutoFillService> canceledRequest = - handleCancelPendingRequest(); - return canceledRequest instanceof PendingFillRequest - ? ((PendingFillRequest) canceledRequest).mRequest.getId() + public int cancelCurrentRequest() { + synchronized (mLock) { + return mPendingFillRequest != null && mPendingFillRequest.cancel(false) + ? mPendingFillRequestId : INVALID_REQUEST_ID; - }, mHandler::post); - } - - public void onFillRequest(@NonNull FillRequest request) { - scheduleRequest(new PendingFillRequest(request, this)); - } - - public void onSaveRequest(@NonNull SaveRequest request) { - scheduleRequest(new PendingSaveRequest(request, this)); - } - - private boolean handleResponseCallbackCommon( - @NonNull PendingRequest<RemoteFillService, IAutoFillService> pendingRequest) { - if (isDestroyed()) return false; - - if (mPendingRequest == pendingRequest) { - mPendingRequest = null; } - return true; } - private void dispatchOnFillRequestSuccess(@NonNull PendingFillRequest pendingRequest, - @Nullable FillResponse response, int requestFlags) { - mHandler.post(() -> { - if (handleResponseCallbackCommon(pendingRequest)) { - mCallbacks.onFillRequestSuccess(pendingRequest.mRequest.getId(), response, - mComponentName.getPackageName(), requestFlags); - } - }); - } - - private void dispatchOnFillRequestFailure(@NonNull PendingFillRequest pendingRequest, - @Nullable CharSequence message) { - mHandler.post(() -> { - if (handleResponseCallbackCommon(pendingRequest)) { - mCallbacks.onFillRequestFailure(pendingRequest.mRequest.getId(), message); - } - }); - } - - private void dispatchOnFillRequestTimeout(@NonNull PendingFillRequest pendingRequest) { - mHandler.post(() -> { - if (handleResponseCallbackCommon(pendingRequest)) { - mCallbacks.onFillRequestTimeout(pendingRequest.mRequest.getId()); - } - }); - } - - private void dispatchOnFillTimeout(@NonNull ICancellationSignal cancellationSignal) { - mHandler.post(() -> { - try { - cancellationSignal.cancel(); - } catch (RemoteException e) { - Slog.w(mTag, "Error calling cancellation signal: " + e); - } - }); - } - - private void dispatchOnSaveRequestSuccess(PendingSaveRequest pendingRequest, - IntentSender intentSender) { - mHandler.post(() -> { - if (handleResponseCallbackCommon(pendingRequest)) { - mCallbacks.onSaveRequestSuccess(mComponentName.getPackageName(), intentSender); - } - }); - } + public void onFillRequest(@NonNull FillRequest request) { + AtomicReference<ICancellationSignal> cancellationSink = new AtomicReference<>(); + AtomicReference<CompletableFuture<FillResponse>> futureRef = new AtomicReference<>(); - private void dispatchOnSaveRequestFailure(PendingSaveRequest pendingRequest, - @Nullable CharSequence message) { - mHandler.post(() -> { - if (handleResponseCallbackCommon(pendingRequest)) { - mCallbacks.onSaveRequestFailure(message, mComponentName.getPackageName()); + CompletableFuture<FillResponse> connectThenFillRequest = postAsync(remoteService -> { + if (sVerbose) { + Slog.v(TAG, "calling onFillRequest() for id=" + request.getId()); } - }); - } - - private static final class PendingFillRequest - extends PendingRequest<RemoteFillService, IAutoFillService> { - private final FillRequest mRequest; - private final IFillCallback mCallback; - private ICancellationSignal mCancellation; - - public PendingFillRequest(FillRequest request, RemoteFillService service) { - super(service); - mRequest = request; - mCallback = new IFillCallback.Stub() { + CompletableFuture<FillResponse> fillRequest = new CompletableFuture<>(); + remoteService.onFillRequest(request, new IFillCallback.Stub() { @Override public void onCancellable(ICancellationSignal cancellation) { - synchronized (mLock) { - final boolean cancelled; - synchronized (mLock) { - mCancellation = cancellation; - cancelled = isCancelledLocked(); - } - if (cancelled) { - try { - cancellation.cancel(); - } catch (RemoteException e) { - Slog.e(mTag, "Error requesting a cancellation", e); - } - } + CompletableFuture<FillResponse> future = futureRef.get(); + if (future != null && future.isCancelled()) { + dispatchCancellationSignal(cancellation); + } else { + cancellationSink.set(cancellation); } } @Override public void onSuccess(FillResponse response) { - if (!finish()) return; - - final RemoteFillService remoteService = getService(); - if (remoteService != null) { - remoteService.dispatchOnFillRequestSuccess(PendingFillRequest.this, - response, request.getFlags()); - } + fillRequest.complete(response); } @Override public void onFailure(int requestId, CharSequence message) { - if (!finish()) return; - - final RemoteFillService remoteService = getService(); - if (remoteService != null) { - remoteService.dispatchOnFillRequestFailure(PendingFillRequest.this, - message); - } - } - }; - } - - @Override - protected void onTimeout(RemoteFillService remoteService) { - // NOTE: Must make these 2 calls asynchronously, because the cancellation signal is - // handled by the service, which could block. - final ICancellationSignal cancellation; - synchronized (mLock) { - cancellation = mCancellation; - } - if (cancellation != null) { - remoteService.dispatchOnFillTimeout(cancellation); - } - remoteService.dispatchOnFillRequestTimeout(PendingFillRequest.this); - } - - @Override - public void run() { - synchronized (mLock) { - if (isCancelledLocked()) { - if (sDebug) Slog.d(mTag, "run() called after canceled: " + mRequest); - return; + fillRequest.completeExceptionally( + new RuntimeException(String.valueOf(message))); } - } - final RemoteFillService remoteService = getService(); - if (remoteService != null) { - if (sVerbose) Slog.v(mTag, "calling onFillRequest() for id=" + mRequest.getId()); - try { - remoteService.mService.onFillRequest(mRequest, mCallback); - } catch (RemoteException e) { - Slog.e(mTag, "Error calling on fill request", e); - - remoteService.dispatchOnFillRequestFailure(PendingFillRequest.this, null); - } - } + }); + return fillRequest; + }).orTimeout(TIMEOUT_REMOTE_REQUEST_MILLIS, TimeUnit.MILLISECONDS); + futureRef.set(connectThenFillRequest); + + synchronized (mLock) { + mPendingFillRequest = connectThenFillRequest; + mPendingFillRequestId = request.getId(); } - @Override - public boolean cancel() { - if (!super.cancel()) return false; - - final ICancellationSignal cancellation; + connectThenFillRequest.whenComplete((res, err) -> Handler.getMain().post(() -> { synchronized (mLock) { - cancellation = mCancellation; + mPendingFillRequest = null; + mPendingFillRequestId = INVALID_REQUEST_ID; } - if (cancellation != null) { - try { - cancellation.cancel(); - } catch (RemoteException e) { - Slog.e(mTag, "Error cancelling a fill request", e); + if (err == null) { + mCallbacks.onFillRequestSuccess(request.getId(), res, + mComponentName.getPackageName(), request.getFlags()); + } else { + Slog.e(TAG, "Error calling on fill request", err); + if (err instanceof TimeoutException) { + dispatchCancellationSignal(cancellationSink.get()); + mCallbacks.onFillRequestTimeout(request.getId()); + } else if (err instanceof CancellationException) { + dispatchCancellationSignal(cancellationSink.get()); + } else { + mCallbacks.onFillRequestFailure(request.getId(), err.getMessage()); } } - return true; - } + })); } - private static final class PendingSaveRequest - extends PendingRequest<RemoteFillService, IAutoFillService> { - private final SaveRequest mRequest; - private final ISaveCallback mCallback; - - public PendingSaveRequest(@NonNull SaveRequest request, - @NonNull RemoteFillService service) { - super(service); - mRequest = request; + public void onSaveRequest(@NonNull SaveRequest request) { + postAsync(service -> { + if (sVerbose) Slog.v(TAG, "calling onSaveRequest()"); - mCallback = new ISaveCallback.Stub() { + CompletableFuture<IntentSender> save = new CompletableFuture<>(); + service.onSaveRequest(request, new ISaveCallback.Stub() { @Override public void onSuccess(IntentSender intentSender) { - if (!finish()) return; - - final RemoteFillService remoteService = getService(); - if (remoteService != null) { - remoteService.dispatchOnSaveRequestSuccess(PendingSaveRequest.this, - intentSender); - } + save.complete(intentSender); } @Override public void onFailure(CharSequence message) { - if (!finish()) return; - - final RemoteFillService remoteService = getService(); - if (remoteService != null) { - remoteService.dispatchOnSaveRequestFailure(PendingSaveRequest.this, - message); - } - } - }; - } - - @Override - protected void onTimeout(RemoteFillService remoteService) { - remoteService.dispatchOnSaveRequestFailure(PendingSaveRequest.this, null); - } - - @Override - public void run() { - final RemoteFillService remoteService = getService(); - if (remoteService != null) { - if (sVerbose) Slog.v(mTag, "calling onSaveRequest()"); - try { - remoteService.mService.onSaveRequest(mRequest, mCallback); - } catch (RemoteException e) { - Slog.e(mTag, "Error calling on save request", e); - - remoteService.dispatchOnSaveRequestFailure(PendingSaveRequest.this, null); + save.completeExceptionally(new RuntimeException(String.valueOf(message))); } - } - } + }); + return save; + }).orTimeout(TIMEOUT_REMOTE_REQUEST_MILLIS, TimeUnit.MILLISECONDS) + .whenComplete((res, err) -> Handler.getMain().post(() -> { + if (err == null) { + mCallbacks.onSaveRequestSuccess(mComponentName.getPackageName(), res); + } else { + mCallbacks.onSaveRequestFailure( + mComponentName.getPackageName(), err.getMessage()); + } + })); + } - @Override - public boolean isFinal() { - return true; - } + public void destroy() { + unbind(); } } diff --git a/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java b/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java new file mode 100644 index 000000000000..80b8583759e7 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.server.autofill; + +import static com.android.server.autofill.Helper.sVerbose; + +import android.Manifest; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.UserIdInt; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.content.pm.ResolveInfo; +import android.content.pm.ServiceInfo; +import android.os.Bundle; +import android.os.IBinder; +import android.os.RemoteCallback; +import android.service.autofill.IInlineSuggestionRenderService; +import android.service.autofill.IInlineSuggestionUiCallback; +import android.service.autofill.InlinePresentation; +import android.service.autofill.InlineSuggestionRenderService; +import android.util.Slog; + +import com.android.internal.infra.AbstractMultiplePendingRequestsRemoteService; + +/** + * Remote service to help connect to InlineSuggestionRenderService in ExtServices. + */ +public final class RemoteInlineSuggestionRenderService extends + AbstractMultiplePendingRequestsRemoteService<RemoteInlineSuggestionRenderService, + IInlineSuggestionRenderService> { + + private static final String TAG = "RemoteInlineSuggestionRenderService"; + + private final long mIdleUnbindTimeoutMs = PERMANENT_BOUND_TIMEOUT_MS; + + RemoteInlineSuggestionRenderService(Context context, ComponentName componentName, + String serviceInterface, int userId, InlineSuggestionRenderCallbacks callback, + boolean bindInstantServiceAllowed, boolean verbose) { + super(context, serviceInterface, componentName, userId, callback, + context.getMainThreadHandler(), + bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0, verbose, + /* initialCapacity= */ 2); + + ensureBound(); + } + + @Override // from AbstractRemoteService + protected IInlineSuggestionRenderService getServiceInterface(@NonNull IBinder service) { + return IInlineSuggestionRenderService.Stub.asInterface(service); + } + + @Override // from AbstractRemoteService + protected long getTimeoutIdleBindMillis() { + return mIdleUnbindTimeoutMs; + } + + @Override // from AbstractRemoteService + protected void handleOnConnectedStateChanged(boolean connected) { + if (connected && getTimeoutIdleBindMillis() != PERMANENT_BOUND_TIMEOUT_MS) { + scheduleUnbind(); + } + super.handleOnConnectedStateChanged(connected); + } + + public void ensureBound() { + scheduleBind(); + } + + /** + * Called by {@link Session} to generate a call to the + * {@link RemoteInlineSuggestionRenderService} to request rendering a slice . + */ + public void renderSuggestion(@NonNull IInlineSuggestionUiCallback callback, + @NonNull InlinePresentation presentation, int width, int height, + @Nullable IBinder hostInputToken, int displayId, int userId, int sessionId) { + scheduleAsyncRequest((s) -> s.renderSuggestion(callback, presentation, width, height, + hostInputToken, displayId, userId, sessionId)); + } + + /** + * Gets the inline suggestions renderer info as a {@link Bundle}. + */ + public void getInlineSuggestionsRendererInfo(@NonNull RemoteCallback callback) { + scheduleAsyncRequest((s) -> s.getInlineSuggestionsRendererInfo(callback)); + } + + /** + * Destroys the remote inline suggestion views associated with the given user id and session id. + */ + public void destroySuggestionViews(int userId, int sessionId) { + scheduleAsyncRequest((s) -> s.destroySuggestionViews(userId, sessionId)); + } + + @Nullable + private static ServiceInfo getServiceInfo(Context context, int userId) { + final String packageName = + context.getPackageManager().getServicesSystemSharedLibraryPackageName(); + if (packageName == null) { + Slog.w(TAG, "no external services package!"); + return null; + } + + final Intent intent = new Intent(InlineSuggestionRenderService.SERVICE_INTERFACE); + intent.setPackage(packageName); + final ResolveInfo resolveInfo = context.getPackageManager().resolveServiceAsUser(intent, + PackageManager.GET_SERVICES | PackageManager.GET_META_DATA, userId); + final ServiceInfo serviceInfo = resolveInfo == null ? null : resolveInfo.serviceInfo; + if (resolveInfo == null || serviceInfo == null) { + Slog.w(TAG, "No valid components found."); + return null; + } + + if (!Manifest.permission.BIND_INLINE_SUGGESTION_RENDER_SERVICE + .equals(serviceInfo.permission)) { + Slog.w(TAG, serviceInfo.name + " does not require permission " + + Manifest.permission.BIND_INLINE_SUGGESTION_RENDER_SERVICE); + return null; + } + + return serviceInfo; + } + + @Nullable + public static ComponentName getServiceComponentName(Context context, @UserIdInt int userId) { + final ServiceInfo serviceInfo = getServiceInfo(context, userId); + if (serviceInfo == null) return null; + + final ComponentName componentName = new ComponentName(serviceInfo.packageName, + serviceInfo.name); + + if (sVerbose) Slog.v(TAG, "getServiceComponentName(): " + componentName); + return componentName; + } + + interface InlineSuggestionRenderCallbacks + extends VultureCallback<RemoteInlineSuggestionRenderService> { + // NOTE: so far we don't need to notify the callback implementation + // (AutofillManagerServiceImpl) of the request results (success, timeouts, etc..), so this + // callback interface is empty. + } +} diff --git a/services/autofill/java/com/android/server/autofill/Session.java b/services/autofill/java/com/android/server/autofill/Session.java index 7d7c5703dd5d..1970b5774bbb 100644 --- a/services/autofill/java/com/android/server/autofill/Session.java +++ b/services/autofill/java/com/android/server/autofill/Session.java @@ -19,7 +19,9 @@ package com.android.server.autofill; import static android.service.autofill.AutofillFieldClassificationService.EXTRA_SCORES; import static android.service.autofill.FillRequest.FLAG_MANUAL_REQUEST; import static android.service.autofill.FillRequest.FLAG_PASSWORD_INPUT_TYPE; +import static android.service.autofill.FillRequest.FLAG_VIEW_NOT_FOCUSED; import static android.service.autofill.FillRequest.INVALID_REQUEST_ID; +import static android.view.autofill.AutofillManager.ACTION_RESPONSE_EXPIRED; import static android.view.autofill.AutofillManager.ACTION_START_SESSION; import static android.view.autofill.AutofillManager.ACTION_VALUE_CHANGED; import static android.view.autofill.AutofillManager.ACTION_VIEW_ENTERED; @@ -71,6 +73,7 @@ import android.service.autofill.FieldClassificationUserData; import android.service.autofill.FillContext; import android.service.autofill.FillRequest; import android.service.autofill.FillResponse; +import android.service.autofill.InlinePresentation; import android.service.autofill.InternalSanitizer; import android.service.autofill.InternalValidator; import android.service.autofill.SaveInfo; @@ -81,6 +84,7 @@ import android.text.TextUtils; import android.util.ArrayMap; import android.util.ArraySet; import android.util.LocalLog; +import android.util.Log; import android.util.Slog; import android.util.SparseArray; import android.util.TimeUtils; @@ -91,6 +95,7 @@ import android.view.autofill.AutofillManager.SmartSuggestionMode; import android.view.autofill.AutofillValue; import android.view.autofill.IAutoFillManagerClient; import android.view.autofill.IAutofillWindowPresenter; +import android.view.inputmethod.InlineSuggestionsRequest; import com.android.internal.R; import com.android.internal.annotations.GuardedBy; @@ -98,7 +103,9 @@ import com.android.internal.logging.MetricsLogger; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; import com.android.internal.util.ArrayUtils; import com.android.server.autofill.ui.AutoFillUI; +import com.android.server.autofill.ui.InlineFillUi; import com.android.server.autofill.ui.PendingUi; +import com.android.server.inputmethod.InputMethodManagerInternal; import java.io.PrintWriter; import java.util.ArrayList; @@ -107,7 +114,11 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Function; /** * A session for a given activity. @@ -134,7 +145,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState private final MetricsLogger mMetricsLogger = new MetricsLogger(); - private static AtomicInteger sIdCounter = new AtomicInteger(); + static final int AUGMENTED_AUTOFILL_REQUEST_ID = 1; + + private static AtomicInteger sIdCounter = new AtomicInteger(2); /** * ID of the session. @@ -143,6 +156,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState */ public final int id; + /** userId the session belongs to */ + public final int userId; + /** uid the session is for */ public final int uid; @@ -250,6 +266,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState @GuardedBy("mLock") private final LocalLog mWtfHistory; + @GuardedBy("mLock") + private boolean mExpiredResponse; + /** * Map of {@link MetricsEvent#AUTOFILL_REQUEST} metrics, keyed by fill request id. */ @@ -291,10 +310,106 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState @GuardedBy("mLock") private boolean mForAugmentedAutofillOnly; + @Nullable + private final AutofillInlineSessionController mInlineSessionController; + /** * Receiver of assist data from the app's {@link Activity}. */ - private final IAssistDataReceiver mAssistReceiver = new IAssistDataReceiver.Stub() { + private final AssistDataReceiverImpl mAssistReceiver = new AssistDataReceiverImpl(); + + void onSwitchInputMethodLocked() { + // One caveat is that for the case where the focus is on a field for which regular autofill + // returns null, and augmented autofill is triggered, and then the user switches the input + // method. Tapping on the field again will not trigger a new augmented autofill request. + // This may be fixed by adding more checks such as whether mCurrentViewId is null. + if (mExpiredResponse) { + return; + } + if (shouldResetSessionStateOnInputMethodSwitch()) { + // Set the old response expired, so the next action (ACTION_VIEW_ENTERED) can trigger + // a new fill request. + mExpiredResponse = true; + // Clear the augmented autofillable ids so augmented autofill will trigger again. + mAugmentedAutofillableIds = null; + // In case the field is augmented autofill only, we clear the current view id, so that + // we won't skip view entered due to same view entered, for the augmented autofill. + if (mForAugmentedAutofillOnly) { + mCurrentViewId = null; + } + } + } + + private boolean shouldResetSessionStateOnInputMethodSwitch() { + // One of below cases will need a new fill request to update the inline spec for the new + // input method. + // 1. The autofill provider supports inline suggestion and the render service is available. + // 2. Had triggered the augmented autofill and the render service is available. Whether the + // augmented autofill triggered by: + // a. Augmented autofill only + // b. The autofill provider respond null + if (mService.getRemoteInlineSuggestionRenderServiceLocked() == null) { + return false; + } + + if (isInlineSuggestionsEnabledByAutofillProviderLocked()) { + return true; + } + + final ViewState state = mViewStates.get(mCurrentViewId); + if (state != null + && (state.getState() & ViewState.STATE_TRIGGERED_AUGMENTED_AUTOFILL) != 0) { + return true; + } + + return false; + } + + /** + * TODO(b/151867668): improve how asynchronous data dependencies are handled, without using + * CountDownLatch. + */ + private final class AssistDataReceiverImpl extends IAssistDataReceiver.Stub { + + @GuardedBy("mLock") + private InlineSuggestionsRequest mPendingInlineSuggestionsRequest; + @GuardedBy("mLock") + private FillRequest mPendingFillRequest; + @GuardedBy("mLock") + private CountDownLatch mCountDownLatch = new CountDownLatch(0); + + @Nullable Consumer<InlineSuggestionsRequest> newAutofillRequestLocked(ViewState viewState, + boolean isInlineRequest) { + mCountDownLatch = new CountDownLatch(isInlineRequest ? 2 : 1); + mPendingFillRequest = null; + mPendingInlineSuggestionsRequest = null; + return isInlineRequest ? (inlineSuggestionsRequest) -> { + synchronized (mLock) { + if (mCountDownLatch.getCount() == 0) { + return; + } + mPendingInlineSuggestionsRequest = inlineSuggestionsRequest; + mCountDownLatch.countDown(); + maybeRequestFillLocked(); + viewState.resetState(ViewState.STATE_PENDING_CREATE_INLINE_REQUEST); + } + } : null; + } + + void maybeRequestFillLocked() { + if (mCountDownLatch.getCount() > 0 || mPendingFillRequest == null) { + return; + } + if (mPendingInlineSuggestionsRequest != null) { + mPendingFillRequest = new FillRequest(mPendingFillRequest.getId(), + mPendingFillRequest.getFillContexts(), mPendingFillRequest.getClientState(), + mPendingFillRequest.getFlags(), mPendingInlineSuggestionsRequest); + } + mRemoteFillService.onFillRequest(mPendingFillRequest); + mPendingInlineSuggestionsRequest = null; + mPendingFillRequest = null; + } + @Override public void onHandleAssistData(Bundle resultData) throws RemoteException { if (mRemoteFillService == null) { @@ -302,6 +417,13 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState + "mForAugmentedAutofillOnly: %s", mForAugmentedAutofillOnly); return; } + // Keeps to prevent it is cleared on multiple threads. + final AutofillId currentViewId = mCurrentViewId; + if (currentViewId == null) { + Slog.w(TAG, "No current view id - session might have finished"); + return; + } + final AssistStructure structure = resultData.getParcelable(ASSIST_KEY_STRUCTURE); if (structure == null) { Slog.e(TAG, "No assist structure - app might have crashed providing it"); @@ -371,7 +493,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState if (mContexts == null) { mContexts = new ArrayList<>(1); } - mContexts.add(new FillContext(requestId, structure, mCurrentViewId)); + mContexts.add(new FillContext(requestId, structure, currentViewId)); cancelCurrentRequestLocked(); @@ -382,10 +504,23 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState final ArrayList<FillContext> contexts = mergePreviousSessionLocked(/* forSave= */ false); - request = new FillRequest(requestId, contexts, mClientState, flags); + request = new FillRequest(requestId, contexts, mClientState, flags, + /*inlineSuggestionsRequest=*/null); + + if (mCountDownLatch.getCount() > 0) { + mPendingFillRequest = request; + mCountDownLatch.countDown(); + maybeRequestFillLocked(); + } else { + // TODO(b/151867668): ideally this case should not happen, but it was + // observed, we should figure out why and fix. + mRemoteFillService.onFillRequest(request); + } } - mRemoteFillService.onFillRequest(request); + if (mActivityToken != null) { + mService.sendActivityAssistDataToContentCapture(mActivityToken, resultData); + } } @Override @@ -547,34 +682,52 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState + "mForAugmentedAutofillOnly: %s", mForAugmentedAutofillOnly); return; } - mRemoteFillService.cancelCurrentRequest().whenComplete((canceledRequest, err) -> { - if (err != null) { - Slog.e(TAG, "cancelCurrentRequest(): unexpected exception", err); - return; - } + final int canceledRequest = mRemoteFillService.cancelCurrentRequest(); - // Remove the FillContext as there will never be a response for the service - if (canceledRequest != INVALID_REQUEST_ID && mContexts != null) { - final int numContexts = mContexts.size(); + // Remove the FillContext as there will never be a response for the service + if (canceledRequest != INVALID_REQUEST_ID && mContexts != null) { + final int numContexts = mContexts.size(); - // It is most likely the last context, hence search backwards - for (int i = numContexts - 1; i >= 0; i--) { - if (mContexts.get(i).getRequestId() == canceledRequest) { - if (sDebug) Slog.d(TAG, "cancelCurrentRequest(): id = " + canceledRequest); - mContexts.remove(i); - break; - } + // It is most likely the last context, hence search backwards + for (int i = numContexts - 1; i >= 0; i--) { + if (mContexts.get(i).getRequestId() == canceledRequest) { + if (sDebug) Slog.d(TAG, "cancelCurrentRequest(): id = " + canceledRequest); + mContexts.remove(i); + break; } } - }); + } } /** - * Reads a new structure and then request a new fill response from the fill service. + * Returns whether inline suggestions are supported by Autofill provider (not augmented + * Autofill provider). + */ + private boolean isInlineSuggestionsEnabledByAutofillProviderLocked() { + return mService.isInlineSuggestionsEnabled(); + } + + private boolean isViewFocusedLocked(int flags) { + return (flags & FLAG_VIEW_NOT_FOCUSED) == 0; + } + + /** + * Clears the existing response for the partition, reads a new structure, and then requests a + * new fill response from the fill service. + * + * <p> Also asks the IME to make an inline suggestions request if it's enabled. */ @GuardedBy("mLock") private void requestNewFillResponseLocked(@NonNull ViewState viewState, int newState, int flags) { + final FillResponse existingResponse = viewState.getResponse(); + if (existingResponse != null) { + setViewStatesLocked( + existingResponse, + ViewState.STATE_INITIAL, + /* clearResponse= */ true); + } + mExpiredResponse = false; if (mForAugmentedAutofillOnly || mRemoteFillService == null) { if (sVerbose) { Slog.v(TAG, "requestNewFillResponse(): triggering augmented autofill instead " @@ -585,10 +738,11 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState triggerAugmentedAutofillLocked(flags); return; } + viewState.setState(newState); int requestId; - + // TODO(b/158623971): Update this to prevent possible overflow do { requestId = sIdCounter.getAndIncrement(); } while (requestId == INVALID_REQUEST_ID); @@ -614,6 +768,35 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState // structure is taken. This causes only one fill request per bust of focus changes. cancelCurrentRequestLocked(); + // Only ask IME to create inline suggestions request if Autofill provider supports it and + // the render service is available except the autofill is triggered manually and the view + // is also not focused. + final RemoteInlineSuggestionRenderService remoteRenderService = + mService.getRemoteInlineSuggestionRenderServiceLocked(); + if (isInlineSuggestionsEnabledByAutofillProviderLocked() + && remoteRenderService != null + && isViewFocusedLocked(flags)) { + Consumer<InlineSuggestionsRequest> inlineSuggestionsRequestConsumer = + mAssistReceiver.newAutofillRequestLocked(viewState, + /*isInlineRequest=*/ true); + if (inlineSuggestionsRequestConsumer != null) { + final AutofillId focusedId = mCurrentViewId; + remoteRenderService.getInlineSuggestionsRendererInfo( + new RemoteCallback((extras) -> { + synchronized (mLock) { + mInlineSessionController.onCreateInlineSuggestionsRequestLocked( + focusedId, inlineSuggestionsRequestConsumer, extras); + } + }, mHandler) + ); + viewState.setState(ViewState.STATE_PENDING_CREATE_INLINE_REQUEST); + } + } else { + mAssistReceiver.newAutofillRequestLocked(viewState, + /*isInlineRequest=*/ false); + } + + // Now request the assist structure data. try { final Bundle receiverExtras = new Bundle(); receiverExtras.putInt(EXTRA_REQUEST_ID, requestId); @@ -637,12 +820,14 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState @NonNull IBinder client, boolean hasCallback, @NonNull LocalLog uiLatencyHistory, @NonNull LocalLog wtfHistory, @Nullable ComponentName serviceComponentName, @NonNull ComponentName componentName, boolean compatMode, - boolean bindInstantServiceAllowed, boolean forAugmentedAutofillOnly, int flags) { + boolean bindInstantServiceAllowed, boolean forAugmentedAutofillOnly, int flags, + @NonNull InputMethodManagerInternal inputMethodManagerInternal) { if (sessionId < 0) { wtf(null, "Non-positive sessionId: %s", sessionId); } id = sessionId; mFlags = flags; + this.userId = userId; this.taskId = taskId; this.uid = uid; mStartTime = SystemClock.elapsedRealtime(); @@ -662,6 +847,20 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState mForAugmentedAutofillOnly = forAugmentedAutofillOnly; setClientLocked(client); + mInlineSessionController = new AutofillInlineSessionController(inputMethodManagerInternal, + userId, componentName, handler, mLock, + new InlineFillUi.InlineUiEventCallback() { + @Override + public void notifyInlineUiShown(AutofillId autofillId) { + notifyFillUiShown(autofillId); + } + + @Override + public void notifyInlineUiHidden(AutofillId autofillId) { + notifyFillUiHidden(autofillId); + } + }); + mMetricsLogger.write(newLogMaker(MetricsEvent.AUTOFILL_SESSION_STARTED) .addTaggedData(MetricsEvent.FIELD_AUTOFILL_FLAGS, flags)); } @@ -773,13 +972,19 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState final long disableDuration = response.getDisableDuration(); if (disableDuration > 0) { final int flags = response.getFlags(); - if ((flags & FillResponse.FLAG_DISABLE_ACTIVITY_ONLY) != 0) { + final boolean disableActivityOnly = + (flags & FillResponse.FLAG_DISABLE_ACTIVITY_ONLY) != 0; + notifyDisableAutofillToClient(disableDuration, + disableActivityOnly ? mComponentName : null); + + if (disableActivityOnly) { mService.disableAutofillForActivity(mComponentName, disableDuration, id, mCompatMode); } else { mService.disableAutofillForApp(mComponentName.getPackageName(), disableDuration, id, mCompatMode); } + // Although "standard" autofill is disabled, it might still trigger augmented autofill if (triggerAugmentedAutofillLocked(requestFlags) != null) { mForAugmentedAutofillOnly = true; @@ -805,6 +1010,10 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState || disableDuration > 0) { // Response is "empty" from an UI point of view, need to notify client. notifyUnavailableToClient(sessionFinishedState, /* autofillableIds= */ null); + synchronized (mLock) { + mInlineSessionController.setInlineFillUiLocked( + InlineFillUi.emptyUi(mCurrentViewId)); + } } if (requestLog != null) { @@ -893,7 +1102,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState mMetricsLogger.write(log); if (intentSender != null) { if (sDebug) Slog.d(TAG, "Starting intent sender on save()"); - startIntentSender(intentSender); + startIntentSenderAndFinishSession(intentSender); } // Nothing left to do... @@ -963,7 +1172,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState // FillServiceCallbacks @Override - public void authenticate(int requestId, int datasetIndex, IntentSender intent, Bundle extras) { + public void authenticate(int requestId, int datasetIndex, IntentSender intent, Bundle extras, + boolean authenticateInline) { if (sDebug) { Slog.d(TAG, "authenticate(): requestId=" + requestId + "; datasetIdx=" + datasetIndex + "; intentSender=" + intent); @@ -987,7 +1197,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState final int authenticationId = AutofillManager.makeAuthenticationId(requestId, datasetIndex); mHandler.sendMessage(obtainMessage( Session::startAuthentication, - this, authenticationId, intent, fillInIntent)); + this, authenticationId, intent, fillInIntent, authenticateInline)); } // VultureCallback @@ -1102,29 +1312,67 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } catch (RemoteException e) { Slog.e(TAG, "Error requesting to hide fill UI", e); } + + mInlineSessionController.hideInlineSuggestionsUiLocked(id); + } + } + + // AutoFillUiCallback + @Override + public void cancelSession() { + synchronized (mLock) { + removeSelfLocked(); } } // AutoFillUiCallback @Override - public void startIntentSender(IntentSender intentSender) { + public void startIntentSenderAndFinishSession(IntentSender intentSender) { + startIntentSender(intentSender, null); + } + + // AutoFillUiCallback + @Override + public void startIntentSender(IntentSender intentSender, Intent intent) { synchronized (mLock) { if (mDestroyed) { Slog.w(TAG, "Call to Session#startIntentSender() rejected - session: " + id + " destroyed"); return; } - removeSelfLocked(); + if (intent == null) { + removeSelfLocked(); + } } mHandler.sendMessage(obtainMessage( Session::doStartIntentSender, - this, intentSender)); + this, intentSender, intent)); + } + + private void notifyFillUiHidden(@NonNull AutofillId autofillId) { + synchronized (mLock) { + try { + mClient.notifyFillUiHidden(this.id, autofillId); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending fill UI hidden notification", e); + } + } } - private void doStartIntentSender(IntentSender intentSender) { + private void notifyFillUiShown(@NonNull AutofillId autofillId) { + synchronized (mLock) { + try { + mClient.notifyFillUiShown(this.id, autofillId); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending fill UI shown notification", e); + } + } + } + + private void doStartIntentSender(IntentSender intentSender, Intent intent) { try { synchronized (mLock) { - mClient.startIntentSender(intentSender, null); + mClient.startIntentSender(intentSender, intent); } } catch (RemoteException e) { Slog.e(TAG, "Error launching auth intent", e); @@ -1138,6 +1386,11 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState + id + " destroyed"); return; } + final int requestId = AutofillManager.getRequestIdFromAuthenticationId(authenticationId); + if (requestId == AUGMENTED_AUTOFILL_REQUEST_ID) { + setAuthenticationResultForAugmentedAutofillLocked(data, authenticationId); + return; + } if (mResponses == null) { // Typically happens when app explicitly called cancel() while the service was showing // the auth UI. @@ -1145,7 +1398,6 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState removeSelf(); return; } - final int requestId = AutofillManager.getRequestIdFromAuthenticationId(authenticationId); final FillResponse authenticatedResponse = mResponses.get(requestId); if (authenticatedResponse == null || data == null) { Slog.w(TAG, "no authenticated response"); @@ -1165,6 +1417,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } } + // The client becomes invisible for the authentication, the response is effective. + mExpiredResponse = false; + final Parcelable result = data.getParcelable(AutofillManager.EXTRA_AUTHENTICATION_RESULT); final Bundle newClientState = data.getBundle(AutofillManager.EXTRA_CLIENT_STATE); if (sDebug) { @@ -1183,7 +1438,10 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState mClientState = newClientState; } final Dataset dataset = (Dataset) result; - authenticatedResponse.getDatasets().set(datasetIdx, dataset); + final Dataset oldDataset = authenticatedResponse.getDatasets().get(datasetIdx); + if (!isPinnedDataset(oldDataset)) { + authenticatedResponse.getDatasets().set(datasetIdx, dataset); + } autoFill(requestId, datasetIdx, dataset, false); } else { Slog.w(TAG, "invalid index (" + datasetIdx + ") for authentication id " @@ -1201,6 +1459,79 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } } + /** + * A dataset can potentially have multiple fields, and it's possible that some of the fields' + * has inline presentation and some don't. It's also possible that some of the fields' + * inline presentation is pinned and some isn't. So the concept of whether a dataset is + * pinned or not is ill-defined. Here we say a dataset is pinned if any of the field has a + * pinned inline presentation in the dataset. It's not ideal but hopefully it is sufficient + * for most of the cases. + */ + private static boolean isPinnedDataset(@Nullable Dataset dataset) { + if (dataset != null && dataset.getFieldIds() != null) { + final int numOfFields = dataset.getFieldIds().size(); + for (int i = 0; i < numOfFields; i++) { + final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation(i); + if (inlinePresentation != null && inlinePresentation.isPinned()) { + return true; + } + } + } + return false; + } + + @GuardedBy("mLock") + void setAuthenticationResultForAugmentedAutofillLocked(Bundle data, int authId) { + final Dataset dataset = (data == null) ? null : + data.getParcelable(AutofillManager.EXTRA_AUTHENTICATION_RESULT); + if (sDebug) { + Slog.d(TAG, "Auth result for augmented autofill: sessionId=" + id + + ", authId=" + authId + ", dataset=" + dataset); + } + if (dataset == null + || dataset.getFieldIds().size() != 1 + || dataset.getFieldIds().get(0) == null + || dataset.getFieldValues().size() != 1 + || dataset.getFieldValues().get(0) == null) { + if (sDebug) { + Slog.d(TAG, "Rejecting empty/invalid auth result"); + } + mService.resetLastAugmentedAutofillResponse(); + removeSelfLocked(); + return; + } + final List<AutofillId> fieldIds = dataset.getFieldIds(); + final List<AutofillValue> autofillValues = dataset.getFieldValues(); + final AutofillId fieldId = fieldIds.get(0); + final AutofillValue value = autofillValues.get(0); + + // Update state to ensure that after filling the field here we don't end up firing another + // autofill request that will end up showing the same suggestions to the user again. When + // the auth activity came up, the field for which the suggestions were shown lost focus and + // mCurrentViewId was cleared. We need to set mCurrentViewId back to the id of the field + // that we are filling. + fieldId.setSessionId(id); + mCurrentViewId = fieldId; + + // Notify the Augmented Autofill provider of the dataset that was selected. + final Bundle clientState = data.getBundle(AutofillManager.EXTRA_CLIENT_STATE); + mService.logAugmentedAutofillSelected(id, dataset.getId(), clientState); + + // Fill the value into the field. + if (sDebug) { + Slog.d(TAG, "Filling after auth: fieldId=" + fieldId + ", value=" + value); + } + try { + mClient.autofill(id, fieldIds, autofillValues, true); + } catch (RemoteException e) { + Slog.w(TAG, "Error filling after auth: fieldId=" + fieldId + ", value=" + value + + ", error=" + e); + } + + // Clear the suggestions since the user already accepted one of them. + mInlineSessionController.setInlineFillUiLocked(InlineFillUi.emptyUi(fieldId)); + } + @GuardedBy("mLock") void setHasCallbackLocked(boolean hasIt) { if (mDestroyed) { @@ -1869,7 +2200,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState mHandler.sendMessage(obtainMessage(Session::logSaveShown, this)); final IAutoFillManagerClient client = getClient(); - mPendingSaveUi = new PendingUi(mActivityToken, id, client); + mPendingSaveUi = new PendingUi(new Binder(), id, client); final CharSequence serviceLabel; final Drawable serviceIcon; @@ -2096,8 +2427,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState updateValuesForSaveLocked(); // Remove pending fill requests as the session is finished. - cancelCurrentRequestLocked(); + final ArrayList<FillContext> contexts = mergePreviousSessionLocked( /* forSave= */ true); final SaveRequest saveRequest = @@ -2180,15 +2511,17 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState * @param id The id of the view that is entered. * @param viewState The view that is entered. * @param flags The flag that was passed by the AutofillManager. + * + * @return {@code true} if a new fill response is requested. */ @GuardedBy("mLock") - private void requestNewFillResponseOnViewEnteredIfNecessaryLocked(@NonNull AutofillId id, + private boolean requestNewFillResponseOnViewEnteredIfNecessaryLocked(@NonNull AutofillId id, @NonNull ViewState viewState, int flags) { if ((flags & FLAG_MANUAL_REQUEST) != 0) { mForAugmentedAutofillOnly = false; if (sDebug) Slog.d(TAG, "Re-starting session on view " + id + " and flags " + flags); requestNewFillResponseLocked(viewState, ViewState.STATE_RESTARTED_SESSION, flags); - return; + return true; } // If it's not, then check if it it should start a partition. @@ -2198,12 +2531,14 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState + viewState.getStateAsString()); } requestNewFillResponseLocked(viewState, ViewState.STATE_STARTED_PARTITION, flags); + return true; } else { if (sVerbose) { Slog.v(TAG, "Not starting new partition for view " + id + ": " + viewState.getStateAsString()); } } + return false; } /** @@ -2211,11 +2546,20 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState * * @param id The id of the view that is entered * - * @return {@code true} iff a new partition should be started + * @return {@code true} if a new partition should be started */ @GuardedBy("mLock") private boolean shouldStartNewPartitionLocked(@NonNull AutofillId id) { + final ViewState currentView = mViewStates.get(id); if (mResponses == null) { + return currentView != null && (currentView.getState() + & ViewState.STATE_PENDING_CREATE_INLINE_REQUEST) == 0; + } + + if (mExpiredResponse) { + if (sDebug) { + Slog.d(TAG, "Starting a new partition because the response has expired."); + } return true; } @@ -2270,12 +2614,24 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState + id + " destroyed"); return; } + if (action == ACTION_RESPONSE_EXPIRED) { + mExpiredResponse = true; + if (sDebug) { + Slog.d(TAG, "Set the response has expired."); + } + return; + } + id.setSessionId(this.id); if (sVerbose) { Slog.v(TAG, "updateLocked(" + this.id + "): id=" + id + ", action=" + actionAsString(action) + ", flags=" + flags); } ViewState viewState = mViewStates.get(id); + if (sVerbose) { + Slog.v(TAG, "updateLocked(" + this.id + "): mCurrentViewId=" + mCurrentViewId + + ", mExpiredResponse=" + mExpiredResponse + ", viewState=" + viewState); + } if (viewState == null) { if (action == ACTION_START_SESSION || action == ACTION_VALUE_CHANGED @@ -2340,65 +2696,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState forceRemoveSelfLocked(AutofillManager.STATE_UNKNOWN_COMPAT_MODE); return; } - if (!Objects.equals(value, viewState.getCurrentValue())) { - if ((value == null || value.isEmpty()) - && viewState.getCurrentValue() != null - && viewState.getCurrentValue().isText() - && viewState.getCurrentValue().getTextValue() != null - && getSaveInfoLocked() != null) { - final int length = viewState.getCurrentValue().getTextValue().length(); - if (sDebug) { - Slog.d(TAG, "updateLocked(" + id + "): resetting value that was " - + length + " chars long"); - } - final LogMaker log = newLogMaker(MetricsEvent.AUTOFILL_VALUE_RESET) - .addTaggedData(MetricsEvent.FIELD_AUTOFILL_PREVIOUS_LENGTH, length); - mMetricsLogger.write(log); - } - - // Always update the internal state. - viewState.setCurrentValue(value); - - // Must check if this update was caused by autofilling the view, in which - // case we just update the value, but not the UI. - final AutofillValue filledValue = viewState.getAutofilledValue(); - if (filledValue != null) { - if (filledValue.equals(value)) { - if (sVerbose) { - Slog.v(TAG, "ignoring autofilled change on id " + id); - } - viewState.resetState(ViewState.STATE_CHANGED); - return; - } - else { - if ((viewState.id.equals(this.mCurrentViewId)) && - (viewState.getState() & ViewState.STATE_AUTOFILLED) != 0) { - // Remove autofilled state once field is changed after autofilling. - if (sVerbose) { - Slog.v(TAG, "field changed after autofill on id " + id); - } - viewState.resetState(ViewState.STATE_AUTOFILLED); - final ViewState currentView = mViewStates.get(mCurrentViewId); - currentView.maybeCallOnFillReady(flags); - } - } - } - - // Update the internal state... - viewState.setState(ViewState.STATE_CHANGED); - - //..and the UI - final String filterText; - if (value == null || !value.isText()) { - filterText = null; - } else { - final CharSequence text = value.getTextValue(); - // Text should never be null, but it doesn't hurt to check to avoid a - // system crash... - filterText = (text == null) ? null : text.toString(); - } - getUiForShowing().filterFillUi(filterText, this); + logIfViewClearedLocked(id, value, viewState); + updateViewStateAndUiOnValueChangedLocked(id, value, viewState, flags); } break; case ACTION_VIEW_ENTERED: @@ -2406,6 +2706,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState Slog.v(TAG, "entered on virtual child " + id + ": " + virtualBounds); } + final boolean isSameViewEntered = Objects.equals(mCurrentViewId, viewState.id); // Update the view states first... mCurrentViewId = viewState.id; if (value != null) { @@ -2417,25 +2718,32 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState return; } - if ((flags & FLAG_MANUAL_REQUEST) == 0 && mAugmentedAutofillableIds != null - && mAugmentedAutofillableIds.contains(id)) { - // View was already reported when server could not handle a response, but it - // triggered augmented autofill - - if (sDebug) Slog.d(TAG, "updateLocked(" + id + "): augmented-autofillable"); + if ((flags & FLAG_MANUAL_REQUEST) == 0) { + // Not a manual request + if (mAugmentedAutofillableIds != null && mAugmentedAutofillableIds.contains( + id)) { + // Regular autofill handled the view and returned null response, but it + // triggered augmented autofill + if (!isSameViewEntered) { + if (sDebug) Slog.d(TAG, "trigger augmented autofill."); + triggerAugmentedAutofillLocked(flags); + } else { + if (sDebug) Slog.d(TAG, "skip augmented autofill for same view."); + } + return; + } else if (mForAugmentedAutofillOnly && isSameViewEntered) { + // Regular autofill is disabled. + if (sDebug) Slog.d(TAG, "skip augmented autofill for same view."); + return; + } + } - // ...then trigger the augmented autofill UI - triggerAugmentedAutofillLocked(flags); + if (requestNewFillResponseOnViewEnteredIfNecessaryLocked(id, viewState, flags)) { return; } - requestNewFillResponseOnViewEnteredIfNecessaryLocked(id, viewState, flags); - - // Remove the UI if the ViewState has changed. - if (!Objects.equals(mCurrentViewId, viewState.id)) { - mUi.hideFillUi(this); - mCurrentViewId = viewState.id; - hideAugmentedAutofillLocked(viewState); + if (isSameViewEntered) { + return; } // If the ViewState is ready to be displayed, onReady() will be called. @@ -2446,6 +2754,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState if (sVerbose) Slog.v(TAG, "Exiting view " + id); mUi.hideFillUi(this); hideAugmentedAutofillLocked(viewState); + // We don't send an empty response to IME so that it doesn't cause UI flicker + // on the IME side if it arrives before the input view is finished on the IME. + mInlineSessionController.resetInlineFillUiLocked(); mCurrentViewId = null; } break; @@ -2475,6 +2786,132 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState return ArrayUtils.contains(response.getIgnoredIds(), id); } + @GuardedBy("mLock") + private void logIfViewClearedLocked(AutofillId id, AutofillValue value, ViewState viewState) { + if ((value == null || value.isEmpty()) + && viewState.getCurrentValue() != null + && viewState.getCurrentValue().isText() + && viewState.getCurrentValue().getTextValue() != null + && getSaveInfoLocked() != null) { + final int length = viewState.getCurrentValue().getTextValue().length(); + if (sDebug) { + Slog.d(TAG, "updateLocked(" + id + "): resetting value that was " + + length + " chars long"); + } + final LogMaker log = newLogMaker(MetricsEvent.AUTOFILL_VALUE_RESET) + .addTaggedData(MetricsEvent.FIELD_AUTOFILL_PREVIOUS_LENGTH, length); + mMetricsLogger.write(log); + } + } + + @GuardedBy("mLock") + private void updateViewStateAndUiOnValueChangedLocked(AutofillId id, AutofillValue value, + ViewState viewState, int flags) { + final String textValue; + if (value == null || !value.isText()) { + textValue = null; + } else { + final CharSequence text = value.getTextValue(); + // Text should never be null, but it doesn't hurt to check to avoid a + // system crash... + textValue = (text == null) ? null : text.toString(); + } + updateFilteringStateOnValueChangedLocked(textValue, viewState); + + viewState.setCurrentValue(value); + + final String filterText = textValue; + + final AutofillValue filledValue = viewState.getAutofilledValue(); + if (filledValue != null) { + if (filledValue.equals(value)) { + // When the update is caused by autofilling the view, just update the + // value, not the UI. + if (sVerbose) { + Slog.v(TAG, "ignoring autofilled change on id " + id); + } + // TODO(b/156099633): remove this once framework gets out of business of resending + // inline suggestions when IME visibility changes. + mInlineSessionController.hideInlineSuggestionsUiLocked(viewState.id); + viewState.resetState(ViewState.STATE_CHANGED); + return; + } else if ((viewState.id.equals(this.mCurrentViewId)) + && (viewState.getState() & ViewState.STATE_AUTOFILLED) != 0) { + // Remove autofilled state once field is changed after autofilling. + if (sVerbose) { + Slog.v(TAG, "field changed after autofill on id " + id); + } + viewState.resetState(ViewState.STATE_AUTOFILLED); + final ViewState currentView = mViewStates.get(mCurrentViewId); + currentView.maybeCallOnFillReady(flags); + } + } + + if (viewState.id.equals(this.mCurrentViewId) + && (viewState.getState() & ViewState.STATE_INLINE_SHOWN) != 0) { + if ((viewState.getState() & ViewState.STATE_INLINE_DISABLED) != 0) { + mInlineSessionController.disableFilterMatching(viewState.id); + } + mInlineSessionController.filterInlineFillUiLocked(mCurrentViewId, filterText); + } else if (viewState.id.equals(this.mCurrentViewId) + && (viewState.getState() & ViewState.STATE_TRIGGERED_AUGMENTED_AUTOFILL) != 0) { + if (!TextUtils.isEmpty(filterText)) { + // TODO: we should be able to replace this with controller#filterInlineFillUiLocked + // to accomplish filtering for augmented autofill. + mInlineSessionController.hideInlineSuggestionsUiLocked(mCurrentViewId); + } + } + + viewState.setState(ViewState.STATE_CHANGED); + getUiForShowing().filterFillUi(filterText, this); + } + + /** + * Disable filtering of inline suggestions for further text changes in this view if any + * character was removed earlier and now any character is being added. Such behaviour may + * indicate the IME attempting to probe the potentially sensitive content of inline suggestions. + */ + @GuardedBy("mLock") + private void updateFilteringStateOnValueChangedLocked(@Nullable String newTextValue, + ViewState viewState) { + if (newTextValue == null) { + // Don't just return here, otherwise the IME can circumvent this logic using non-text + // values. + newTextValue = ""; + } + final AutofillValue currentValue = viewState.getCurrentValue(); + final String currentTextValue; + if (currentValue == null || !currentValue.isText()) { + currentTextValue = ""; + } else { + currentTextValue = currentValue.getTextValue().toString(); + } + + if ((viewState.getState() & ViewState.STATE_CHAR_REMOVED) == 0) { + if (!containsCharsInOrder(newTextValue, currentTextValue)) { + viewState.setState(ViewState.STATE_CHAR_REMOVED); + } + } else if (!containsCharsInOrder(currentTextValue, newTextValue)) { + // Characters were added or replaced. + viewState.setState(ViewState.STATE_INLINE_DISABLED); + } + } + + /** + * Returns true if {@code s1} contains all characters of {@code s2}, in order. + */ + private static boolean containsCharsInOrder(String s1, String s2) { + int prevIndex = -1; + for (char ch : s2.toCharArray()) { + int index = TextUtils.indexOf(s1, ch, prevIndex + 1); + if (index == -1) { + return false; + } + prevIndex = index; + } + return true; + } + @Override public void onFillReady(@NonNull FillResponse response, @NonNull AutofillId filledId, @Nullable AutofillValue value) { @@ -2501,10 +2938,26 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState wtf(null, "onFillReady(): no service label or icon"); return; } + + if (response.supportsInlineSuggestions()) { + synchronized (mLock) { + if (requestShowInlineSuggestionsLocked(response, filterText)) { + final ViewState currentView = mViewStates.get(mCurrentViewId); + currentView.setState(ViewState.STATE_INLINE_SHOWN); + //TODO(b/137800469): Fix it to log showed only when IME asks for inflation, + // rather than here where framework sends back the response. + mService.logDatasetShown(id, mClientState); + return; + } + } + } + getUiForShowing().showFillUi(filledId, response, filterText, mService.getServicePackageName(), mComponentName, serviceLabel, serviceIcon, this, id, mCompatMode); + mService.logDatasetShown(id, mClientState); + synchronized (mLock) { if (mUiShownTime == 0) { // Log first time UI is shown. @@ -2530,6 +2983,42 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } } + /** + * Returns whether we made a request to show inline suggestions. + */ + private boolean requestShowInlineSuggestionsLocked(@NonNull FillResponse response, + @Nullable String filterText) { + if (mCurrentViewId == null) { + Log.w(TAG, "requestShowInlineSuggestionsLocked(): no view currently focused"); + return false; + } + final AutofillId focusedId = mCurrentViewId; + + final Optional<InlineSuggestionsRequest> inlineSuggestionsRequest = + mInlineSessionController.getInlineSuggestionsRequestLocked(); + if (!inlineSuggestionsRequest.isPresent()) { + Log.w(TAG, "InlineSuggestionsRequest unavailable"); + return false; + } + + final RemoteInlineSuggestionRenderService remoteRenderService = + mService.getRemoteInlineSuggestionRenderServiceLocked(); + if (remoteRenderService == null) { + Log.w(TAG, "RemoteInlineSuggestionRenderService not found"); + return false; + } + + InlineFillUi inlineFillUi = InlineFillUi.forAutofill( + inlineSuggestionsRequest.get(), response, focusedId, filterText, + /*uiCallback*/this, /*onErrorCallback*/ () -> { + synchronized (mLock) { + mInlineSessionController.setInlineFillUiLocked( + InlineFillUi.emptyUi(focusedId)); + } + }, remoteRenderService, userId, id); + return mInlineSessionController.setInlineFillUiLocked(inlineFillUi); + } + boolean isDestroyed() { synchronized (mLock) { return mDestroyed; @@ -2558,6 +3047,17 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } } + private void notifyDisableAutofillToClient(long disableDuration, ComponentName componentName) { + synchronized (mLock) { + if (mCurrentViewId == null) return; + try { + mClient.notifyDisableAutofill(disableDuration, componentName); + } catch (RemoteException e) { + Slog.e(TAG, "Error notifying client disable autofill: id=" + mCurrentViewId, e); + } + } + } + @GuardedBy("mLock") private void updateTrackedIdsLocked() { // Only track the views of the last response as only those are reported back to the @@ -2699,12 +3199,15 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState notifyUnavailableToClient(AutofillManager.STATE_FINISHED, autofillableIds); removeSelf(); } else { - if (sVerbose) { - if ((flags & FLAG_PASSWORD_INPUT_TYPE) != 0) { + if ((flags & FLAG_PASSWORD_INPUT_TYPE) != 0) { + if (sVerbose) { Slog.v(TAG, "keeping session " + id + " when service returned null and " + "augmented service is disabled for password fields. " + "AutofillableIds: " + autofillableIds); - } else { + } + mInlineSessionController.hideInlineSuggestionsUiLocked(mCurrentViewId); + } else { + if (sVerbose) { Slog.v(TAG, "keeping session " + id + " when service returned null but " + "it can be augmented. AutofillableIds: " + autofillableIds); } @@ -2721,13 +3224,16 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState /** * Tries to trigger Augmented Autofill when the standard service could not fulfill a request. * + * <p> The request may not have been sent when this method returns as it may be waiting for + * the inline suggestion request asynchronously. + * * @return callback to destroy the autofill UI, or {@code null} if not supported. */ // TODO(b/123099468): might need to call it in other places, like when the service returns a // non-null response but without datasets (for example, just SaveInfo) @GuardedBy("mLock") private Runnable triggerAugmentedAutofillLocked(int flags) { - // (TODO: b/141703197) Fix later by passing info to service. + // TODO: (b/141703197) Fix later by passing info to service. if ((flags & FLAG_PASSWORD_INPUT_TYPE) != 0) { return null; } @@ -2766,19 +3272,13 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState final boolean isWhitelisted = mService .isWhitelistedForAugmentedAutofillLocked(mComponentName); - final String historyItem = - "aug:id=" + id + " u=" + uid + " m=" + mode - + " a=" + ComponentName.flattenToShortString(mComponentName) - + " f=" + mCurrentViewId - + " s=" + remoteService.getComponentName() - + " w=" + isWhitelisted; - mService.getMaster().logRequestLocked(historyItem); - if (!isWhitelisted) { if (sVerbose) { Slog.v(TAG, "triggerAugmentedAutofillLocked(): " + ComponentName.flattenToShortString(mComponentName) + " not whitelisted "); } + logAugmentedAutofillRequestLocked(mode, remoteService.getComponentName(), + mCurrentViewId, isWhitelisted, /* isInline= */ null); return null; } @@ -2801,11 +3301,62 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState remoteService.getComponentName().getPackageName()); mAugmentedRequestsLogs.add(log); - final AutofillId focusedId = AutofillId.withoutSession(mCurrentViewId); - - remoteService.onRequestAutofillLocked(id, mClient, taskId, mComponentName, focusedId, - currentValue); + final AutofillId focusedId = mCurrentViewId; + final Function<InlineFillUi, Boolean> inlineSuggestionsResponseCallback = + response -> { + synchronized (mLock) { + return mInlineSessionController.setInlineFillUiLocked(response); + } + }; + final Consumer<InlineSuggestionsRequest> requestAugmentedAutofill = + (inlineSuggestionsRequest) -> { + synchronized (mLock) { + logAugmentedAutofillRequestLocked(mode, remoteService.getComponentName(), + focusedId, isWhitelisted, inlineSuggestionsRequest != null); + remoteService.onRequestAutofillLocked(id, mClient, taskId, mComponentName, + AutofillId.withoutSession(focusedId), currentValue, + inlineSuggestionsRequest, inlineSuggestionsResponseCallback, + /*onErrorCallback=*/ () -> { + synchronized (mLock) { + cancelAugmentedAutofillLocked(); + + // Also cancel augmented in IME + mInlineSessionController.setInlineFillUiLocked( + InlineFillUi.emptyUi(mCurrentViewId)); + } + }, mService.getRemoteInlineSuggestionRenderServiceLocked(), userId); + } + }; + + // When the inline suggestion render service is available and the view is focused, there + // are 3 cases when augmented autofill should ask IME for inline suggestion request, + // because standard autofill flow didn't: + // 1. the field is augmented autofill only (when standard autofill provider is None or + // when it returns null response) + // 2. standard autofill provider doesn't support inline suggestion + // 3. we re-entered the autofill session and standard autofill was not re-triggered, this is + // recognized by seeing mExpiredResponse == true + final RemoteInlineSuggestionRenderService remoteRenderService = + mService.getRemoteInlineSuggestionRenderServiceLocked(); + if (remoteRenderService != null + && (mForAugmentedAutofillOnly + || !isInlineSuggestionsEnabledByAutofillProviderLocked() + || mExpiredResponse) + && isViewFocusedLocked(flags)) { + if (sDebug) Slog.d(TAG, "Create inline request for augmented autofill"); + remoteRenderService.getInlineSuggestionsRendererInfo(new RemoteCallback( + (extras) -> { + synchronized (mLock) { + mInlineSessionController.onCreateInlineSuggestionsRequestLocked( + focusedId, /*requestConsumer=*/ requestAugmentedAutofill, + extras); + } + }, mHandler)); + } else { + requestAugmentedAutofill.accept( + mInlineSessionController.getInlineSuggestionsRequestLocked().orElse(null)); + } if (mAugmentedAutofillDestroyer == null) { mAugmentedAutofillDestroyer = () -> remoteService.onDestroyAutofillWindowsRequest(); } @@ -2813,6 +3364,20 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } @GuardedBy("mLock") + private void logAugmentedAutofillRequestLocked(int mode, + ComponentName augmentedRemoteServiceName, AutofillId focusedId, boolean isWhitelisted, + Boolean isInline) { + final String historyItem = + "aug:id=" + id + " u=" + uid + " m=" + mode + + " a=" + ComponentName.flattenToShortString(mComponentName) + + " f=" + focusedId + + " s=" + augmentedRemoteServiceName + + " w=" + isWhitelisted + + " i=" + isInline; + mService.getMaster().logRequestLocked(historyItem); + } + + @GuardedBy("mLock") private void cancelAugmentedAutofillLocked() { final RemoteAugmentedAutofillService remoteService = mService .getRemoteAugmentedAutofillServiceLocked(); @@ -2909,7 +3474,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } /** - * Sets the state of all views in the given dataset and response. + * Sets the state and response of all views in the given dataset. */ @GuardedBy("mLock") private void setViewStatesLocked(@Nullable FillResponse response, @NonNull Dataset dataset, @@ -2924,10 +3489,10 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState if (datasetId != null) { viewState.setDatasetId(datasetId); } - if (response != null) { - viewState.setResponse(response); - } else if (clearResponse) { + if (clearResponse) { viewState.setResponse(null); + } else if (response != null) { + viewState.setResponse(response); } } } @@ -2968,7 +3533,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState if (generateEvent) { mService.logDatasetSelected(dataset.getId(), id, mClientState); } - + if (mCurrentViewId != null) { + mInlineSessionController.hideInlineSuggestionsUiLocked(mCurrentViewId); + } autoFillApp(dataset); return; } @@ -2983,7 +3550,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } final int authenticationId = AutofillManager.makeAuthenticationId(requestId, datasetIndex); - startAuthentication(authenticationId, dataset.getAuthentication(), fillInIntent); + startAuthentication(authenticationId, dataset.getAuthentication(), fillInIntent, + /* authenticateInline= */false); } } @@ -3007,10 +3575,11 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } private void startAuthentication(int authenticationId, IntentSender intent, - Intent fillInIntent) { + Intent fillInIntent, boolean authenticateInline) { try { synchronized (mLock) { - mClient.authenticate(id, authenticationId, intent, fillInIntent); + mClient.authenticate(id, authenticationId, intent, fillInIntent, + authenticateInline); } } catch (RemoteException e) { Slog.e(TAG, "Error launching auth intent", e); @@ -3186,6 +3755,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState final List<AutofillId> ids = new ArrayList<>(entryCount); final List<AutofillValue> values = new ArrayList<>(entryCount); boolean waitingDatasetAuth = false; + boolean hideHighlight = (entryCount == 1 + && dataset.getFieldIds().get(0).equals(mCurrentViewId)); for (int i = 0; i < entryCount; i++) { if (dataset.getFieldValues().get(i) == null) { continue; @@ -3209,7 +3780,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } if (sDebug) Slog.d(TAG, "autoFillApp(): the buck is on the app: " + dataset); - mClient.autofill(id, ids, values); + mClient.autofill(id, ids, values, hideHighlight); if (dataset.getId() != null) { if (mSelectedDatasetIds == null) { mSelectedDatasetIds = new ArrayList<>(); @@ -3247,9 +3818,19 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState if (mDestroyed) { return null; } + unlinkClientVultureLocked(); mUi.destroyAll(mPendingSaveUi, this, true); mUi.clearCallback(this); + if (mCurrentViewId != null) { + mInlineSessionController.destroyLocked(mCurrentViewId); + } + final RemoteInlineSuggestionRenderService remoteRenderService = + mService.getRemoteInlineSuggestionRenderServiceLocked(); + if (remoteRenderService != null) { + remoteRenderService.destroySuggestionViews(userId, id); + } + mDestroyed = true; // Log metrics @@ -3402,6 +3983,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState for (int i = 0; i < responseCount; i++) { if (mResponses.keyAt(i) > lastResponseId) { lastResponseIdx = i; + lastResponseId = mResponses.keyAt(i); } } } @@ -3460,6 +4042,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState return "VIEW_EXITED"; case ACTION_VALUE_CHANGED: return "VALUE_CHANGED"; + case ACTION_RESPONSE_EXPIRED: + return "RESPONSE_EXPIRED"; default: return "UNKNOWN_" + action; } diff --git a/services/autofill/java/com/android/server/autofill/ViewState.java b/services/autofill/java/com/android/server/autofill/ViewState.java index 84886f83027d..adb1e3e43731 100644 --- a/services/autofill/java/com/android/server/autofill/ViewState.java +++ b/services/autofill/java/com/android/server/autofill/ViewState.java @@ -74,6 +74,14 @@ final class ViewState { public static final int STATE_AUTOFILLED_ONCE = 0x800; /** View triggered the latest augmented autofill request. */ public static final int STATE_TRIGGERED_AUGMENTED_AUTOFILL = 0x1000; + /** Inline suggestions were shown for this View. */ + public static final int STATE_INLINE_SHOWN = 0x2000; + /** A character was removed from the View value (not by the service). */ + public static final int STATE_CHAR_REMOVED = 0x4000; + /** Showing inline suggestions is not allowed for this View. */ + public static final int STATE_INLINE_DISABLED = 0x8000; + /** The View is waiting for an inline suggestions request from IME.*/ + public static final int STATE_PENDING_CREATE_INLINE_REQUEST = 0x10000; public final AutofillId id; diff --git a/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java b/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java index fe86ab3a3f26..71c3c16a2c06 100644 --- a/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java +++ b/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java @@ -22,6 +22,7 @@ import android.annotation.NonNull; import android.annotation.Nullable; import android.content.ComponentName; import android.content.Context; +import android.content.Intent; import android.content.IntentSender; import android.graphics.drawable.Drawable; import android.metrics.LogMaker; @@ -73,17 +74,22 @@ public final class AutoFillUI { private final @NonNull OverlayControl mOverlayControl; private final @NonNull UiModeManagerInternal mUiModeMgr; + private @Nullable Runnable mCreateFillUiRunnable; + private @Nullable AutoFillUiCallback mSaveUiCallback; + public interface AutoFillUiCallback { void authenticate(int requestId, int datasetIndex, @NonNull IntentSender intent, - @Nullable Bundle extras); + @Nullable Bundle extras, boolean authenticateInline); void fill(int requestId, int datasetIndex, @NonNull Dataset dataset); void save(); void cancelSave(); void requestShowFillUi(AutofillId id, int width, int height, IAutofillWindowPresenter presenter); void requestHideFillUi(AutofillId id); - void startIntentSender(IntentSender intentSender); + void startIntentSenderAndFinishSession(IntentSender intentSender); + void startIntentSender(IntentSender intentSender, Intent intent); void dispatchUnhandledKey(AutofillId id, KeyEvent keyEvent); + void cancelSession(); } public AutoFillUI(@NonNull Context context) { @@ -96,9 +102,13 @@ public final class AutoFillUI { mHandler.post(() -> { if (mCallback != callback) { if (mCallback != null) { - hideAllUiThread(mCallback); + if (isSaveUiShowing()) { + // keeps showing the save UI + hideFillUiUiThread(callback, true); + } else { + hideAllUiThread(mCallback); + } } - mCallback = callback; } }); @@ -191,7 +201,7 @@ public final class AutoFillUI { .addTaggedData(MetricsEvent.FIELD_AUTOFILL_NUM_DATASETS, response.getDatasets() == null ? 0 : response.getDatasets().size()); - mHandler.post(() -> { + final Runnable createFillUiRunnable = () -> { if (callback != mCallback) { return; } @@ -207,7 +217,8 @@ public final class AutoFillUI { if (mCallback != null) { mCallback.authenticate(response.getRequestId(), AutofillManager.AUTHENTICATION_ID_DATASET_ID_UNDEFINED, - response.getAuthentication(), response.getClientState()); + response.getAuthentication(), response.getClientState(), + /* authenticateInline= */ false); } } @@ -253,7 +264,7 @@ public final class AutoFillUI { @Override public void startIntentSender(IntentSender intentSender) { if (mCallback != null) { - mCallback.startIntentSender(intentSender); + mCallback.startIntentSenderAndFinishSession(intentSender); } } @@ -263,8 +274,23 @@ public final class AutoFillUI { mCallback.dispatchUnhandledKey(focusedId, keyEvent); } } + + @Override + public void cancelSession() { + if (mCallback != null) { + mCallback.cancelSession(); + } + } }); - }); + }; + + if (isSaveUiShowing()) { + // postpone creating the fill UI for showing the save UI + if (sDebug) Slog.d(TAG, "postpone fill UI request.."); + mCreateFillUiRunnable = createFillUiRunnable; + } else { + mHandler.post(createFillUiRunnable); + } } /** @@ -296,23 +322,22 @@ public final class AutoFillUI { return; } hideAllUiThread(callback); + mSaveUiCallback = callback; mSaveUi = new SaveUi(mContext, pendingSaveUi, serviceLabel, serviceIcon, servicePackageName, componentName, info, valueFinder, mOverlayControl, new SaveUi.OnSaveListener() { @Override public void onSave() { log.setType(MetricsEvent.TYPE_ACTION); - hideSaveUiUiThread(mCallback); - if (mCallback != null) { - mCallback.save(); - } + hideSaveUiUiThread(callback); + callback.save(); destroySaveUiUiThread(pendingSaveUi, true); } @Override public void onCancel(IntentSender listener) { log.setType(MetricsEvent.TYPE_DISMISS); - hideSaveUiUiThread(mCallback); + hideSaveUiUiThread(callback); if (listener != null) { try { listener.sendIntent(mContext, 0, null, null, null); @@ -321,9 +346,7 @@ public final class AutoFillUI { + listener, e); } } - if (mCallback != null) { - mCallback.cancelSave(); - } + callback.cancelSave(); destroySaveUiUiThread(pendingSaveUi, true); } @@ -332,12 +355,15 @@ public final class AutoFillUI { if (log.getType() == MetricsEvent.TYPE_UNKNOWN) { log.setType(MetricsEvent.TYPE_CLOSE); - if (mCallback != null) { - mCallback.cancelSave(); - } + callback.cancelSave(); } mMetricsLogger.write(log); } + + @Override + public void startIntentSender(IntentSender intentSender, Intent intent) { + callback.startIntentSender(intentSender, intent); + } }, mUiModeMgr.isNightMode(), isUpdate, compatMode); }); } @@ -370,6 +396,10 @@ public final class AutoFillUI { mHandler.post(() -> destroyAllUiThread(pendingSaveUi, callback, notifyClient)); } + public boolean isSaveUiShowing() { + return mSaveUi == null ? false : mSaveUi.isShowing(); + } + public void dump(PrintWriter pw) { pw.println("Autofill UI"); final String prefix = " "; @@ -404,7 +434,8 @@ public final class AutoFillUI { Slog.v(TAG, "hideSaveUiUiThread(): mSaveUi=" + mSaveUi + ", callback=" + callback + ", mCallback=" + mCallback); } - if (mSaveUi != null && (callback == null || callback == mCallback)) { + + if (mSaveUi != null && mSaveUiCallback == callback) { return mSaveUi.hide(); } return null; @@ -423,6 +454,7 @@ public final class AutoFillUI { if (sDebug) Slog.d(TAG, "destroySaveUiUiThread(): " + pendingSaveUi); mSaveUi.destroy(); mSaveUi = null; + mSaveUiCallback = null; if (pendingSaveUi != null && notifyClient) { try { if (sDebug) Slog.d(TAG, "destroySaveUiUiThread(): notifying client"); @@ -431,6 +463,12 @@ public final class AutoFillUI { Slog.e(TAG, "Error notifying client to set save UI state to hidden: " + e); } } + + if (mCreateFillUiRunnable != null) { + if (sDebug) Slog.d(TAG, "start the pending fill UI request.."); + mHandler.post(mCreateFillUiRunnable); + mCreateFillUiRunnable = null; + } } @android.annotation.UiThread diff --git a/services/autofill/java/com/android/server/autofill/ui/CustomScrollView.java b/services/autofill/java/com/android/server/autofill/ui/CustomScrollView.java index e68263a51415..14bd7d78f3bf 100644 --- a/services/autofill/java/com/android/server/autofill/ui/CustomScrollView.java +++ b/services/autofill/java/com/android/server/autofill/ui/CustomScrollView.java @@ -64,24 +64,24 @@ public class CustomScrollView extends ScrollView { return; } + mWidth = MeasureSpec.getSize(widthMeasureSpec); calculateDimensions(); setMeasuredDimension(mWidth, mHeight); } private void calculateDimensions() { - if (mWidth != -1) return; + if (mHeight != -1) return; final TypedValue typedValue = new TypedValue(); final Point point = new Point(); final Context context = getContext(); - context.getDisplay().getSize(point); + context.getDisplayNoVerify().getSize(point); context.getTheme().resolveAttribute(R.attr.autofillSaveCustomSubtitleMaxHeight, typedValue, true); final View child = getChildAt(0); final int childHeight = child.getMeasuredHeight(); final int maxHeight = (int) typedValue.getFraction(point.y, point.y); - mWidth = point.x; mHeight = Math.min(childHeight, maxHeight); if (sDebug) { Slog.d(TAG, "calculateDimensions(): maxHeight=" + maxHeight diff --git a/services/autofill/java/com/android/server/autofill/ui/FillUi.java b/services/autofill/java/com/android/server/autofill/ui/FillUi.java index dbd4d8c168ba..890208720f97 100644 --- a/services/autofill/java/com/android/server/autofill/ui/FillUi.java +++ b/services/autofill/java/com/android/server/autofill/ui/FillUi.java @@ -88,6 +88,7 @@ final class FillUi { void requestHideFillUi(); void startIntentSender(IntentSender intentSender); void dispatchUnhandledKey(KeyEvent keyEvent); + void cancelSession(); } private final @NonNull Point mTempPoint = new Point(); @@ -129,9 +130,9 @@ final class FillUi { } FillUi(@NonNull Context context, @NonNull FillResponse response, - @NonNull AutofillId focusedViewId, @NonNull @Nullable String filterText, - @NonNull OverlayControl overlayControl, @NonNull CharSequence serviceLabel, - @NonNull Drawable serviceIcon, boolean nightMode, @NonNull Callback callback) { + @NonNull AutofillId focusedViewId, @Nullable String filterText, + @NonNull OverlayControl overlayControl, @NonNull CharSequence serviceLabel, + @NonNull Drawable serviceIcon, boolean nightMode, @NonNull Callback callback) { if (sVerbose) Slog.v(TAG, "nightMode: " + nightMode); mThemeId = nightMode ? THEME_ID_DARK : THEME_ID_LIGHT; mCallback = callback; @@ -164,7 +165,7 @@ final class FillUi { // In full screen we only initialize size once assuming screen size never changes if (mFullScreen) { final Point outPoint = mTempPoint; - mContext.getDisplay().getSize(outPoint); + mContext.getDisplayNoVerify().getSize(outPoint); // full with of screen and half height of screen mContentWidth = LayoutParams.MATCH_PARENT; mContentHeight = outPoint.y / 2; @@ -263,6 +264,7 @@ final class FillUi { mHeader = headerPresentation.applyWithTheme(mContext, null, clickBlocker, mThemeId); final LinearLayout headerContainer = decor.findViewById(R.id.autofill_dataset_header); + applyCancelAction(mHeader, response.getCancelIds()); if (sVerbose) Slog.v(TAG, "adding header"); headerContainer.addView(mHeader); headerContainer.setVisibility(View.VISIBLE); @@ -279,6 +281,7 @@ final class FillUi { } mFooter = footerPresentation.applyWithTheme( mContext, null, clickBlocker, mThemeId); + applyCancelAction(mFooter, response.getCancelIds()); // Footer not supported on some platform e.g. TV if (sVerbose) Slog.v(TAG, "adding footer"); footerContainer.addView(mFooter); @@ -310,6 +313,8 @@ final class FillUi { Slog.e(TAG, "Error inflating remote views", e); continue; } + // TODO: Extract the shared filtering logic here and in FillUi to a common + // method. final DatasetFieldFilter filter = dataset.getFilter(index); Pattern filterPattern = null; String valueText = null; @@ -330,6 +335,7 @@ final class FillUi { } } + applyCancelAction(view, response.getCancelIds()); items.add(new ViewItem(dataset, filterPattern, filterable, valueText, view)); } } @@ -355,6 +361,37 @@ final class FillUi { } } + private void applyCancelAction(View rootView, int[] ids) { + if (ids == null) { + return; + } + + if (sDebug) Slog.d(TAG, "fill UI has " + ids.length + " actions"); + if (!(rootView instanceof ViewGroup)) { + Slog.w(TAG, "cannot apply actions because fill UI root is not a " + + "ViewGroup: " + rootView); + return; + } + + // Apply click actions. + final ViewGroup root = (ViewGroup) rootView; + for (int i = 0; i < ids.length; i++) { + final int id = ids[i]; + final View child = root.findViewById(id); + if (child == null) { + Slog.w(TAG, "Ignoring cancel action for view " + id + + " because it's not on " + root); + continue; + } + child.setOnClickListener((v) -> { + if (sVerbose) { + Slog.v(TAG, " Cancelling session after " + v + " clicked"); + } + mCallback.cancelSession(); + }); + } + } + void requestShowFillUi() { mCallback.requestShowFillUi(mContentWidth, mContentHeight, mWindowPresenter); } @@ -522,7 +559,7 @@ final class FillUi { } private static void resolveMaxWindowSize(Context context, Point outPoint) { - context.getDisplay().getSize(outPoint); + context.getDisplayNoVerify().getSize(outPoint); final TypedValue typedValue = sTempTypedValue; context.getTheme().resolveAttribute(R.attr.autofillDatasetPickerMaxWidth, typedValue, true); @@ -567,6 +604,7 @@ final class FillUi { * Returns whether this item matches the value input by the user so it can be included * in the filtered datasets. */ + // TODO: Extract the shared filtering logic here and in FillUi to a common method. public boolean matches(CharSequence filterText) { if (TextUtils.isEmpty(filterText)) { // Always show item when the user input is empty diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java b/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java new file mode 100644 index 000000000000..7fbf4b9c590c --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sVerbose; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.os.Handler; +import android.util.Slog; + +import com.android.internal.view.inline.IInlineContentCallback; +import com.android.internal.view.inline.IInlineContentProvider; +import com.android.server.FgThread; + +/** + * We create one instance of this class for each {@link android.view.inputmethod.InlineSuggestion} + * instance. Each inline suggestion instance will only be sent to the remote IME process once. In + * case of filtering and resending the suggestion when keyboard state changes between hide and + * show, a new instance of this class will be created using {@link #copy()}, with the same backing + * {@link RemoteInlineSuggestionUi}. When the + * {@link #provideContent(int, int, IInlineContentCallback)} is called the first time (it's only + * allowed to be called at most once), the passed in width/height is used to determine whether + * the existing {@link RemoteInlineSuggestionUi} provided in the constructor can be reused, or a + * new one should be created to suit the new size requirement for the view. In normal cases, + * we should not expect the size requirement to change, although in theory the public API allows + * the IME to do that. + * + * <p>This design is to enable us to be able to reuse the backing remote view while still keeping + * the callbacks relatively well aligned. For example, if we allow multiple remote IME binder + * callbacks to call into one instance of this class, then binder A may call in with width/height + * X for which we create a view (i.e. {@link RemoteInlineSuggestionUi}) for it, + * + * See also {@link RemoteInlineSuggestionUi} for relevant information. + */ +final class InlineContentProviderImpl extends IInlineContentProvider.Stub { + + // TODO(b/153615023): consider not holding strong reference to heavy objects in this stub, to + // avoid memory leak in case the client app is holding the remote reference for a longer + // time than expected. Essentially we need strong reference in the system process to + // the member variables, but weak reference to them in the IInlineContentProvider.Stub. + + private static final String TAG = InlineContentProviderImpl.class.getSimpleName(); + + private final Handler mHandler = FgThread.getHandler();; + + @NonNull + private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector; + @Nullable + private RemoteInlineSuggestionUi mRemoteInlineSuggestionUi; + + private boolean mProvideContentCalled = false; + + InlineContentProviderImpl( + @NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector, + @Nullable RemoteInlineSuggestionUi remoteInlineSuggestionUi) { + mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector; + mRemoteInlineSuggestionUi = remoteInlineSuggestionUi; + } + + /** + * Returns a new instance of this class, with the same {@code mInlineSuggestionRenderer} and + * {@code mRemoteInlineSuggestionUi}. The latter may or may not be reusable depending on the + * size information provided when the client calls {@link #provideContent(int, int, + * IInlineContentCallback)}. + */ + @NonNull + public InlineContentProviderImpl copy() { + return new InlineContentProviderImpl(mRemoteInlineSuggestionViewConnector, + mRemoteInlineSuggestionUi); + } + + /** + * Provides a SurfacePackage associated with the inline suggestion view to the IME. If such + * view doesn't exit, then create a new one. This method should be called once per lifecycle + * of this object. Any further calls to the method will be ignored. + */ + @Override + public void provideContent(int width, int height, IInlineContentCallback callback) { + mHandler.post(() -> handleProvideContent(width, height, callback)); + } + + @Override + public void requestSurfacePackage() { + mHandler.post(this::handleGetSurfacePackage); + } + + @Override + public void onSurfacePackageReleased() { + mHandler.post(this::handleOnSurfacePackageReleased); + } + + private void handleProvideContent(int width, int height, IInlineContentCallback callback) { + if (sVerbose) Slog.v(TAG, "handleProvideContent"); + if (mProvideContentCalled) { + // This method should only be called once. + return; + } + mProvideContentCalled = true; + if (mRemoteInlineSuggestionUi == null || !mRemoteInlineSuggestionUi.match(width, height)) { + mRemoteInlineSuggestionUi = new RemoteInlineSuggestionUi( + mRemoteInlineSuggestionViewConnector, + width, height, mHandler); + } + mRemoteInlineSuggestionUi.setInlineContentCallback(callback); + mRemoteInlineSuggestionUi.requestSurfacePackage(); + } + + private void handleGetSurfacePackage() { + if (sVerbose) Slog.v(TAG, "handleGetSurfacePackage"); + if (!mProvideContentCalled || mRemoteInlineSuggestionUi == null) { + // provideContent should be called first, and remote UI should not be null. + return; + } + mRemoteInlineSuggestionUi.requestSurfacePackage(); + } + + private void handleOnSurfacePackageReleased() { + if (sVerbose) Slog.v(TAG, "handleOnSurfacePackageReleased"); + if (!mProvideContentCalled || mRemoteInlineSuggestionUi == null) { + // provideContent should be called first, and remote UI should not be null. + return; + } + mRemoteInlineSuggestionUi.surfacePackageReleased(); + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineFillUi.java b/services/autofill/java/com/android/server/autofill/ui/InlineFillUi.java new file mode 100644 index 000000000000..25e9d5c90764 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/InlineFillUi.java @@ -0,0 +1,317 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sVerbose; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.Intent; +import android.content.IntentSender; +import android.service.autofill.Dataset; +import android.service.autofill.FillResponse; +import android.service.autofill.InlinePresentation; +import android.text.TextUtils; +import android.util.Pair; +import android.util.Slog; +import android.util.SparseArray; +import android.view.autofill.AutofillId; +import android.view.autofill.AutofillValue; +import android.view.inputmethod.InlineSuggestion; +import android.view.inputmethod.InlineSuggestionsRequest; +import android.view.inputmethod.InlineSuggestionsResponse; + +import com.android.internal.view.inline.IInlineContentProvider; +import com.android.server.autofill.RemoteInlineSuggestionRenderService; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + + +/** + * UI for a particular field (i.e. {@link AutofillId}) based on an inline autofill response from + * the autofill service or the augmented autofill service. It wraps multiple inline suggestions. + * + * <p> This class is responsible for filtering the suggestions based on the filtered text. + * It'll create {@link InlineSuggestion} instances by reusing the backing remote views (from the + * renderer service) if possible. + */ +public final class InlineFillUi { + + private static final String TAG = "InlineFillUi"; + + /** + * The id of the field which the current Ui is for. + */ + @NonNull + final AutofillId mAutofillId; + + /** + * The list of inline suggestions, before applying any filtering + */ + @NonNull + private final ArrayList<InlineSuggestion> mInlineSuggestions; + + /** + * The corresponding data sets for the inline suggestions. The list may be null if the current + * Ui is the authentication UI for the response. If non-null, the size of data sets should equal + * that of inline suggestions. + */ + @Nullable + private final ArrayList<Dataset> mDatasets; + + /** + * The filter text which will be applied on the inline suggestion list before they are returned + * as a response. + */ + @Nullable + private String mFilterText; + + /** + * Whether prefix/regex based filtering is disabled. + */ + private boolean mFilterMatchingDisabled; + + /** + * Returns an empty inline autofill UI. + */ + @NonNull + public static InlineFillUi emptyUi(@NonNull AutofillId autofillId) { + return new InlineFillUi(autofillId, new SparseArray<>(), null); + } + + /** + * Returns an inline autofill UI for a field based on an Autofilll response. + */ + @NonNull + public static InlineFillUi forAutofill(@NonNull InlineSuggestionsRequest request, + @NonNull FillResponse response, + @NonNull AutofillId focusedViewId, @Nullable String filterText, + @NonNull AutoFillUI.AutoFillUiCallback uiCallback, + @NonNull Runnable onErrorCallback, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId) { + + if (InlineSuggestionFactory.responseNeedAuthentication(response)) { + InlineSuggestion inlineAuthentication = + InlineSuggestionFactory.createInlineAuthentication(request, response, + uiCallback, onErrorCallback, remoteRenderService, userId, sessionId); + return new InlineFillUi(focusedViewId, inlineAuthentication, filterText); + } else if (response.getDatasets() != null) { + SparseArray<Pair<Dataset, InlineSuggestion>> inlineSuggestions = + InlineSuggestionFactory.createAutofillInlineSuggestions(request, + response.getRequestId(), + response.getDatasets(), focusedViewId, uiCallback, onErrorCallback, + remoteRenderService, userId, sessionId); + return new InlineFillUi(focusedViewId, inlineSuggestions, filterText); + } + return new InlineFillUi(focusedViewId, new SparseArray<>(), filterText); + } + + /** + * Returns an inline autofill UI for a field based on an Autofilll response. + */ + @NonNull + public static InlineFillUi forAugmentedAutofill(@NonNull InlineSuggestionsRequest request, + @NonNull List<Dataset> datasets, + @NonNull AutofillId focusedViewId, @Nullable String filterText, + @NonNull InlineSuggestionUiCallback uiCallback, + @NonNull Runnable onErrorCallback, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId) { + SparseArray<Pair<Dataset, InlineSuggestion>> inlineSuggestions = + InlineSuggestionFactory.createAugmentedAutofillInlineSuggestions(request, datasets, + focusedViewId, + uiCallback, onErrorCallback, remoteRenderService, userId, sessionId); + return new InlineFillUi(focusedViewId, inlineSuggestions, filterText); + } + + InlineFillUi(@NonNull AutofillId autofillId, + @NonNull SparseArray<Pair<Dataset, InlineSuggestion>> inlineSuggestions, + @Nullable String filterText) { + mAutofillId = autofillId; + int size = inlineSuggestions.size(); + mDatasets = new ArrayList<>(size); + mInlineSuggestions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + Pair<Dataset, InlineSuggestion> value = inlineSuggestions.valueAt(i); + mDatasets.add(value.first); + mInlineSuggestions.add(value.second); + } + mFilterText = filterText; + } + + InlineFillUi(@NonNull AutofillId autofillId, InlineSuggestion inlineSuggestion, + @Nullable String filterText) { + mAutofillId = autofillId; + mDatasets = null; + mInlineSuggestions = new ArrayList<>(); + mInlineSuggestions.add(inlineSuggestion); + mFilterText = filterText; + } + + @NonNull + public AutofillId getAutofillId() { + return mAutofillId; + } + + public void setFilterText(@Nullable String filterText) { + mFilterText = filterText; + } + + /** + * Returns the list of filtered inline suggestions suitable for being sent to the IME. + */ + @NonNull + public InlineSuggestionsResponse getInlineSuggestionsResponse() { + final int size = mInlineSuggestions.size(); + if (size == 0) { + return new InlineSuggestionsResponse(Collections.emptyList()); + } + final List<InlineSuggestion> inlineSuggestions = new ArrayList<>(); + if (mDatasets == null || mDatasets.size() != size) { + // authentication case + for (int i = 0; i < size; i++) { + inlineSuggestions.add(copy(i, mInlineSuggestions.get(i))); + } + return new InlineSuggestionsResponse(inlineSuggestions); + } + for (int i = 0; i < size; i++) { + final Dataset dataset = mDatasets.get(i); + final int fieldIndex = dataset.getFieldIds().indexOf(mAutofillId); + if (fieldIndex < 0) { + Slog.w(TAG, "AutofillId=" + mAutofillId + " not found in dataset"); + continue; + } + final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation( + fieldIndex); + if (inlinePresentation == null) { + Slog.w(TAG, "InlinePresentation not found in dataset"); + continue; + } + if (!inlinePresentation.isPinned() // don't filter pinned suggestions + && !includeDataset(dataset, fieldIndex)) { + continue; + } + inlineSuggestions.add(copy(i, mInlineSuggestions.get(i))); + } + return new InlineSuggestionsResponse(inlineSuggestions); + } + + /** + * Returns a copy of the suggestion, that internally copies the {@link IInlineContentProvider} + * so that it's not reused by the remote IME process across different inline suggestions. + * See {@link InlineContentProviderImpl} for why this is needed. + * + * <p>Note that although it copies the {@link IInlineContentProvider}, the underlying remote + * view (in the renderer service) is still reused. + */ + @NonNull + private InlineSuggestion copy(int index, @NonNull InlineSuggestion inlineSuggestion) { + final IInlineContentProvider contentProvider = inlineSuggestion.getContentProvider(); + if (contentProvider instanceof InlineContentProviderImpl) { + // We have to create a new inline suggestion instance to ensure we don't reuse the + // same {@link IInlineContentProvider}, but the underlying views are reused when + // calling {@link InlineContentProviderImpl#copy()}. + InlineSuggestion newInlineSuggestion = new InlineSuggestion(inlineSuggestion + .getInfo(), ((InlineContentProviderImpl) contentProvider).copy()); + // The remote view is only set when the content provider is called to inflate the view, + // which happens after it's sent to the IME (i.e. not now), so we keep the latest + // content provider (through newInlineSuggestion) to make sure the next time we copy it, + // we get to reuse the view. + mInlineSuggestions.set(index, newInlineSuggestion); + return newInlineSuggestion; + } + return inlineSuggestion; + } + + // TODO: Extract the shared filtering logic here and in FillUi to a common method. + private boolean includeDataset(Dataset dataset, int fieldIndex) { + // Show everything when the user input is empty. + if (TextUtils.isEmpty(mFilterText)) { + return true; + } + + final String constraintLowerCase = mFilterText.toString().toLowerCase(); + + // Use the filter provided by the service, if available. + final Dataset.DatasetFieldFilter filter = dataset.getFilter(fieldIndex); + if (filter != null) { + Pattern filterPattern = filter.pattern; + if (filterPattern == null) { + if (sVerbose) { + Slog.v(TAG, "Explicitly disabling filter for dataset id" + dataset.getId()); + } + return false; + } + if (mFilterMatchingDisabled) { + return false; + } + return filterPattern.matcher(constraintLowerCase).matches(); + } + + final AutofillValue value = dataset.getFieldValues().get(fieldIndex); + if (value == null || !value.isText()) { + return dataset.getAuthentication() == null; + } + if (mFilterMatchingDisabled) { + return false; + } + final String valueText = value.getTextValue().toString().toLowerCase(); + return valueText.toLowerCase().startsWith(constraintLowerCase); + } + + /** + * Disables prefix/regex based filtering. Other filtering rules (see {@link + * android.service.autofill.Dataset}) still apply. + */ + public void disableFilterMatching() { + mFilterMatchingDisabled = true; + } + + /** + * Callback from the inline suggestion Ui. + */ + public interface InlineSuggestionUiCallback { + /** + * Callback to autofill a dataset to the client app. + */ + void autofill(@NonNull Dataset dataset, int datasetIndex); + + /** + * Callback to start Intent in client app. + */ + void startIntentSender(@NonNull IntentSender intentSender, @NonNull Intent intent); + } + + /** + * Callback for inline suggestion Ui related events. + */ + public interface InlineUiEventCallback { + /** + * Callback to notify inline ui is shown. + */ + void notifyInlineUiShown(@NonNull AutofillId autofillId); + + /** + * Callback to notify inline ui is hidden. + */ + void notifyInlineUiHidden(@NonNull AutofillId autofillId); + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java new file mode 100644 index 000000000000..8fcb8aa9393c --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java @@ -0,0 +1,242 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sDebug; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.Intent; +import android.content.IntentSender; +import android.os.IBinder; +import android.service.autofill.Dataset; +import android.service.autofill.FillResponse; +import android.service.autofill.InlinePresentation; +import android.util.Pair; +import android.util.Slog; +import android.util.SparseArray; +import android.view.autofill.AutofillId; +import android.view.autofill.AutofillManager; +import android.view.inputmethod.InlineSuggestion; +import android.view.inputmethod.InlineSuggestionInfo; +import android.view.inputmethod.InlineSuggestionsRequest; +import android.view.inputmethod.InlineSuggestionsResponse; +import android.widget.inline.InlinePresentationSpec; + +import com.android.internal.view.inline.IInlineContentProvider; +import com.android.server.autofill.RemoteInlineSuggestionRenderService; + +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +final class InlineSuggestionFactory { + private static final String TAG = "InlineSuggestionFactory"; + + public static boolean responseNeedAuthentication(@NonNull FillResponse response) { + return response.getAuthentication() != null && response.getInlinePresentation() != null; + } + + public static InlineSuggestion createInlineAuthentication( + @NonNull InlineSuggestionsRequest request, @NonNull FillResponse response, + @NonNull AutoFillUI.AutoFillUiCallback client, @NonNull Runnable onErrorCallback, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, int userId, + int sessionId) { + final BiConsumer<Dataset, Integer> onClickFactory = (dataset, datasetIndex) -> { + client.authenticate(response.getRequestId(), + datasetIndex, response.getAuthentication(), response.getClientState(), + /* authenticateInline= */ true); + }; + final Consumer<IntentSender> intentSenderConsumer = (intentSender) -> + client.startIntentSender(intentSender, new Intent()); + InlinePresentation inlineAuthentication = response.getInlinePresentation(); + return createInlineAuthSuggestion( + mergedInlinePresentation(request, 0, inlineAuthentication), + remoteRenderService, userId, sessionId, + onClickFactory, onErrorCallback, intentSenderConsumer, + request.getHostInputToken(), request.getHostDisplayId()); + } + + /** + * Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by the + * autofill service, potentially filtering the datasets. + */ + @Nullable + public static SparseArray<Pair<Dataset, InlineSuggestion>> createAutofillInlineSuggestions( + @NonNull InlineSuggestionsRequest request, int requestId, + @NonNull List<Dataset> datasets, + @NonNull AutofillId autofillId, + @NonNull AutoFillUI.AutoFillUiCallback client, @NonNull Runnable onErrorCallback, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId) { + if (sDebug) Slog.d(TAG, "createInlineSuggestionsResponse called"); + final Consumer<IntentSender> intentSenderConsumer = (intentSender) -> + client.startIntentSender(intentSender, new Intent()); + final BiConsumer<Dataset, Integer> onClickFactory = (dataset, datasetIndex) -> { + client.fill(requestId, datasetIndex, dataset); + }; + + return createInlineSuggestionsInternal(/* isAugmented= */ false, request, + datasets, autofillId, + onErrorCallback, onClickFactory, intentSenderConsumer, remoteRenderService, userId, + sessionId); + } + + /** + * Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by augmented + * autofill service. + */ + @Nullable + public static SparseArray<Pair<Dataset, InlineSuggestion>> + createAugmentedAutofillInlineSuggestions( + @NonNull InlineSuggestionsRequest request, @NonNull List<Dataset> datasets, + @NonNull AutofillId autofillId, + @NonNull InlineFillUi.InlineSuggestionUiCallback inlineSuggestionUiCallback, + @NonNull Runnable onErrorCallback, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId) { + if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called"); + return createInlineSuggestionsInternal(/* isAugmented= */ true, request, + datasets, autofillId, onErrorCallback, + (dataset, datasetIndex) -> + inlineSuggestionUiCallback.autofill(dataset, datasetIndex), + (intentSender) -> + inlineSuggestionUiCallback.startIntentSender(intentSender, new Intent()), + remoteRenderService, userId, sessionId); + } + + @Nullable + private static SparseArray<Pair<Dataset, InlineSuggestion>> createInlineSuggestionsInternal( + boolean isAugmented, @NonNull InlineSuggestionsRequest request, + @NonNull List<Dataset> datasets, @NonNull AutofillId autofillId, + @NonNull Runnable onErrorCallback, @NonNull BiConsumer<Dataset, Integer> onClickFactory, + @NonNull Consumer<IntentSender> intentSenderConsumer, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId) { + SparseArray<Pair<Dataset, InlineSuggestion>> response = new SparseArray<>(datasets.size()); + for (int datasetIndex = 0; datasetIndex < datasets.size(); datasetIndex++) { + final Dataset dataset = datasets.get(datasetIndex); + final int fieldIndex = dataset.getFieldIds().indexOf(autofillId); + if (fieldIndex < 0) { + Slog.w(TAG, "AutofillId=" + autofillId + " not found in dataset"); + continue; + } + final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation( + fieldIndex); + if (inlinePresentation == null) { + Slog.w(TAG, "InlinePresentation not found in dataset"); + continue; + } + InlineSuggestion inlineSuggestion = createInlineSuggestion(isAugmented, dataset, + datasetIndex, + mergedInlinePresentation(request, datasetIndex, inlinePresentation), + onClickFactory, remoteRenderService, userId, sessionId, + onErrorCallback, intentSenderConsumer, + request.getHostInputToken(), request.getHostDisplayId()); + response.append(datasetIndex, Pair.create(dataset, inlineSuggestion)); + } + return response; + } + + private static InlineSuggestion createInlineSuggestion(boolean isAugmented, + @NonNull Dataset dataset, int datasetIndex, + @NonNull InlinePresentation inlinePresentation, + @NonNull BiConsumer<Dataset, Integer> onClickFactory, + @NonNull RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId, + @NonNull Runnable onErrorCallback, @NonNull Consumer<IntentSender> intentSenderConsumer, + @Nullable IBinder hostInputToken, + int displayId) { + final String suggestionSource = isAugmented ? InlineSuggestionInfo.SOURCE_PLATFORM + : InlineSuggestionInfo.SOURCE_AUTOFILL; + final String suggestionType = + dataset.getAuthentication() == null ? InlineSuggestionInfo.TYPE_SUGGESTION + : InlineSuggestionInfo.TYPE_ACTION; + final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo( + inlinePresentation.getInlinePresentationSpec(), suggestionSource, + inlinePresentation.getAutofillHints(), suggestionType, + inlinePresentation.isPinned()); + + final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo, + createInlineContentProvider(inlinePresentation, + () -> onClickFactory.accept(dataset, datasetIndex), onErrorCallback, + intentSenderConsumer, remoteRenderService, userId, sessionId, + hostInputToken, displayId)); + + return inlineSuggestion; + } + + private static InlineSuggestion createInlineAuthSuggestion( + @NonNull InlinePresentation inlinePresentation, + @NonNull RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId, + @NonNull BiConsumer<Dataset, Integer> onClickFactory, @NonNull Runnable onErrorCallback, + @NonNull Consumer<IntentSender> intentSenderConsumer, + @Nullable IBinder hostInputToken, int displayId) { + final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo( + inlinePresentation.getInlinePresentationSpec(), + InlineSuggestionInfo.SOURCE_AUTOFILL, inlinePresentation.getAutofillHints(), + InlineSuggestionInfo.TYPE_ACTION, inlinePresentation.isPinned()); + + return new InlineSuggestion(inlineSuggestionInfo, + createInlineContentProvider(inlinePresentation, + () -> onClickFactory.accept(null, + AutofillManager.AUTHENTICATION_ID_DATASET_ID_UNDEFINED), + onErrorCallback, intentSenderConsumer, remoteRenderService, userId, + sessionId, hostInputToken, displayId)); + } + + /** + * Returns an {@link InlinePresentation} with the style spec from the request/host, and + * everything else from the provided {@code inlinePresentation}. + */ + private static InlinePresentation mergedInlinePresentation( + @NonNull InlineSuggestionsRequest request, + int index, @NonNull InlinePresentation inlinePresentation) { + final List<InlinePresentationSpec> specs = request.getInlinePresentationSpecs(); + if (specs.isEmpty()) { + return inlinePresentation; + } + InlinePresentationSpec specFromHost = specs.get(Math.min(specs.size() - 1, index)); + InlinePresentationSpec mergedInlinePresentation = new InlinePresentationSpec.Builder( + inlinePresentation.getInlinePresentationSpec().getMinSize(), + inlinePresentation.getInlinePresentationSpec().getMaxSize()).setStyle( + specFromHost.getStyle()).build(); + return new InlinePresentation(inlinePresentation.getSlice(), mergedInlinePresentation, + inlinePresentation.isPinned()); + } + + private static IInlineContentProvider createInlineContentProvider( + @NonNull InlinePresentation inlinePresentation, @Nullable Runnable onClickAction, + @NonNull Runnable onErrorCallback, + @NonNull Consumer<IntentSender> intentSenderConsumer, + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId, + @Nullable IBinder hostInputToken, + int displayId) { + RemoteInlineSuggestionViewConnector + remoteInlineSuggestionViewConnector = new RemoteInlineSuggestionViewConnector( + remoteRenderService, userId, sessionId, inlinePresentation, hostInputToken, + displayId, onClickAction, onErrorCallback, intentSenderConsumer); + InlineContentProviderImpl inlineContentProvider = new InlineContentProviderImpl( + remoteInlineSuggestionViewConnector, null); + return inlineContentProvider; + } + + private InlineSuggestionFactory() { + } +}
\ No newline at end of file diff --git a/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java new file mode 100644 index 000000000000..368f71760b0d --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java @@ -0,0 +1,300 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sDebug; +import static com.android.server.autofill.Helper.sVerbose; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.IntentSender; +import android.os.Handler; +import android.os.IBinder; +import android.os.RemoteException; +import android.service.autofill.IInlineSuggestionUi; +import android.service.autofill.IInlineSuggestionUiCallback; +import android.service.autofill.ISurfacePackageResultCallback; +import android.util.Slog; +import android.view.SurfaceControlViewHost; + +import com.android.internal.view.inline.IInlineContentCallback; + +/** + * The instance of this class lives in the system server, orchestrating the communication between + * the remote process owning embedded view (i.e. ExtServices) and the remote process hosting the + * embedded view (i.e. IME). It's also responsible for releasing the embedded view from the owning + * process when it's not longer needed in the hosting process. + * + * <p>An instance of this class may be reused to associate with multiple instances of + * {@link InlineContentProviderImpl}s, each of which wraps a callback from the IME. But at any + * given time, there is only one active IME callback which this class will callback into. + * + * <p>This class is thread safe, because all the outside calls are piped into a single handler + * thread to be processed. + */ +final class RemoteInlineSuggestionUi { + + private static final String TAG = RemoteInlineSuggestionUi.class.getSimpleName(); + + // The delay time to release the remote inline suggestion view (in the renderer + // process) after receiving a signal about the surface package being released due to being + // detached from the window in the host app (in the IME process). The release will be + // canceled if the host app reattaches the view to a window within this delay time. + // TODO(b/154683107): try out using the Chroreographer to schedule the release right at the + // next frame. Basically if the view is not re-attached to the window immediately in the next + // frame after it was detached, then it will be released. + private static final long RELEASE_REMOTE_VIEW_HOST_DELAY_MS = 200; + + @NonNull + private final Handler mHandler; + @NonNull + private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector; + private final int mWidth; + private final int mHeight; + @NonNull + private final InlineSuggestionUiCallbackImpl mInlineSuggestionUiCallback; + + @Nullable + private IInlineContentCallback mInlineContentCallback; // from IME + + /** + * Remote inline suggestion view, backed by an instance of {@link SurfaceControlViewHost} in + * the render service process. We takes care of releasing it when there is no remote + * reference to it (from IME), and we will create a new instance of the view when it's needed + * by IME again. + */ + @Nullable + private IInlineSuggestionUi mInlineSuggestionUi; + private int mRefCount = 0; + private boolean mWaitingForUiCreation = false; + private int mActualWidth; + private int mActualHeight; + + @Nullable + private Runnable mDelayedReleaseViewRunnable; + + RemoteInlineSuggestionUi( + @NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector, + int width, int height, Handler handler) { + mHandler = handler; + mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector; + mWidth = width; + mHeight = height; + mInlineSuggestionUiCallback = new InlineSuggestionUiCallbackImpl(); + } + + /** + * Updates the callback from the IME process. It'll swap out the previous IME callback, and + * all the subsequent callback events (onClick, onLongClick, touch event transfer, etc) will + * be directed to the new callback. + */ + void setInlineContentCallback(@NonNull IInlineContentCallback inlineContentCallback) { + mHandler.post(() -> { + mInlineContentCallback = inlineContentCallback; + }); + } + + /** + * Handles the request from the IME process to get a new surface package. May create a new + * view in the renderer process if the existing view is already released. + */ + void requestSurfacePackage() { + mHandler.post(this::handleRequestSurfacePackage); + } + + /** + * Handles the signal from the IME process that the previously sent surface package has been + * released. + */ + void surfacePackageReleased() { + mHandler.post(() -> handleUpdateRefCount(-1)); + } + + /** + * Returns true if the provided size matches the remote view's size. + */ + boolean match(int width, int height) { + return mWidth == width && mHeight == height; + } + + private void handleRequestSurfacePackage() { + cancelPendingReleaseViewRequest(); + + if (mInlineSuggestionUi == null) { + if (mWaitingForUiCreation) { + // This could happen in the following case: the remote embedded view was released + // when previously detached from window. An event after that to re-attached to + // the window will cause us calling the renderSuggestion again. Now, before the + // render call returns a new surface package, if the view is detached and + // re-attached to the window, causing this method to be called again, we will get + // to this state. This request will be ignored and the surface package will still + // be sent back once the view is rendered. + if (sDebug) Slog.d(TAG, "Inline suggestion ui is not ready"); + } else { + mRemoteInlineSuggestionViewConnector.renderSuggestion(mWidth, mHeight, + mInlineSuggestionUiCallback); + mWaitingForUiCreation = true; + } + } else { + try { + mInlineSuggestionUi.getSurfacePackage(new ISurfacePackageResultCallback.Stub() { + @Override + public void onResult(SurfaceControlViewHost.SurfacePackage result) { + mHandler.post(() -> { + if (sVerbose) Slog.v(TAG, "Sending refreshed SurfacePackage to IME"); + try { + mInlineContentCallback.onContent(result, mActualWidth, + mActualHeight); + handleUpdateRefCount(1); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onContent"); + } + }); + } + }); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling getSurfacePackage."); + } + } + } + + private void handleUpdateRefCount(int delta) { + cancelPendingReleaseViewRequest(); + mRefCount += delta; + if (mRefCount <= 0) { + mDelayedReleaseViewRunnable = () -> { + if (mInlineSuggestionUi != null) { + try { + if (sVerbose) Slog.v(TAG, "releasing the host"); + mInlineSuggestionUi.releaseSurfaceControlViewHost(); + mInlineSuggestionUi = null; + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling releaseSurfaceControlViewHost"); + } + } + mDelayedReleaseViewRunnable = null; + }; + mHandler.postDelayed(mDelayedReleaseViewRunnable, RELEASE_REMOTE_VIEW_HOST_DELAY_MS); + } + } + + private void cancelPendingReleaseViewRequest() { + if (mDelayedReleaseViewRunnable != null) { + mHandler.removeCallbacks(mDelayedReleaseViewRunnable); + mDelayedReleaseViewRunnable = null; + } + } + + /** + * This is called when a new inline suggestion UI is inflated from the ext services. + */ + private void handleInlineSuggestionUiReady(IInlineSuggestionUi content, + SurfaceControlViewHost.SurfacePackage surfacePackage, int width, int height) { + mInlineSuggestionUi = content; + mRefCount = 0; + mWaitingForUiCreation = false; + mActualWidth = width; + mActualHeight = height; + if (mInlineContentCallback != null) { + try { + if (sVerbose) Slog.v(TAG, "Sending new UI content to IME"); + handleUpdateRefCount(1); + mInlineContentCallback.onContent(surfacePackage, mActualWidth, mActualHeight); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onContent"); + } + } + if (surfacePackage != null) { + surfacePackage.release(); + } + } + + private void handleOnClick() { + // Autofill the value + mRemoteInlineSuggestionViewConnector.onClick(); + + // Notify the remote process (IME) that hosts the embedded UI that it's clicked + if (mInlineContentCallback != null) { + try { + mInlineContentCallback.onClick(); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onClick"); + } + } + } + + private void handleOnLongClick() { + // Notify the remote process (IME) that hosts the embedded UI that it's long clicked + if (mInlineContentCallback != null) { + try { + mInlineContentCallback.onLongClick(); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onLongClick"); + } + } + } + + private void handleOnError() { + mRemoteInlineSuggestionViewConnector.onError(); + } + + private void handleOnTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + mRemoteInlineSuggestionViewConnector.onTransferTouchFocusToImeWindow(sourceInputToken, + displayId); + } + + private void handleOnStartIntentSender(IntentSender intentSender) { + mRemoteInlineSuggestionViewConnector.onStartIntentSender(intentSender); + } + + /** + * Responsible for communicating with the inline suggestion view owning process. + */ + private class InlineSuggestionUiCallbackImpl extends IInlineSuggestionUiCallback.Stub { + + @Override + public void onClick() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnClick); + } + + @Override + public void onLongClick() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnLongClick); + } + + @Override + public void onContent(IInlineSuggestionUi content, + SurfaceControlViewHost.SurfacePackage surface, int width, int height) { + mHandler.post(() -> handleInlineSuggestionUiReady(content, surface, width, height)); + } + + @Override + public void onError() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnError); + } + + @Override + public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + mHandler.post(() -> handleOnTransferTouchFocusToImeWindow(sourceInputToken, displayId)); + } + + @Override + public void onStartIntentSender(IntentSender intentSender) { + mHandler.post(() -> handleOnStartIntentSender(intentSender)); + } + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java new file mode 100644 index 000000000000..7257255d1ee4 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sDebug; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.IntentSender; +import android.os.IBinder; +import android.service.autofill.IInlineSuggestionUiCallback; +import android.service.autofill.InlinePresentation; +import android.util.Slog; + +import com.android.server.LocalServices; +import com.android.server.autofill.RemoteInlineSuggestionRenderService; +import com.android.server.inputmethod.InputMethodManagerInternal; + +import java.util.function.Consumer; + +/** + * Wraps the parameters needed to create a new inline suggestion view in the remote renderer + * service, and handles the callback from the events on the created remote view. + */ +final class RemoteInlineSuggestionViewConnector { + private static final String TAG = RemoteInlineSuggestionViewConnector.class.getSimpleName(); + + @Nullable + private final RemoteInlineSuggestionRenderService mRemoteRenderService; + @NonNull + private final InlinePresentation mInlinePresentation; + @Nullable + private final IBinder mHostInputToken; + private final int mDisplayId; + private final int mUserId; + private final int mSessionId; + + @NonNull + private final Runnable mOnAutofillCallback; + @NonNull + private final Runnable mOnErrorCallback; + @NonNull + private final Consumer<IntentSender> mStartIntentSenderFromClientApp; + + RemoteInlineSuggestionViewConnector( + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + int userId, int sessionId, + @NonNull InlinePresentation inlinePresentation, + @Nullable IBinder hostInputToken, + int displayId, + @NonNull Runnable onAutofillCallback, + @NonNull Runnable onErrorCallback, + @NonNull Consumer<IntentSender> startIntentSenderFromClientApp) { + mRemoteRenderService = remoteRenderService; + mInlinePresentation = inlinePresentation; + mHostInputToken = hostInputToken; + mDisplayId = displayId; + mUserId = userId; + mSessionId = sessionId; + + mOnAutofillCallback = onAutofillCallback; + mOnErrorCallback = onErrorCallback; + mStartIntentSenderFromClientApp = startIntentSenderFromClientApp; + } + + /** + * Calls the remote renderer service to create a new inline suggestion view. + * + * @return true if the call is made to the remote renderer service, false otherwise. + */ + public boolean renderSuggestion(int width, int height, + @NonNull IInlineSuggestionUiCallback callback) { + if (mRemoteRenderService != null) { + if (sDebug) Slog.d(TAG, "Request to recreate the UI"); + mRemoteRenderService.renderSuggestion(callback, mInlinePresentation, width, height, + mHostInputToken, mDisplayId, mUserId, mSessionId); + return true; + } + return false; + } + + /** + * Handles the callback for the event of remote view being clicked. + */ + public void onClick() { + mOnAutofillCallback.run(); + } + + /** + * Handles the callback for the remote error when creating or interacting with the view. + */ + public void onError() { + mOnErrorCallback.run(); + } + + /** + * Handles the callback for transferring the touch event on the remote view to the IME + * process. + */ + public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + final InputMethodManagerInternal inputMethodManagerInternal = + LocalServices.getService(InputMethodManagerInternal.class); + if (!inputMethodManagerInternal.transferTouchFocusToImeWindow(sourceInputToken, + displayId)) { + Slog.e(TAG, "Cannot transfer touch focus from suggestion to IME"); + mOnErrorCallback.run(); + } + } + + /** + * Handles starting an intent sender from the client app's process. + */ + public void onStartIntentSender(IntentSender intentSender) { + mStartIntentSenderFromClientApp.accept(intentSender); + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/SaveUi.java b/services/autofill/java/com/android/server/autofill/ui/SaveUi.java index 73f5cb8326ea..1c3116699b2d 100644 --- a/services/autofill/java/com/android/server/autofill/ui/SaveUi.java +++ b/services/autofill/java/com/android/server/autofill/ui/SaveUi.java @@ -27,12 +27,13 @@ import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.IntentSender; +import android.content.pm.PackageManager; import android.content.res.Resources; import android.graphics.drawable.Drawable; import android.metrics.LogMaker; import android.os.Handler; import android.os.IBinder; -import android.os.RemoteException; +import android.os.UserHandle; import android.service.autofill.BatchUpdates; import android.service.autofill.CustomDescription; import android.service.autofill.InternalOnClickAction; @@ -41,6 +42,10 @@ import android.service.autofill.InternalValidator; import android.service.autofill.SaveInfo; import android.service.autofill.ValueFinder; import android.text.Html; +import android.text.SpannableStringBuilder; +import android.text.TextUtils; +import android.text.method.LinkMovementMethod; +import android.text.style.ClickableSpan; import android.util.ArraySet; import android.util.Pair; import android.util.Slog; @@ -61,11 +66,14 @@ import android.widget.TextView; import com.android.internal.R; import com.android.internal.logging.MetricsLogger; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; +import com.android.internal.util.ArrayUtils; import com.android.server.UiThread; import com.android.server.autofill.Helper; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; /** * Autofill Save Prompt @@ -79,10 +87,13 @@ final class SaveUi { private static final int THEME_ID_DARK = com.android.internal.R.style.Theme_DeviceDefault_Autofill_Save; + private static final int SCROLL_BAR_DEFAULT_DELAY_BEFORE_FADE_MS = 500; + public interface OnSaveListener { void onSave(); void onCancel(IntentSender listener); void onDestroy(); + void startIntentSender(IntentSender intentSender, Intent intent); } /** @@ -129,6 +140,15 @@ final class SaveUi { mDone = true; mRealListener.onDestroy(); } + + @Override + public void startIntentSender(IntentSender intentSender, Intent intent) { + if (sDebug) Slog.d(TAG, "OneTimeListener.startIntentSender(): " + mDone); + if (mDone) { + return; + } + mRealListener.startIntentSender(intentSender, intent); + } } private final Handler mHandler = UiThread.getHandler(); @@ -147,6 +167,7 @@ final class SaveUi { private final ComponentName mComponentName; private final boolean mCompatMode; private final int mThemeId; + private final int mType; private boolean mDestroyed; @@ -158,35 +179,77 @@ final class SaveUi { boolean nightMode, boolean isUpdate, boolean compatMode) { if (sVerbose) Slog.v(TAG, "nightMode: " + nightMode); mThemeId = nightMode ? THEME_ID_DARK : THEME_ID_LIGHT; - mPendingUi= pendingUi; + mPendingUi = pendingUi; mListener = new OneActionThenDestroyListener(listener); mOverlayControl = overlayControl; mServicePackageName = servicePackageName; mComponentName = componentName; mCompatMode = compatMode; - context = new ContextThemeWrapper(context, mThemeId); + context = new ContextThemeWrapper(context, mThemeId) { + @Override + public void startActivity(Intent intent) { + if (resolveActivity(intent) == null) { + if (sDebug) { + Slog.d(TAG, "Can not startActivity for save UI with intent=" + intent); + } + return; + } + intent.putExtra(AutofillManager.EXTRA_RESTORE_CROSS_ACTIVITY, true); + + PendingIntent p = PendingIntent.getActivityAsUser( + this, /* requestCode= */ 0, intent, /* flags= */ 0, /* options= */ null, + UserHandle.CURRENT); + if (sDebug) { + Slog.d(TAG, "startActivity add save UI restored with intent=" + intent); + } + // Apply restore mechanism + startIntentSenderWithRestore(p, intent); + } + + private ComponentName resolveActivity(Intent intent) { + final PackageManager packageManager = getPackageManager(); + final ComponentName componentName = intent.resolveActivity(packageManager); + if (componentName != null) { + return componentName; + } + intent.addFlags(Intent.FLAG_ACTIVITY_MATCH_EXTERNAL); + return intent.resolveActivity(packageManager); + } + }; final LayoutInflater inflater = LayoutInflater.from(context); final View view = inflater.inflate(R.layout.autofill_save, null); final TextView titleView = view.findViewById(R.id.autofill_save_title); final ArraySet<String> types = new ArraySet<>(3); - final int type = info.getType(); + mType = info.getType(); - if ((type & SaveInfo.SAVE_DATA_TYPE_PASSWORD) != 0) { + if ((mType & SaveInfo.SAVE_DATA_TYPE_PASSWORD) != 0) { types.add(context.getString(R.string.autofill_save_type_password)); } - if ((type & SaveInfo.SAVE_DATA_TYPE_ADDRESS) != 0) { + if ((mType & SaveInfo.SAVE_DATA_TYPE_ADDRESS) != 0) { types.add(context.getString(R.string.autofill_save_type_address)); } - if ((type & SaveInfo.SAVE_DATA_TYPE_CREDIT_CARD) != 0) { + + // fallback to generic card type if set multiple types + final int cardTypeMask = SaveInfo.SAVE_DATA_TYPE_CREDIT_CARD + | SaveInfo.SAVE_DATA_TYPE_DEBIT_CARD + | SaveInfo.SAVE_DATA_TYPE_PAYMENT_CARD; + final int count = Integer.bitCount(mType & cardTypeMask); + if (count > 1 || (mType & SaveInfo.SAVE_DATA_TYPE_GENERIC_CARD) != 0) { + types.add(context.getString(R.string.autofill_save_type_generic_card)); + } else if ((mType & SaveInfo.SAVE_DATA_TYPE_PAYMENT_CARD) != 0) { + types.add(context.getString(R.string.autofill_save_type_payment_card)); + } else if ((mType & SaveInfo.SAVE_DATA_TYPE_CREDIT_CARD) != 0) { types.add(context.getString(R.string.autofill_save_type_credit_card)); + } else if ((mType & SaveInfo.SAVE_DATA_TYPE_DEBIT_CARD) != 0) { + types.add(context.getString(R.string.autofill_save_type_debit_card)); } - if ((type & SaveInfo.SAVE_DATA_TYPE_USERNAME) != 0) { + if ((mType & SaveInfo.SAVE_DATA_TYPE_USERNAME) != 0) { types.add(context.getString(R.string.autofill_save_type_username)); } - if ((type & SaveInfo.SAVE_DATA_TYPE_EMAIL_ADDRESS) != 0) { + if ((mType & SaveInfo.SAVE_DATA_TYPE_EMAIL_ADDRESS) != 0) { types.add(context.getString(R.string.autofill_save_type_email_address)); } @@ -228,29 +291,41 @@ final class SaveUi { } else { mSubTitle = info.getDescription(); if (mSubTitle != null) { - writeLog(MetricsEvent.AUTOFILL_SAVE_CUSTOM_SUBTITLE, type); + writeLog(MetricsEvent.AUTOFILL_SAVE_CUSTOM_SUBTITLE); final ViewGroup subtitleContainer = view.findViewById(R.id.autofill_save_custom_subtitle); final TextView subtitleView = new TextView(context); subtitleView.setText(mSubTitle); + applyMovementMethodIfNeed(subtitleView); subtitleContainer.addView(subtitleView, new LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT)); subtitleContainer.setVisibility(View.VISIBLE); + subtitleContainer.setScrollBarDefaultDelayBeforeFade( + SCROLL_BAR_DEFAULT_DELAY_BEFORE_FADE_MS); } if (sDebug) Slog.d(TAG, "on constructor: title=" + mTitle + ", subTitle=" + mSubTitle); } final TextView noButton = view.findViewById(R.id.autofill_save_no); - if (info.getNegativeActionStyle() == SaveInfo.NEGATIVE_BUTTON_STYLE_REJECT) { - noButton.setText(R.string.save_password_notnow); - } else { - noButton.setText(R.string.autofill_save_no); + final int negativeActionStyle = info.getNegativeActionStyle(); + switch (negativeActionStyle) { + case SaveInfo.NEGATIVE_BUTTON_STYLE_REJECT: + noButton.setText(R.string.autofill_save_notnow); + break; + case SaveInfo.NEGATIVE_BUTTON_STYLE_NEVER: + noButton.setText(R.string.autofill_save_never); + break; + case SaveInfo.NEGATIVE_BUTTON_STYLE_CANCEL: + default: + noButton.setText(R.string.autofill_save_no); } noButton.setOnClickListener((v) -> mListener.onCancel(info.getNegativeActionListener())); final TextView yesButton = view.findViewById(R.id.autofill_save_yes); - if (isUpdate) { + if (info.getPositiveActionStyle() == SaveInfo.POSITIVE_BUTTON_STYLE_CONTINUE) { + yesButton.setText(R.string.autofill_continue_yes); + } else if (isUpdate) { yesButton.setText(R.string.autofill_update_yes); } yesButton.setOnClickListener((v) -> mListener.onSave()); @@ -267,7 +342,7 @@ final class SaveUi { window.addFlags(WindowManager.LayoutParams.FLAG_ALT_FOCUSABLE_IM | WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL | WindowManager.LayoutParams.FLAG_WATCH_OUTSIDE_TOUCH); - window.addPrivateFlags(WindowManager.LayoutParams.PRIVATE_FLAG_SHOW_FOR_ALL_USERS); + window.addPrivateFlags(WindowManager.LayoutParams.SYSTEM_FLAG_SHOW_FOR_ALL_USERS); window.setSoftInputMode(WindowManager.LayoutParams.SOFT_INPUT_ADJUST_PAN); window.setGravity(Gravity.BOTTOM | Gravity.CENTER); window.setCloseOnTouchOutside(true); @@ -285,8 +360,7 @@ final class SaveUi { if (customDescription == null) { return false; } - final int type = info.getType(); - writeLog(MetricsEvent.AUTOFILL_SAVE_CUSTOM_DESCRIPTION, type); + writeLog(MetricsEvent.AUTOFILL_SAVE_CUSTOM_DESCRIPTION); final RemoteViews template = customDescription.getPresentation(); if (template == null) { @@ -309,35 +383,16 @@ final class SaveUi { final RemoteViews.OnClickHandler handler = (view, pendingIntent, response) -> { Intent intent = response.getLaunchOptions(view).first; - final LogMaker log = - newLogMaker(MetricsEvent.AUTOFILL_SAVE_LINK_TAPPED, type); - // We need to hide the Save UI before launching the pending intent, and - // restore back it once the activity is finished, and that's achieved by - // adding a custom extra in the activity intent. final boolean isValid = isValidLink(pendingIntent, intent); if (!isValid) { + final LogMaker log = newLogMaker(MetricsEvent.AUTOFILL_SAVE_LINK_TAPPED, mType); log.setType(MetricsEvent.TYPE_UNKNOWN); mMetricsLogger.write(log); return false; } - if (sVerbose) Slog.v(TAG, "Intercepting custom description intent"); - final IBinder token = mPendingUi.getToken(); - intent.putExtra(AutofillManager.EXTRA_RESTORE_SESSION_TOKEN, token); - try { - mPendingUi.client.startIntentSender(pendingIntent.getIntentSender(), - intent); - mPendingUi.setState(PendingUi.STATE_PENDING); - if (sDebug) Slog.d(TAG, "hiding UI until restored with token " + token); - hide(); - log.setType(MetricsEvent.TYPE_OPEN); - mMetricsLogger.write(log); - return true; - } catch (RemoteException e) { - Slog.w(TAG, "error triggering pending intent: " + intent); - log.setType(MetricsEvent.TYPE_FAILURE); - mMetricsLogger.write(log); - return false; - } + + startIntentSenderWithRestore(pendingIntent, intent); + return true; }; try { @@ -417,11 +472,16 @@ final class SaveUi { } } + applyTextViewStyle(customSubtitleView); + // Finally, add the custom description to the save UI. final ViewGroup subtitleContainer = saveUiView.findViewById(R.id.autofill_save_custom_subtitle); subtitleContainer.addView(customSubtitleView); subtitleContainer.setVisibility(View.VISIBLE); + subtitleContainer.setScrollBarDefaultDelayBeforeFade( + SCROLL_BAR_DEFAULT_DELAY_BEFORE_FADE_MS); + return true; } catch (Exception e) { Slog.e(TAG, "Error applying custom description. ", e); @@ -429,6 +489,60 @@ final class SaveUi { return false; } + private void startIntentSenderWithRestore(@NonNull PendingIntent pendingIntent, + @NonNull Intent intent) { + if (sVerbose) Slog.v(TAG, "Intercepting custom description intent"); + + // We need to hide the Save UI before launching the pending intent, and + // restore back it once the activity is finished, and that's achieved by + // adding a custom extra in the activity intent. + final IBinder token = mPendingUi.getToken(); + intent.putExtra(AutofillManager.EXTRA_RESTORE_SESSION_TOKEN, token); + + mListener.startIntentSender(pendingIntent.getIntentSender(), intent); + mPendingUi.setState(PendingUi.STATE_PENDING); + + if (sDebug) Slog.d(TAG, "hiding UI until restored with token " + token); + hide(); + + final LogMaker log = newLogMaker(MetricsEvent.AUTOFILL_SAVE_LINK_TAPPED, mType); + log.setType(MetricsEvent.TYPE_OPEN); + mMetricsLogger.write(log); + } + + private void applyTextViewStyle(@NonNull View rootView) { + final List<TextView> textViews = new ArrayList<>(); + final Predicate<View> predicate = (view) -> { + if (view instanceof TextView) { + // Collects TextViews + textViews.add((TextView) view); + } + return false; + }; + + // Traverses all TextViews, enables movement method if the TextView contains URLSpan + rootView.findViewByPredicate(predicate); + final int size = textViews.size(); + for (int i = 0; i < size; i++) { + applyMovementMethodIfNeed(textViews.get(i)); + } + } + + private void applyMovementMethodIfNeed(@NonNull TextView textView) { + final CharSequence message = textView.getText(); + if (TextUtils.isEmpty(message)) { + return; + } + + final SpannableStringBuilder ssb = new SpannableStringBuilder(message); + final ClickableSpan[] spans = ssb.getSpans(0, ssb.length(), ClickableSpan.class); + if (ArrayUtils.isEmpty(spans)) { + return; + } + + textView.setMovementMethod(LinkMovementMethod.getInstance()); + } + private void setServiceIcon(Context context, View view, Drawable serviceIcon) { final ImageView iconView = view.findViewById(R.id.autofill_save_icon); final Resources res = context.getResources(); @@ -478,8 +592,8 @@ final class SaveUi { mPendingUi.sessionId, mCompatMode); } - private void writeLog(int category, int saveType) { - mMetricsLogger.write(newLogMaker(category, saveType)); + private void writeLog(int category) { + mMetricsLogger.write(newLogMaker(category, mType)); } /** @@ -534,6 +648,10 @@ final class SaveUi { return mPendingUi; } + boolean isShowing() { + return mDialog.isShowing(); + } + void destroy() { try { if (sDebug) Slog.d(TAG, "destroy()"); |