Use a separate code path for rich content in augmented autofill

Image suggestions (and other rich content) are not handled the same way
as primitive autofill values. These suggestions are also only applicable
to augmented autofill. Therefore, instead of reusing AutofillType and
AutofillValue, we use a separate code path to insert rich content. A
follow-on change will remove AUTOFILL_TYPE_RICH_CONTENT and the
corresponding code on AutofillValue.

Bug: 168837034
Test: Manual and unit tests
  atest CtsAutoFillServiceTestCases:DatasetTest
  atest CtsAutoFillServiceTestCases:InlineAugmentedAuthTest
  atest CtsAutoFillServiceTestCases:InlineAugmentedLoginActivityTest
Change-Id: I4fa3baf2b545908fc25f3a6e28a7addc7004786b

8 files changed, 203 insertions, 35 deletions

diff --git a/api/system-current.txt b/api/system-current.txt
index f30f756ae3f6..68ec4b28792f 100644
--- a/api/system-current.txt
+++ b/api/system-current.txt
@@ -10025,6 +10025,7 @@ package android.service.autofill {
 
   public static final class Dataset.Builder {
     ctor public Dataset.Builder(@NonNull android.service.autofill.InlinePresentation);
+    method @NonNull public android.service.autofill.Dataset.Builder setContent(@NonNull android.view.autofill.AutofillId, @Nullable android.content.ClipData);
     method @NonNull public android.service.autofill.Dataset.Builder setFieldInlinePresentation(@NonNull android.view.autofill.AutofillId, @Nullable android.view.autofill.AutofillValue, @Nullable java.util.regex.Pattern, @NonNull android.service.autofill.InlinePresentation);
   }
 
diff --git a/api/test-current.txt b/api/test-current.txt
index 82838ea605ff..b925a27a13c3 100644
--- a/api/test-current.txt
+++ b/api/test-current.txt
@@ -1555,6 +1555,19 @@ package android.service.autofill {
     method @Nullable public android.util.SparseArray<android.service.autofill.InternalOnClickAction> getActions();
   }
 
+  public final class Dataset implements android.os.Parcelable {
+    method @Nullable public android.content.IntentSender getAuthentication();
+    method @Nullable public android.content.ClipData getFieldContent();
+    method @Nullable public java.util.ArrayList<android.view.autofill.AutofillId> getFieldIds();
+    method @Nullable public java.util.ArrayList<android.view.autofill.AutofillValue> getFieldValues();
+    method @Nullable public String getId();
+    method public boolean isEmpty();
+  }
+
+  public static final class Dataset.Builder {
+    method @NonNull public android.service.autofill.Dataset.Builder setContent(@NonNull android.view.autofill.AutofillId, @Nullable android.content.ClipData);
+  }
+
   public final class DateTransformation extends android.service.autofill.InternalTransformation implements android.os.Parcelable android.service.autofill.Transformation {
     method public void apply(@NonNull android.service.autofill.ValueFinder, @NonNull android.widget.RemoteViews, int) throws java.lang.Exception;
   }
diff --git a/core/java/android/service/autofill/Dataset.java b/core/java/android/service/autofill/Dataset.java
index 18d79927388b..8ae1b6bf702d 100644
--- a/core/java/android/service/autofill/Dataset.java
+++ b/core/java/android/service/autofill/Dataset.java
@@ -20,7 +20,10 @@ import static android.view.autofill.Helper.sDebug;
 
 import android.annotation.NonNull;
 import android.annotation.Nullable;
+import android.annotation.SuppressLint;
 import android.annotation.SystemApi;
+import android.annotation.TestApi;
+import android.content.ClipData;
 import android.content.IntentSender;
 import android.os.Parcel;
 import android.os.Parcelable;
@@ -97,7 +100,6 @@ import java.util.regex.Pattern;
  * with the lower case value of the view's text are shown.
  *   <li>All other datasets are hidden.
  * </ol>
- *
  */
 public final class Dataset implements Parcelable {
 
@@ -106,6 +108,7 @@ public final class Dataset implements Parcelable {
     private final ArrayList<RemoteViews> mFieldPresentations;
     private final ArrayList<InlinePresentation> mFieldInlinePresentations;
     private final ArrayList<DatasetFieldFilter> mFieldFilters;
+    @Nullable private final ClipData mFieldContent;
     private final RemoteViews mPresentation;
     @Nullable private final InlinePresentation mInlinePresentation;
     private final IntentSender mAuthentication;
@@ -117,6 +120,7 @@ public final class Dataset implements Parcelable {
         mFieldPresentations = builder.mFieldPresentations;
         mFieldInlinePresentations = builder.mFieldInlinePresentations;
         mFieldFilters = builder.mFieldFilters;
+        mFieldContent = builder.mFieldContent;
         mPresentation = builder.mPresentation;
         mInlinePresentation = builder.mInlinePresentation;
         mAuthentication = builder.mAuthentication;
@@ -124,11 +128,15 @@ public final class Dataset implements Parcelable {
     }
 
     /** @hide */
+    @TestApi
+    @SuppressLint("ConcreteCollection")
     public @Nullable ArrayList<AutofillId> getFieldIds() {
         return mFieldIds;
     }
 
     /** @hide */
+    @TestApi
+    @SuppressLint("ConcreteCollection")
     public @Nullable ArrayList<AutofillValue> getFieldValues() {
         return mFieldValues;
     }
@@ -140,24 +148,37 @@ public final class Dataset implements Parcelable {
     }
 
     /** @hide */
-    @Nullable
-    public InlinePresentation getFieldInlinePresentation(int index) {
+    public @Nullable InlinePresentation getFieldInlinePresentation(int index) {
         final InlinePresentation inlinePresentation = mFieldInlinePresentations.get(index);
         return inlinePresentation != null ? inlinePresentation : mInlinePresentation;
     }
 
     /** @hide */
-    @Nullable
-    public DatasetFieldFilter getFilter(int index) {
+    public @Nullable DatasetFieldFilter getFilter(int index) {
         return mFieldFilters.get(index);
     }
 
+    /**
+     * Returns the content to be filled for a non-text suggestion. This is only applicable to
+     * augmented autofill. The target field for the content is available via {@link #getFieldIds()}
+     * (guaranteed to have a single field id set when the return value here is non-null). See
+     * {@link Builder#setContent(AutofillId, ClipData)} for more info.
+     *
+     * @hide
+     */
+    @TestApi
+    public @Nullable ClipData getFieldContent() {
+        return mFieldContent;
+    }
+
     /** @hide */
+    @TestApi
     public @Nullable IntentSender getAuthentication() {
         return mAuthentication;
     }
 
     /** @hide */
+    @TestApi
     public boolean isEmpty() {
         return mFieldIds == null || mFieldIds.isEmpty();
     }
@@ -179,6 +200,9 @@ public final class Dataset implements Parcelable {
         if (mFieldValues != null) {
             builder.append(", fieldValues=").append(mFieldValues);
         }
+        if (mFieldContent != null) {
+            builder.append(", fieldContent=").append(mFieldContent);
+        }
         if (mFieldPresentations != null) {
             builder.append(", fieldPresentations=").append(mFieldPresentations.size());
         }
@@ -207,7 +231,8 @@ public final class Dataset implements Parcelable {
      *
      * @hide
      */
-    public String getId() {
+    @TestApi
+    public @Nullable String getId() {
         return mId;
     }
 
@@ -221,6 +246,7 @@ public final class Dataset implements Parcelable {
         private ArrayList<RemoteViews> mFieldPresentations;
         private ArrayList<InlinePresentation> mFieldInlinePresentations;
         private ArrayList<DatasetFieldFilter> mFieldFilters;
+        @Nullable private ClipData mFieldContent;
         private RemoteViews mPresentation;
         @Nullable private InlinePresentation mInlinePresentation;
         private IntentSender mAuthentication;
@@ -366,6 +392,36 @@ public final class Dataset implements Parcelable {
         }
 
         /**
+         * Sets the content for a field.
+         *
+         * <p>Only called by augmented autofill.
+         *
+         * <p>For a given field, either a {@link AutofillValue value} or content can be filled, but
+         * not both. Furthermore, when filling content, only a single field can be filled.
+         *
+         * @param id id returned by
+         * {@link android.app.assist.AssistStructure.ViewNode#getAutofillId()}.
+         * @param content content to be autofilled. Pass {@code null} if you do not have the content
+         * but the target view is a logical part of the dataset. For example, if the dataset needs
+         * authentication.
+         *
+         * @throws IllegalStateException if {@link #build()} was already called.
+         *
+         * @return this builder.
+         *
+         * @hide
+         */
+        @TestApi
+        @SystemApi
+        @SuppressLint("MissingGetterMatchingBuilder")
+        public @NonNull Builder setContent(@NonNull AutofillId id, @Nullable ClipData content) {
+            throwIfDestroyed();
+            setLifeTheUniverseAndEverything(id, null, null, null, null);
+            mFieldContent = content;
+            return this;
+        }
+
+        /**
          * Sets the value of a field.
          *
          * <b>Note:</b> Prior to Android {@link android.os.Build.VERSION_CODES#P}, this method would
@@ -659,6 +715,15 @@ public final class Dataset implements Parcelable {
             if (mFieldIds == null) {
                 throw new IllegalStateException("at least one value must be set");
             }
+            if (mFieldContent != null) {
+                if (mFieldIds.size() > 1) {
+                    throw new IllegalStateException(
+                            "when filling content, only one field can be filled");
+                }
+                if (mFieldValues.get(0) != null) {
+                    throw new IllegalStateException("cannot fill both content and values");
+                }
+            }
             return new Dataset(this);
         }
 
@@ -687,6 +752,7 @@ public final class Dataset implements Parcelable {
         parcel.writeTypedList(mFieldPresentations, flags);
         parcel.writeTypedList(mFieldInlinePresentations, flags);
         parcel.writeTypedList(mFieldFilters, flags);
+        parcel.writeParcelable(mFieldContent, flags);
         parcel.writeParcelable(mAuthentication, flags);
         parcel.writeString(mId);
     }
@@ -694,18 +760,8 @@ public final class Dataset implements Parcelable {
     public static final @NonNull Creator<Dataset> CREATOR = new Creator<Dataset>() {
         @Override
         public Dataset createFromParcel(Parcel parcel) {
-            // Always go through the builder to ensure the data ingested by
-            // the system obeys the contract of the builder to avoid attacks
-            // using specially crafted parcels.
             final RemoteViews presentation = parcel.readParcelable(null);
             final InlinePresentation inlinePresentation = parcel.readParcelable(null);
-            final Builder builder = presentation != null
-                    ? inlinePresentation == null
-                            ? new Builder(presentation)
-                            : new Builder(presentation).setInlinePresentation(inlinePresentation)
-                    : inlinePresentation == null
-                            ? new Builder()
-                            : new Builder(inlinePresentation);
             final ArrayList<AutofillId> ids =
                     parcel.createTypedArrayList(AutofillId.CREATOR);
             final ArrayList<AutofillValue> values =
@@ -716,6 +772,21 @@ public final class Dataset implements Parcelable {
                     parcel.createTypedArrayList(InlinePresentation.CREATOR);
             final ArrayList<DatasetFieldFilter> filters =
                     parcel.createTypedArrayList(DatasetFieldFilter.CREATOR);
+            final ClipData fieldContent = parcel.readParcelable(null);
+            final IntentSender authentication = parcel.readParcelable(null);
+            final String datasetId = parcel.readString();
+
+            // Always go through the builder to ensure the data ingested by
+            // the system obeys the contract of the builder to avoid attacks
+            // using specially crafted parcels.
+            final Builder builder = (presentation != null) ? new Builder(presentation)
+                    : new Builder();
+            if (inlinePresentation != null) {
+                builder.setInlinePresentation(inlinePresentation);
+            }
+            if (fieldContent != null) {
+                builder.setContent(ids.get(0), fieldContent);
+            }
             final int inlinePresentationsSize = inlinePresentations.size();
             for (int i = 0; i < ids.size(); i++) {
                 final AutofillId id = ids.get(i);
@@ -727,8 +798,8 @@ public final class Dataset implements Parcelable {
                 builder.setLifeTheUniverseAndEverything(id, value, fieldPresentation,
                         fieldInlinePresentation, filter);
             }
-            builder.setAuthentication(parcel.readParcelable(null));
-            builder.setId(parcel.readString());
+            builder.setAuthentication(authentication);
+            builder.setId(datasetId);
             return builder.build();
         }
 
diff --git a/core/java/android/view/autofill/AutofillManager.java b/core/java/android/view/autofill/AutofillManager.java
index fb66b5298839..81db62857c17 100644
--- a/core/java/android/view/autofill/AutofillManager.java
+++ b/core/java/android/view/autofill/AutofillManager.java
@@ -19,6 +19,7 @@ package android.view.autofill;
 import static android.service.autofill.FillRequest.FLAG_MANUAL_REQUEST;
 import static android.service.autofill.FillRequest.FLAG_PASSWORD_INPUT_TYPE;
 import static android.service.autofill.FillRequest.FLAG_VIEW_NOT_FOCUSED;
+import static android.view.OnReceiveContentCallback.Payload.SOURCE_AUTOFILL;
 import static android.view.autofill.Helper.sDebug;
 import static android.view.autofill.Helper.sVerbose;
 import static android.view.autofill.Helper.toList;
@@ -32,6 +33,7 @@ import android.annotation.SystemApi;
 import android.annotation.SystemService;
 import android.annotation.TestApi;
 import android.content.AutofillOptions;
+import android.content.ClipData;
 import android.content.ComponentName;
 import android.content.Context;
 import android.content.Intent;
@@ -60,6 +62,7 @@ import android.util.Slog;
 import android.util.SparseArray;
 import android.view.Choreographer;
 import android.view.KeyEvent;
+import android.view.OnReceiveContentCallback;
 import android.view.View;
 import android.view.accessibility.AccessibilityEvent;
 import android.view.accessibility.AccessibilityManager;
@@ -2350,6 +2353,49 @@ public final class AutofillManager {
         }
     }
 
+    private void autofillContent(int sessionId, AutofillId id, ClipData clip) {
+        synchronized (mLock) {
+            if (sessionId != mSessionId) {
+                return;
+            }
+            final AutofillClient client = getClient();
+            if (client == null) {
+                return;
+            }
+            final View view = client.autofillClientFindViewByAutofillIdTraversal(id);
+            if (view == null) {
+                // Most likely view has been removed after the initial request was sent to the
+                // the service; this is fine, but we need to update the view status in the
+                // server side so it can be triggered again.
+                Log.d(TAG, "autofillContent(): no view with id " + id);
+                reportAutofillContentFailure(id);
+                return;
+            }
+            OnReceiveContentCallback.Payload payload =
+                    new OnReceiveContentCallback.Payload.Builder(clip, SOURCE_AUTOFILL)
+                            .build();
+            boolean handled = view.onReceiveContent(payload);
+            if (!handled) {
+                Log.w(TAG, "autofillContent(): receiver returned false: id=" + id
+                        + ", view=" + view + ", clip=" + clip);
+                reportAutofillContentFailure(id);
+                return;
+            }
+            mMetricsLogger.write(newLog(MetricsEvent.AUTOFILL_DATASET_APPLIED)
+                    .addTaggedData(MetricsEvent.FIELD_AUTOFILL_NUM_VALUES, 1)
+                    .addTaggedData(MetricsEvent.FIELD_AUTOFILL_NUM_VIEWS_FILLED, 1));
+        }
+    }
+
+    private void reportAutofillContentFailure(AutofillId id) {
+        try {
+            mService.setAutofillFailure(mSessionId, Collections.singletonList(id),
+                    mContext.getUserId());
+        } catch (RemoteException e) {
+            throw e.rethrowFromSystemServer();
+        }
+    }
+
     private LogMaker newLog(int category) {
         final LogMaker log = new LogMaker(category)
                 .addTaggedData(MetricsEvent.FIELD_AUTOFILL_SESSION_ID, mSessionId);
@@ -3391,6 +3437,14 @@ public final class AutofillManager {
         }
 
         @Override
+        public void autofillContent(int sessionId, AutofillId id, ClipData content) {
+            final AutofillManager afm = mAfm.get();
+            if (afm != null) {
+                afm.post(() -> afm.autofillContent(sessionId, id, content));
+            }
+        }
+
+        @Override
         public void authenticate(int sessionId, int authenticationId, IntentSender intent,
                 Intent fillInIntent, boolean authenticateInline) {
             final AutofillManager afm = mAfm.get();
diff --git a/core/java/android/view/autofill/IAutoFillManagerClient.aidl b/core/java/android/view/autofill/IAutoFillManagerClient.aidl
index f8ccea5d8356..1f833f66c257 100644
--- a/core/java/android/view/autofill/IAutoFillManagerClient.aidl
+++ b/core/java/android/view/autofill/IAutoFillManagerClient.aidl
@@ -18,6 +18,7 @@ package android.view.autofill;
 
 import java.util.List;
 
+import android.content.ClipData;
 import android.content.ComponentName;
 import android.content.Intent;
 import android.content.IntentSender;
@@ -48,6 +49,11 @@ oneway interface IAutoFillManagerClient {
             boolean hideHighlight);
 
     /**
+     * Autofills the activity with rich content data (e.g. an image) from a dataset.
+     */
+    void autofillContent(int sessionId, in AutofillId id, in ClipData content);
+
+    /**
       * Authenticates a fill response or a data set.
       */
     void authenticate(int sessionId, int authenticationId, in IntentSender intent,
diff --git a/non-updatable-api/system-current.txt b/non-updatable-api/system-current.txt
index fe40892a959b..c71f1459e73d 100644
--- a/non-updatable-api/system-current.txt
+++ b/non-updatable-api/system-current.txt
@@ -8880,6 +8880,7 @@ package android.service.autofill {
 
   public static final class Dataset.Builder {
     ctor public Dataset.Builder(@NonNull android.service.autofill.InlinePresentation);
+    method @NonNull public android.service.autofill.Dataset.Builder setContent(@NonNull android.view.autofill.AutofillId, @Nullable android.content.ClipData);
     method @NonNull public android.service.autofill.Dataset.Builder setFieldInlinePresentation(@NonNull android.view.autofill.AutofillId, @Nullable android.view.autofill.AutofillValue, @Nullable java.util.regex.Pattern, @NonNull android.service.autofill.InlinePresentation);
   }
 
diff --git a/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java b/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java
index 92b8608f4f6c..bd26d44bed6f 100644
--- a/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java
+++ b/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java
@@ -25,6 +25,7 @@ import android.annotation.NonNull;
 import android.annotation.Nullable;
 import android.annotation.UserIdInt;
 import android.app.AppGlobals;
+import android.content.ClipData;
 import android.content.ComponentName;
 import android.content.Context;
 import android.content.Intent;
@@ -296,11 +297,29 @@ final class RemoteAugmentedAutofillService
                                         dataset.getId(), clientState);
                                 try {
                                     final ArrayList<AutofillId> fieldIds = dataset.getFieldIds();
-                                    final int size = fieldIds.size();
-                                    final boolean hideHighlight = size == 1
-                                            && fieldIds.get(0).equals(focusedId);
-                                    client.autofill(sessionId, fieldIds, dataset.getFieldValues(),
-                                            hideHighlight);
+                                    final ClipData content = dataset.getFieldContent();
+                                    if (content != null) {
+                                        final AutofillId fieldId = fieldIds.get(0);
+                                        if (sDebug) {
+                                            Slog.d(TAG, "Calling client autofillContent(): "
+                                                    + "id=" + fieldId + ", content=" + content);
+                                        }
+                                        client.autofillContent(sessionId, fieldId, content);
+                                    } else {
+                                        final int size = fieldIds.size();
+                                        final boolean hideHighlight = size == 1
+                                                && fieldIds.get(0).equals(focusedId);
+                                        if (sDebug) {
+                                            Slog.d(TAG, "Calling client autofill(): "
+                                                    + "ids=" + fieldIds
+                                                    + ", values=" + dataset.getFieldValues());
+                                        }
+                                        client.autofill(
+                                                sessionId,
+                                                fieldIds,
+                                                dataset.getFieldValues(),
+                                                hideHighlight);
+                                    }
                                     inlineSuggestionsCallback.apply(
                                             InlineFillUi.emptyUi(focusedId));
                                 } catch (RemoteException e) {
diff --git a/services/autofill/java/com/android/server/autofill/Session.java b/services/autofill/java/com/android/server/autofill/Session.java
index f596b072d713..0302b2251f10 100644
--- a/services/autofill/java/com/android/server/autofill/Session.java
+++ b/services/autofill/java/com/android/server/autofill/Session.java
@@ -47,6 +47,7 @@ import android.app.IAssistDataReceiver;
 import android.app.assist.AssistStructure;
 import android.app.assist.AssistStructure.AutofillOverlay;
 import android.app.assist.AssistStructure.ViewNode;
+import android.content.ClipData;
 import android.content.ComponentName;
 import android.content.Context;
 import android.content.Intent;
@@ -1493,11 +1494,12 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
             Slog.d(TAG, "Auth result for augmented autofill: sessionId=" + id
                     + ", authId=" + authId + ", dataset=" + dataset);
         }
-        if (dataset == null
-                || dataset.getFieldIds().size() != 1
-                || dataset.getFieldIds().get(0) == null
-                || dataset.getFieldValues().size() != 1
-                || dataset.getFieldValues().get(0) == null) {
+        final AutofillId fieldId = (dataset != null && dataset.getFieldIds().size() == 1)
+                ? dataset.getFieldIds().get(0) : null;
+        final AutofillValue value = (dataset != null && dataset.getFieldValues().size() == 1)
+                ? dataset.getFieldValues().get(0) : null;
+        final ClipData content = (dataset != null) ? dataset.getFieldContent() : null;
+        if (fieldId == null || (value == null && content == null)) {
             if (sDebug) {
                 Slog.d(TAG, "Rejecting empty/invalid auth result");
             }
@@ -1505,10 +1507,6 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
             removeSelfLocked();
             return;
         }
-        final List<AutofillId> fieldIds = dataset.getFieldIds();
-        final List<AutofillValue> autofillValues = dataset.getFieldValues();
-        final AutofillId fieldId = fieldIds.get(0);
-        final AutofillValue value = autofillValues.get(0);
 
         // Update state to ensure that after filling the field here we don't end up firing another
         // autofill request that will end up showing the same suggestions to the user again. When
@@ -1524,13 +1522,18 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
 
         // Fill the value into the field.
         if (sDebug) {
-            Slog.d(TAG, "Filling after auth: fieldId=" + fieldId + ", value=" + value);
+            Slog.d(TAG, "Filling after auth: fieldId=" + fieldId + ", value=" + value
+                    + ", content=" + content);
         }
         try {
-            mClient.autofill(id, fieldIds, autofillValues, true);
+            if (content != null) {
+                mClient.autofillContent(id, fieldId, content);
+            } else {
+                mClient.autofill(id, dataset.getFieldIds(), dataset.getFieldValues(), true);
+            }
         } catch (RemoteException e) {
             Slog.w(TAG, "Error filling after auth: fieldId=" + fieldId + ", value=" + value
-                    + ", error=" + e);
+                    + ", content=" + content, e);
         }
 
         // Clear the suggestions since the user already accepted one of them.