/* * Copyright (C) 2020 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.server.musicrecognition; import static android.Manifest.permission.RECORD_AUDIO; import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_AUDIO_UNAVAILABLE; import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_SERVICE_KILLED; import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_SERVICE_UNAVAILABLE; import static android.media.musicrecognition.MusicRecognitionManager.RecognitionFailureCode; import android.Manifest; import android.annotation.NonNull; import android.annotation.Nullable; import android.app.AppGlobals; import android.app.AppOpsManager; import android.content.ComponentName; import android.content.pm.PackageManager; import android.content.pm.ServiceInfo; import android.media.AudioRecord; import android.media.MediaMetadata; import android.media.musicrecognition.IMusicRecognitionManagerCallback; import android.media.musicrecognition.IMusicRecognitionServiceCallback; import android.media.musicrecognition.RecognitionRequest; import android.os.Bundle; import android.os.IBinder; import android.os.ParcelFileDescriptor; import android.os.RemoteException; import android.util.Pair; import android.util.Slog; import com.android.internal.annotations.GuardedBy; import com.android.server.infra.AbstractPerUserSystemService; import java.io.IOException; import java.io.OutputStream; import java.util.Objects; import java.util.concurrent.CompletableFuture; /** * Handles per-user requests received by * {@link MusicRecognitionManagerService}. Opens an audio stream from the * dsp and writes it into a pipe to {@link RemoteMusicRecognitionService}. */ public final class MusicRecognitionManagerPerUserService extends AbstractPerUserSystemService implements RemoteMusicRecognitionService.Callbacks { private static final String TAG = MusicRecognitionManagerPerUserService.class.getSimpleName(); private static final String MUSIC_RECOGNITION_MANAGER_ATTRIBUTION_TAG = "MusicRecognitionManagerService"; private static final String KEY_MUSIC_RECOGNITION_SERVICE_ATTRIBUTION_TAG = "android.media.musicrecognition.attributiontag"; // Number of bytes per sample of audio (which is a short). private static final int BYTES_PER_SAMPLE = 2; private static final int MAX_STREAMING_SECONDS = 24; @Nullable @GuardedBy("mLock") private RemoteMusicRecognitionService mRemoteService; private final AppOpsManager mAppOpsManager; private final String mAttributionMessage; // Service info of the remote MusicRecognitionService (which the audio gets forwarded to). private ServiceInfo mServiceInfo; private CompletableFuture mAttributionTagFuture; MusicRecognitionManagerPerUserService( @NonNull MusicRecognitionManagerService primary, @NonNull Object lock, int userId) { super(primary, lock, userId); // When attributing audio-access, this establishes that audio access is performed by // MusicRecognitionManager (on behalf of the receiving service, whose attribution tag, // provided by mAttributionTagFuture, is used for the actual calls to startProxyOp(...). mAppOpsManager = getContext().createAttributionContext( MUSIC_RECOGNITION_MANAGER_ATTRIBUTION_TAG).getSystemService(AppOpsManager.class); mAttributionMessage = String.format("MusicRecognitionManager.invokedByUid.%s", userId); mAttributionTagFuture = null; mServiceInfo = null; } @NonNull @GuardedBy("mLock") @Override protected ServiceInfo newServiceInfoLocked(@NonNull ComponentName serviceComponent) throws PackageManager.NameNotFoundException { ServiceInfo si; try { si = AppGlobals.getPackageManager().getServiceInfo(serviceComponent, PackageManager.GET_META_DATA, mUserId); } catch (RemoteException e) { throw new PackageManager.NameNotFoundException( "Could not get service for " + serviceComponent); } if (!Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE.equals(si.permission)) { Slog.w(TAG, "MusicRecognitionService from '" + si.packageName + "' does not require permission " + Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE); throw new SecurityException("Service does not require permission " + Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE); } // TODO(b/158194857): check process which owns the service has RECORD_AUDIO permission. How? return si; } @GuardedBy("mLock") @Nullable private RemoteMusicRecognitionService ensureRemoteServiceLocked( IMusicRecognitionManagerCallback clientCallback) { if (mRemoteService == null) { final String serviceName = getComponentNameLocked(); if (serviceName == null) { if (mMaster.verbose) { Slog.v(TAG, "ensureRemoteServiceLocked(): not set"); } return null; } ComponentName serviceComponent = ComponentName.unflattenFromString(serviceName); mRemoteService = new RemoteMusicRecognitionService(getContext(), serviceComponent, mUserId, this, new MusicRecognitionServiceCallback(clientCallback), mMaster.isBindInstantServiceAllowed(), mMaster.verbose); try { mServiceInfo = getContext().getPackageManager().getServiceInfo( mRemoteService.getComponentName(), PackageManager.GET_META_DATA); mAttributionTagFuture = mRemoteService.getAttributionTag(); Slog.i(TAG, "Remote service bound: " + mRemoteService.getComponentName()); } catch (PackageManager.NameNotFoundException e) { Slog.e(TAG, "Service was not found.", e); } } return mRemoteService; } /** * Read audio from the given capture session using an AudioRecord and writes it to a * ParcelFileDescriptor. */ @GuardedBy("mLock") public void beginRecognitionLocked( @NonNull RecognitionRequest recognitionRequest, @NonNull IBinder callback) { IMusicRecognitionManagerCallback clientCallback = IMusicRecognitionManagerCallback.Stub.asInterface(callback); mRemoteService = ensureRemoteServiceLocked(clientCallback); if (mRemoteService == null) { try { clientCallback.onRecognitionFailed( RECOGNITION_FAILED_SERVICE_UNAVAILABLE); } catch (RemoteException e) { // Ignored. } return; } Pair clientPipe = createPipe(); if (clientPipe == null) { try { clientCallback.onRecognitionFailed( RECOGNITION_FAILED_AUDIO_UNAVAILABLE); } catch (RemoteException ignored) { // Ignored. } return; } ParcelFileDescriptor audioSink = clientPipe.second; ParcelFileDescriptor clientRead = clientPipe.first; mAttributionTagFuture.thenAcceptAsync( tag -> { streamAudio(tag, recognitionRequest, clientCallback, audioSink); }, mMaster.mExecutorService); // Send the pipe down to the lookup service while we write to it asynchronously. mRemoteService.onAudioStreamStarted(clientRead, recognitionRequest.getAudioFormat()); } /** * Streams audio based on given request to the given audioSink. Notifies callback of errors. * * @param recognitionRequest the recognition request specifying audio parameters. * @param clientCallback the callback to notify on errors. * @param audioSink the sink to which to stream audio to. */ private void streamAudio(@Nullable String attributionTag, @NonNull RecognitionRequest recognitionRequest, IMusicRecognitionManagerCallback clientCallback, ParcelFileDescriptor audioSink) { int maxAudioLengthSeconds = Math.min(recognitionRequest.getMaxAudioLengthSeconds(), MAX_STREAMING_SECONDS); if (maxAudioLengthSeconds <= 0) { // TODO(b/192992319): A request to stream 0s of audio can be used to initialize the // music recognition service implementation, hence not reporting an error here. // The TODO for Android T is to move this functionality into an init() API call. Slog.i(TAG, "No audio requested. Closing stream."); try { audioSink.close(); clientCallback.onAudioStreamClosed(); } catch (IOException e) { Slog.e(TAG, "Problem closing stream.", e); } catch (RemoteException ignored) { // Ignored. } return; } try { startRecordAudioOp(attributionTag); } catch (SecurityException e) { // A security exception can occur if the MusicRecognitionService (receiving the audio) // does not (or does no longer) hold the necessary permissions to record audio. Slog.e(TAG, "RECORD_AUDIO op not permitted on behalf of " + mServiceInfo.getComponentName(), e); try { clientCallback.onRecognitionFailed( RECOGNITION_FAILED_AUDIO_UNAVAILABLE); } catch (RemoteException ignored) { // Ignored. } return; } AudioRecord audioRecord = createAudioRecord(recognitionRequest, maxAudioLengthSeconds); try (OutputStream fos = new ParcelFileDescriptor.AutoCloseOutputStream(audioSink)) { streamAudio(recognitionRequest, maxAudioLengthSeconds, audioRecord, fos); } catch (IOException e) { Slog.e(TAG, "Audio streaming stopped.", e); } finally { audioRecord.release(); finishRecordAudioOp(attributionTag); try { clientCallback.onAudioStreamClosed(); } catch (RemoteException ignored) { // Ignored. } } } /** Performs the actual streaming from audioRecord into outputStream. **/ private void streamAudio(@NonNull RecognitionRequest recognitionRequest, int maxAudioLengthSeconds, AudioRecord audioRecord, OutputStream outputStream) throws IOException { int halfSecondBufferSize = audioRecord.getBufferSizeInFrames() / maxAudioLengthSeconds; byte[] byteBuffer = new byte[halfSecondBufferSize]; int bytesRead = 0; int totalBytesRead = 0; int ignoreBytes = recognitionRequest.getIgnoreBeginningFrames() * BYTES_PER_SAMPLE; audioRecord.startRecording(); while (bytesRead >= 0 && totalBytesRead < audioRecord.getBufferSizeInFrames() * BYTES_PER_SAMPLE && mRemoteService != null) { bytesRead = audioRecord.read(byteBuffer, 0, byteBuffer.length); if (bytesRead > 0) { totalBytesRead += bytesRead; // If we are ignoring the first x bytes, update that counter. if (ignoreBytes > 0) { ignoreBytes -= bytesRead; // If we've dipped negative, we've skipped through all ignored bytes // and then some. Write out the bytes we shouldn't have skipped. if (ignoreBytes < 0) { outputStream.write(byteBuffer, bytesRead + ignoreBytes, -ignoreBytes); } } else { outputStream.write(byteBuffer); } } } Slog.i(TAG, String.format("Streamed %s bytes from audio record", totalBytesRead)); } /** * Callback invoked by {@link android.service.musicrecognition.MusicRecognitionService} to pass * back the music search result. */ final class MusicRecognitionServiceCallback extends IMusicRecognitionServiceCallback.Stub { private final IMusicRecognitionManagerCallback mClientCallback; private MusicRecognitionServiceCallback(IMusicRecognitionManagerCallback clientCallback) { mClientCallback = clientCallback; } @Override public void onRecognitionSucceeded(MediaMetadata result, Bundle extras) { try { sanitizeBundle(extras); mClientCallback.onRecognitionSucceeded(result, extras); } catch (RemoteException ignored) { // Ignored. } destroyService(); } @Override public void onRecognitionFailed(@RecognitionFailureCode int failureCode) { try { mClientCallback.onRecognitionFailed(failureCode); } catch (RemoteException ignored) { // Ignored. } destroyService(); } private IMusicRecognitionManagerCallback getClientCallback() { return mClientCallback; } } @Override public void onServiceDied(@NonNull RemoteMusicRecognitionService service) { try { service.getServerCallback().getClientCallback().onRecognitionFailed( RECOGNITION_FAILED_SERVICE_KILLED); } catch (RemoteException e) { // Ignored. } Slog.w(TAG, "remote service died: " + service); destroyService(); } @GuardedBy("mLock") private void destroyService() { synchronized (mLock) { if (mRemoteService != null) { mRemoteService.destroy(); mRemoteService = null; } } } /** * Tracks that the RECORD_AUDIO operation started (attributes it to the service receiving the * audio). */ private void startRecordAudioOp(@Nullable String attributionTag) { int status = mAppOpsManager.startProxyOp( Objects.requireNonNull(AppOpsManager.permissionToOp(RECORD_AUDIO)), mServiceInfo.applicationInfo.uid, mServiceInfo.packageName, attributionTag, mAttributionMessage); // The above should already throw a SecurityException. This is just a fallback. if (status != AppOpsManager.MODE_ALLOWED) { throw new SecurityException(String.format( "Failed to obtain RECORD_AUDIO permission (status: %d) for " + "receiving service: %s", status, mServiceInfo.getComponentName())); } Slog.i(TAG, String.format( "Starting audio streaming. Attributing to %s (%d) with tag '%s'", mServiceInfo.packageName, mServiceInfo.applicationInfo.uid, attributionTag)); } /** Tracks that the RECORD_AUDIO operation finished. */ private void finishRecordAudioOp(@Nullable String attributionTag) { mAppOpsManager.finishProxyOp( Objects.requireNonNull(AppOpsManager.permissionToOp(RECORD_AUDIO)), mServiceInfo.applicationInfo.uid, mServiceInfo.packageName, attributionTag); } /** Establishes an audio stream from the DSP audio source. */ private static AudioRecord createAudioRecord( @NonNull RecognitionRequest recognitionRequest, int maxAudioLengthSeconds) { int sampleRate = recognitionRequest.getAudioFormat().getSampleRate(); int bufferSize = getBufferSizeInBytes(sampleRate, maxAudioLengthSeconds); return new AudioRecord(recognitionRequest.getAudioAttributes(), recognitionRequest.getAudioFormat(), bufferSize, recognitionRequest.getCaptureSession()); } /** * Returns the number of bytes required to store {@code bufferLengthSeconds} of audio sampled at * {@code sampleRate} Hz, using the format returned by DSP audio capture. */ private static int getBufferSizeInBytes(int sampleRate, int bufferLengthSeconds) { return BYTES_PER_SAMPLE * sampleRate * bufferLengthSeconds; } private static Pair createPipe() { ParcelFileDescriptor[] fileDescriptors; try { fileDescriptors = ParcelFileDescriptor.createPipe(); } catch (IOException e) { Slog.e(TAG, "Failed to create audio stream pipe", e); return null; } if (fileDescriptors.length != 2) { Slog.e(TAG, "Failed to create audio stream pipe, " + "unexpected number of file descriptors"); return null; } if (!fileDescriptors[0].getFileDescriptor().valid() || !fileDescriptors[1].getFileDescriptor().valid()) { Slog.e(TAG, "Failed to create audio stream pipe, didn't " + "receive a pair of valid file descriptors."); return null; } return Pair.create(fileDescriptors[0], fileDescriptors[1]); } /** Removes remote objects from the bundle. */ private static void sanitizeBundle(@Nullable Bundle bundle) { if (bundle == null) { return; } for (String key : bundle.keySet()) { Object o = bundle.get(key); if (o instanceof Bundle) { sanitizeBundle((Bundle) o); } else if (o instanceof IBinder || o instanceof ParcelFileDescriptor) { bundle.remove(key); } } } }