AudioSource.java

/*
 * Copyright 2020 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.camera.video.internal;

import static androidx.camera.video.internal.AudioSource.InternalState.CONFIGURED;
import static androidx.camera.video.internal.AudioSource.InternalState.RELEASED;
import static androidx.camera.video.internal.AudioSource.InternalState.STARTED;

import android.Manifest;
import android.annotation.SuppressLint;
import android.content.Context;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioRecord;
import android.media.AudioRecordingConfiguration;
import android.media.AudioTimestamp;
import android.os.Build;

import androidx.annotation.IntRange;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.RequiresPermission;
import androidx.camera.core.Logger;
import androidx.camera.core.impl.Observable;
import androidx.camera.core.impl.annotation.ExecutedBy;
import androidx.camera.core.impl.utils.executor.CameraXExecutors;
import androidx.camera.core.impl.utils.futures.FutureCallback;
import androidx.camera.core.impl.utils.futures.Futures;
import androidx.camera.video.internal.compat.Api23Impl;
import androidx.camera.video.internal.compat.Api24Impl;
import androidx.camera.video.internal.compat.Api29Impl;
import androidx.camera.video.internal.compat.Api31Impl;
import androidx.camera.video.internal.encoder.InputBuffer;
import androidx.concurrent.futures.CallbackToFutureAdapter;
import androidx.core.util.Preconditions;

import com.google.auto.value.AutoValue;
import com.google.common.util.concurrent.ListenableFuture;

import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * AudioSource is used to obtain audio raw data and write to the buffer from {@link BufferProvider}.
 *
 * <p>The audio raw data could be one of sources from the device. The target source can be
 * specified with {@link Settings.Builder#setAudioSource(int)}.
 *
 * <p>Calling {@link #start} will start reading audio data from the target source and then write
 * the data into the buffer from {@link BufferProvider}. Calling {@link #stop} will stop sending
 * audio data. However, to really read/write data to buffer, the {@link BufferProvider}'s state
 * must be {@link BufferProvider.State#ACTIVE}. So recording may temporarily pause when the
 * {@link BufferProvider}'s state is {@link BufferProvider.State#INACTIVE}.
 *
 * @see BufferProvider
 * @see AudioRecord
 */
@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
public final class AudioSource {
    private static final String TAG = "AudioSource";
    // Common sample rate options to choose from in descending order.
    public static final List<Integer> COMMON_SAMPLE_RATES = Collections.unmodifiableList(
            Arrays.asList(48000, 44100, 22050, 11025, 8000, 4800));

    enum InternalState {
        /** The initial state or when {@link #stop} is called after started. */
        CONFIGURED,

        /** The state is when it is in {@link #CONFIGURED} state and {@link #start} is called. */
        STARTED,

        /** The state is when {@link #release} is called. */
        RELEASED,
    }

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    final Executor mExecutor;

    private AudioManager.AudioRecordingCallback mAudioRecordingCallback;

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    AtomicBoolean mSourceSilence = new AtomicBoolean(false);

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    final AudioRecord mAudioRecord;

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    final int mBufferSize;

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    InternalState mState = CONFIGURED;

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    BufferProvider.State mBufferProviderState = BufferProvider.State.INACTIVE;

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    boolean mIsSendingAudio;

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    Executor mCallbackExecutor;

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    AudioSourceCallback mAudioSourceCallback;

    // The following should only be accessed by mExecutor
    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    BufferProvider<InputBuffer> mBufferProvider;
    private FutureCallback<InputBuffer> mAcquireBufferCallback;
    private Observable.Observer<BufferProvider.State> mStateObserver;


    /**
     * Creates an AudioSource for the given settings.
     *
     * <p>It should be verified the combination of sample rate, channel count and audio format is
     * supported with {@link #isSettingsSupported(int, int, int)} before passing the settings to
     * this constructor, or an {@link UnsupportedOperationException} will be thrown.
     *
     * @param settings           The settings that will be used to configure the audio source.
     * @param executor           An executor that will be used to read audio samples in the
     *                           background. The
     *                           threads of this executor may be blocked while waiting for samples.
     * @param attributionContext A {@link Context} object that will be used to attribute the
     *                           audio to the contained {@link android.content.AttributionSource}.
     *                           Audio attribution is only available on API 31+. Setting this on
     *                           lower API levels or if the context does not contain an
     *                           attribution source, setting this context will have no effect.
     *                           This context will not be retained beyond the scope of the
     *                           constructor.
     * @throws UnsupportedOperationException if the combination of sample rate, channel count,
     *                                       and audio format in the provided settings is
     *                                       unsupported.
     * @throws AudioSourceAccessException    if the audio device is not available or cannot be
     *                                       initialized with the given settings.
     */
    @RequiresPermission(Manifest.permission.RECORD_AUDIO)
    public AudioSource(@NonNull Settings settings, @NonNull Executor executor,
            @Nullable Context attributionContext)
            throws AudioSourceAccessException {
        if (!isSettingsSupported(settings.getSampleRate(), settings.getChannelCount(),
                settings.getAudioFormat())) {
            throw new UnsupportedOperationException(String.format(
                    "The combination of sample rate %d, channel count %d and audio format"
                            + " %d is not supported.",
                    settings.getSampleRate(), settings.getChannelCount(),
                    settings.getAudioFormat()));
        }

        int minBufferSize = getMinBufferSize(settings.getSampleRate(), settings.getChannelCount(),
                settings.getAudioFormat());
        // The minBufferSize should be a positive value since the settings had already been checked
        // by the isSettingsSupported().
        Preconditions.checkState(minBufferSize > 0);

        mExecutor = CameraXExecutors.newSequentialExecutor(executor);
        mBufferSize = minBufferSize * 2;
        try {
            if (Build.VERSION.SDK_INT >= 23) {
                AudioFormat audioFormatObj = new AudioFormat.Builder()
                        .setSampleRate(settings.getSampleRate())
                        .setChannelMask(channelCountToChannelMask(settings.getChannelCount()))
                        .setEncoding(settings.getAudioFormat())
                        .build();
                AudioRecord.Builder audioRecordBuilder = Api23Impl.createAudioRecordBuilder();
                if (Build.VERSION.SDK_INT >= 31 && attributionContext != null) {
                    Api31Impl.setContext(audioRecordBuilder, attributionContext);
                }
                Api23Impl.setAudioSource(audioRecordBuilder, settings.getAudioSource());
                Api23Impl.setAudioFormat(audioRecordBuilder, audioFormatObj);
                Api23Impl.setBufferSizeInBytes(audioRecordBuilder, mBufferSize);
                mAudioRecord = Api23Impl.build(audioRecordBuilder);
            } else {
                mAudioRecord = new AudioRecord(settings.getAudioSource(),
                        settings.getSampleRate(),
                        channelCountToChannelConfig(settings.getChannelCount()),
                        settings.getAudioFormat(),
                        mBufferSize);
            }
        } catch (IllegalArgumentException e) {
            throw new AudioSourceAccessException("Unable to create AudioRecord", e);
        }

        if (mAudioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
            mAudioRecord.release();
            throw new AudioSourceAccessException("Unable to initialize AudioRecord");
        }

        if (Build.VERSION.SDK_INT >= 29) {
            mAudioRecordingCallback = new AudioRecordingApi29Callback();
            Api29Impl.registerAudioRecordingCallback(mAudioRecord, mExecutor,
                    mAudioRecordingCallback);
        }
    }

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    @RequiresApi(29)
    class AudioRecordingApi29Callback extends AudioManager.AudioRecordingCallback {
        @Override
        public void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs) {
            super.onRecordingConfigChanged(configs);
            if (mCallbackExecutor != null && mAudioSourceCallback != null) {
                for (AudioRecordingConfiguration config : configs) {
                    if (Api24Impl.getClientAudioSessionId(config)
                            == mAudioRecord.getAudioSessionId()) {
                        boolean isSilenced = Api29Impl.isClientSilenced(config);
                        if (mSourceSilence.getAndSet(isSilenced) != isSilenced) {
                            mCallbackExecutor.execute(
                                    () -> mAudioSourceCallback.onSilenced(isSilenced));
                        }
                        break;
                    }
                }
            }
        }
    }


    /**
     * Sets the {@link BufferProvider}.
     *
     * <p>A buffer provider is required to stream audio. If no buffer provider is provided, then
     * audio will be dropped until one is provided and active.
     *
     * @param bufferProvider The new buffer provider to use.
     */
    public void setBufferProvider(@NonNull BufferProvider<InputBuffer> bufferProvider) {
        mExecutor.execute(() -> {
            switch (mState) {
                case CONFIGURED:
                    // Fall-through
                case STARTED:
                    if (mBufferProvider != bufferProvider) {
                        resetBufferProvider(bufferProvider);
                    }
                    break;
                case RELEASED:
                    throw new AssertionError("AudioRecorder is released");
            }
        });
    }

    /**
     * Starts the AudioSource.
     *
     * <p>Before starting, a {@link BufferProvider} should be set with
     * {@link #setBufferProvider(BufferProvider)}. If a buffer provider is not set, audio data
     * will be dropped.
     *
     * <p>Audio data will start being sent to the {@link BufferProvider} when
     * {@link BufferProvider}'s state is {@link BufferProvider.State#ACTIVE}.
     */
    public void start() {
        mExecutor.execute(() -> {
            switch (mState) {
                case CONFIGURED:
                    setState(STARTED);
                    updateSendingAudio();
                    break;
                case STARTED:
                    // Do nothing
                    break;
                case RELEASED:
                    throw new AssertionError("AudioRecorder is released");
            }
        });
    }

    /**
     * Stops the AudioSource.
     *
     * <p>Audio data will stop being sent to the {@link BufferProvider}.
     */
    public void stop() {
        mExecutor.execute(() -> {
            switch (mState) {
                case STARTED:
                    setState(CONFIGURED);
                    updateSendingAudio();
                    break;
                case CONFIGURED:
                    // Do nothing
                    break;
                case RELEASED:
                    Logger.w(TAG, "AudioRecorder is released. "
                            + "Calling stop() is a no-op.");
            }
        });
    }

    /**
     * Releases the AudioSource.
     *
     * <p>Once the AudioSource is released, it can not be used any more.
     */
    @NonNull
    public ListenableFuture<Void> release() {
        return CallbackToFutureAdapter.getFuture(completer -> {
            mExecutor.execute(() -> {
                try {
                    switch (mState) {
                        case STARTED:
                            // Fall-through
                        case CONFIGURED:
                            resetBufferProvider(null);
                            if (Build.VERSION.SDK_INT >= 29) {
                                Api29Impl.unregisterAudioRecordingCallback(mAudioRecord,
                                        mAudioRecordingCallback);
                            }
                            mAudioRecord.release();
                            stopSendingAudio();
                            setState(RELEASED);
                            break;
                        case RELEASED:
                            // Do nothing
                            break;
                    }
                    completer.set(null);
                } catch (Throwable t) {
                    completer.setException(t);
                }
            });

            return "AudioSource-release";
        });
    }

    /**
     * Sets callback to receive configuration status.
     *
     * <p>The callback must be set before the audio source is started.
     *
     * @param executor the callback executor
     * @param callback the configuration callback
     */
    public void setAudioSourceCallback(@NonNull Executor executor,
            @NonNull AudioSourceCallback callback) {
        mExecutor.execute(() -> {
            switch (mState) {
                case CONFIGURED:
                    mCallbackExecutor = executor;
                    mAudioSourceCallback = callback;
                    break;
                case STARTED:
                    // Fall-through
                case RELEASED:
                    throw new AssertionError("The audio recording callback must be "
                            + "registered before the audio source is started.");
            }
        });
    }

    @ExecutedBy("mExecutor")
    private void resetBufferProvider(@Nullable BufferProvider<InputBuffer> bufferProvider) {
        if (mBufferProvider != null) {
            mBufferProvider.removeObserver(mStateObserver);
            mBufferProvider = null;
            mStateObserver = null;
            mAcquireBufferCallback = null;
        }
        mBufferProviderState = BufferProvider.State.INACTIVE;
        updateSendingAudio();
        if (bufferProvider != null) {
            mBufferProvider = bufferProvider;
            mStateObserver = new Observable.Observer<BufferProvider.State>() {
                @ExecutedBy("mExecutor")
                @Override
                public void onNewData(@Nullable BufferProvider.State state) {
                    if (mBufferProvider == bufferProvider) {
                        Logger.d(TAG, "Receive BufferProvider state change: "
                                + mBufferProviderState + " to " + state);
                        mBufferProviderState = state;
                        updateSendingAudio();
                    }
                }

                @ExecutedBy("mExecutor")
                @Override
                public void onError(@NonNull Throwable throwable) {
                    if (mBufferProvider == bufferProvider) {
                        notifyError(throwable);
                    }
                }
            };

            mAcquireBufferCallback = new FutureCallback<InputBuffer>() {
                @ExecutedBy("mExecutor")
                @Override
                public void onSuccess(InputBuffer inputBuffer) {
                    if (!mIsSendingAudio || mBufferProvider != bufferProvider) {
                        inputBuffer.cancel();
                        return;
                    }
                    ByteBuffer byteBuffer = inputBuffer.getByteBuffer();

                    int length = mAudioRecord.read(byteBuffer, mBufferSize);
                    if (length > 0) {
                        byteBuffer.limit(length);
                        inputBuffer.setPresentationTimeUs(generatePresentationTimeUs());
                        inputBuffer.submit();
                    } else {
                        Logger.w(TAG, "Unable to read data from AudioRecord.");
                        inputBuffer.cancel();
                    }
                    sendNextAudio();
                }

                @ExecutedBy("mExecutor")
                @Override
                public void onFailure(@NonNull Throwable throwable) {
                    if (mBufferProvider != bufferProvider) {
                        Logger.d(TAG, "Unable to get input buffer, the BufferProvider "
                                + "could be transitioning to INACTIVE state.");
                        notifyError(throwable);
                    }
                }
            };
            mBufferProvider.addObserver(mExecutor, mStateObserver);
        }
    }

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    void notifyError(Throwable throwable) {
        if (mCallbackExecutor != null && mAudioSourceCallback != null) {
            mCallbackExecutor.execute(() -> mAudioSourceCallback.onError(throwable));
        }
    }

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    @ExecutedBy("mExecutor")
    void updateSendingAudio() {
        if (mState == STARTED && mBufferProviderState == BufferProvider.State.ACTIVE) {
            startSendingAudio();
        } else {
            stopSendingAudio();
        }
    }

    @ExecutedBy("mExecutor")
    private void startSendingAudio() {
        if (mIsSendingAudio) {
            // Already started, ignore
            return;
        }
        try {
            Logger.d(TAG, "startSendingAudio");
            mAudioRecord.startRecording();
            if (mAudioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) {
                throw new IllegalStateException("Unable to start AudioRecord with state: "
                                + mAudioRecord.getRecordingState());
            }
        } catch (IllegalStateException e) {
            Logger.w(TAG, "Failed to start AudioRecord", e);
            setState(CONFIGURED);
            notifyError(new AudioSourceAccessException("Unable to start the audio record.", e));
            return;
        }
        mIsSendingAudio = true;
        sendNextAudio();
    }

    @ExecutedBy("mExecutor")
    private void stopSendingAudio() {
        if (!mIsSendingAudio) {
            // Already stopped, ignore.
            return;
        }
        mIsSendingAudio = false;
        try {
            Logger.d(TAG, "stopSendingAudio");
            mAudioRecord.stop();
            if (mAudioRecord.getRecordingState() != AudioRecord.RECORDSTATE_STOPPED) {
                throw new IllegalStateException("Unable to stop AudioRecord with state: "
                        + mAudioRecord.getRecordingState());
            }
        } catch (IllegalStateException e) {
            Logger.w(TAG, "Failed to stop AudioRecord", e);
            notifyError(e);
        }
    }

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    @ExecutedBy("mExecutor")
    void sendNextAudio() {
        Futures.addCallback(mBufferProvider.acquireBuffer(), mAcquireBufferCallback, mExecutor);
    }

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    @ExecutedBy("mExecutor")
    void setState(InternalState state) {
        Logger.d(TAG, "Transitioning internal state: " + mState + " --> " + state);
        mState = state;
    }

    @SuppressWarnings("WeakerAccess") /* synthetic accessor */
    long generatePresentationTimeUs() {
        long presentationTimeUs = -1;
        if (Build.VERSION.SDK_INT >= 24) {
            AudioTimestamp audioTimestamp = new AudioTimestamp();
            if (Api24Impl.getTimestamp(mAudioRecord, audioTimestamp,
                    AudioTimestamp.TIMEBASE_MONOTONIC) == AudioRecord.SUCCESS) {
                presentationTimeUs = TimeUnit.NANOSECONDS.toMicros(audioTimestamp.nanoTime);
            } else {
                Logger.w(TAG, "Unable to get audio timestamp");
            }
        }
        if (presentationTimeUs == -1) {
            presentationTimeUs = TimeUnit.NANOSECONDS.toMicros(System.nanoTime());
        }
        return presentationTimeUs;
    }

    /** Check if the combination of sample rate, channel count and audio format is supported. */
    public static boolean isSettingsSupported(int sampleRate, int channelCount, int audioFormat) {
        if (sampleRate <= 0 || channelCount <= 0) {
            return false;
        }
        return getMinBufferSize(sampleRate, channelCount, audioFormat) > 0;
    }

    private static int channelCountToChannelConfig(int channelCount) {
        return channelCount == 1 ? AudioFormat.CHANNEL_IN_MONO : AudioFormat.CHANNEL_IN_STEREO;
    }

    private static int channelCountToChannelMask(int channelCount) {
        // Currently equivalent to channelCountToChannelConfig, but keep this logic separate
        // since technically channel masks are different from the legacy channel config and we don't
        // want any future updates to break things.
        return channelCount == 1 ? AudioFormat.CHANNEL_IN_MONO : AudioFormat.CHANNEL_IN_STEREO;
    }

    private static int getMinBufferSize(int sampleRate, int channelCount, int audioFormat) {
        return AudioRecord.getMinBufferSize(sampleRate, channelCountToChannelConfig(channelCount),
                audioFormat);
    }

    /**
     * Settings required to configure the audio source.
     */
    @AutoValue
    public abstract static class Settings {

        /** Creates a builder for these settings. */
        @SuppressLint("Range") // Need to initialize as invalid values
        @NonNull
        public static Settings.Builder builder() {
            return new AutoValue_AudioSource_Settings.Builder()
                    .setAudioSource(-1)
                    .setSampleRate(-1)
                    .setChannelCount(-1)
                    .setAudioFormat(-1);
        }

        /** Creates a {@link Builder} initialized with the same settings as this instance. */
        @NonNull
        public abstract Builder toBuilder();

        /**
         * Gets the device audio source.
         *
         * @see android.media.MediaRecorder.AudioSource#MIC
         * @see android.media.MediaRecorder.AudioSource#CAMCORDER
         */
        public abstract int getAudioSource();

        /**
         * Gets the audio sample rate.
         */
        @IntRange(from = 1)
        public abstract int getSampleRate();

        /**
         * Gets the channel count.
         */
        @IntRange(from = 1)
        public abstract int getChannelCount();

        /**
         * Sets the audio format.
         *
         * @see AudioFormat#ENCODING_PCM_16BIT
         */
        public abstract int getAudioFormat();

        // Should not be instantiated directly
        Settings() {
        }

        /**
         * A Builder for {@link AudioSource.Settings}
         */
        @AutoValue.Builder
        public abstract static class Builder {
            /**
             * Sets the device audio source.
             *
             * @see android.media.MediaRecorder.AudioSource#MIC
             * @see android.media.MediaRecorder.AudioSource#CAMCORDER
             */
            @NonNull
            public abstract Builder setAudioSource(int audioSource);

            /**
             * Sets the audio sample rate in Hertz.
             */
            @NonNull
            public abstract Builder setSampleRate(@IntRange(from = 1) int sampleRate);

            /**
             * Sets the channel count.
             */
            @NonNull
            public abstract Builder setChannelCount(@IntRange(from = 1) int channelCount);

            /**
             * Sets the audio format.
             *
             * @see AudioFormat#ENCODING_PCM_16BIT
             */
            @NonNull
            public abstract Builder setAudioFormat(int audioFormat);

            abstract Settings autoBuild(); // Actual build method. Not public.

            /**
             * Returns the built config after performing settings validation.
             *
             * <p>It should be verified that combination of sample rate, channel count and audio
             * format is supported by {@link AudioSource#isSettingsSupported(int, int, int)} or
             * an {@link UnsupportedOperationException} will be thrown when passing the settings
             * to the
             * {@linkplain AudioSource#AudioSource(Settings, Executor, Context) AudioSource
             * constructor}.
             *
             * @throws IllegalArgumentException if a setting is missing or invalid.
             */
            @NonNull
            public final Settings build() {
                Settings settings = autoBuild();
                String missingOrInvalid = "";
                if (settings.getAudioSource() == -1) {
                    missingOrInvalid += " audioSource";
                }
                if (settings.getSampleRate() <= 0) {
                    missingOrInvalid += " sampleRate";
                }
                if (settings.getChannelCount() <= 0) {
                    missingOrInvalid += " channelCount";
                }
                if (settings.getAudioFormat() == -1) {
                    missingOrInvalid += " audioFormat";
                }

                if (!missingOrInvalid.isEmpty()) {
                    throw new IllegalArgumentException("Required settings missing or "
                            + "non-positive:" + missingOrInvalid);
                }

                return settings;
            }

            // Should not be instantiated directly
            Builder() {
            }
        }
    }

    /**
     * The callback for receiving the audio source status.
     */
    public interface AudioSourceCallback {
        /**
         * The method called when the audio source is silenced.
         *
         * <p>The audio source is silenced when the audio record is occupied by privilege
         * application. When it happens, the audio source will keep providing audio data with
         * silence sample.
         */
        void onSilenced(boolean silenced);

        /**
         * The method called when the audio source encountered errors.
         */
        void onError(@NonNull Throwable t);
    }
}