ExtensionsManager.java

/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package androidx.camera.extensions;

import android.content.Context;
import android.hardware.camera2.CameraMetadata;
import android.hardware.camera2.params.StreamConfigurationMap;
import android.util.Range;

import androidx.annotation.GuardedBy;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.RestrictTo;
import androidx.annotation.VisibleForTesting;
import androidx.camera.core.CameraProvider;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageCapture;
import androidx.camera.core.Logger;
import androidx.camera.core.Preview;
import androidx.camera.core.impl.utils.ContextUtil;
import androidx.camera.core.impl.utils.executor.CameraXExecutors;
import androidx.camera.core.impl.utils.futures.Futures;
import androidx.camera.extensions.impl.InitializerImpl;
import androidx.camera.extensions.internal.ExtensionVersion;
import androidx.camera.extensions.internal.Version;
import androidx.camera.extensions.internal.VersionName;
import androidx.concurrent.futures.CallbackToFutureAdapter;
import androidx.lifecycle.LifecycleOwner;

import com.google.common.util.concurrent.ListenableFuture;

import java.util.concurrent.ExecutionException;

/**
 * Provides interfaces for third party app developers to get capabilities info of extension
 * functions.
 *
 * <p>Many Android devices contain powerful cameras, with manufacturers devoting a lot of effort
 * to build many cutting-edge features, or special effects, into these camera devices.
 * <code>CameraX Extensions</code> allows third party apps to enable the available extension
 * modes on the supported devices. The extension modes which might be supported via <code>CameraX
 * Extensions</code> are {@link ExtensionMode#BOKEH}, {@link ExtensionMode#HDR},
 * {@link ExtensionMode#NIGHT}, {@link ExtensionMode#FACE_RETOUCH} and {@link ExtensionMode#AUTO}
 * . The known supported devices are listed in the
 * <a href="https://developer.android.com/training/camera/supported-devices">Supported devices</a>
 * page.
 *
 * <p><code>CameraX Extensions</code> are built on the top of <code>CameraX Core</code> libraries
 * . To enable an extension mode, an {@link ExtensionsManager} instance needs to be retrieved
 * first with {@link #getInstanceAsync(Context, CameraProvider)}. Only a single
 * {@link ExtensionsManager} instance can exist within a process. After retrieving the
 * {@link ExtensionsManager} instance, the availability of a specific extension mode can be
 * checked by {@link #isExtensionAvailable(CameraSelector, int)}. For an available extension
 * mode, an extension enabled {@link CameraSelector} can be obtained by calling
 * {@link #getExtensionEnabledCameraSelector(CameraSelector, int)}. After binding use cases by
 * the extension enabled {@link CameraSelector}, the extension mode will be applied to the bound
 * {@link Preview} and {@link ImageCapture}. The following sample code describes how to enable an
 * extension mode for use cases.
 * </p>
 * <pre>
 * void bindUseCasesWithBokehMode() {
 *     // Create a camera provider
 *     ProcessCameraProvider cameraProvider = ... // Get the provider instance
 *     // Call the getInstance function to retrieve a ListenableFuture object
 *     ListenableFuture future = ExtensionsManager.getInstance(context, cameraProvider);
 *
 *     // Obtain the ExtensionsManager instance from the returned ListenableFuture object
 *     future.addListener(() -> {
 *         try {
 *             ExtensionsManager extensionsManager = future.get()
 *
 *             // Query if extension is available.
 *             if (mExtensionsManager.isExtensionAvailable(DEFAULT_BACK_CAMERA,
 *                        ExtensionMode.BOKEH)) {
 *                 // Needs to unbind all use cases before enabling different extension mode.
 *                 cameraProvider.unbindAll();
 *
 *                 // Retrieve extension enabled camera selector
 *                 CameraSelector extensionCameraSelector;
 *                 extensionCameraSelector = extensionsManager.getExtensionEnabledCameraSelector(
 *                         DEFAULT_BACK_CAMERA, ExtensionMode.BOKEH);
 *
 *                 // Bind image capture and preview use cases with the extension enabled camera
 *                 // selector.
 *                 ImageCapture imageCapture = new ImageCapture.Builder().build();
 *                 Preview preview = new Preview.Builder().build();
 *                 cameraProvider.bindToLifecycle(lifecycleOwner, extensionCameraSelector,
 *                         imageCapture, preview);
 *             }
 *         } catch (ExecutionException | InterruptedException e) {
 *             // This should not happen unless the future is cancelled or the thread is
 *             // interrupted by applications.
 *         }
 *     }, ContextCompact.getMainExecutor(context));
 * }
 * </pre>
 *
 * <p>Without enabling <code>CameraX Extensions</code>, any device should be able to support the
 * use cases combination of {@link ImageCapture}, {@link Preview} and {@link ImageAnalysis}. To
 * support the <code>CameraX Extensions</code> functionality, the {@link ImageCapture} or
 * {@link Preview} might need to occupy a different format of stream. This might restrict the app
 * to not be able to bind {@link ImageCapture}, {@link Preview} and {@link ImageAnalysis} at the
 * same time if the device's hardware level is not
 * {@link CameraMetadata#INFO_SUPPORTED_HARDWARE_LEVEL_FULL} or above. If enabling an extension
 * mode is more important and the {@link ImageAnalysis} could be optional to the app design, the
 * extension mode can be enabled successfully when only binding {@link ImageCapture},
 * {@link Preview} even if the device's hardware level is
 * {@link CameraMetadata#INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED}.
 *
 * <p><code>CameraX Extensions</code> currently can only support {@link ImageCapture} and
 * {@link Preview}. The {@linkplain androidx.camera.video.VideoCapture} can't be supported yet.
 * If the app binds {@linkplain androidx.camera.video.VideoCapture} and
 * enables any extension mode, an {@link IllegalArgumentException} will be thrown.
 *
 * <p>For some devices, the vendor library implementation might only support a subset of the all
 * supported sizes retrieved by {@link StreamConfigurationMap#getOutputSizes(int)}. <code>CameraX
 * </code> will select the supported sizes for the use cases according to the use cases'
 * configuration and combination.
 */
@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
public final class ExtensionsManager {
    private static final String TAG = "ExtensionsManager";

    enum ExtensionsAvailability {
        /**
         * The device extensions library exists and has been correctly loaded.
         */
        LIBRARY_AVAILABLE,
        /**
         * The device extensions library exists. However, there was some error loading the library.
         */
        LIBRARY_UNAVAILABLE_ERROR_LOADING,
        /**
         * The device extensions library exists. However, the library is missing implementations.
         */
        LIBRARY_UNAVAILABLE_MISSING_IMPLEMENTATION,
        /**
         * There are no extensions available on this device.
         */
        NONE
    }

    // Singleton instance of the Extensions object
    private static final Object EXTENSIONS_LOCK = new Object();

    @GuardedBy("EXTENSIONS_LOCK")
    private static ListenableFuture<ExtensionsManager> sInitializeFuture;

    @GuardedBy("EXTENSIONS_LOCK")
    private static ListenableFuture<Void> sDeinitializeFuture;

    @GuardedBy("EXTENSIONS_LOCK")
    private static ExtensionsManager sExtensionsManager;

    private final ExtensionsAvailability mExtensionsAvailability;

    private final ExtensionsInfo mExtensionsInfo;

    /**
     * Retrieves the {@link ExtensionsManager} associated with the current process.
     *
     * <p>An application must wait until the {@link ListenableFuture} completes to get an
     * {@link ExtensionsManager} instance. The {@link ExtensionsManager} instance can be used to
     * access the extensions related functions.
     *
     * @param context The context to initialize the extensions library.
     * @param cameraProvider     A {@link CameraProvider} will be used to query the information
     *                           of cameras on the device. The {@link CameraProvider} can be the
     *                           {@link androidx.camera.lifecycle.ProcessCameraProvider}
     *                           which is obtained by
     *                 {@link androidx.camera.lifecycle.ProcessCameraProvider#getInstance(Context)}.
     */
    @NonNull
    public static ListenableFuture<ExtensionsManager> getInstanceAsync(@NonNull Context context,
            @NonNull CameraProvider cameraProvider) {
        return getInstanceAsync(context, cameraProvider, VersionName.getCurrentVersion());
    }

    static ListenableFuture<ExtensionsManager> getInstanceAsync(@NonNull Context context,
            @NonNull CameraProvider cameraProvider, @NonNull VersionName versionName) {
        synchronized (EXTENSIONS_LOCK) {
            if (sDeinitializeFuture != null && !sDeinitializeFuture.isDone()) {
                throw new IllegalStateException("Not yet done deinitializing extensions");
            }
            sDeinitializeFuture = null;

            // Will be initialized, with an empty implementation which will report all extensions
            // as unavailable
            if (ExtensionVersion.getRuntimeVersion() == null) {
                return Futures.immediateFuture(
                        getOrCreateExtensionsManager(ExtensionsAvailability.NONE, cameraProvider));
            }

            // Prior to 1.1 no additional initialization logic required
            if (ExtensionVersion.getRuntimeVersion().compareTo(Version.VERSION_1_1) < 0) {
                return Futures.immediateFuture(
                        getOrCreateExtensionsManager(ExtensionsAvailability.LIBRARY_AVAILABLE,
                                cameraProvider));
            }

            if (sInitializeFuture == null) {
                sInitializeFuture = CallbackToFutureAdapter.getFuture(completer -> {
                    try {
                        InitializerImpl.init(versionName.toVersionString(),
                                ContextUtil.getApplicationContext(context),
                                new InitializerImpl.OnExtensionsInitializedCallback() {
                                    @Override
                                    public void onSuccess() {
                                        Logger.d(TAG, "Successfully initialized extensions");
                                        completer.set(getOrCreateExtensionsManager(
                                                ExtensionsAvailability.LIBRARY_AVAILABLE,
                                                cameraProvider));
                                    }

                                    @Override
                                    public void onFailure(int error) {
                                        Logger.e(TAG, "Failed to initialize extensions");
                                        completer.set(getOrCreateExtensionsManager(
                                                ExtensionsAvailability
                                                        .LIBRARY_UNAVAILABLE_ERROR_LOADING,
                                                cameraProvider));
                                    }
                                },
                                CameraXExecutors.directExecutor());
                    } catch (NoSuchMethodError | NoClassDefFoundError | AbstractMethodError e) {
                        Logger.e(TAG, "Failed to initialize extensions. Some classes or methods "
                                + "are missed in the vendor library. " + e);
                        completer.set(getOrCreateExtensionsManager(
                                ExtensionsAvailability.LIBRARY_UNAVAILABLE_MISSING_IMPLEMENTATION,
                                cameraProvider));
                    } catch (RuntimeException e) {
                        // Catches all unexpected runtime exceptions and still returns an
                        // ExtensionsManager instance which performs default behavior.
                        Logger.e(TAG,
                                "Failed to initialize extensions. Something wents wrong when "
                                        + "initializing the vendor library. "
                                        + e);
                        completer.set(getOrCreateExtensionsManager(
                                ExtensionsAvailability.LIBRARY_UNAVAILABLE_ERROR_LOADING,
                                cameraProvider));
                    }

                    return "Initialize extensions";
                });
            }

            return sInitializeFuture;
        }
    }

    /**
     * Shutdown the extensions.
     *
     * <p> For the moment only used for testing to shutdown the extensions. Calling this function
     * can deinitialize the extensions vendor library and release the created
     * {@link ExtensionsManager} instance. Tests should wait until the returned future is
     * complete. Then, tests can call the
     * {@link ExtensionsManager#getInstanceAsync(Context, CameraProvider)} function again to
     * initialize a new {@link ExtensionsManager} instance.
     *
     * @hide
     */
    // TODO: Will need to be rewritten to be threadsafe with use in conjunction with
    //  ExtensionsManager.init(...) if this is to be released for use outside of testing.
    @RestrictTo(RestrictTo.Scope.TESTS)
    @NonNull
    public ListenableFuture<Void> shutdown() {
        synchronized (EXTENSIONS_LOCK) {
            if (ExtensionVersion.getRuntimeVersion() == null) {
                sInitializeFuture = null;
                sExtensionsManager = null;
                return Futures.immediateFuture(null);
            }

            // If initialization not yet attempted then deinit should succeed immediately.
            if (sInitializeFuture == null) {
                return Futures.immediateFuture(null);
            }

            // If already in progress of deinit then return the future
            if (sDeinitializeFuture != null) {
                return sDeinitializeFuture;
            }

            ExtensionsAvailability availability;

            // Wait for the extension to be initialized before deinitializing. Block since
            // this is only used for testing.
            try {
                sInitializeFuture.get();
                sInitializeFuture = null;
                availability = sExtensionsManager.mExtensionsAvailability;
                sExtensionsManager = null;
            } catch (ExecutionException | InterruptedException e) {
                sDeinitializeFuture = Futures.immediateFailedFuture(e);
                return sDeinitializeFuture;
            }

            // Once extension has been initialized start the deinit call
            if (availability == ExtensionsAvailability.LIBRARY_AVAILABLE) {
                sDeinitializeFuture = CallbackToFutureAdapter.getFuture(completer -> {
                    try {
                        InitializerImpl.deinit(
                                new InitializerImpl.OnExtensionsDeinitializedCallback() {
                                    @Override
                                    public void onSuccess() {
                                        completer.set(null);
                                    }

                                    @Override
                                    public void onFailure(int error) {
                                        completer.setException(new Exception("Failed to "
                                                + "deinitialize extensions."));
                                    }
                                },
                                CameraXExecutors.directExecutor());
                    } catch (NoSuchMethodError | NoClassDefFoundError e) {
                        completer.setException(e);
                    }
                    return null;
                });
            } else {
                sDeinitializeFuture = Futures.immediateFuture(null);
            }
            return sDeinitializeFuture;
        }
    }

    static ExtensionsManager getOrCreateExtensionsManager(
            @NonNull ExtensionsAvailability extensionsAvailability,
            @NonNull CameraProvider cameraProvider) {
        synchronized (EXTENSIONS_LOCK) {
            if (sExtensionsManager != null) {
                return sExtensionsManager;
            }

            sExtensionsManager = new ExtensionsManager(extensionsAvailability, cameraProvider);

            return sExtensionsManager;
        }
    }

    /**
     * Returns a modified {@link CameraSelector} that will enable the specified extension mode.
     *
     * <p>The returned extension {@link CameraSelector} can be used to bind use cases to a
     * desired {@link LifecycleOwner} and then the specified extension mode will be enabled on
     * the camera.
     *
     * @param baseCameraSelector The base {@link CameraSelector} on top of which the extension
     *                           config is applied.
     *                           {@link #isExtensionAvailable(CameraSelector, int)} can be used
     *                           to check whether any camera can support the specified extension
     *                           mode for the base camera selector.
     * @param mode               The target extension mode.
     * @return a {@link CameraSelector} for the specified Extensions mode.
     * @throws IllegalArgumentException If this device doesn't support extensions function, no
     *                                  camera can be found to support the specified extension
     *                                  mode, or the base {@link CameraSelector} has contained
     *                                  extension related configuration in it.
     */
    @NonNull
    public CameraSelector getExtensionEnabledCameraSelector(
            @NonNull CameraSelector baseCameraSelector, @ExtensionMode.Mode int mode) {
        // Directly return the input baseCameraSelector if the target extension mode is NONE.
        if (mode == ExtensionMode.NONE) {
            return baseCameraSelector;
        }

        if (mExtensionsAvailability != ExtensionsAvailability.LIBRARY_AVAILABLE) {
            throw new IllegalArgumentException("This device doesn't support extensions function! "
                    + "isExtensionAvailable should be checked first before calling "
                    + "getExtensionEnabledCameraSelector.");
        }

        return mExtensionsInfo.getExtensionCameraSelectorAndInjectCameraConfig(baseCameraSelector,
                mode);
    }

    /**
     * Returns true if the particular extension mode is available for the specified
     * {@link CameraSelector}.
     *
     * @param baseCameraSelector The base {@link CameraSelector} to find a camera to use.
     * @param mode               The target extension mode to support.
     */
    public boolean isExtensionAvailable(@NonNull CameraSelector baseCameraSelector,
            @ExtensionMode.Mode int mode) {
        if (mode == ExtensionMode.NONE) {
            return true;
        }

        if (mExtensionsAvailability != ExtensionsAvailability.LIBRARY_AVAILABLE) {
            // Returns false if extensions are not available.
            return false;
        }

        return mExtensionsInfo.isExtensionAvailable(baseCameraSelector, mode);
    }

    /**
     * Returns the estimated capture latency range in milliseconds for the target camera and
     * extension mode.
     *
     * <p>This includes the time spent processing the multi-frame capture request along with any
     * additional time for encoding of the processed buffer in the framework if necessary.
     *
     * @param cameraSelector    The {@link CameraSelector} to find a camera which supports the
     *                          specified extension mode.
     * @param mode              The extension mode to check.
     * @return the range of estimated minimal and maximal capture latency in milliseconds.
     * Returns null if no capture latency info can be provided.
     * @throws IllegalArgumentException If this device doesn't support extensions function, or no
     *                                  camera can be found to support the specified extension mode.
     */
    @Nullable
    public Range<Long> getEstimatedCaptureLatencyRange(@NonNull CameraSelector cameraSelector,
            @ExtensionMode.Mode int mode) {
        if (mode == ExtensionMode.NONE
                || mExtensionsAvailability != ExtensionsAvailability.LIBRARY_AVAILABLE) {
            throw new IllegalArgumentException(
                    "No camera can be found to support the specified extensions mode! "
                            + "isExtensionAvailable should be checked first before calling "
                            + "getEstimatedCaptureLatencyRange.");
        }

        return mExtensionsInfo.getEstimatedCaptureLatencyRange(cameraSelector, mode, null);
    }

    @VisibleForTesting
    @NonNull
    ExtensionsAvailability getExtensionsAvailability() {
        return mExtensionsAvailability;
    }

    private ExtensionsManager(@NonNull ExtensionsAvailability extensionsAvailability,
            @NonNull CameraProvider cameraProvider) {
        mExtensionsAvailability = extensionsAvailability;
        mExtensionsInfo = new ExtensionsInfo(cameraProvider);
    }
}