ZslControlImpl.java

/*
 * Copyright 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.camera.camera2.internal;

import static android.graphics.ImageFormat.PRIVATE;
import static android.hardware.camera2.CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_PRIVATE_REPROCESSING;

import static androidx.camera.camera2.internal.ZslUtil.isCapabilitySupported;

import android.graphics.ImageFormat;
import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.params.InputConfiguration;
import android.hardware.camera2.params.StreamConfigurationMap;
import android.media.Image;
import android.media.ImageWriter;
import android.os.Build;
import android.util.Size;
import android.view.Surface;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.OptIn;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import androidx.camera.camera2.internal.compat.CameraCharacteristicsCompat;
import androidx.camera.camera2.internal.compat.quirk.DeviceQuirks;
import androidx.camera.camera2.internal.compat.quirk.ZslDisablerQuirk;
import androidx.camera.core.ExperimentalGetImage;
import androidx.camera.core.ImageProxy;
import androidx.camera.core.Logger;
import androidx.camera.core.MetadataImageReader;
import androidx.camera.core.SafeCloseImageReaderProxy;
import androidx.camera.core.impl.CameraCaptureCallback;
import androidx.camera.core.impl.DeferrableSurface;
import androidx.camera.core.impl.ImmediateSurface;
import androidx.camera.core.impl.SessionConfig;
import androidx.camera.core.impl.utils.CompareSizesByArea;
import androidx.camera.core.impl.utils.executor.CameraXExecutors;
import androidx.camera.core.internal.compat.ImageWriterCompat;
import androidx.camera.core.internal.utils.ZslRingBuffer;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;

/**
 * Implementation for {@link ZslControl}.
 */
@RequiresApi(23)
final class ZslControlImpl implements ZslControl {

    private static final String TAG = "ZslControlImpl";

    @VisibleForTesting
    static final int RING_BUFFER_CAPACITY = 3;

    @VisibleForTesting
    static final int MAX_IMAGES = RING_BUFFER_CAPACITY * 3;

    @NonNull
    private final Map<Integer, Size> mReprocessingInputSizeMap;

    @NonNull
    private final CameraCharacteristicsCompat mCameraCharacteristicsCompat;

    @VisibleForTesting
    @SuppressWarnings("WeakerAccess")
    @NonNull
    final ZslRingBuffer mImageRingBuffer;

    private boolean mIsZslDisabledByUseCaseConfig = false;
    private boolean mIsZslDisabledByFlashMode = false;
    private boolean mIsPrivateReprocessingSupported = false;

    private boolean mShouldZslDisabledByQuirks = false;

    @SuppressWarnings("WeakerAccess")
    SafeCloseImageReaderProxy mReprocessingImageReader;
    private CameraCaptureCallback mMetadataMatchingCaptureCallback;
    private DeferrableSurface mReprocessingImageDeferrableSurface;

    @Nullable
    ImageWriter mReprocessingImageWriter;

    ZslControlImpl(@NonNull CameraCharacteristicsCompat cameraCharacteristicsCompat) {
        mCameraCharacteristicsCompat = cameraCharacteristicsCompat;
        mIsPrivateReprocessingSupported =
                isCapabilitySupported(mCameraCharacteristicsCompat,
                        REQUEST_AVAILABLE_CAPABILITIES_PRIVATE_REPROCESSING);

        mReprocessingInputSizeMap = createReprocessingInputSizeMap(mCameraCharacteristicsCompat);

        mShouldZslDisabledByQuirks = DeviceQuirks.get(ZslDisablerQuirk.class) != null;

        mImageRingBuffer = new ZslRingBuffer(
                RING_BUFFER_CAPACITY,
                imageProxy -> imageProxy.close());
    }

    @Override
    public void setZslDisabledByUserCaseConfig(boolean disabled) {
        mIsZslDisabledByUseCaseConfig = disabled;
    }

    @Override
    public boolean isZslDisabledByUserCaseConfig() {
        return mIsZslDisabledByUseCaseConfig;
    }

    @Override
    public void setZslDisabledByFlashMode(boolean disabled) {
        mIsZslDisabledByFlashMode = disabled;
    }

    @Override
    public boolean isZslDisabledByFlashMode() {
        return mIsZslDisabledByFlashMode;
    }

    @Override
    public void addZslConfig(@NonNull SessionConfig.Builder sessionConfigBuilder) {
        cleanup();

        // Early return only if use case config doesn't support zsl. If flash mode doesn't
        // support zsl, we still create reprocessing capture session but will create a
        // regular capture request when taking pictures. So when user switches flash mode, we
        // could create reprocessing capture request if flash mode allows.
        if (mIsZslDisabledByUseCaseConfig) {
            return;
        }

        if (mShouldZslDisabledByQuirks) {
            return;
        }

        // Due to b/232268355 and feedback from pixel team that private format will have better
        // performance, we will use private only for zsl.
        if (!mIsPrivateReprocessingSupported
                || mReprocessingInputSizeMap.isEmpty()
                || !mReprocessingInputSizeMap.containsKey(PRIVATE)
                || !isJpegValidOutputForInputFormat(mCameraCharacteristicsCompat, PRIVATE)) {
            return;
        }

        int reprocessingImageFormat = PRIVATE;
        Size resolution = mReprocessingInputSizeMap.get(reprocessingImageFormat);
        MetadataImageReader metadataImageReader = new MetadataImageReader(
                resolution.getWidth(),
                resolution.getHeight(),
                reprocessingImageFormat,
                MAX_IMAGES);
        mMetadataMatchingCaptureCallback = metadataImageReader.getCameraCaptureCallback();
        mReprocessingImageReader = new SafeCloseImageReaderProxy(metadataImageReader);
        metadataImageReader.setOnImageAvailableListener(
                imageReader -> {
                    try {
                        ImageProxy imageProxy = imageReader.acquireLatestImage();
                        if (imageProxy != null) {
                            mImageRingBuffer.enqueue(imageProxy);
                        }
                    } catch (IllegalStateException e) {
                        Logger.e(TAG, "Failed to acquire latest image IllegalStateException = "
                                + e.getMessage());
                    }

                }, CameraXExecutors.ioExecutor());

        // Init the reprocessing image reader surface and add into the target surfaces of capture
        mReprocessingImageDeferrableSurface = new ImmediateSurface(
                mReprocessingImageReader.getSurface(),
                new Size(mReprocessingImageReader.getWidth(),
                        mReprocessingImageReader.getHeight()),
                reprocessingImageFormat);

        SafeCloseImageReaderProxy reprocessingImageReaderProxy = mReprocessingImageReader;
        mReprocessingImageDeferrableSurface.getTerminationFuture().addListener(
                reprocessingImageReaderProxy::safeClose,
                CameraXExecutors.mainThreadExecutor());
        sessionConfigBuilder.addSurface(mReprocessingImageDeferrableSurface);

        // Init capture and session state callback and enqueue the total capture result
        sessionConfigBuilder.addCameraCaptureCallback(mMetadataMatchingCaptureCallback);
        sessionConfigBuilder.addSessionStateCallback(
                new CameraCaptureSession.StateCallback() {
                    @Override
                    public void onConfigured(
                            @NonNull CameraCaptureSession cameraCaptureSession) {
                        Surface surface = cameraCaptureSession.getInputSurface();
                        if (surface != null) {
                            mReprocessingImageWriter =
                                    ImageWriterCompat.newInstance(surface, 1);
                        }
                    }

                    @Override
                    public void onConfigureFailed(
                            @NonNull CameraCaptureSession cameraCaptureSession) { }
                });

        // Set input configuration for reprocessing capture request
        sessionConfigBuilder.setInputConfiguration(new InputConfiguration(
                mReprocessingImageReader.getWidth(),
                mReprocessingImageReader.getHeight(),
                mReprocessingImageReader.getImageFormat()));
    }

    @Nullable
    @Override
    public ImageProxy dequeueImageFromBuffer() {
        ImageProxy imageProxy = null;
        try {
            imageProxy = mImageRingBuffer.dequeue();
        } catch (NoSuchElementException e) {
            Logger.e(TAG, "dequeueImageFromBuffer no such element");
        }

        return imageProxy;
    }

    @Override
    public boolean enqueueImageToImageWriter(@NonNull ImageProxy imageProxy) {
        @OptIn(markerClass = ExperimentalGetImage.class)
        Image image = imageProxy.getImage();

        if (Build.VERSION.SDK_INT >= 23 && mReprocessingImageWriter != null && image != null) {
            try {
                ImageWriterCompat.queueInputImage(mReprocessingImageWriter, image);
            } catch (IllegalStateException e) {
                Logger.e(TAG, "enqueueImageToImageWriter throws IllegalStateException = "
                        + e.getMessage());
                return false;
            }
            return true;
        }
        return false;
    }

    private void cleanup() {
        // We might need synchronization here when clearing ring buffer while image is enqueued
        // at the same time. Will test this case.
        ZslRingBuffer imageRingBuffer = mImageRingBuffer;
        while (!imageRingBuffer.isEmpty()) {
            ImageProxy imageProxy = imageRingBuffer.dequeue();
            imageProxy.close();
        }

        DeferrableSurface reprocessingImageDeferrableSurface = mReprocessingImageDeferrableSurface;
        if (reprocessingImageDeferrableSurface != null) {
            SafeCloseImageReaderProxy reprocessingImageReaderProxy = mReprocessingImageReader;
            if (reprocessingImageReaderProxy != null) {
                reprocessingImageDeferrableSurface.getTerminationFuture().addListener(
                        reprocessingImageReaderProxy::safeClose,
                        CameraXExecutors.mainThreadExecutor());
                mReprocessingImageReader = null;
            }
            reprocessingImageDeferrableSurface.close();
            mReprocessingImageDeferrableSurface = null;
        }

        ImageWriter reprocessingImageWriter = mReprocessingImageWriter;
        if (reprocessingImageWriter != null) {
            reprocessingImageWriter.close();
            mReprocessingImageWriter = null;
        }
    }

    @NonNull
    private Map<Integer, Size> createReprocessingInputSizeMap(
            @NonNull CameraCharacteristicsCompat cameraCharacteristicsCompat) {
        StreamConfigurationMap map =
                cameraCharacteristicsCompat.get(
                        CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);

        if (map == null || map.getInputFormats() == null) {
            return new HashMap<>();
        }

        Map<Integer, Size> inputSizeMap = new HashMap<>();
        for (int format: map.getInputFormats()) {
            Size[] inputSizes = map.getInputSizes(format);
            if (inputSizes != null) {
                // Sort by descending order
                Arrays.sort(inputSizes, new CompareSizesByArea(true));

                // TODO(b/233696144): Check if selecting an input size closer to output size will
                //  improve performance or not.
                inputSizeMap.put(format, inputSizes[0]);
            }
        }
        return inputSizeMap;
    }

    private boolean isJpegValidOutputForInputFormat(
            @NonNull CameraCharacteristicsCompat cameraCharacteristicsCompat,
            int inputFormat) {
        StreamConfigurationMap map =
                cameraCharacteristicsCompat.get(
                        CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);

        if (map == null) {
            return false;
        }

        int[] validOutputFormats = map.getValidOutputFormatsForInput(inputFormat);
        if (validOutputFormats == null) {
            return false;
        }
        for (int outputFormat : validOutputFormats) {
            if (outputFormat == ImageFormat.JPEG) {
                return true;
            }
        }
        return false;
    }
}