BasicExtenderSessionProcessor.java

/*
 * Copyright 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.camera.extensions.internal.sessionprocessor;

import static androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_IMAGE_PROCESSOR;
import static androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY;

import android.content.Context;
import android.graphics.ImageFormat;
import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.CameraDevice;
import android.hardware.camera2.CaptureRequest;
import android.hardware.camera2.CaptureResult;
import android.hardware.camera2.TotalCaptureResult;
import android.util.Pair;

import androidx.annotation.GuardedBy;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.OptIn;
import androidx.annotation.RequiresApi;
import androidx.camera.camera2.impl.Camera2CameraCaptureResultConverter;
import androidx.camera.camera2.interop.CaptureRequestOptions;
import androidx.camera.camera2.interop.ExperimentalCamera2Interop;
import androidx.camera.core.Logger;
import androidx.camera.core.impl.CameraCaptureFailure;
import androidx.camera.core.impl.CameraCaptureResult;
import androidx.camera.core.impl.Config;
import androidx.camera.core.impl.OutputSurface;
import androidx.camera.core.impl.RequestProcessor;
import androidx.camera.core.impl.SessionProcessor;
import androidx.camera.extensions.impl.CaptureProcessorImpl;
import androidx.camera.extensions.impl.CaptureStageImpl;
import androidx.camera.extensions.impl.ImageCaptureExtenderImpl;
import androidx.camera.extensions.impl.PreviewExtenderImpl;
import androidx.camera.extensions.impl.PreviewImageProcessorImpl;
import androidx.camera.extensions.impl.RequestUpdateProcessorImpl;
import androidx.core.util.Preconditions;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * A {@link SessionProcessor} based on OEMs' basic extender implementation.
 */
@OptIn(markerClass = ExperimentalCamera2Interop.class)
@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
public class BasicExtenderSessionProcessor extends SessionProcessorBase {
    private static final String TAG = "BasicSessionProcessor";

    private static final int PREVIEW_PROCESS_MAX_IMAGES = 2;
    @NonNull
    private final Context mContext;
    @NonNull
    private final PreviewExtenderImpl mPreviewExtenderImpl;
    @NonNull
    private final ImageCaptureExtenderImpl mImageCaptureExtenderImpl;

    final Object mLock = new Object();
    volatile StillCaptureProcessor mStillCaptureProcessor = null;
    volatile PreviewProcessor mPreviewProcessor = null;
    volatile RequestUpdateProcessorImpl mRequestUpdateProcessor = null;
    private volatile Camera2OutputConfig mPreviewOutputConfig;
    private volatile Camera2OutputConfig mCaptureOutputConfig;
    @Nullable
    private volatile Camera2OutputConfig mAnalysisOutputConfig = null;
    private volatile OutputSurface mPreviewOutputSurfaceConfig;
    private volatile OutputSurface mCaptureOutputSurfaceConfig;
    private volatile RequestProcessor mRequestProcessor;
    volatile boolean mIsCapturing = false;
    private final AtomicInteger mNextCaptureSequenceId = new AtomicInteger(0);
    static AtomicInteger sLastOutputConfigId = new AtomicInteger(0);
    @GuardedBy("mLock")
    private final Map<CaptureRequest.Key<?>, Object> mParameters = new LinkedHashMap<>();

    public BasicExtenderSessionProcessor(@NonNull PreviewExtenderImpl previewExtenderImpl,
            @NonNull ImageCaptureExtenderImpl imageCaptureExtenderImpl,
            @NonNull Context context) {
        mPreviewExtenderImpl = previewExtenderImpl;
        mImageCaptureExtenderImpl = imageCaptureExtenderImpl;
        mContext = context;
    }

    @NonNull
    @Override
    protected Camera2SessionConfig initSessionInternal(@NonNull String cameraId,
            @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap,
            @NonNull OutputSurface previewSurfaceConfig,
            @NonNull OutputSurface imageCaptureSurfaceConfig,
            @Nullable OutputSurface imageAnalysisSurfaceConfig) {
        Logger.d(TAG, "PreviewExtenderImpl.onInit");
        mPreviewExtenderImpl.onInit(cameraId, cameraCharacteristicsMap.get(cameraId),
                mContext);
        Logger.d(TAG, "ImageCaptureExtenderImpl.onInit");
        mImageCaptureExtenderImpl.onInit(cameraId, cameraCharacteristicsMap.get(cameraId),
                mContext);

        mPreviewOutputSurfaceConfig = previewSurfaceConfig;
        mCaptureOutputSurfaceConfig = imageCaptureSurfaceConfig;

        // Preview
        PreviewExtenderImpl.ProcessorType processorType =
                mPreviewExtenderImpl.getProcessorType();
        Logger.d(TAG, "preview processorType=" + processorType);
        if (processorType == PROCESSOR_TYPE_IMAGE_PROCESSOR) {
            mPreviewOutputConfig = ImageReaderOutputConfig.create(
                    sLastOutputConfigId.getAndIncrement(),
                    previewSurfaceConfig.getSize(),
                    ImageFormat.YUV_420_888,
                    PREVIEW_PROCESS_MAX_IMAGES);
            PreviewImageProcessorImpl previewImageProcessor =
                    (PreviewImageProcessorImpl) mPreviewExtenderImpl.getProcessor();
            mPreviewProcessor = new PreviewProcessor(
                    previewImageProcessor, mPreviewOutputSurfaceConfig.getSurface(),
                    mPreviewOutputSurfaceConfig.getSize());
        } else if (processorType == PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
            mPreviewOutputConfig = SurfaceOutputConfig.create(
                    sLastOutputConfigId.getAndIncrement(),
                    previewSurfaceConfig.getSurface());
            mRequestUpdateProcessor =
                    (RequestUpdateProcessorImpl) mPreviewExtenderImpl.getProcessor();
        } else {
            mPreviewOutputConfig = SurfaceOutputConfig.create(
                    sLastOutputConfigId.getAndIncrement(),
                    previewSurfaceConfig.getSurface());
        }

        // Image Capture
        CaptureProcessorImpl captureProcessor = mImageCaptureExtenderImpl.getCaptureProcessor();
        Logger.d(TAG, "CaptureProcessor=" + captureProcessor);

        if (captureProcessor != null) {
            mCaptureOutputConfig = ImageReaderOutputConfig.create(
                    sLastOutputConfigId.getAndIncrement(),
                    imageCaptureSurfaceConfig.getSize(),
                    ImageFormat.YUV_420_888,
                    mImageCaptureExtenderImpl.getMaxCaptureStage());
            mStillCaptureProcessor = new StillCaptureProcessor(
                    captureProcessor, mCaptureOutputSurfaceConfig.getSurface(),
                    mCaptureOutputSurfaceConfig.getSize());
        } else {
            mCaptureOutputConfig = SurfaceOutputConfig.create(
                    sLastOutputConfigId.getAndIncrement(),
                    imageCaptureSurfaceConfig.getSurface());
        }

        // Image Analysis
        if (imageAnalysisSurfaceConfig != null) {
            mAnalysisOutputConfig = SurfaceOutputConfig.create(
                    sLastOutputConfigId.getAndIncrement(),
                    imageAnalysisSurfaceConfig.getSurface());
        }

        Camera2SessionConfigBuilder builder =
                new Camera2SessionConfigBuilder()
                        .addOutputConfig(mPreviewOutputConfig)
                        .addOutputConfig(mCaptureOutputConfig)
                        .setSessionTemplateId(CameraDevice.TEMPLATE_PREVIEW);

        if (mAnalysisOutputConfig != null) {
            builder.addOutputConfig(mAnalysisOutputConfig);
        }

        CaptureStageImpl captureStagePreview = mPreviewExtenderImpl.onPresetSession();
        Logger.d(TAG, "preview onPresetSession:" + captureStagePreview);

        CaptureStageImpl captureStageCapture = mImageCaptureExtenderImpl.onPresetSession();
        Logger.d(TAG, "capture onPresetSession:" + captureStageCapture);

        if (captureStagePreview != null && captureStagePreview.getParameters() != null) {
            for (Pair<CaptureRequest.Key, Object> parameter :
                    captureStagePreview.getParameters()) {
                builder.addSessionParameter(parameter.first, parameter.second);
            }
        }

        if (captureStageCapture != null && captureStageCapture.getParameters() != null) {
            for (Pair<CaptureRequest.Key, Object> parameter :
                    captureStageCapture.getParameters()) {
                builder.addSessionParameter(parameter.first, parameter.second);
            }
        }
        return builder.build();
    }

    @Override
    protected void deInitSessionInternal() {
        Logger.d(TAG, "preview onDeInit");
        mPreviewExtenderImpl.onDeInit();
        Logger.d(TAG, "capture onDeInit");
        mImageCaptureExtenderImpl.onDeInit();

        if (mPreviewProcessor != null) {
            mPreviewProcessor.close();
            mPreviewProcessor = null;
        }
        if (mStillCaptureProcessor != null) {
            mStillCaptureProcessor.close();
            mStillCaptureProcessor = null;
        }
    }

    @Override
    public void setParameters(@NonNull Config config) {
        synchronized (mLock) {
            HashMap<CaptureRequest.Key<?>, Object> map = new HashMap<>();

            CaptureRequestOptions options =
                    CaptureRequestOptions.Builder.from(config).build();

            for (Config.Option<?> option : options.listOptions()) {
                @SuppressWarnings("unchecked")
                CaptureRequest.Key<Object> key = (CaptureRequest.Key<Object>) option.getToken();
                map.put(key, options.retrieveOption(option));
            }
            mParameters.clear();
            mParameters.putAll(map);
            applyRotationAndJpegQualityToProcessor();
        }
    }

    @Override
    public void onCaptureSessionStart(@NonNull RequestProcessor requestProcessor) {
        mRequestProcessor = requestProcessor;

        List<CaptureStageImpl> captureStages = new ArrayList<>();
        CaptureStageImpl captureStage1 = mPreviewExtenderImpl.onEnableSession();
        Logger.d(TAG, "preview onEnableSession: " + captureStage1);
        if (captureStage1 != null) {
            captureStages.add(captureStage1);
        }
        CaptureStageImpl captureStage2 = mImageCaptureExtenderImpl.onEnableSession();
        Logger.d(TAG, "capture onEnableSession:" + captureStage2);
        if (captureStage2 != null) {
            captureStages.add(captureStage2);
        }

        if (!captureStages.isEmpty()) {
            submitRequestByCaptureStages(requestProcessor, captureStages);
        }

        if (mPreviewProcessor != null) {
            setImageProcessor(mPreviewOutputConfig.getId(),
                    new ImageProcessor() {
                        @Override
                        public void onNextImageAvailable(int outputStreamId, long timestampNs,
                                @NonNull ImageReference imageReference,
                                @Nullable String physicalCameraId) {
                            if (mPreviewProcessor != null) {
                                mPreviewProcessor.notifyImage(imageReference);
                            }
                        }
                    });
            mPreviewProcessor.start();
        }
    }

    private void applyParameters(RequestBuilder builder) {
        synchronized (mLock) {
            for (CaptureRequest.Key<?> key : mParameters.keySet()) {
                Object value = mParameters.get(key);
                if (value != null) {
                    builder.setParameters(key, value);
                }
            }
        }
    }

    private void applyRotationAndJpegQualityToProcessor() {
        synchronized (mLock) {
            if (mStillCaptureProcessor == null) {
                return;
            }
            Integer orientationObj = (Integer) mParameters.get(CaptureRequest.JPEG_ORIENTATION);
            if (orientationObj != null) {
                mStillCaptureProcessor.setRotationDegrees(orientationObj);
            }

            Byte qualityObj = (Byte) mParameters.get(CaptureRequest.JPEG_QUALITY);
            if (qualityObj != null) {
                mStillCaptureProcessor.setJpegQuality((int) qualityObj);
            }
        }
    }


    private void submitRequestByCaptureStages(RequestProcessor requestProcessor,
            List<CaptureStageImpl> captureStageList) {
        List<RequestProcessor.Request> requestList = new ArrayList<>();
        for (CaptureStageImpl captureStage : captureStageList) {
            RequestBuilder builder = new RequestBuilder();
            builder.addTargetOutputConfigIds(mPreviewOutputConfig.getId());
            if (mAnalysisOutputConfig != null) {
                builder.addTargetOutputConfigIds(mAnalysisOutputConfig.getId());
            }
            for (Pair<CaptureRequest.Key, Object> keyObjectPair : captureStage.getParameters()) {
                builder.setParameters(keyObjectPair.first, keyObjectPair.second);
            }
            builder.setTemplateId(CameraDevice.TEMPLATE_PREVIEW);
            requestList.add(builder.build());
        }
        requestProcessor.submit(requestList, new RequestProcessor.Callback() {
        });
    }

    @Override
    public void onCaptureSessionEnd() {
        List<CaptureStageImpl> captureStages = new ArrayList<>();
        CaptureStageImpl captureStage1 = mPreviewExtenderImpl.onDisableSession();
        Logger.d(TAG, "preview onDisableSession: " + captureStage1);
        if (captureStage1 != null) {
            captureStages.add(captureStage1);
        }
        CaptureStageImpl captureStage2 = mImageCaptureExtenderImpl.onDisableSession();
        Logger.d(TAG, "capture onDisableSession:" + captureStage2);
        if (captureStage2 != null) {
            captureStages.add(captureStage2);
        }

        if (!captureStages.isEmpty()) {
            submitRequestByCaptureStages(mRequestProcessor, captureStages);
        }
        mRequestProcessor = null;
        mIsCapturing = false;
    }

    @Override
    public int startRepeating(@NonNull CaptureCallback captureCallback) {
        int repeatingCaptureSequenceId = mNextCaptureSequenceId.getAndIncrement();
        if (mRequestProcessor == null) {
            captureCallback.onCaptureFailed(repeatingCaptureSequenceId);
            captureCallback.onCaptureSequenceAborted(repeatingCaptureSequenceId);
        } else {
            updateRepeating(repeatingCaptureSequenceId, captureCallback);
        }

        return repeatingCaptureSequenceId;
    }

    void updateRepeating(int repeatingCaptureSequenceId, @NonNull CaptureCallback captureCallback) {
        if (mRequestProcessor == null) {
            Logger.d(TAG, "mRequestProcessor is null, ignore repeating request");
            return;
        }
        RequestBuilder builder = new RequestBuilder();
        builder.addTargetOutputConfigIds(mPreviewOutputConfig.getId());
        if (mAnalysisOutputConfig != null) {
            builder.addTargetOutputConfigIds(mAnalysisOutputConfig.getId());
        }
        builder.setTemplateId(CameraDevice.TEMPLATE_PREVIEW);
        applyParameters(builder);
        applyPreviewStagesParameters(builder);

        RequestProcessor.Callback callback = new RequestProcessor.Callback() {
            @Override
            public void onCaptureCompleted(@NonNull RequestProcessor.Request request,
                    @NonNull CameraCaptureResult cameraCaptureResult) {
                CaptureResult captureResult =
                        Camera2CameraCaptureResultConverter.getCaptureResult(
                                cameraCaptureResult);
                Preconditions.checkArgument(captureResult instanceof TotalCaptureResult,
                        "Cannot get TotalCaptureResult from the cameraCaptureResult ");
                TotalCaptureResult totalCaptureResult = (TotalCaptureResult) captureResult;

                if (mPreviewProcessor != null) {
                    mPreviewProcessor.notifyCaptureResult(totalCaptureResult);
                }

                if (mRequestUpdateProcessor != null) {
                    CaptureStageImpl captureStage =
                            mRequestUpdateProcessor.process(totalCaptureResult);

                    if (captureStage != null) {
                        updateRepeating(repeatingCaptureSequenceId, captureCallback);
                    }
                }

                captureCallback.onCaptureSequenceCompleted(repeatingCaptureSequenceId);
            }
        };

        Logger.d(TAG, "requestProcessor setRepeating");
        mRequestProcessor.setRepeating(builder.build(), callback);
    }

    private void applyPreviewStagesParameters(RequestBuilder builder) {
        CaptureStageImpl captureStage = mPreviewExtenderImpl.getCaptureStage();
        if (captureStage != null) {
            for (Pair<CaptureRequest.Key, Object> keyObjectPair :
                    captureStage.getParameters()) {
                builder.setParameters(keyObjectPair.first, keyObjectPair.second);
            }
        }
    }

    @Override
    public void stopRepeating() {
        mRequestProcessor.stopRepeating();
    }

    @Override
    public int startCapture(@NonNull CaptureCallback captureCallback) {
        int captureSequenceId = mNextCaptureSequenceId.getAndIncrement();

        if (mRequestProcessor == null || mIsCapturing) {
            Logger.d(TAG, "startCapture failed");
            captureCallback.onCaptureFailed(captureSequenceId);
            captureCallback.onCaptureSequenceAborted(captureSequenceId);
            return captureSequenceId;
        }
        mIsCapturing = true;

        List<RequestProcessor.Request> requestList = new ArrayList<>();
        List<CaptureStageImpl> captureStages = mImageCaptureExtenderImpl.getCaptureStages();
        List<Integer> captureIdList = new ArrayList<>();

        for (CaptureStageImpl captureStage : captureStages) {
            RequestBuilder builder = new RequestBuilder();
            builder.addTargetOutputConfigIds(mCaptureOutputConfig.getId());
            builder.setTemplateId(CameraDevice.TEMPLATE_STILL_CAPTURE);
            builder.setCaptureStageId(captureStage.getId());

            captureIdList.add(captureStage.getId());

            applyParameters(builder);
            applyPreviewStagesParameters(builder);

            for (Pair<CaptureRequest.Key, Object> keyObjectPair :
                    captureStage.getParameters()) {
                builder.setParameters(keyObjectPair.first, keyObjectPair.second);
            }
            requestList.add(builder.build());
        }

        Logger.d(TAG, "Wait for capture stage id: " + captureIdList);

        RequestProcessor.Callback callback = new RequestProcessor.Callback() {
            boolean mIsCaptureFailed = false;
            boolean mIsCaptureStarted = false;

            @Override
            public void onCaptureStarted(@NonNull RequestProcessor.Request request,
                    long frameNumber, long timestamp) {
                if (!mIsCaptureStarted) {
                    mIsCaptureStarted = true;
                    captureCallback.onCaptureStarted(captureSequenceId, timestamp);
                }
            }

            @Override
            public void onCaptureCompleted(@NonNull RequestProcessor.Request request,
                    @NonNull CameraCaptureResult cameraCaptureResult) {
                CaptureResult captureResult =
                        Camera2CameraCaptureResultConverter.getCaptureResult(
                                cameraCaptureResult);
                Preconditions.checkArgument(captureResult instanceof TotalCaptureResult,
                        "Cannot get capture TotalCaptureResult from the cameraCaptureResult ");
                TotalCaptureResult totalCaptureResult = (TotalCaptureResult) captureResult;

                RequestBuilder.RequestProcessorRequest requestProcessorRequest =
                        (RequestBuilder.RequestProcessorRequest) request;

                if (mStillCaptureProcessor != null) {
                    mStillCaptureProcessor.notifyCaptureResult(
                            totalCaptureResult,
                            requestProcessorRequest.getCaptureStageId());
                } else {
                    captureCallback.onCaptureProcessStarted(captureSequenceId);
                    captureCallback.onCaptureSequenceCompleted(captureSequenceId);
                    mIsCapturing = false;
                }
            }

            @Override
            public void onCaptureFailed(@NonNull RequestProcessor.Request request,
                    @NonNull CameraCaptureFailure captureFailure) {
                if (!mIsCaptureFailed) {
                    mIsCaptureFailed = true;
                    captureCallback.onCaptureFailed(captureSequenceId);
                    captureCallback.onCaptureSequenceAborted(captureSequenceId);
                    mIsCapturing = false;
                }
            }

            @Override
            public void onCaptureSequenceAborted(int sequenceId) {
                captureCallback.onCaptureSequenceAborted(captureSequenceId);
                mIsCapturing = false;
            }
        };

        Logger.d(TAG, "startCapture");
        if (mStillCaptureProcessor != null) {
            mStillCaptureProcessor.startCapture(captureIdList,
                    new StillCaptureProcessor.OnCaptureResultCallback() {
                        @Override
                        public void onCompleted() {
                            captureCallback.onCaptureSequenceCompleted(captureSequenceId);
                            mIsCapturing = false;
                        }

                        @Override
                        public void onError(@NonNull Exception e) {
                            captureCallback.onCaptureFailed(captureSequenceId);
                            mIsCapturing = false;
                        }
                    });
        }
        setImageProcessor(mCaptureOutputConfig.getId(),
                new ImageProcessor() {
                    boolean mIsFirstFrame = true;

                    @Override
                    public void onNextImageAvailable(int outputStreamId, long timestampNs,
                            @NonNull ImageReference imageReference,
                            @Nullable String physicalCameraId) {
                        Logger.d(TAG,
                                "onNextImageAvailable  outputStreamId=" + outputStreamId);
                        if (mStillCaptureProcessor != null) {
                            mStillCaptureProcessor.notifyImage(imageReference);
                        }

                        if (mIsFirstFrame) {
                            captureCallback.onCaptureProcessStarted(captureSequenceId);
                            mIsFirstFrame = false;
                        }
                    }
                });
        mRequestProcessor.submit(requestList, callback);
        return captureSequenceId;
    }

    @Override
    public void abortCapture(int captureSequenceId) {
        mRequestProcessor.abortCaptures();
    }

    @Override
    public int startTrigger(@NonNull Config config, @NonNull CaptureCallback callback) {
        Logger.d(TAG, "startTrigger");
        int captureSequenceId = mNextCaptureSequenceId.getAndIncrement();
        RequestBuilder builder = new RequestBuilder();
        builder.addTargetOutputConfigIds(mPreviewOutputConfig.getId());
        if (mAnalysisOutputConfig != null) {
            builder.addTargetOutputConfigIds(mAnalysisOutputConfig.getId());
        }
        builder.setTemplateId(CameraDevice.TEMPLATE_PREVIEW);
        applyParameters(builder);
        applyPreviewStagesParameters(builder);

        CaptureRequestOptions options =
                CaptureRequestOptions.Builder.from(config).build();
        for (Config.Option<?> option : options.listOptions()) {
            @SuppressWarnings("unchecked")
            CaptureRequest.Key<Object> key = (CaptureRequest.Key<Object>) option.getToken();
            builder.setParameters(key, options.retrieveOption(option));
        }

        mRequestProcessor.submit(builder.build(), new RequestProcessor.Callback() {
            @Override
            public void onCaptureCompleted(@NonNull RequestProcessor.Request request,
                    @NonNull CameraCaptureResult captureResult) {
                callback.onCaptureSequenceCompleted(captureSequenceId);
            }

            @Override
            public void onCaptureFailed(@NonNull RequestProcessor.Request request,
                    @NonNull CameraCaptureFailure captureFailure) {
                callback.onCaptureFailed(captureSequenceId);
            }
        });

        return captureSequenceId;
    }
}