ImageAnalysisAbstractAnalyzer.java

/*
 * Copyright 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.camera.core;

import static androidx.camera.core.ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888;
import static androidx.camera.core.ImageProcessingUtil.applyPixelShiftForYUV;
import static androidx.camera.core.ImageProcessingUtil.convertYUVToRGB;
import static androidx.camera.core.ImageProcessingUtil.rotateYUV;

import android.graphics.Matrix;
import android.graphics.Rect;
import android.graphics.RectF;
import android.media.ImageWriter;
import android.os.Build;

import androidx.annotation.GuardedBy;
import androidx.annotation.IntRange;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import androidx.camera.core.impl.ImageReaderProxy;
import androidx.camera.core.impl.utils.futures.Futures;
import androidx.camera.core.internal.compat.ImageWriterCompat;
import androidx.concurrent.futures.CallbackToFutureAdapter;
import androidx.core.os.OperationCanceledException;

import com.google.common.util.concurrent.ListenableFuture;

import java.nio.ByteBuffer;
import java.util.concurrent.Executor;

/**
 * Abstract Analyzer that wraps around {@link ImageAnalysis.Analyzer} and implements
 * {@link ImageReaderProxy.OnImageAvailableListener}.
 *
 * This is an extension of {@link ImageAnalysis}. It has the same lifecycle and share part of the
 * states.
 */
@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
abstract class ImageAnalysisAbstractAnalyzer implements ImageReaderProxy.OnImageAvailableListener {

    private static final String TAG = "ImageAnalysisAnalyzer";
    private static final RectF NORMALIZED_RECT = new RectF(-1, -1, 1, 1);

    // Member variables from ImageAnalysis.
    @GuardedBy("mAnalyzerLock")
    private ImageAnalysis.Analyzer mSubscribedAnalyzer;

    // Relative rotation degree provided to user in image info based on sensor to buffer rotation
    // degrees and target rotation degrees.
    @IntRange(from = 0, to = 359)
    private volatile int mRelativeRotation;

    // Cache buffer rotation degree for previous frame to decide whether new image reader proxy
    // needs to be created.
    @IntRange(from = 0, to = 359)
    private volatile int mPrevBufferRotationDegrees;

    @ImageAnalysis.OutputImageFormat
    private volatile int mOutputImageFormat = ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888;
    private volatile boolean mOutputImageRotationEnabled;
    private volatile boolean mOnePixelShiftEnabled;

    @GuardedBy("mAnalyzerLock")
    private Executor mUserExecutor;

    @GuardedBy("mAnalyzerLock")
    @Nullable
    private SafeCloseImageReaderProxy mProcessedImageReaderProxy;

    @GuardedBy("mAnalyzerLock")
    @Nullable
    private ImageWriter mProcessedImageWriter;

    @GuardedBy("mAnalyzerLock")
    private Rect mOriginalViewPortCropRect = new Rect();

    @GuardedBy("mAnalyzerLock")
    private Rect mUpdatedViewPortCropRect = new Rect();

    @GuardedBy("mAnalyzerLock")
    private Matrix mOriginalSensorToBufferTransformMatrix = new Matrix();

    @GuardedBy("mAnalyzerLock")
    private Matrix mUpdatedSensorToBufferTransformMatrix = new Matrix();

    @GuardedBy("mAnalyzerLock")
    @Nullable
    @VisibleForTesting ByteBuffer mRGBConvertedBuffer;

    @GuardedBy("mAnalyzerLock")
    @Nullable
    @VisibleForTesting ByteBuffer mYRotatedBuffer;

    @GuardedBy("mAnalyzerLock")
    @Nullable
    @VisibleForTesting ByteBuffer mURotatedBuffer;

    @GuardedBy("mAnalyzerLock")
    @Nullable
    @VisibleForTesting ByteBuffer mVRotatedBuffer;

    // Lock that synchronizes the access to mSubscribedAnalyzer/mUserExecutor to prevent mismatch.
    private final Object mAnalyzerLock = new Object();

    // Flag that reflects the attaching state of the holding ImageAnalysis object.
    protected boolean mIsAttached = true;

    @Override
    public void onImageAvailable(@NonNull ImageReaderProxy imageReaderProxy) {
        try {
            ImageProxy imageProxy = acquireImage(imageReaderProxy);
            if (imageProxy != null) {
                onValidImageAvailable(imageProxy);
            }
        } catch (IllegalStateException e) {
            // This happens if image is not closed in STRATEGY_BLOCK_PRODUCER mode. Catch the
            // exception and fail with an error log.
            // TODO(b/175851631): it may also happen when STRATEGY_KEEP_ONLY_LATEST not closing
            //  the cached image properly. We are unclear why it happens but catching the
            //  exception should improve the situation by not crashing.
            Logger.e(TAG, "Failed to acquire image.", e);
        }
    }

    /**
     * Implemented by children to acquireImage via {@link ImageReaderProxy#acquireLatestImage()} or
     * {@link ImageReaderProxy#acquireNextImage()}.
     */
    @Nullable
    abstract ImageProxy acquireImage(@NonNull ImageReaderProxy imageReaderProxy);

    /**
     * Called when a new valid {@link ImageProxy} becomes available via
     * {@link ImageReaderProxy.OnImageAvailableListener}.
     */
    abstract void onValidImageAvailable(@NonNull ImageProxy imageProxy);

    /**
     * Called by {@link ImageAnalysis} to release cached images.
     */
    abstract void clearCache();

    /**
     * Analyzes a {@link ImageProxy} using the wrapped {@link ImageAnalysis.Analyzer}.
     *
     * <p> The analysis will run on the executor provided by {@link #setAnalyzer(Executor,
     * ImageAnalysis.Analyzer)}. Once the analysis successfully finishes the returned
     * ListenableFuture will succeed. If the future fails then it means the {@link
     * ImageAnalysis.Analyzer} was not called so the image needs to be closed.
     *
     * @return The future which will complete once analysis has finished or it failed.
     */
    ListenableFuture<Void> analyzeImage(@NonNull ImageProxy imageProxy) {
        Executor executor;
        ImageAnalysis.Analyzer analyzer;
        SafeCloseImageReaderProxy processedImageReaderProxy;
        ImageWriter processedImageWriter;
        ByteBuffer rgbConvertedBuffer;
        ByteBuffer yRotatedBuffer;
        ByteBuffer uRotatedBuffer;
        ByteBuffer vRotatedBuffer;
        int currentBufferRotationDegrees = mOutputImageRotationEnabled ? mRelativeRotation : 0;
        boolean outputImageDirty;

        synchronized (mAnalyzerLock) {
            executor = mUserExecutor;
            analyzer = mSubscribedAnalyzer;

            // Set dirty flag to indicate the output image transform matrix (for both YUV and RGB)
            // and image reader proxy (for YUV) needs to be recreated.
            outputImageDirty = mOutputImageRotationEnabled
                    && currentBufferRotationDegrees != mPrevBufferRotationDegrees;

            // Cache the image reader proxy and image write for reuse and only recreate when
            // relative rotation degree changes.
            if (outputImageDirty) {
                recreateImageReaderProxy(imageProxy, currentBufferRotationDegrees);
            }

            // Cache memory buffer for image rotation
            if (mOutputImageRotationEnabled) {
                createHelperBuffer(imageProxy);
            }

            processedImageReaderProxy = mProcessedImageReaderProxy;
            processedImageWriter = mProcessedImageWriter;
            rgbConvertedBuffer = mRGBConvertedBuffer;
            yRotatedBuffer = mYRotatedBuffer;
            uRotatedBuffer = mURotatedBuffer;
            vRotatedBuffer = mVRotatedBuffer;
        }

        ListenableFuture<Void> future;

        if (analyzer != null && executor != null && mIsAttached) {
            ImageProxy processedImageProxy = null;

            if (processedImageReaderProxy != null) {
                if (mOutputImageFormat == OUTPUT_IMAGE_FORMAT_RGBA_8888) {
                    processedImageProxy =
                            convertYUVToRGB(
                                    imageProxy,
                                    processedImageReaderProxy,
                                    rgbConvertedBuffer,
                                    currentBufferRotationDegrees,
                                    mOnePixelShiftEnabled);
                } else if (mOutputImageFormat == ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888) {
                    // Apply one pixel shift before other processing, e.g. rotation.
                    if (mOnePixelShiftEnabled) {
                        applyPixelShiftForYUV(imageProxy);
                    }
                    if (processedImageWriter != null
                            && yRotatedBuffer != null
                            && uRotatedBuffer != null
                            && vRotatedBuffer != null) {
                        processedImageProxy = rotateYUV(
                                imageProxy,
                                processedImageReaderProxy,
                                processedImageWriter,
                                yRotatedBuffer,
                                uRotatedBuffer,
                                vRotatedBuffer,
                                currentBufferRotationDegrees);
                    }
                }
            }

            // Flag to indicate YUV2RGB conversion or YUV/RGB rotation failed, not including one
            // pixel shift process for YUV.
            final boolean outputProcessedImageFailed = processedImageProxy == null;
            final ImageProxy outputImageProxy = outputProcessedImageFailed ? imageProxy :
                    processedImageProxy;

            // recalculate transform matrix and update crop rect only if
            // rotation succeeded and relative rotation degree changed
            Rect cropRect = new Rect();
            Matrix transformMatrix = new Matrix();
            synchronized (mAnalyzerLock) {
                if (outputImageDirty && !outputProcessedImageFailed) {
                    recalculateTransformMatrixAndCropRect(
                            imageProxy.getWidth(),
                            imageProxy.getHeight(),
                            outputImageProxy.getWidth(),
                            outputImageProxy.getHeight());
                }
                mPrevBufferRotationDegrees = currentBufferRotationDegrees;

                cropRect.set(mUpdatedViewPortCropRect);
                transformMatrix.set(mUpdatedSensorToBufferTransformMatrix);
            }

            // When the analyzer exists and ImageAnalysis is active.
            future = CallbackToFutureAdapter.getFuture(
                    completer -> {
                        executor.execute(() -> {
                            if (mIsAttached) {
                                ImageInfo imageInfo = ImmutableImageInfo.create(
                                        imageProxy.getImageInfo().getTagBundle(),
                                        imageProxy.getImageInfo().getTimestamp(),
                                        mOutputImageRotationEnabled ? 0
                                                : mRelativeRotation,
                                        transformMatrix);

                                ImageProxy outputSettableImageProxy = new SettableImageProxy(
                                        outputImageProxy, imageInfo);
                                if (!cropRect.isEmpty()) {
                                    outputSettableImageProxy.setCropRect(cropRect);
                                }
                                analyzer.analyze(outputSettableImageProxy);
                                completer.set(null);
                            } else {
                                completer.setException(new OperationCanceledException(
                                        "ImageAnalysis is detached"));
                            }
                        });
                        return "analyzeImage";
                    });
        } else {
            future = Futures.immediateFailedFuture(new OperationCanceledException(
                    "No analyzer or executor currently set."));
        }

        return future;
    }

    @NonNull
    private static SafeCloseImageReaderProxy createImageReaderProxy(
            int imageWidth,
            int imageHeight,
            int rotation,
            int format,
            int maxImages) {
        boolean flipWH = (rotation == 90 || rotation == 270);
        int width = flipWH ? imageHeight : imageWidth;
        int height = flipWH ? imageWidth : imageHeight;

        return new SafeCloseImageReaderProxy(
                ImageReaderProxys.createIsolatedReader(
                        width,
                        height,
                        format,
                        maxImages));
    }

    void setRelativeRotation(int relativeRotation) {
        mRelativeRotation = relativeRotation;
    }

    void setOutputImageRotationEnabled(boolean outputImageRotationEnabled) {
        mOutputImageRotationEnabled = outputImageRotationEnabled;
    }

    void setOutputImageFormat(@ImageAnalysis.OutputImageFormat int outputImageFormat) {
        mOutputImageFormat = outputImageFormat;
    }

    void setOnePixelShiftEnabled(boolean onePixelShiftEnabled) {
        mOnePixelShiftEnabled = onePixelShiftEnabled;
    }

    void setViewPortCropRect(@NonNull Rect viewPortCropRect) {
        synchronized (mAnalyzerLock) {
            mOriginalViewPortCropRect = viewPortCropRect;
            mUpdatedViewPortCropRect = new Rect(mOriginalViewPortCropRect);
        }
    }

    void setSensorToBufferTransformMatrix(@NonNull Matrix sensorToBufferTransformMatrix) {
        synchronized (mAnalyzerLock) {
            mOriginalSensorToBufferTransformMatrix = sensorToBufferTransformMatrix;
            mUpdatedSensorToBufferTransformMatrix =
                    new Matrix(mOriginalSensorToBufferTransformMatrix);
        }
    }

    void setProcessedImageReaderProxy(
            @NonNull SafeCloseImageReaderProxy processedImageReaderProxy) {
        synchronized (mAnalyzerLock) {
            mProcessedImageReaderProxy = processedImageReaderProxy;
        }

    }

    void setAnalyzer(@Nullable Executor userExecutor,
            @Nullable ImageAnalysis.Analyzer subscribedAnalyzer) {
        synchronized (mAnalyzerLock) {
            if (subscribedAnalyzer == null) {
                clearCache();
            }
            mSubscribedAnalyzer = subscribedAnalyzer;
            mUserExecutor = userExecutor;
        }
    }

    /**
     * Initialize the callback.
     */
    void attach() {
        mIsAttached = true;
    }

    /**
     * Closes the callback so that it will stop posting to analyzer.
     */
    void detach() {
        mIsAttached = false;
        clearCache();
    }

    @GuardedBy("mAnalyzerLock")
    private void createHelperBuffer(@NonNull ImageProxy imageProxy) {
        if (mOutputImageFormat == ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888) {
            if (mYRotatedBuffer == null) {
                mYRotatedBuffer = ByteBuffer.allocateDirect(
                        imageProxy.getWidth() * imageProxy.getHeight());
            }
            mYRotatedBuffer.position(0);

            if (mURotatedBuffer == null) {
                mURotatedBuffer = ByteBuffer.allocateDirect(
                        imageProxy.getWidth() * imageProxy.getHeight() / 4);
            }
            mURotatedBuffer.position(0);

            if (mVRotatedBuffer == null) {
                mVRotatedBuffer = ByteBuffer.allocateDirect(
                        imageProxy.getWidth() * imageProxy.getHeight() / 4);
            }
            mVRotatedBuffer.position(0);
        } else if (mOutputImageFormat == OUTPUT_IMAGE_FORMAT_RGBA_8888) {
            if (mRGBConvertedBuffer == null) {
                mRGBConvertedBuffer = ByteBuffer.allocateDirect(
                        imageProxy.getWidth() * imageProxy.getHeight() * 4);
            }
        }
    }

    @GuardedBy("mAnalyzerLock")
    private void recreateImageReaderProxy(
            @NonNull ImageProxy imageProxy,
            @IntRange(from = 0, to = 359) int relativeRotation) {
        if (mProcessedImageReaderProxy == null) {
            return;
        }

        mProcessedImageReaderProxy.safeClose();
        mProcessedImageReaderProxy = createImageReaderProxy(
                imageProxy.getWidth(),
                imageProxy.getHeight(),
                relativeRotation,
                mProcessedImageReaderProxy.getImageFormat(),
                mProcessedImageReaderProxy.getMaxImages());

        if (Build.VERSION.SDK_INT >= 23
                && mOutputImageFormat == ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888) {

            if (mProcessedImageWriter != null) {
                ImageWriterCompat.close(mProcessedImageWriter);
            }

            mProcessedImageWriter = ImageWriterCompat.newInstance(
                    mProcessedImageReaderProxy.getSurface(),
                    mProcessedImageReaderProxy.getMaxImages());
        }
    }

    @GuardedBy("mAnalyzerLock")
    private void recalculateTransformMatrixAndCropRect(
            int originalWidth,
            int originalHeight,
            int rotatedWidth,
            int rotatedHeight) {
        Matrix additionalTransformMatrix = getAdditionalTransformMatrixAppliedByProcessor(
                originalWidth,
                originalHeight,
                rotatedWidth,
                rotatedHeight,
                mRelativeRotation);
        mUpdatedViewPortCropRect = getUpdatedCropRect(
                mOriginalViewPortCropRect, additionalTransformMatrix);
        mUpdatedSensorToBufferTransformMatrix.setConcat(mOriginalSensorToBufferTransformMatrix,
                additionalTransformMatrix);
    }

    @NonNull
    static Rect getUpdatedCropRect(
            @NonNull Rect originalCropRect,
            @NonNull Matrix additionalTransformMatrix) {
        RectF rectF = new RectF(originalCropRect);
        additionalTransformMatrix.mapRect(rectF);
        Rect rect = new Rect();
        rectF.round(rect);
        return rect;
    }

    @VisibleForTesting
    @NonNull
    static Matrix getAdditionalTransformMatrixAppliedByProcessor(
            int originalWidth,
            int originalHeight,
            int rotatedWidth,
            int rotatedHeight,
            @IntRange(from = 0, to = 359) int relativeRotation) {
        Matrix matrix = new Matrix();
        if (relativeRotation > 0) {
            matrix.setRectToRect(
                    new RectF(0, 0, originalWidth, originalHeight),
                    NORMALIZED_RECT,
                    Matrix.ScaleToFit.FILL);
            matrix.postRotate(relativeRotation);
            matrix.postConcat(getNormalizedToBuffer(new RectF(0, 0, rotatedWidth,
                    rotatedHeight)));
        }
        return matrix;
    }

    @NonNull
    private static Matrix getNormalizedToBuffer(@NonNull RectF buffer) {
        Matrix normalizedToBuffer = new Matrix();
        normalizedToBuffer.setRectToRect(NORMALIZED_RECT, buffer, Matrix.ScaleToFit.FILL);
        return normalizedToBuffer;
    }
}