SurfaceEdge.java

/*
 * Copyright 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.camera.core.processing;

import static androidx.camera.core.impl.ImageOutputConfig.ROTATION_NOT_SPECIFIED;
import static androidx.camera.core.impl.utils.Threads.checkMainThread;
import static androidx.camera.core.impl.utils.executor.CameraXExecutors.directExecutor;
import static androidx.camera.core.impl.utils.executor.CameraXExecutors.mainThreadExecutor;
import static androidx.camera.core.impl.utils.futures.Futures.immediateFailedFuture;
import static androidx.camera.core.impl.utils.futures.Futures.immediateFuture;
import static androidx.camera.core.impl.utils.futures.Futures.transformAsync;
import static androidx.core.util.Preconditions.checkArgument;
import static androidx.core.util.Preconditions.checkNotNull;
import static androidx.core.util.Preconditions.checkState;

import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.os.Build;
import android.util.Range;
import android.util.Size;
import android.view.Surface;
import android.view.SurfaceView;
import android.view.TextureView;

import androidx.annotation.MainThread;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import androidx.camera.core.CameraEffect;
import androidx.camera.core.SurfaceOutput;
import androidx.camera.core.SurfaceProcessor;
import androidx.camera.core.SurfaceRequest;
import androidx.camera.core.SurfaceRequest.TransformationInfo;
import androidx.camera.core.UseCase;
import androidx.camera.core.impl.CameraInternal;
import androidx.camera.core.impl.DeferrableSurface;
import androidx.camera.core.impl.SessionConfig;
import androidx.camera.core.impl.StreamSpec;
import androidx.camera.core.impl.utils.futures.Futures;
import androidx.concurrent.futures.CallbackToFutureAdapter;

import com.google.common.util.concurrent.ListenableFuture;

import java.util.HashSet;
import java.util.Set;

/**
 * An edge between two {@link Node} that is based on a {@link DeferrableSurface}.
 *
 * <p>This class contains a single {@link DeferrableSurface} with additional info such as size,
 * crop rect and transformation. It also connects the downstream {@link DeferrableSurface} or
 * {@link SurfaceRequest} that provides the {@link Surface}.
 *
 * <p>To set up a connection, configure both downstream/upstream nodes. Both downstream/upstream
 * nodes can only be configured once for each connection. Trying to configure them again throws
 * {@link IllegalStateException}.
 *
 * <p>To connect a downstream node(Surface provider):
 * <ul>
 * <li>For external source, call {@link #createSurfaceRequest} and send the
 * {@link SurfaceRequest} to the app. For example, sending the {@link SurfaceRequest} to
 * PreviewView or Recorder.
 * <li>For internal source, call {@link #setProvider} with the {@link DeferrableSurface}
 * from another {@link UseCase}. For example, when sharing one stream to two use cases.
 * </ul>
 *
 * <p>To connect a upstream node(surface consumer):
 * <ul>
 * <li>For external source, call {@link #createSurfaceOutputFuture} and send the
 * {@link SurfaceOutput} to the app. For example, sending the {@link SurfaceOutput} to
 * {@link SurfaceProcessor}.
 * <li>For internal source, call {@link #getDeferrableSurface()} and set the
 * {@link DeferrableSurface} on {@link SessionConfig}.
 * </ul>
 *
 * <p>The connection ends when the {@link #close()} or {@link #invalidate()} is called.
 * The difference is that {@link #close()} only notifies the upstream pipeline that the
 * {@link Surface} should no longer be used, and {@link #invalidate()} cleans the current
 * connection so it can be connected again.
 */
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public class SurfaceEdge {

    private final Matrix mSensorToBufferTransform;
    private final boolean mHasCameraTransform;
    private final Rect mCropRect;
    private final boolean mMirroring;
    @CameraEffect.Targets
    private final int mTargets;
    private final StreamSpec mStreamSpec;
    // Guarded by main thread.
    private int mRotationDegrees;

    // Guarded by main thread.
    @Nullable
    private SurfaceOutputImpl mConsumerToNotify;
    // Guarded by main thread.
    private boolean mHasConsumer = false;

    // Guarded by main thread.
    @Nullable
    private SurfaceRequest mProviderSurfaceRequest;

    // Guarded by main thread.
    @NonNull
    private SettableSurface mSettableSurface;

    // Guarded by main thread.
    @NonNull
    private final Set<Runnable> mOnInvalidatedListeners = new HashSet<>();

    // Guarded by main thread.
    // Tombstone flag indicates whether the edge has been closed. Once closed, the edge should
    // never be used again.
    private boolean mIsClosed = false;

    /**
     * Please see the getters to understand the parameters.
     */
    public SurfaceEdge(
            @CameraEffect.Targets int targets,
            @NonNull StreamSpec streamSpec,
            @NonNull Matrix sensorToBufferTransform,
            boolean hasCameraTransform,
            @NonNull Rect cropRect,
            int rotationDegrees,
            boolean mirroring) {
        mTargets = targets;
        mStreamSpec = streamSpec;
        mSensorToBufferTransform = sensorToBufferTransform;
        mHasCameraTransform = hasCameraTransform;
        mCropRect = cropRect;
        mRotationDegrees = rotationDegrees;
        mMirroring = mirroring;
        mSettableSurface = new SettableSurface(streamSpec.getResolution());
    }

    /**
     * Adds a Runnable that gets invoked when the downstream pipeline is invalidated.
     *
     * <p>The added listeners are invoked when the downstream pipeline wants to replace the
     * previously provided {@link Surface}. For example, when {@link SurfaceRequest#invalidate()}
     * is called. When that happens, the edge should notify the upstream pipeline to get the new
     * Surface.
     */
    @MainThread
    public void addOnInvalidatedListener(@NonNull Runnable onInvalidated) {
        checkMainThread();
        checkNotClosed();
        mOnInvalidatedListeners.add(onInvalidated);
    }

    /**
     * Gets the {@link DeferrableSurface} for upstream nodes.
     *
     * <p>This method throws {@link IllegalStateException} if the current {@link SurfaceEdge}
     * already has a Surface consumer. To remove the current Surface consumer, call
     * {@link #invalidate()} to reset the connection.
     */
    @NonNull
    @MainThread
    public DeferrableSurface getDeferrableSurface() {
        checkMainThread();
        checkNotClosed();
        checkAndSetHasConsumer();
        return mSettableSurface;
    }

    /**
     * Sets the downstream {@link DeferrableSurface}.
     *
     * <p>Once connected, the value {@link #getDeferrableSurface()} and the provider will be
     * in sync on the following matters: 1) surface provision, 2) ref-counting, 3) closure and 4)
     * termination. See the list below for details:
     * <ul>
     * <li>Surface. the provider and the parent share the same Surface object.
     * <li>Ref-counting. The ref-count of the {@link #getDeferrableSurface()} represents whether
     * it's safe to release the Surface. The ref-count of the provider represents whether the
     * {@link #getDeferrableSurface()} is terminated. As long as the parent is not terminated, the
     * provider cannot release the surface because someone might be accessing the surface.
     * <li>Closure. When {@link #getDeferrableSurface()} is closed, if the surface is provided
     * via {@link SurfaceOutput}, it will invoke {@link SurfaceOutputImpl#requestClose()} to
     * decrease the ref-counter; if the surface is used by the camera-camera2, wait for the
     * ref-counter to go to zero on its own. For the provider, closing after providing the
     * surface has no effect; closing before providing the surface propagates the exception
     * upstream.
     * <li>Termination. On {@link #getDeferrableSurface()} termination, close the provider and
     * decrease the ref-count to notify that the Surface can be safely released. The provider
     * cannot be terminated before the {@link #getDeferrableSurface()} does.
     * </ul>
     *
     * <p>This method is for organizing the pipeline internally. For example, using the output of
     * one {@link UseCase} as the input of another {@link UseCase} for stream sharing.
     *
     * <p>This method is idempotent. Calling it with the same provider no-ops. Calling it with a
     * different provider throws {@link IllegalStateException}.
     *
     * @throws DeferrableSurface.SurfaceClosedException when the provider is already closed.
     */
    @MainThread
    public void setProvider(@NonNull DeferrableSurface provider)
            throws DeferrableSurface.SurfaceClosedException {
        checkMainThread();
        checkNotClosed();
        mSettableSurface.setProvider(provider);
    }

    /**
     * Creates a {@link SurfaceRequest} that is linked to this {@link SurfaceEdge}.
     *
     * <p>The {@link SurfaceRequest} is for requesting a {@link Surface} from an external source
     * such as {@code PreviewView} or {@code VideoCapture}. {@link SurfaceEdge} uses the
     * {@link Surface} provided by {@link SurfaceRequest#provideSurface} as its source. For how
     * the ref-counting works, please see the Javadoc of {@link #setProvider}.
     *
     * <p>It throws {@link IllegalStateException} if the current {@link SurfaceEdge}
     * already has a provider.
     */
    @MainThread
    @NonNull
    public SurfaceRequest createSurfaceRequest(@NonNull CameraInternal cameraInternal) {
        return createSurfaceRequest(cameraInternal, null);
    }

    /**
     * Creates a {@link SurfaceRequest} that is linked to this {@link SurfaceEdge}.
     *
     * <p>The {@link SurfaceRequest} is for requesting a {@link Surface} from an external source
     * such as {@code PreviewView} or {@code VideoCapture}. {@link SurfaceEdge} uses the
     * {@link Surface} provided by {@link SurfaceRequest#provideSurface} as its source. For how
     * the ref-counting works, please see the Javadoc of {@link #setProvider}.
     *
     * <p>It throws {@link IllegalStateException} if the current {@link SurfaceEdge}
     * already has a provider.
     *
     * <p>This overload optionally allows allows specifying the expected frame rate range in which
     * the surface should operate.
     */
    @MainThread
    @NonNull
    public SurfaceRequest createSurfaceRequest(@NonNull CameraInternal cameraInternal,
            @Nullable Range<Integer> expectedFpsRange) {
        checkMainThread();
        checkNotClosed();
        // TODO(b/238230154) figure out how to support HDR.
        SurfaceRequest surfaceRequest = new SurfaceRequest(mStreamSpec.getResolution(),
                cameraInternal, expectedFpsRange,
                () -> mainThreadExecutor().execute(() -> {
                    if (!mIsClosed) {
                        invalidate();
                    }
                }));
        try {
            DeferrableSurface deferrableSurface = surfaceRequest.getDeferrableSurface();
            if (mSettableSurface.setProvider(deferrableSurface)) {
                mSettableSurface.getTerminationFuture().addListener(deferrableSurface::close,
                        directExecutor());
            }
        } catch (DeferrableSurface.SurfaceClosedException e) {
            // This should never happen. We just created the SurfaceRequest. It can't be closed.
            throw new AssertionError("Surface is somehow already closed", e);
        } catch (RuntimeException e) {
            // This should never happen. It indicates a bug in CameraX code. Close the
            // SurfaceRequest just to be safe.
            surfaceRequest.willNotProvideSurface();
            throw e;
        }
        mProviderSurfaceRequest = surfaceRequest;
        notifyTransformationInfoUpdate();
        return surfaceRequest;
    }

    /**
     * Creates a {@link SurfaceOutput} that is linked to this {@link SurfaceEdge}.
     *
     * <p>The {@link SurfaceOutput} is for providing a surface to an external target such
     * as {@link SurfaceProcessor}.
     *
     * <p>This method returns a {@link ListenableFuture<SurfaceOutput>} that completes when the
     * {@link #getDeferrableSurface()} completes. The {@link SurfaceOutput} contains the surface
     * and ref-counts the {@link SurfaceEdge}.
     *
     * <p>Do not provide the {@link SurfaceOutput} to external target if the
     * {@link ListenableFuture} fails.
     *
     * <p>This method throws {@link IllegalStateException} if the current {@link SurfaceEdge}
     * already has a Surface consumer. To remove the current Surface consumer, call
     * {@link #invalidate()} to reset the connection.
     *
     * @param inputSize       resolution of input image buffer
     * @param cropRect        crop rect of input image buffer
     * @param rotationDegrees expected rotation to the input image buffer
     * @param mirroring       expected mirroring to the input image buffer
     */
    @MainThread
    @NonNull
    public ListenableFuture<SurfaceOutput> createSurfaceOutputFuture(@NonNull Size inputSize,
            @NonNull Rect cropRect, int rotationDegrees, boolean mirroring) {
        checkMainThread();
        checkNotClosed();
        checkAndSetHasConsumer();
        SettableSurface settableSurface = mSettableSurface;
        return transformAsync(mSettableSurface.getSurface(),
                surface -> {
                    checkNotNull(surface);
                    try {
                        settableSurface.incrementUseCount();
                    } catch (DeferrableSurface.SurfaceClosedException e) {
                        return immediateFailedFuture(e);
                    }
                    SurfaceOutputImpl surfaceOutputImpl = new SurfaceOutputImpl(surface,
                            getTargets(), mStreamSpec.getResolution(), inputSize, cropRect,
                            rotationDegrees, mirroring);
                    surfaceOutputImpl.getCloseFuture().addListener(
                            settableSurface::decrementUseCount,
                            directExecutor());
                    mConsumerToNotify = surfaceOutputImpl;
                    return immediateFuture(surfaceOutputImpl);
                }, mainThreadExecutor());
    }

    /**
     * Resets connection and notifies that a new connection is ready.
     *
     * <p>Call this method to notify that the {@link Surface} previously provided via
     * {@link #createSurfaceRequest} or {@link #setProvider} should no longer be used. The
     * upstream pipeline should call {@link #getDeferrableSurface()} or
     * {@link #createSurfaceOutputFuture} to get the new {@link Surface}.
     *
     * <p>Only call this method when the surface provider is ready to provide a new {@link Surface}.
     * For example, when {@link SurfaceRequest#invalidate()} is invoked or when a downstream
     * {@link UseCase} resets.
     *
     * @see #close()
     */
    @MainThread
    public void invalidate() {
        checkMainThread();
        checkNotClosed();
        if (mSettableSurface.canSetProvider()) {
            // If the edge is still connectable, no-ops.
            return;
        }
        disconnectWithoutCheckingClosed();
        mHasConsumer = false;
        mSettableSurface = new SettableSurface(mStreamSpec.getResolution());
        for (Runnable onInvalidated : mOnInvalidatedListeners) {
            onInvalidated.run();
        }
    }

    /**
     * Closes the edge.
     *
     * <p> Disconnects the edge and sets a tombstone so it will never be used again. This method
     * is idempotent.
     *
     * @see #disconnect()
     */
    @MainThread
    public final void close() {
        checkMainThread();
        disconnectWithoutCheckingClosed();
        mIsClosed = true;
    }

    /**
     * Disconnects the edge.
     *
     * <p> Once disconnected, upstream should stop sending images to the edge, and downstream
     * should stop expecting images from the edge.
     *
     * <p> This method notifies the upstream via {@link SettableSurface#close()}/
     * {@link SurfaceOutputImpl}. By calling {@link SettableSurface#close()}, it also decrements the
     * ref-count on downstream Surfaces so they can be released.
     *
     * @see DeferrableSurface#close().
     * @see #invalidate()
     */
    @MainThread
    public final void disconnect() {
        checkMainThread();
        checkNotClosed();
        disconnectWithoutCheckingClosed();
    }

    private void disconnectWithoutCheckingClosed() {
        mSettableSurface.close();
        if (mConsumerToNotify != null) {
            mConsumerToNotify.requestClose();
            mConsumerToNotify = null;
        }
    }

    /**
     * This field indicates that what purpose the {@link Surface} will be used for.
     */
    @CameraEffect.Targets
    public int getTargets() {
        return mTargets;
    }

    /**
     * Gets the {@link Matrix} represents the transformation from camera sensor to the current
     * {@link Surface}.
     *
     * <p>This value represents the transformation from sensor coordinates to the current buffer
     * coordinates, which is required to transform coordinates between UseCases. For example, in
     * AR, transforming the coordinates of the detected face in ImageAnalysis to coordinates in
     * PreviewView.
     *
     * <p> If the {@link SurfaceEdge} is directly connected to a camera output and its
     * aspect ratio matches the aspect ratio of the sensor, this value is usually an identity
     * matrix, with the exception of device quirks. Each time a intermediate {@link Node}
     * transforms the image buffer, it has to append the same transformation to this
     * {@link Matrix} and pass it to the downstream {@link Node}.
     */
    @NonNull
    public Matrix getSensorToBufferTransform() {
        return mSensorToBufferTransform;
    }

    /**
     * Whether the current {@link Surface} contains the camera transformation info.
     *
     * <p>Camera2 writes the camera transform to the {@link Surface}. The info is typically used by
     * {@link SurfaceView}/{@link TextureView} to correct the preview. Once it's buffer copied by
     * post-processing, the info is lost. The app (e.g. PreviewView) needs to handle the
     * transformation differently based on this flag.
     */
    public boolean hasCameraTransform() {
        return mHasCameraTransform;
    }

    // The following values represent the scenario that if this buffer is given directly to the
    // app, these are the additional transformation needs to be applied by the app. Every time we
    // make a change to the buffer, these values need to be updated as well.

    /**
     * Gets the crop rect based on {@link UseCase} config.
     */
    @NonNull
    public Rect getCropRect() {
        return mCropRect;
    }

    /**
     * Gets the clockwise rotation degrees based on {@link UseCase} config.
     */
    public int getRotationDegrees() {
        return mRotationDegrees;
    }

    /**
     * Sets the rotation degrees.
     *
     * <p>If the surface provider is created via {@link #createSurfaceRequest(CameraInternal)}, the
     * returned SurfaceRequest will receive the rotation update by
     * {@link SurfaceRequest.TransformationInfoListener}.
     */
    @MainThread
    public void setRotationDegrees(int rotationDegrees) {
        checkMainThread();
        if (mRotationDegrees == rotationDegrees) {
            return;
        }
        mRotationDegrees = rotationDegrees;
        notifyTransformationInfoUpdate();
    }

    @MainThread
    private void notifyTransformationInfoUpdate() {
        checkMainThread();
        if (mProviderSurfaceRequest != null) {
            mProviderSurfaceRequest.updateTransformationInfo(
                    TransformationInfo.of(mCropRect, mRotationDegrees, ROTATION_NOT_SPECIFIED,
                            hasCameraTransform()));
        }
    }

    /**
     * Check the edge only has one consumer defensively.
     */
    private void checkAndSetHasConsumer() {
        checkState(!mHasConsumer, "Consumer can only be linked once.");
        mHasConsumer = true;
    }

    /**
     * Gets whether the buffer needs to be horizontally mirrored based on {@link UseCase} config.
     */
    public boolean getMirroring() {
        return mMirroring;
    }

    /**
     * Returns {@link StreamSpec} associated with this edge.
     */
    @NonNull
    public StreamSpec getStreamSpec() {
        return mStreamSpec;
    }

    private void checkNotClosed() {
        checkState(!mIsClosed, "Edge is already closed.");
    }

    @VisibleForTesting
    @NonNull
    public DeferrableSurface getDeferrableSurfaceForTesting() {
        return mSettableSurface;
    }

    @VisibleForTesting
    public boolean isClosed() {
        return mIsClosed;
    }

    /**
     * @return true if this edge is connected to a Surface provider.
     */
    @VisibleForTesting
    public boolean hasProvider() {
        return mSettableSurface.hasProvider();
    }

    /**
     * A {@link DeferrableSurface} that sets another {@link DeferrableSurface} as the source.
     *
     * <p>This class provides mechanisms to link an {@link DeferrableSurface}, and propagates
     * Surface releasing/closure to the {@link DeferrableSurface}.
     */
    static class SettableSurface extends DeferrableSurface {

        final ListenableFuture<Surface> mSurfaceFuture = CallbackToFutureAdapter.getFuture(
                completer -> {
                    mCompleter = completer;
                    return "SettableFuture hashCode: " + hashCode();
                });

        CallbackToFutureAdapter.Completer<Surface> mCompleter;

        private DeferrableSurface mProvider;

        SettableSurface(@NonNull Size size) {
            super(size, ImageFormat.PRIVATE);
        }

        @NonNull
        @Override
        protected ListenableFuture<Surface> provideSurface() {
            return mSurfaceFuture;
        }

        @MainThread
        boolean canSetProvider() {
            checkMainThread();
            return mProvider == null && !isClosed();
        }

        @VisibleForTesting
        boolean hasProvider() {
            return mProvider != null;
        }

        /**
         * Sets the {@link DeferrableSurface} that provides the surface.
         *
         * <p>This method is idempotent. Calling it with the same provider no-ops.
         *
         * @return true if the provider is set; false if the same provider has already been set.
         * @throws IllegalStateException    if the provider has already been set.
         * @throws IllegalArgumentException if the provider's size is different than the size of
         *                                  this {@link SettableSurface}.
         * @throws SurfaceClosedException   if the provider is already closed.
         * @see SurfaceEdge#setProvider(DeferrableSurface)
         */
        @MainThread
        public boolean setProvider(@NonNull DeferrableSurface provider)
                throws SurfaceClosedException {
            checkMainThread();
            checkNotNull(provider);
            if (mProvider == provider) {
                // Same provider has already been set. Ignore.
                return false;
            }
            checkState(mProvider == null, "A different provider has been set. To change the "
                    + "provider, call SurfaceEdge#invalidate before calling "
                    + "SurfaceEdge#setProvider");
            checkArgument(getPrescribedSize().equals(provider.getPrescribedSize()),
                    "The provider's size must match the parent");
            checkState(!isClosed(), "The parent is closed. Call SurfaceEdge#invalidate() before "
                    + "setting a new provider.");
            mProvider = provider;
            Futures.propagate(provider.getSurface(), mCompleter);
            provider.incrementUseCount();
            getTerminationFuture().addListener(provider::decrementUseCount, directExecutor());
            return true;
        }
    }
}