/*
* Copyright 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.camera.camera2.internal;
import android.hardware.camera2.CameraAccessException;
import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CameraDevice;
import android.hardware.camera2.CaptureRequest;
import android.os.Handler;
import android.view.Surface;
import androidx.annotation.GuardedBy;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.camera.camera2.internal.SynchronizedCaptureSessionOpener.SynchronizedSessionFeature;
import androidx.camera.camera2.internal.annotation.CameraExecutor;
import androidx.camera.camera2.internal.compat.params.SessionConfigurationCompat;
import androidx.camera.core.Logger;
import androidx.camera.core.impl.DeferrableSurface;
import androidx.camera.core.impl.utils.executor.CameraXExecutors;
import androidx.camera.core.impl.utils.futures.FutureChain;
import androidx.camera.core.impl.utils.futures.Futures;
import androidx.concurrent.futures.CallbackToFutureAdapter;
import com.google.common.util.concurrent.ListenableFuture;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
/**
* <p>The SynchronizedCaptureSessionImpl synchronizing methods between the other
* SynchronizedCaptureSessions to fix b/135050586, b/145725334, b/144817309, b/146773463. The
* SynchronizedCaptureSessionBaseImpl would be a non-synchronizing version.
*
* <p>In b/144817309, the onClosed() callback on
* {@link android.hardware.camera2.CameraCaptureSession.StateCallback}
* might not be invoked if the capture session is not the latest one. To align the fixed
* framework behavior, we manually call the onClosed() when a new CameraCaptureSession is created.
*
* <p>The b/135050586, b/145725334 need to close the {@link DeferrableSurface} to force the
* {@link DeferrableSurface} recreate in the new CaptureSession.
*
* <p>b/146773463: It needs to check all the releasing capture sessions are ready for opening
* next capture session.
*/
class SynchronizedCaptureSessionImpl extends SynchronizedCaptureSessionBaseImpl {
private static final String TAG = "SyncCaptureSessionImpl";
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
private final Object mObjectLock = new Object();
@NonNull
private final Set<String> mEnabledFeature;
@NonNull
private final ListenableFuture<Void> mStartStreamingFuture;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
CallbackToFutureAdapter.Completer<Void> mStartStreamingCompleter;
@Nullable
private final ListenableFuture<Void> mClosingDeferrableSurfaceFuture;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
@Nullable
CallbackToFutureAdapter.Completer<Void> mClosingDeferrableSurfaceCompleter;
@Nullable
@GuardedBy("mObjectLock")
private List<DeferrableSurface> mDeferrableSurfaces;
@Nullable
@GuardedBy("mObjectLock")
ListenableFuture<Void> mOpeningCaptureSession;
@Nullable
@GuardedBy("mObjectLock")
ListenableFuture<List<Surface>> mStartingSurface;
/** Whether the capture session has submitted the repeating request. */
@GuardedBy("mObjectLock")
private boolean mHasSubmittedRepeating;
SynchronizedCaptureSessionImpl(
@NonNull Set<String> enabledFeature,
@NonNull CaptureSessionRepository repository,
@NonNull @CameraExecutor Executor executor,
@NonNull ScheduledExecutorService scheduledExecutorService,
@NonNull Handler compatHandler) {
super(repository, executor, scheduledExecutorService, compatHandler);
mEnabledFeature = enabledFeature;
if (enabledFeature.contains(SynchronizedCaptureSessionOpener.FEATURE_WAIT_FOR_REQUEST)) {
mStartStreamingFuture = CallbackToFutureAdapter.getFuture(completer -> {
// Opening and releasing the capture session quickly and constantly is a problem for
// LEGACY devices. See: b/146773463. It needs to check all the releasing capture
// sessions are ready for opening next capture session.
mStartStreamingCompleter = completer;
return "StartStreamingFuture[session=" + SynchronizedCaptureSessionImpl.this
+ "]";
});
} else {
mStartStreamingFuture = Futures.immediateFuture(null);
}
if (mEnabledFeature.contains(
SynchronizedCaptureSessionOpener.FEATURE_DEFERRABLE_SURFACE_CLOSE)) {
mClosingDeferrableSurfaceFuture = CallbackToFutureAdapter.getFuture(completer -> {
mClosingDeferrableSurfaceCompleter = completer;
return "ClosingDeferrableSurfaceFuture[session="
+ SynchronizedCaptureSessionImpl.this + "]";
});
} else {
mClosingDeferrableSurfaceFuture = Futures.immediateFuture(null);
}
}
@NonNull
@Override
public ListenableFuture<Void> openCaptureSession(@NonNull CameraDevice cameraDevice,
@NonNull SessionConfigurationCompat sessionConfigurationCompat) {
synchronized (mObjectLock) {
List<ListenableFuture<Void>> futureList =
getBlockerFuture(SynchronizedCaptureSessionOpener.FEATURE_WAIT_FOR_REQUEST,
mCaptureSessionRepository.getClosingCaptureSession());
mOpeningCaptureSession =
FutureChain.from(Futures.successfulAsList(futureList)).transformAsync(
v -> super.openCaptureSession(cameraDevice, sessionConfigurationCompat),
CameraXExecutors.directExecutor());
return Futures.nonCancellationPropagating(mOpeningCaptureSession);
}
}
@NonNull
@Override
public ListenableFuture<Void> getSynchronizedBlocker(
@SynchronizedSessionFeature @NonNull String feature) {
switch (feature) {
case SynchronizedCaptureSessionOpener.FEATURE_WAIT_FOR_REQUEST:
// Returns the future which is completed once the session starts streaming
// frames.
return Futures.nonCancellationPropagating(mStartStreamingFuture);
case SynchronizedCaptureSessionOpener.FEATURE_DEFERRABLE_SURFACE_CLOSE:
return Futures.nonCancellationPropagating(mClosingDeferrableSurfaceFuture);
default:
return super.getSynchronizedBlocker(feature);
}
}
private List<ListenableFuture<Void>> getBlockerFuture(
@SynchronizedSessionFeature @NonNull String feature,
List<SynchronizedCaptureSession> sessions) {
List<ListenableFuture<Void>> futureList = new ArrayList<>();
for (SynchronizedCaptureSession session : sessions) {
futureList.add(session.getSynchronizedBlocker(feature));
}
return futureList;
}
@NonNull
@Override
public ListenableFuture<List<Surface>> startWithDeferrableSurface(
@NonNull List<DeferrableSurface> deferrableSurfaces, long timeout) {
synchronized (mObjectLock) {
mDeferrableSurfaces = deferrableSurfaces;
List<ListenableFuture<Void>> futureList = Collections.emptyList();
if (mEnabledFeature.contains(SynchronizedCaptureSessionOpener.FEATURE_FORCE_CLOSE)) {
Map<SynchronizedCaptureSession, List<DeferrableSurface>> registeredSurfaceMap =
mCaptureSessionRepository.registerDeferrableSurface(this,
deferrableSurfaces);
List<SynchronizedCaptureSession> sessionsWithSameSurface = new ArrayList<>();
for (Map.Entry<SynchronizedCaptureSession, List<DeferrableSurface>> entry :
registeredSurfaceMap.entrySet()) {
if (entry.getKey() != this && !Collections.disjoint(entry.getValue(),
mDeferrableSurfaces)) {
sessionsWithSameSurface.add(entry.getKey());
}
}
// Only blocking this method when the other SynchronizedCaptureSessions using the
// same deferrableSurface instance.
futureList = getBlockerFuture(
SynchronizedCaptureSessionOpener.FEATURE_DEFERRABLE_SURFACE_CLOSE,
sessionsWithSameSurface);
}
mStartingSurface =
FutureChain.from(Futures.successfulAsList(futureList)).transformAsync(
v -> super.startWithDeferrableSurface(deferrableSurfaces, timeout),
getExecutor());
return Futures.nonCancellationPropagating(mStartingSurface);
}
}
@Override
public boolean stop() {
synchronized (mObjectLock) {
if (isCameraCaptureSessionOpen()) {
closeConfiguredDeferrableSurfaces();
} else {
if (mOpeningCaptureSession != null) {
mOpeningCaptureSession.cancel(true);
}
if (mStartingSurface != null) {
mStartingSurface.cancel(true);
}
stopDeferrableSurface();
}
return super.stop();
}
}
@Override
public int setSingleRepeatingRequest(@NonNull CaptureRequest request,
@NonNull CameraCaptureSession.CaptureCallback listener) throws CameraAccessException {
if (mEnabledFeature.contains(SynchronizedCaptureSessionOpener.FEATURE_WAIT_FOR_REQUEST)) {
synchronized (mObjectLock) {
mHasSubmittedRepeating = true;
CameraCaptureSession.CaptureCallback comboCaptureCallback =
Camera2CaptureCallbacks.createComboCallback(mCaptureCallback, listener);
return super.setSingleRepeatingRequest(request, comboCaptureCallback);
}
} else {
return super.setSingleRepeatingRequest(request, listener);
}
}
@Override
public void onConfigured(@NonNull SynchronizedCaptureSession session) {
debugLog("Session onConfigured()");
if (mEnabledFeature.contains(SynchronizedCaptureSessionOpener.FEATURE_FORCE_CLOSE)) {
Set<SynchronizedCaptureSession> staleCreatingSessions = new LinkedHashSet<>();
for (SynchronizedCaptureSession s :
mCaptureSessionRepository.getCreatingCaptureSessions()) {
// Collect the sessions that started configuring before the current session. The
// current session and the session that starts configure after the current session
// are not included since they don't need to be closed.
if (s == session) {
break;
}
staleCreatingSessions.add(s);
}
// Once the CaptureSession is configured, the stale CaptureSessions should not have
// chance to complete the configuration flow. Force change to configure fail since
// the configureFail will treat the CaptureSession is closed. More detail please see
// b/158540776.
forceOnConfigureFailed(staleCreatingSessions);
}
super.onConfigured(session);
// Once the new CameraCaptureSession is created, all the previous opened
// CameraCaptureSession can be treated as closed (more detail in b/144817309),
// trigger its associated StateCallback#onClosed callback to finish the
// session close flow.
if (mEnabledFeature.contains(SynchronizedCaptureSessionOpener.FEATURE_FORCE_CLOSE)) {
Set<SynchronizedCaptureSession> openedSessions = new LinkedHashSet<>();
for (SynchronizedCaptureSession s : mCaptureSessionRepository.getCaptureSessions()) {
// The entrySet keys of the LinkedHashMap should be insertion-ordered, so we
// get the previous capture sessions by iterate it from the beginning.
if (s == session) {
break;
}
openedSessions.add(s);
}
forceOnClosed(openedSessions);
}
}
@Override
public void close() {
debugLog("Session call close()");
if (mEnabledFeature.contains(SynchronizedCaptureSessionOpener.FEATURE_WAIT_FOR_REQUEST)) {
synchronized (mObjectLock) {
if (!mHasSubmittedRepeating) {
// If the release() is called before any repeating requests have been issued,
// then the startStreamingFuture should be cancelled.
mStartStreamingFuture.cancel(true);
}
}
}
mStartStreamingFuture.addListener(() -> {
// Checks the capture session is ready before closing. See: b/146773463.
debugLog("Session call super.close()");
super.close();
}, getExecutor());
}
void closeConfiguredDeferrableSurfaces() {
synchronized (mObjectLock) {
if (mDeferrableSurfaces == null) {
debugLog("deferrableSurface == null, maybe forceClose, skip close");
return;
}
if (mEnabledFeature.contains(
SynchronizedCaptureSessionOpener.FEATURE_DEFERRABLE_SURFACE_CLOSE)) {
// Do not close for non-LEGACY devices. Reusing {@link DeferrableSurface} is only a
// problem for LEGACY devices. See: b/135050586.
// Another problem is the behavior of TextureView below API 23. It releases {@link
// SurfaceTexture}. Hence, request to close and recreate {@link DeferrableSurface}.
// See: b/145725334.
for (DeferrableSurface deferrableSurface : mDeferrableSurfaces) {
deferrableSurface.close();
}
debugLog("deferrableSurface closed");
stopDeferrableSurface();
}
}
}
void stopDeferrableSurface() {
if (mEnabledFeature.contains(
SynchronizedCaptureSessionOpener.FEATURE_DEFERRABLE_SURFACE_CLOSE)) {
mCaptureSessionRepository.unregisterDeferrableSurface(this);
if (mClosingDeferrableSurfaceCompleter != null) {
mClosingDeferrableSurfaceCompleter.set(null);
}
}
}
@Override
public void onClosed(@NonNull SynchronizedCaptureSession session) {
closeConfiguredDeferrableSurfaces();
debugLog("onClosed()");
super.onClosed(session);
}
static void forceOnClosed(@NonNull Set<SynchronizedCaptureSession> sessions) {
for (SynchronizedCaptureSession session : sessions) {
session.getStateCallback().onClosed(session);
}
}
private void forceOnConfigureFailed(@NonNull Set<SynchronizedCaptureSession> sessions) {
for (SynchronizedCaptureSession session : sessions) {
session.getStateCallback().onConfigureFailed(session);
}
}
void debugLog(String message) {
Logger.d(TAG, "[" + SynchronizedCaptureSessionImpl.this + "] " + message);
}
private final CameraCaptureSession.CaptureCallback mCaptureCallback =
new CameraCaptureSession.CaptureCallback() {
@Override
public void onCaptureStarted(@NonNull CameraCaptureSession session,
@NonNull CaptureRequest request, long timestamp, long frameNumber) {
if (mStartStreamingCompleter != null) {
mStartStreamingCompleter.set(null);
mStartStreamingCompleter = null;
}
}
@Override
public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session,
int sequenceId) {
if (mStartStreamingCompleter != null) {
mStartStreamingCompleter.setCancelled();
mStartStreamingCompleter = null;
}
}
};
}