/*
* Copyright 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.camera.core;
import android.media.ImageReader;
import android.os.Handler;
import android.util.Log;
import android.util.Size;
import android.view.Surface;
import androidx.annotation.GuardedBy;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.camera.core.impl.CameraCaptureCallback;
import androidx.camera.core.impl.CaptureBundle;
import androidx.camera.core.impl.CaptureProcessor;
import androidx.camera.core.impl.CaptureStage;
import androidx.camera.core.impl.ImageReaderProxy;
import androidx.camera.core.impl.utils.executor.CameraXExecutors;
import androidx.camera.core.impl.utils.futures.FutureCallback;
import androidx.camera.core.impl.utils.futures.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executor;
/**
* An {@link ImageReaderProxy} which takes one or more {@link android.media.Image}, processes it,
* then output the final result {@link ImageProxy} to
* {@link ImageReaderProxy.OnImageAvailableListener}.
*
* <p>ProcessingImageReader takes {@link CaptureBundle} as the expected set of
* {@link CaptureStage}. Once all the ImageProxy from the captures are ready. It invokes
* the {@link CaptureProcessor} set, then returns a single output ImageProxy to
* OnImageAvailableListener.
*/
class ProcessingImageReader implements ImageReaderProxy {
private static final String TAG = "ProcessingImageReader";
private final Object mLock = new Object();
// Callback when Image is ready from InputImageReader.
private ImageReaderProxy.OnImageAvailableListener mTransformedListener =
new ImageReaderProxy.OnImageAvailableListener() {
@Override
public void onImageAvailable(@NonNull ImageReaderProxy reader) {
imageIncoming(reader);
}
};
// Callback when Image is ready from OutputImageReader.
private ImageReaderProxy.OnImageAvailableListener mImageProcessedListener =
new ImageReaderProxy.OnImageAvailableListener() {
// TODO(b/141958189): Suppressed during upgrade to AGP 3.6.
@SuppressWarnings("GuardedBy")
@Override
public void onImageAvailable(@NonNull ImageReaderProxy reader) {
// Callback the output OnImageAvailableListener.
if (mExecutor != null) {
mExecutor.execute(
new Runnable() {
@Override
public void run() {
mListener.onImageAvailable(ProcessingImageReader.this);
}
});
} else {
mListener.onImageAvailable(ProcessingImageReader.this);
}
// Resets SettableImageProxyBundle after the processor finishes processing.
mSettableImageProxyBundle.reset();
setupSettableImageProxyBundleCallbacks();
}
};
// Callback when all the ImageProxies in SettableImageProxyBundle are ready.
private FutureCallback<List<ImageProxy>> mCaptureStageReadyCallback =
new FutureCallback<List<ImageProxy>>() {
// TODO(b/141958189): Suppressed during upgrade to AGP 3.6.
@SuppressWarnings("GuardedBy")
@Override
public void onSuccess(@Nullable List<ImageProxy> imageProxyList) {
mCaptureProcessor.process(mSettableImageProxyBundle);
}
@Override
public void onFailure(Throwable throwable) {
}
};
@GuardedBy("mLock")
private boolean mClosed = false;
@GuardedBy("mLock")
private final ImageReaderProxy mInputImageReader;
@GuardedBy("mLock")
private final ImageReaderProxy mOutputImageReader;
@GuardedBy("mLock")
@Nullable
ImageReaderProxy.OnImageAvailableListener mListener;
@GuardedBy("mLock")
@Nullable
Executor mExecutor;
@NonNull
CaptureProcessor mCaptureProcessor;
@GuardedBy("mLock")
SettableImageProxyBundle mSettableImageProxyBundle = null;
private final List<Integer> mCaptureIdList = new ArrayList<>();
/**
* Create a {@link ProcessingImageReader} with specific configurations.
*
* @param width Width of the ImageReader
* @param height Height of the ImageReader
* @param format Image format
* @param maxImages Maximum Image number the ImageReader can hold. The capacity should
* be greater than the captureBundle size in order to hold all the
* Images needed with this processing.
* @param handler Handler for executing
* {@link ImageReaderProxy.OnImageAvailableListener}
* @param captureBundle The {@link CaptureBundle} includes the processing information
* @param captureProcessor The {@link CaptureProcessor} to be invoked when the Images are ready
*/
ProcessingImageReader(int width, int height, int format, int maxImages,
@Nullable Handler handler,
@NonNull CaptureBundle captureBundle, @NonNull CaptureProcessor captureProcessor) {
mInputImageReader = new MetadataImageReader(
width,
height,
format,
maxImages,
handler);
mOutputImageReader = new AndroidImageReaderProxy(
ImageReader.newInstance(width, height, format, maxImages));
init(CameraXExecutors.newHandlerExecutor(handler), captureBundle, captureProcessor);
}
ProcessingImageReader(ImageReaderProxy imageReader, @Nullable Handler handler,
@NonNull CaptureBundle captureBundle,
@NonNull CaptureProcessor captureProcessor) {
if (imageReader.getMaxImages() < captureBundle.getCaptureStages().size()) {
throw new IllegalArgumentException(
"MetadataImageReader is smaller than CaptureBundle.");
}
mInputImageReader = imageReader;
mOutputImageReader = new AndroidImageReaderProxy(
ImageReader.newInstance(imageReader.getWidth(),
imageReader.getHeight(), imageReader.getImageFormat(),
imageReader.getMaxImages()));
init(CameraXExecutors.newHandlerExecutor(handler), captureBundle, captureProcessor);
}
@SuppressWarnings("GuardedBy") // TODO(b/141958189): Suppressed during upgrade to AGP 3.6.
private void init(@NonNull Executor executor, @NonNull CaptureBundle captureBundle,
@NonNull CaptureProcessor captureProcessor) {
mExecutor = executor;
mInputImageReader.setOnImageAvailableListener(mTransformedListener, executor);
mOutputImageReader.setOnImageAvailableListener(mImageProcessedListener, executor);
mCaptureProcessor = captureProcessor;
mCaptureProcessor.onOutputSurface(mOutputImageReader.getSurface(), getImageFormat());
mCaptureProcessor.onResolutionUpdate(
new Size(mInputImageReader.getWidth(), mInputImageReader.getHeight()));
setCaptureBundle(captureBundle);
}
@Override
@Nullable
public ImageProxy acquireLatestImage() {
synchronized (mLock) {
return mOutputImageReader.acquireLatestImage();
}
}
@Override
@Nullable
public ImageProxy acquireNextImage() {
synchronized (mLock) {
return mOutputImageReader.acquireNextImage();
}
}
@Override
public void close() {
synchronized (mLock) {
if (mClosed) {
return;
}
mInputImageReader.close();
mOutputImageReader.close();
mSettableImageProxyBundle.close();
mClosed = true;
}
}
@Override
public int getHeight() {
synchronized (mLock) {
return mInputImageReader.getHeight();
}
}
@Override
public int getWidth() {
synchronized (mLock) {
return mInputImageReader.getWidth();
}
}
@Override
public int getImageFormat() {
synchronized (mLock) {
return mInputImageReader.getImageFormat();
}
}
@Override
public int getMaxImages() {
synchronized (mLock) {
return mInputImageReader.getMaxImages();
}
}
@NonNull
@Override
public Surface getSurface() {
synchronized (mLock) {
return mInputImageReader.getSurface();
}
}
@Override
public void setOnImageAvailableListener(
@NonNull final ImageReaderProxy.OnImageAvailableListener listener,
@Nullable Handler handler) {
setOnImageAvailableListener(listener, CameraXExecutors.newHandlerExecutor(handler));
}
@Override
public void setOnImageAvailableListener(@NonNull OnImageAvailableListener listener,
@NonNull Executor executor) {
// TODO(b/115747543) support callback on executor
synchronized (mLock) {
mListener = listener;
mExecutor = executor;
mInputImageReader.setOnImageAvailableListener(mTransformedListener, executor);
mOutputImageReader.setOnImageAvailableListener(mImageProcessedListener, executor);
}
}
/** Sets a CaptureBundle */
public void setCaptureBundle(@NonNull CaptureBundle captureBundle) {
synchronized (mLock) {
if (captureBundle.getCaptureStages() != null) {
if (mInputImageReader.getMaxImages() < captureBundle.getCaptureStages().size()) {
throw new IllegalArgumentException(
"CaptureBundle is lager than InputImageReader.");
}
mCaptureIdList.clear();
for (CaptureStage captureStage : captureBundle.getCaptureStages()) {
if (captureStage != null) {
mCaptureIdList.add(captureStage.getId());
}
}
}
mSettableImageProxyBundle = new SettableImageProxyBundle(mCaptureIdList);
setupSettableImageProxyBundleCallbacks();
}
}
/** Returns necessary camera callbacks to retrieve metadata from camera result. */
@SuppressWarnings("GuardedBy") // TODO(b/141958189): Suppressed during upgrade to AGP 3.6.
@Nullable
CameraCaptureCallback getCameraCaptureCallback() {
if (mInputImageReader instanceof MetadataImageReader) {
return ((MetadataImageReader) mInputImageReader).getCameraCaptureCallback();
} else {
return null;
}
}
@SuppressWarnings("GuardedBy") // TODO(b/141958189): Suppressed during upgrade to AGP 3.6.
void setupSettableImageProxyBundleCallbacks() {
List<ListenableFuture<ImageProxy>> futureList = new ArrayList<>();
for (Integer id : mCaptureIdList) {
futureList.add(mSettableImageProxyBundle.getImageProxy(id));
}
Futures.addCallback(Futures.allAsList(futureList), mCaptureStageReadyCallback,
CameraXExecutors.directExecutor());
}
// Incoming Image from InputImageReader. Acquires it and add to SettableImageProxyBundle.
void imageIncoming(ImageReaderProxy imageReader) {
synchronized (mLock) {
if (mClosed) {
return;
}
ImageProxy image = null;
try {
image = imageReader.acquireNextImage();
} catch (IllegalStateException e) {
Log.e(TAG, "Failed to acquire latest image.", e);
} finally {
if (image != null) {
Integer tag = (Integer) image.getImageInfo().getTag();
if (!mCaptureIdList.contains(tag)) {
Log.w(TAG, "ImageProxyBundle does not contain this id: " + tag);
image.close();
} else {
mSettableImageProxyBundle.addImageProxy(image);
}
}
}
}
}
}