CaptureProcessorPipeline.java
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.camera.core;
import android.graphics.ImageFormat;
import android.media.ImageReader;
import android.util.Size;
import android.view.Surface;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import androidx.camera.core.impl.CaptureProcessor;
import androidx.camera.core.impl.ImageProxyBundle;
import androidx.camera.core.impl.ImageReaderProxy;
import androidx.camera.core.impl.TagBundle;
import androidx.camera.core.impl.utils.executor.CameraXExecutors;
import androidx.core.util.Preconditions;
import com.google.common.util.concurrent.ListenableFuture;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.RejectedExecutionException;
/**
* A CaptureProcessor which can link two CaptureProcessors.
*/
@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
class CaptureProcessorPipeline implements CaptureProcessor {
private static final String TAG = "CaptureProcessorPipeline";
private final CaptureProcessor mPreCaptureProcessor;
private final CaptureProcessor mPostCaptureProcessor;
final Executor mExecutor;
private final int mMaxImages;
private ImageReaderProxy mIntermediateImageReader = null;
private ImageInfo mSourceImageInfo = null;
/**
* Creates a {@link CaptureProcessorPipeline} to link two CaptureProcessors to process the
* captured images.
*
* @param preCaptureProcessor The pre-processing {@link CaptureProcessor} which must output
* YUV_420_888 {@link ImageProxy} for the post-processing
* {@link CaptureProcessor} to process.
* @param maxImages the maximum image buffer count used to create the intermediate
* {@link ImageReaderProxy} to receive the processed
* {@link ImageProxy} from the
* pre-processing {@link CaptureProcessor}.
* @param postCaptureProcessor The post-processing {@link CaptureProcessor} which can process
* an {@link ImageProxy} of YUV_420_888 format . It must be able
* to process the image without referencing to the
* {@link TagBundle} and capture id.
* @param executor the {@link Executor} used by the post-processing
* {@link CaptureProcessor}
* to process the {@link ImageProxy}.
*/
CaptureProcessorPipeline(@NonNull CaptureProcessor preCaptureProcessor, int maxImages,
@NonNull CaptureProcessor postCaptureProcessor, @NonNull Executor executor) {
mPreCaptureProcessor = preCaptureProcessor;
mPostCaptureProcessor = postCaptureProcessor;
mExecutor = executor;
mMaxImages = maxImages;
}
@Override
public void onOutputSurface(@NonNull Surface surface, int imageFormat) {
// Updates the output surface to the post-processing CaptureProcessor
mPostCaptureProcessor.onOutputSurface(surface, imageFormat);
}
@Override
public void process(@NonNull ImageProxyBundle bundle) {
List<Integer> ids = bundle.getCaptureIds();
ListenableFuture<ImageProxy> imageProxyListenableFuture = bundle.getImageProxy(ids.get(0));
Preconditions.checkArgument(imageProxyListenableFuture.isDone());
ImageProxy imageProxy;
try {
imageProxy = imageProxyListenableFuture.get();
ImageInfo imageInfo = imageProxy.getImageInfo();
// Stores the imageInfo of source image that will be used when when the processed
// ImageProxy is received from the pre-processing CaptureProcessor.
mSourceImageInfo = imageInfo;
} catch (ExecutionException | InterruptedException e) {
throw new IllegalArgumentException("Can not successfully extract ImageProxy from the "
+ "ImageProxyBundle.");
}
// Calls the pre-processing CaptureProcessor to process the ImageProxyBundle
mPreCaptureProcessor.process(bundle);
}
@Override
public void onResolutionUpdate(@NonNull Size size) {
// Creates an intermediate ImageReader to receive the processed image from the
// pre-processing CaptureProcessor.
mIntermediateImageReader = new AndroidImageReaderProxy(
ImageReader.newInstance(size.getWidth(), size.getHeight(),
ImageFormat.YUV_420_888, mMaxImages));
mPreCaptureProcessor.onOutputSurface(mIntermediateImageReader.getSurface(),
ImageFormat.YUV_420_888);
mPreCaptureProcessor.onResolutionUpdate(size);
// Updates the resolution information to the post-processing CaptureProcessor.
mPostCaptureProcessor.onResolutionUpdate(size);
// Register the ImageAvailableListener to receive the processed image from the
// pre-processing CaptureProcessor.
mIntermediateImageReader.setOnImageAvailableListener(
imageReader -> {
ImageProxy image = imageReader.acquireNextImage();
try {
mExecutor.execute(() -> postProcess(image));
} catch (RejectedExecutionException e) {
Logger.e(TAG, "The executor for post-processing might have been "
+ "shutting down or terminated!");
image.close();
}
},
CameraXExecutors.directExecutor());
}
void postProcess(ImageProxy imageProxy) {
Size resolution = new Size(imageProxy.getWidth(), imageProxy.getHeight());
// Retrieves information from ImageInfo of source image to create a
// SettableImageProxyBundle and calls the post-processing CaptureProcessor to process it.
Preconditions.checkNotNull(mSourceImageInfo);
String tagBundleKey = mSourceImageInfo.getTagBundle().listKeys().iterator().next();
int captureId = (Integer) mSourceImageInfo.getTagBundle().getTag(tagBundleKey);
SettableImageProxy settableImageProxy =
new SettableImageProxy(imageProxy, resolution, mSourceImageInfo);
mSourceImageInfo = null;
SettableImageProxyBundle settableImageProxyBundle = new SettableImageProxyBundle(
Collections.singletonList(captureId), tagBundleKey);
settableImageProxyBundle.addImageProxy(settableImageProxy);
mPostCaptureProcessor.process(settableImageProxyBundle);
}
/**
* Closes the objects generated when creating the {@link CaptureProcessorPipeline}.
*/
void close() {
if (mIntermediateImageReader != null) {
mIntermediateImageReader.clearOnImageAvailableListener();
mIntermediateImageReader.close();
}
}
}