BasicVendorExtender.java
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.camera.extensions.internal;
import android.content.Context;
import android.graphics.ImageFormat;
import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.params.StreamConfigurationMap;
import android.util.Pair;
import android.util.Range;
import android.util.Size;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.OptIn;
import androidx.annotation.RequiresApi;
import androidx.camera.camera2.interop.Camera2CameraInfo;
import androidx.camera.camera2.interop.ExperimentalCamera2Interop;
import androidx.camera.core.CameraInfo;
import androidx.camera.core.Logger;
import androidx.camera.core.impl.SessionProcessor;
import androidx.camera.extensions.ExtensionMode;
import androidx.camera.extensions.impl.AutoImageCaptureExtenderImpl;
import androidx.camera.extensions.impl.AutoPreviewExtenderImpl;
import androidx.camera.extensions.impl.BeautyImageCaptureExtenderImpl;
import androidx.camera.extensions.impl.BeautyPreviewExtenderImpl;
import androidx.camera.extensions.impl.BokehImageCaptureExtenderImpl;
import androidx.camera.extensions.impl.BokehPreviewExtenderImpl;
import androidx.camera.extensions.impl.HdrImageCaptureExtenderImpl;
import androidx.camera.extensions.impl.HdrPreviewExtenderImpl;
import androidx.camera.extensions.impl.ImageCaptureExtenderImpl;
import androidx.camera.extensions.impl.NightImageCaptureExtenderImpl;
import androidx.camera.extensions.impl.NightPreviewExtenderImpl;
import androidx.camera.extensions.impl.PreviewExtenderImpl;
import androidx.camera.extensions.internal.compat.workaround.ExtensionDisabledValidator;
import androidx.camera.extensions.internal.sessionprocessor.BasicExtenderSessionProcessor;
import androidx.core.util.Preconditions;
import org.jetbrains.annotations.TestOnly;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* Basic vendor interface implementation
*/
@RequiresApi(23) // TODO(b/200306659): Remove and replace with annotation on package-info.java
// replaceImageFormatIfMissing accesses ImageFormat#PRIVATE which is public since API level 23.
public class BasicVendorExtender implements VendorExtender {
private static final String TAG = "BasicVendorExtender";
private final ExtensionDisabledValidator mExtensionDisabledValidator =
new ExtensionDisabledValidator();
private PreviewExtenderImpl mPreviewExtenderImpl = null;
private ImageCaptureExtenderImpl mImageCaptureExtenderImpl = null;
private CameraInfo mCameraInfo;
public BasicVendorExtender(@ExtensionMode.Mode int mode) {
try {
switch (mode) {
case ExtensionMode.BOKEH:
mPreviewExtenderImpl = new BokehPreviewExtenderImpl();
mImageCaptureExtenderImpl = new BokehImageCaptureExtenderImpl();
break;
case ExtensionMode.HDR:
mPreviewExtenderImpl = new HdrPreviewExtenderImpl();
mImageCaptureExtenderImpl = new HdrImageCaptureExtenderImpl();
break;
case ExtensionMode.NIGHT:
mPreviewExtenderImpl = new NightPreviewExtenderImpl();
mImageCaptureExtenderImpl = new NightImageCaptureExtenderImpl();
break;
case ExtensionMode.FACE_RETOUCH:
mPreviewExtenderImpl = new BeautyPreviewExtenderImpl();
mImageCaptureExtenderImpl = new BeautyImageCaptureExtenderImpl();
break;
case ExtensionMode.AUTO:
mPreviewExtenderImpl = new AutoPreviewExtenderImpl();
mImageCaptureExtenderImpl = new AutoImageCaptureExtenderImpl();
break;
case ExtensionMode.NONE:
default:
throw new IllegalArgumentException("Should not activate ExtensionMode.NONE");
}
} catch (NoClassDefFoundError e) {
Logger.e(TAG, "OEM implementation for extension mode " + mode + "does not exist!");
}
}
@TestOnly
BasicVendorExtender(ImageCaptureExtenderImpl imageCaptureExtenderImpl,
PreviewExtenderImpl previewExtenderImpl) {
mPreviewExtenderImpl = previewExtenderImpl;
mImageCaptureExtenderImpl = imageCaptureExtenderImpl;
}
@Override
public boolean isExtensionAvailable(@NonNull String cameraId,
@NonNull Map<String, CameraCharacteristics> characteristicsMap) {
if (mExtensionDisabledValidator.shouldDisableExtension()) {
return false;
}
// Returns false if implementation classes do not exist.
if (mPreviewExtenderImpl == null || mImageCaptureExtenderImpl == null) {
return false;
}
CameraCharacteristics cameraCharacteristics = characteristicsMap.get(cameraId);
return mPreviewExtenderImpl.isExtensionAvailable(cameraId, cameraCharacteristics)
&& mImageCaptureExtenderImpl.isExtensionAvailable(cameraId, cameraCharacteristics);
}
@OptIn(markerClass = ExperimentalCamera2Interop.class)
@Override
public void init(@NonNull CameraInfo cameraInfo) {
mCameraInfo = cameraInfo;
if (mPreviewExtenderImpl == null || mImageCaptureExtenderImpl == null) {
return;
}
String cameraId = Camera2CameraInfo.from(cameraInfo).getCameraId();
CameraCharacteristics cameraCharacteristics =
Camera2CameraInfo.extractCameraCharacteristics(cameraInfo);
mPreviewExtenderImpl.init(cameraId, cameraCharacteristics);
mImageCaptureExtenderImpl.init(cameraId, cameraCharacteristics);
Logger.d(TAG, "PreviewExtender processorType= " + mPreviewExtenderImpl.getProcessorType());
Logger.d(TAG, "ImageCaptureExtender processor= "
+ mImageCaptureExtenderImpl.getCaptureProcessor());
}
@Nullable
@Override
public Range<Long> getEstimatedCaptureLatencyRange(@Nullable Size size) {
Preconditions.checkNotNull(mCameraInfo, "VendorExtender#init() must be called first");
if (mImageCaptureExtenderImpl != null && ExtensionVersion.getRuntimeVersion().compareTo(
Version.VERSION_1_2) >= 0) {
try {
return mImageCaptureExtenderImpl.getEstimatedCaptureLatencyRange(size);
} catch (NoSuchMethodError e) {
}
}
return null;
}
@OptIn(markerClass = ExperimentalCamera2Interop.class)
private Size[] getOutputSizes(int imageFormat) {
StreamConfigurationMap map = Camera2CameraInfo.from(mCameraInfo)
.getCameraCharacteristic(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
return map.getOutputSizes(imageFormat);
}
private int getCaptureInputImageFormat() {
if (mImageCaptureExtenderImpl != null
&& mImageCaptureExtenderImpl.getCaptureProcessor() != null) {
return ImageFormat.YUV_420_888;
} else {
return ImageFormat.JPEG;
}
}
private int getPreviewInputImageFormat() {
if (mPreviewExtenderImpl != null
&& mPreviewExtenderImpl.getProcessorType()
== PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
return ImageFormat.YUV_420_888;
} else {
return ImageFormat.PRIVATE;
}
}
@NonNull
@Override
public List<Pair<Integer, Size[]>> getSupportedPreviewOutputResolutions() {
Preconditions.checkNotNull(mCameraInfo, "VendorExtender#init() must be called first");
if (mPreviewExtenderImpl != null && ExtensionVersion.getRuntimeVersion().compareTo(
Version.VERSION_1_1) >= 0) {
try {
List<Pair<Integer, Size[]>> result =
mPreviewExtenderImpl.getSupportedResolutions();
if (result != null) {
// Ensure the PRIVATE format is in the list.
// PreviewExtenderImpl.getSupportedResolutions() returns the supported size
// for input surface. We need to ensure output surface format is supported.
return replaceImageFormatIfMissing(result,
ImageFormat.YUV_420_888 /* formatToBeReplaced */,
ImageFormat.PRIVATE /* newFormat */);
}
} catch (NoSuchMethodError e) {
}
}
// Returns output sizes from stream configuration map if OEM returns null or OEM does not
// implement the function. BasicVendorExtender's SessionProcessor will always output
// to PRIVATE surface, but the input image which connect to the camera could be
// either YUV or PRIVATE. Since the input image from input surface is guaranteed to be
// able to output to the output surface, therefore we fetch the sizes from the
// input image format for the output format.
int inputImageFormat = getPreviewInputImageFormat();
return Arrays.asList(new Pair<>(ImageFormat.PRIVATE, getOutputSizes(inputImageFormat)));
}
@NonNull
@Override
public List<Pair<Integer, Size[]>> getSupportedCaptureOutputResolutions() {
Preconditions.checkNotNull(mCameraInfo, "VendorExtender#init() must be called first");
if (mImageCaptureExtenderImpl != null && ExtensionVersion.getRuntimeVersion().compareTo(
Version.VERSION_1_1) >= 0) {
try {
List<Pair<Integer, Size[]>> result =
mImageCaptureExtenderImpl.getSupportedResolutions();
if (result != null) {
// Ensure the JPEG format is in the list.
// ImageCaptureExtenderImpl.getSupportedResolutions() returns the supported
// size for input surface. We need to ensure output surface format is supported.
return replaceImageFormatIfMissing(result,
ImageFormat.YUV_420_888 /* formatToBeReplaced */,
ImageFormat.JPEG /* newFormat */);
}
} catch (NoSuchMethodError e) {
}
}
// Returns output sizes from stream configuration map if OEM returns null or OEM does not
// implement the function. BasicVendorExtender's SessionProcessor will always output
// JPEG Images, but the input image which connect to the camera could be either YUV or
// JPEG. Since the input image from input surface is guaranteed to be able to output to
// the output surface, therefore we fetch the sizes from the input image format for the
// output format.
int inputImageFormat = getCaptureInputImageFormat();
return Arrays.asList(new Pair<>(ImageFormat.JPEG, getOutputSizes(inputImageFormat)));
}
private List<Pair<Integer, Size[]>> replaceImageFormatIfMissing(
List<Pair<Integer, Size[]>> input, int formatToBeReplaced, int newFormat) {
for (Pair<Integer, Size[]> pair : input) {
if (pair.first == newFormat) {
return input;
}
}
List<Pair<Integer, Size[]>> output = new ArrayList<>();
boolean formatFound = false;
for (Pair<Integer, Size[]> pair : input) {
if (pair.first == formatToBeReplaced) {
formatFound = true;
output.add(new Pair<>(newFormat, pair.second));
} else {
output.add(pair);
}
}
if (!formatFound) {
throw new IllegalArgumentException(
"Supported resolution should contain " + newFormat + " format.");
}
return output;
}
@NonNull
@Override
public Size[] getSupportedYuvAnalysisResolutions() {
Preconditions.checkNotNull(mCameraInfo, "VendorExtender#init() must be called first");
return getOutputSizes(ImageFormat.YUV_420_888);
}
@Nullable
@Override
public SessionProcessor createSessionProcessor(@NonNull Context context) {
Preconditions.checkNotNull(mCameraInfo, "VendorExtender#init() must be called first");
return new BasicExtenderSessionProcessor(mPreviewExtenderImpl, mImageCaptureExtenderImpl,
context);
}
}