SingleFrameGlTextureProcessor.java

/*
 * Copyright 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package androidx.media3.effect;

import static androidx.media3.common.util.Assertions.checkState;

import android.util.Pair;
import androidx.annotation.CallSuper;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

/**
 * Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels
 * into an output frame, with changes to pixels specific to the implementation.
 *
 * <p>{@code SingleFrameGlTextureProcessor} implementations must produce exactly one output frame
 * per input frame with the same presentation timestamp. For more flexibility, implement {@link
 * GlTextureProcessor} directly.
 *
 * <p>All methods in this class must be called on the thread that owns the OpenGL context.
 */
@UnstableApi
public abstract class SingleFrameGlTextureProcessor implements GlTextureProcessor {

  private final boolean useHdr;

  private InputListener inputListener;
  private OutputListener outputListener;
  private ErrorListener errorListener;
  private int inputWidth;
  private int inputHeight;
  private @MonotonicNonNull TextureInfo outputTexture;
  private boolean outputTextureInUse;

  /**
   * Creates a {@code SingleFrameGlTextureProcessor} instance.
   *
   * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
   *     in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
   */
  public SingleFrameGlTextureProcessor(boolean useHdr) {
    this.useHdr = useHdr;
    inputListener = new InputListener() {};
    outputListener = new OutputListener() {};
    errorListener = (frameProcessingException) -> {};
  }

  /**
   * Configures the texture processor based on the input dimensions.
   *
   * <p>This method must be called before {@linkplain #drawFrame(int,long) drawing} the first frame
   * and before drawing subsequent frames with different input dimensions.
   *
   * @param inputWidth The input width, in pixels.
   * @param inputHeight The input height, in pixels.
   * @return The output width and height of frames processed through {@link #drawFrame(int, long)}.
   */
  public abstract Pair<Integer, Integer> configure(int inputWidth, int inputHeight);

  /**
   * Draws one frame.
   *
   * <p>This method may only be called after the texture processor has been {@link #configure(int,
   * int) configured}. The caller is responsible for focussing the correct render target before
   * calling this method.
   *
   * <p>A minimal implementation should tell OpenGL to use its shader program, bind the shader
   * program's vertex attributes and uniforms, and issue a drawing command.
   *
   * @param inputTexId Identifier of a 2D OpenGL texture containing the input frame.
   * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
   * @throws FrameProcessingException If an error occurs while processing or drawing the frame.
   */
  public abstract void drawFrame(int inputTexId, long presentationTimeUs)
      throws FrameProcessingException;

  @Override
  public final void setInputListener(InputListener inputListener) {
    this.inputListener = inputListener;
    if (!outputTextureInUse) {
      inputListener.onReadyToAcceptInputFrame();
    }
  }

  @Override
  public final void setOutputListener(OutputListener outputListener) {
    this.outputListener = outputListener;
  }

  @Override
  public final void setErrorListener(ErrorListener errorListener) {
    this.errorListener = errorListener;
  }

  @Override
  public final void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
    checkState(
        !outputTextureInUse,
        "The texture processor does not currently accept input frames. Release prior output frames"
            + " first.");

    try {
      if (outputTexture == null
          || inputTexture.width != inputWidth
          || inputTexture.height != inputHeight) {
        configureOutputTexture(inputTexture.width, inputTexture.height);
      }
      outputTextureInUse = true;
      GlUtil.focusFramebufferUsingCurrentContext(
          outputTexture.fboId, outputTexture.width, outputTexture.height);
      GlUtil.clearOutputFrame();
      drawFrame(inputTexture.texId, presentationTimeUs);
      inputListener.onInputFrameProcessed(inputTexture);
      outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
    } catch (FrameProcessingException | GlUtil.GlException | RuntimeException e) {
      errorListener.onFrameProcessingError(
          e instanceof FrameProcessingException
              ? (FrameProcessingException) e
              : new FrameProcessingException(e));
    }
  }

  @EnsuresNonNull("outputTexture")
  private void configureOutputTexture(int inputWidth, int inputHeight) throws GlUtil.GlException {
    this.inputWidth = inputWidth;
    this.inputHeight = inputHeight;
    Pair<Integer, Integer> outputSize = configure(inputWidth, inputHeight);
    if (outputTexture == null
        || outputSize.first != outputTexture.width
        || outputSize.second != outputTexture.height) {
      if (outputTexture != null) {
        GlUtil.deleteTexture(outputTexture.texId);
      }
      int outputTexId = GlUtil.createTexture(outputSize.first, outputSize.second, useHdr);
      int outputFboId = GlUtil.createFboForTexture(outputTexId);
      outputTexture =
          new TextureInfo(outputTexId, outputFboId, outputSize.first, outputSize.second);
    }
  }

  @Override
  public final void releaseOutputFrame(TextureInfo outputTexture) {
    outputTextureInUse = false;
    inputListener.onReadyToAcceptInputFrame();
  }

  @Override
  public final void signalEndOfCurrentInputStream() {
    outputListener.onCurrentOutputStreamEnded();
  }

  @Override
  @CallSuper
  public void release() throws FrameProcessingException {
    if (outputTexture != null) {
      try {
        GlUtil.deleteTexture(outputTexture.texId);
      } catch (GlUtil.GlException e) {
        throw new FrameProcessingException(e);
      }
    }
  }
}