BitmapTextureManager.java

/*
 * Copyright 2023 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package androidx.media3.effect;

import static androidx.media3.common.util.Assertions.checkNotNull;
import static java.lang.Math.round;

import android.graphics.Bitmap;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

/**
 * Forwards a video frame produced from a {@link Bitmap} to a {@link GlShaderProgram} for
 * consumption.
 *
 * <p>Public methods in this class can be called from any thread.
 */
@UnstableApi
/* package */ final class BitmapTextureManager implements TextureManager {
  private final GlShaderProgram shaderProgram;
  private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
  // The queue holds all bitmaps with one or more frames pending to be sent downstream.
  private final Queue<BitmapFrameSequenceInfo> pendingBitmaps;

  private @MonotonicNonNull GlTextureInfo currentGlTextureInfo;
  private int downstreamShaderProgramCapacity;
  private int framesToQueueForCurrentBitmap;
  private double currentPresentationTimeUs;
  private boolean useHdr;
  private boolean inputEnded;

  /**
   * Creates a new instance.
   *
   * @param shaderProgram The {@link GlShaderProgram} for which this {@code BitmapTextureManager}
   *     will be set as the {@link GlShaderProgram.InputListener}.
   * @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor} that the
   *     methods of this class run on.
   */
  public BitmapTextureManager(
      GlShaderProgram shaderProgram,
      VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) {
    this.shaderProgram = shaderProgram;
    this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
    pendingBitmaps = new LinkedBlockingQueue<>();
  }

  @Override
  public void onReadyToAcceptInputFrame() {
    videoFrameProcessingTaskExecutor.submit(
        () -> {
          downstreamShaderProgramCapacity++;
          maybeQueueToShaderProgram();
        });
  }

  @Override
  public void queueInputBitmap(
      Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
    videoFrameProcessingTaskExecutor.submit(
        () -> setupBitmap(inputBitmap, durationUs, offsetUs, frameRate, useHdr));
  }

  @Override
  public int getPendingFrameCount() {
    // Always treat all queued bitmaps as immediately processed.
    return 0;
  }

  @Override
  public void signalEndOfCurrentInputStream() {
    // Do nothing here. End of current input signaling is handled in maybeQueueToShaderProgram().
  }

  @Override
  public void signalEndOfInput() {
    videoFrameProcessingTaskExecutor.submit(
        () -> {
          if (framesToQueueForCurrentBitmap == 0 && pendingBitmaps.isEmpty()) {
            shaderProgram.signalEndOfCurrentInputStream();
          } else {
            inputEnded = true;
          }
        });
  }

  @Override
  public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) {
    // Do nothing.
  }

  @Override
  public void release() {
    videoFrameProcessingTaskExecutor.submit(
        () -> {
          if (currentGlTextureInfo != null) {
            GlUtil.deleteTexture(currentGlTextureInfo.texId);
          }
        });
  }

  // Methods that must be called on the GL thread.

  private void setupBitmap(
      Bitmap bitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr)
      throws VideoFrameProcessingException {
    this.useHdr = useHdr;
    int framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND));
    double frameDurationUs = C.MICROS_PER_SECOND / frameRate;
    pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, offsetUs, frameDurationUs, framesToAdd));

    maybeQueueToShaderProgram();
  }

  private void maybeQueueToShaderProgram() throws VideoFrameProcessingException {
    if (pendingBitmaps.isEmpty() || downstreamShaderProgramCapacity == 0) {
      return;
    }

    BitmapFrameSequenceInfo currentBitmapInfo = checkNotNull(pendingBitmaps.peek());
    if (framesToQueueForCurrentBitmap == 0) {
      Bitmap bitmap = currentBitmapInfo.bitmap;
      framesToQueueForCurrentBitmap = currentBitmapInfo.numberOfFrames;
      currentPresentationTimeUs = currentBitmapInfo.offsetUs;
      int currentTexId;
      try {
        if (currentGlTextureInfo != null) {
          GlUtil.deleteTexture(currentGlTextureInfo.texId);
        }
        currentTexId =
            GlUtil.createTexture(
                bitmap.getWidth(),
                bitmap.getHeight(),
                /* useHighPrecisionColorComponents= */ useHdr);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, currentTexId);
        GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0);
        GlUtil.checkGlError();
      } catch (GlUtil.GlException e) {
        throw VideoFrameProcessingException.from(e);
      }
      currentGlTextureInfo =
          new GlTextureInfo(
              currentTexId,
              /* fboId= */ C.INDEX_UNSET,
              /* rboId= */ C.INDEX_UNSET,
              bitmap.getWidth(),
              bitmap.getHeight());
    }

    framesToQueueForCurrentBitmap--;
    downstreamShaderProgramCapacity--;

    shaderProgram.queueInputFrame(
        checkNotNull(currentGlTextureInfo), round(currentPresentationTimeUs));

    currentPresentationTimeUs += currentBitmapInfo.frameDurationUs;
    if (framesToQueueForCurrentBitmap == 0) {
      pendingBitmaps.remove();
      if (pendingBitmaps.isEmpty() && inputEnded) {
        // Only signal end of stream after all pending bitmaps are processed.
        // TODO(b/269424561): Call signalEndOfCurrentInputStream on every bitmap
        shaderProgram.signalEndOfCurrentInputStream();
      }
    }
  }

  /** Information to generate all the frames associated with a specific {@link Bitmap}. */
  private static final class BitmapFrameSequenceInfo {
    public final Bitmap bitmap;
    public final long offsetUs;
    public final double frameDurationUs;
    public final int numberOfFrames;

    public BitmapFrameSequenceInfo(
        Bitmap bitmap, long offsetUs, double frameDurationUs, int numberOfFrames) {
      this.bitmap = bitmap;
      this.offsetUs = offsetUs;
      this.frameDurationUs = frameDurationUs;
      this.numberOfFrames = numberOfFrames;
    }
  }
}