SceneRenderer.java

/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package androidx.media3.exoplayer.video.spherical;

import static androidx.media3.common.util.GlUtil.checkGlError;

import android.graphics.SurfaceTexture;
import android.media.MediaFormat;
import android.opengl.GLES20;
import android.opengl.Matrix;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.Assertions;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.TimedValueQueue;
import androidx.media3.exoplayer.video.VideoFrameMetadataListener;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

/** Renders a GL Scene. */
/* package */ final class SceneRenderer
    implements VideoFrameMetadataListener, CameraMotionListener {

  private static final String TAG = "SceneRenderer";

  private final AtomicBoolean frameAvailable;
  private final AtomicBoolean resetRotationAtNextFrame;
  private final ProjectionRenderer projectionRenderer;
  private final FrameRotationQueue frameRotationQueue;
  private final TimedValueQueue<Long> sampleTimestampQueue;
  private final TimedValueQueue<Projection> projectionQueue;
  private final float[] rotationMatrix;
  private final float[] tempMatrix;

  // Used by GL thread only
  private int textureId;
  private @MonotonicNonNull SurfaceTexture surfaceTexture;

  // Used by other threads only
  private volatile @C.StereoMode int defaultStereoMode;
  private @C.StereoMode int lastStereoMode;
  @Nullable private byte[] lastProjectionData;

  // Methods called on any thread.

  public SceneRenderer() {
    frameAvailable = new AtomicBoolean();
    resetRotationAtNextFrame = new AtomicBoolean(true);
    projectionRenderer = new ProjectionRenderer();
    frameRotationQueue = new FrameRotationQueue();
    sampleTimestampQueue = new TimedValueQueue<>();
    projectionQueue = new TimedValueQueue<>();
    rotationMatrix = new float[16];
    tempMatrix = new float[16];
    defaultStereoMode = C.STEREO_MODE_MONO;
    lastStereoMode = Format.NO_VALUE;
  }

  /**
   * Sets the default stereo mode. If the played video doesn't contain a stereo mode the default one
   * is used.
   *
   * @param stereoMode A {@link C.StereoMode} value.
   */
  public void setDefaultStereoMode(@C.StereoMode int stereoMode) {
    defaultStereoMode = stereoMode;
  }

  // Methods called on GL thread.

  /** Initializes the renderer. */
  public SurfaceTexture init() {
    try {
      // Set the background frame color. This is only visible if the display mesh isn't a full
      // sphere.
      GLES20.glClearColor(0.5f, 0.5f, 0.5f, 1.0f);
      checkGlError();

      projectionRenderer.init();
      checkGlError();

      textureId = GlUtil.createExternalTexture();
    } catch (GlUtil.GlException e) {
      Log.e(TAG, "Failed to initialize the renderer", e);
    }
    surfaceTexture = new SurfaceTexture(textureId);
    surfaceTexture.setOnFrameAvailableListener(surfaceTexture -> frameAvailable.set(true));
    return surfaceTexture;
  }

  /**
   * Draws the scene with a given eye pose and type.
   *
   * @param viewProjectionMatrix 16 element GL matrix.
   * @param rightEye Whether the right eye view should be drawn. If {@code false}, the left eye view
   *     is drawn.
   */
  public void drawFrame(float[] viewProjectionMatrix, boolean rightEye) {
    // glClear isn't strictly necessary when rendering fully spherical panoramas, but it can improve
    // performance on tiled renderers by causing the GPU to discard previous data.
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    try {
      checkGlError();
    } catch (GlUtil.GlException e) {
      Log.e(TAG, "Failed to draw a frame", e);
    }

    if (frameAvailable.compareAndSet(true, false)) {
      Assertions.checkNotNull(surfaceTexture).updateTexImage();
      try {
        checkGlError();
      } catch (GlUtil.GlException e) {
        Log.e(TAG, "Failed to draw a frame", e);
      }
      if (resetRotationAtNextFrame.compareAndSet(true, false)) {
        GlUtil.setToIdentity(rotationMatrix);
      }
      long lastFrameTimestampNs = surfaceTexture.getTimestamp();
      Long sampleTimestampUs = sampleTimestampQueue.poll(lastFrameTimestampNs);
      if (sampleTimestampUs != null) {
        frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestampUs);
      }
      Projection projection = projectionQueue.pollFloor(lastFrameTimestampNs);
      if (projection != null) {
        projectionRenderer.setProjection(projection);
      }
    }
    Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0);
    projectionRenderer.draw(textureId, tempMatrix, rightEye);
  }

  /** Cleans up GL resources. */
  public void shutdown() {
    projectionRenderer.shutdown();
  }

  // Methods called on playback thread.

  // VideoFrameMetadataListener implementation.

  @Override
  public void onVideoFrameAboutToBeRendered(
      long presentationTimeUs,
      long releaseTimeNs,
      Format format,
      @Nullable MediaFormat mediaFormat) {
    sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
    setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
  }

  // CameraMotionListener implementation.

  @Override
  public void onCameraMotion(long timeUs, float[] rotation) {
    frameRotationQueue.setRotation(timeUs, rotation);
  }

  @Override
  public void onCameraMotionReset() {
    sampleTimestampQueue.clear();
    frameRotationQueue.reset();
    resetRotationAtNextFrame.set(true);
  }

  /**
   * Sets projection data and stereo mode of the media to be played.
   *
   * @param projectionData Contains the projection data to be rendered.
   * @param stereoMode A {@link C.StereoMode} value.
   * @param timeNs When then new projection should be used.
   */
  private void setProjection(
      @Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) {
    byte[] oldProjectionData = lastProjectionData;
    int oldStereoMode = lastStereoMode;
    lastProjectionData = projectionData;
    lastStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode;
    if (oldStereoMode == lastStereoMode && Arrays.equals(oldProjectionData, lastProjectionData)) {
      return;
    }

    Projection projectionFromData = null;
    if (lastProjectionData != null) {
      projectionFromData = ProjectionDecoder.decode(lastProjectionData, lastStereoMode);
    }
    Projection projection =
        projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData)
            ? projectionFromData
            : Projection.createEquirectangular(lastStereoMode);
    projectionQueue.add(timeNs, projection);
  }
}