ExoPlayerAssetLoader.java

/*
 * Copyright 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.media3.transformer;

import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS;
import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_MS;
import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_MAX_BUFFER_MS;
import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_MIN_BUFFER_MS;
import static androidx.media3.transformer.ExportException.ERROR_CODE_FAILED_RUNTIME_CHECK;
import static androidx.media3.transformer.ExportException.ERROR_CODE_UNSPECIFIED;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
import static java.lang.Math.min;

import android.content.Context;
import android.os.Handler;
import android.os.Looper;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.Player;
import androidx.media3.common.Timeline;
import androidx.media3.common.Tracks;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.exoplayer.DefaultLoadControl;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.RenderersFactory;
import androidx.media3.exoplayer.audio.AudioRendererEventListener;
import androidx.media3.exoplayer.metadata.MetadataOutput;
import androidx.media3.exoplayer.source.DefaultMediaSourceFactory;
import androidx.media3.exoplayer.source.MediaSource;
import androidx.media3.exoplayer.text.TextOutput;
import androidx.media3.exoplayer.trackselection.DefaultTrackSelector;
import androidx.media3.exoplayer.video.VideoRendererEventListener;
import androidx.media3.extractor.DefaultExtractorsFactory;
import androidx.media3.extractor.mp4.Mp4Extractor;
import com.google.common.collect.ImmutableMap;

/** An {@link AssetLoader} implementation that uses an {@link ExoPlayer} to load samples. */
@UnstableApi
public final class ExoPlayerAssetLoader implements AssetLoader {

  /** An {@link AssetLoader.Factory} for {@link ExoPlayerAssetLoader} instances. */
  public static final class Factory implements AssetLoader.Factory {

    private final Context context;
    private final Codec.DecoderFactory decoderFactory;
    private final boolean forceInterpretHdrAsSdr;
    private final Clock clock;
    @Nullable private final MediaSource.Factory mediaSourceFactory;

    /**
     * Creates an instance using a {@link DefaultMediaSourceFactory}.
     *
     * @param context The {@link Context}.
     * @param decoderFactory The {@link Codec.DecoderFactory} to use to decode the samples (if
     *     necessary).
     * @param forceInterpretHdrAsSdr Whether to apply {@link
     *     TransformationRequest#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}.
     * @param clock The {@link Clock} to use. It should always be {@link Clock#DEFAULT}, except for
     *     testing.
     */
    public Factory(
        Context context,
        Codec.DecoderFactory decoderFactory,
        boolean forceInterpretHdrAsSdr,
        Clock clock) {
      this.context = context;
      this.decoderFactory = decoderFactory;
      this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr;
      this.clock = clock;
      this.mediaSourceFactory = null;
    }

    /**
     * Creates an instance.
     *
     * @param context The {@link Context}.
     * @param decoderFactory The {@link Codec.DecoderFactory} to use to decode the samples (if
     *     necessary).
     * @param forceInterpretHdrAsSdr Whether to apply {@link
     *     TransformationRequest#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}.
     * @param clock The {@link Clock} to use. It should always be {@link Clock#DEFAULT}, except for
     *     testing.
     * @param mediaSourceFactory The {@link MediaSource.Factory} to use to retrieve the samples to
     *     transform.
     */
    public Factory(
        Context context,
        Codec.DecoderFactory decoderFactory,
        boolean forceInterpretHdrAsSdr,
        Clock clock,
        MediaSource.Factory mediaSourceFactory) {
      this.context = context;
      this.decoderFactory = decoderFactory;
      this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr;
      this.clock = clock;
      this.mediaSourceFactory = mediaSourceFactory;
    }

    @Override
    public AssetLoader createAssetLoader(
        EditedMediaItem editedMediaItem, Looper looper, Listener listener) {
      MediaSource.Factory mediaSourceFactory = this.mediaSourceFactory;
      if (mediaSourceFactory == null) {
        DefaultExtractorsFactory defaultExtractorsFactory = new DefaultExtractorsFactory();
        if (editedMediaItem.flattenForSlowMotion) {
          defaultExtractorsFactory.setMp4ExtractorFlags(Mp4Extractor.FLAG_READ_SEF_DATA);
        }
        mediaSourceFactory = new DefaultMediaSourceFactory(context, defaultExtractorsFactory);
      }
      return new ExoPlayerAssetLoader(
          context,
          editedMediaItem,
          mediaSourceFactory,
          decoderFactory,
          forceInterpretHdrAsSdr,
          looper,
          listener,
          clock);
    }
  }

  private final EditedMediaItem editedMediaItem;
  private final CapturingDecoderFactory decoderFactory;
  private final ExoPlayer player;

  private @Transformer.ProgressState int progressState;

  private ExoPlayerAssetLoader(
      Context context,
      EditedMediaItem editedMediaItem,
      MediaSource.Factory mediaSourceFactory,
      Codec.DecoderFactory decoderFactory,
      boolean forceInterpretHdrAsSdr,
      Looper looper,
      Listener listener,
      Clock clock) {
    this.editedMediaItem = editedMediaItem;
    this.decoderFactory = new CapturingDecoderFactory(decoderFactory);

    DefaultTrackSelector trackSelector = new DefaultTrackSelector(context);
    trackSelector.setParameters(
        new DefaultTrackSelector.Parameters.Builder(context)
            .setForceHighestSupportedBitrate(true)
            .build());
    // Arbitrarily decrease buffers for playback so that samples start being sent earlier to the
    // pipelines (rebuffers are less problematic for the export use case).
    DefaultLoadControl loadControl =
        new DefaultLoadControl.Builder()
            .setBufferDurationsMs(
                DEFAULT_MIN_BUFFER_MS,
                DEFAULT_MAX_BUFFER_MS,
                DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10,
                DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10)
            .build();
    ExoPlayer.Builder playerBuilder =
        new ExoPlayer.Builder(
                context,
                new RenderersFactoryImpl(
                    editedMediaItem.removeAudio,
                    editedMediaItem.removeVideo,
                    editedMediaItem.flattenForSlowMotion,
                    this.decoderFactory,
                    forceInterpretHdrAsSdr,
                    listener))
            .setMediaSourceFactory(mediaSourceFactory)
            .setTrackSelector(trackSelector)
            .setLoadControl(loadControl)
            .setLooper(looper)
            .setUsePlatformDiagnostics(false);
    if (clock != Clock.DEFAULT) {
      // Transformer.Builder#setClock is also @VisibleForTesting, so if we're using a non-default
      // clock we must be in a test context.
      @SuppressWarnings("VisibleForTests")
      ExoPlayer.Builder unusedForAnnotation = playerBuilder.setClock(clock);
    }
    player = playerBuilder.build();
    player.addListener(new PlayerListener(listener));

    progressState = PROGRESS_STATE_NOT_STARTED;
  }

  @Override
  public void start() {
    player.setMediaItem(editedMediaItem.mediaItem);
    player.prepare();
    progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
  }

  @Override
  public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) {
    if (progressState == PROGRESS_STATE_AVAILABLE) {
      long durationMs = player.getDuration();
      long positionMs = player.getCurrentPosition();
      progressHolder.progress = min((int) (positionMs * 100 / durationMs), 99);
    }
    return progressState;
  }

  @Override
  public ImmutableMap<Integer, String> getDecoderNames() {
    ImmutableMap.Builder<Integer, String> decoderNamesByTrackType = new ImmutableMap.Builder<>();
    @Nullable String audioDecoderName = decoderFactory.getAudioDecoderName();
    if (audioDecoderName != null) {
      decoderNamesByTrackType.put(C.TRACK_TYPE_AUDIO, audioDecoderName);
    }
    @Nullable String videoDecoderName = decoderFactory.getVideoDecoderName();
    if (videoDecoderName != null) {
      decoderNamesByTrackType.put(C.TRACK_TYPE_VIDEO, videoDecoderName);
    }
    return decoderNamesByTrackType.buildOrThrow();
  }

  @Override
  public void release() {
    player.release();
    progressState = PROGRESS_STATE_NOT_STARTED;
  }

  private static final class RenderersFactoryImpl implements RenderersFactory {

    private final TransformerMediaClock mediaClock;
    private final boolean removeAudio;
    private final boolean removeVideo;
    private final boolean flattenForSlowMotion;
    private final Codec.DecoderFactory decoderFactory;
    private final boolean forceInterpretHdrAsSdr;
    private final Listener assetLoaderListener;

    public RenderersFactoryImpl(
        boolean removeAudio,
        boolean removeVideo,
        boolean flattenForSlowMotion,
        Codec.DecoderFactory decoderFactory,
        boolean forceInterpretHdrAsSdr,
        Listener assetLoaderListener) {
      this.removeAudio = removeAudio;
      this.removeVideo = removeVideo;
      this.flattenForSlowMotion = flattenForSlowMotion;
      this.decoderFactory = decoderFactory;
      this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr;
      this.assetLoaderListener = assetLoaderListener;
      mediaClock = new TransformerMediaClock();
    }

    @Override
    public Renderer[] createRenderers(
        Handler eventHandler,
        VideoRendererEventListener videoRendererEventListener,
        AudioRendererEventListener audioRendererEventListener,
        TextOutput textRendererOutput,
        MetadataOutput metadataRendererOutput) {
      int rendererCount = removeAudio || removeVideo ? 1 : 2;
      Renderer[] renderers = new Renderer[rendererCount];
      int index = 0;
      if (!removeAudio) {
        renderers[index] =
            new ExoAssetLoaderAudioRenderer(decoderFactory, mediaClock, assetLoaderListener);
        index++;
      }
      if (!removeVideo) {
        renderers[index] =
            new ExoAssetLoaderVideoRenderer(
                flattenForSlowMotion,
                decoderFactory,
                forceInterpretHdrAsSdr,
                mediaClock,
                assetLoaderListener);
        index++;
      }
      return renderers;
    }
  }

  private final class PlayerListener implements Player.Listener {

    private final Listener assetLoaderListener;

    public PlayerListener(Listener assetLoaderListener) {
      this.assetLoaderListener = assetLoaderListener;
    }

    @Override
    public void onTimelineChanged(Timeline timeline, int reason) {
      try {
        if (progressState != PROGRESS_STATE_WAITING_FOR_AVAILABILITY) {
          return;
        }
        Timeline.Window window = new Timeline.Window();
        timeline.getWindow(/* windowIndex= */ 0, window);
        if (!window.isPlaceholder) {
          long durationUs = window.durationUs;
          // Make progress permanently unavailable if the duration is unknown, so that it doesn't
          // jump to a high value at the end of the export if the duration is set once the media is
          // entirely loaded.
          progressState =
              durationUs <= 0 || durationUs == C.TIME_UNSET
                  ? PROGRESS_STATE_UNAVAILABLE
                  : PROGRESS_STATE_AVAILABLE;
          assetLoaderListener.onDurationUs(window.durationUs);
        }
      } catch (RuntimeException e) {
        assetLoaderListener.onError(
            ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED));
      }
    }

    @Override
    public void onTracksChanged(Tracks tracks) {
      try {
        int trackCount = 0;
        if (tracks.isTypeSelected(C.TRACK_TYPE_AUDIO)) {
          trackCount++;
        }
        if (tracks.isTypeSelected(C.TRACK_TYPE_VIDEO)) {
          trackCount++;
        }

        if (trackCount > 0) {
          assetLoaderListener.onTrackCount(trackCount);
          // Start the renderers after having registered all the tracks to make sure the AssetLoader
          // listener callbacks are called in the right order.
          player.play();
        } else {
          assetLoaderListener.onError(
              ExportException.createForAssetLoader(
                  new IllegalStateException("The asset loader has no track to output."),
                  ERROR_CODE_FAILED_RUNTIME_CHECK));
        }
      } catch (RuntimeException e) {
        assetLoaderListener.onError(
            ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED));
      }
    }

    @Override
    public void onPlayerError(PlaybackException error) {
      @ExportException.ErrorCode
      int errorCode =
          checkNotNull(
              ExportException.NAME_TO_ERROR_CODE.getOrDefault(
                  error.getErrorCodeName(), ERROR_CODE_UNSPECIFIED));
      assetLoaderListener.onError(ExportException.createForAssetLoader(error, errorCode));
    }
  }
}