/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.video;
import static androidx.media3.common.util.Assertions.checkNotNull;
import android.content.Context;
import android.hardware.display.DisplayManager;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Message;
import android.view.Choreographer;
import android.view.Choreographer.FrameCallback;
import android.view.Display;
import android.view.Surface;
import android.view.WindowManager;
import androidx.annotation.DoNotInline;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.Renderer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Helps a video {@link Renderer} release frames to a {@link Surface}. The helper:
*
* <ul>
* <li>Adjusts frame release timestamps to achieve a smoother visual result. The release
* timestamps are smoothed, and aligned with the default display's vsync signal.
* <li>Adjusts the {@link Surface} frame rate to inform the underlying platform of a fixed frame
* rate, when there is one.
* </ul>
*/
@UnstableApi
public final class VideoFrameReleaseHelper {
private static final String TAG = "VideoFrameReleaseHelper";
/**
* The minimum sum of frame durations used to calculate the current fixed frame rate estimate, for
* the estimate to be treated as a high confidence estimate.
*/
private static final long MINIMUM_MATCHING_FRAME_DURATION_FOR_HIGH_CONFIDENCE_NS = 5_000_000_000L;
/**
* The minimum change in media frame rate that will trigger a change in surface frame rate, given
* a high confidence estimate.
*/
private static final float MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_HIGH_CONFIDENCE = 0.02f;
/**
* The minimum change in media frame rate that will trigger a change in surface frame rate, given
* a low confidence estimate.
*/
private static final float MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_LOW_CONFIDENCE = 1f;
/**
* The minimum number of frames without a frame rate estimate, for the surface frame rate to be
* cleared.
*/
private static final int MINIMUM_FRAMES_WITHOUT_SYNC_TO_CLEAR_SURFACE_FRAME_RATE =
2 * FixedFrameRateEstimator.CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC;
/** The period between sampling display VSYNC timestamps, in milliseconds. */
private static final long VSYNC_SAMPLE_UPDATE_PERIOD_MS = 500;
/**
* The maximum adjustment that can be made to a frame release timestamp, in nanoseconds, excluding
* the part of the adjustment that aligns frame release timestamps with the display VSYNC.
*/
private static final long MAX_ALLOWED_ADJUSTMENT_NS = 20_000_000;
/**
* If a frame is targeted to a display VSYNC with timestamp {@code vsyncTime}, the adjusted frame
* release timestamp will be calculated as {@code releaseTime = vsyncTime - ((vsyncDuration *
* VSYNC_OFFSET_PERCENTAGE) / 100)}.
*/
private static final long VSYNC_OFFSET_PERCENTAGE = 80;
private final FixedFrameRateEstimator frameRateEstimator;
@Nullable private final DisplayHelper displayHelper;
@Nullable private final VSyncSampler vsyncSampler;
private boolean started;
@Nullable private Surface surface;
/** The media frame rate specified in the {@link Format}. */
private float formatFrameRate;
/**
* The media frame rate used to calculate the playback frame rate of the {@link Surface}. This may
* be different to {@link #formatFrameRate} if {@link #formatFrameRate} is unspecified or
* inaccurate.
*/
private float surfaceMediaFrameRate;
/** The playback frame rate set on the {@link Surface}. */
private float surfacePlaybackFrameRate;
private float playbackSpeed;
private @C.VideoChangeFrameRateStrategy int changeFrameRateStrategy;
private long vsyncDurationNs;
private long vsyncOffsetNs;
private long frameIndex;
private long pendingLastAdjustedFrameIndex;
private long pendingLastAdjustedReleaseTimeNs;
private long lastAdjustedFrameIndex;
private long lastAdjustedReleaseTimeNs;
/**
* Constructs an instance.
*
* @param context A context from which information about the default display can be retrieved.
*/
public VideoFrameReleaseHelper(@Nullable Context context) {
frameRateEstimator = new FixedFrameRateEstimator();
displayHelper = maybeBuildDisplayHelper(context);
vsyncSampler = displayHelper != null ? VSyncSampler.getInstance() : null;
vsyncDurationNs = C.TIME_UNSET;
vsyncOffsetNs = C.TIME_UNSET;
formatFrameRate = Format.NO_VALUE;
playbackSpeed = 1f;
changeFrameRateStrategy = C.VIDEO_CHANGE_FRAME_RATE_STRATEGY_ONLY_IF_SEAMLESS;
}
/**
* Change the {@link C.VideoChangeFrameRateStrategy} used when calling {@link
* Surface#setFrameRate}.
*/
public void setChangeFrameRateStrategy(
@C.VideoChangeFrameRateStrategy int changeFrameRateStrategy) {
if (this.changeFrameRateStrategy == changeFrameRateStrategy) {
return;
}
this.changeFrameRateStrategy = changeFrameRateStrategy;
updateSurfacePlaybackFrameRate(/* forceUpdate= */ true);
}
/** Called when the renderer is started. */
public void onStarted() {
started = true;
resetAdjustment();
if (displayHelper != null) {
checkNotNull(vsyncSampler).addObserver();
displayHelper.register(this::updateDefaultDisplayRefreshRateParams);
}
updateSurfacePlaybackFrameRate(/* forceUpdate= */ false);
}
/**
* Called when the renderer changes which {@link Surface} it's rendering to renders to.
*
* @param surface The new {@link Surface}, or {@code null} if the renderer does not have one.
*/
public void onSurfaceChanged(@Nullable Surface surface) {
if (Util.SDK_INT >= 17 && Api17.isPlaceholderSurface(surface)) {
// We don't care about dummy surfaces for release timing, since they're not visible.
surface = null;
}
if (this.surface == surface) {
return;
}
clearSurfaceFrameRate();
this.surface = surface;
updateSurfacePlaybackFrameRate(/* forceUpdate= */ true);
}
/** Called when the renderer's position is reset. */
public void onPositionReset() {
resetAdjustment();
}
/**
* Called when the renderer's playback speed changes.
*
* @param playbackSpeed The factor by which playback is sped up.
*/
public void onPlaybackSpeed(float playbackSpeed) {
this.playbackSpeed = playbackSpeed;
resetAdjustment();
updateSurfacePlaybackFrameRate(/* forceUpdate= */ false);
}
/**
* Called when the renderer's output format changes.
*
* @param formatFrameRate The format's frame rate, or {@link Format#NO_VALUE} if unknown.
*/
public void onFormatChanged(float formatFrameRate) {
this.formatFrameRate = formatFrameRate;
frameRateEstimator.reset();
updateSurfaceMediaFrameRate();
}
/**
* Called by the renderer for each frame, prior to it being skipped, dropped or rendered.
*
* @param framePresentationTimeUs The frame presentation timestamp, in microseconds.
*/
public void onNextFrame(long framePresentationTimeUs) {
if (pendingLastAdjustedFrameIndex != C.INDEX_UNSET) {
lastAdjustedFrameIndex = pendingLastAdjustedFrameIndex;
lastAdjustedReleaseTimeNs = pendingLastAdjustedReleaseTimeNs;
}
frameIndex++;
frameRateEstimator.onNextFrame(framePresentationTimeUs * 1000);
updateSurfaceMediaFrameRate();
}
/** Called when the renderer is stopped. */
public void onStopped() {
started = false;
if (displayHelper != null) {
displayHelper.unregister();
checkNotNull(vsyncSampler).removeObserver();
}
clearSurfaceFrameRate();
}
// Frame release time adjustment.
/**
* Adjusts the release timestamp for the next frame. This is the frame whose presentation
* timestamp was most recently passed to {@link #onNextFrame}.
*
* <p>This method may be called any number of times for each frame, including zero times (for
* skipped frames, or when rendering the first frame prior to playback starting), or more than
* once (if the caller wishes to give the helper the opportunity to refine a release time closer
* to when the frame needs to be released).
*
* @param releaseTimeNs The frame's unadjusted release time, in nanoseconds and in the same time
* base as {@link System#nanoTime()}.
* @return The adjusted frame release timestamp, in nanoseconds and in the same time base as
* {@link System#nanoTime()}.
*/
public long adjustReleaseTime(long releaseTimeNs) {
// Until we know better, the adjustment will be a no-op.
long adjustedReleaseTimeNs = releaseTimeNs;
if (lastAdjustedFrameIndex != C.INDEX_UNSET && frameRateEstimator.isSynced()) {
long frameDurationNs = frameRateEstimator.getFrameDurationNs();
long candidateAdjustedReleaseTimeNs =
lastAdjustedReleaseTimeNs
+ (long) ((frameDurationNs * (frameIndex - lastAdjustedFrameIndex)) / playbackSpeed);
if (adjustmentAllowed(releaseTimeNs, candidateAdjustedReleaseTimeNs)) {
adjustedReleaseTimeNs = candidateAdjustedReleaseTimeNs;
} else {
resetAdjustment();
}
}
pendingLastAdjustedFrameIndex = frameIndex;
pendingLastAdjustedReleaseTimeNs = adjustedReleaseTimeNs;
if (vsyncSampler == null || vsyncDurationNs == C.TIME_UNSET) {
return adjustedReleaseTimeNs;
}
long sampledVsyncTimeNs = vsyncSampler.sampledVsyncTimeNs;
if (sampledVsyncTimeNs == C.TIME_UNSET) {
return adjustedReleaseTimeNs;
}
// Find the timestamp of the closest vsync. This is the vsync that we're targeting.
long snappedTimeNs = closestVsync(adjustedReleaseTimeNs, sampledVsyncTimeNs, vsyncDurationNs);
// Apply an offset so that we release before the target vsync, but after the previous one.
return snappedTimeNs - vsyncOffsetNs;
}
private void resetAdjustment() {
frameIndex = 0;
lastAdjustedFrameIndex = C.INDEX_UNSET;
pendingLastAdjustedFrameIndex = C.INDEX_UNSET;
}
private static boolean adjustmentAllowed(
long unadjustedReleaseTimeNs, long adjustedReleaseTimeNs) {
return Math.abs(unadjustedReleaseTimeNs - adjustedReleaseTimeNs) <= MAX_ALLOWED_ADJUSTMENT_NS;
}
// Surface frame rate adjustment.
/**
* Updates the media frame rate that's used to calculate the playback frame rate of the current
* {@link #surface}. If the frame rate is updated then {@link #updateSurfacePlaybackFrameRate} is
* called to update the surface.
*/
private void updateSurfaceMediaFrameRate() {
if (Util.SDK_INT < 30 || surface == null) {
return;
}
float candidateFrameRate =
frameRateEstimator.isSynced() ? frameRateEstimator.getFrameRate() : formatFrameRate;
if (candidateFrameRate == surfaceMediaFrameRate) {
return;
}
// The candidate is different to the current surface media frame rate. Decide whether to update
// the surface media frame rate.
boolean shouldUpdate;
if (candidateFrameRate != Format.NO_VALUE && surfaceMediaFrameRate != Format.NO_VALUE) {
boolean candidateIsHighConfidence =
frameRateEstimator.isSynced()
&& frameRateEstimator.getMatchingFrameDurationSumNs()
>= MINIMUM_MATCHING_FRAME_DURATION_FOR_HIGH_CONFIDENCE_NS;
float minimumChangeForUpdate =
candidateIsHighConfidence
? MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_HIGH_CONFIDENCE
: MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_LOW_CONFIDENCE;
shouldUpdate = Math.abs(candidateFrameRate - surfaceMediaFrameRate) >= minimumChangeForUpdate;
} else if (candidateFrameRate != Format.NO_VALUE) {
shouldUpdate = true;
} else {
shouldUpdate =
frameRateEstimator.getFramesWithoutSyncCount()
>= MINIMUM_FRAMES_WITHOUT_SYNC_TO_CLEAR_SURFACE_FRAME_RATE;
}
if (shouldUpdate) {
surfaceMediaFrameRate = candidateFrameRate;
updateSurfacePlaybackFrameRate(/* forceUpdate= */ false);
}
}
/**
* Updates the playback frame rate of the current {@link #surface} based on the playback speed,
* frame rate of the content, and whether the renderer is started.
*
* <p>Does nothing if {@link #changeFrameRateStrategy} is {@link
* C#VIDEO_CHANGE_FRAME_RATE_STRATEGY_OFF}.
*
* @param forceUpdate Whether to call {@link Surface#setFrameRate} even if the frame rate is
* unchanged.
*/
private void updateSurfacePlaybackFrameRate(boolean forceUpdate) {
if (Util.SDK_INT < 30
|| surface == null
|| changeFrameRateStrategy == C.VIDEO_CHANGE_FRAME_RATE_STRATEGY_OFF) {
return;
}
float surfacePlaybackFrameRate = 0;
if (started && surfaceMediaFrameRate != Format.NO_VALUE) {
surfacePlaybackFrameRate = surfaceMediaFrameRate * playbackSpeed;
}
// We always set the frame-rate if we have a new surface, since we have no way of knowing what
// it might have been set to previously.
if (!forceUpdate && this.surfacePlaybackFrameRate == surfacePlaybackFrameRate) {
return;
}
this.surfacePlaybackFrameRate = surfacePlaybackFrameRate;
Api30.setSurfaceFrameRate(surface, surfacePlaybackFrameRate);
}
/**
* Clears the frame-rate of the current {@link #surface}.
*
* <p>Does nothing if {@link #changeFrameRateStrategy} is {@link
* C#VIDEO_CHANGE_FRAME_RATE_STRATEGY_OFF}.
*/
private void clearSurfaceFrameRate() {
if (Util.SDK_INT < 30
|| surface == null
|| changeFrameRateStrategy == C.VIDEO_CHANGE_FRAME_RATE_STRATEGY_OFF
|| surfacePlaybackFrameRate == 0) {
return;
}
surfacePlaybackFrameRate = 0;
Api30.setSurfaceFrameRate(surface, /* frameRate= */ 0);
}
// Display refresh rate and vsync logic.
private void updateDefaultDisplayRefreshRateParams(@Nullable Display defaultDisplay) {
if (defaultDisplay != null) {
double defaultDisplayRefreshRate = defaultDisplay.getRefreshRate();
vsyncDurationNs = (long) (C.NANOS_PER_SECOND / defaultDisplayRefreshRate);
vsyncOffsetNs = (vsyncDurationNs * VSYNC_OFFSET_PERCENTAGE) / 100;
} else {
Log.w(TAG, "Unable to query display refresh rate");
vsyncDurationNs = C.TIME_UNSET;
vsyncOffsetNs = C.TIME_UNSET;
}
}
private static long closestVsync(long releaseTime, long sampledVsyncTime, long vsyncDuration) {
long vsyncCount = (releaseTime - sampledVsyncTime) / vsyncDuration;
long snappedTimeNs = sampledVsyncTime + (vsyncDuration * vsyncCount);
long snappedBeforeNs;
long snappedAfterNs;
if (releaseTime <= snappedTimeNs) {
snappedBeforeNs = snappedTimeNs - vsyncDuration;
snappedAfterNs = snappedTimeNs;
} else {
snappedBeforeNs = snappedTimeNs;
snappedAfterNs = snappedTimeNs + vsyncDuration;
}
long snappedAfterDiff = snappedAfterNs - releaseTime;
long snappedBeforeDiff = releaseTime - snappedBeforeNs;
return snappedAfterDiff < snappedBeforeDiff ? snappedAfterNs : snappedBeforeNs;
}
@Nullable
private static DisplayHelper maybeBuildDisplayHelper(@Nullable Context context) {
@Nullable DisplayHelper displayHelper = null;
if (context != null) {
context = context.getApplicationContext();
if (Util.SDK_INT >= 17) {
displayHelper = DisplayHelperV17.maybeBuildNewInstance(context);
}
if (displayHelper == null) {
displayHelper = DisplayHelperV16.maybeBuildNewInstance(context);
}
}
return displayHelper;
}
// Nested classes.
@RequiresApi(30)
private static final class Api30 {
@DoNotInline
public static void setSurfaceFrameRate(Surface surface, float frameRate) {
int compatibility =
frameRate == 0
? Surface.FRAME_RATE_COMPATIBILITY_DEFAULT
: Surface.FRAME_RATE_COMPATIBILITY_FIXED_SOURCE;
try {
surface.setFrameRate(frameRate, compatibility);
} catch (IllegalStateException e) {
Log.e(TAG, "Failed to call Surface.setFrameRate", e);
}
}
}
/** Helper for listening to changes to the default display. */
private interface DisplayHelper {
/** Listener for changes to the default display. */
interface Listener {
/**
* Called when the default display changes.
*
* @param defaultDisplay The default display, or {@code null} if a corresponding {@link
* Display} object could not be obtained.
*/
void onDefaultDisplayChanged(@Nullable Display defaultDisplay);
}
/**
* Enables the helper, invoking {@link Listener#onDefaultDisplayChanged(Display)} to pass the
* initial default display.
*/
void register(Listener listener);
/** Disables the helper. */
void unregister();
}
private static final class DisplayHelperV16 implements DisplayHelper {
@Nullable
public static DisplayHelper maybeBuildNewInstance(Context context) {
WindowManager windowManager =
(WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
return windowManager != null ? new DisplayHelperV16(windowManager) : null;
}
private final WindowManager windowManager;
private DisplayHelperV16(WindowManager windowManager) {
this.windowManager = windowManager;
}
@Override
public void register(Listener listener) {
listener.onDefaultDisplayChanged(windowManager.getDefaultDisplay());
}
@Override
public void unregister() {
// Do nothing.
}
}
@RequiresApi(17)
private static final class DisplayHelperV17
implements DisplayHelper, DisplayManager.DisplayListener {
@Nullable
public static DisplayHelper maybeBuildNewInstance(Context context) {
DisplayManager displayManager =
(DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
return displayManager != null ? new DisplayHelperV17(displayManager) : null;
}
private final DisplayManager displayManager;
@Nullable private Listener listener;
private DisplayHelperV17(DisplayManager displayManager) {
this.displayManager = displayManager;
}
@Override
public void register(Listener listener) {
this.listener = listener;
displayManager.registerDisplayListener(this, Util.createHandlerForCurrentLooper());
listener.onDefaultDisplayChanged(getDefaultDisplay());
}
@Override
public void unregister() {
displayManager.unregisterDisplayListener(this);
listener = null;
}
@Override
public void onDisplayChanged(int displayId) {
if (listener != null && displayId == Display.DEFAULT_DISPLAY) {
listener.onDefaultDisplayChanged(getDefaultDisplay());
}
}
@Override
public void onDisplayAdded(int displayId) {
// Do nothing.
}
@Override
public void onDisplayRemoved(int displayId) {
// Do nothing.
}
private Display getDefaultDisplay() {
return displayManager.getDisplay(Display.DEFAULT_DISPLAY);
}
}
/**
* Samples display vsync timestamps. A single instance using a single {@link Choreographer} is
* shared by all {@link VideoFrameReleaseHelper} instances. This is done to avoid a resource leak
* in the platform on API levels prior to 23. See [Internal: b/12455729].
*/
private static final class VSyncSampler implements FrameCallback, Handler.Callback {
public volatile long sampledVsyncTimeNs;
private static final int CREATE_CHOREOGRAPHER = 0;
private static final int MSG_ADD_OBSERVER = 1;
private static final int MSG_REMOVE_OBSERVER = 2;
private static final VSyncSampler INSTANCE = new VSyncSampler();
private final Handler handler;
private final HandlerThread choreographerOwnerThread;
private @MonotonicNonNull Choreographer choreographer;
private int observerCount;
public static VSyncSampler getInstance() {
return INSTANCE;
}
private VSyncSampler() {
sampledVsyncTimeNs = C.TIME_UNSET;
choreographerOwnerThread = new HandlerThread("ExoPlayer:FrameReleaseChoreographer");
choreographerOwnerThread.start();
handler = Util.createHandler(choreographerOwnerThread.getLooper(), /* callback= */ this);
handler.sendEmptyMessage(CREATE_CHOREOGRAPHER);
}
/**
* Notifies the sampler that a {@link VideoFrameReleaseHelper} is observing {@link
* #sampledVsyncTimeNs}, and hence that the value should be periodically updated.
*/
public void addObserver() {
handler.sendEmptyMessage(MSG_ADD_OBSERVER);
}
/**
* Notifies the sampler that a {@link VideoFrameReleaseHelper} is no longer observing {@link
* #sampledVsyncTimeNs}.
*/
public void removeObserver() {
handler.sendEmptyMessage(MSG_REMOVE_OBSERVER);
}
@Override
public void doFrame(long vsyncTimeNs) {
sampledVsyncTimeNs = vsyncTimeNs;
checkNotNull(choreographer).postFrameCallbackDelayed(this, VSYNC_SAMPLE_UPDATE_PERIOD_MS);
}
@Override
public boolean handleMessage(Message message) {
switch (message.what) {
case CREATE_CHOREOGRAPHER:
createChoreographerInstanceInternal();
return true;
case MSG_ADD_OBSERVER:
addObserverInternal();
return true;
case MSG_REMOVE_OBSERVER:
removeObserverInternal();
return true;
default:
return false;
}
}
private void createChoreographerInstanceInternal() {
try {
choreographer = Choreographer.getInstance();
} catch (RuntimeException e) {
// See [Internal: b/213926330].
Log.w(TAG, "Vsync sampling disabled due to platform error", e);
}
}
private void addObserverInternal() {
if (choreographer != null) {
observerCount++;
if (observerCount == 1) {
choreographer.postFrameCallback(this);
}
}
}
private void removeObserverInternal() {
if (choreographer != null) {
observerCount--;
if (observerCount == 0) {
choreographer.removeFrameCallback(this);
sampledVsyncTimeNs = C.TIME_UNSET;
}
}
}
}
@RequiresApi(17)
private static final class Api17 {
@DoNotInline
public static boolean isPlaceholderSurface(@Nullable Surface surface) {
return surface instanceof PlaceholderSurface;
}
}
}