GlShaderProgram.java
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.effect;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.UnstableApi;
import java.util.concurrent.Executor;
/**
* Processes frames from one OpenGL 2D texture to another.
*
* <p>The {@code GlShaderProgram} consumes input frames it accepts via {@link
* #queueInputFrame(GlTextureInfo, long)} and surrenders each texture back to the caller via its
* {@linkplain InputListener#onInputFrameProcessed(GlTextureInfo) listener} once the texture's
* contents have been processed.
*
* <p>The {@code GlShaderProgram} produces output frames asynchronously and notifies its owner when
* they are available via its {@linkplain OutputListener#onOutputFrameAvailable(GlTextureInfo, long)
* listener}. The {@code GlShaderProgram} instance's owner must surrender the texture back to the
* {@code GlShaderProgram} via {@link #releaseOutputFrame(GlTextureInfo)} when it has finished
* processing it.
*
* <p>{@code GlShaderProgram} implementations can choose to produce output frames before receiving
* input frames or process several input frames before producing an output frame. However, {@code
* GlShaderProgram} implementations cannot assume that they will receive more than one input frame
* at a time, so they must process each input frame they accept even if they cannot produce output
* yet.
*
* <p>The methods in this interface must be called on the thread that owns the parent OpenGL
* context. If the implementation uses another OpenGL context, e.g., on another thread, it must
* configure it to share data with the context of thread the interface methods are called on.
*/
@UnstableApi
public interface GlShaderProgram {
/**
* Listener for input-related video frame processing events.
*
* <p>This listener can be called from any thread.
*/
interface InputListener {
/**
* Called when the {@link GlShaderProgram} is ready to accept another input frame.
*
* <p>For each time this method is called, {@link #queueInputFrame(GlTextureInfo, long)} can be
* called once.
*/
default void onReadyToAcceptInputFrame() {}
/**
* Called when the {@link GlShaderProgram} has processed an input frame.
*
* <p>The implementation shall not assume the {@link GlShaderProgram} is {@linkplain
* #onReadyToAcceptInputFrame ready to accept another input frame} when this method is called.
*
* @param inputTexture The {@link GlTextureInfo} that was used to {@linkplain
* #queueInputFrame(GlTextureInfo, long) queue} the input frame.
*/
default void onInputFrameProcessed(GlTextureInfo inputTexture) {}
/**
* Called when the {@link GlShaderProgram} has been flushed.
*
* <p>The implementation shall not assume the {@link GlShaderProgram} is {@linkplain
* #onReadyToAcceptInputFrame ready to accept another input frame} when this method is called.
*/
default void onFlush() {}
}
/**
* Listener for output-related video frame processing events.
*
* <p>This listener can be called from any thread.
*/
interface OutputListener {
/**
* Called when the {@link GlShaderProgram} has produced an output frame.
*
* <p>After the listener's owner has processed the output frame, it must call {@link
* #releaseOutputFrame(GlTextureInfo)}. The output frame should be released as soon as possible,
* as there is no guarantee that the {@link GlShaderProgram} will produce further output frames
* before this output frame is released.
*
* @param outputTexture A {@link GlTextureInfo} describing the texture containing the output
* frame.
* @param presentationTimeUs The presentation timestamp of the output frame, in microseconds.
*/
default void onOutputFrameAvailable(GlTextureInfo outputTexture, long presentationTimeUs) {}
/**
* Called when the {@link GlShaderProgram} will not produce further output frames belonging to
* the current output stream. May be called multiple times for one output stream.
*/
default void onCurrentOutputStreamEnded() {}
}
/**
* Listener for video frame processing errors.
*
* <p>This listener can be called from any thread.
*/
interface ErrorListener {
/**
* Called when an exception occurs during asynchronous video frame processing.
*
* <p>If an error occurred, consuming and producing further frames will not work as expected and
* the {@link GlShaderProgram} should be released.
*/
void onError(VideoFrameProcessingException e);
}
/**
* Sets the {@link InputListener}.
*
* <p>The {@link InputListener} should be invoked on the thread that owns the parent OpenGL
* context. For example, {@link DefaultVideoFrameProcessor} invokes the {@link InputListener}
* methods on its internal thread.
*/
void setInputListener(InputListener inputListener);
/**
* Sets the {@link OutputListener}.
*
* <p>The {@link OutputListener} should be invoked on the thread that owns the parent OpenGL
* context. For example, {@link DefaultVideoFrameProcessor} invokes the {@link OutputListener}
* methods on its internal thread.
*/
void setOutputListener(OutputListener outputListener);
/**
* Sets the {@link ErrorListener}.
*
* <p>The {@link ErrorListener} is invoked on the provided {@link Executor}.
*/
void setErrorListener(Executor executor, ErrorListener errorListener);
/**
* Sets the {@link GlObjectsProvider}.
*
* <p>This method should not be called after any of the frame processing methods.
*/
void setGlObjectsProvider(GlObjectsProvider glObjectsProvider);
/**
* Processes an input frame if possible.
*
* <p>The {@code GlShaderProgram} owns the accepted frame until it calls {@link
* InputListener#onInputFrameProcessed(GlTextureInfo)}. The caller should not overwrite or release
* the texture before the {@code GlShaderProgram} has finished processing it.
*
* <p>This method must only be called when the {@code GlShaderProgram} can {@linkplain
* InputListener#onReadyToAcceptInputFrame() accept an input frame}.
*
* @param inputTexture A {@link GlTextureInfo} describing the texture containing the input frame.
* @param presentationTimeUs The presentation timestamp of the input frame, in microseconds.
*/
void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs);
/**
* Notifies the {@code GlShaderProgram} that the frame on the given output texture is no longer
* used and can be overwritten.
*/
void releaseOutputFrame(GlTextureInfo outputTexture);
/**
* Notifies the {@code GlShaderProgram} that no further input frames belonging to the current
* input stream will be queued.
*
* <p>Input frames that are queued after this method is called belong to a different input stream.
*/
void signalEndOfCurrentInputStream();
/**
* Flushes the {@code GlShaderProgram}.
*
* <p>The {@code GlShaderProgram} should reclaim the ownership of its allocated textures,
* {@linkplain InputListener#onFlush notify} its {@link InputListener} about the flush event, and
* {@linkplain InputListener#onReadyToAcceptInputFrame report its availability} if necessary.
*/
void flush();
/**
* Releases all resources.
*
* @throws VideoFrameProcessingException If an error occurs while releasing resources.
*/
void release() throws VideoFrameProcessingException;
}