using System;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.Video;
namespace InteractiveVideo
{
/// How the video is fitted into the RawImage rectangle.
public enum VideoFitMode
{
/// Stretch to the RawImage rect (use this when an AspectRatioFitter already shapes the rect).
Stretch,
/// Keep the video aspect, letterbox/pillarbox inside the rect (bars are drawn by the highlight shader).
FitInside,
/// Keep the video aspect, fill the rect and crop the overflow.
FitOutside,
}
///
/// Plays the MP4 into a RenderTexture shown by a RawImage, and drives the tracking manager with the current
/// VideoPlayer.frame (polled in Update, optionally also via frameReady, always on seekCompleted).
///
/// It is also the one authority on where the video pixels sit inside the RawImage rect
/// (), so the click handler, the debug overlay and the shader agree on
/// letterboxing. Screen-to-video mapping helpers live here for that reason.
///
[DisallowMultipleComponent]
[AddComponentMenu("Interactive Video/Interactive Video Player")]
public sealed class InteractiveVideoPlayer : MonoBehaviour
{
[Header("Components")]
[SerializeField] private VideoPlayer videoPlayer;
[SerializeField] private RawImage videoDisplay;
[Tooltip("Optional. When empty, a RenderTexture matching the video size is created on prepare and reused.")]
[SerializeField] private RenderTexture videoTexture;
[SerializeField] private VideoTrackingManager trackingManager;
[SerializeField] private VideoHighlightController highlightController;
[Header("Playback")]
[SerializeField] private bool playOnStart = true;
[Tooltip("Also react to VideoPlayer.frameReady callbacks (Update polling stays on as the fallback).")]
[SerializeField] private bool useFrameReadyEvents;
[Header("Display")]
[SerializeField] private VideoFitMode fitMode = VideoFitMode.FitInside;
[SerializeField] private Color letterboxColor = Color.black;
/// Raised once the VideoPlayer has prepared and the RenderTexture is wired.
public event Action Prepared;
/// Raised when the processed video frame changes (also on seek). Argument: frame, or -1.
public event Action FrameChanged;
/// Raised when changes (resize, new video).
public event Action DisplayRectChanged;
public event Action PlaybackError;
public VideoPlayer Player => videoPlayer;
public RawImage Display => videoDisplay;
public RenderTexture VideoTexture => videoTexture;
public VideoTrackingManager TrackingManager => trackingManager;
public VideoHighlightController HighlightController => highlightController;
public bool IsPrepared => videoPlayer != null && videoPlayer.isPrepared;
public bool IsPlaying => videoPlayer != null && videoPlayer.isPlaying;
public bool IsPaused => videoPlayer != null && videoPlayer.isPaused;
/// Last processed frame, -1 when none.
public long CurrentFrame { get; private set; } = -1;
public long FrameCount => IsPrepared ? (long)videoPlayer.frameCount : 0;
public float FrameRate => IsPrepared ? videoPlayer.frameRate : 0f;
public double Duration => IsPrepared ? videoPlayer.length : 0d;
///
/// Where the video pixels are inside the RawImage rect, normalised to the rect (0..1, Y up).
/// (0,0,1,1) = the whole rect; FitInside gives bars; FitOutside gives a rect larger than the unit square.
///
public Rect DisplayedVideoRect { get; private set; } = new Rect(0, 0, 1, 1);
public VideoFitMode FitMode { get => fitMode; set { fitMode = value; UpdateDisplayRect(true); } }
private bool _ownsTexture;
private bool _playWhenPrepared;
private Vector2 _lastRectSize = new Vector2(-1, -1);
private uint _lastVideoWidth, _lastVideoHeight;
private bool _warnedNoRawImage;
// ------------------------------------------------------------------ lifecycle
private void Reset()
{
videoPlayer = GetComponent();
trackingManager = GetComponent();
highlightController = GetComponent();
}
private void Awake()
{
if (videoPlayer == null) videoPlayer = GetComponent();
if (videoPlayer == null)
{
Debug.LogError("[InteractiveVideo] InteractiveVideoPlayer needs a VideoPlayer reference.", this);
enabled = false;
return;
}
videoPlayer.playOnAwake = false;
videoPlayer.renderMode = VideoRenderMode.RenderTexture;
// The RenderTexture matches the video aspect; letterboxing is done in the shader from DisplayedVideoRect,
// so the RT must be filled edge to edge to keep UV == normalised video coordinates.
videoPlayer.aspectRatio = VideoAspectRatio.Stretch;
videoPlayer.waitForFirstFrame = true;
videoPlayer.skipOnDrop = true;
videoPlayer.sendFrameReadyEvents = useFrameReadyEvents;
videoPlayer.prepareCompleted += OnPrepareCompleted;
videoPlayer.seekCompleted += OnSeekCompleted;
videoPlayer.frameReady += OnFrameReady;
videoPlayer.errorReceived += OnErrorReceived;
videoPlayer.loopPointReached += OnLoopPointReached;
if (videoTexture != null) ApplyTexture(videoTexture);
}
private void Start()
{
bool hasSource = videoPlayer.source == VideoSource.VideoClip ? videoPlayer.clip != null : !string.IsNullOrEmpty(videoPlayer.url);
if (hasSource)
{
_playWhenPrepared = playOnStart;
if (!videoPlayer.isPrepared) videoPlayer.Prepare();
else OnPrepareCompleted(videoPlayer);
}
UpdateDisplayRect(true);
}
private void OnDestroy()
{
if (videoPlayer != null)
{
videoPlayer.prepareCompleted -= OnPrepareCompleted;
videoPlayer.seekCompleted -= OnSeekCompleted;
videoPlayer.frameReady -= OnFrameReady;
videoPlayer.errorReceived -= OnErrorReceived;
videoPlayer.loopPointReached -= OnLoopPointReached;
}
if (_ownsTexture && videoTexture != null)
{
videoTexture.Release();
Destroy(videoTexture);
videoTexture = null;
}
}
private void Update()
{
UpdateDisplayRect(false);
if (!IsPrepared)
{
if (CurrentFrame != -1) ProcessFrame(-1, false);
return;
}
// Polling VideoPlayer.frame is enough for sync; dedupe so nothing runs while the frame is unchanged
// (paused video, or a 60 Hz Update over a 30 fps video).
long frame = videoPlayer.frame;
if (frame == CurrentFrame) return;
ProcessFrame(frame, false);
}
// ------------------------------------------------------------------ public API
/// Loads a clip asset. Tracking JSON is loaded separately through the tracking manager.
public void Load(VideoClip clip, bool play = true)
{
if (videoPlayer == null) return;
videoPlayer.Stop();
videoPlayer.source = VideoSource.VideoClip;
videoPlayer.clip = clip;
BeginPrepare(play);
}
/// Loads from a URL or file path (StreamingAssets, persistentDataPath, http...).
public void Load(string url, bool play = true)
{
if (videoPlayer == null) return;
videoPlayer.Stop();
videoPlayer.source = VideoSource.Url;
videoPlayer.url = url;
BeginPrepare(play);
}
/// Loads a video and its tracking JSON together.
public void Load(VideoClip clip, TextAsset tracking, bool play = true)
{
trackingManager?.LoadTracking(tracking);
Load(clip, play);
}
public void Load(string url, string trackingJson, bool play = true)
{
trackingManager?.LoadTrackingJson(trackingJson, url);
Load(url, play);
}
public void Play()
{
if (videoPlayer == null) return;
if (!videoPlayer.isPrepared) { _playWhenPrepared = true; if (!videoPlayer.isPlaying) videoPlayer.Prepare(); return; }
videoPlayer.Play();
}
public void Pause() { if (videoPlayer != null) videoPlayer.Pause(); }
public void TogglePlayPause() { if (IsPlaying) Pause(); else Play(); }
public void Stop()
{
if (videoPlayer == null) return;
videoPlayer.Stop();
ProcessFrame(-1, true);
}
/// Seeks to a frame. Tracking updates as soon as the VideoPlayer reports seekCompleted.
public void SeekToFrame(long frame)
{
if (!IsPrepared) return;
long max = Math.Max(0, FrameCount - 1);
videoPlayer.frame = Math.Clamp(frame, 0, max);
}
public void SeekToTime(double seconds)
{
if (!IsPrepared) return;
videoPlayer.time = Math.Clamp(seconds, 0d, Math.Max(0d, videoPlayer.length));
}
/// Seeks by normalised position 0..1 (for scrub bars).
public void SeekNormalized(float t)
{
if (!IsPrepared) return;
SeekToFrame((long)Math.Round(Mathf.Clamp01(t) * Math.Max(0, FrameCount - 1)));
}
// ------------------------------------------------------------------ coordinate mapping (RawImage <-> video)
///
/// RawImage local point (from RectTransformUtility / InverseTransformPoint) to video UV (0..1, Y up).
/// False when the point lies in a letterbox bar or outside the rect: nothing there can be clicked.
///
public bool TryLocalPointToVideoUv(Vector2 localPoint, out Vector2 uv)
{
uv = default;
if (videoDisplay == null) return false;
Rect rect = videoDisplay.rectTransform.rect;
if (rect.width <= 0f || rect.height <= 0f) return false;
Vector2 n = new Vector2((localPoint.x - rect.xMin) / rect.width, (localPoint.y - rect.yMin) / rect.height);
Rect d = DisplayedVideoRect;
if (d.width <= 0f || d.height <= 0f) return false;
uv = new Vector2((n.x - d.xMin) / d.width, (n.y - d.yMin) / d.height);
return VideoCoordinates.IsInsideUnitSquare(uv) && VideoCoordinates.IsInsideUnitSquare(n);
}
/// Screen point (mouse/touch) to video UV. Camera may be null for Screen Space Overlay canvases.
public bool TryScreenPointToVideoUv(Vector2 screenPoint, Camera eventCamera, out Vector2 uv)
{
uv = default;
if (videoDisplay == null) return false;
if (!RectTransformUtility.ScreenPointToLocalPointInRectangle(videoDisplay.rectTransform, screenPoint, eventCamera, out Vector2 local))
return false;
return TryLocalPointToVideoUv(local, out uv);
}
/// Screen point to TRACKING coordinates (0..1, Y down). This is what hit tests take.
public bool TryScreenPointToTracking(Vector2 screenPoint, Camera eventCamera, out Vector2 tracking)
{
if (TryScreenPointToVideoUv(screenPoint, eventCamera, out Vector2 uv))
{
tracking = VideoCoordinates.UvToTracking(uv);
return true;
}
tracking = default;
return false;
}
public bool TryLocalPointToTracking(Vector2 localPoint, out Vector2 tracking)
{
if (TryLocalPointToVideoUv(localPoint, out Vector2 uv))
{
tracking = VideoCoordinates.UvToTracking(uv);
return true;
}
tracking = default;
return false;
}
/// Video UV to a RawImage local point (for overlays).
public Vector2 VideoUvToLocalPoint(Vector2 uv)
{
if (videoDisplay == null) return Vector2.zero;
Rect rect = videoDisplay.rectTransform.rect;
Rect d = DisplayedVideoRect;
Vector2 n = new Vector2(d.xMin + uv.x * d.width, d.yMin + uv.y * d.height);
return new Vector2(rect.xMin + n.x * rect.width, rect.yMin + n.y * rect.height);
}
public Vector2 TrackingToLocalPoint(Vector2 tracking) => VideoUvToLocalPoint(VideoCoordinates.TrackingToUv(tracking));
// ------------------------------------------------------------------ internals
private void BeginPrepare(bool play)
{
_playWhenPrepared = play;
ProcessFrame(-1, true);
videoPlayer.Prepare();
}
private void OnPrepareCompleted(VideoPlayer source)
{
EnsureRenderTexture();
UpdateDisplayRect(true);
Prepared?.Invoke();
if (_playWhenPrepared)
{
_playWhenPrepared = false;
videoPlayer.Play();
}
else
{
// Show the first frame and align tracking even when not auto-playing.
ProcessFrame(videoPlayer.frame, true);
}
}
private void OnSeekCompleted(VideoPlayer source)
{
// Users scrub backwards and forwards: never assume frames arrive in order.
ProcessFrame(videoPlayer.frame, true);
}
private void OnFrameReady(VideoPlayer source, long frameIdx)
{
if (useFrameReadyEvents) ProcessFrame(frameIdx, false);
}
private void OnLoopPointReached(VideoPlayer source)
{
if (!source.isLooping) ProcessFrame(videoPlayer.frame, true);
}
private void OnErrorReceived(VideoPlayer source, string message)
{
Debug.LogError($"[InteractiveVideo] VideoPlayer error: {message}", this);
PlaybackError?.Invoke(message);
}
private void ProcessFrame(long frame, bool force)
{
if (!force && frame == CurrentFrame) return;
CurrentFrame = frame;
if (trackingManager != null)
{
if (force) { trackingManager.SetCurrentFrame(frame); trackingManager.Refresh(); }
else trackingManager.SetCurrentFrame(frame);
}
FrameChanged?.Invoke(frame);
}
private void EnsureRenderTexture()
{
int w = (int)videoPlayer.width, h = (int)videoPlayer.height;
if (w <= 0 || h <= 0) return;
bool needNew = videoTexture == null || (_ownsTexture && (videoTexture.width != w || videoTexture.height != h));
if (needNew)
{
if (_ownsTexture && videoTexture != null)
{
videoTexture.Release();
Destroy(videoTexture);
}
videoTexture = new RenderTexture(w, h, 0, RenderTextureFormat.ARGB32)
{
name = "InteractiveVideo_Video",
useMipMap = false,
autoGenerateMips = false,
wrapMode = TextureWrapMode.Clamp,
filterMode = FilterMode.Bilinear,
};
videoTexture.Create();
_ownsTexture = true;
// A new RenderTexture holds whatever memory it was given until the first frame arrives; clear it
// so the RawImage shows black (not garbage) between Prepare and the first decoded frame.
var previous = RenderTexture.active;
RenderTexture.active = videoTexture;
GL.Clear(true, true, Color.black);
RenderTexture.active = previous;
}
ApplyTexture(videoTexture);
}
private void ApplyTexture(RenderTexture rt)
{
videoPlayer.targetTexture = rt;
if (videoDisplay != null) videoDisplay.texture = rt;
else if (!_warnedNoRawImage)
{
_warnedNoRawImage = true;
Debug.LogWarning("[InteractiveVideo] No RawImage assigned; the video renders into the RenderTexture only.", this);
}
}
private void UpdateDisplayRect(bool force)
{
if (videoDisplay == null) return;
Vector2 size = videoDisplay.rectTransform.rect.size;
uint vw = IsPrepared ? videoPlayer.width : 0, vh = IsPrepared ? videoPlayer.height : 0;
if (!force && size == _lastRectSize && vw == _lastVideoWidth && vh == _lastVideoHeight) return;
_lastRectSize = size;
_lastVideoWidth = vw;
_lastVideoHeight = vh;
Rect r = new Rect(0, 0, 1, 1);
if (fitMode != VideoFitMode.Stretch && vw > 0 && vh > 0 && size.x > 0f && size.y > 0f)
{
float videoAspect = (float)vw / vh;
float rectAspect = size.x / size.y;
bool rectWider = rectAspect > videoAspect;
if (fitMode == VideoFitMode.FitOutside) rectWider = !rectWider;
if (rectWider)
{
float w = videoAspect / rectAspect; // < 1 for FitInside (pillarbox), > 1 for FitOutside
r = new Rect((1f - w) * 0.5f, 0f, w, 1f);
}
else
{
float h = rectAspect / videoAspect;
r = new Rect(0f, (1f - h) * 0.5f, 1f, h);
}
}
if (r != DisplayedVideoRect || force)
{
DisplayedVideoRect = r;
highlightController?.SetDisplayRect(r, letterboxColor);
DisplayRectChanged?.Invoke(r);
}
}
}
}