using System; using UnityEngine; using UnityEngine.UI; using UnityEngine.Video; namespace InteractiveVideo { /// How the video is fitted into the RawImage rectangle. public enum VideoFitMode { /// Stretch to the RawImage rect (use this when an AspectRatioFitter already shapes the rect). Stretch, /// Keep the video aspect, letterbox/pillarbox inside the rect (bars are drawn by the highlight shader). FitInside, /// Keep the video aspect, fill the rect and crop the overflow. FitOutside, } /// /// Plays the MP4 into a RenderTexture shown by a RawImage, and drives the tracking manager with the current /// VideoPlayer.frame (polled in Update, optionally also via frameReady, always on seekCompleted). /// /// It is also the one authority on where the video pixels sit inside the RawImage rect /// (), so the click handler, the debug overlay and the shader agree on /// letterboxing. Screen-to-video mapping helpers live here for that reason. /// [DisallowMultipleComponent] [AddComponentMenu("Interactive Video/Interactive Video Player")] public sealed class InteractiveVideoPlayer : MonoBehaviour { [Header("Components")] [SerializeField] private VideoPlayer videoPlayer; [SerializeField] private RawImage videoDisplay; [Tooltip("Optional. When empty, a RenderTexture matching the video size is created on prepare and reused.")] [SerializeField] private RenderTexture videoTexture; [SerializeField] private VideoTrackingManager trackingManager; [SerializeField] private VideoHighlightController highlightController; [Header("Playback")] [SerializeField] private bool playOnStart = true; [Tooltip("Also react to VideoPlayer.frameReady callbacks (Update polling stays on as the fallback).")] [SerializeField] private bool useFrameReadyEvents; [Header("Display")] [SerializeField] private VideoFitMode fitMode = VideoFitMode.FitInside; [SerializeField] private Color letterboxColor = Color.black; /// Raised once the VideoPlayer has prepared and the RenderTexture is wired. public event Action Prepared; /// Raised when the processed video frame changes (also on seek). Argument: frame, or -1. public event Action FrameChanged; /// Raised when changes (resize, new video). public event Action DisplayRectChanged; public event Action PlaybackError; public VideoPlayer Player => videoPlayer; public RawImage Display => videoDisplay; public RenderTexture VideoTexture => videoTexture; public VideoTrackingManager TrackingManager => trackingManager; public VideoHighlightController HighlightController => highlightController; public bool IsPrepared => videoPlayer != null && videoPlayer.isPrepared; public bool IsPlaying => videoPlayer != null && videoPlayer.isPlaying; public bool IsPaused => videoPlayer != null && videoPlayer.isPaused; /// Last processed frame, -1 when none. public long CurrentFrame { get; private set; } = -1; public long FrameCount => IsPrepared ? (long)videoPlayer.frameCount : 0; public float FrameRate => IsPrepared ? videoPlayer.frameRate : 0f; public double Duration => IsPrepared ? videoPlayer.length : 0d; /// /// Where the video pixels are inside the RawImage rect, normalised to the rect (0..1, Y up). /// (0,0,1,1) = the whole rect; FitInside gives bars; FitOutside gives a rect larger than the unit square. /// public Rect DisplayedVideoRect { get; private set; } = new Rect(0, 0, 1, 1); public VideoFitMode FitMode { get => fitMode; set { fitMode = value; UpdateDisplayRect(true); } } private bool _ownsTexture; private bool _playWhenPrepared; private Vector2 _lastRectSize = new Vector2(-1, -1); private uint _lastVideoWidth, _lastVideoHeight; private bool _warnedNoRawImage; // ------------------------------------------------------------------ lifecycle private void Reset() { videoPlayer = GetComponent(); trackingManager = GetComponent(); highlightController = GetComponent(); } private void Awake() { if (videoPlayer == null) videoPlayer = GetComponent(); if (videoPlayer == null) { Debug.LogError("[InteractiveVideo] InteractiveVideoPlayer needs a VideoPlayer reference.", this); enabled = false; return; } videoPlayer.playOnAwake = false; videoPlayer.renderMode = VideoRenderMode.RenderTexture; // The RenderTexture matches the video aspect; letterboxing is done in the shader from DisplayedVideoRect, // so the RT must be filled edge to edge to keep UV == normalised video coordinates. videoPlayer.aspectRatio = VideoAspectRatio.Stretch; videoPlayer.waitForFirstFrame = true; videoPlayer.skipOnDrop = true; videoPlayer.sendFrameReadyEvents = useFrameReadyEvents; videoPlayer.prepareCompleted += OnPrepareCompleted; videoPlayer.seekCompleted += OnSeekCompleted; videoPlayer.frameReady += OnFrameReady; videoPlayer.errorReceived += OnErrorReceived; videoPlayer.loopPointReached += OnLoopPointReached; if (videoTexture != null) ApplyTexture(videoTexture); } private void Start() { bool hasSource = videoPlayer.source == VideoSource.VideoClip ? videoPlayer.clip != null : !string.IsNullOrEmpty(videoPlayer.url); if (hasSource) { _playWhenPrepared = playOnStart; if (!videoPlayer.isPrepared) videoPlayer.Prepare(); else OnPrepareCompleted(videoPlayer); } UpdateDisplayRect(true); } private void OnDestroy() { if (videoPlayer != null) { videoPlayer.prepareCompleted -= OnPrepareCompleted; videoPlayer.seekCompleted -= OnSeekCompleted; videoPlayer.frameReady -= OnFrameReady; videoPlayer.errorReceived -= OnErrorReceived; videoPlayer.loopPointReached -= OnLoopPointReached; } if (_ownsTexture && videoTexture != null) { videoTexture.Release(); Destroy(videoTexture); videoTexture = null; } } private void Update() { UpdateDisplayRect(false); if (!IsPrepared) { if (CurrentFrame != -1) ProcessFrame(-1, false); return; } // Polling VideoPlayer.frame is enough for sync; dedupe so nothing runs while the frame is unchanged // (paused video, or a 60 Hz Update over a 30 fps video). long frame = videoPlayer.frame; if (frame == CurrentFrame) return; ProcessFrame(frame, false); } // ------------------------------------------------------------------ public API /// Loads a clip asset. Tracking JSON is loaded separately through the tracking manager. public void Load(VideoClip clip, bool play = true) { if (videoPlayer == null) return; videoPlayer.Stop(); videoPlayer.source = VideoSource.VideoClip; videoPlayer.clip = clip; BeginPrepare(play); } /// Loads from a URL or file path (StreamingAssets, persistentDataPath, http...). public void Load(string url, bool play = true) { if (videoPlayer == null) return; videoPlayer.Stop(); videoPlayer.source = VideoSource.Url; videoPlayer.url = url; BeginPrepare(play); } /// Loads a video and its tracking JSON together. public void Load(VideoClip clip, TextAsset tracking, bool play = true) { trackingManager?.LoadTracking(tracking); Load(clip, play); } public void Load(string url, string trackingJson, bool play = true) { trackingManager?.LoadTrackingJson(trackingJson, url); Load(url, play); } public void Play() { if (videoPlayer == null) return; if (!videoPlayer.isPrepared) { _playWhenPrepared = true; if (!videoPlayer.isPlaying) videoPlayer.Prepare(); return; } videoPlayer.Play(); } public void Pause() { if (videoPlayer != null) videoPlayer.Pause(); } public void TogglePlayPause() { if (IsPlaying) Pause(); else Play(); } public void Stop() { if (videoPlayer == null) return; videoPlayer.Stop(); ProcessFrame(-1, true); } /// Seeks to a frame. Tracking updates as soon as the VideoPlayer reports seekCompleted. public void SeekToFrame(long frame) { if (!IsPrepared) return; long max = Math.Max(0, FrameCount - 1); videoPlayer.frame = Math.Clamp(frame, 0, max); } public void SeekToTime(double seconds) { if (!IsPrepared) return; videoPlayer.time = Math.Clamp(seconds, 0d, Math.Max(0d, videoPlayer.length)); } /// Seeks by normalised position 0..1 (for scrub bars). public void SeekNormalized(float t) { if (!IsPrepared) return; SeekToFrame((long)Math.Round(Mathf.Clamp01(t) * Math.Max(0, FrameCount - 1))); } // ------------------------------------------------------------------ coordinate mapping (RawImage <-> video) /// /// RawImage local point (from RectTransformUtility / InverseTransformPoint) to video UV (0..1, Y up). /// False when the point lies in a letterbox bar or outside the rect: nothing there can be clicked. /// public bool TryLocalPointToVideoUv(Vector2 localPoint, out Vector2 uv) { uv = default; if (videoDisplay == null) return false; Rect rect = videoDisplay.rectTransform.rect; if (rect.width <= 0f || rect.height <= 0f) return false; Vector2 n = new Vector2((localPoint.x - rect.xMin) / rect.width, (localPoint.y - rect.yMin) / rect.height); Rect d = DisplayedVideoRect; if (d.width <= 0f || d.height <= 0f) return false; uv = new Vector2((n.x - d.xMin) / d.width, (n.y - d.yMin) / d.height); return VideoCoordinates.IsInsideUnitSquare(uv) && VideoCoordinates.IsInsideUnitSquare(n); } /// Screen point (mouse/touch) to video UV. Camera may be null for Screen Space Overlay canvases. public bool TryScreenPointToVideoUv(Vector2 screenPoint, Camera eventCamera, out Vector2 uv) { uv = default; if (videoDisplay == null) return false; if (!RectTransformUtility.ScreenPointToLocalPointInRectangle(videoDisplay.rectTransform, screenPoint, eventCamera, out Vector2 local)) return false; return TryLocalPointToVideoUv(local, out uv); } /// Screen point to TRACKING coordinates (0..1, Y down). This is what hit tests take. public bool TryScreenPointToTracking(Vector2 screenPoint, Camera eventCamera, out Vector2 tracking) { if (TryScreenPointToVideoUv(screenPoint, eventCamera, out Vector2 uv)) { tracking = VideoCoordinates.UvToTracking(uv); return true; } tracking = default; return false; } public bool TryLocalPointToTracking(Vector2 localPoint, out Vector2 tracking) { if (TryLocalPointToVideoUv(localPoint, out Vector2 uv)) { tracking = VideoCoordinates.UvToTracking(uv); return true; } tracking = default; return false; } /// Video UV to a RawImage local point (for overlays). public Vector2 VideoUvToLocalPoint(Vector2 uv) { if (videoDisplay == null) return Vector2.zero; Rect rect = videoDisplay.rectTransform.rect; Rect d = DisplayedVideoRect; Vector2 n = new Vector2(d.xMin + uv.x * d.width, d.yMin + uv.y * d.height); return new Vector2(rect.xMin + n.x * rect.width, rect.yMin + n.y * rect.height); } public Vector2 TrackingToLocalPoint(Vector2 tracking) => VideoUvToLocalPoint(VideoCoordinates.TrackingToUv(tracking)); // ------------------------------------------------------------------ internals private void BeginPrepare(bool play) { _playWhenPrepared = play; ProcessFrame(-1, true); videoPlayer.Prepare(); } private void OnPrepareCompleted(VideoPlayer source) { EnsureRenderTexture(); UpdateDisplayRect(true); Prepared?.Invoke(); if (_playWhenPrepared) { _playWhenPrepared = false; videoPlayer.Play(); } else { // Show the first frame and align tracking even when not auto-playing. ProcessFrame(videoPlayer.frame, true); } } private void OnSeekCompleted(VideoPlayer source) { // Users scrub backwards and forwards: never assume frames arrive in order. ProcessFrame(videoPlayer.frame, true); } private void OnFrameReady(VideoPlayer source, long frameIdx) { if (useFrameReadyEvents) ProcessFrame(frameIdx, false); } private void OnLoopPointReached(VideoPlayer source) { if (!source.isLooping) ProcessFrame(videoPlayer.frame, true); } private void OnErrorReceived(VideoPlayer source, string message) { Debug.LogError($"[InteractiveVideo] VideoPlayer error: {message}", this); PlaybackError?.Invoke(message); } private void ProcessFrame(long frame, bool force) { if (!force && frame == CurrentFrame) return; CurrentFrame = frame; if (trackingManager != null) { if (force) { trackingManager.SetCurrentFrame(frame); trackingManager.Refresh(); } else trackingManager.SetCurrentFrame(frame); } FrameChanged?.Invoke(frame); } private void EnsureRenderTexture() { int w = (int)videoPlayer.width, h = (int)videoPlayer.height; if (w <= 0 || h <= 0) return; bool needNew = videoTexture == null || (_ownsTexture && (videoTexture.width != w || videoTexture.height != h)); if (needNew) { if (_ownsTexture && videoTexture != null) { videoTexture.Release(); Destroy(videoTexture); } videoTexture = new RenderTexture(w, h, 0, RenderTextureFormat.ARGB32) { name = "InteractiveVideo_Video", useMipMap = false, autoGenerateMips = false, wrapMode = TextureWrapMode.Clamp, filterMode = FilterMode.Bilinear, }; videoTexture.Create(); _ownsTexture = true; // A new RenderTexture holds whatever memory it was given until the first frame arrives; clear it // so the RawImage shows black (not garbage) between Prepare and the first decoded frame. var previous = RenderTexture.active; RenderTexture.active = videoTexture; GL.Clear(true, true, Color.black); RenderTexture.active = previous; } ApplyTexture(videoTexture); } private void ApplyTexture(RenderTexture rt) { videoPlayer.targetTexture = rt; if (videoDisplay != null) videoDisplay.texture = rt; else if (!_warnedNoRawImage) { _warnedNoRawImage = true; Debug.LogWarning("[InteractiveVideo] No RawImage assigned; the video renders into the RenderTexture only.", this); } } private void UpdateDisplayRect(bool force) { if (videoDisplay == null) return; Vector2 size = videoDisplay.rectTransform.rect.size; uint vw = IsPrepared ? videoPlayer.width : 0, vh = IsPrepared ? videoPlayer.height : 0; if (!force && size == _lastRectSize && vw == _lastVideoWidth && vh == _lastVideoHeight) return; _lastRectSize = size; _lastVideoWidth = vw; _lastVideoHeight = vh; Rect r = new Rect(0, 0, 1, 1); if (fitMode != VideoFitMode.Stretch && vw > 0 && vh > 0 && size.x > 0f && size.y > 0f) { float videoAspect = (float)vw / vh; float rectAspect = size.x / size.y; bool rectWider = rectAspect > videoAspect; if (fitMode == VideoFitMode.FitOutside) rectWider = !rectWider; if (rectWider) { float w = videoAspect / rectAspect; // < 1 for FitInside (pillarbox), > 1 for FitOutside r = new Rect((1f - w) * 0.5f, 0f, w, 1f); } else { float h = rectAspect / videoAspect; r = new Rect(0f, (1f - h) * 0.5f, 1f, h); } } if (r != DisplayedVideoRect || force) { DisplayedVideoRect = r; highlightController?.SetDisplayRect(r, letterboxColor); DisplayRectChanged?.Invoke(r); } } } }