using System; using System.Collections.Generic; using UnityEngine; namespace InteractiveVideo { /// /// Owns the loaded , resolves the current video frame into tracked object states /// (nearest tracked frame, or interpolation between the two surrounding tracked frames), answers hit tests, /// and holds the selection. It knows nothing about rendering, UI or the VideoPlayer. /// /// Coordinates everywhere here are TRACKING coordinates (0..1, Y down). See . /// /// Threading/allocation: resolved frames are written into pooled buffers; the instances handed out by /// , etc. are rewritten on the next /// frame change. Read what you need, do not cache them. /// [DisallowMultipleComponent] [AddComponentMenu("Interactive Video/Video Tracking Manager")] public sealed class VideoTrackingManager : MonoBehaviour { [Header("Data")] [Tooltip("Tracking JSON produced by the preprocessing pipeline. Can also be supplied at runtime via LoadTracking().")] [SerializeField] private TextAsset trackingJson; [SerializeField] private bool loadOnAwake = true; [Header("Frame resolution")] [Tooltip("Interpolate polygons between the surrounding tracked frames (same trackId, same point count). Off = nearest tracked frame.")] [SerializeField] private bool enableInterpolation = true; [Tooltip("How far (in video frames) the nearest tracked frame may be before the video frame counts as untracked. -1 = unlimited.")] [SerializeField] private int maxFrameGap = 45; [Tooltip("Do not interpolate across gaps in the tracking data wider than this many video frames; fall back to nearest.")] [SerializeField] private int maxInterpolationSpan = 90; [Header("Hit testing")] [Tooltip("Objects marked visible=false cannot be clicked.")] [SerializeField] private bool ignoreInvisibleInHitTest = true; [Tooltip("When several polygons contain the point (a glass held in a hand), pick the smallest one.")] [SerializeField] private bool preferSmallestHit = true; [Header("Debug")] [Tooltip("Read by TrackingDebugOverlay: draws bounding boxes, polygons, vertices, ids and the frame numbers.")] [SerializeField] private bool showTrackingDebug; // ---- events ---- /// Raised after new tracking data was loaded (or unloaded: argument null). public event Action TrackingLoaded; /// Raised when the resolved frame content changed. Argument: the video frame. public event Action FrameChanged; /// Raised when an object was added to the selection. The instance is the state at the current frame (may be null if the object is not in the current frame). public event Action ObjectSelected; /// Raised when an object was removed from the selection. public event Action ObjectDeselected; /// Raised after any change to the selection set. public event Action SelectionChanged; // ---- state ---- public TrackingData Data { get; private set; } public bool HasData => Data != null && Data.Frames.Count > 0; /// The video frame last passed to ; -1 when unknown/stopped. public long CurrentFrame { get; private set; } = -1; /// /// Increments every time the resolved object states may have changed (new tracked frame, interpolation /// step, data load). Consumers compare it to skip work when nothing moved. /// public int ResolvedVersion { get; private set; } /// Resolved states for , or null when the frame has no tracking. public TrackingFrame CurrentTrackingFrame => _currentHasData ? _current.Frame : null; /// The tracked frame the current resolution is based on (nearest), -1 if none. For debug display. public long CurrentSourceFrame { get; private set; } = -1; /// True when the current states were interpolated between two tracked frames. public bool CurrentIsInterpolated { get; private set; } public bool EnableInterpolation { get => enableInterpolation; set { if (enableInterpolation != value) { enableInterpolation = value; Refresh(); } } } public int MaxFrameGap { get => maxFrameGap; set { maxFrameGap = value; Refresh(); } } public bool ShowTrackingDebug { get => showTrackingDebug; set => showTrackingDebug = value; } private readonly ResolvedFrameBuffer _current = new ResolvedFrameBuffer(); private readonly ResolvedFrameBuffer _scratch = new ResolvedFrameBuffer(); private bool _currentHasData; private long _lastSourceFrame = long.MinValue; private bool _lastWasInterpolated; private readonly List _selectedTrackIds = new List(4); // ------------------------------------------------------------------ lifecycle private void Awake() { if (loadOnAwake && trackingJson != null && Data == null) LoadTracking(trackingJson); } // ------------------------------------------------------------------ loading public bool LoadTracking(TextAsset asset) => LoadTracking(VideoTrackingLoader.Load(asset)); public bool LoadTrackingJson(string json, string sourceName = "tracking") => LoadTracking(VideoTrackingLoader.LoadFromJson(json, sourceName)); public bool LoadTrackingFile(string path) => LoadTracking(VideoTrackingLoader.LoadFromFile(path)); /// Installs already-parsed data. Passing null unloads. The selection is kept only for ids that still exist. public bool LoadTracking(TrackingData data) { Data = data; _lastSourceFrame = long.MinValue; if (Data != null) { for (int i = _selectedTrackIds.Count - 1; i >= 0; i--) if (!Data.ObjectsById.ContainsKey(_selectedTrackIds[i])) _selectedTrackIds.RemoveAt(i); } else { _selectedTrackIds.Clear(); } Refresh(); TrackingLoaded?.Invoke(Data); SelectionChanged?.Invoke(); return Data != null; } public void UnloadTracking() => LoadTracking((TrackingData)null); // ------------------------------------------------------------------ frame /// /// Called by the video player whenever VideoPlayer.frame changed (including seeks). Cheap when the /// resolved content does not change (e.g. 30 fps video over 10 fps tracking without interpolation). /// Pass -1 when there is no current frame (video stopped / not prepared). /// public void SetCurrentFrame(long videoFrame) { if (videoFrame == CurrentFrame && videoFrame >= 0) return; CurrentFrame = videoFrame; Resolve(); } /// Re-resolves the current frame (after settings or data changed). public void Refresh() { _lastSourceFrame = long.MinValue; Resolve(); } private void Resolve() { _currentHasData = ResolveInto(CurrentFrame, _current, out long sourceFrame, out bool interpolated); CurrentSourceFrame = sourceFrame; CurrentIsInterpolated = interpolated; // Only announce a change when the resolved content can differ from last time. bool changed = interpolated || _lastWasInterpolated || sourceFrame != _lastSourceFrame; _lastSourceFrame = sourceFrame; _lastWasInterpolated = interpolated; if (!changed) return; ResolvedVersion++; FrameChanged?.Invoke(CurrentFrame); } // ------------------------------------------------------------------ queries /// Nearest raw tracked frame for a video frame (within maxFrameGap), or null. public TrackingFrame GetTrackingFrame(long videoFrame) { if (!HasData || videoFrame < 0) return null; FindSurroundingFrames(videoFrame, out var prev, out var next); var nearest = PickNearest(videoFrame, prev, next); if (nearest == null) return null; if (maxFrameGap >= 0 && Math.Abs(nearest.Frame - videoFrame) > maxFrameGap) return null; return nearest; } /// /// The tracked frame at or before, and the first tracked frame after, a video frame. Either may be null. /// If the video frame is itself tracked, prev is that frame. /// public bool FindSurroundingFrames(long videoFrame, out TrackingFrame prev, out TrackingFrame next) { prev = null; next = null; if (!HasData) return false; int iPrev = Data.FindFrameIndexAtOrBefore(videoFrame); if (iPrev >= 0) prev = Data.Frames[iPrev]; int iNext = iPrev + 1; if (iNext < Data.Frames.Count) next = Data.Frames[iNext]; return prev != null || next != null; } /// /// Resolved object states at any video frame, appended to (not cleared). /// For the current frame this is free; other frames are resolved into a scratch buffer. /// Returns the number of objects added. /// public int GetObjectsAtFrame(long videoFrame, List results) { var frame = ResolveForQuery(videoFrame); if (frame == null) return 0; results.AddRange(frame.Objects); return frame.Objects.Count; } /// Hit test at the current frame. is normalised, Y down. public TrackedObjectInstance GetObjectAtPoint(Vector2 trackingPoint) => GetObjectAtPoint(CurrentFrame, trackingPoint); /// /// Hit test: bounding box first, then point-in-polygon. When several polygons contain the point the /// smallest one wins (so a glass held in a hand beats the person), unless preferSmallestHit is off, /// in which case the last one in file order wins. /// public TrackedObjectInstance GetObjectAtPoint(long videoFrame, Vector2 trackingPoint) { var frame = ResolveForQuery(videoFrame); if (frame == null) return null; TrackedObjectInstance best = null; float bestArea = float.MaxValue; for (int i = 0; i < frame.Objects.Count; i++) { var o = frame.Objects[i]; if (ignoreInvisibleInHitTest && !o.Visible) continue; if (!o.HasPolygon) continue; if (!o.Bounds.Contains(trackingPoint)) continue; // cheap reject if (!PolygonUtility.ContainsPoint(o.Polygon, trackingPoint)) continue; float area = o.Bounds.width * o.Bounds.height; if (best == null || !preferSmallestHit || area < bestArea) { best = o; bestArea = area; } } return best; } /// State of one track at the current frame. False when the frame is untracked or the object is absent. public bool TryGetCurrentObject(string trackId, out TrackedObjectInstance instance) { instance = null; return _currentHasData && trackId != null && _current.Frame.TryGet(trackId, out instance); } /// State of one track at any frame. public bool TryGetObjectState(long videoFrame, string trackId, out TrackedObjectInstance instance) { instance = null; var frame = ResolveForQuery(videoFrame); return frame != null && trackId != null && frame.TryGet(trackId, out instance); } public TrackedObject GetDefinition(string trackId) { if (Data == null || trackId == null) return null; Data.ObjectsById.TryGetValue(trackId, out var def); return def; } // ------------------------------------------------------------------ selection /// Selected track ids. Single selection today; the list form keeps multi-select possible. public IReadOnlyList SelectedTrackIds => _selectedTrackIds; public string SelectedTrackId => _selectedTrackIds.Count > 0 ? _selectedTrackIds[0] : null; public bool HasSelection => _selectedTrackIds.Count > 0; public bool IsSelected(string trackId) => trackId != null && _selectedTrackIds.Contains(trackId); /// The primary selected object's state at the current frame (false when none selected or not in frame). public bool TryGetSelectedObject(out TrackedObjectInstance instance) { instance = null; return _selectedTrackIds.Count > 0 && TryGetCurrentObject(_selectedTrackIds[0], out instance); } /// /// Selects a track. With false (default) it replaces the selection. /// Unknown ids are refused with a warning. Returns true if the selection changed. /// public bool Select(string trackId, bool additive = false) { if (string.IsNullOrEmpty(trackId)) return false; if (Data == null || !Data.ObjectsById.ContainsKey(trackId)) { Debug.LogWarning($"[InteractiveVideo] Cannot select unknown track id '{trackId}'."); return false; } if (!additive) { if (_selectedTrackIds.Count == 1 && _selectedTrackIds[0] == trackId) return false; for (int i = _selectedTrackIds.Count - 1; i >= 0; i--) { if (_selectedTrackIds[i] == trackId) continue; string removed = _selectedTrackIds[i]; _selectedTrackIds.RemoveAt(i); ObjectDeselected?.Invoke(removed); } if (_selectedTrackIds.Count == 1) { SelectionChanged?.Invoke(); return true; } } else if (_selectedTrackIds.Contains(trackId)) { return false; } _selectedTrackIds.Add(trackId); TryGetCurrentObject(trackId, out var state); ObjectSelected?.Invoke(trackId, state); SelectionChanged?.Invoke(); return true; } public bool Deselect(string trackId) { if (trackId == null || !_selectedTrackIds.Remove(trackId)) return false; ObjectDeselected?.Invoke(trackId); SelectionChanged?.Invoke(); return true; } public void ClearSelection() { if (_selectedTrackIds.Count == 0) return; for (int i = _selectedTrackIds.Count - 1; i >= 0; i--) { string removed = _selectedTrackIds[i]; _selectedTrackIds.RemoveAt(i); ObjectDeselected?.Invoke(removed); } SelectionChanged?.Invoke(); } // ------------------------------------------------------------------ resolution internals private TrackingFrame ResolveForQuery(long videoFrame) { if (videoFrame < 0) return null; if (videoFrame == CurrentFrame) return _currentHasData ? _current.Frame : null; return ResolveInto(videoFrame, _scratch, out _, out _) ? _scratch.Frame : null; } private static TrackingFrame PickNearest(long videoFrame, TrackingFrame prev, TrackingFrame next) { if (prev == null) return next; if (next == null) return prev; return (videoFrame - prev.Frame) <= (next.Frame - videoFrame) ? prev : next; } /// Fills with the states for a video frame. False = no tracking there. private bool ResolveInto(long videoFrame, ResolvedFrameBuffer target, out long sourceFrame, out bool interpolated) { sourceFrame = -1; interpolated = false; target.Reset(videoFrame); if (!HasData || videoFrame < 0) return false; FindSurroundingFrames(videoFrame, out var prev, out var next); var nearest = PickNearest(videoFrame, prev, next); if (nearest == null) return false; if (maxFrameGap >= 0 && Math.Abs(nearest.Frame - videoFrame) > maxFrameGap) return false; sourceFrame = nearest.Frame; bool canInterpolate = enableInterpolation && prev != null && next != null && prev != next && prev.Frame != videoFrame && (maxInterpolationSpan < 0 || next.Frame - prev.Frame <= maxInterpolationSpan); if (!canInterpolate) { for (int i = 0; i < nearest.Objects.Count; i++) target.Acquire().CopyFrom(nearest.Objects[i]); return true; } interpolated = true; var other = nearest == prev ? next : prev; float t = (float)(videoFrame - prev.Frame) / (next.Frame - prev.Frame); for (int i = 0; i < nearest.Objects.Count; i++) { var n = nearest.Objects[i]; var inst = target.Acquire(); inst.CopyFrom(n); if (!n.HasPolygon || !n.Visible) continue; if (!other.TryGet(n.TrackId, out var o) || !o.Visible || o.Polygon.Count != n.Polygon.Count) continue; var a = nearest == prev ? n : o; // state at prev.Frame var b = nearest == prev ? o : n; // state at next.Frame PolygonUtility.LerpPolygon(a.Polygon, b.Polygon, t, inst.Polygon); inst.Bounds = Rect.MinMaxRect( Mathf.LerpUnclamped(a.Bounds.xMin, b.Bounds.xMin, t), Mathf.LerpUnclamped(a.Bounds.yMin, b.Bounds.yMin, t), Mathf.LerpUnclamped(a.Bounds.xMax, b.Bounds.xMax, t), Mathf.LerpUnclamped(a.Bounds.yMax, b.Bounds.yMax, t)); inst.Confidence = Mathf.LerpUnclamped(a.Confidence, b.Confidence, t); inst.Frame = videoFrame; } return true; } /// A TrackingFrame plus a pool of instances so resolving never allocates after warm-up. private sealed class ResolvedFrameBuffer { public readonly TrackingFrame Frame = new TrackingFrame(); private readonly List _pool = new List(8); public void Reset(long videoFrame) { Frame.Frame = videoFrame; Frame.Objects.Clear(); } public TrackedObjectInstance Acquire() { int index = Frame.Objects.Count; if (index >= _pool.Count) _pool.Add(new TrackedObjectInstance()); var inst = _pool[index]; Frame.Objects.Add(inst); return inst; } } } }