using System; using System.Collections.Generic; using System.IO; using UnityEngine; namespace InteractiveVideo { /// /// Turns tracking JSON into . Called once per video, never per frame. /// /// Expected shape (all coordinates normalised 0..1, Y down; see ): /// { /// "video": "scene_001.mp4", "width": 1920, "height": 1080, "fps": 30, /// "objects": [ { "id": "person_01", "type": "person", "name": "Father" } ], /// "frames": [ { "frame": 120, "objects": [ { "trackId": "person_01", "bbox": [xMin, yMin, xMax, yMax], /// "polygon": [[x, y], ...], "visible": true, "occluded": false, "confidence": 0.97, /// "action": "drinking water" } ] } ] /// } /// /// Tolerated deviations: missing "objects" (identities become implicit), missing "bbox" (computed from the /// polygon), polygon points given as {"x":..,"y":..} objects, missing optional fields. /// public static class VideoTrackingLoader { /// Loads from a TextAsset. Returns null (after logging) when the asset is missing or invalid. public static TrackingData Load(TextAsset asset) { if (asset == null) { Debug.LogWarning("[InteractiveVideo] No tracking JSON asset assigned."); return null; } return LoadFromJson(asset.text, asset.name); } /// Loads from a file path (e.g. StreamingAssets or a download cache). public static TrackingData LoadFromFile(string path) { if (string.IsNullOrEmpty(path) || !File.Exists(path)) { Debug.LogWarning($"[InteractiveVideo] Tracking file not found: '{path}'."); return null; } try { return LoadFromJson(File.ReadAllText(path), Path.GetFileName(path)); } catch (Exception e) { Debug.LogError($"[InteractiveVideo] Could not read tracking file '{path}': {e.Message}"); return null; } } /// Parses JSON text. Returns null (after logging) on invalid input. public static TrackingData LoadFromJson(string json, string sourceName = "tracking") { if (string.IsNullOrWhiteSpace(json)) { Debug.LogWarning($"[InteractiveVideo] Tracking JSON '{sourceName}' is empty."); return null; } Dictionary root; try { root = LightweightJson.AsObject(LightweightJson.Parse(json)); } catch (FormatException e) { Debug.LogError($"[InteractiveVideo] Invalid tracking JSON '{sourceName}': {e.Message}"); return null; } if (root == null) { Debug.LogError($"[InteractiveVideo] Tracking JSON '{sourceName}' must be an object at the top level."); return null; } try { return Build(root, sourceName); } catch (Exception e) { Debug.LogError($"[InteractiveVideo] Tracking JSON '{sourceName}' has an unexpected structure: {e.Message}"); return null; } } private static TrackingData Build(Dictionary root, string sourceName) { var data = new TrackingData { Video = LightweightJson.GetString(root, "video"), Width = (int)LightweightJson.GetNumber(root, "width"), Height = (int)LightweightJson.GetNumber(root, "height"), Fps = (float)LightweightJson.GetNumber(root, "fps", 30), }; // ---- object identities ---- var objects = LightweightJson.TryGet(root, "objects", out var objectsRaw) ? LightweightJson.AsArray(objectsRaw) : null; if (objects != null) { foreach (var item in objects) { var o = LightweightJson.AsObject(item); if (o == null) continue; string id = LightweightJson.GetString(o, "id"); if (string.IsNullOrEmpty(id)) { Debug.LogWarning($"[InteractiveVideo] '{sourceName}': an entry in \"objects\" has no id and was skipped."); continue; } if (data.ObjectsById.ContainsKey(id)) { Debug.LogWarning($"[InteractiveVideo] '{sourceName}': duplicate object id '{id}'; the first one is kept."); continue; } var def = new TrackedObject { Id = id, Type = LightweightJson.GetString(o, "type", "object"), Name = LightweightJson.GetString(o, "name", id), }; data.Objects.Add(def); data.ObjectsById.Add(id, def); } } // ---- frames ---- var frames = LightweightJson.TryGet(root, "frames", out var framesRaw) ? LightweightJson.AsArray(framesRaw) : null; if (frames == null || frames.Count == 0) { Debug.LogWarning($"[InteractiveVideo] '{sourceName}' contains no frames."); } else { int skippedPolygons = 0; var implicitIds = new HashSet(); foreach (var item in frames) { var f = LightweightJson.AsObject(item); if (f == null || !LightweightJson.TryGet(f, "frame", out var frameNumberRaw)) continue; var frame = new TrackingFrame { Frame = (long)LightweightJson.ToNumber(frameNumberRaw) }; var frameObjects = LightweightJson.TryGet(f, "objects", out var foRaw) ? LightweightJson.AsArray(foRaw) : null; if (frameObjects != null) { foreach (var stateItem in frameObjects) { var s = LightweightJson.AsObject(stateItem); if (s == null) continue; string trackId = LightweightJson.GetString(s, "trackId") ?? LightweightJson.GetString(s, "id"); if (string.IsNullOrEmpty(trackId)) continue; if (!data.ObjectsById.TryGetValue(trackId, out var def)) { def = new TrackedObject { Id = trackId, Type = "object", Name = trackId, IsImplicit = true }; data.Objects.Add(def); data.ObjectsById.Add(trackId, def); implicitIds.Add(trackId); } var inst = new TrackedObjectInstance { TrackId = trackId, Definition = def, Frame = frame.Frame, Visible = LightweightJson.GetBool(s, "visible", true), Occluded = LightweightJson.GetBool(s, "occluded", false), Confidence = (float)LightweightJson.GetNumber(s, "confidence", 1), Action = LightweightJson.GetString(s, "action"), }; ReadPolygon(s, inst.Polygon); if (inst.Polygon.Count > 0 && inst.Polygon.Count < 3) skippedPolygons++; inst.Bounds = ReadBounds(s, inst.Polygon); frame.Objects.Add(inst); } } data.Frames.Add(frame); } if (implicitIds.Count > 0) Debug.LogWarning($"[InteractiveVideo] '{sourceName}': {implicitIds.Count} track id(s) appear in frames but not in \"objects\": {string.Join(", ", implicitIds)}. Placeholder identities were created."); if (skippedPolygons > 0) Debug.LogWarning($"[InteractiveVideo] '{sourceName}': {skippedPolygons} polygon(s) have fewer than 3 points and cannot be clicked or highlighted."); } data.RebuildIndex(); return data; } private static void ReadPolygon(Dictionary state, List into) { into.Clear(); if (!LightweightJson.TryGet(state, "polygon", out var raw)) return; var points = LightweightJson.AsArray(raw); if (points == null) return; foreach (var p in points) { if (p is List pair && pair.Count >= 2) { into.Add(new Vector2((float)LightweightJson.ToNumber(pair[0]), (float)LightweightJson.ToNumber(pair[1]))); } else if (p is Dictionary xy) { into.Add(new Vector2((float)LightweightJson.GetNumber(xy, "x"), (float)LightweightJson.GetNumber(xy, "y"))); } } } /// /// bbox is [xMin, yMin, xMax, yMax] in tracking coordinates. When it is missing or does not enclose the /// polygon, the polygon bounds are used (unioned), so the bbox pre-test can never reject a valid polygon hit. /// private static Rect ReadBounds(Dictionary state, List polygon) { Rect fromPolygon = PolygonUtility.ComputeBounds(polygon); if (LightweightJson.TryGet(state, "bbox", out var raw) && raw is List b && b.Count >= 4) { var r = Rect.MinMaxRect( (float)LightweightJson.ToNumber(b[0]), (float)LightweightJson.ToNumber(b[1]), (float)LightweightJson.ToNumber(b[2]), (float)LightweightJson.ToNumber(b[3])); if (polygon.Count == 0) return r; return Rect.MinMaxRect( Mathf.Min(r.xMin, fromPolygon.xMin), Mathf.Min(r.yMin, fromPolygon.yMin), Mathf.Max(r.xMax, fromPolygon.xMax), Mathf.Max(r.yMax, fromPolygon.yMax)); } return fromPolygon; } } }