supervision 0.1.7 → 0.2.0-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/dist/constants/media-renderer.d.ts +3 -0
  2. package/dist/constants/media-renderer.d.ts.map +1 -1
  3. package/dist/detections/chunked-detection-frame-source.d.ts.map +1 -1
  4. package/dist/index.d.ts +8 -4
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +9281 -5038
  7. package/dist/index.js.map +1 -1
  8. package/dist/mask-preparation.worker.js +409 -185
  9. package/dist/mask-preparation.worker.js.map +1 -1
  10. package/dist/media/display-pixel-ratio.d.ts +15 -0
  11. package/dist/media/display-pixel-ratio.d.ts.map +1 -0
  12. package/dist/media/engine-import-failure.d.ts +16 -0
  13. package/dist/media/engine-import-failure.d.ts.map +1 -0
  14. package/dist/media/media-errors.d.ts +6 -0
  15. package/dist/media/media-errors.d.ts.map +1 -1
  16. package/dist/media/media-normalization.d.ts.map +1 -1
  17. package/dist/media/media-probe.d.ts.map +1 -1
  18. package/dist/media/media-source-state.d.ts.map +1 -1
  19. package/dist/media/media-source.d.ts +17 -0
  20. package/dist/media/media-source.d.ts.map +1 -1
  21. package/dist/media/mediabunny-media-source.d.ts.map +1 -1
  22. package/dist/media/video-engine-media-source.d.ts +54 -0
  23. package/dist/media/video-engine-media-source.d.ts.map +1 -0
  24. package/dist/media/video-engine-media-source.js +3 -0
  25. package/dist/media/video-engine-media-source.js.map +1 -0
  26. package/dist/playback/media-playback-controller.d.ts +7 -1
  27. package/dist/playback/media-playback-controller.d.ts.map +1 -1
  28. package/dist/render-preparation/mask-frame-artifact.d.ts +48 -12
  29. package/dist/render-preparation/mask-frame-artifact.d.ts.map +1 -1
  30. package/dist/render-preparation/mask-frame-compositor.d.ts +12 -5
  31. package/dist/render-preparation/mask-frame-compositor.d.ts.map +1 -1
  32. package/dist/render-preparation/mask-frame-preparer.d.ts.map +1 -1
  33. package/dist/render-preparation/mask-preparation-worker-count.d.ts +4 -4
  34. package/dist/render-preparation/mask-preparation-worker-protocol.d.ts +13 -3
  35. package/dist/render-preparation/mask-preparation-worker-protocol.d.ts.map +1 -1
  36. package/dist/render-preparation/prepared-render-window.d.ts +44 -2
  37. package/dist/render-preparation/prepared-render-window.d.ts.map +1 -1
  38. package/dist/render-preparation/prepared-window-timeline.d.ts +2 -6
  39. package/dist/render-preparation/prepared-window-timeline.d.ts.map +1 -1
  40. package/dist/renderers/injected-pixi.d.ts +38 -0
  41. package/dist/renderers/injected-pixi.d.ts.map +1 -0
  42. package/dist/renderers/mask-palette.d.ts +14 -0
  43. package/dist/renderers/mask-palette.d.ts.map +1 -0
  44. package/dist/renderers/mask-vertex.d.ts +14 -0
  45. package/dist/renderers/mask-vertex.d.ts.map +1 -0
  46. package/dist/renderers/media-renderer-core.d.ts +9 -0
  47. package/dist/renderers/media-renderer-core.d.ts.map +1 -1
  48. package/dist/renderers/media-renderer-scene.d.ts +50 -2
  49. package/dist/renderers/media-renderer-scene.d.ts.map +1 -1
  50. package/dist/renderers/media-renderer-state.d.ts +20 -1
  51. package/dist/renderers/media-renderer-state.d.ts.map +1 -1
  52. package/dist/renderers/media-renderer-transport.d.ts +72 -0
  53. package/dist/renderers/media-renderer-transport.d.ts.map +1 -0
  54. package/dist/renderers/pixi-box-layer.d.ts +5 -0
  55. package/dist/renderers/pixi-box-layer.d.ts.map +1 -1
  56. package/dist/renderers/pixi-focus-layer.d.ts +19 -2
  57. package/dist/renderers/pixi-focus-layer.d.ts.map +1 -1
  58. package/dist/renderers/pixi-frame-present.d.ts +77 -0
  59. package/dist/renderers/pixi-frame-present.d.ts.map +1 -0
  60. package/dist/renderers/pixi-id-mask-shader.d.ts +6 -25
  61. package/dist/renderers/pixi-id-mask-shader.d.ts.map +1 -1
  62. package/dist/renderers/pixi-interaction-layer.d.ts.map +1 -1
  63. package/dist/renderers/pixi-interaction-presentation-layer.d.ts +12 -2
  64. package/dist/renderers/pixi-interaction-presentation-layer.d.ts.map +1 -1
  65. package/dist/renderers/pixi-mask-halo.d.ts +18 -23
  66. package/dist/renderers/pixi-mask-halo.d.ts.map +1 -1
  67. package/dist/renderers/pixi-mask-layer.d.ts +94 -32
  68. package/dist/renderers/pixi-mask-layer.d.ts.map +1 -1
  69. package/dist/renderers/pixi-media-scene.d.ts +67 -0
  70. package/dist/renderers/pixi-media-scene.d.ts.map +1 -1
  71. package/dist/renderers/pixi-polygon-layer.d.ts +9 -2
  72. package/dist/renderers/pixi-polygon-layer.d.ts.map +1 -1
  73. package/dist/renderers/pixi-region-coverage-mask.d.ts +62 -0
  74. package/dist/renderers/pixi-region-coverage-mask.d.ts.map +1 -0
  75. package/dist/renderers/pixi-region-effect.d.ts +42 -0
  76. package/dist/renderers/pixi-region-effect.d.ts.map +1 -0
  77. package/dist/renderers/pixi-region-layer.d.ts +43 -2
  78. package/dist/renderers/pixi-region-layer.d.ts.map +1 -1
  79. package/dist/renderers/pixi-shader-lifecycle.d.ts +14 -0
  80. package/dist/renderers/pixi-shader-lifecycle.d.ts.map +1 -0
  81. package/dist/renderers/pixi-vector-layer.d.ts.map +1 -1
  82. package/dist/renderers/prepared-annotation-window.d.ts +47 -0
  83. package/dist/renderers/prepared-annotation-window.d.ts.map +1 -0
  84. package/dist/renderers/presented-frame-channel.d.ts +138 -0
  85. package/dist/renderers/presented-frame-channel.d.ts.map +1 -0
  86. package/dist/renderers/scene-render-scheduler.d.ts +20 -0
  87. package/dist/renderers/scene-render-scheduler.d.ts.map +1 -0
  88. package/dist/sessions/media-session-defaults.d.ts +10 -6
  89. package/dist/sessions/media-session-defaults.d.ts.map +1 -1
  90. package/dist/sessions/media-session-media.d.ts.map +1 -1
  91. package/dist/sessions/media-session-state.d.ts.map +1 -1
  92. package/dist/sessions/media-session.d.ts.map +1 -1
  93. package/dist/tracking.worker.js +73 -4
  94. package/dist/tracking.worker.js.map +1 -1
  95. package/dist/types/media-normalization.d.ts +34 -0
  96. package/dist/types/media-normalization.d.ts.map +1 -1
  97. package/dist/types/media-renderer.d.ts +31 -1
  98. package/dist/types/media-renderer.d.ts.map +1 -1
  99. package/dist/types/media-session.d.ts +89 -15
  100. package/dist/types/media-session.d.ts.map +1 -1
  101. package/dist/types/render-preparation.d.ts +149 -5
  102. package/dist/types/render-preparation.d.ts.map +1 -1
  103. package/dist/video-engine-media-source-CUXnOJOV.js +450 -0
  104. package/dist/video-engine-media-source-CUXnOJOV.js.map +1 -0
  105. package/dist/web-video-engine/analysis-session.d.ts +75 -0
  106. package/dist/web-video-engine/analysis.d.ts +6 -0
  107. package/dist/web-video-engine/analysis.js +1850 -0
  108. package/dist/web-video-engine/cache-budget.d.ts +18 -0
  109. package/dist/web-video-engine/canvas-sink-scrub-cursor.d.ts +89 -0
  110. package/dist/web-video-engine/clock.d.ts +82 -0
  111. package/dist/web-video-engine/constants.d.ts +261 -0
  112. package/dist/web-video-engine/create-scrub-cursor.d.ts +65 -0
  113. package/dist/web-video-engine/decode-resolution.d.ts +114 -0
  114. package/dist/web-video-engine/decode-scheduler.d.ts +371 -0
  115. package/dist/web-video-engine/decode-session.d.ts +283 -0
  116. package/dist/web-video-engine/decode-source.d.ts +258 -0
  117. package/dist/web-video-engine/diagnostics-store.d.ts +17 -0
  118. package/dist/web-video-engine/diagnostics.d.ts +316 -0
  119. package/dist/web-video-engine/embedded-engine-worker.d.ts +1 -0
  120. package/dist/web-video-engine/engine-core.d.ts +269 -0
  121. package/dist/web-video-engine/engine.d.ts +25 -0
  122. package/dist/web-video-engine/engine.js +912 -0
  123. package/dist/web-video-engine/engine.worker.d.ts +1 -0
  124. package/dist/web-video-engine/engine.worker.js +33624 -0
  125. package/dist/web-video-engine/frame-cache.d.ts +236 -0
  126. package/dist/web-video-engine/frame-extractor.d.ts +27 -0
  127. package/dist/web-video-engine/frame-timeline-DreEwsRY.js +616 -0
  128. package/dist/web-video-engine/frame-timeline.d.ts +80 -0
  129. package/dist/web-video-engine/frame-walker.d.ts +73 -0
  130. package/dist/web-video-engine/index.d.ts +13 -0
  131. package/dist/web-video-engine/index.d.ts.map +1 -0
  132. package/dist/web-video-engine/index.js +3 -0
  133. package/dist/web-video-engine/index.js.map +1 -0
  134. package/dist/web-video-engine/key-packet.d.ts +64 -0
  135. package/dist/web-video-engine/keyframe-index.d.ts +90 -0
  136. package/dist/web-video-engine/mirror-store.d.ts +55 -0
  137. package/dist/web-video-engine/renderer.d.ts +93 -0
  138. package/dist/web-video-engine/rotation.d.ts +34 -0
  139. package/dist/web-video-engine/scrub-controller.d.ts +375 -0
  140. package/dist/web-video-engine/scrub-cursor.d.ts +347 -0
  141. package/dist/web-video-engine/scrub-trajectory.d.ts +29 -0
  142. package/dist/web-video-engine/source-residency.d.ts +91 -0
  143. package/dist/web-video-engine/trace-recorder.d.ts +163 -0
  144. package/dist/web-video-engine/types.d.ts +200 -0
  145. package/dist/web-video-engine/video-engine.d.ts +437 -0
  146. package/dist/web-video-engine/webgpu-renderer.d.ts +61 -0
  147. package/dist/web-video-engine/worker-bridge.d.ts +11 -0
  148. package/dist/web-video-engine/worker-dispatch.d.ts +16 -0
  149. package/dist/web-video-engine/worker-protocol.d.ts +282 -0
  150. package/node_modules/supervision-js-core/dist/detections/buffered-detection-timeline.d.ts.map +1 -1
  151. package/node_modules/supervision-js-core/dist/detections/composite-detection-frame-source.d.ts.map +1 -1
  152. package/node_modules/supervision-js-core/dist/index.d.ts +5 -4
  153. package/node_modules/supervision-js-core/dist/index.d.ts.map +1 -1
  154. package/node_modules/supervision-js-core/dist/index.js +1167 -280
  155. package/node_modules/supervision-js-core/dist/index.js.map +1 -1
  156. package/node_modules/supervision-js-core/dist/styles/default-annotation-presentation.d.ts.map +1 -1
  157. package/node_modules/supervision-js-core/dist/styles/interaction-style.d.ts +4 -36
  158. package/node_modules/supervision-js-core/dist/styles/interaction-style.d.ts.map +1 -1
  159. package/node_modules/supervision-js-core/dist/styles/polyline-style.d.ts +5 -0
  160. package/node_modules/supervision-js-core/dist/styles/polyline-style.d.ts.map +1 -1
  161. package/node_modules/supervision-js-core/dist/types/annotation-renderer.d.ts +92 -7
  162. package/node_modules/supervision-js-core/dist/types/annotation-renderer.d.ts.map +1 -1
  163. package/node_modules/supervision-js-core/dist/types/detection-timeline.d.ts +191 -17
  164. package/node_modules/supervision-js-core/dist/types/detection-timeline.d.ts.map +1 -1
  165. package/node_modules/supervision-js-core/dist/types/media-rendering.d.ts +83 -5
  166. package/node_modules/supervision-js-core/dist/types/media-rendering.d.ts.map +1 -1
  167. package/node_modules/supervision-js-core/dist/types/polyline-style.d.ts +2 -0
  168. package/node_modules/supervision-js-core/dist/types/polyline-style.d.ts.map +1 -1
  169. package/node_modules/supervision-js-core/dist/types/session-lifecycle.d.ts +32 -0
  170. package/node_modules/supervision-js-core/dist/types/session-lifecycle.d.ts.map +1 -1
  171. package/node_modules/supervision-js-core/dist/utils/detection-conversions.d.ts.map +1 -1
  172. package/node_modules/supervision-js-core/dist/utils/detection-frames.d.ts +5 -0
  173. package/node_modules/supervision-js-core/dist/utils/detection-frames.d.ts.map +1 -1
  174. package/node_modules/supervision-js-core/dist/utils/detection-masks.d.ts +9 -0
  175. package/node_modules/supervision-js-core/dist/utils/detection-masks.d.ts.map +1 -1
  176. package/node_modules/supervision-js-core/dist/utils/detection-ranges.d.ts +18 -0
  177. package/node_modules/supervision-js-core/dist/utils/detection-ranges.d.ts.map +1 -0
  178. package/node_modules/supervision-js-core/dist/utils/id-mask-frame.d.ts +34 -2
  179. package/node_modules/supervision-js-core/dist/utils/id-mask-frame.d.ts.map +1 -1
  180. package/node_modules/supervision-js-core/dist/utils/wait-bound.d.ts +7 -0
  181. package/node_modules/supervision-js-core/dist/utils/wait-bound.d.ts.map +1 -0
  182. package/package.json +24 -3
@@ -0,0 +1,347 @@
1
+ import type { CreateScrubCursorOptions } from "./create-scrub-cursor";
2
+ import type { FrameCacheStats } from "./frame-cache";
3
+ import type { FrameId, FrameTimeline } from "./frame-timeline";
4
+ import type { GopStats } from "./keyframe-index";
5
+ import type { Rotation } from "./rotation";
6
+ import type { DecodePath, Sec } from "./types";
7
+ /**
8
+ * How good a painted frame is. "exact" is a full decode-resolution frame;
9
+ * "preview" is the coarse downscaled stand-in the cache serves at once while a
10
+ * crisp one decodes. The two look materially different on screen, so anything
11
+ * reporting what was painted has to say which it was.
12
+ */
13
+ export type FrameQuality = "exact" | "preview";
14
+ export declare enum ScrubCursorState {
15
+ Idle = "idle",
16
+ Seeking = "seeking",
17
+ Closed = "closed"
18
+ }
19
+ /** Fields every paint-ready frame carries, independent of how its pixels reach
20
+ * the renderer. */
21
+ export interface ScrubFrameBase {
22
+ readonly timestampS: Sec;
23
+ readonly width: number;
24
+ readonly height: number;
25
+ readonly isKeyFrame: boolean;
26
+ readonly quality: FrameQuality;
27
+ }
28
+ /**
29
+ * The structural slice of mediabunny's VideoSample the runtime touches: enough
30
+ * to draw it into a 2D canvas, hand its pixels to WebGPU without an intermediate
31
+ * transfer copy where the selected path supports that, and release it. Kept as
32
+ * a slice rather than the concrete class so the runtime never
33
+ * imports mediabunny and stays fake-injectable under test.
34
+ *
35
+ * close() is the caller's obligation and is made idempotent at the cursor
36
+ * boundary, so a sample drawn into the cache, painted, and then closed on
37
+ * teardown is closed at most once for real.
38
+ */
39
+ export interface VideoSampleLike {
40
+ /**
41
+ * A fresh VideoFrame per call; the caller closes the returned frame.
42
+ *
43
+ * The frame carries the stored pixels and NOT the track's rotation, on this
44
+ * and on mediabunny's own implementation, which drops it deliberately. So
45
+ * whoever takes one owes it the turn named by `rotation` below.
46
+ */
47
+ toVideoFrame(): VideoFrame;
48
+ /** The sample owns storage independent of a decoder output pool, so a host
49
+ * transfer may rewrap it without copying its pixels again. */
50
+ readonly independentPixels?: true;
51
+ /** The turn the pixels still need, already applied by draw() and dropped by
52
+ * toVideoFrame(). */
53
+ readonly rotation: Rotation;
54
+ draw(context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, dx: number, dy: number, dWidth?: number, dHeight?: number): void;
55
+ close(): void;
56
+ readonly timestamp: number;
57
+ readonly duration: number;
58
+ }
59
+ /**
60
+ * A paint-ready frame yielded by the cursor, tagged by how its pixels reach the
61
+ * renderer.
62
+ *
63
+ * A canvas frame carries a canvas and no close obligation: it is a cache blit
64
+ * or a CanvasSink decode. WebGPU uploads it with copyExternalImageToTexture,
65
+ * which accepts neither an SVG image nor a video element. A sample frame carries
66
+ * a live VideoSample that an eligible VideoSampleSink route can import directly
67
+ * into WebGPU. A DecodeSession route may first materialize independently owned
68
+ * pixels. Whoever stashes the sample owns its close, so only fresh decodes on
69
+ * the sample source produce one.
70
+ */
71
+ export interface CanvasScrubFrame extends ScrubFrameBase {
72
+ readonly kind: "canvas";
73
+ readonly source: OffscreenCanvas | HTMLCanvasElement;
74
+ }
75
+ export interface SampleScrubFrame extends ScrubFrameBase {
76
+ readonly kind: "sample";
77
+ readonly sample: VideoSampleLike;
78
+ }
79
+ export type ScrubFrame = CanvasScrubFrame | SampleScrubFrame;
80
+ /**
81
+ * The turn a frame's raw pixels still need. A canvas frame has none left: a
82
+ * CanvasSink decode arrives upright, and a cache blit was drawn through a
83
+ * sample's own draw, which applies it. Only a live sample still owes one.
84
+ */
85
+ export declare function frameRotation(frame: ScrubFrame): Rotation;
86
+ /**
87
+ * Wraps a raw sample so its close() runs at most once, then no-ops. The
88
+ * lifetime is hard to keep linear: a sample is drawn into the cache, stashed for
89
+ * paint, then closed after paint, and may be closed again on teardown if a
90
+ * gesture left it unpainted. An idempotent close lets every path call it
91
+ * defensively without double-free.
92
+ */
93
+ export declare function idempotentSample(sample: VideoSampleLike): VideoSampleLike;
94
+ export type ScrubFrameListener = (frame: ScrubFrame) => void;
95
+ /** A frame considered for the screen, and what is already on it. */
96
+ export interface ScreenCandidate {
97
+ readonly timestampMs: number;
98
+ readonly quality: FrameQuality;
99
+ }
100
+ /**
101
+ * Whether a frame earns the screen over what is already showing, judged from
102
+ * where the user is pointing rather than from what happens to be up. Closer
103
+ * wins; equally close wins only by being sharper, since repainting the same
104
+ * ground with the same or worse pixels is a visible change for no information.
105
+ *
106
+ * Both painters share this. They used to disagree: one applied it to coarse
107
+ * frames only and the other to everything, so which rule you got depended on
108
+ * which path served the frame.
109
+ */
110
+ export declare function earnsScreen(candidate: ScreenCandidate, showing: ScreenCandidate | null, targetMs: number): boolean;
111
+ /** Rolling scrub-decode latency, in milliseconds, since the cursor opened.
112
+ * p50Ms/p95Ms are percentiles over a bounded recent-sample ring, so a single
113
+ * slow region shows up in p95 without dragging the all-time avg. targetVsLandedMs
114
+ * is how far the last exact seek landed from where it aimed (long-GOP forces a
115
+ * distant anchor); timeToCrispMs is the gap between a preview paint and the crisp
116
+ * decode that replaced it on the same seek. */
117
+ export interface ScrubLatencyStats {
118
+ readonly samples: number;
119
+ readonly lastMs: number;
120
+ readonly avgMs: number;
121
+ readonly maxMs: number;
122
+ readonly p50Ms: number;
123
+ readonly p95Ms: number;
124
+ readonly targetVsLandedMs: number;
125
+ readonly timeToCrispMs: number;
126
+ }
127
+ /** Decodes the scheduler has run, split by why. foreground is a gesture-driven
128
+ * seek/step/play pull; the prefetch counts are background sweep fills per tier;
129
+ * keyframeAnchored is a key-only seek landing on its anchor. Counters only, never
130
+ * reset, so ratios over a session describe where decode bandwidth went. */
131
+ export interface DecodeCounters {
132
+ readonly foreground: number;
133
+ readonly prefetchExact: number;
134
+ readonly prefetchPreview: number;
135
+ readonly keyframeAnchored: number;
136
+ /** Frames the decode machinery has produced since open, all paths, walk
137
+ * pre-roll included where the path can see it. The pipeline ledger's
138
+ * decoded column. */
139
+ readonly framesOut: number;
140
+ /** Forward-playback pulls in flight (0 or 1); a free read of the drain latch. */
141
+ readonly nextPending: number;
142
+ }
143
+ /** Seeks the scheduler has serviced. exact is a full-resolution scrub; key is a
144
+ * key-only navigation seek; coalesceDepth counts how often a pending target was
145
+ * overwritten before it drained (the decoder falling behind a fast drag). */
146
+ export interface SeekCounters {
147
+ readonly exact: number;
148
+ readonly key: number;
149
+ readonly coalesceDepth: number;
150
+ }
151
+ /** Background prefetch sweep liveness: whether a sweep is decoding right now and
152
+ * the generation token (bumped by every foreground gesture, so a jumping value
153
+ * is a churning sweep that never warms the cache). */
154
+ export interface PrefetchState {
155
+ readonly inFlight: boolean;
156
+ readonly generation: number;
157
+ }
158
+ /**
159
+ * Observability snapshot a caching cursor can report: how often the cache
160
+ * answered a scrub, and how long the misses took to decode. Cache hit-rate and
161
+ * decode latency together describe perceived scrub responsiveness.
162
+ */
163
+ /**
164
+ * Why the playhead is moving. A hand on a timeline keeps feeding positions, so
165
+ * the runtime reads their direction and speed to decide what to prepare next.
166
+ * A jump produces one position and no motion to read, so prediction from the
167
+ * positions around it would be prediction from a gesture nobody is making.
168
+ */
169
+ export type SeekIntent = "gesture" | "jump";
170
+ /** The decode window the scheduler is filling around the playhead, in ms. */
171
+ /** The decode targets the next background sweep would aim at, given what the
172
+ * runtime currently knows. A bounds pair cannot represent a hole, and holes
173
+ * are what a coverage instrument exists to show. */
174
+ export interface PrefetchPlan {
175
+ readonly targetsMs: number[];
176
+ }
177
+ export interface SchedulerStats {
178
+ /** The runtime's live read of how the consumer is moving (idle/scrubbing/
179
+ * stepping/playing). The single highest-signal field for a human watching
180
+ * the engine interpret their gesture. */
181
+ readonly mode: string;
182
+ /** Which decode machinery this source was opened through. Resolved once at
183
+ * open from the track and the realm; a hang-recovery rebuild re-resolves
184
+ * it, so a path change here means the source was re-opened. */
185
+ readonly decodePath: DecodePath;
186
+ readonly cache: FrameCacheStats;
187
+ readonly scrub: ScrubLatencyStats;
188
+ readonly decode: DecodeCounters;
189
+ readonly seek: SeekCounters;
190
+ /** GOP-gap distribution over the keyframes discovered so far. A long maxGopS
191
+ * is the badly-encoded-source tell every off-anchor scrub pays for. */
192
+ readonly gop: GopStats;
193
+ /** Container round-trips the keyframe index has made resolving anchors; the
194
+ * rest resolve from memory. */
195
+ readonly probeRoundTrips: number;
196
+ /** Discovered keyframe timestamps (ms), ascending. Lazy: grows as the source
197
+ * is scrubbed/swept, so it shows what the runtime actually knows. */
198
+ readonly keyframesMs: number[];
199
+ /** The targets the scheduler would prefetch around the playhead, or null
200
+ * while playing (the forward stream iterator covers ahead). */
201
+ readonly prefetch: PrefetchPlan | null;
202
+ /** Background sweep liveness (in-flight + generation token). */
203
+ readonly prefetchState: PrefetchState;
204
+ /** Cache lookup tolerances, so a timeline can draw the "served" bands. */
205
+ readonly exactToleranceMs: number;
206
+ readonly previewToleranceMs: number;
207
+ /** True after a decode hung on a non-re-openable source: the decoder could
208
+ * not be rebuilt, so the runtime is degraded (decodes no-op) but not frozen.
209
+ * Lets a consumer surface the state rather than show a silent freeze. */
210
+ readonly decoderDead: boolean;
211
+ /** True once the decoder is judged unable to decode this source at all: it
212
+ * refused to configure, it errored, or it acknowledged decode requests and
213
+ * produced nothing. Distinct from decoderDead, which is a rebuild the
214
+ * source made impossible; this one survives every rebuild, so the runtime
215
+ * stops rebuilding. The transport reads Errored at the same moment. */
216
+ readonly decoderStalled: boolean;
217
+ /** State of the seek drain. Every play pull is refused while a seek drains,
218
+ * so a drain that never finishes shows on screen as playback dying with no
219
+ * other symptom: without this the panel can say the pump is dead but not
220
+ * that a seek is the reason. */
221
+ readonly drain: DrainState;
222
+ }
223
+ export interface DrainState {
224
+ /** A seek is being serviced right now, which refuses every play pull. */
225
+ readonly draining: boolean;
226
+ /** Target waiting behind the one being serviced, in ms, or null. */
227
+ readonly pendingTargetMs: number | null;
228
+ /** A hung-decode rebuild is in progress. */
229
+ readonly recovering: boolean;
230
+ }
231
+ /**
232
+ * Resolved track facts the engine reads after open(). width/height are the
233
+ * source's native resolution (used for aspect, metadata, cache sizing).
234
+ * decodeWidth/decodeHeight are the resolution frames are actually decoded to
235
+ * after the decode-resolution strategy runs, and size the visible canvas
236
+ * backing store. They equal native unless a downscaling strategy is in play.
237
+ */
238
+ export interface ScrubTrackInfo {
239
+ readonly width: number;
240
+ readonly height: number;
241
+ readonly decodeWidth: number;
242
+ readonly decodeHeight: number;
243
+ /**
244
+ * The track's quarter turn. Every dimension above is the display size, so all
245
+ * four already account for it. This exists for the pixels, which do not.
246
+ */
247
+ readonly rotation: Rotation;
248
+ readonly nativeFps: number | null;
249
+ readonly durationS: Sec;
250
+ /**
251
+ * Timestamp of the track's first sample, in seconds. Usually near zero but
252
+ * may be positive (a trimmed clip) or negative (offset timing). The seed
253
+ * seek and seek clamping use it as the origin so a non-zero start does not
254
+ * mis-seek to t=0.
255
+ */
256
+ readonly firstTimestampS: Sec;
257
+ /** Every real frame of this track, by its container tick timestamp. The one
258
+ * place a frame's identity is defined; everything else snaps into it. */
259
+ readonly timeline: FrameTimeline;
260
+ }
261
+ /**
262
+ * Random-access cursor over a video source: the seam between the engine and
263
+ * the mediabunny primitives it wraps.
264
+ *
265
+ * The implementation (CanvasSinkScrubCursor) rides CanvasSink.getCanvas(t)
266
+ * for random-access seeks, which does the keyframe walk and GOP decode
267
+ * internally, and a sink.canvases(start) iterator for forward playback that
268
+ * lives only between attachPlay and detachPlay. The interface is kept as a
269
+ * seam so a future mediabunny SampleCursor backend can replace the
270
+ * implementation without touching consumers.
271
+ *
272
+ * Contract notes consumers rely on:
273
+ * - seekTo coalesces concurrent seeks latest-wins, so rapid pointermove
274
+ * scrubs collapse to the most recent target.
275
+ * - seekToKey lands on the sample at or before t and marks the emitted
276
+ * frame as a keyframe result.
277
+ * - next is VFR-correct and a no-op while paused (no iterator attached),
278
+ * so it never advances media time across a paused canvas. The 1/fps step
279
+ * approximation lives in WebVideoEngine.step, not here.
280
+ * - isIdle is true only when no seek is draining and no pull is in flight.
281
+ * - subscribe replays the most recent frame to a new listener so the seed
282
+ * frame from open() is never dropped.
283
+ * - close is final; reusing a closed cursor is a programming error.
284
+ */
285
+ export interface ScrubCursor {
286
+ readonly state: ScrubCursorState;
287
+ readonly track: ScrubTrackInfo;
288
+ readonly isIdle: boolean;
289
+ open(): Promise<void>;
290
+ seekTo(timestamp: Sec, intent?: SeekIntent): void;
291
+ seekToKey(timestamp: Sec): void;
292
+ next(): void;
293
+ /**
294
+ * Attach a forward-playback iterator anchored at startS. Required for
295
+ * next() to advance; paused mode keeps no iterator.
296
+ */
297
+ attachPlay(startS: number): void;
298
+ /** Detach the forward-playback iterator. */
299
+ detachPlay(): void;
300
+ /**
301
+ * Decode the named frame of the source and emit it through the listener
302
+ * chain. Returns the emitted ScrubFrame, or null at a boundary or once
303
+ * closed. Steps are index arithmetic on the frame table, so a caller names
304
+ * the frame it wants and not a time near it.
305
+ */
306
+ seekToFrame(frame: FrameId): Promise<ScrubFrame | null>;
307
+ idle(): Promise<void>;
308
+ /**
309
+ * Resolves at the seek drain's next settle: whichever target it is servicing
310
+ * has landed. The wait spans one decode. idle() spans the whole gesture,
311
+ * since a drag re-arms the drain on every pointer move, so a caller that has
312
+ * to answer while the hand is still down asks this one.
313
+ */
314
+ seekSettled(): Promise<void>;
315
+ /**
316
+ * Registers a frame listener. The cache replays the most recently emitted
317
+ * frame synchronously on subscribe so the seed frame from open() is
318
+ * delivered to controllers that wire up afterwards.
319
+ */
320
+ subscribe(listener: ScrubFrameListener): () => void;
321
+ /**
322
+ * Synchronous best-effort cache lookup the render loop calls on scrub to
323
+ * paint a frame this same tick, before the full-res decode resolves. The
324
+ * uncached implementation returns null; the caching scheduler returns the
325
+ * closest cached frame within tolerance, or null on a miss.
326
+ */
327
+ peekCached(timeMs: number): ScrubFrame | null;
328
+ /**
329
+ * Observability snapshot (cache hit-rate, scrub-decode latency). Present
330
+ * only on the caching backend; the uncached cursor omits it.
331
+ */
332
+ /** Decoded frames the consumer may hold ahead of the playhead; see
333
+ * FrameProvider.playReadAhead. Absent on cursors with no decode path of
334
+ * their own, where the caller falls back to the canvas depth. */
335
+ readonly playReadAhead?: number;
336
+ getStats?(): SchedulerStats;
337
+ /**
338
+ * Eagerly walks the whole track's keyframes into the index, metadata only,
339
+ * so the diagnostics keyframe lane reflects the entire file rather than just
340
+ * the swept regions. Idempotent and off the hot seek path. Present only on
341
+ * the caching backend (the uncached cursor keeps no index); callers invoke
342
+ * it from the diagnostics layer, never during a gesture.
343
+ */
344
+ ensureKeyframeIndex?(): Promise<void>;
345
+ close(): Promise<void>;
346
+ }
347
+ export type ScrubCursorFactory = (options: CreateScrubCursorOptions) => Promise<ScrubCursor>;
@@ -0,0 +1,29 @@
1
+ import type { Sec } from "./types";
2
+ /**
3
+ * Reads which way a scrub gesture is travelling, so a prefetch window can spend
4
+ * its budget on the side of the playhead the hand is moving toward.
5
+ *
6
+ * The heading spans a bounded ring of recent samples, so one jittery pointer
7
+ * move cannot swing it. A step opposing the ring's heading drops the samples
8
+ * behind it, leaving the reversal as the only evidence of where the gesture is
9
+ * going.
10
+ */
11
+ export declare class ScrubTrajectory {
12
+ private readonly capacity;
13
+ private readonly positionsS;
14
+ private readonly timesMs;
15
+ private count;
16
+ private writeIndex;
17
+ constructor(capacity?: number);
18
+ sample(positionS: Sec, atMs: number): void;
19
+ /**
20
+ * Which way the gesture is travelling: 1 forward, -1 backward, 0 when there
21
+ * is not enough of a gesture to say. The direction survives being uncertain
22
+ * about how far, which is all a window split needs.
23
+ */
24
+ heading(): -1 | 0 | 1;
25
+ reset(): void;
26
+ private netDisplacementS;
27
+ private indexFromNewest;
28
+ private indexFromOldest;
29
+ }
@@ -0,0 +1,91 @@
1
+ /**
2
+ * Byte residency for a URL source: the bytes this process holds, the fetch that
3
+ * serves them to the demuxer, and a background walk that fills the rest in.
4
+ *
5
+ * The browser HTTP cache does not do this job for media. A 200 response is
6
+ * stored whole and read back in about a millisecond, but the demuxer never
7
+ * issues one: it opens the file with `Range: bytes=N-` and abandons the
8
+ * response once it has what it wanted, so every read is stored as sparse data
9
+ * against a single cache entry. Sparse data is evicted long before a hundred
10
+ * megabytes of it accumulates, and a whole-file fetch issued after the demuxer
11
+ * has already touched the URL lands in that same sparse entry and is discarded
12
+ * with it. Bytes held here are held for as long as this session wants them,
13
+ * whatever the cache does.
14
+ */
15
+ /** A held run of the source file, `end` exclusive. */
16
+ export interface ResidentRange {
17
+ readonly start: number;
18
+ readonly end: number;
19
+ }
20
+ export interface ResidencySnapshot {
21
+ /** Held runs, ascending, never touching or overlapping. */
22
+ readonly ranges: readonly ResidentRange[];
23
+ readonly residentBytes: number;
24
+ /** Source length once a response has disclosed it, else null. */
25
+ readonly totalBytes: number | null;
26
+ /** Bytes the background walk has pulled, excluding what playback pulled. */
27
+ readonly prefetchedBytes: number;
28
+ readonly warming: boolean;
29
+ }
30
+ export interface SourceResidencyOptions {
31
+ readonly url: string;
32
+ /** Ceiling on held bytes. Runs furthest from the focus offset are dropped first. */
33
+ readonly budgetBytes: number;
34
+ /** Largest single background request. */
35
+ readonly chunkBytes?: number;
36
+ readonly requestInit?: RequestInit;
37
+ readonly fetchImpl?: typeof fetch;
38
+ }
39
+ export interface SourceResidency {
40
+ /** Hand to mediabunny's `UrlSource` as its `fetchFn`. */
41
+ readonly fetchFn: typeof fetch;
42
+ snapshot(): ResidencySnapshot;
43
+ /**
44
+ * Offset the background walk works outward from, so the bytes nearest the
45
+ * playhead arrive first and a viewer who seeks nearby stops waiting soonest.
46
+ */
47
+ focusAt(offset: number): void;
48
+ startWarming(): void;
49
+ stopWarming(): void;
50
+ dispose(): void;
51
+ }
52
+ /** Granularity at which a streaming read is banked. Mediabunny never asks
53
+ * the network for less than half a mebibyte, so every read it makes banks
54
+ * at least one block. */
55
+ export declare const READ_BLOCK_BYTES: number;
56
+ /** How long a delivered chunk may sit unclaimed before the walk stops deferring
57
+ * to the read that produced it. It measures the gap between a chunk and the
58
+ * consumer's next pull, not the wait for the server, so a slow link reads as
59
+ * a live read while a reader that walked away frees the link. */
60
+ export declare const FOREGROUND_STALL_MS = 250;
61
+ /**
62
+ * Bytes held in one buffer per contiguous run, so a lookup is a single scan and
63
+ * a served range is a subarray with nothing copied. Insert is where copying
64
+ * happens: a run keeps spare capacity so an append that extends it writes into
65
+ * that room, and only a backward or overlapping insert re-materializes the run.
66
+ */
67
+ export declare class ByteStore {
68
+ #private;
69
+ get residentBytes(): number;
70
+ /** Bytes retained by backing arrays, including append headroom. */
71
+ get allocatedBytes(): number;
72
+ ranges(): ResidentRange[];
73
+ /** The held run from `offset` to the end of its run, or null. */
74
+ runAt(offset: number): Uint8Array | null;
75
+ /** Where the gap containing `offset` ends, or null when nothing is held past it. */
76
+ nextHeldStart(offset: number): number | null;
77
+ insert(start: number, bytes: Uint8Array): void;
78
+ /**
79
+ * Brings held bytes under the budget: whole runs go first, furthest from
80
+ * `focus`, and a lone run that has outgrown the budget is narrowed to a
81
+ * `budgetBytes` window at `focus`.
82
+ */
83
+ evictTo(budgetBytes: number, focus: number): void;
84
+ clear(): void;
85
+ }
86
+ /**
87
+ * Outstanding claims on the link, for this module's own tests: they drive every
88
+ * way a read can end and assert the count comes back to zero.
89
+ */
90
+ export declare function outstandingForegroundHolds(residency: SourceResidency): number;
91
+ export declare function createSourceResidency(options: SourceResidencyOptions): SourceResidency;
@@ -0,0 +1,163 @@
1
+ import type { FrameQuality } from "./scrub-cursor";
2
+ import { type DiagnosticsSnapshot, type Warning } from "./diagnostics";
3
+ /**
4
+ * Worker-realm trace recorder. Two fixed-capacity ring buffers capture a rolling
5
+ * window of what the runtime did: an EVENT ring (paints, scrubs, seeks, and
6
+ * status transitions, the moments EngineCore drives directly) and a SNAPSHOT
7
+ * ring (the same DiagnosticsSnapshot references already assembled for the
8
+ * broadcast). Both overwrite oldest in place, so a long session has a constant,
9
+ * predictable footprint and the appends are O(1) with no growth or GC churn.
10
+ *
11
+ * Stall and decode-spike depth are not separate events: stalls ride the snapshot
12
+ * ring (realtime.stalls), and per-decode latency rides the scrub stats, so the
13
+ * event ring stays to the moments the engine itself initiates.
14
+ *
15
+ * Allocated on arm, freed on disarm or export, so a disarmed recorder costs
16
+ * zero memory. assemble() walks both rings ascending into one self-describing
17
+ * versioned JSON document an agent or a human can read cold: the summary carries
18
+ * the exact Warning diagnoses the UI shows, so the trace explains itself.
19
+ *
20
+ * Pure data. It never touches decode, the sink, or the cursor.
21
+ */
22
+ export declare const TRACE_SCHEMA = "roboflow.videoEngine.trace/2";
23
+ /** One captured runtime event. type is the moment ("paint", "scrub", "seek",
24
+ * "status"); tMs is worker-clock relative to arm; the rest are optional payload
25
+ * fields the relevant moment fills. All plain data. */
26
+ export interface TraceEvent {
27
+ readonly type: "paint" | "scrub" | "seek" | "status" | "rate";
28
+ readonly tMs: number;
29
+ readonly mediaTimeMs?: number;
30
+ readonly paintSeq?: number;
31
+ /** paint: which frame of the source these pixels are, by the engine's frame
32
+ * table. The snapshot ring samples at 10Hz and paints outrun it, so without
33
+ * this the frame most paints put up is named nowhere in the trace. */
34
+ readonly frameIndex?: number;
35
+ readonly catchUpMs?: number;
36
+ /** paint: whether a full decode or the coarse stand-in reached the canvas.
37
+ * Paints come from sources with materially different pixel quality, so a
38
+ * trace without this cannot tell a crisp frame from a blurry one. */
39
+ readonly quality?: FrameQuality;
40
+ /** seek/scrub: the target the gesture aimed at, and where it landed.
41
+ * paint: the position the paint was serving (the engine clock at paint). */
42
+ readonly targetMs?: number;
43
+ readonly landedMs?: number;
44
+ /** seek: whether it was a key-only navigation seek. */
45
+ readonly keyOnly?: boolean;
46
+ /** status: the new playback status string. */
47
+ readonly status?: string;
48
+ /** rate: the new playback rate. Every paint after it is served by a clock
49
+ * running at a different slope, which the timestamps alone do not show. */
50
+ readonly rate?: number;
51
+ }
52
+ /** A broadcast snapshot with the moment it was taken, in the same worker-clock
53
+ * milliseconds since arm that every event carries, so the two series line up. */
54
+ export type TracedSnapshot = DiagnosticsSnapshot & {
55
+ readonly tMs: number;
56
+ };
57
+ /** Environment facts captured once at export, so a trace read elsewhere carries
58
+ * the device context that shaped the numbers. */
59
+ export interface TraceEnvironment {
60
+ readonly userAgent: string;
61
+ readonly webgpuAvailable: boolean;
62
+ /** Null when nobody measured one: the engine has no window to read it off, so
63
+ * it knows the ratio only when a host hands it a display box or a canvas. */
64
+ readonly devicePixelRatio: number | null;
65
+ readonly hardwareConcurrency: number;
66
+ }
67
+ /** Rolled-up totals for a glanceable read, with the same Warning diagnoses the
68
+ * UI shows so the trace is self-explaining. effectivePaintFps is measured across
69
+ * every snapshot pushed since arm, so it covers more than the snapshots the ring
70
+ * still holds; no snapshot carries such a reading. cacheHitRatePct is null until
71
+ * something looks the cache up, and cacheLookups is its denominator, so a reader
72
+ * can tell a cold cache from a failing one. */
73
+ export interface TraceSummary {
74
+ readonly scrubP50Ms: number;
75
+ readonly scrubP95Ms: number;
76
+ readonly scrubMaxMs: number;
77
+ readonly effectivePaintFps: number | null;
78
+ readonly lateFrames: number;
79
+ readonly stalls: number;
80
+ readonly maxCatchUpMs: number;
81
+ readonly cacheHitRatePct: number | null;
82
+ readonly cacheLookups: number;
83
+ readonly exactEvictions: number;
84
+ readonly warnings: Warning[];
85
+ }
86
+ /**
87
+ * How much of the capture one ring can still answer for. A capture older than a
88
+ * ring's capacity keeps only its tail, so durationMs describes the session and
89
+ * this describes the evidence: coveredMs runs from the oldest entry still held,
90
+ * and dropped counts what was overwritten to make room.
91
+ */
92
+ export interface TraceRingCoverage {
93
+ readonly capacity: number;
94
+ readonly retained: number;
95
+ readonly dropped: number;
96
+ readonly oldestTMs: number | null;
97
+ readonly newestTMs: number | null;
98
+ readonly coveredMs: number;
99
+ }
100
+ /** Per-ring coverage. The two rings fill at different rates, so they reach back
101
+ * different distances and neither one is "the captured window". */
102
+ export interface TraceCoverage {
103
+ readonly events: TraceRingCoverage;
104
+ readonly snapshots: TraceRingCoverage;
105
+ }
106
+ /** The assembled trace document. Versioned by schema so a reader knows the shape. */
107
+ export interface EngineTrace {
108
+ readonly schema: typeof TRACE_SCHEMA;
109
+ readonly capturedAt: number;
110
+ readonly armOriginMs: number;
111
+ readonly durationMs: number;
112
+ readonly coverage: TraceCoverage;
113
+ /** Why the capture stopped early, or null when a reader closed it. */
114
+ readonly truncatedReason: string | null;
115
+ readonly environment: TraceEnvironment;
116
+ readonly events: TraceEvent[];
117
+ readonly snapshots: TracedSnapshot[];
118
+ readonly summary: TraceSummary;
119
+ }
120
+ export interface TraceRecorderEnvironment {
121
+ readonly userAgent: string;
122
+ readonly webgpuAvailable: boolean;
123
+ readonly devicePixelRatio: number | null;
124
+ readonly hardwareConcurrency: number;
125
+ }
126
+ export declare class TraceRecorder {
127
+ private readonly env;
128
+ private readonly now;
129
+ private readonly events;
130
+ private readonly snapshots;
131
+ private readonly armOriginMs;
132
+ private paintedFrames;
133
+ private playingMs;
134
+ private lastSnapshotAtMs;
135
+ private lastPaints;
136
+ private lastSnapshotPlaying;
137
+ private truncatedReason;
138
+ constructor(env: TraceRecorderEnvironment, now?: () => number, eventCap?: number, snapshotCap?: number);
139
+ pushEvent(event: TraceEvent): void;
140
+ /**
141
+ * Stamps the broadcast snapshot with when it was taken and rings it. The
142
+ * stamp is the whole point of keeping a series: without it the snapshots are
143
+ * an unordered pile with no way to line them up against the events, which
144
+ * carry one, or to tell a freeze from a slow stretch.
145
+ */
146
+ pushSnapshot(snapshot: DiagnosticsSnapshot): void;
147
+ /** Records that the capture was cut short by something other than a reader
148
+ * closing it, so an assembled trace says why it ends where it does. The
149
+ * first reason sticks: it is the one that ended the capture. */
150
+ truncate(reason: string): void;
151
+ /**
152
+ * Paints between two consecutive snapshots over the wall time between them.
153
+ * An interval counts only when playback was up at both ends, since one
154
+ * folded-in pause paints nothing and sinks the rate far below what the user
155
+ * watched. The paint counter restarts with each play session, so a negative
156
+ * delta means a new session began and that interval is dropped.
157
+ */
158
+ private accumulatePaintRate;
159
+ /** Worker-clock milliseconds since arm; the stamp every pushed event uses. */
160
+ elapsedMs(): number;
161
+ assemble(): EngineTrace;
162
+ private summarize;
163
+ }