nybls 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,27 @@
1
+ {
2
+ "name": "nybls",
3
+ "owner": {
4
+ "name": "Ares",
5
+ "url": "https://github.com/nKOxxx"
6
+ },
7
+ "plugins": [
8
+ {
9
+ "name": "nybls",
10
+ "source": "./",
11
+ "description": "Watch videos properly: transcript first, contact sheets for coverage, then only the frames the model names a gap for \u2014 with a spend ledger and evidence receipts.",
12
+ "version": "0.1.0",
13
+ "license": "MIT",
14
+ "keywords": [
15
+ "video",
16
+ "youtube",
17
+ "vision",
18
+ "frames",
19
+ "transcript"
20
+ ],
21
+ "homepage": "https://github.com/nKOxxx/nybls"
22
+ }
23
+ ],
24
+ "metadata": {
25
+ "description": "Budgeted, iterative video watching for AI agents \u2014 sip the video, don't slurp it."
26
+ }
27
+ }
@@ -0,0 +1,11 @@
1
+ {
2
+ "name": "nybls",
3
+ "displayName": "nybls",
4
+ "version": "0.1.0",
5
+ "description": "Budgeted, iterative video watching for AI agents — transcript first, then only the frames the model asks for, with receipts.",
6
+ "author": { "name": "Ares", "url": "https://github.com/nKOxxx" },
7
+ "homepage": "https://github.com/nKOxxx/nybls",
8
+ "repository": "https://github.com/nKOxxx/nybls",
9
+ "license": "MIT",
10
+ "keywords": ["video", "youtube", "vision", "frames", "transcript", "analysis"]
11
+ }
nybls-0.1.0/.gitignore ADDED
@@ -0,0 +1,8 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.pyc
4
+ tests/sample/
5
+ brand/nybls_60.png
6
+ brand/nybls_180.png
7
+ .DS_Store
8
+ .env
nybls-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ares
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
nybls-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,242 @@
1
+ Metadata-Version: 2.5
2
+ Name: nybls
3
+ Version: 0.1.0
4
+ Summary: Budgeted, iterative video watching for AI agents — transcript first, then only the frames the model asks for, with receipts.
5
+ Project-URL: Homepage, https://github.com/nKOxxx/nybls
6
+ Project-URL: Repository, https://github.com/nKOxxx/nybls
7
+ Project-URL: Issues, https://github.com/nKOxxx/nybls/issues
8
+ Author: Ares
9
+ License: MIT License
10
+
11
+ Copyright (c) 2026 Ares
12
+
13
+ Permission is hereby granted, free of charge, to any person obtaining a copy
14
+ of this software and associated documentation files (the "Software"), to deal
15
+ in the Software without restriction, including without limitation the rights
16
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17
+ copies of the Software, and to permit persons to whom the Software is
18
+ furnished to do so, subject to the following conditions:
19
+
20
+ The above copyright notice and this permission notice shall be included in all
21
+ copies or substantial portions of the Software.
22
+
23
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29
+ SOFTWARE.
30
+ License-File: LICENSE
31
+ Keywords: agent,claude,ffmpeg,llm,mcp,transcript,video,vision
32
+ Classifier: Development Status :: 3 - Alpha
33
+ Classifier: Environment :: Console
34
+ Classifier: Intended Audience :: Developers
35
+ Classifier: License :: OSI Approved :: MIT License
36
+ Classifier: Programming Language :: Python :: 3.10
37
+ Classifier: Programming Language :: Python :: 3.11
38
+ Classifier: Programming Language :: Python :: 3.12
39
+ Classifier: Topic :: Multimedia :: Video
40
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
41
+ Requires-Python: >=3.10
42
+ Requires-Dist: imagehash>=4.3.2
43
+ Requires-Dist: pillow>=10.0
44
+ Requires-Dist: scenedetect>=0.7.1
45
+ Provides-Extra: dev
46
+ Requires-Dist: pytest>=8.0; extra == 'dev'
47
+ Provides-Extra: macos
48
+ Requires-Dist: ocrmac>=1.0; extra == 'macos'
49
+ Description-Content-Type: text/markdown
50
+
51
+ <p align="center">
52
+ <img src="brand/nybls_1024.png" alt="nybls" width="140">
53
+ </p>
54
+
55
+ <h1 align="center">nybls</h1>
56
+
57
+ <p align="center">
58
+ <em>Other tools slurp the whole video. This one sips —<br>
59
+ three frames out of fifty-three thousand, and it shows you which.</em>
60
+ </p>
61
+
62
+ ---
63
+
64
+ ## The problem
65
+
66
+ Almost every "AI watches YouTube" tool reads the **subtitles**. That is not a
67
+ cynical read of the field, it is what the code does: the YouTube loaders in the
68
+ major agent frameworks and self-hosted chat apps are transcript scrapers, and
69
+ Google's own documentation says a YouTube source in NotebookLM imports "only the
70
+ text transcript of the video."
71
+
72
+ So a video with no speech returns nothing. A chart on screen is invisible. A
73
+ demo that shows rather than tells is invisible. And you cannot tell the
74
+ difference between a tool that looked and a tool that guessed, because neither
75
+ shows you what it saw.
76
+
77
+ The other half of the field does the opposite and dumps 100 uniformly-sampled
78
+ frames into the model at once — expensive, mostly redundant, and still blind to
79
+ whatever fell between the samples.
80
+
81
+ ## What nybls does instead
82
+
83
+ nybls gives an AI agent five ways to ask for pieces of a video, and a protocol
84
+ that makes it ask well:
85
+
86
+ 1. **Read the free stuff first** — transcript, scene boundaries, metadata. Zero images.
87
+ 2. **Look broadly, cheaply** — one contact sheet is six timestamped thumbnails for
88
+ the cost of a single image.
89
+ 3. **Then say what it is missing** — the agent must name the time interval and what
90
+ it expects to find there before it may request more.
91
+ 4. **Drill in only there** — full-resolution frames, or a zoom crop to read small text.
92
+ 5. **Stop, and show the receipts** — every answer ends with the frames it examined
93
+ and what they cost.
94
+
95
+ Real run, this repo's test corpus — a 35-minute car-restoration video:
96
+
97
+ ```
98
+ watched 35 min · examined 3 images (~3,069 visual tokens, ~$0.01)
99
+ of ~53,171 total frames · budget 3/142 units
100
+ ```
101
+
102
+ Three frames. It found the reveal at 12:05, identified that the second half of
103
+ the video switches to a different car, and reported honestly that a zoom aimed
104
+ at a badge landed on a headlight instead — so it did not name the model.
105
+
106
+ ## Why "sips" is the whole point
107
+
108
+ The research this is built on is unambiguous: **selecting fewer frames well beats
109
+ sampling many frames blindly.** Stanford's VideoAgent matched the accuracy of 180
110
+ uniformly-sampled frames using 8.4 frames chosen by a confidence-gated loop.
111
+ Adaptive keyframe selection beats uniform sampling at equal frame count. Grids
112
+ larger than about six tiles measurably destroy a model's ability to localize
113
+ detail. Every rule in the nybls protocol traces to a published result — see
114
+ [docs/RESEARCH.md](docs/RESEARCH.md).
115
+
116
+ ## Install
117
+
118
+ **Prerequisites** (nybls orchestrates these; it does not vendor them):
119
+
120
+ | Tool | Why | Install |
121
+ |---|---|---|
122
+ | `ffmpeg` | frame extraction, crops, contact sheets | `brew install ffmpeg` |
123
+ | `yt-dlp` | downloading from YouTube and ~1,800 other sites | `brew install yt-dlp` |
124
+ | `deno` | required by yt-dlp for full YouTube support | `brew install deno` |
125
+ | `whisper-cpp` | local transcription when a video has no captions | `brew install whisper-cpp` |
126
+
127
+ Then:
128
+
129
+ ```bash
130
+ pip install nybls
131
+ ```
132
+
133
+ On macOS, add Apple Vision OCR (works with Arabic, no extra models):
134
+
135
+ ```bash
136
+ pip install "nybls[macos]"
137
+ ```
138
+
139
+ For local transcription, download a Whisper model once:
140
+
141
+ ```bash
142
+ mkdir -p ~/.nybls/models && curl -L -o ~/.nybls/models/ggml-large-v3-turbo.bin \
143
+ https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin
144
+ ```
145
+
146
+ ## Use it
147
+
148
+ ```bash
149
+ nybls probe "https://www.youtube.com/watch?v=VIDEO_ID"
150
+ ```
151
+
152
+ ```
153
+ id: VIDEO_ID
154
+ title: But what is a neural network? | Deep learning chapter 1
155
+ duration: 18.7 min · 1280x720 · 24 scenes
156
+ transcript: subtitles:video.en.vtt -> ~/.nybls/store/VIDEO_ID/transcript.txt
157
+ budget: 75 image units
158
+ cost so far: 0 images. Read the transcript first; then request sheets.
159
+ ```
160
+
161
+ | Command | What it does | Cost |
162
+ |---|---|---|
163
+ | `nybls probe <url\|file>` | download, transcribe, detect scenes, set the budget | 0 images |
164
+ | `nybls sheet <id> [--range S E]` | 3x2 contact sheet, timestamped | 1 image |
165
+ | `nybls frames <id> --at 92,570` | full-resolution stills | 1 per frame |
166
+ | `nybls zoom <id> --at 570 --box x,y,w,h` | crop into a region to read detail | 1 image |
167
+ | `nybls ledger <id>` | what has been spent | 0 |
168
+
169
+ Everything is written to `~/.nybls/store/<id>/` as ordinary PNG and text files.
170
+ There is no API integration: your agent reads the files.
171
+
172
+ ### With Claude Code
173
+
174
+ Install as a plugin — the skill and its protocol come with it:
175
+
176
+ ```
177
+ /plugin marketplace add nKOxxx/nybls
178
+ ```
179
+
180
+ ```
181
+ /plugin install nybls@nybls
182
+ ```
183
+
184
+ Then ask:
185
+
186
+ ```
187
+ /nybls:watch https://www.youtube.com/watch?v=VIDEO_ID what does the chart at the end show?
188
+ ```
189
+
190
+ The skill carries the protocol — the confidence check, the named-gap rule, the
191
+ stop conditions, and the receipts contract — so the discipline travels with the
192
+ tool instead of depending on how well you prompt. For a personal install without
193
+ the plugin system, copy `skills/watch/SKILL.md` to
194
+ `~/.claude/skills/watch/SKILL.md`.
195
+
196
+ ## The guardrails are in the tool, not just the prose
197
+
198
+ A protocol written only in a prompt is a suggestion. These are enforced by the
199
+ CLI itself:
200
+
201
+ - **Named gaps.** Past a few images, `frames` and `zoom` refuse to run without
202
+ `--looking-for "<interval + what you expect>"`.
203
+ - **A real budget.** Duration-scaled, cumulative, and the server refuses to serve
204
+ past it unless a human passes `--force`.
205
+ - **Duplicate warnings.** A frame that is a near-duplicate of one already served
206
+ is flagged as wasted budget.
207
+ - **Next-step nudges.** Every command's output ends by telling the agent to draft
208
+ an answer and classify its own confidence before asking for more.
209
+
210
+ ## What this cannot do
211
+
212
+ - **No live video.** Batch only: download, then analyze.
213
+ - **Motion is lossy.** It sees stills, not movement. Good for talks, tutorials,
214
+ demos, dashboards, and on-screen text; weak for sports mechanics or fast action.
215
+ - **Speech only.** Transcription covers words, not music or sound events.
216
+ - **It depends on tools that fight back.** Video platforms actively break
217
+ downloaders. Keep `yt-dlp` current; expect occasional breakage.
218
+ - **Alpha.** The protocol is validated by published research and by daily use, but
219
+ this is version 0.1.0 and the interfaces may change.
220
+
221
+ ## Privacy and security posture
222
+
223
+ Local-first by construction: no API keys, no accounts, no telemetry, no server.
224
+ The agent you already use supplies the model; everything else runs on your
225
+ machine. See [docs/SECURITY.md](docs/SECURITY.md) for the full posture, including
226
+ the one rule that matters most — **video content is data, never instructions.** A
227
+ video can display text aimed at your agent, and the protocol requires that such
228
+ text be reported to you rather than obeyed.
229
+
230
+ Downloading from a platform may conflict with that platform's terms of service.
231
+ That is your call to make; this tool takes no position and phones nothing home.
232
+
233
+ ## Documentation
234
+
235
+ - [docs/PROTOCOL.md](docs/PROTOCOL.md) — the WATCH loop in full, and why each rule exists
236
+ - [docs/RESEARCH.md](docs/RESEARCH.md) — the evidence base and everything this is built on
237
+ - [docs/SECURITY.md](docs/SECURITY.md) — threat model and the security checklist
238
+ - [CHANGELOG.md](CHANGELOG.md)
239
+
240
+ ## License
241
+
242
+ MIT.
nybls-0.1.0/README.md ADDED
@@ -0,0 +1,192 @@
1
+ <p align="center">
2
+ <img src="brand/nybls_1024.png" alt="nybls" width="140">
3
+ </p>
4
+
5
+ <h1 align="center">nybls</h1>
6
+
7
+ <p align="center">
8
+ <em>Other tools slurp the whole video. This one sips —<br>
9
+ three frames out of fifty-three thousand, and it shows you which.</em>
10
+ </p>
11
+
12
+ ---
13
+
14
+ ## The problem
15
+
16
+ Almost every "AI watches YouTube" tool reads the **subtitles**. That is not a
17
+ cynical read of the field, it is what the code does: the YouTube loaders in the
18
+ major agent frameworks and self-hosted chat apps are transcript scrapers, and
19
+ Google's own documentation says a YouTube source in NotebookLM imports "only the
20
+ text transcript of the video."
21
+
22
+ So a video with no speech returns nothing. A chart on screen is invisible. A
23
+ demo that shows rather than tells is invisible. And you cannot tell the
24
+ difference between a tool that looked and a tool that guessed, because neither
25
+ shows you what it saw.
26
+
27
+ The other half of the field does the opposite and dumps 100 uniformly-sampled
28
+ frames into the model at once — expensive, mostly redundant, and still blind to
29
+ whatever fell between the samples.
30
+
31
+ ## What nybls does instead
32
+
33
+ nybls gives an AI agent five ways to ask for pieces of a video, and a protocol
34
+ that makes it ask well:
35
+
36
+ 1. **Read the free stuff first** — transcript, scene boundaries, metadata. Zero images.
37
+ 2. **Look broadly, cheaply** — one contact sheet is six timestamped thumbnails for
38
+ the cost of a single image.
39
+ 3. **Then say what it is missing** — the agent must name the time interval and what
40
+ it expects to find there before it may request more.
41
+ 4. **Drill in only there** — full-resolution frames, or a zoom crop to read small text.
42
+ 5. **Stop, and show the receipts** — every answer ends with the frames it examined
43
+ and what they cost.
44
+
45
+ Real run, this repo's test corpus — a 35-minute car-restoration video:
46
+
47
+ ```
48
+ watched 35 min · examined 3 images (~3,069 visual tokens, ~$0.01)
49
+ of ~53,171 total frames · budget 3/142 units
50
+ ```
51
+
52
+ Three frames. It found the reveal at 12:05, identified that the second half of
53
+ the video switches to a different car, and reported honestly that a zoom aimed
54
+ at a badge landed on a headlight instead — so it did not name the model.
55
+
56
+ ## Why "sips" is the whole point
57
+
58
+ The research this is built on is unambiguous: **selecting fewer frames well beats
59
+ sampling many frames blindly.** Stanford's VideoAgent matched the accuracy of 180
60
+ uniformly-sampled frames using 8.4 frames chosen by a confidence-gated loop.
61
+ Adaptive keyframe selection beats uniform sampling at equal frame count. Grids
62
+ larger than about six tiles measurably destroy a model's ability to localize
63
+ detail. Every rule in the nybls protocol traces to a published result — see
64
+ [docs/RESEARCH.md](docs/RESEARCH.md).
65
+
66
+ ## Install
67
+
68
+ **Prerequisites** (nybls orchestrates these; it does not vendor them):
69
+
70
+ | Tool | Why | Install |
71
+ |---|---|---|
72
+ | `ffmpeg` | frame extraction, crops, contact sheets | `brew install ffmpeg` |
73
+ | `yt-dlp` | downloading from YouTube and ~1,800 other sites | `brew install yt-dlp` |
74
+ | `deno` | required by yt-dlp for full YouTube support | `brew install deno` |
75
+ | `whisper-cpp` | local transcription when a video has no captions | `brew install whisper-cpp` |
76
+
77
+ Then:
78
+
79
+ ```bash
80
+ pip install nybls
81
+ ```
82
+
83
+ On macOS, add Apple Vision OCR (works with Arabic, no extra models):
84
+
85
+ ```bash
86
+ pip install "nybls[macos]"
87
+ ```
88
+
89
+ For local transcription, download a Whisper model once:
90
+
91
+ ```bash
92
+ mkdir -p ~/.nybls/models && curl -L -o ~/.nybls/models/ggml-large-v3-turbo.bin \
93
+ https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin
94
+ ```
95
+
96
+ ## Use it
97
+
98
+ ```bash
99
+ nybls probe "https://www.youtube.com/watch?v=VIDEO_ID"
100
+ ```
101
+
102
+ ```
103
+ id: VIDEO_ID
104
+ title: But what is a neural network? | Deep learning chapter 1
105
+ duration: 18.7 min · 1280x720 · 24 scenes
106
+ transcript: subtitles:video.en.vtt -> ~/.nybls/store/VIDEO_ID/transcript.txt
107
+ budget: 75 image units
108
+ cost so far: 0 images. Read the transcript first; then request sheets.
109
+ ```
110
+
111
+ | Command | What it does | Cost |
112
+ |---|---|---|
113
+ | `nybls probe <url\|file>` | download, transcribe, detect scenes, set the budget | 0 images |
114
+ | `nybls sheet <id> [--range S E]` | 3x2 contact sheet, timestamped | 1 image |
115
+ | `nybls frames <id> --at 92,570` | full-resolution stills | 1 per frame |
116
+ | `nybls zoom <id> --at 570 --box x,y,w,h` | crop into a region to read detail | 1 image |
117
+ | `nybls ledger <id>` | what has been spent | 0 |
118
+
119
+ Everything is written to `~/.nybls/store/<id>/` as ordinary PNG and text files.
120
+ There is no API integration: your agent reads the files.
121
+
122
+ ### With Claude Code
123
+
124
+ Install as a plugin — the skill and its protocol come with it:
125
+
126
+ ```
127
+ /plugin marketplace add nKOxxx/nybls
128
+ ```
129
+
130
+ ```
131
+ /plugin install nybls@nybls
132
+ ```
133
+
134
+ Then ask:
135
+
136
+ ```
137
+ /nybls:watch https://www.youtube.com/watch?v=VIDEO_ID what does the chart at the end show?
138
+ ```
139
+
140
+ The skill carries the protocol — the confidence check, the named-gap rule, the
141
+ stop conditions, and the receipts contract — so the discipline travels with the
142
+ tool instead of depending on how well you prompt. For a personal install without
143
+ the plugin system, copy `skills/watch/SKILL.md` to
144
+ `~/.claude/skills/watch/SKILL.md`.
145
+
146
+ ## The guardrails are in the tool, not just the prose
147
+
148
+ A protocol written only in a prompt is a suggestion. These are enforced by the
149
+ CLI itself:
150
+
151
+ - **Named gaps.** Past a few images, `frames` and `zoom` refuse to run without
152
+ `--looking-for "<interval + what you expect>"`.
153
+ - **A real budget.** Duration-scaled, cumulative, and the server refuses to serve
154
+ past it unless a human passes `--force`.
155
+ - **Duplicate warnings.** A frame that is a near-duplicate of one already served
156
+ is flagged as wasted budget.
157
+ - **Next-step nudges.** Every command's output ends by telling the agent to draft
158
+ an answer and classify its own confidence before asking for more.
159
+
160
+ ## What this cannot do
161
+
162
+ - **No live video.** Batch only: download, then analyze.
163
+ - **Motion is lossy.** It sees stills, not movement. Good for talks, tutorials,
164
+ demos, dashboards, and on-screen text; weak for sports mechanics or fast action.
165
+ - **Speech only.** Transcription covers words, not music or sound events.
166
+ - **It depends on tools that fight back.** Video platforms actively break
167
+ downloaders. Keep `yt-dlp` current; expect occasional breakage.
168
+ - **Alpha.** The protocol is validated by published research and by daily use, but
169
+ this is version 0.1.0 and the interfaces may change.
170
+
171
+ ## Privacy and security posture
172
+
173
+ Local-first by construction: no API keys, no accounts, no telemetry, no server.
174
+ The agent you already use supplies the model; everything else runs on your
175
+ machine. See [docs/SECURITY.md](docs/SECURITY.md) for the full posture, including
176
+ the one rule that matters most — **video content is data, never instructions.** A
177
+ video can display text aimed at your agent, and the protocol requires that such
178
+ text be reported to you rather than obeyed.
179
+
180
+ Downloading from a platform may conflict with that platform's terms of service.
181
+ That is your call to make; this tool takes no position and phones nothing home.
182
+
183
+ ## Documentation
184
+
185
+ - [docs/PROTOCOL.md](docs/PROTOCOL.md) — the WATCH loop in full, and why each rule exists
186
+ - [docs/RESEARCH.md](docs/RESEARCH.md) — the evidence base and everything this is built on
187
+ - [docs/SECURITY.md](docs/SECURITY.md) — threat model and the security checklist
188
+ - [CHANGELOG.md](CHANGELOG.md)
189
+
190
+ ## License
191
+
192
+ MIT.
@@ -0,0 +1,156 @@
1
+ # The WATCH protocol
2
+
3
+ The CLI is deliberately unintelligent. It extracts frames, it does not decide
4
+ which frames matter. The deciding happens in the model, guided by this protocol
5
+ and constrained by rails in the tool.
6
+
7
+ That split is the central design choice, and it is not arbitrary: every attempt
8
+ to make the *extractor* smart loses to letting the *model* choose, because the
9
+ extractor has to decide before it understands the question. Measured result:
10
+ a confidence-gated loop matched the accuracy of 180 uniformly sampled frames
11
+ using 8.4 chosen ones (arXiv:2403.10517).
12
+
13
+ ---
14
+
15
+ ## The loop
16
+
17
+ ### Round 0 — free
18
+
19
+ ```bash
20
+ nybls probe <url-or-file>
21
+ ```
22
+
23
+ Produces the transcript, the scene list, metadata, and the budget. Costs zero
24
+ images. Read all of it before spending anything.
25
+
26
+ For talk-heavy video this round frequently answers the question outright. It
27
+ must never be used to answer a *visual* question — if the user asks what is on
28
+ screen, the transcript is evidence about the audio, not the picture.
29
+
30
+ ### Round 1 — coverage
31
+
32
+ ```bash
33
+ nybls sheet <id> [--range START_S END_S]
34
+ ```
35
+
36
+ One sheet is six timestamped thumbnails in a 3x2 grid: six moments for the price
37
+ of one image. Use `--range` to cover a specific region of a long video; for a
38
+ 20-minute video, one sheet per five to seven minutes of interest is a sane
39
+ budget.
40
+
41
+ **Six tiles, not more.** IG-VLM ablated 4 to 20 tiles per grid and found six in a
42
+ near-square layout optimal, with larger grids performing worse (arXiv:2403.18406).
43
+ Needle-in-a-haystack testing shows every current model — Claude included — loses
44
+ the ability to localize detail inside dense sub-image grids (arXiv:2406.11230).
45
+ Sheets tell you *where to look*. They are not for reading.
46
+
47
+ ### The evaluator — mandatory, every round
48
+
49
+ Draft the answer. Then classify, explicitly:
50
+
51
+ | Verdict | Meaning | Action |
52
+ |---|---|---|
53
+ | `sufficient` | the evidence supports the answer | stop, write the output |
54
+ | `partial` | the shape is right, detail is missing | name the gap, request only that |
55
+ | `insufficient` | cannot answer | name the gap, request only that |
56
+
57
+ This step is not decoration. In component ablations of adaptive video agents, the
58
+ "can I answer yet?" evaluator was the single highest-impact part of the system —
59
+ removing it hurt accuracy more than removing the smart sampler (arXiv:2410.20252).
60
+ Interval-level model confidence also correlates with correctness, which is what
61
+ makes confidence-gated search work (arXiv:2507.02946).
62
+
63
+ ### Rounds 2–3 — targeted
64
+
65
+ ```bash
66
+ nybls frames <id> --at 92,570 --looking-for "09:30, the chart he refers to"
67
+ nybls zoom <id> --at 570 --box 0.3,0.2,0.4,0.4 --looking-for "axis labels"
68
+ ```
69
+
70
+ Request frames for a **named** interval and a **named** expectation. Never
71
+ re-sample globally; that is what Round 1 was for.
72
+
73
+ Prefer a zoom over more frames when the answer is inside one frame. Adding
74
+ spatial zoom to a temporal search gained 2.6 to 5.9 accuracy points under an
75
+ identical frame budget (arXiv:2504.02259, building on arXiv:2312.14135).
76
+
77
+ ### Stop
78
+
79
+ Stop at the first of:
80
+
81
+ - confidence is `sufficient`
82
+ - three rounds after probe — accuracy saturates there (arXiv:2403.10517)
83
+ - the budget is exhausted
84
+
85
+ If you stop at `partial`, **say so in the output.** A confident answer built on
86
+ insufficient evidence is the failure mode this whole design exists to prevent.
87
+
88
+ ---
89
+
90
+ ## Budget
91
+
92
+ ```
93
+ budget = clamp(8, ceil(duration_minutes * 4), 200) # image units
94
+ ```
95
+
96
+ One sheet, one full frame, and one zoom each cost one unit. The ledger also
97
+ tracks estimated visual tokens, computed the way the Claude API bills them
98
+ (ceil(w/28) * ceil(h/28) per image).
99
+
100
+ Duration-scaled budgets with a floor and ceiling follow the pattern in
101
+ arXiv:2510.04428. The ceiling matters more than the floor: past roughly 32
102
+ well-chosen frames, video QA accuracy plateaus (arXiv:2502.19680), so a large
103
+ budget is permission to look carefully, not an instruction to look often.
104
+
105
+ Sampling density is genuinely task-dependent — some questions are answerable at
106
+ one frame per minute, others need one frame per second in a narrow window
107
+ (arXiv:2503.12496). Coarse first, dense only where the question points.
108
+
109
+ ---
110
+
111
+ ## Rails: what the tool enforces
112
+
113
+ Prose can be ignored. These cannot:
114
+
115
+ | Rail | Behavior |
116
+ |---|---|
117
+ | Named gaps | past 3 spent images, `frames`/`zoom` refuse without `--looking-for` |
118
+ | Budget stop | requests beyond the budget are refused; only a human may `--force` |
119
+ | Duplicate detection | a near-duplicate of an already-served frame is flagged as waste |
120
+ | Per-call cap | at most 10 frames per invocation |
121
+ | Next-step nudges | every command's output restates the next protocol step |
122
+
123
+ The rails exist because the protocol must work for an agent that was not
124
+ carefully prompted. Discipline that depends on the operator does not survive
125
+ distribution.
126
+
127
+ ---
128
+
129
+ ## Output contract
130
+
131
+ Every answer ends with three blocks:
132
+
133
+ 1. **Answer** — grounded in what was seen and heard, with inline `[mm:ss]` citations.
134
+ 2. **Evidence strip** — every image examined, and what it showed.
135
+ 3. **Ledger** — verbatim output of `nybls ledger <id>`.
136
+
137
+ The evidence strip is what separates a claim from an assertion. It is also the
138
+ product's most visible difference: a transcript-only tool cannot produce one.
139
+
140
+ ---
141
+
142
+ ## Worked example
143
+
144
+ A 35-minute video, question: what happens in it?
145
+
146
+ | Round | Action | Spend | Outcome |
147
+ |---|---|---|---|
148
+ | 0 | probe, read transcript | 0 | narrative clear; the visual payoff is not |
149
+ | 1 | one whole-video sheet | 1 | six moments; the second half is a *different car* |
150
+ | — | evaluator | — | `partial` — the titular reveal is not in these six |
151
+ | 2 | sheet of the final 5.5 minutes | 1 | not there either; the reveal is earlier |
152
+ | 3 | zoom on a badge | 1 | landed on a headlight — reported honestly, model not named |
153
+
154
+ Final: 3 images, ~3,069 visual tokens, about one cent, against 53,171 frames.
155
+ The reveal was located at 12:05 by combining a Round 1 thumbnail with a free
156
+ transcript read — not by spending more images.