nybls 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nybls-0.1.0/.claude-plugin/marketplace.json +27 -0
- nybls-0.1.0/.claude-plugin/plugin.json +11 -0
- nybls-0.1.0/.gitignore +8 -0
- nybls-0.1.0/LICENSE +21 -0
- nybls-0.1.0/PKG-INFO +242 -0
- nybls-0.1.0/README.md +192 -0
- nybls-0.1.0/docs/PROTOCOL.md +156 -0
- nybls-0.1.0/docs/RESEARCH.md +183 -0
- nybls-0.1.0/docs/SECURITY.md +64 -0
- nybls-0.1.0/nybls_core/__init__.py +0 -0
- nybls-0.1.0/nybls_core/cli.py +277 -0
- nybls-0.1.0/nybls_core/ingest.py +92 -0
- nybls-0.1.0/nybls_core/ledger.py +42 -0
- nybls-0.1.0/nybls_core/media.py +108 -0
- nybls-0.1.0/nybls_core/receiver.py +240 -0
- nybls-0.1.0/nybls_core/store.py +57 -0
- nybls-0.1.0/nybls_core/transcribe.py +76 -0
- nybls-0.1.0/pyproject.toml +47 -0
- nybls-0.1.0/skills/watch/SKILL.md +48 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "nybls",
|
|
3
|
+
"owner": {
|
|
4
|
+
"name": "Ares",
|
|
5
|
+
"url": "https://github.com/nKOxxx"
|
|
6
|
+
},
|
|
7
|
+
"plugins": [
|
|
8
|
+
{
|
|
9
|
+
"name": "nybls",
|
|
10
|
+
"source": "./",
|
|
11
|
+
"description": "Watch videos properly: transcript first, contact sheets for coverage, then only the frames the model names a gap for \u2014 with a spend ledger and evidence receipts.",
|
|
12
|
+
"version": "0.1.0",
|
|
13
|
+
"license": "MIT",
|
|
14
|
+
"keywords": [
|
|
15
|
+
"video",
|
|
16
|
+
"youtube",
|
|
17
|
+
"vision",
|
|
18
|
+
"frames",
|
|
19
|
+
"transcript"
|
|
20
|
+
],
|
|
21
|
+
"homepage": "https://github.com/nKOxxx/nybls"
|
|
22
|
+
}
|
|
23
|
+
],
|
|
24
|
+
"metadata": {
|
|
25
|
+
"description": "Budgeted, iterative video watching for AI agents \u2014 sip the video, don't slurp it."
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "nybls",
|
|
3
|
+
"displayName": "nybls",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"description": "Budgeted, iterative video watching for AI agents — transcript first, then only the frames the model asks for, with receipts.",
|
|
6
|
+
"author": { "name": "Ares", "url": "https://github.com/nKOxxx" },
|
|
7
|
+
"homepage": "https://github.com/nKOxxx/nybls",
|
|
8
|
+
"repository": "https://github.com/nKOxxx/nybls",
|
|
9
|
+
"license": "MIT",
|
|
10
|
+
"keywords": ["video", "youtube", "vision", "frames", "transcript", "analysis"]
|
|
11
|
+
}
|
nybls-0.1.0/.gitignore
ADDED
nybls-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ares
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
nybls-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: nybls
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Budgeted, iterative video watching for AI agents — transcript first, then only the frames the model asks for, with receipts.
|
|
5
|
+
Project-URL: Homepage, https://github.com/nKOxxx/nybls
|
|
6
|
+
Project-URL: Repository, https://github.com/nKOxxx/nybls
|
|
7
|
+
Project-URL: Issues, https://github.com/nKOxxx/nybls/issues
|
|
8
|
+
Author: Ares
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2026 Ares
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Keywords: agent,claude,ffmpeg,llm,mcp,transcript,video,vision
|
|
32
|
+
Classifier: Development Status :: 3 - Alpha
|
|
33
|
+
Classifier: Environment :: Console
|
|
34
|
+
Classifier: Intended Audience :: Developers
|
|
35
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
39
|
+
Classifier: Topic :: Multimedia :: Video
|
|
40
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
41
|
+
Requires-Python: >=3.10
|
|
42
|
+
Requires-Dist: imagehash>=4.3.2
|
|
43
|
+
Requires-Dist: pillow>=10.0
|
|
44
|
+
Requires-Dist: scenedetect>=0.7.1
|
|
45
|
+
Provides-Extra: dev
|
|
46
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
47
|
+
Provides-Extra: macos
|
|
48
|
+
Requires-Dist: ocrmac>=1.0; extra == 'macos'
|
|
49
|
+
Description-Content-Type: text/markdown
|
|
50
|
+
|
|
51
|
+
<p align="center">
|
|
52
|
+
<img src="brand/nybls_1024.png" alt="nybls" width="140">
|
|
53
|
+
</p>
|
|
54
|
+
|
|
55
|
+
<h1 align="center">nybls</h1>
|
|
56
|
+
|
|
57
|
+
<p align="center">
|
|
58
|
+
<em>Other tools slurp the whole video. This one sips —<br>
|
|
59
|
+
three frames out of fifty-three thousand, and it shows you which.</em>
|
|
60
|
+
</p>
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## The problem
|
|
65
|
+
|
|
66
|
+
Almost every "AI watches YouTube" tool reads the **subtitles**. That is not a
|
|
67
|
+
cynical read of the field, it is what the code does: the YouTube loaders in the
|
|
68
|
+
major agent frameworks and self-hosted chat apps are transcript scrapers, and
|
|
69
|
+
Google's own documentation says a YouTube source in NotebookLM imports "only the
|
|
70
|
+
text transcript of the video."
|
|
71
|
+
|
|
72
|
+
So a video with no speech returns nothing. A chart on screen is invisible. A
|
|
73
|
+
demo that shows rather than tells is invisible. And you cannot tell the
|
|
74
|
+
difference between a tool that looked and a tool that guessed, because neither
|
|
75
|
+
shows you what it saw.
|
|
76
|
+
|
|
77
|
+
The other half of the field does the opposite and dumps 100 uniformly-sampled
|
|
78
|
+
frames into the model at once — expensive, mostly redundant, and still blind to
|
|
79
|
+
whatever fell between the samples.
|
|
80
|
+
|
|
81
|
+
## What nybls does instead
|
|
82
|
+
|
|
83
|
+
nybls gives an AI agent five ways to ask for pieces of a video, and a protocol
|
|
84
|
+
that makes it ask well:
|
|
85
|
+
|
|
86
|
+
1. **Read the free stuff first** — transcript, scene boundaries, metadata. Zero images.
|
|
87
|
+
2. **Look broadly, cheaply** — one contact sheet is six timestamped thumbnails for
|
|
88
|
+
the cost of a single image.
|
|
89
|
+
3. **Then say what it is missing** — the agent must name the time interval and what
|
|
90
|
+
it expects to find there before it may request more.
|
|
91
|
+
4. **Drill in only there** — full-resolution frames, or a zoom crop to read small text.
|
|
92
|
+
5. **Stop, and show the receipts** — every answer ends with the frames it examined
|
|
93
|
+
and what they cost.
|
|
94
|
+
|
|
95
|
+
Real run, this repo's test corpus — a 35-minute car-restoration video:
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
watched 35 min · examined 3 images (~3,069 visual tokens, ~$0.01)
|
|
99
|
+
of ~53,171 total frames · budget 3/142 units
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Three frames. It found the reveal at 12:05, identified that the second half of
|
|
103
|
+
the video switches to a different car, and reported honestly that a zoom aimed
|
|
104
|
+
at a badge landed on a headlight instead — so it did not name the model.
|
|
105
|
+
|
|
106
|
+
## Why "sips" is the whole point
|
|
107
|
+
|
|
108
|
+
The research this is built on is unambiguous: **selecting fewer frames well beats
|
|
109
|
+
sampling many frames blindly.** Stanford's VideoAgent matched the accuracy of 180
|
|
110
|
+
uniformly-sampled frames using 8.4 frames chosen by a confidence-gated loop.
|
|
111
|
+
Adaptive keyframe selection beats uniform sampling at equal frame count. Grids
|
|
112
|
+
larger than about six tiles measurably destroy a model's ability to localize
|
|
113
|
+
detail. Every rule in the nybls protocol traces to a published result — see
|
|
114
|
+
[docs/RESEARCH.md](docs/RESEARCH.md).
|
|
115
|
+
|
|
116
|
+
## Install
|
|
117
|
+
|
|
118
|
+
**Prerequisites** (nybls orchestrates these; it does not vendor them):
|
|
119
|
+
|
|
120
|
+
| Tool | Why | Install |
|
|
121
|
+
|---|---|---|
|
|
122
|
+
| `ffmpeg` | frame extraction, crops, contact sheets | `brew install ffmpeg` |
|
|
123
|
+
| `yt-dlp` | downloading from YouTube and ~1,800 other sites | `brew install yt-dlp` |
|
|
124
|
+
| `deno` | required by yt-dlp for full YouTube support | `brew install deno` |
|
|
125
|
+
| `whisper-cpp` | local transcription when a video has no captions | `brew install whisper-cpp` |
|
|
126
|
+
|
|
127
|
+
Then:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
pip install nybls
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
On macOS, add Apple Vision OCR (works with Arabic, no extra models):
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
pip install "nybls[macos]"
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
For local transcription, download a Whisper model once:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
mkdir -p ~/.nybls/models && curl -L -o ~/.nybls/models/ggml-large-v3-turbo.bin \
|
|
143
|
+
https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## Use it
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
nybls probe "https://www.youtube.com/watch?v=VIDEO_ID"
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
id: VIDEO_ID
|
|
154
|
+
title: But what is a neural network? | Deep learning chapter 1
|
|
155
|
+
duration: 18.7 min · 1280x720 · 24 scenes
|
|
156
|
+
transcript: subtitles:video.en.vtt -> ~/.nybls/store/VIDEO_ID/transcript.txt
|
|
157
|
+
budget: 75 image units
|
|
158
|
+
cost so far: 0 images. Read the transcript first; then request sheets.
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
| Command | What it does | Cost |
|
|
162
|
+
|---|---|---|
|
|
163
|
+
| `nybls probe <url\|file>` | download, transcribe, detect scenes, set the budget | 0 images |
|
|
164
|
+
| `nybls sheet <id> [--range S E]` | 3x2 contact sheet, timestamped | 1 image |
|
|
165
|
+
| `nybls frames <id> --at 92,570` | full-resolution stills | 1 per frame |
|
|
166
|
+
| `nybls zoom <id> --at 570 --box x,y,w,h` | crop into a region to read detail | 1 image |
|
|
167
|
+
| `nybls ledger <id>` | what has been spent | 0 |
|
|
168
|
+
|
|
169
|
+
Everything is written to `~/.nybls/store/<id>/` as ordinary PNG and text files.
|
|
170
|
+
There is no API integration: your agent reads the files.
|
|
171
|
+
|
|
172
|
+
### With Claude Code
|
|
173
|
+
|
|
174
|
+
Install as a plugin — the skill and its protocol come with it:
|
|
175
|
+
|
|
176
|
+
```
|
|
177
|
+
/plugin marketplace add nKOxxx/nybls
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
```
|
|
181
|
+
/plugin install nybls@nybls
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Then ask:
|
|
185
|
+
|
|
186
|
+
```
|
|
187
|
+
/nybls:watch https://www.youtube.com/watch?v=VIDEO_ID what does the chart at the end show?
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
The skill carries the protocol — the confidence check, the named-gap rule, the
|
|
191
|
+
stop conditions, and the receipts contract — so the discipline travels with the
|
|
192
|
+
tool instead of depending on how well you prompt. For a personal install without
|
|
193
|
+
the plugin system, copy `skills/watch/SKILL.md` to
|
|
194
|
+
`~/.claude/skills/watch/SKILL.md`.
|
|
195
|
+
|
|
196
|
+
## The guardrails are in the tool, not just the prose
|
|
197
|
+
|
|
198
|
+
A protocol written only in a prompt is a suggestion. These are enforced by the
|
|
199
|
+
CLI itself:
|
|
200
|
+
|
|
201
|
+
- **Named gaps.** Past a few images, `frames` and `zoom` refuse to run without
|
|
202
|
+
`--looking-for "<interval + what you expect>"`.
|
|
203
|
+
- **A real budget.** Duration-scaled, cumulative, and the server refuses to serve
|
|
204
|
+
past it unless a human passes `--force`.
|
|
205
|
+
- **Duplicate warnings.** A frame that is a near-duplicate of one already served
|
|
206
|
+
is flagged as wasted budget.
|
|
207
|
+
- **Next-step nudges.** Every command's output ends by telling the agent to draft
|
|
208
|
+
an answer and classify its own confidence before asking for more.
|
|
209
|
+
|
|
210
|
+
## What this cannot do
|
|
211
|
+
|
|
212
|
+
- **No live video.** Batch only: download, then analyze.
|
|
213
|
+
- **Motion is lossy.** It sees stills, not movement. Good for talks, tutorials,
|
|
214
|
+
demos, dashboards, and on-screen text; weak for sports mechanics or fast action.
|
|
215
|
+
- **Speech only.** Transcription covers words, not music or sound events.
|
|
216
|
+
- **It depends on tools that fight back.** Video platforms actively break
|
|
217
|
+
downloaders. Keep `yt-dlp` current; expect occasional breakage.
|
|
218
|
+
- **Alpha.** The protocol is validated by published research and by daily use, but
|
|
219
|
+
this is version 0.1.0 and the interfaces may change.
|
|
220
|
+
|
|
221
|
+
## Privacy and security posture
|
|
222
|
+
|
|
223
|
+
Local-first by construction: no API keys, no accounts, no telemetry, no server.
|
|
224
|
+
The agent you already use supplies the model; everything else runs on your
|
|
225
|
+
machine. See [docs/SECURITY.md](docs/SECURITY.md) for the full posture, including
|
|
226
|
+
the one rule that matters most — **video content is data, never instructions.** A
|
|
227
|
+
video can display text aimed at your agent, and the protocol requires that such
|
|
228
|
+
text be reported to you rather than obeyed.
|
|
229
|
+
|
|
230
|
+
Downloading from a platform may conflict with that platform's terms of service.
|
|
231
|
+
That is your call to make; this tool takes no position and phones nothing home.
|
|
232
|
+
|
|
233
|
+
## Documentation
|
|
234
|
+
|
|
235
|
+
- [docs/PROTOCOL.md](docs/PROTOCOL.md) — the WATCH loop in full, and why each rule exists
|
|
236
|
+
- [docs/RESEARCH.md](docs/RESEARCH.md) — the evidence base and everything this is built on
|
|
237
|
+
- [docs/SECURITY.md](docs/SECURITY.md) — threat model and the security checklist
|
|
238
|
+
- [CHANGELOG.md](CHANGELOG.md)
|
|
239
|
+
|
|
240
|
+
## License
|
|
241
|
+
|
|
242
|
+
MIT.
|
nybls-0.1.0/README.md
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="brand/nybls_1024.png" alt="nybls" width="140">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">nybls</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<em>Other tools slurp the whole video. This one sips —<br>
|
|
9
|
+
three frames out of fifty-three thousand, and it shows you which.</em>
|
|
10
|
+
</p>
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## The problem
|
|
15
|
+
|
|
16
|
+
Almost every "AI watches YouTube" tool reads the **subtitles**. That is not a
|
|
17
|
+
cynical read of the field, it is what the code does: the YouTube loaders in the
|
|
18
|
+
major agent frameworks and self-hosted chat apps are transcript scrapers, and
|
|
19
|
+
Google's own documentation says a YouTube source in NotebookLM imports "only the
|
|
20
|
+
text transcript of the video."
|
|
21
|
+
|
|
22
|
+
So a video with no speech returns nothing. A chart on screen is invisible. A
|
|
23
|
+
demo that shows rather than tells is invisible. And you cannot tell the
|
|
24
|
+
difference between a tool that looked and a tool that guessed, because neither
|
|
25
|
+
shows you what it saw.
|
|
26
|
+
|
|
27
|
+
The other half of the field does the opposite and dumps 100 uniformly-sampled
|
|
28
|
+
frames into the model at once — expensive, mostly redundant, and still blind to
|
|
29
|
+
whatever fell between the samples.
|
|
30
|
+
|
|
31
|
+
## What nybls does instead
|
|
32
|
+
|
|
33
|
+
nybls gives an AI agent five ways to ask for pieces of a video, and a protocol
|
|
34
|
+
that makes it ask well:
|
|
35
|
+
|
|
36
|
+
1. **Read the free stuff first** — transcript, scene boundaries, metadata. Zero images.
|
|
37
|
+
2. **Look broadly, cheaply** — one contact sheet is six timestamped thumbnails for
|
|
38
|
+
the cost of a single image.
|
|
39
|
+
3. **Then say what it is missing** — the agent must name the time interval and what
|
|
40
|
+
it expects to find there before it may request more.
|
|
41
|
+
4. **Drill in only there** — full-resolution frames, or a zoom crop to read small text.
|
|
42
|
+
5. **Stop, and show the receipts** — every answer ends with the frames it examined
|
|
43
|
+
and what they cost.
|
|
44
|
+
|
|
45
|
+
Real run, this repo's test corpus — a 35-minute car-restoration video:
|
|
46
|
+
|
|
47
|
+
```
|
|
48
|
+
watched 35 min · examined 3 images (~3,069 visual tokens, ~$0.01)
|
|
49
|
+
of ~53,171 total frames · budget 3/142 units
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Three frames. It found the reveal at 12:05, identified that the second half of
|
|
53
|
+
the video switches to a different car, and reported honestly that a zoom aimed
|
|
54
|
+
at a badge landed on a headlight instead — so it did not name the model.
|
|
55
|
+
|
|
56
|
+
## Why "sips" is the whole point
|
|
57
|
+
|
|
58
|
+
The research this is built on is unambiguous: **selecting fewer frames well beats
|
|
59
|
+
sampling many frames blindly.** Stanford's VideoAgent matched the accuracy of 180
|
|
60
|
+
uniformly-sampled frames using 8.4 frames chosen by a confidence-gated loop.
|
|
61
|
+
Adaptive keyframe selection beats uniform sampling at equal frame count. Grids
|
|
62
|
+
larger than about six tiles measurably destroy a model's ability to localize
|
|
63
|
+
detail. Every rule in the nybls protocol traces to a published result — see
|
|
64
|
+
[docs/RESEARCH.md](docs/RESEARCH.md).
|
|
65
|
+
|
|
66
|
+
## Install
|
|
67
|
+
|
|
68
|
+
**Prerequisites** (nybls orchestrates these; it does not vendor them):
|
|
69
|
+
|
|
70
|
+
| Tool | Why | Install |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| `ffmpeg` | frame extraction, crops, contact sheets | `brew install ffmpeg` |
|
|
73
|
+
| `yt-dlp` | downloading from YouTube and ~1,800 other sites | `brew install yt-dlp` |
|
|
74
|
+
| `deno` | required by yt-dlp for full YouTube support | `brew install deno` |
|
|
75
|
+
| `whisper-cpp` | local transcription when a video has no captions | `brew install whisper-cpp` |
|
|
76
|
+
|
|
77
|
+
Then:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
pip install nybls
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
On macOS, add Apple Vision OCR (works with Arabic, no extra models):
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install "nybls[macos]"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
For local transcription, download a Whisper model once:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
mkdir -p ~/.nybls/models && curl -L -o ~/.nybls/models/ggml-large-v3-turbo.bin \
|
|
93
|
+
https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Use it
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
nybls probe "https://www.youtube.com/watch?v=VIDEO_ID"
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
id: VIDEO_ID
|
|
104
|
+
title: But what is a neural network? | Deep learning chapter 1
|
|
105
|
+
duration: 18.7 min · 1280x720 · 24 scenes
|
|
106
|
+
transcript: subtitles:video.en.vtt -> ~/.nybls/store/VIDEO_ID/transcript.txt
|
|
107
|
+
budget: 75 image units
|
|
108
|
+
cost so far: 0 images. Read the transcript first; then request sheets.
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
| Command | What it does | Cost |
|
|
112
|
+
|---|---|---|
|
|
113
|
+
| `nybls probe <url\|file>` | download, transcribe, detect scenes, set the budget | 0 images |
|
|
114
|
+
| `nybls sheet <id> [--range S E]` | 3x2 contact sheet, timestamped | 1 image |
|
|
115
|
+
| `nybls frames <id> --at 92,570` | full-resolution stills | 1 per frame |
|
|
116
|
+
| `nybls zoom <id> --at 570 --box x,y,w,h` | crop into a region to read detail | 1 image |
|
|
117
|
+
| `nybls ledger <id>` | what has been spent | 0 |
|
|
118
|
+
|
|
119
|
+
Everything is written to `~/.nybls/store/<id>/` as ordinary PNG and text files.
|
|
120
|
+
There is no API integration: your agent reads the files.
|
|
121
|
+
|
|
122
|
+
### With Claude Code
|
|
123
|
+
|
|
124
|
+
Install as a plugin — the skill and its protocol come with it:
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
/plugin marketplace add nKOxxx/nybls
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
```
|
|
131
|
+
/plugin install nybls@nybls
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Then ask:
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
/nybls:watch https://www.youtube.com/watch?v=VIDEO_ID what does the chart at the end show?
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
The skill carries the protocol — the confidence check, the named-gap rule, the
|
|
141
|
+
stop conditions, and the receipts contract — so the discipline travels with the
|
|
142
|
+
tool instead of depending on how well you prompt. For a personal install without
|
|
143
|
+
the plugin system, copy `skills/watch/SKILL.md` to
|
|
144
|
+
`~/.claude/skills/watch/SKILL.md`.
|
|
145
|
+
|
|
146
|
+
## The guardrails are in the tool, not just the prose
|
|
147
|
+
|
|
148
|
+
A protocol written only in a prompt is a suggestion. These are enforced by the
|
|
149
|
+
CLI itself:
|
|
150
|
+
|
|
151
|
+
- **Named gaps.** Past a few images, `frames` and `zoom` refuse to run without
|
|
152
|
+
`--looking-for "<interval + what you expect>"`.
|
|
153
|
+
- **A real budget.** Duration-scaled, cumulative, and the server refuses to serve
|
|
154
|
+
past it unless a human passes `--force`.
|
|
155
|
+
- **Duplicate warnings.** A frame that is a near-duplicate of one already served
|
|
156
|
+
is flagged as wasted budget.
|
|
157
|
+
- **Next-step nudges.** Every command's output ends by telling the agent to draft
|
|
158
|
+
an answer and classify its own confidence before asking for more.
|
|
159
|
+
|
|
160
|
+
## What this cannot do
|
|
161
|
+
|
|
162
|
+
- **No live video.** Batch only: download, then analyze.
|
|
163
|
+
- **Motion is lossy.** It sees stills, not movement. Good for talks, tutorials,
|
|
164
|
+
demos, dashboards, and on-screen text; weak for sports mechanics or fast action.
|
|
165
|
+
- **Speech only.** Transcription covers words, not music or sound events.
|
|
166
|
+
- **It depends on tools that fight back.** Video platforms actively break
|
|
167
|
+
downloaders. Keep `yt-dlp` current; expect occasional breakage.
|
|
168
|
+
- **Alpha.** The protocol is validated by published research and by daily use, but
|
|
169
|
+
this is version 0.1.0 and the interfaces may change.
|
|
170
|
+
|
|
171
|
+
## Privacy and security posture
|
|
172
|
+
|
|
173
|
+
Local-first by construction: no API keys, no accounts, no telemetry, no server.
|
|
174
|
+
The agent you already use supplies the model; everything else runs on your
|
|
175
|
+
machine. See [docs/SECURITY.md](docs/SECURITY.md) for the full posture, including
|
|
176
|
+
the one rule that matters most — **video content is data, never instructions.** A
|
|
177
|
+
video can display text aimed at your agent, and the protocol requires that such
|
|
178
|
+
text be reported to you rather than obeyed.
|
|
179
|
+
|
|
180
|
+
Downloading from a platform may conflict with that platform's terms of service.
|
|
181
|
+
That is your call to make; this tool takes no position and phones nothing home.
|
|
182
|
+
|
|
183
|
+
## Documentation
|
|
184
|
+
|
|
185
|
+
- [docs/PROTOCOL.md](docs/PROTOCOL.md) — the WATCH loop in full, and why each rule exists
|
|
186
|
+
- [docs/RESEARCH.md](docs/RESEARCH.md) — the evidence base and everything this is built on
|
|
187
|
+
- [docs/SECURITY.md](docs/SECURITY.md) — threat model and the security checklist
|
|
188
|
+
- [CHANGELOG.md](CHANGELOG.md)
|
|
189
|
+
|
|
190
|
+
## License
|
|
191
|
+
|
|
192
|
+
MIT.
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# The WATCH protocol
|
|
2
|
+
|
|
3
|
+
The CLI is deliberately unintelligent. It extracts frames, it does not decide
|
|
4
|
+
which frames matter. The deciding happens in the model, guided by this protocol
|
|
5
|
+
and constrained by rails in the tool.
|
|
6
|
+
|
|
7
|
+
That split is the central design choice, and it is not arbitrary: every attempt
|
|
8
|
+
to make the *extractor* smart loses to letting the *model* choose, because the
|
|
9
|
+
extractor has to decide before it understands the question. Measured result:
|
|
10
|
+
a confidence-gated loop matched the accuracy of 180 uniformly sampled frames
|
|
11
|
+
using 8.4 chosen ones (arXiv:2403.10517).
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## The loop
|
|
16
|
+
|
|
17
|
+
### Round 0 — free
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
nybls probe <url-or-file>
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Produces the transcript, the scene list, metadata, and the budget. Costs zero
|
|
24
|
+
images. Read all of it before spending anything.
|
|
25
|
+
|
|
26
|
+
For talk-heavy video this round frequently answers the question outright. It
|
|
27
|
+
must never be used to answer a *visual* question — if the user asks what is on
|
|
28
|
+
screen, the transcript is evidence about the audio, not the picture.
|
|
29
|
+
|
|
30
|
+
### Round 1 — coverage
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
nybls sheet <id> [--range START_S END_S]
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
One sheet is six timestamped thumbnails in a 3x2 grid: six moments for the price
|
|
37
|
+
of one image. Use `--range` to cover a specific region of a long video; for a
|
|
38
|
+
20-minute video, one sheet per five to seven minutes of interest is a sane
|
|
39
|
+
budget.
|
|
40
|
+
|
|
41
|
+
**Six tiles, not more.** IG-VLM ablated 4 to 20 tiles per grid and found six in a
|
|
42
|
+
near-square layout optimal, with larger grids performing worse (arXiv:2403.18406).
|
|
43
|
+
Needle-in-a-haystack testing shows every current model — Claude included — loses
|
|
44
|
+
the ability to localize detail inside dense sub-image grids (arXiv:2406.11230).
|
|
45
|
+
Sheets tell you *where to look*. They are not for reading.
|
|
46
|
+
|
|
47
|
+
### The evaluator — mandatory, every round
|
|
48
|
+
|
|
49
|
+
Draft the answer. Then classify, explicitly:
|
|
50
|
+
|
|
51
|
+
| Verdict | Meaning | Action |
|
|
52
|
+
|---|---|---|
|
|
53
|
+
| `sufficient` | the evidence supports the answer | stop, write the output |
|
|
54
|
+
| `partial` | the shape is right, detail is missing | name the gap, request only that |
|
|
55
|
+
| `insufficient` | cannot answer | name the gap, request only that |
|
|
56
|
+
|
|
57
|
+
This step is not decoration. In component ablations of adaptive video agents, the
|
|
58
|
+
"can I answer yet?" evaluator was the single highest-impact part of the system —
|
|
59
|
+
removing it hurt accuracy more than removing the smart sampler (arXiv:2410.20252).
|
|
60
|
+
Interval-level model confidence also correlates with correctness, which is what
|
|
61
|
+
makes confidence-gated search work (arXiv:2507.02946).
|
|
62
|
+
|
|
63
|
+
### Rounds 2–3 — targeted
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
nybls frames <id> --at 92,570 --looking-for "09:30, the chart he refers to"
|
|
67
|
+
nybls zoom <id> --at 570 --box 0.3,0.2,0.4,0.4 --looking-for "axis labels"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Request frames for a **named** interval and a **named** expectation. Never
|
|
71
|
+
re-sample globally; that is what Round 1 was for.
|
|
72
|
+
|
|
73
|
+
Prefer a zoom over more frames when the answer is inside one frame. Adding
|
|
74
|
+
spatial zoom to a temporal search gained 2.6 to 5.9 accuracy points under an
|
|
75
|
+
identical frame budget (arXiv:2504.02259, building on arXiv:2312.14135).
|
|
76
|
+
|
|
77
|
+
### Stop
|
|
78
|
+
|
|
79
|
+
Stop at the first of:
|
|
80
|
+
|
|
81
|
+
- confidence is `sufficient`
|
|
82
|
+
- three rounds after probe — accuracy saturates there (arXiv:2403.10517)
|
|
83
|
+
- the budget is exhausted
|
|
84
|
+
|
|
85
|
+
If you stop at `partial`, **say so in the output.** A confident answer built on
|
|
86
|
+
insufficient evidence is the failure mode this whole design exists to prevent.
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## Budget
|
|
91
|
+
|
|
92
|
+
```
|
|
93
|
+
budget = clamp(8, ceil(duration_minutes * 4), 200) # image units
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
One sheet, one full frame, and one zoom each cost one unit. The ledger also
|
|
97
|
+
tracks estimated visual tokens, computed the way the Claude API bills them
|
|
98
|
+
(ceil(w/28) * ceil(h/28) per image).
|
|
99
|
+
|
|
100
|
+
Duration-scaled budgets with a floor and ceiling follow the pattern in
|
|
101
|
+
arXiv:2510.04428. The ceiling matters more than the floor: past roughly 32
|
|
102
|
+
well-chosen frames, video QA accuracy plateaus (arXiv:2502.19680), so a large
|
|
103
|
+
budget is permission to look carefully, not an instruction to look often.
|
|
104
|
+
|
|
105
|
+
Sampling density is genuinely task-dependent — some questions are answerable at
|
|
106
|
+
one frame per minute, others need one frame per second in a narrow window
|
|
107
|
+
(arXiv:2503.12496). Coarse first, dense only where the question points.
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Rails: what the tool enforces
|
|
112
|
+
|
|
113
|
+
Prose can be ignored. These cannot:
|
|
114
|
+
|
|
115
|
+
| Rail | Behavior |
|
|
116
|
+
|---|---|
|
|
117
|
+
| Named gaps | past 3 spent images, `frames`/`zoom` refuse without `--looking-for` |
|
|
118
|
+
| Budget stop | requests beyond the budget are refused; only a human may `--force` |
|
|
119
|
+
| Duplicate detection | a near-duplicate of an already-served frame is flagged as waste |
|
|
120
|
+
| Per-call cap | at most 10 frames per invocation |
|
|
121
|
+
| Next-step nudges | every command's output restates the next protocol step |
|
|
122
|
+
|
|
123
|
+
The rails exist because the protocol must work for an agent that was not
|
|
124
|
+
carefully prompted. Discipline that depends on the operator does not survive
|
|
125
|
+
distribution.
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## Output contract
|
|
130
|
+
|
|
131
|
+
Every answer ends with three blocks:
|
|
132
|
+
|
|
133
|
+
1. **Answer** — grounded in what was seen and heard, with inline `[mm:ss]` citations.
|
|
134
|
+
2. **Evidence strip** — every image examined, and what it showed.
|
|
135
|
+
3. **Ledger** — verbatim output of `nybls ledger <id>`.
|
|
136
|
+
|
|
137
|
+
The evidence strip is what separates a claim from an assertion. It is also the
|
|
138
|
+
product's most visible difference: a transcript-only tool cannot produce one.
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## Worked example
|
|
143
|
+
|
|
144
|
+
A 35-minute video, question: what happens in it?
|
|
145
|
+
|
|
146
|
+
| Round | Action | Spend | Outcome |
|
|
147
|
+
|---|---|---|---|
|
|
148
|
+
| 0 | probe, read transcript | 0 | narrative clear; the visual payoff is not |
|
|
149
|
+
| 1 | one whole-video sheet | 1 | six moments; the second half is a *different car* |
|
|
150
|
+
| — | evaluator | — | `partial` — the titular reveal is not in these six |
|
|
151
|
+
| 2 | sheet of the final 5.5 minutes | 1 | not there either; the reveal is earlier |
|
|
152
|
+
| 3 | zoom on a badge | 1 | landed on a headlight — reported honestly, model not named |
|
|
153
|
+
|
|
154
|
+
Final: 3 images, ~3,069 visual tokens, about one cent, against 53,171 frames.
|
|
155
|
+
The reveal was located at 12:05 by combining a Round 1 thumbnail with a free
|
|
156
|
+
transcript read — not by spending more images.
|