stream-clipper 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. stream_clipper-0.1.0/LICENSE +21 -0
  2. stream_clipper-0.1.0/MANIFEST.in +1 -0
  3. stream_clipper-0.1.0/PKG-INFO +238 -0
  4. stream_clipper-0.1.0/README.md +224 -0
  5. stream_clipper-0.1.0/pipeline/__init__.py +1 -0
  6. stream_clipper-0.1.0/pipeline/banter_model.py +4 -0
  7. stream_clipper-0.1.0/pipeline/benchmark_evidence.py +103 -0
  8. stream_clipper-0.1.0/pipeline/chat_spikes.py +171 -0
  9. stream_clipper-0.1.0/pipeline/clip_mcp.py +149 -0
  10. stream_clipper-0.1.0/pipeline/clip_quality.py +150 -0
  11. stream_clipper-0.1.0/pipeline/clips_ground_truth.py +206 -0
  12. stream_clipper-0.1.0/pipeline/fortnite_hud.py +145 -0
  13. stream_clipper-0.1.0/pipeline/live_drafts.py +121 -0
  14. stream_clipper-0.1.0/pipeline/live_monitor.py +1146 -0
  15. stream_clipper-0.1.0/pipeline/moment_ranking.py +256 -0
  16. stream_clipper-0.1.0/pipeline/push_to_postfling.py +319 -0
  17. stream_clipper-0.1.0/pipeline/speech_excerpt.py +144 -0
  18. stream_clipper-0.1.0/pipeline/twitch_clock.py +357 -0
  19. stream_clipper-0.1.0/pipeline/twitch_slate.py +33 -0
  20. stream_clipper-0.1.0/pyproject.toml +28 -0
  21. stream_clipper-0.1.0/requirements-mcp.txt +1 -0
  22. stream_clipper-0.1.0/setup.cfg +4 -0
  23. stream_clipper-0.1.0/stream_clipper.egg-info/PKG-INFO +238 -0
  24. stream_clipper-0.1.0/stream_clipper.egg-info/SOURCES.txt +34 -0
  25. stream_clipper-0.1.0/stream_clipper.egg-info/dependency_links.txt +1 -0
  26. stream_clipper-0.1.0/stream_clipper.egg-info/entry_points.txt +4 -0
  27. stream_clipper-0.1.0/stream_clipper.egg-info/requires.txt +3 -0
  28. stream_clipper-0.1.0/stream_clipper.egg-info/top_level.txt +1 -0
  29. stream_clipper-0.1.0/tests/test_clip_mcp.py +170 -0
  30. stream_clipper-0.1.0/tests/test_clip_quality.py +112 -0
  31. stream_clipper-0.1.0/tests/test_fortnite_hud.py +86 -0
  32. stream_clipper-0.1.0/tests/test_moment_ranking.py +206 -0
  33. stream_clipper-0.1.0/tests/test_pipeline.py +434 -0
  34. stream_clipper-0.1.0/tests/test_postfling.py +312 -0
  35. stream_clipper-0.1.0/tests/test_speech_excerpt.py +47 -0
  36. stream_clipper-0.1.0/tests/test_twitch_clock.py +103 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 lisn0
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ include requirements-mcp.txt
@@ -0,0 +1,238 @@
1
+ Metadata-Version: 2.4
2
+ Name: stream-clipper
3
+ Version: 0.1.0
4
+ Summary: Watch Twitch chat, record live streams, and export candidate clips.
5
+ License-Expression: MIT
6
+ Project-URL: Repository, https://github.com/lisn0/stream-clipper
7
+ Project-URL: Issues, https://github.com/lisn0/stream-clipper/issues
8
+ Requires-Python: >=3.10
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Provides-Extra: mcp
12
+ Requires-Dist: mcp<2,>=1.28; extra == "mcp"
13
+ Dynamic: license-file
14
+
15
+ # Stream Clipping
16
+
17
+ A local Python tool that watches explicitly selected Twitch channels, detects
18
+ chat activity spikes, records live video, and cuts candidate clips. It can also
19
+ prepare completed clips as drafts in PostFling for human review.
20
+
21
+ ## Requirements
22
+
23
+ - Python 3.10 or newer. The core pipeline uses only the Python standard library.
24
+ - A [Twitch application](https://dev.twitch.tv/docs/authentication/register-app/)
25
+ client ID and secret for live-stream polling.
26
+ - [yt-dlp](https://github.com/yt-dlp/yt-dlp) and
27
+ [FFmpeg and ffprobe](https://ffmpeg.org/download.html) on `PATH` for recording
28
+ and export. The chat-only checks do not require video tools.
29
+ - Optional dependencies include Tesseract with English data for
30
+ `--fortnite-hud` and [faster-whisper](https://github.com/SYSTRAN/faster-whisper)
31
+ with a cached Whisper Small model for speech-assisted options.
32
+
33
+ ## Installation
34
+
35
+ Install the CLI from PyPI in a Python virtual environment:
36
+
37
+ ```sh
38
+ python -m pip install stream-clipper
39
+ ```
40
+
41
+ Use `stream-clipping` to run the monitor and `stream-clipping-postfling` to
42
+ preview or upload clips for draft review. These commands accept the same
43
+ arguments as `pipeline/live_monitor.py` and `pipeline/push_to_postfling.py`
44
+ in the source examples below. You can also run those scripts from a checkout
45
+ without installing the package.
46
+
47
+ ## Twitch use and authorization
48
+
49
+ The monitor reads stream status from Twitch Helix and chat from Twitch IRC.
50
+ Live recording uses yt-dlp. Use it only where you have authorization to record
51
+ the broadcast and retain the footage and chat under
52
+ [Twitch's Terms of Service](https://www.twitch.tv/p/terms-of-service).
53
+ Obtain the necessary rights before sharing clips, including rights for any
54
+ third-party material in the clip. Twitch also documents its
55
+ [official clip creation and retrieval API](https://dev.twitch.tv/docs/api/clips/).
56
+
57
+ The monitor requires an explicit `--channels` list; there is no implicit channel
58
+ selection. Use channel login names separated by commas.
59
+
60
+ ## Configure and run
61
+
62
+ Create a local Twitch credential file (keep it outside source control):
63
+
64
+ ```text
65
+ TWITCH_CLIENT_ID=your_client_id
66
+ TWITCH_CLIENT_SECRET=your_client_secret
67
+ ```
68
+
69
+ Save it as `secrets/twitch.env` or another local path. Do not commit it. First
70
+ run a short dry-run to check Twitch polling and chat connectivity. This mode
71
+ does not record video:
72
+
73
+ ```sh
74
+ python3 pipeline/live_monitor.py \
75
+ --channels your_authorized_channel \
76
+ --env secrets/twitch.env \
77
+ --dry-run
78
+ ```
79
+
80
+ After reviewing the dry-run output, start live recording and clipping:
81
+
82
+ ```sh
83
+ python3 pipeline/live_monitor.py \
84
+ --channels your_authorized_channel \
85
+ --env secrets/twitch.env \
86
+ --staging ./output/live
87
+ ```
88
+
89
+ Stop with Ctrl-C. The staging directory defaults to `./output/live`. The
90
+ monitor keeps recordings and event logs there. Finished MP4s and their
91
+ `*_insights.json` sidecars go in `clips/` under staging. The default disk
92
+ reserve pauses recording as free space falls. Recordings and exported clips
93
+ can accumulate across sessions, so set a retention policy for staging.
94
+
95
+ By default, the chat log stores event metadata such as time and message rate.
96
+ `--save-chat-samples` adds short raw chat excerpts. Restrict access to staging
97
+ and retain recordings, chat logs, and clips only as long as necessary.
98
+
99
+ For a bounded connectivity check, use `--smoke SECONDS` with exactly one
100
+ explicit channel and the credential file. That channel must be live. Smoke mode
101
+ checks Twitch polling and chat but does not record. Use an active chat:
102
+
103
+ ```sh
104
+ python3 pipeline/live_monitor.py \
105
+ --channels your_authorized_channel \
106
+ --env secrets/twitch.env \
107
+ --smoke 30
108
+ ```
109
+
110
+ ## Optional PostFling draft review
111
+
112
+ PostFling is optional. Its uploader defaults to `review` mode, which uploads
113
+ the selected clips and saves editable workspace drafts for a person to review.
114
+ It does not initiate publishing in that mode. Preview locally first with
115
+ `--dry-run`:
116
+
117
+ ```sh
118
+ python3 pipeline/push_to_postfling.py \
119
+ --clips-dir ./output/live/clips \
120
+ --channel YOUR_POSTFLING_CHANNEL \
121
+ --platform YOUR_PLATFORM \
122
+ --dry-run
123
+ ```
124
+
125
+ To save drafts, load `POSTFLING_EMAIL` and `POSTFLING_PASSWORD` into the
126
+ process environment from your secret store and omit `--dry-run`:
127
+
128
+ ```sh
129
+ python3 pipeline/push_to_postfling.py \
130
+ --clips-dir ./output/live/clips \
131
+ --channel YOUR_POSTFLING_CHANNEL \
132
+ --platform YOUR_PLATFORM
133
+ ```
134
+
135
+ The default origin is `http://127.0.0.1:8477` for local testing. For a remote
136
+ instance, set `POSTFLING_URL` to the HTTPS origin, such as
137
+ `https://workspace.example`, without an API path. The client rejects redirects.
138
+
139
+ Direct delivery is an explicit `--mode direct` action. Use it only when you
140
+ have the required rights and authorization for the content and destination.
141
+ Review each draft and destination before choosing direct delivery.
142
+
143
+ ## Optional MCP server
144
+
145
+ The stdio server lets an MCP client browse completed live clips while the
146
+ monitor runs separately. Use an absolute path to a clips directory writable
147
+ only by you and the monitor. Create it or wait for the monitor to save a clip
148
+ before connecting the client.
149
+
150
+ With `uvx`, the client can launch the PyPI package with its MCP dependencies:
151
+
152
+ ```sh
153
+ uvx --from 'stream-clipper[mcp]' stream-clipping-mcp \
154
+ --clips-dir /absolute/path/to/output/live/clips
155
+ ```
156
+
157
+ ```json
158
+ {
159
+ "mcpServers": {
160
+ "stream-clipping": {
161
+ "command": "uvx",
162
+ "args": [
163
+ "--from", "stream-clipper[mcp]", "stream-clipping-mcp",
164
+ "--clips-dir", "/absolute/path/to/output/live/clips"
165
+ ]
166
+ }
167
+ }
168
+ }
169
+ ```
170
+
171
+ If your client cannot find `uvx` on its `PATH`, use the absolute path to it.
172
+ You can also run `python -m pip install 'stream-clipper[mcp]'` in a virtual
173
+ environment and point the client at the installed `stream-clipping-mcp`
174
+ executable with `--clips-dir` and the absolute directory path as arguments.
175
+
176
+ ### Launch from a source checkout
177
+
178
+ Install the MCP Python SDK in a virtual environment for the client to launch.
179
+
180
+ ```sh
181
+ python3 -m venv .venv
182
+ .venv/bin/python -m pip install -r requirements-mcp.txt
183
+ ```
184
+
185
+ Use absolute paths in your client's MCP server configuration.
186
+
187
+ ```json
188
+ {
189
+ "mcpServers": {
190
+ "stream-clipping": {
191
+ "command": "/absolute/path/to/stream-clipping/.venv/bin/python",
192
+ "args": [
193
+ "/absolute/path/to/stream-clipping/pipeline/clip_mcp.py",
194
+ "--clips-dir", "/absolute/path/to/output/live/clips"
195
+ ]
196
+ }
197
+ }
198
+ }
199
+ ```
200
+
201
+ ### MCP tools
202
+
203
+ `list_clips` shows the newest completed MP4 and insights pairs first. It returns
204
+ clip IDs, Twitch logins, Unix timestamps, and optional duration, score, and
205
+ trigger fields. Use `limit` (1–100, default 25) and `offset` (default 0) to page
206
+ through results.
207
+
208
+ To enable `save_postfling_draft`, append `--postfling-config` and the absolute
209
+ path to a JSON file to `args`. Use the same workspace and Twitch-login mapping
210
+ as the live monitor. Supply `POSTFLING_EMAIL` and `POSTFLING_PASSWORD` to the
211
+ server process through your client's secret environment or launch wrapper.
212
+
213
+ ```json
214
+ {
215
+ "origin": "https://workspace.example",
216
+ "workspace_id": "YOUR_WORKSPACE_ID",
217
+ "destinations": {
218
+ "your_authorized_channel": "YOUR_DESTINATION_CHANNEL_ID"
219
+ }
220
+ }
221
+ ```
222
+
223
+ Call `save_postfling_draft` with a `clip_id` from `list_clips`. The tool uploads
224
+ that MP4 to the configured PostFling workspace and saves an editable draft for
225
+ the mapped destination. Review and publish through PostFling.
226
+
227
+ ## Tests
228
+
229
+ With FFmpeg and ffprobe installed, run the full suite from the repository root:
230
+
231
+ ```sh
232
+ python3 -B -m unittest discover -s tests -v
233
+ ```
234
+
235
+ ## License
236
+
237
+ Project code is under the [MIT License](LICENSE). The external tools and model
238
+ weights retain their own licenses.
@@ -0,0 +1,224 @@
1
+ # Stream Clipping
2
+
3
+ A local Python tool that watches explicitly selected Twitch channels, detects
4
+ chat activity spikes, records live video, and cuts candidate clips. It can also
5
+ prepare completed clips as drafts in PostFling for human review.
6
+
7
+ ## Requirements
8
+
9
+ - Python 3.10 or newer. The core pipeline uses only the Python standard library.
10
+ - A [Twitch application](https://dev.twitch.tv/docs/authentication/register-app/)
11
+ client ID and secret for live-stream polling.
12
+ - [yt-dlp](https://github.com/yt-dlp/yt-dlp) and
13
+ [FFmpeg and ffprobe](https://ffmpeg.org/download.html) on `PATH` for recording
14
+ and export. The chat-only checks do not require video tools.
15
+ - Optional dependencies include Tesseract with English data for
16
+ `--fortnite-hud` and [faster-whisper](https://github.com/SYSTRAN/faster-whisper)
17
+ with a cached Whisper Small model for speech-assisted options.
18
+
19
+ ## Installation
20
+
21
+ Install the CLI from PyPI in a Python virtual environment:
22
+
23
+ ```sh
24
+ python -m pip install stream-clipper
25
+ ```
26
+
27
+ Use `stream-clipping` to run the monitor and `stream-clipping-postfling` to
28
+ preview or upload clips for draft review. These commands accept the same
29
+ arguments as `pipeline/live_monitor.py` and `pipeline/push_to_postfling.py`
30
+ in the source examples below. You can also run those scripts from a checkout
31
+ without installing the package.
32
+
33
+ ## Twitch use and authorization
34
+
35
+ The monitor reads stream status from Twitch Helix and chat from Twitch IRC.
36
+ Live recording uses yt-dlp. Use it only where you have authorization to record
37
+ the broadcast and retain the footage and chat under
38
+ [Twitch's Terms of Service](https://www.twitch.tv/p/terms-of-service).
39
+ Obtain the necessary rights before sharing clips, including rights for any
40
+ third-party material in the clip. Twitch also documents its
41
+ [official clip creation and retrieval API](https://dev.twitch.tv/docs/api/clips/).
42
+
43
+ The monitor requires an explicit `--channels` list; there is no implicit channel
44
+ selection. Use channel login names separated by commas.
45
+
46
+ ## Configure and run
47
+
48
+ Create a local Twitch credential file (keep it outside source control):
49
+
50
+ ```text
51
+ TWITCH_CLIENT_ID=your_client_id
52
+ TWITCH_CLIENT_SECRET=your_client_secret
53
+ ```
54
+
55
+ Save it as `secrets/twitch.env` or another local path. Do not commit it. First
56
+ run a short dry-run to check Twitch polling and chat connectivity. This mode
57
+ does not record video:
58
+
59
+ ```sh
60
+ python3 pipeline/live_monitor.py \
61
+ --channels your_authorized_channel \
62
+ --env secrets/twitch.env \
63
+ --dry-run
64
+ ```
65
+
66
+ After reviewing the dry-run output, start live recording and clipping:
67
+
68
+ ```sh
69
+ python3 pipeline/live_monitor.py \
70
+ --channels your_authorized_channel \
71
+ --env secrets/twitch.env \
72
+ --staging ./output/live
73
+ ```
74
+
75
+ Stop with Ctrl-C. The staging directory defaults to `./output/live`. The
76
+ monitor keeps recordings and event logs there. Finished MP4s and their
77
+ `*_insights.json` sidecars go in `clips/` under staging. The default disk
78
+ reserve pauses recording as free space falls. Recordings and exported clips
79
+ can accumulate across sessions, so set a retention policy for staging.
80
+
81
+ By default, the chat log stores event metadata such as time and message rate.
82
+ `--save-chat-samples` adds short raw chat excerpts. Restrict access to staging
83
+ and retain recordings, chat logs, and clips only as long as necessary.
84
+
85
+ For a bounded connectivity check, use `--smoke SECONDS` with exactly one
86
+ explicit channel and the credential file. That channel must be live. Smoke mode
87
+ checks Twitch polling and chat but does not record. Use an active chat:
88
+
89
+ ```sh
90
+ python3 pipeline/live_monitor.py \
91
+ --channels your_authorized_channel \
92
+ --env secrets/twitch.env \
93
+ --smoke 30
94
+ ```
95
+
96
+ ## Optional PostFling draft review
97
+
98
+ PostFling is optional. Its uploader defaults to `review` mode, which uploads
99
+ the selected clips and saves editable workspace drafts for a person to review.
100
+ It does not initiate publishing in that mode. Preview locally first with
101
+ `--dry-run`:
102
+
103
+ ```sh
104
+ python3 pipeline/push_to_postfling.py \
105
+ --clips-dir ./output/live/clips \
106
+ --channel YOUR_POSTFLING_CHANNEL \
107
+ --platform YOUR_PLATFORM \
108
+ --dry-run
109
+ ```
110
+
111
+ To save drafts, load `POSTFLING_EMAIL` and `POSTFLING_PASSWORD` into the
112
+ process environment from your secret store and omit `--dry-run`:
113
+
114
+ ```sh
115
+ python3 pipeline/push_to_postfling.py \
116
+ --clips-dir ./output/live/clips \
117
+ --channel YOUR_POSTFLING_CHANNEL \
118
+ --platform YOUR_PLATFORM
119
+ ```
120
+
121
+ The default origin is `http://127.0.0.1:8477` for local testing. For a remote
122
+ instance, set `POSTFLING_URL` to the HTTPS origin, such as
123
+ `https://workspace.example`, without an API path. The client rejects redirects.
124
+
125
+ Direct delivery is an explicit `--mode direct` action. Use it only when you
126
+ have the required rights and authorization for the content and destination.
127
+ Review each draft and destination before choosing direct delivery.
128
+
129
+ ## Optional MCP server
130
+
131
+ The stdio server lets an MCP client browse completed live clips while the
132
+ monitor runs separately. Use an absolute path to a clips directory writable
133
+ only by you and the monitor. Create it or wait for the monitor to save a clip
134
+ before connecting the client.
135
+
136
+ With `uvx`, the client can launch the PyPI package with its MCP dependencies:
137
+
138
+ ```sh
139
+ uvx --from 'stream-clipper[mcp]' stream-clipping-mcp \
140
+ --clips-dir /absolute/path/to/output/live/clips
141
+ ```
142
+
143
+ ```json
144
+ {
145
+ "mcpServers": {
146
+ "stream-clipping": {
147
+ "command": "uvx",
148
+ "args": [
149
+ "--from", "stream-clipper[mcp]", "stream-clipping-mcp",
150
+ "--clips-dir", "/absolute/path/to/output/live/clips"
151
+ ]
152
+ }
153
+ }
154
+ }
155
+ ```
156
+
157
+ If your client cannot find `uvx` on its `PATH`, use the absolute path to it.
158
+ You can also run `python -m pip install 'stream-clipper[mcp]'` in a virtual
159
+ environment and point the client at the installed `stream-clipping-mcp`
160
+ executable with `--clips-dir` and the absolute directory path as arguments.
161
+
162
+ ### Launch from a source checkout
163
+
164
+ Install the MCP Python SDK in a virtual environment for the client to launch.
165
+
166
+ ```sh
167
+ python3 -m venv .venv
168
+ .venv/bin/python -m pip install -r requirements-mcp.txt
169
+ ```
170
+
171
+ Use absolute paths in your client's MCP server configuration.
172
+
173
+ ```json
174
+ {
175
+ "mcpServers": {
176
+ "stream-clipping": {
177
+ "command": "/absolute/path/to/stream-clipping/.venv/bin/python",
178
+ "args": [
179
+ "/absolute/path/to/stream-clipping/pipeline/clip_mcp.py",
180
+ "--clips-dir", "/absolute/path/to/output/live/clips"
181
+ ]
182
+ }
183
+ }
184
+ }
185
+ ```
186
+
187
+ ### MCP tools
188
+
189
+ `list_clips` shows the newest completed MP4 and insights pairs first. It returns
190
+ clip IDs, Twitch logins, Unix timestamps, and optional duration, score, and
191
+ trigger fields. Use `limit` (1–100, default 25) and `offset` (default 0) to page
192
+ through results.
193
+
194
+ To enable `save_postfling_draft`, append `--postfling-config` and the absolute
195
+ path to a JSON file to `args`. Use the same workspace and Twitch-login mapping
196
+ as the live monitor. Supply `POSTFLING_EMAIL` and `POSTFLING_PASSWORD` to the
197
+ server process through your client's secret environment or launch wrapper.
198
+
199
+ ```json
200
+ {
201
+ "origin": "https://workspace.example",
202
+ "workspace_id": "YOUR_WORKSPACE_ID",
203
+ "destinations": {
204
+ "your_authorized_channel": "YOUR_DESTINATION_CHANNEL_ID"
205
+ }
206
+ }
207
+ ```
208
+
209
+ Call `save_postfling_draft` with a `clip_id` from `list_clips`. The tool uploads
210
+ that MP4 to the configured PostFling workspace and saves an editable draft for
211
+ the mapped destination. Review and publish through PostFling.
212
+
213
+ ## Tests
214
+
215
+ With FFmpeg and ffprobe installed, run the full suite from the repository root:
216
+
217
+ ```sh
218
+ python3 -B -m unittest discover -s tests -v
219
+ ```
220
+
221
+ ## License
222
+
223
+ Project code is under the [MIT License](LICENSE). The external tools and model
224
+ weights retain their own licenses.
@@ -0,0 +1 @@
1
+ """Twitch stream clipping and draft review tools."""
@@ -0,0 +1,4 @@
1
+ """Retired entry point for the transcript editing pilot."""
2
+
3
+ if __name__ == "__main__":
4
+ raise SystemExit("The transcript editor has been retired.")
@@ -0,0 +1,103 @@
1
+ """Save a bounded sample of live source segments while the benchmark runs."""
2
+
3
+ import argparse
4
+ import json
5
+ import os
6
+ import shutil
7
+ import time
8
+ from pathlib import Path
9
+
10
+
11
+ MAX_REVIEW_BYTES = 384 * 1024**2
12
+
13
+
14
+ def sample_type(row):
15
+ decision = row["decision"]
16
+ if decision in {"exported", "boundaries need review", "footage rejected"}:
17
+ return decision
18
+ ranking = row["ranking"]
19
+ if ranking["chat_qualified"] or ranking["viewer_creators"] >= 3:
20
+ return "strong single signal"
21
+ return None
22
+
23
+
24
+ def save_source(staging, output, row, start, end, remaining_bytes):
25
+ channel = row["channel"]
26
+ sources = []
27
+ for recording in staging.glob(f"{channel}-*"):
28
+ matches = []
29
+ for path in recording.glob("*.ts"):
30
+ pdt = int(path.stem.rsplit("-", 1)[1]) / 1_000_000
31
+ if start - 4 <= pdt <= end:
32
+ matches.append((path, pdt))
33
+ if len(matches) > len(sources):
34
+ sources = matches
35
+ if not sources:
36
+ return 0
37
+ size = sum(path.stat().st_size for path, _ in sources)
38
+ if size > remaining_bytes:
39
+ return 0
40
+ sources.sort(key=lambda item: item[1])
41
+ output.mkdir(parents=True, exist_ok=True)
42
+ segment_times = []
43
+ for path, pdt in sources:
44
+ target = output / path.name
45
+ if not target.exists():
46
+ os.link(path, target)
47
+ segment_times.append(pdt)
48
+ review = {**row, "review_source": {"requested": [start, end], "segment_starts": segment_times}}
49
+ (output / "decision.json").write_text(json.dumps(review, indent=2) + "\n")
50
+ return size
51
+
52
+
53
+ def main():
54
+ parser = argparse.ArgumentParser(description=__doc__)
55
+ parser.add_argument("staging", type=Path)
56
+ parser.add_argument("monitor_pid", type=int)
57
+ args = parser.parse_args()
58
+ audit = args.staging / "quality.jsonl"
59
+ offset = 0
60
+ saved = set()
61
+ retained_bytes = 0
62
+ origin = time.time()
63
+ while True:
64
+ running = True
65
+ try:
66
+ os.kill(args.monitor_pid, 0)
67
+ except ProcessLookupError:
68
+ running = False
69
+ if audit.exists():
70
+ with audit.open() as decisions:
71
+ decisions.seek(offset)
72
+ while line := decisions.readline():
73
+ if not line.endswith("\n"):
74
+ break
75
+ offset = decisions.tell()
76
+ row = json.loads(line)
77
+ kind = sample_type(row)
78
+ if kind is None or shutil.disk_usage(args.staging).free <= 7 * 1024**3:
79
+ continue
80
+ ranking = row["ranking"]
81
+ anchor = ranking["chat_anchor"] or row["timestamp"]
82
+ bucket = (int((anchor - origin) // 1800), kind)
83
+ if bucket in saved:
84
+ continue
85
+ start = min(row.get("source_start", anchor), anchor - 35)
86
+ end = max(row.get("source_end", anchor), anchor + 20)
87
+ output = args.staging / "review-source" / f"{row['channel']}-{row['timestamp']:.6f}"
88
+ try:
89
+ size = save_source(args.staging, output, row, start, end,
90
+ MAX_REVIEW_BYTES - retained_bytes)
91
+ if size:
92
+ retained_bytes += size
93
+ saved.add(bucket)
94
+ except OSError as exc:
95
+ print(f"Could not retain review context for {output.name}: {exc}", flush=True)
96
+ if not running:
97
+ break
98
+ time.sleep(10)
99
+ print(f"Retained {len(saved)} live source samples, {retained_bytes} bytes", flush=True)
100
+
101
+
102
+ if __name__ == "__main__":
103
+ main()