ytcap 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ytcap-0.1.0/LICENSE.md +21 -0
- ytcap-0.1.0/PKG-INFO +302 -0
- ytcap-0.1.0/README.md +275 -0
- ytcap-0.1.0/pyproject.toml +52 -0
- ytcap-0.1.0/setup.cfg +4 -0
- ytcap-0.1.0/src/ytcap/__init__.py +5 -0
- ytcap-0.1.0/src/ytcap/__main__.py +9 -0
- ytcap-0.1.0/src/ytcap/app/__init__.py +3 -0
- ytcap-0.1.0/src/ytcap/app/existing_outputs.py +80 -0
- ytcap-0.1.0/src/ytcap/app/export_subtitles.py +318 -0
- ytcap-0.1.0/src/ytcap/app/process_batch.py +296 -0
- ytcap-0.1.0/src/ytcap/app/process_playlist.py +368 -0
- ytcap-0.1.0/src/ytcap/app/process_video.py +227 -0
- ytcap-0.1.0/src/ytcap/cli.py +78 -0
- ytcap-0.1.0/src/ytcap/commands/__init__.py +1 -0
- ytcap-0.1.0/src/ytcap/commands/batch.py +73 -0
- ytcap-0.1.0/src/ytcap/commands/common.py +28 -0
- ytcap-0.1.0/src/ytcap/commands/export.py +51 -0
- ytcap-0.1.0/src/ytcap/commands/inspect.py +67 -0
- ytcap-0.1.0/src/ytcap/commands/playlist.py +115 -0
- ytcap-0.1.0/src/ytcap/commands/video.py +96 -0
- ytcap-0.1.0/src/ytcap/errors.py +55 -0
- ytcap-0.1.0/src/ytcap/exporters/__init__.py +1 -0
- ytcap-0.1.0/src/ytcap/exporters/failed_writer.py +40 -0
- ytcap-0.1.0/src/ytcap/exporters/json_writer.py +46 -0
- ytcap-0.1.0/src/ytcap/exporters/jsonl_writer.py +140 -0
- ytcap-0.1.0/src/ytcap/exporters/output_paths.py +117 -0
- ytcap-0.1.0/src/ytcap/logging_config.py +19 -0
- ytcap-0.1.0/src/ytcap/models/__init__.py +1 -0
- ytcap-0.1.0/src/ytcap/models/subtitle.py +39 -0
- ytcap-0.1.0/src/ytcap/models/video_metadata.py +110 -0
- ytcap-0.1.0/src/ytcap/services/__init__.py +3 -0
- ytcap-0.1.0/src/ytcap/services/batch_parser.py +47 -0
- ytcap-0.1.0/src/ytcap/services/subtitle_format.py +20 -0
- ytcap-0.1.0/src/ytcap/services/subtitle_parser.py +189 -0
- ytcap-0.1.0/src/ytcap/services/subtitle_segmenter.py +138 -0
- ytcap-0.1.0/src/ytcap/services/subtitle_selector.py +70 -0
- ytcap-0.1.0/src/ytcap/services/ytdlp_adapter.py +349 -0
- ytcap-0.1.0/src/ytcap.egg-info/PKG-INFO +302 -0
- ytcap-0.1.0/src/ytcap.egg-info/SOURCES.txt +42 -0
- ytcap-0.1.0/src/ytcap.egg-info/dependency_links.txt +1 -0
- ytcap-0.1.0/src/ytcap.egg-info/entry_points.txt +2 -0
- ytcap-0.1.0/src/ytcap.egg-info/requires.txt +5 -0
- ytcap-0.1.0/src/ytcap.egg-info/top_level.txt +1 -0
ytcap-0.1.0/LICENSE.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Özkan Öztürk
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ytcap-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ytcap
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Extract YouTube metadata and subtitles into JSON and JSONL outputs.
|
|
5
|
+
Author: Özkan Öztürk
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ozkozturk/ytcap
|
|
8
|
+
Project-URL: Documentation, https://github.com/ozkozturk/ytcap#readme
|
|
9
|
+
Keywords: youtube,subtitles,metadata,jsonl,yt-dlp
|
|
10
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Multimedia :: Video
|
|
18
|
+
Classifier: Topic :: Text Processing
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE.md
|
|
22
|
+
Requires-Dist: yt-dlp>=2026.06.09
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: build; extra == "dev"
|
|
25
|
+
Requires-Dist: twine; extra == "dev"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# ytcap
|
|
29
|
+
|
|
30
|
+
`ytcap` is a Python CLI project for extracting **video metadata** and **subtitle files** from YouTube video, batch, and playlist sources, then turning them into reusable JSON and JSONL outputs.
|
|
31
|
+
|
|
32
|
+
It is designed for workflows where you have YouTube video URLs, video IDs, or playlists and want structured metadata plus subtitles for search, indexing, dataset preparation, education, or analysis.
|
|
33
|
+
|
|
34
|
+
## Project Status
|
|
35
|
+
|
|
36
|
+
This repository is in an early planning and implementation stage. The first target release is:
|
|
37
|
+
|
|
38
|
+
```text
|
|
39
|
+
0.1.0
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Currently implemented:
|
|
43
|
+
|
|
44
|
+
- Importable Python package scaffold.
|
|
45
|
+
- Package version exposed as `ytcap.__version__`.
|
|
46
|
+
- Basic CLI entry point with `ytcap --help` and `ytcap --version`.
|
|
47
|
+
- CLI command structure for `inspect`, `video`, and `export`.
|
|
48
|
+
- `batch` command to process multiple video URLs or IDs from a text file, with run manifest logging, `--resume` and `--skip-existing` support.
|
|
49
|
+
- `yt-dlp` adapter support for `inspect` metadata extraction.
|
|
50
|
+
- Normalized video metadata mapping and inspect JSON summary output.
|
|
51
|
+
- Tested subtitle source selection for `manual`, `auto`, and `any` normalized tracks.
|
|
52
|
+
- Controlled subtitle format validation for `srt` and `vtt`.
|
|
53
|
+
- Standard output directory layout creation for `video --out`.
|
|
54
|
+
- `video` command metadata JSON writing and selected SRT/VTT subtitle file download.
|
|
55
|
+
- SRT/VTT cue parsing, cue-level JSONL writer helpers, and basic sentence-level segmentation helpers.
|
|
56
|
+
- `export` command conversion of existing SRT/VTT files to cue-level or sentence-level JSONL.
|
|
57
|
+
- `playlist` command to process videos inside a YouTube playlist with `--limit`, `--start`, and `--end` range controls, run manifest logging, `--resume`, `--skip-existing`, and `--dry-run`.
|
|
58
|
+
- Safe validation for dynamic output filename parts to prevent path traversal from user input or extractor metadata.
|
|
59
|
+
|
|
60
|
+
## Core Decisions
|
|
61
|
+
|
|
62
|
+
| Decision | Target |
|
|
63
|
+
|---|---|
|
|
64
|
+
| Language | Python |
|
|
65
|
+
| Minimum Python version | Python 3.11+ |
|
|
66
|
+
| CLI approach | Standard library first: `argparse` |
|
|
67
|
+
| Test approach | Standard library first: `unittest` |
|
|
68
|
+
| YouTube Data API | Not used |
|
|
69
|
+
| Metadata and subtitle extraction | `yt-dlp>=2026.06.09` is the core extractor dependency |
|
|
70
|
+
| Video/audio downloads | Out of scope for the MVP |
|
|
71
|
+
| Output format | JSON for metadata, JSONL for segment and sentence output |
|
|
72
|
+
| Distribution target | PyPI and `pipx install ytcap` |
|
|
73
|
+
| Additional dependencies | Require justification and approval before being added |
|
|
74
|
+
|
|
75
|
+
## Planned Capabilities
|
|
76
|
+
|
|
77
|
+
The MVP is intended to:
|
|
78
|
+
|
|
79
|
+
- Accept a single YouTube video URL or video ID.
|
|
80
|
+
- Save normalized video metadata as JSON.
|
|
81
|
+
- Find subtitles for a requested language.
|
|
82
|
+
- Prefer manual subtitles when available, with optional fallback to automatic subtitles.
|
|
83
|
+
- Save subtitles as SRT or VTT.
|
|
84
|
+
- Convert subtitles to cue-level or sentence-level JSONL.
|
|
85
|
+
- Report videos with missing or failed subtitle extraction.
|
|
86
|
+
|
|
87
|
+
Later releases may add:
|
|
88
|
+
|
|
89
|
+
- A dedicated retry command for failed records.
|
|
90
|
+
- PyPI publication.
|
|
91
|
+
- Automated test and release workflows through GitHub Actions.
|
|
92
|
+
|
|
93
|
+
## Non-Goals
|
|
94
|
+
|
|
95
|
+
`ytcap` intentionally does not:
|
|
96
|
+
|
|
97
|
+
- Use the official YouTube Data API.
|
|
98
|
+
- Manage API keys or OAuth flows.
|
|
99
|
+
- Download video or audio files as an MVP goal.
|
|
100
|
+
- Redistribute YouTube content or bypass access restrictions.
|
|
101
|
+
- Start with advanced NLP sentence segmentation libraries; simple, testable heuristics come first.
|
|
102
|
+
|
|
103
|
+
## Planned CLI Examples
|
|
104
|
+
|
|
105
|
+
These commands show the intended user experience. Some commands may not be implemented yet.
|
|
106
|
+
The current `inspect` command uses `yt-dlp` for metadata and subtitle
|
|
107
|
+
availability summaries. The `video` command extracts metadata through
|
|
108
|
+
`yt-dlp`, writes normalized metadata JSON, selects a matching subtitle track,
|
|
109
|
+
and saves the selected SRT/VTT subtitle file. SRT/VTT cue parsers,
|
|
110
|
+
cue-level JSONL writer helpers, and basic punctuation-based sentence
|
|
111
|
+
segmentation helpers are wired into the `export` command for existing
|
|
112
|
+
subtitle files.
|
|
113
|
+
|
|
114
|
+
### Inspect One Video
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
ytcap inspect --url "https://www.youtube.com/watch?v=VIDEO_ID"
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
This should answer:
|
|
121
|
+
|
|
122
|
+
- Is the video reachable?
|
|
123
|
+
- Can metadata be extracted?
|
|
124
|
+
- Which subtitle languages are available?
|
|
125
|
+
- Are subtitles manual, automatic, or both?
|
|
126
|
+
|
|
127
|
+
### Extract Metadata and Subtitles
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
ytcap video --url "https://www.youtube.com/watch?v=VIDEO_ID" --lang en --source any --format srt --out ./data
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
| Part | Meaning |
|
|
134
|
+
|---|---|
|
|
135
|
+
| `ytcap` | CLI application |
|
|
136
|
+
| `video` | Single-video processing command |
|
|
137
|
+
| `--url` | Video URL |
|
|
138
|
+
| `--lang en` | Request English subtitles |
|
|
139
|
+
| `--source any` | Try manual subtitles first, then automatic subtitles |
|
|
140
|
+
| `--format srt` | Save subtitles as SRT |
|
|
141
|
+
| `--out ./data` | Write outputs under `./data` |
|
|
142
|
+
|
|
143
|
+
Implemented source selection behavior uses exact language and format matches:
|
|
144
|
+
`manual` selects only manual subtitles, `auto` selects only automatic subtitles,
|
|
145
|
+
and `any` tries manual first before falling back to automatic subtitles.
|
|
146
|
+
Implemented subtitle format validation currently accepts `srt` and `vtt`; other
|
|
147
|
+
values return an `UNSUPPORTED_FORMAT` error before extraction work starts.
|
|
148
|
+
When run without `--dry-run`, `video` writes normalized metadata to
|
|
149
|
+
`videos/{video_id}.info.json` and the selected subtitle file to
|
|
150
|
+
`subtitles/{video_id}.{lang}.{source}.{format}` under the output directory.
|
|
151
|
+
If the requested subtitle cannot be selected or downloaded, the command returns
|
|
152
|
+
a controlled error without leaving a new partial metadata file behind.
|
|
153
|
+
|
|
154
|
+
### List Available Subtitles
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
ytcap inspect --url "https://www.youtube.com/watch?v=VIDEO_ID" --list-subs
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Convert Existing Subtitles to JSONL
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
ytcap export --input ./data/subtitles/VIDEO_ID.en.manual.srt --segments cue --out ./data/normalized
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
ytcap export --input ./data/subtitles --segments sentence --out ./data/normalized
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
The `export` command reads existing `.srt` and `.vtt` files and writes JSONL
|
|
171
|
+
records to `{video_id}.{lang}.{segments}.jsonl` under the output directory. It
|
|
172
|
+
infers `video_id`, language, and source from names such as
|
|
173
|
+
`VIDEO_ID.en.manual.srt`; when the source is missing, JSONL records use
|
|
174
|
+
`"source":"unknown"`. `--video-id` and `--lang` may override metadata for a
|
|
175
|
+
single file input.
|
|
176
|
+
|
|
177
|
+
Dynamic filename parts such as video ID, language, source, format, segment type,
|
|
178
|
+
and run ID are validated before paths are built. Empty values, path separators,
|
|
179
|
+
control characters, absolute paths, `.` and `..` are rejected.
|
|
180
|
+
|
|
181
|
+
### Process a Batch File
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
ytcap batch --input videos.txt --lang en --source any --format srt --resume --skip-existing --out ./data
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
This command parses the input file and processes each URL/ID. It creates a run
|
|
188
|
+
manifest under `runs/{run_id}.manifest.json` keeping track of execution
|
|
189
|
+
statistics, output files, and errors. Failed attempts are appended to
|
|
190
|
+
`failed/failed.jsonl`. `--resume` skips entries completed in the latest
|
|
191
|
+
manifest and retries previous failures, while `--skip-existing` skips videos
|
|
192
|
+
whose metadata and subtitle files already exist for the requested language,
|
|
193
|
+
source, and format. `--dry-run` reports the batch plan without writing files or
|
|
194
|
+
creating output directories.
|
|
195
|
+
|
|
196
|
+
#### Batch Input File Format
|
|
197
|
+
|
|
198
|
+
The `--input` file for the `batch` command is a plain text file containing one YouTube video URL or video ID per line.
|
|
199
|
+
- Empty lines and lines containing only whitespace are ignored.
|
|
200
|
+
- Lines starting with `#` (with optional leading whitespace) are ignored as comment lines.
|
|
201
|
+
- Inline comments starting with `#` are supported, and the comment text plus any preceding whitespace are ignored.
|
|
202
|
+
|
|
203
|
+
Example input file:
|
|
204
|
+
```text
|
|
205
|
+
# This is a comment line
|
|
206
|
+
dQw4w9WgXcQ # Rick Astley - Never Gonna Give You Up
|
|
207
|
+
https://youtu.be/jNQXAC9IVRw # Another video URL
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
### Process a Playlist
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
ytcap playlist --url "https://www.youtube.com/playlist?list=PLAYLIST_ID" --start 1 --limit 50 --lang en --source any --format srt --out ./data
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
The `playlist` command uses `yt-dlp` flat playlist extraction to collect video
|
|
217
|
+
entries without the official YouTube Data API, then processes each video with
|
|
218
|
+
the same metadata and subtitle flow as `video`. `--start` is 1-based, `--end`
|
|
219
|
+
is inclusive, and `--limit` caps the selected range. `--resume` continues only
|
|
220
|
+
from a matching playlist run manifest, while `--skip-existing` skips videos
|
|
221
|
+
only when matching metadata and subtitle files already exist.
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
## Output Layout
|
|
225
|
+
|
|
226
|
+
```text
|
|
227
|
+
data/
|
|
228
|
+
videos/
|
|
229
|
+
VIDEO_ID.info.json
|
|
230
|
+
subtitles/
|
|
231
|
+
VIDEO_ID.en.manual.srt
|
|
232
|
+
normalized/
|
|
233
|
+
VIDEO_ID.en.cue.jsonl
|
|
234
|
+
runs/
|
|
235
|
+
RUN_ID.manifest.json
|
|
236
|
+
failed/
|
|
237
|
+
failed.jsonl
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
Example cue-level JSONL line:
|
|
241
|
+
|
|
242
|
+
```json
|
|
243
|
+
{"schema_version":"0.1","type":"cue","video_id":"VIDEO_ID","language":"en","source":"manual","start":1.0,"end":3.5,"text":"Example subtitle text.","cue_index":1}
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
Sentence-level segmentation uses a simple standard-library heuristic that
|
|
247
|
+
splits on `.`, `?`, and `!`. Timing is marked with a strategy such as
|
|
248
|
+
`cue_exact`, `cue_merge`, `heuristic`, or `unknown` because sentence boundaries
|
|
249
|
+
can fall inside or across subtitle cues.
|
|
250
|
+
|
|
251
|
+
## Documentation
|
|
252
|
+
|
|
253
|
+
| File | Purpose |
|
|
254
|
+
|---|---|
|
|
255
|
+
| `USAGE.md` | Usage boundaries, limitations, and responsible use notes |
|
|
256
|
+
| `CLI_REFERENCE.md` | Planned commands, flags, behavior, and error codes |
|
|
257
|
+
| `OUTPUT_FORMAT.md` | Target JSON and JSONL output formats |
|
|
258
|
+
| `RELEASE.md` | Packaging and release process |
|
|
259
|
+
| `CONTRIBUTING.md` | Contributor expectations |
|
|
260
|
+
| `SECURITY.md` | Security policy and sensitive data rules |
|
|
261
|
+
|
|
262
|
+
## Development Install
|
|
263
|
+
|
|
264
|
+
For local development, use Python 3.11 or newer:
|
|
265
|
+
|
|
266
|
+
```bash
|
|
267
|
+
python3 -m venv .venv
|
|
268
|
+
source .venv/bin/activate
|
|
269
|
+
python -m pip install -e .
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
This installs `yt-dlp>=2026.06.09` as the runtime extractor dependency. Unit
|
|
273
|
+
tests use fixtures and mocks instead of making real YouTube or network calls.
|
|
274
|
+
|
|
275
|
+
Smoke-test the current CLI:
|
|
276
|
+
|
|
277
|
+
```bash
|
|
278
|
+
ytcap --help
|
|
279
|
+
ytcap --version
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
## Installation Target
|
|
283
|
+
|
|
284
|
+
The long-term installation target is:
|
|
285
|
+
|
|
286
|
+
```bash
|
|
287
|
+
pipx install ytcap
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
Expected usage:
|
|
291
|
+
|
|
292
|
+
```bash
|
|
293
|
+
ytcap --help
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
```bash
|
|
297
|
+
ytcap video --url "https://www.youtube.com/watch?v=VIDEO_ID" --lang en --source any
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
## License
|
|
301
|
+
|
|
302
|
+
This project is planned for release under the MIT License. See `LICENSE.md` for details.
|
ytcap-0.1.0/README.md
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
# ytcap
|
|
2
|
+
|
|
3
|
+
`ytcap` is a Python CLI project for extracting **video metadata** and **subtitle files** from YouTube video, batch, and playlist sources, then turning them into reusable JSON and JSONL outputs.
|
|
4
|
+
|
|
5
|
+
It is designed for workflows where you have YouTube video URLs, video IDs, or playlists and want structured metadata plus subtitles for search, indexing, dataset preparation, education, or analysis.
|
|
6
|
+
|
|
7
|
+
## Project Status
|
|
8
|
+
|
|
9
|
+
This repository is in an early planning and implementation stage. The first target release is:
|
|
10
|
+
|
|
11
|
+
```text
|
|
12
|
+
0.1.0
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Currently implemented:
|
|
16
|
+
|
|
17
|
+
- Importable Python package scaffold.
|
|
18
|
+
- Package version exposed as `ytcap.__version__`.
|
|
19
|
+
- Basic CLI entry point with `ytcap --help` and `ytcap --version`.
|
|
20
|
+
- CLI command structure for `inspect`, `video`, and `export`.
|
|
21
|
+
- `batch` command to process multiple video URLs or IDs from a text file, with run manifest logging, `--resume` and `--skip-existing` support.
|
|
22
|
+
- `yt-dlp` adapter support for `inspect` metadata extraction.
|
|
23
|
+
- Normalized video metadata mapping and inspect JSON summary output.
|
|
24
|
+
- Tested subtitle source selection for `manual`, `auto`, and `any` normalized tracks.
|
|
25
|
+
- Controlled subtitle format validation for `srt` and `vtt`.
|
|
26
|
+
- Standard output directory layout creation for `video --out`.
|
|
27
|
+
- `video` command metadata JSON writing and selected SRT/VTT subtitle file download.
|
|
28
|
+
- SRT/VTT cue parsing, cue-level JSONL writer helpers, and basic sentence-level segmentation helpers.
|
|
29
|
+
- `export` command conversion of existing SRT/VTT files to cue-level or sentence-level JSONL.
|
|
30
|
+
- `playlist` command to process videos inside a YouTube playlist with `--limit`, `--start`, and `--end` range controls, run manifest logging, `--resume`, `--skip-existing`, and `--dry-run`.
|
|
31
|
+
- Safe validation for dynamic output filename parts to prevent path traversal from user input or extractor metadata.
|
|
32
|
+
|
|
33
|
+
## Core Decisions
|
|
34
|
+
|
|
35
|
+
| Decision | Target |
|
|
36
|
+
|---|---|
|
|
37
|
+
| Language | Python |
|
|
38
|
+
| Minimum Python version | Python 3.11+ |
|
|
39
|
+
| CLI approach | Standard library first: `argparse` |
|
|
40
|
+
| Test approach | Standard library first: `unittest` |
|
|
41
|
+
| YouTube Data API | Not used |
|
|
42
|
+
| Metadata and subtitle extraction | `yt-dlp>=2026.06.09` is the core extractor dependency |
|
|
43
|
+
| Video/audio downloads | Out of scope for the MVP |
|
|
44
|
+
| Output format | JSON for metadata, JSONL for segment and sentence output |
|
|
45
|
+
| Distribution target | PyPI and `pipx install ytcap` |
|
|
46
|
+
| Additional dependencies | Require justification and approval before being added |
|
|
47
|
+
|
|
48
|
+
## Planned Capabilities
|
|
49
|
+
|
|
50
|
+
The MVP is intended to:
|
|
51
|
+
|
|
52
|
+
- Accept a single YouTube video URL or video ID.
|
|
53
|
+
- Save normalized video metadata as JSON.
|
|
54
|
+
- Find subtitles for a requested language.
|
|
55
|
+
- Prefer manual subtitles when available, with optional fallback to automatic subtitles.
|
|
56
|
+
- Save subtitles as SRT or VTT.
|
|
57
|
+
- Convert subtitles to cue-level or sentence-level JSONL.
|
|
58
|
+
- Report videos with missing or failed subtitle extraction.
|
|
59
|
+
|
|
60
|
+
Later releases may add:
|
|
61
|
+
|
|
62
|
+
- A dedicated retry command for failed records.
|
|
63
|
+
- PyPI publication.
|
|
64
|
+
- Automated test and release workflows through GitHub Actions.
|
|
65
|
+
|
|
66
|
+
## Non-Goals
|
|
67
|
+
|
|
68
|
+
`ytcap` intentionally does not:
|
|
69
|
+
|
|
70
|
+
- Use the official YouTube Data API.
|
|
71
|
+
- Manage API keys or OAuth flows.
|
|
72
|
+
- Download video or audio files as an MVP goal.
|
|
73
|
+
- Redistribute YouTube content or bypass access restrictions.
|
|
74
|
+
- Start with advanced NLP sentence segmentation libraries; simple, testable heuristics come first.
|
|
75
|
+
|
|
76
|
+
## Planned CLI Examples
|
|
77
|
+
|
|
78
|
+
These commands show the intended user experience. Some commands may not be implemented yet.
|
|
79
|
+
The current `inspect` command uses `yt-dlp` for metadata and subtitle
|
|
80
|
+
availability summaries. The `video` command extracts metadata through
|
|
81
|
+
`yt-dlp`, writes normalized metadata JSON, selects a matching subtitle track,
|
|
82
|
+
and saves the selected SRT/VTT subtitle file. SRT/VTT cue parsers,
|
|
83
|
+
cue-level JSONL writer helpers, and basic punctuation-based sentence
|
|
84
|
+
segmentation helpers are wired into the `export` command for existing
|
|
85
|
+
subtitle files.
|
|
86
|
+
|
|
87
|
+
### Inspect One Video
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
ytcap inspect --url "https://www.youtube.com/watch?v=VIDEO_ID"
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
This should answer:
|
|
94
|
+
|
|
95
|
+
- Is the video reachable?
|
|
96
|
+
- Can metadata be extracted?
|
|
97
|
+
- Which subtitle languages are available?
|
|
98
|
+
- Are subtitles manual, automatic, or both?
|
|
99
|
+
|
|
100
|
+
### Extract Metadata and Subtitles
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
ytcap video --url "https://www.youtube.com/watch?v=VIDEO_ID" --lang en --source any --format srt --out ./data
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
| Part | Meaning |
|
|
107
|
+
|---|---|
|
|
108
|
+
| `ytcap` | CLI application |
|
|
109
|
+
| `video` | Single-video processing command |
|
|
110
|
+
| `--url` | Video URL |
|
|
111
|
+
| `--lang en` | Request English subtitles |
|
|
112
|
+
| `--source any` | Try manual subtitles first, then automatic subtitles |
|
|
113
|
+
| `--format srt` | Save subtitles as SRT |
|
|
114
|
+
| `--out ./data` | Write outputs under `./data` |
|
|
115
|
+
|
|
116
|
+
Implemented source selection behavior uses exact language and format matches:
|
|
117
|
+
`manual` selects only manual subtitles, `auto` selects only automatic subtitles,
|
|
118
|
+
and `any` tries manual first before falling back to automatic subtitles.
|
|
119
|
+
Implemented subtitle format validation currently accepts `srt` and `vtt`; other
|
|
120
|
+
values return an `UNSUPPORTED_FORMAT` error before extraction work starts.
|
|
121
|
+
When run without `--dry-run`, `video` writes normalized metadata to
|
|
122
|
+
`videos/{video_id}.info.json` and the selected subtitle file to
|
|
123
|
+
`subtitles/{video_id}.{lang}.{source}.{format}` under the output directory.
|
|
124
|
+
If the requested subtitle cannot be selected or downloaded, the command returns
|
|
125
|
+
a controlled error without leaving a new partial metadata file behind.
|
|
126
|
+
|
|
127
|
+
### List Available Subtitles
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
ytcap inspect --url "https://www.youtube.com/watch?v=VIDEO_ID" --list-subs
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### Convert Existing Subtitles to JSONL
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
ytcap export --input ./data/subtitles/VIDEO_ID.en.manual.srt --segments cue --out ./data/normalized
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
ytcap export --input ./data/subtitles --segments sentence --out ./data/normalized
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The `export` command reads existing `.srt` and `.vtt` files and writes JSONL
|
|
144
|
+
records to `{video_id}.{lang}.{segments}.jsonl` under the output directory. It
|
|
145
|
+
infers `video_id`, language, and source from names such as
|
|
146
|
+
`VIDEO_ID.en.manual.srt`; when the source is missing, JSONL records use
|
|
147
|
+
`"source":"unknown"`. `--video-id` and `--lang` may override metadata for a
|
|
148
|
+
single file input.
|
|
149
|
+
|
|
150
|
+
Dynamic filename parts such as video ID, language, source, format, segment type,
|
|
151
|
+
and run ID are validated before paths are built. Empty values, path separators,
|
|
152
|
+
control characters, absolute paths, `.` and `..` are rejected.
|
|
153
|
+
|
|
154
|
+
### Process a Batch File
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
ytcap batch --input videos.txt --lang en --source any --format srt --resume --skip-existing --out ./data
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
This command parses the input file and processes each URL/ID. It creates a run
|
|
161
|
+
manifest under `runs/{run_id}.manifest.json` keeping track of execution
|
|
162
|
+
statistics, output files, and errors. Failed attempts are appended to
|
|
163
|
+
`failed/failed.jsonl`. `--resume` skips entries completed in the latest
|
|
164
|
+
manifest and retries previous failures, while `--skip-existing` skips videos
|
|
165
|
+
whose metadata and subtitle files already exist for the requested language,
|
|
166
|
+
source, and format. `--dry-run` reports the batch plan without writing files or
|
|
167
|
+
creating output directories.
|
|
168
|
+
|
|
169
|
+
#### Batch Input File Format
|
|
170
|
+
|
|
171
|
+
The `--input` file for the `batch` command is a plain text file containing one YouTube video URL or video ID per line.
|
|
172
|
+
- Empty lines and lines containing only whitespace are ignored.
|
|
173
|
+
- Lines starting with `#` (with optional leading whitespace) are ignored as comment lines.
|
|
174
|
+
- Inline comments starting with `#` are supported, and the comment text plus any preceding whitespace are ignored.
|
|
175
|
+
|
|
176
|
+
Example input file:
|
|
177
|
+
```text
|
|
178
|
+
# This is a comment line
|
|
179
|
+
dQw4w9WgXcQ # Rick Astley - Never Gonna Give You Up
|
|
180
|
+
https://youtu.be/jNQXAC9IVRw # Another video URL
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
### Process a Playlist
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
ytcap playlist --url "https://www.youtube.com/playlist?list=PLAYLIST_ID" --start 1 --limit 50 --lang en --source any --format srt --out ./data
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
The `playlist` command uses `yt-dlp` flat playlist extraction to collect video
|
|
190
|
+
entries without the official YouTube Data API, then processes each video with
|
|
191
|
+
the same metadata and subtitle flow as `video`. `--start` is 1-based, `--end`
|
|
192
|
+
is inclusive, and `--limit` caps the selected range. `--resume` continues only
|
|
193
|
+
from a matching playlist run manifest, while `--skip-existing` skips videos
|
|
194
|
+
only when matching metadata and subtitle files already exist.
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
## Output Layout
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
data/
|
|
201
|
+
videos/
|
|
202
|
+
VIDEO_ID.info.json
|
|
203
|
+
subtitles/
|
|
204
|
+
VIDEO_ID.en.manual.srt
|
|
205
|
+
normalized/
|
|
206
|
+
VIDEO_ID.en.cue.jsonl
|
|
207
|
+
runs/
|
|
208
|
+
RUN_ID.manifest.json
|
|
209
|
+
failed/
|
|
210
|
+
failed.jsonl
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Example cue-level JSONL line:
|
|
214
|
+
|
|
215
|
+
```json
|
|
216
|
+
{"schema_version":"0.1","type":"cue","video_id":"VIDEO_ID","language":"en","source":"manual","start":1.0,"end":3.5,"text":"Example subtitle text.","cue_index":1}
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Sentence-level segmentation uses a simple standard-library heuristic that
|
|
220
|
+
splits on `.`, `?`, and `!`. Timing is marked with a strategy such as
|
|
221
|
+
`cue_exact`, `cue_merge`, `heuristic`, or `unknown` because sentence boundaries
|
|
222
|
+
can fall inside or across subtitle cues.
|
|
223
|
+
|
|
224
|
+
## Documentation
|
|
225
|
+
|
|
226
|
+
| File | Purpose |
|
|
227
|
+
|---|---|
|
|
228
|
+
| `USAGE.md` | Usage boundaries, limitations, and responsible use notes |
|
|
229
|
+
| `CLI_REFERENCE.md` | Planned commands, flags, behavior, and error codes |
|
|
230
|
+
| `OUTPUT_FORMAT.md` | Target JSON and JSONL output formats |
|
|
231
|
+
| `RELEASE.md` | Packaging and release process |
|
|
232
|
+
| `CONTRIBUTING.md` | Contributor expectations |
|
|
233
|
+
| `SECURITY.md` | Security policy and sensitive data rules |
|
|
234
|
+
|
|
235
|
+
## Development Install
|
|
236
|
+
|
|
237
|
+
For local development, use Python 3.11 or newer:
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
python3 -m venv .venv
|
|
241
|
+
source .venv/bin/activate
|
|
242
|
+
python -m pip install -e .
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
This installs `yt-dlp>=2026.06.09` as the runtime extractor dependency. Unit
|
|
246
|
+
tests use fixtures and mocks instead of making real YouTube or network calls.
|
|
247
|
+
|
|
248
|
+
Smoke-test the current CLI:
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
ytcap --help
|
|
252
|
+
ytcap --version
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
## Installation Target
|
|
256
|
+
|
|
257
|
+
The long-term installation target is:
|
|
258
|
+
|
|
259
|
+
```bash
|
|
260
|
+
pipx install ytcap
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Expected usage:
|
|
264
|
+
|
|
265
|
+
```bash
|
|
266
|
+
ytcap --help
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
```bash
|
|
270
|
+
ytcap video --url "https://www.youtube.com/watch?v=VIDEO_ID" --lang en --source any
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
## License
|
|
274
|
+
|
|
275
|
+
This project is planned for release under the MIT License. See `LICENSE.md` for details.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0.3"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ytcap"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Extract YouTube metadata and subtitles into JSON and JSONL outputs."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE.md"]
|
|
13
|
+
authors = [
|
|
14
|
+
{name = "Özkan Öztürk"}
|
|
15
|
+
]
|
|
16
|
+
keywords = ["youtube", "subtitles", "metadata", "jsonl", "yt-dlp"]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
19
|
+
"Environment :: Console",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Multimedia :: Video",
|
|
26
|
+
"Topic :: Text Processing",
|
|
27
|
+
]
|
|
28
|
+
dependencies = [
|
|
29
|
+
"yt-dlp>=2026.06.09",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
dev = [
|
|
34
|
+
"build",
|
|
35
|
+
"twine",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[project.urls]
|
|
39
|
+
Homepage = "https://github.com/ozkozturk/ytcap"
|
|
40
|
+
Documentation = "https://github.com/ozkozturk/ytcap#readme"
|
|
41
|
+
|
|
42
|
+
[project.scripts]
|
|
43
|
+
ytcap = "ytcap.cli:main"
|
|
44
|
+
|
|
45
|
+
[tool.setuptools]
|
|
46
|
+
package-dir = {"" = "src"}
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.dynamic]
|
|
49
|
+
version = {attr = "ytcap.__version__"}
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
ytcap-0.1.0/setup.cfg
ADDED