tube2note 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tube2note-0.1.1/LICENSE +21 -0
- tube2note-0.1.1/PKG-INFO +73 -0
- tube2note-0.1.1/README.md +55 -0
- tube2note-0.1.1/pyproject.toml +27 -0
- tube2note-0.1.1/setup.cfg +4 -0
- tube2note-0.1.1/tube2note.egg-info/PKG-INFO +73 -0
- tube2note-0.1.1/tube2note.egg-info/SOURCES.txt +10 -0
- tube2note-0.1.1/tube2note.egg-info/dependency_links.txt +1 -0
- tube2note-0.1.1/tube2note.egg-info/entry_points.txt +2 -0
- tube2note-0.1.1/tube2note.egg-info/requires.txt +1 -0
- tube2note-0.1.1/tube2note.egg-info/top_level.txt +1 -0
- tube2note-0.1.1/tube2note.py +1040 -0
tube2note-0.1.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 tube2note contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tube2note-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tube2note
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: YouTube channels, playlists and videos to a single Markdown file for NotebookLM
|
|
5
|
+
License: MIT
|
|
6
|
+
Keywords: youtube,transcript,markdown,notebooklm,cli,rag
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Utilities
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: yt-dlp
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# tube2note — YouTube → Markdown for NotebookLM
|
|
20
|
+
|
|
21
|
+
Turn a YouTube channel, playlist, or list of videos into **one Markdown file** (or a folder of them) ready to feed NotebookLM / any RAG pipeline. Single file, zero-install besides `yt-dlp`, resumable, polite to YouTube's rate limits.
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install yt-dlp
|
|
25
|
+
python3 tube2note.py -o channel.md "https://www.youtube.com/@SomeChannel/videos"
|
|
26
|
+
# or guided mode:
|
|
27
|
+
python3 tube2note.py
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Why not just paste YouTube links into NotebookLM?
|
|
31
|
+
|
|
32
|
+
- NotebookLM caps YouTube imports (~100 videos) and needs caption files per video.
|
|
33
|
+
- tube2note merges everything into Markdown sources you control: one file per 500k-word cap, timestamps, per-video files, resume after interruptions.
|
|
34
|
+
|
|
35
|
+
## Features
|
|
36
|
+
|
|
37
|
+
- **Channel / playlist / video URLs** (auto-detected listing, Shorts/tabs handled)
|
|
38
|
+
- **Guided TUI**: intro, first-run guide, auto-detected file name / video count / languages, confirm table
|
|
39
|
+
- **Anti-throttle engine**: chunks with breaks, jittered pacing, single retry on 429, long cooldown after 5 throttles in a row
|
|
40
|
+
- **Resume**: `.done` log — Ctrl+C anytime, continue later; skip log reconciled every run
|
|
41
|
+
- **Layouts**: `single` (one .md), `videos` (per-video files + `INDEX.md`), `tree` (`Channel/Video/transcript.md` + `INDEX.md` + YAML frontmatter)
|
|
42
|
+
- **Name templates**: `--name-template "{channel}/{title} [{id}]"`
|
|
43
|
+
- **Profiles + env**: `--profile X`, `YT2MD_*` vars, per-folder `.yt2md.json` overrides, `setup` wizard
|
|
44
|
+
- **NotebookLM-aware**: timestamps option, `--split-words` auto-split under the 500k-word cap
|
|
45
|
+
- **Status & dry-run**: `status [dir]` progress table, `--dry-run` estimate before downloading
|
|
46
|
+
|
|
47
|
+
## Quickstart
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
git clone <repo-url> && cd tube2note
|
|
51
|
+
pip install yt-dlp
|
|
52
|
+
python3 tube2note.py -o notes.md "<playlist_url>" "<video_url>" ...
|
|
53
|
+
python3 tube2note.py -o channel.md --max 200 --layout tree -d ./out "https://www.youtube.com/@SomeChannel/videos"
|
|
54
|
+
tube2note status ./out
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## NotebookLM limits (verified 2026)
|
|
58
|
+
|
|
59
|
+
- **Per source: 500,000 words / 200 MB** — identical on every plan.
|
|
60
|
+
- Sources per notebook: 50 (Free) → up to 600 (Ultra).
|
|
61
|
+
- Rule of thumb: one merged `.md` until 500k words, then `--split-words 400000` into multiple sources.
|
|
62
|
+
|
|
63
|
+
## FAQ
|
|
64
|
+
|
|
65
|
+
**HTTP 429?** YouTube throttles sustained subtitle downloads. tube2note slows down automatically (chunks, cooldowns). If throttled hard: stop, wait ~1h (retries extend the ban), resume — progress is saved.
|
|
66
|
+
|
|
67
|
+
**No subtitles for a video?** Skipped and listed at the end (`## Skipped`) + in `INDEX.md`.
|
|
68
|
+
|
|
69
|
+
**Big channels?** Use `--max`, `--chunk 25`, longer `--chunk-cooldown`, or run overnight. Resume anytime.
|
|
70
|
+
|
|
71
|
+
## License
|
|
72
|
+
|
|
73
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# tube2note — YouTube → Markdown for NotebookLM
|
|
2
|
+
|
|
3
|
+
Turn a YouTube channel, playlist, or list of videos into **one Markdown file** (or a folder of them) ready to feed NotebookLM / any RAG pipeline. Single file, zero-install besides `yt-dlp`, resumable, polite to YouTube's rate limits.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install yt-dlp
|
|
7
|
+
python3 tube2note.py -o channel.md "https://www.youtube.com/@SomeChannel/videos"
|
|
8
|
+
# or guided mode:
|
|
9
|
+
python3 tube2note.py
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Why not just paste YouTube links into NotebookLM?
|
|
13
|
+
|
|
14
|
+
- NotebookLM caps YouTube imports (~100 videos) and needs caption files per video.
|
|
15
|
+
- tube2note merges everything into Markdown sources you control: one file per 500k-word cap, timestamps, per-video files, resume after interruptions.
|
|
16
|
+
|
|
17
|
+
## Features
|
|
18
|
+
|
|
19
|
+
- **Channel / playlist / video URLs** (auto-detected listing, Shorts/tabs handled)
|
|
20
|
+
- **Guided TUI**: intro, first-run guide, auto-detected file name / video count / languages, confirm table
|
|
21
|
+
- **Anti-throttle engine**: chunks with breaks, jittered pacing, single retry on 429, long cooldown after 5 throttles in a row
|
|
22
|
+
- **Resume**: `.done` log — Ctrl+C anytime, continue later; skip log reconciled every run
|
|
23
|
+
- **Layouts**: `single` (one .md), `videos` (per-video files + `INDEX.md`), `tree` (`Channel/Video/transcript.md` + `INDEX.md` + YAML frontmatter)
|
|
24
|
+
- **Name templates**: `--name-template "{channel}/{title} [{id}]"`
|
|
25
|
+
- **Profiles + env**: `--profile X`, `YT2MD_*` vars, per-folder `.yt2md.json` overrides, `setup` wizard
|
|
26
|
+
- **NotebookLM-aware**: timestamps option, `--split-words` auto-split under the 500k-word cap
|
|
27
|
+
- **Status & dry-run**: `status [dir]` progress table, `--dry-run` estimate before downloading
|
|
28
|
+
|
|
29
|
+
## Quickstart
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
git clone <repo-url> && cd tube2note
|
|
33
|
+
pip install yt-dlp
|
|
34
|
+
python3 tube2note.py -o notes.md "<playlist_url>" "<video_url>" ...
|
|
35
|
+
python3 tube2note.py -o channel.md --max 200 --layout tree -d ./out "https://www.youtube.com/@SomeChannel/videos"
|
|
36
|
+
tube2note status ./out
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## NotebookLM limits (verified 2026)
|
|
40
|
+
|
|
41
|
+
- **Per source: 500,000 words / 200 MB** — identical on every plan.
|
|
42
|
+
- Sources per notebook: 50 (Free) → up to 600 (Ultra).
|
|
43
|
+
- Rule of thumb: one merged `.md` until 500k words, then `--split-words 400000` into multiple sources.
|
|
44
|
+
|
|
45
|
+
## FAQ
|
|
46
|
+
|
|
47
|
+
**HTTP 429?** YouTube throttles sustained subtitle downloads. tube2note slows down automatically (chunks, cooldowns). If throttled hard: stop, wait ~1h (retries extend the ban), resume — progress is saved.
|
|
48
|
+
|
|
49
|
+
**No subtitles for a video?** Skipped and listed at the end (`## Skipped`) + in `INDEX.md`.
|
|
50
|
+
|
|
51
|
+
**Big channels?** Use `--max`, `--chunk 25`, longer `--chunk-cooldown`, or run overnight. Resume anytime.
|
|
52
|
+
|
|
53
|
+
## License
|
|
54
|
+
|
|
55
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tube2note"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "YouTube channels, playlists and videos to a single Markdown file for NotebookLM"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
keywords = ["youtube", "transcript", "markdown", "notebooklm", "cli", "rag"]
|
|
13
|
+
dependencies = ["yt-dlp"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Utilities",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.scripts]
|
|
24
|
+
tube2note = "tube2note:main"
|
|
25
|
+
|
|
26
|
+
[tool.setuptools]
|
|
27
|
+
py-modules = ["tube2note"]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tube2note
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: YouTube channels, playlists and videos to a single Markdown file for NotebookLM
|
|
5
|
+
License: MIT
|
|
6
|
+
Keywords: youtube,transcript,markdown,notebooklm,cli,rag
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Utilities
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: yt-dlp
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# tube2note — YouTube → Markdown for NotebookLM
|
|
20
|
+
|
|
21
|
+
Turn a YouTube channel, playlist, or list of videos into **one Markdown file** (or a folder of them) ready to feed NotebookLM / any RAG pipeline. Single file, zero-install besides `yt-dlp`, resumable, polite to YouTube's rate limits.
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install yt-dlp
|
|
25
|
+
python3 tube2note.py -o channel.md "https://www.youtube.com/@SomeChannel/videos"
|
|
26
|
+
# or guided mode:
|
|
27
|
+
python3 tube2note.py
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Why not just paste YouTube links into NotebookLM?
|
|
31
|
+
|
|
32
|
+
- NotebookLM caps YouTube imports (~100 videos) and needs caption files per video.
|
|
33
|
+
- tube2note merges everything into Markdown sources you control: one file per 500k-word cap, timestamps, per-video files, resume after interruptions.
|
|
34
|
+
|
|
35
|
+
## Features
|
|
36
|
+
|
|
37
|
+
- **Channel / playlist / video URLs** (auto-detected listing, Shorts/tabs handled)
|
|
38
|
+
- **Guided TUI**: intro, first-run guide, auto-detected file name / video count / languages, confirm table
|
|
39
|
+
- **Anti-throttle engine**: chunks with breaks, jittered pacing, single retry on 429, long cooldown after 5 throttles in a row
|
|
40
|
+
- **Resume**: `.done` log — Ctrl+C anytime, continue later; skip log reconciled every run
|
|
41
|
+
- **Layouts**: `single` (one .md), `videos` (per-video files + `INDEX.md`), `tree` (`Channel/Video/transcript.md` + `INDEX.md` + YAML frontmatter)
|
|
42
|
+
- **Name templates**: `--name-template "{channel}/{title} [{id}]"`
|
|
43
|
+
- **Profiles + env**: `--profile X`, `YT2MD_*` vars, per-folder `.yt2md.json` overrides, `setup` wizard
|
|
44
|
+
- **NotebookLM-aware**: timestamps option, `--split-words` auto-split under the 500k-word cap
|
|
45
|
+
- **Status & dry-run**: `status [dir]` progress table, `--dry-run` estimate before downloading
|
|
46
|
+
|
|
47
|
+
## Quickstart
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
git clone <repo-url> && cd tube2note
|
|
51
|
+
pip install yt-dlp
|
|
52
|
+
python3 tube2note.py -o notes.md "<playlist_url>" "<video_url>" ...
|
|
53
|
+
python3 tube2note.py -o channel.md --max 200 --layout tree -d ./out "https://www.youtube.com/@SomeChannel/videos"
|
|
54
|
+
tube2note status ./out
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## NotebookLM limits (verified 2026)
|
|
58
|
+
|
|
59
|
+
- **Per source: 500,000 words / 200 MB** — identical on every plan.
|
|
60
|
+
- Sources per notebook: 50 (Free) → up to 600 (Ultra).
|
|
61
|
+
- Rule of thumb: one merged `.md` until 500k words, then `--split-words 400000` into multiple sources.
|
|
62
|
+
|
|
63
|
+
## FAQ
|
|
64
|
+
|
|
65
|
+
**HTTP 429?** YouTube throttles sustained subtitle downloads. tube2note slows down automatically (chunks, cooldowns). If throttled hard: stop, wait ~1h (retries extend the ban), resume — progress is saved.
|
|
66
|
+
|
|
67
|
+
**No subtitles for a video?** Skipped and listed at the end (`## Skipped`) + in `INDEX.md`.
|
|
68
|
+
|
|
69
|
+
**Big channels?** Use `--max`, `--chunk 25`, longer `--chunk-cooldown`, or run overnight. Resume anytime.
|
|
70
|
+
|
|
71
|
+
## License
|
|
72
|
+
|
|
73
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
tube2note.py
|
|
5
|
+
tube2note.egg-info/PKG-INFO
|
|
6
|
+
tube2note.egg-info/SOURCES.txt
|
|
7
|
+
tube2note.egg-info/dependency_links.txt
|
|
8
|
+
tube2note.egg-info/entry_points.txt
|
|
9
|
+
tube2note.egg-info/requires.txt
|
|
10
|
+
tube2note.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
yt-dlp
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tube2note
|
|
@@ -0,0 +1,1040 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""tube2note: merge YouTube channel/playlist/video subtitles into Markdown files.
|
|
3
|
+
Usage:
|
|
4
|
+
python3 tube2note.py -o notes.md "<playlist_url>" "<video_url>" ...
|
|
5
|
+
python3 tube2note.py -o channel.md --max 50 --lang tr,en "https://www.youtube.com/@channel/videos"
|
|
6
|
+
yt # guided interactive mode
|
|
7
|
+
yt setup # personalize defaults (folder, layout, ...)
|
|
8
|
+
python3 tube2note.py status [dir] # progress table of saved collections
|
|
9
|
+
python3 tube2note.py --dry-run "<playlist_url>" # preview only, no download
|
|
10
|
+
Input: channel / playlist / single video URLs. Output: one Markdown file to feed NotebookLM.
|
|
11
|
+
Requires: pip install yt-dlp (no ffmpeg needed)
|
|
12
|
+
"""
|
|
13
|
+
import argparse
|
|
14
|
+
import datetime
|
|
15
|
+
import html
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import random
|
|
19
|
+
import re
|
|
20
|
+
import shutil
|
|
21
|
+
import sys
|
|
22
|
+
import time
|
|
23
|
+
import urllib.request
|
|
24
|
+
|
|
25
|
+
from yt_dlp import YoutubeDL
|
|
26
|
+
|
|
27
|
+
TAG_RE = re.compile(r"<[^>]+>")
|
|
28
|
+
ANSI_RE = re.compile(r"\033\[[0-9;]*m")
|
|
29
|
+
UI_ON = sys.stdout.isatty() and os.environ.get("NO_COLOR") is None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _len(s):
|
|
33
|
+
return len(ANSI_RE.sub("", s))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _pad(s, w):
|
|
37
|
+
return s + " " * max(0, w - _len(s))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _c(code, s):
|
|
41
|
+
return f"\033[{code}m{s}\033[0m" if UI_ON else s
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def bold(s):
|
|
45
|
+
return _c("1", s)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def green(s):
|
|
49
|
+
return _c("32", s)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def yellow(s):
|
|
53
|
+
return _c("33", s)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def red(s):
|
|
57
|
+
return _c("31", s)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def dim(s):
|
|
61
|
+
return _c("2", s)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def cyan(s):
|
|
65
|
+
return _c("36", s)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def panel(title, lines):
|
|
69
|
+
rows = [title] + list(lines)
|
|
70
|
+
w = max(_len(l) for l in rows)
|
|
71
|
+
edge = "─" * (w + 2)
|
|
72
|
+
out = [f"┌{edge}┐", f"│ {_pad(bold(title), w)} │", f"├{edge}┤"]
|
|
73
|
+
out += [f"│ {_pad(l, w)} │" for l in lines]
|
|
74
|
+
out.append(f"└{edge}┘")
|
|
75
|
+
return "\n".join(out)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def table(headers, rows):
|
|
79
|
+
widths = [_len(h) for h in headers]
|
|
80
|
+
for r in rows:
|
|
81
|
+
for j, c in enumerate(r):
|
|
82
|
+
widths[j] = max(widths[j], _len(str(c)))
|
|
83
|
+
sep = "─┼─".join("─" * w for w in widths)
|
|
84
|
+
head = " │ ".join(_pad(bold(h), w) for h, w in zip(headers, widths))
|
|
85
|
+
out = [head, sep]
|
|
86
|
+
for r in rows:
|
|
87
|
+
out.append(" │ ".join(_pad(str(c), w) for c, w in zip(r, widths)))
|
|
88
|
+
return "\n".join(out)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def bar(frac, width=24):
|
|
92
|
+
frac = max(0.0, min(1.0, frac))
|
|
93
|
+
fill = int(frac * width)
|
|
94
|
+
return f"[{'█' * fill}{'░' * (width - fill)}] {frac * 100:4.0f}%"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _term_width():
|
|
98
|
+
try:
|
|
99
|
+
return max(40, shutil.get_terminal_size().columns)
|
|
100
|
+
except Exception:
|
|
101
|
+
return 80
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _short(s, w):
|
|
105
|
+
s = str(s)
|
|
106
|
+
return s if len(s) <= w else s[: max(0, w - 1)] + "…"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _render_dash(done_n, todo_n, title, ok_n, skip_n, words, t0, status=""):
|
|
110
|
+
"""Pure: build the 3 dashboard lines (testable without a terminal)."""
|
|
111
|
+
w = _term_width()
|
|
112
|
+
el = (time.time() - t0) / 60
|
|
113
|
+
frac = done_n / todo_n if todo_n else 0
|
|
114
|
+
l1 = _short(f"{bar(frac)} {done_n}/{todo_n} · {words} words · {el:.0f} min", w)
|
|
115
|
+
l2 = _short(f"▶ {title}", w)
|
|
116
|
+
l3 = _short(f"ok {ok_n} · skipped {skip_n}" + (f" · {status}" if status else ""), w)
|
|
117
|
+
return [l1, l2, l3]
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
_dash_lines = 0 # lines currently drawn (0 = nothing on screen yet)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
_VERBOSE = False
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def dash_update(done_n, todo_n, title, ok_n, skip_n, words, t0, status=""):
|
|
127
|
+
"""Redraw the single in-place dashboard (TTY) or plain lines (logs)."""
|
|
128
|
+
global _dash_lines
|
|
129
|
+
if _VERBOSE:
|
|
130
|
+
el = (time.time() - t0) / 60
|
|
131
|
+
extra = f" · {status}" if status else ""
|
|
132
|
+
print(f"{done_n}/{todo_n} videos · {words} words · {el:.0f} min{extra}", flush=True)
|
|
133
|
+
return
|
|
134
|
+
if not UI_ON or todo_n <= 0:
|
|
135
|
+
if todo_n > 0 and (done_n % 10 == 0 or done_n >= todo_n):
|
|
136
|
+
print(f"{done_n}/{todo_n} videos · {words} words", flush=True)
|
|
137
|
+
return
|
|
138
|
+
lines = _render_dash(done_n, todo_n, title, ok_n, skip_n, words, t0, status)
|
|
139
|
+
if _dash_lines:
|
|
140
|
+
sys.stdout.write(f"\x1b[{_dash_lines}A")
|
|
141
|
+
for l in lines:
|
|
142
|
+
sys.stdout.write("\r\x1b[K" + l + "\n")
|
|
143
|
+
sys.stdout.flush()
|
|
144
|
+
_dash_lines = len(lines)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def log(msg):
|
|
148
|
+
"""Log line that cleanly breaks the live dashboard."""
|
|
149
|
+
global _dash_lines
|
|
150
|
+
if _dash_lines and UI_ON:
|
|
151
|
+
sys.stdout.write("\n")
|
|
152
|
+
_dash_lines = 0
|
|
153
|
+
print(msg, flush=True)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def dash_end():
|
|
157
|
+
global _dash_lines
|
|
158
|
+
_dash_lines = 0
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _to_secs(t):
|
|
162
|
+
try:
|
|
163
|
+
p = t.strip().split(":")
|
|
164
|
+
if len(p) == 3:
|
|
165
|
+
return int(p[0]) * 3600 + int(p[1]) * 60 + float(p[2])
|
|
166
|
+
return int(p[0]) * 60 + float(p[1])
|
|
167
|
+
except (ValueError, IndexError, AttributeError):
|
|
168
|
+
return 0.0
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _fmt_ts(s):
|
|
172
|
+
s = int(s)
|
|
173
|
+
h, m = s // 3600, s % 3600 // 60
|
|
174
|
+
return f"{h}:{m:02d}:{s % 60:02d}" if h else f"{m:02d}:{s % 60:02d}"
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def vtt_to_text(vtt: str, ts: bool = False) -> str:
|
|
178
|
+
lines = [] # (start_secs, text)
|
|
179
|
+
start = 0.0
|
|
180
|
+
for block in vtt.splitlines():
|
|
181
|
+
s = block.strip()
|
|
182
|
+
if not s or s == "WEBVTT" or s.startswith("NOTE") or s.startswith("STYLE") or s.startswith("REGION"):
|
|
183
|
+
continue
|
|
184
|
+
if "-->" in s:
|
|
185
|
+
start = _to_secs(s.split("-->")[0])
|
|
186
|
+
continue
|
|
187
|
+
if s.isdigit():
|
|
188
|
+
continue
|
|
189
|
+
s = TAG_RE.sub("", s)
|
|
190
|
+
s = html.unescape(s).replace(" ", " ").strip()
|
|
191
|
+
if s and (not lines or lines[-1][1] != s): # drop back-to-back duplicates from auto captions
|
|
192
|
+
lines.append((start, s))
|
|
193
|
+
# join into paragraphs of ~20 lines so the flow stays readable
|
|
194
|
+
paras, buf = [], []
|
|
195
|
+
for i, (st, ln) in enumerate(lines, 1):
|
|
196
|
+
buf.append((st, ln))
|
|
197
|
+
if i % 20 == 0:
|
|
198
|
+
paras.append(buf)
|
|
199
|
+
buf = []
|
|
200
|
+
if buf:
|
|
201
|
+
paras.append(buf)
|
|
202
|
+
out = []
|
|
203
|
+
for p in paras:
|
|
204
|
+
body = " ".join(t for _, t in p)
|
|
205
|
+
out.append(f"[{_fmt_ts(p[0][0])}] {body}" if ts else body)
|
|
206
|
+
return "\n\n".join(out)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _self_test():
|
|
210
|
+
vtt = "WEBVTT\n\n00:00.000 --> 00:01.000\nmerhaba <b>dünya</b>\n\n00:01.000 --> 00:02.000\nmerhaba <b>dünya</b>\n"
|
|
211
|
+
assert vtt_to_text(vtt) == "merhaba dünya", vtt_to_text(vtt)
|
|
212
|
+
assert vtt_to_text("WEBVTT\n\n00:01.500 --> 00:03.000\nhello\n", ts=True) == "[00:01] hello"
|
|
213
|
+
class E(Exception):
|
|
214
|
+
code = 429
|
|
215
|
+
assert _is_throttle(E()) and not _is_throttle(ValueError())
|
|
216
|
+
assert slug("PickY Audio!") == "picky-audio.md"
|
|
217
|
+
assert _merge(dict(DEFAULTS), {"defaults": {"lang": "en"}, "profiles": {"p": {"lang": "de"}}},
|
|
218
|
+
"p", {"chunk": 5}, {"YT2MD_LANG": "fr"}) == {**DEFAULTS, "lang": "fr", "chunk": 5}
|
|
219
|
+
assert _merge(dict(DEFAULTS), {"defaults": {}, "profiles": {"p": {"lang": "de"}}},
|
|
220
|
+
"p", {}, {})["lang"] == "de"
|
|
221
|
+
assert sanitize_filename("a<b>c:.md") == "a-b-c-.md"
|
|
222
|
+
assert sanitize_filename("CON") == "_CON"
|
|
223
|
+
assert len(sanitize_filename("x" * 300).encode()) <= 200
|
|
224
|
+
assert render_template("{channel}/{title} [{id}]", {"channel": "C", "title": "T", "id": "ID1"}) == "C/T [ID1].md"
|
|
225
|
+
assert render_template("{nope}/x", {"a": "b"}) == "x.md"
|
|
226
|
+
assert _unique_path("a/b.md", "ID1", {"a/b.md"}) == "a/b_ID1.md"
|
|
227
|
+
assert "50%" in bar(0.5) and bar(2.0).startswith("[█")
|
|
228
|
+
assert "Name" in table(["Name", "Val"], [["a", "1"]])
|
|
229
|
+
assert "tube2note" in panel("tube2note", ["x"])
|
|
230
|
+
assert YT_RE.search("https://www.youtube.com/watch?v=dQw4w9WgXcQ")
|
|
231
|
+
assert not YT_RE.search("https://example.com/foo")
|
|
232
|
+
d = _render_dash(1, 2, "hello world, this title is quite long", 1, 0, 10, time.time())
|
|
233
|
+
assert len(d) == 3 and "50%" in d[0] and "hello" in d[1] and "ok 1" in d[2]
|
|
234
|
+
log("test-line")
|
|
235
|
+
dash_update(1, 1, "t", 1, 0, 5, time.time())
|
|
236
|
+
dash_end()
|
|
237
|
+
info = {"subtitles": {"en": [{"url": "http://x/v?lang=en&fmt=vtt", "ext": "vtt"}]},
|
|
238
|
+
"automatic_captions": {"tr": [{"url": "http://x/?tlang=tr", "ext": "vtt"}]}}
|
|
239
|
+
assert pick_sub(info, ["tr", "en"])[0] == "en" # translated tracks must lose to originals
|
|
240
|
+
print("self-test ok")
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _flatten(ydl, info, depth=0):
|
|
244
|
+
"""Channel -> tabs (Videos/Shorts/Live) -> videos: flatten up to two levels."""
|
|
245
|
+
entries = info.get("entries")
|
|
246
|
+
if info.get("_type") not in ("playlist", "multi_video", "compat_batch") or not entries:
|
|
247
|
+
vid = info.get("id") or ""
|
|
248
|
+
if len(vid) == 11: # video IDs are 11 chars; channels (UC..) / playlists (PL..) are not
|
|
249
|
+
return [{"id": vid, "title": info.get("title") or vid,
|
|
250
|
+
"channel": info.get("channel") or info.get("uploader"),
|
|
251
|
+
"url": f"https://www.youtube.com/watch?v={vid}"}]
|
|
252
|
+
return []
|
|
253
|
+
out = []
|
|
254
|
+
for e in entries:
|
|
255
|
+
if e is None:
|
|
256
|
+
continue
|
|
257
|
+
if e.get("entries"):
|
|
258
|
+
out.extend(_flatten(ydl, e, depth + 1))
|
|
259
|
+
elif e.get("ie_key") == "Youtube" or len(e.get("id", "")) == 11:
|
|
260
|
+
vid = e.get("id")
|
|
261
|
+
out.append({"id": vid, "title": e.get("title") or vid,
|
|
262
|
+
"channel": e.get("channel") or e.get("uploader"),
|
|
263
|
+
"url": f"https://www.youtube.com/watch?v={vid}"})
|
|
264
|
+
elif depth < 2 and e.get("url"):
|
|
265
|
+
try:
|
|
266
|
+
sub = ydl.extract_info(e["url"], download=False)
|
|
267
|
+
except Exception:
|
|
268
|
+
continue
|
|
269
|
+
if sub:
|
|
270
|
+
out.extend(_flatten(ydl, sub, depth + 1))
|
|
271
|
+
return out
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def expand(urls, max_n):
|
|
275
|
+
ydl_opts = {"quiet": True, "no_warnings": True, "extract_flat": True, "socket_timeout": 20}
|
|
276
|
+
out, hint = [], None
|
|
277
|
+
with YoutubeDL(ydl_opts) as ydl:
|
|
278
|
+
for u in urls:
|
|
279
|
+
try:
|
|
280
|
+
info = ydl.extract_info(u, download=False)
|
|
281
|
+
except Exception as e:
|
|
282
|
+
print(f"! could not list {u}: {e}", file=sys.stderr)
|
|
283
|
+
continue
|
|
284
|
+
if not info:
|
|
285
|
+
continue
|
|
286
|
+
if hint is None and len(urls) == 1 and info.get("title"):
|
|
287
|
+
hint = info.get("title")
|
|
288
|
+
out.extend(_flatten(ydl, info))
|
|
289
|
+
if len(out) >= max_n:
|
|
290
|
+
break
|
|
291
|
+
# dedupe in case the same video came from two lists
|
|
292
|
+
seen, uniq = set(), []
|
|
293
|
+
for v in out:
|
|
294
|
+
if v["id"] not in seen:
|
|
295
|
+
seen.add(v["id"])
|
|
296
|
+
uniq.append(v)
|
|
297
|
+
return uniq[:max_n], hint
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def pick_sub(info, langs):
|
|
301
|
+
# ponytail: original tracks only; tlang translations are the most 429-prone kind
|
|
302
|
+
subs, autos = info.get("subtitles") or {}, info.get("automatic_captions") or {}
|
|
303
|
+
for pool in (subs, autos):
|
|
304
|
+
for lg in langs:
|
|
305
|
+
if lg in pool:
|
|
306
|
+
fmts = [f for f in pool[lg] if "tlang=" not in (f.get("url") or "")]
|
|
307
|
+
if fmts:
|
|
308
|
+
return lg, fmts, pool is autos
|
|
309
|
+
return None, None, False
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def fetch_vtt(formats):
|
|
313
|
+
want = [f for f in formats if f.get("ext") == "vtt"] or formats
|
|
314
|
+
# ponytail: take the first vtt, trying every format is waste
|
|
315
|
+
url = want[0]["url"]
|
|
316
|
+
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
|
317
|
+
with urllib.request.urlopen(req, timeout=20) as r:
|
|
318
|
+
return r.read().decode("utf-8", errors="ignore")
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _is_throttle(e):
|
|
322
|
+
return getattr(e, "code", None) in (429, 500, 502, 503)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _countdown(secs, label, tick=None):
|
|
326
|
+
# long breaks: plain lines in logs, live status line on TTY via tick()
|
|
327
|
+
if secs <= 0:
|
|
328
|
+
return
|
|
329
|
+
if not UI_ON:
|
|
330
|
+
print(f"{label}: sleeping ~{secs // 60} min...", flush=True)
|
|
331
|
+
time.sleep(secs)
|
|
332
|
+
print(f"{label}: done.", flush=True)
|
|
333
|
+
return
|
|
334
|
+
end = time.time() + max(1, secs)
|
|
335
|
+
while True:
|
|
336
|
+
left = int(end - time.time())
|
|
337
|
+
if left <= 0:
|
|
338
|
+
break
|
|
339
|
+
if tick:
|
|
340
|
+
tick(left)
|
|
341
|
+
time.sleep(min(5, left))
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def slug(s, fallback="tube2note"):
|
|
345
|
+
s = re.sub(r"[^a-z0-9]+", "-", sanitize_filename(s, "").lower()).strip("-")
|
|
346
|
+
return (s[:60] or fallback) + ".md"
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
WIN_RESERVED = {"con", "prn", "aux", "nul"} | {f"com{i}" for i in range(1, 10)} | {f"lpt{i}" for i in range(1, 10)}
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def sanitize_filename(s, fallback="untitled"):
|
|
353
|
+
"""Cross-platform safe single path segment (Windows/macOS/Linux/Android)."""
|
|
354
|
+
tr = str.maketrans("şğüöçıİŞĞÜÖÇ", "sguociisguoc")
|
|
355
|
+
s = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]', "-", (s or "").translate(tr)).strip(" .")
|
|
356
|
+
if s.lower() in WIN_RESERVED:
|
|
357
|
+
s = "_" + s
|
|
358
|
+
s = (s or fallback).encode("utf-8")[:200].decode("utf-8", "ignore")
|
|
359
|
+
return s or fallback
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
DEFAULT_TEMPLATES = {
|
|
363
|
+
"single": None,
|
|
364
|
+
"videos": "videos/{title} [{id}]",
|
|
365
|
+
"tree": "{channel}/{title}/transcript",
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def render_template(tmpl, fields):
|
|
370
|
+
"""Fill {field} placeholders; unknown fields become empty. Sanitize per segment."""
|
|
371
|
+
rel = re.sub(r"\{(\w+)\}", lambda m: str(fields.get(m.group(1), "")), tmpl or "")
|
|
372
|
+
segs = [sanitize_filename(p) for p in rel.split("/") if p.strip() and p.strip() != "."]
|
|
373
|
+
rel = "/".join(segs)
|
|
374
|
+
if rel and not rel.lower().endswith(".md"):
|
|
375
|
+
rel += ".md"
|
|
376
|
+
return rel or "untitled.md"
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def detect_langs(videos, probe=3, want=("tr", "en")):
|
|
380
|
+
"""Sample the first few videos for ORIGINAL subtitle languages: (suggestion, found).
|
|
381
|
+
tlang translations are ignored (most 429-prone kind)."""
|
|
382
|
+
direct, anykey = {}, {}
|
|
383
|
+
with YoutubeDL({"quiet": True, "no_warnings": True, "skip_download": True,
|
|
384
|
+
"socket_timeout": 20}) as ydl:
|
|
385
|
+
for v in videos[:probe]:
|
|
386
|
+
try:
|
|
387
|
+
info = ydl.extract_info(v["url"], download=False)
|
|
388
|
+
except Exception:
|
|
389
|
+
continue
|
|
390
|
+
for pool in (info.get("subtitles") or {}, info.get("automatic_captions") or {}):
|
|
391
|
+
for k, fmts in pool.items():
|
|
392
|
+
anykey[k] = anykey.get(k, 0) + 1
|
|
393
|
+
if any("tlang=" not in (f.get("url") or "") for f in fmts):
|
|
394
|
+
direct[k] = direct.get(k, 0) + 1
|
|
395
|
+
shown = sorted(direct) or sorted(anykey)
|
|
396
|
+
for pool in (direct, anykey):
|
|
397
|
+
for w in want:
|
|
398
|
+
if w in pool:
|
|
399
|
+
return w, shown
|
|
400
|
+
for pool in (direct, anykey):
|
|
401
|
+
if pool:
|
|
402
|
+
return sorted(pool, key=lambda k: -pool[k])[0], shown
|
|
403
|
+
return "tr,en", []
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def split_output(out, budget):
|
|
407
|
+
"""Split a finished md into <=budget-word parts at video boundaries. Returns part paths."""
|
|
408
|
+
raw = open(out, encoding="utf-8").read()
|
|
409
|
+
head, sep, body = raw.partition("---\n\n")
|
|
410
|
+
head = head + sep if sep else ""
|
|
411
|
+
sections = [s for s in body.split("---\n\n") if s.strip()]
|
|
412
|
+
base, ext = os.path.splitext(out)
|
|
413
|
+
parts, cur, curw, idx = [], [], 0, 0
|
|
414
|
+
for s in sections:
|
|
415
|
+
w = len(s.split())
|
|
416
|
+
if cur and curw + w > budget:
|
|
417
|
+
parts.append((idx + 1, cur))
|
|
418
|
+
cur, curw, idx = [], 0, idx + 1
|
|
419
|
+
cur.append(s)
|
|
420
|
+
curw += w
|
|
421
|
+
if cur:
|
|
422
|
+
parts.append((idx + 1, cur))
|
|
423
|
+
if len(parts) <= 1:
|
|
424
|
+
return []
|
|
425
|
+
paths = []
|
|
426
|
+
for i, secs in parts:
|
|
427
|
+
lines = head.split("\n")
|
|
428
|
+
if lines:
|
|
429
|
+
lines[0] += f" (part {i}/{len(parts)})"
|
|
430
|
+
p = f"{base}_part{i:02d}{ext}"
|
|
431
|
+
with open(p, "w", encoding="utf-8") as f:
|
|
432
|
+
f.write("\n".join(lines) + ("---\n\n".join(secs)) + "---\n")
|
|
433
|
+
paths.append(p)
|
|
434
|
+
return paths
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
YT_RE = re.compile(r"(youtube\.com/(watch|shorts|playlist|@|channel/|c/|user/|live|embed)|youtu\.be/|[?&](list|v)=)")
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def show_intro():
|
|
441
|
+
print(panel("tube2note — YouTube to NotebookLM", [
|
|
442
|
+
"Turn a channel, playlist or videos into ONE Markdown file.",
|
|
443
|
+
"",
|
|
444
|
+
" 1. Paste link(s) 2. Check the auto-detected summary",
|
|
445
|
+
" 3. Press Enter 4. Upload the .md to NotebookLM",
|
|
446
|
+
"",
|
|
447
|
+
dim("NotebookLM cap: 500,000 words per source file."),
|
|
448
|
+
]))
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def is_first_run():
|
|
452
|
+
p = os.path.expanduser("~/.config/yt2md/seen")
|
|
453
|
+
if os.path.exists(p):
|
|
454
|
+
return False
|
|
455
|
+
try:
|
|
456
|
+
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
457
|
+
open(p, "w").write("1")
|
|
458
|
+
except OSError:
|
|
459
|
+
return False
|
|
460
|
+
return True
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def show_guide():
|
|
464
|
+
print(panel("Quick guide", [
|
|
465
|
+
"Paste any YouTube link: channel, playlist or video.",
|
|
466
|
+
"The tool lists videos, guesses the file name + languages.",
|
|
467
|
+
"Big jobs run in chunks with breaks (anti-429 protection).",
|
|
468
|
+
"Progress is saved: Ctrl+C anytime, resume later.",
|
|
469
|
+
"When done, upload the .md (or _part files) to NotebookLM.",
|
|
470
|
+
]))
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
CONFIG_PATH = os.path.expanduser("~/.config/yt2md/config.json")
|
|
474
|
+
DEFAULTS = {"outdir": ".", "layout": "single", "timestamps": False, "chunk": 50,
|
|
475
|
+
"chunk_cooldown_min": 10, "lang": "tr,en", "template": ""}
|
|
476
|
+
|
|
477
|
+
ENV_MAP = {"outdir": "YT2MD_OUTDIR", "layout": "YT2MD_LAYOUT", "lang": "YT2MD_LANG",
|
|
478
|
+
"chunk": "YT2MD_CHUNK", "timestamps": "YT2MD_TIMESTAMPS",
|
|
479
|
+
"chunk_cooldown_min": "YT2MD_COOLDOWN_MIN", "template": "YT2MD_TEMPLATE"}
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def load_config():
|
|
483
|
+
"""Config file: {"defaults": {...}, "profiles": {name: {...}}}. Old flat files count as defaults."""
|
|
484
|
+
raw = {}
|
|
485
|
+
try:
|
|
486
|
+
data = json.load(open(CONFIG_PATH, encoding="utf-8"))
|
|
487
|
+
if isinstance(data, dict):
|
|
488
|
+
raw = data
|
|
489
|
+
except (OSError, ValueError, TypeError):
|
|
490
|
+
pass
|
|
491
|
+
if "defaults" in raw or "profiles" in raw:
|
|
492
|
+
defaults, profiles = raw.get("defaults") or {}, raw.get("profiles") or {}
|
|
493
|
+
else:
|
|
494
|
+
defaults, profiles = raw, {}
|
|
495
|
+
if not isinstance(defaults, dict):
|
|
496
|
+
defaults = {}
|
|
497
|
+
if not isinstance(profiles, dict):
|
|
498
|
+
profiles = {}
|
|
499
|
+
return {"defaults": defaults, "profiles": profiles}
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def save_config(store):
|
|
503
|
+
os.makedirs(os.path.dirname(CONFIG_PATH), exist_ok=True)
|
|
504
|
+
json.dump(store, open(CONFIG_PATH, "w", encoding="utf-8"), indent=2)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _merge(base, store, profile, flags, env):
|
|
508
|
+
"""Precedence: flags > env > profile > config defaults > builtins. Pure (testable)."""
|
|
509
|
+
cfg = dict(base)
|
|
510
|
+
cfg.update({k: v for k, v in store.get("defaults", {}).items() if k in base})
|
|
511
|
+
if profile and profile in store.get("profiles", {}):
|
|
512
|
+
cfg.update({k: v for k, v in store["profiles"][profile].items() if k in base})
|
|
513
|
+
for key, var in ENV_MAP.items():
|
|
514
|
+
if env.get(var, "") != "":
|
|
515
|
+
cfg[key] = env[var]
|
|
516
|
+
cfg.update({k: v for k, v in flags.items() if v is not None})
|
|
517
|
+
for key in ("chunk", "chunk_cooldown_min"):
|
|
518
|
+
try:
|
|
519
|
+
cfg[key] = max(0, int(cfg[key]))
|
|
520
|
+
except (ValueError, TypeError):
|
|
521
|
+
cfg[key] = base[key]
|
|
522
|
+
if isinstance(cfg.get("timestamps"), str):
|
|
523
|
+
cfg["timestamps"] = cfg["timestamps"].lower() in ("1", "y", "yes", "true")
|
|
524
|
+
if cfg.get("layout") not in ("single", "videos", "tree"):
|
|
525
|
+
cfg["layout"] = base["layout"]
|
|
526
|
+
return cfg
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def resolve_config(flags=None, profile=None):
|
|
530
|
+
return _merge(dict(DEFAULTS), load_config(), profile, flags or {}, dict(os.environ))
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def cmd_setup(advanced=False):
|
|
534
|
+
"""Personalize: 3 sticky defaults (folder, layout, language); pacing under --advanced."""
|
|
535
|
+
store = load_config()
|
|
536
|
+
cfg = _merge(dict(DEFAULTS), store, None, {}, {})
|
|
537
|
+
print(panel("Personalize tube2note", ["CLI flags and YT2MD_* env vars always win over these."]))
|
|
538
|
+
cfg["outdir"] = input(f"Default folder [{cfg['outdir']}] > ").strip() or cfg["outdir"]
|
|
539
|
+
lay = input(f"Output layout (single/videos/tree) [{cfg['layout']}] > ").strip().lower() or cfg["layout"]
|
|
540
|
+
cfg["layout"] = lay if lay in ("single", "videos", "tree") else "single"
|
|
541
|
+
cfg["lang"] = input(f"Default languages [{cfg['lang']}] > ").strip() or cfg["lang"]
|
|
542
|
+
if advanced:
|
|
543
|
+
ts = input(f"Timestamps? (y/n) [{'y' if cfg['timestamps'] else 'n'}] > ").strip().lower()
|
|
544
|
+
if ts in ("y", "yes", "n", "no"):
|
|
545
|
+
cfg["timestamps"] = ts in ("y", "yes")
|
|
546
|
+
try:
|
|
547
|
+
cfg["chunk"] = max(0, int(input(f"Chunk size [{cfg['chunk']}] > ").strip() or cfg["chunk"]))
|
|
548
|
+
except ValueError:
|
|
549
|
+
pass
|
|
550
|
+
try:
|
|
551
|
+
cfg["chunk_cooldown_min"] = max(0, int(input(f"Chunk break minutes [{cfg['chunk_cooldown_min']}] > ").strip()
|
|
552
|
+
or cfg["chunk_cooldown_min"]))
|
|
553
|
+
except ValueError:
|
|
554
|
+
pass
|
|
555
|
+
cfg["template"] = input(f"Name template [{cfg['template'] or 'layout default'}] > ").strip()
|
|
556
|
+
store["defaults"] = {k: cfg[k] for k in DEFAULTS if k in cfg}
|
|
557
|
+
name = input("Save as profile name [skip] > ").strip()
|
|
558
|
+
if name:
|
|
559
|
+
store.setdefault("profiles", {})[name] = dict(store["defaults"])
|
|
560
|
+
print(f"Profile '{name}': use with --profile {name} or YT2MD_PROFILE={name}")
|
|
561
|
+
save_config(store)
|
|
562
|
+
print(green("Saved to ") + CONFIG_PATH)
|
|
563
|
+
return cfg
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def tui():
|
|
567
|
+
cfg = resolve_config(profile=os.environ.get("YT2MD_PROFILE") or None)
|
|
568
|
+
show_intro()
|
|
569
|
+
if is_first_run():
|
|
570
|
+
show_guide()
|
|
571
|
+
if input("Personalize defaults now? (folder, layout...) [Y/n] > ").strip().lower() not in ("n", "no"):
|
|
572
|
+
cfg = cmd_setup()
|
|
573
|
+
while True:
|
|
574
|
+
raw = input("\nURLs (space/comma separated, several allowed) > ").strip()
|
|
575
|
+
if raw.lower() in ("q", "quit", "exit"):
|
|
576
|
+
return
|
|
577
|
+
urls = [u.strip(" ,") for u in re.split(r"[,\s]+", raw) if u.strip(" ,")]
|
|
578
|
+
if not urls:
|
|
579
|
+
print("No URLs, try again.")
|
|
580
|
+
continue
|
|
581
|
+
bad = [u for u in urls if not YT_RE.search(u)]
|
|
582
|
+
if bad:
|
|
583
|
+
print(red("Not a YouTube link: ") + ", ".join(bad))
|
|
584
|
+
continue
|
|
585
|
+
print("Listing videos, wait...")
|
|
586
|
+
videos, hint = expand(urls, 5000)
|
|
587
|
+
if not videos:
|
|
588
|
+
print("No videos found.")
|
|
589
|
+
continue
|
|
590
|
+
guess = slug(hint)
|
|
591
|
+
print("Sampling subtitle languages...")
|
|
592
|
+
sug, found = detect_langs(videos)
|
|
593
|
+
est = len(videos) * 12 / 60
|
|
594
|
+
print(table(["Setting", "Value"], [
|
|
595
|
+
["Source", (hint or urls[0])[:60]],
|
|
596
|
+
["Videos", str(len(videos))],
|
|
597
|
+
["File", guess],
|
|
598
|
+
["Languages", sug + (f" (found: {', '.join(found[:8])})" if found else "")],
|
|
599
|
+
["Est. time", f"~{est:.0f} min paced" if est >= 1 else "<1 min"],
|
|
600
|
+
]))
|
|
601
|
+
out = input(f"Output file [{guess}] > ").strip() or guess
|
|
602
|
+
outdir = input(f"Folder [{cfg['outdir']}] > ").strip() or cfg["outdir"]
|
|
603
|
+
lay = input(f"Layout (single/videos/tree) [{cfg['layout']}] > ").strip().lower() or cfg["layout"]
|
|
604
|
+
lay = lay if lay in ("single", "videos", "tree") else "single"
|
|
605
|
+
lang = input(f"Languages [{sug}] > ").strip() or sug
|
|
606
|
+
mx = input(f"Max videos [{len(videos)}] > ").strip() or str(len(videos))
|
|
607
|
+
try:
|
|
608
|
+
max_n = max(1, int(mx))
|
|
609
|
+
except ValueError:
|
|
610
|
+
max_n = len(videos)
|
|
611
|
+
videos = videos[:max_n]
|
|
612
|
+
ch = input(f"Chunk size [{cfg['chunk']}] > ").strip() or str(cfg["chunk"])
|
|
613
|
+
try:
|
|
614
|
+
ch = max(0, int(ch))
|
|
615
|
+
except ValueError:
|
|
616
|
+
ch = cfg["chunk"]
|
|
617
|
+
cd = input(f"Chunk break minutes [{cfg['chunk_cooldown_min']}] > ").strip() or str(cfg["chunk_cooldown_min"])
|
|
618
|
+
try:
|
|
619
|
+
chc = max(0, int(cd)) * 60
|
|
620
|
+
except ValueError:
|
|
621
|
+
chc = cfg["chunk_cooldown_min"] * 60
|
|
622
|
+
yn = "y" if cfg["timestamps"] else "n"
|
|
623
|
+
ts = input(f"Timestamps? [{yn}] > ").strip().lower()
|
|
624
|
+
ts = cfg["timestamps"] if ts == "" else ts in ("y", "yes")
|
|
625
|
+
tmp = input(f"Name template [{cfg['template'] or 'layout default'}] > ").strip()
|
|
626
|
+
tmp = tmp or cfg["template"]
|
|
627
|
+
sp = input("Auto-split words for NotebookLM [0=off] > ").strip() or "0"
|
|
628
|
+
try:
|
|
629
|
+
sp = max(0, int(sp))
|
|
630
|
+
except ValueError:
|
|
631
|
+
sp = 0
|
|
632
|
+
print(f"\n{len(videos)} videos, output: {outdir}/{out}, langs: {lang}, layout: {lay}, "
|
|
633
|
+
f"chunk: {ch}/{chc // 60}min, timestamps: {ts}, template: {tmp or 'default'}, split: {sp or 'off'}")
|
|
634
|
+
go = input("[Enter]=start, q=cancel > ").strip()
|
|
635
|
+
if go.lower() in ("q", "quit"):
|
|
636
|
+
continue
|
|
637
|
+
try:
|
|
638
|
+
run_job(urls, out, lang, max_n, 2.0, False, ch, chc, 1800, videos, outdir, ts, sp,
|
|
639
|
+
layout=lay, template=tmp)
|
|
640
|
+
except KeyboardInterrupt:
|
|
641
|
+
print("\nCancelled.")
|
|
642
|
+
again = input("\nNew job? [Enter]=yes, q=quit > ").strip()
|
|
643
|
+
if again.lower() in ("q", "quit", "exit"):
|
|
644
|
+
return
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def _frontmatter(title, wurl, channel, vid, lg, auto):
|
|
648
|
+
t = " ".join((title or vid).split()).replace('"', "'")
|
|
649
|
+
c = " ".join((channel or "unknown").split()).replace('"', "'")
|
|
650
|
+
return (f"---\ntitle: \"{t}\"\nsource: {wurl}\nchannel: \"{c}\"\n"
|
|
651
|
+
f"video_id: {vid}\nlanguage: {lg}{' (auto)' if auto else ''}\n---\n\n")
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def _unique_path(vp, vid, used):
|
|
655
|
+
"""Disambiguate slug collisions: second same-named file gets _<video_id>."""
|
|
656
|
+
if vp in used:
|
|
657
|
+
base, ext = os.path.splitext(vp)
|
|
658
|
+
vp = f"{base}_{vid}{ext}"
|
|
659
|
+
used.add(vp)
|
|
660
|
+
return vp
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def _count_lines(path):
|
|
664
|
+
try:
|
|
665
|
+
return sum(1 for ln in open(path, encoding="utf-8") if ln.strip())
|
|
666
|
+
except OSError:
|
|
667
|
+
return 0
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _skip_map(skip_log):
|
|
671
|
+
"""Parse 'title | url | reason' lines into {video_id_or_url: reason}."""
|
|
672
|
+
m = {}
|
|
673
|
+
if os.path.exists(skip_log):
|
|
674
|
+
for ln in open(skip_log, encoding="utf-8"):
|
|
675
|
+
parts = [p.strip() for p in ln.split(" | ")]
|
|
676
|
+
if len(parts) == 3:
|
|
677
|
+
t, u, s = parts
|
|
678
|
+
r = re.search(r"[?&]v=([A-Za-z0-9_-]{11})", u)
|
|
679
|
+
m[r.group(1) if r else u] = s
|
|
680
|
+
return m
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _write_index(root, videos, completed, words_by_id, skip_log):
|
|
684
|
+
"""INDEX.md: every video with status, words and per-video file."""
|
|
685
|
+
reasons = _skip_map(skip_log)
|
|
686
|
+
rows = []
|
|
687
|
+
for v in videos:
|
|
688
|
+
vid = v["id"]
|
|
689
|
+
st = "done" if vid in completed else reasons.get(vid, "pending")
|
|
690
|
+
rows.append([v.get("title") or vid, st, str(words_by_id.get(vid, "-"))])
|
|
691
|
+
with open(os.path.join(root, "INDEX.md"), "w", encoding="utf-8") as f:
|
|
692
|
+
f.write(f"# Index\n\n- Videos: {len(videos)}\n- Done: {len(completed)}\n\n")
|
|
693
|
+
f.write(table(["Title", "Status", "Words"], rows))
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
COLLECTION_KEYS = ("layout", "lang", "timestamps", "chunk", "chunk_cooldown_min")
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
def _collection_override(root):
|
|
700
|
+
"""Per-collection .yt2md.json overrides (safe subset only)."""
|
|
701
|
+
try:
|
|
702
|
+
data = json.load(open(os.path.join(root, ".yt2md.json"), encoding="utf-8"))
|
|
703
|
+
except (OSError, ValueError, TypeError):
|
|
704
|
+
return {}
|
|
705
|
+
if not isinstance(data, dict):
|
|
706
|
+
return {}
|
|
707
|
+
return {k: data[k] for k in COLLECTION_KEYS if k in data}
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def run_job(urls, out, lang_str, max_n, sleep, fresh=False, chunk=50, chunk_cooldown=600,
|
|
711
|
+
throttle_cooldown=1800, videos=None, outdir=".", ts=False, split_words=0,
|
|
712
|
+
verbose=False, layout="single", template=""):
|
|
713
|
+
if videos is None:
|
|
714
|
+
videos, _ = expand(urls, max_n)
|
|
715
|
+
if outdir and outdir != ".":
|
|
716
|
+
outdir = os.path.expanduser(outdir)
|
|
717
|
+
os.makedirs(outdir, exist_ok=True)
|
|
718
|
+
out = os.path.join(outdir, out)
|
|
719
|
+
root = os.path.dirname(os.path.abspath(out))
|
|
720
|
+
coll = _collection_override(root)
|
|
721
|
+
if coll.get("layout") in ("single", "videos", "tree"):
|
|
722
|
+
layout = coll["layout"]
|
|
723
|
+
if isinstance(coll.get("lang"), str) and coll["lang"].strip():
|
|
724
|
+
lang_str = coll["lang"]
|
|
725
|
+
if isinstance(coll.get("timestamps"), bool):
|
|
726
|
+
ts = coll["timestamps"]
|
|
727
|
+
if coll.get("chunk") is not None:
|
|
728
|
+
try:
|
|
729
|
+
chunk = max(0, int(coll["chunk"]))
|
|
730
|
+
except (ValueError, TypeError):
|
|
731
|
+
pass
|
|
732
|
+
if coll.get("chunk_cooldown_min") is not None: # minutes, like the TUI prompt
|
|
733
|
+
try:
|
|
734
|
+
chunk_cooldown = max(0, int(coll["chunk_cooldown_min"])) * 60
|
|
735
|
+
except (ValueError, TypeError):
|
|
736
|
+
pass
|
|
737
|
+
langs = [s.strip() for s in lang_str.split(",") if s.strip()]
|
|
738
|
+
total = len(videos)
|
|
739
|
+
print(f"{total} videos found", flush=True)
|
|
740
|
+
if not videos:
|
|
741
|
+
return
|
|
742
|
+
done_log, skip_log = out + ".done", out + ".skip"
|
|
743
|
+
done = set()
|
|
744
|
+
if not fresh and os.path.exists(done_log):
|
|
745
|
+
with open(done_log, encoding="utf-8") as f:
|
|
746
|
+
done = {ln.strip() for ln in f if ln.strip()}
|
|
747
|
+
print(f"resuming: {len(done)} videos already done", flush=True)
|
|
748
|
+
fresh_start = fresh or not os.path.exists(out)
|
|
749
|
+
mode = "w" if fresh_start else "a"
|
|
750
|
+
fout = open(out, mode, encoding="utf-8")
|
|
751
|
+
dlog = open(done_log, "w" if fresh_start else "a", encoding="utf-8")
|
|
752
|
+
slog = open(skip_log, "w" if fresh_start else "a", encoding="utf-8")
|
|
753
|
+
wlog = open(out + ".words", "w" if fresh_start else "a", encoding="utf-8")
|
|
754
|
+
if fresh_start:
|
|
755
|
+
done = set()
|
|
756
|
+
if mode == "w":
|
|
757
|
+
today = datetime.date.today().isoformat()
|
|
758
|
+
fout.write(f"# YouTube Research Notes\n\n- Date: {today}\n- Videos: target {total}\n"
|
|
759
|
+
f"- Languages: {','.join(langs)}\n\nFeed this file to NotebookLM as a source.\n\n---\n\n")
|
|
760
|
+
todo = max(0, total - len(done))
|
|
761
|
+
completed = set(done)
|
|
762
|
+
words_by_id = {}
|
|
763
|
+
if not fresh_start and os.path.exists(out + ".words"):
|
|
764
|
+
try:
|
|
765
|
+
for ln in open(out + ".words", encoding="utf-8"):
|
|
766
|
+
p = ln.split()
|
|
767
|
+
if len(p) == 2:
|
|
768
|
+
words_by_id[p[0]] = int(p[1])
|
|
769
|
+
except (OSError, ValueError):
|
|
770
|
+
pass
|
|
771
|
+
print(f"target: {todo} videos (chunk: {chunk}, chunk break: {chunk_cooldown // 60} min, layout: {layout})", flush=True)
|
|
772
|
+
ok, skip, words, consec, since_break, status = 0, [], 0, 0, 0, ""
|
|
773
|
+
used_paths = set()
|
|
774
|
+
global _VERBOSE
|
|
775
|
+
_VERBOSE = verbose
|
|
776
|
+
t0 = time.time()
|
|
777
|
+
ydl_opts = {"quiet": True, "no_warnings": True, "skip_download": True,
|
|
778
|
+
"writesubtitles": False, "socket_timeout": 20,
|
|
779
|
+
"subtitlesformat": "vtt/best"}
|
|
780
|
+
with YoutubeDL(ydl_opts) as ydl:
|
|
781
|
+
for i, v in enumerate(videos, 1):
|
|
782
|
+
if v["id"] in done:
|
|
783
|
+
continue
|
|
784
|
+
if chunk > 0 and since_break >= chunk and (ok + len(skip)) < todo:
|
|
785
|
+
if verbose:
|
|
786
|
+
log(f"--- CHUNK done, {chunk_cooldown // 60} min break ---")
|
|
787
|
+
_countdown(chunk_cooldown, "chunk break", lambda left: dash_update(
|
|
788
|
+
ok + len(skip), todo, v["title"], ok, len(skip), words, t0,
|
|
789
|
+
status=f"chunk break {left // 60:02d}:{left % 60:02d} left"))
|
|
790
|
+
status = ""
|
|
791
|
+
since_break = 0
|
|
792
|
+
since_break += 1
|
|
793
|
+
dash_update(ok + len(skip), todo, f"[{i}/{total}] {v['title']}",
|
|
794
|
+
ok, len(skip), words, t0, status)
|
|
795
|
+
if verbose:
|
|
796
|
+
log(f"[{i}/{total}] {v['title'][:70]}")
|
|
797
|
+
try:
|
|
798
|
+
info = ydl.extract_info(v["url"], download=False)
|
|
799
|
+
except Exception as e:
|
|
800
|
+
if verbose:
|
|
801
|
+
log(f" ! skipped: {e}")
|
|
802
|
+
skip.append((v["title"], v["url"], str(e)))
|
|
803
|
+
consec = consec + 1 if _is_throttle(e) else 0
|
|
804
|
+
status = "throttled" if _is_throttle(e) else "extract failed"
|
|
805
|
+
if consec >= 5:
|
|
806
|
+
if verbose:
|
|
807
|
+
log(" ! 5 throttles in a row -> long cooldown")
|
|
808
|
+
_countdown(throttle_cooldown, "throttle cooldown", lambda left: dash_update(
|
|
809
|
+
ok + len(skip), todo, v["title"], ok, len(skip), words, t0,
|
|
810
|
+
status=f"throttle cooldown {left // 60:02d}:{left % 60:02d} left"))
|
|
811
|
+
consec, status = 0, ""
|
|
812
|
+
continue
|
|
813
|
+
title = info.get("title") or v["title"]
|
|
814
|
+
wurl = info.get("webpage_url") or v["url"]
|
|
815
|
+
lg, fmts, auto = pick_sub(info, langs)
|
|
816
|
+
if not fmts:
|
|
817
|
+
if verbose:
|
|
818
|
+
log(" ! no subtitles, skipped")
|
|
819
|
+
skip.append((title, wurl, "no subtitles"))
|
|
820
|
+
consec, status = 0, "no subtitles"
|
|
821
|
+
continue
|
|
822
|
+
text, last = None, None
|
|
823
|
+
for _ in (1, 2): # ponytail: 60s + ONE retry on 429; hot retries extend the ban
|
|
824
|
+
try:
|
|
825
|
+
time.sleep(random.uniform(5, 10))
|
|
826
|
+
text = vtt_to_text(fetch_vtt(fmts), ts).strip()
|
|
827
|
+
break
|
|
828
|
+
except Exception as e:
|
|
829
|
+
last = e
|
|
830
|
+
if not _is_throttle(e):
|
|
831
|
+
break
|
|
832
|
+
status = "429: 60s break + single retry"
|
|
833
|
+
if verbose:
|
|
834
|
+
log(" ! 429: 60s break + single retry")
|
|
835
|
+
time.sleep(60)
|
|
836
|
+
if text is None:
|
|
837
|
+
if verbose:
|
|
838
|
+
log(f" ! subtitle download failed: {last}")
|
|
839
|
+
skip.append((title, wurl, str(last)))
|
|
840
|
+
consec = consec + 1 if _is_throttle(last) else 0
|
|
841
|
+
status = "throttled" if _is_throttle(last) else "subtitle failed"
|
|
842
|
+
if consec >= 5:
|
|
843
|
+
if verbose:
|
|
844
|
+
log(" ! 5 throttles in a row -> long cooldown")
|
|
845
|
+
_countdown(throttle_cooldown, "throttle cooldown", lambda left: dash_update(
|
|
846
|
+
ok + len(skip), todo, title, ok, len(skip), words, t0,
|
|
847
|
+
status=f"throttle cooldown {left // 60:02d}:{left % 60:02d} left"))
|
|
848
|
+
consec, status = 0, ""
|
|
849
|
+
continue
|
|
850
|
+
consec, status = 0, ""
|
|
851
|
+
if len(text) < 50:
|
|
852
|
+
skip.append((title, wurl, "subtitle too short"))
|
|
853
|
+
consec, status = 0, "subtitle too short"
|
|
854
|
+
continue
|
|
855
|
+
fout.write(f"## {i}. {title}\n\n- Source: {wurl}\n"
|
|
856
|
+
f"- Video ID: {v['id']}\n- Subtitle lang: {lg}{' (auto)' if auto else ''}\n\n{text}\n\n---\n\n")
|
|
857
|
+
fout.flush()
|
|
858
|
+
dlog.write(v["id"] + "\n")
|
|
859
|
+
dlog.flush()
|
|
860
|
+
ok += 1
|
|
861
|
+
nwords = len(text.split())
|
|
862
|
+
words += nwords
|
|
863
|
+
completed.add(v["id"])
|
|
864
|
+
done.add(v["id"])
|
|
865
|
+
words_by_id[v["id"]] = nwords
|
|
866
|
+
wlog.write(f"{v['id']} {nwords}\n")
|
|
867
|
+
wlog.flush()
|
|
868
|
+
if layout != "single":
|
|
869
|
+
fields = {"channel": v.get("channel") or info.get("channel") or "channel",
|
|
870
|
+
"title": title or v["id"], "id": v["id"],
|
|
871
|
+
"index": f"{i:02d}", "date": datetime.date.today().isoformat(),
|
|
872
|
+
"lang": lg}
|
|
873
|
+
vp = _unique_path(os.path.join(root, render_template(
|
|
874
|
+
template or DEFAULT_TEMPLATES[layout], fields)), v["id"], used_paths)
|
|
875
|
+
os.makedirs(os.path.dirname(vp), exist_ok=True)
|
|
876
|
+
with open(vp, "w", encoding="utf-8") as vf:
|
|
877
|
+
vf.write(_frontmatter(title, wurl, v.get("channel") or info.get("channel"),
|
|
878
|
+
v["id"], lg, auto))
|
|
879
|
+
vf.write(f"## {title}\n\n{text}\n")
|
|
880
|
+
dash_update(ok + len(skip), todo, v["title"], ok, len(skip), words, t0)
|
|
881
|
+
time.sleep(sleep)
|
|
882
|
+
dash_update(todo, todo, "done", ok, len(skip), words, t0)
|
|
883
|
+
dash_end()
|
|
884
|
+
for t, u, s in skip:
|
|
885
|
+
slog.write(f"{t} | {u} | {s}\n")
|
|
886
|
+
fout.close()
|
|
887
|
+
dlog.close()
|
|
888
|
+
slog.close()
|
|
889
|
+
wlog.close()
|
|
890
|
+
if layout != "single":
|
|
891
|
+
_write_index(root, videos, completed, words_by_id, skip_log)
|
|
892
|
+
print(f"index: {os.path.join(root, 'INDEX.md')}", flush=True)
|
|
893
|
+
try: # reconcile .skip: drop fixed videos + dedupe, so counts/tail stay honest
|
|
894
|
+
seen, kept = set(), []
|
|
895
|
+
if os.path.exists(skip_log):
|
|
896
|
+
for ln in open(skip_log, encoding="utf-8"):
|
|
897
|
+
s = ln.strip()
|
|
898
|
+
if not s:
|
|
899
|
+
continue
|
|
900
|
+
r = re.search(r"[?&]v=([A-Za-z0-9_-]{11})", s)
|
|
901
|
+
key = r.group(1) if r else s
|
|
902
|
+
if key not in completed and key not in seen:
|
|
903
|
+
seen.add(key)
|
|
904
|
+
kept.append(s)
|
|
905
|
+
with open(skip_log, "w", encoding="utf-8") as f:
|
|
906
|
+
f.write("\n".join(kept) + ("\n" if kept else ""))
|
|
907
|
+
except OSError:
|
|
908
|
+
pass
|
|
909
|
+
ndone = _count_lines(done_log)
|
|
910
|
+
nskip = _count_lines(skip_log)
|
|
911
|
+
if ndone + nskip >= len(videos):
|
|
912
|
+
if nskip > 0:
|
|
913
|
+
tail_done = False
|
|
914
|
+
try:
|
|
915
|
+
with open(out, "rb") as _f:
|
|
916
|
+
_f.seek(max(0, os.path.getsize(out) - 5000))
|
|
917
|
+
tail_done = b"## Skipped" in _f.read()
|
|
918
|
+
except OSError:
|
|
919
|
+
pass
|
|
920
|
+
if not tail_done:
|
|
921
|
+
with open(out, "a", encoding="utf-8") as f:
|
|
922
|
+
f.write("\n## Skipped\n\n")
|
|
923
|
+
for ln in open(skip_log, encoding="utf-8"):
|
|
924
|
+
f.write(f"- {ln}")
|
|
925
|
+
dash_end()
|
|
926
|
+
print(panel("Done", [
|
|
927
|
+
f"{green(str(ndone))} videos -> {out} ({nskip} skipped)",
|
|
928
|
+
f"~{words} words this run",
|
|
929
|
+
]))
|
|
930
|
+
if split_words > 0:
|
|
931
|
+
total_words = len(open(out, encoding="utf-8").read().split())
|
|
932
|
+
if total_words > split_words:
|
|
933
|
+
parts = split_output(out, split_words)
|
|
934
|
+
if len(parts) > 1:
|
|
935
|
+
print(panel("Upload to NotebookLM", ["Add each part as a separate source:"]
|
|
936
|
+
+ [f" {j}. {p}" for j, p in enumerate(parts, 1)]))
|
|
937
|
+
else:
|
|
938
|
+
print(f"Single file is enough ({total_words} words).")
|
|
939
|
+
else:
|
|
940
|
+
print(f"No split needed ({total_words} words <= {split_words}).")
|
|
941
|
+
else:
|
|
942
|
+
print("Upload to NotebookLM: add this file as a source.")
|
|
943
|
+
print(dim("Cap is 500,000 words/file — use --split-words if bigger."))
|
|
944
|
+
else:
|
|
945
|
+
dash_end()
|
|
946
|
+
print(f"Checkpoint: {ok} videos, {words} words -> {out} (total: {ndone}/{len(videos)})")
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
def cmd_status(d="."):
|
|
950
|
+
d = os.path.expanduser(d)
|
|
951
|
+
try:
|
|
952
|
+
files = sorted(os.listdir(d))
|
|
953
|
+
except OSError as e:
|
|
954
|
+
print(f"Cannot list {d}: {e}")
|
|
955
|
+
return
|
|
956
|
+
rows = []
|
|
957
|
+
for f in files:
|
|
958
|
+
if not f.endswith(".md") or "_part" in f:
|
|
959
|
+
continue
|
|
960
|
+
path = os.path.join(d, f)
|
|
961
|
+
dn = _count_lines(path + ".done")
|
|
962
|
+
sk = _count_lines(path + ".skip")
|
|
963
|
+
rows.append([f, str(dn), str(sk), f"{os.path.getsize(path) // 1024} KB"])
|
|
964
|
+
if not rows:
|
|
965
|
+
print(f"No collections in {d}.")
|
|
966
|
+
return
|
|
967
|
+
print(table(["Collection", "Done", "Skipped", "Size"], rows))
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def cmd_dryrun(urls, max_n, lang_str):
|
|
971
|
+
videos, hint = expand(urls, max_n)
|
|
972
|
+
if not videos:
|
|
973
|
+
print("No videos found.")
|
|
974
|
+
return
|
|
975
|
+
sug, found = detect_langs(videos)
|
|
976
|
+
est = len(videos) * 12 / 60
|
|
977
|
+
print(table(["Setting", "Value"], [
|
|
978
|
+
["Source", (hint or urls[0])[:60]],
|
|
979
|
+
["Videos", str(len(videos))],
|
|
980
|
+
["Languages", sug + (f" (found: {', '.join(found[:8])})" if found else "")],
|
|
981
|
+
["Est. time", f"~{est:.0f} min paced" if est >= 1 else "<1 min"],
|
|
982
|
+
["Est. words", f"~{len(videos) * 2000} (rough: ~2k/video)"],
|
|
983
|
+
]))
|
|
984
|
+
print("Dry run: nothing downloaded. Drop --dry-run to start.")
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
def main():
|
|
988
|
+
if len(sys.argv) > 1 and sys.argv[1] == "status":
|
|
989
|
+
cmd_status(sys.argv[2] if len(sys.argv) > 2 else ".")
|
|
990
|
+
return
|
|
991
|
+
if len(sys.argv) > 1 and sys.argv[1] == "setup":
|
|
992
|
+
cmd_setup("--advanced" in sys.argv)
|
|
993
|
+
return
|
|
994
|
+
ap = argparse.ArgumentParser(description="YouTube -> single Markdown (NotebookLM feed)")
|
|
995
|
+
ap.add_argument("urls", nargs="*", help="channel / playlist / video URLs")
|
|
996
|
+
ap.add_argument("-o", "--out", default="tube2note.md")
|
|
997
|
+
ap.add_argument("--lang", default=None, help="subtitle language priority, comma separated")
|
|
998
|
+
ap.add_argument("--max", type=int, default=100, help="max number of videos")
|
|
999
|
+
ap.add_argument("--sleep", type=float, default=2.0, help="pause between videos (s)")
|
|
1000
|
+
ap.add_argument("--chunk", type=int, default=None, help="long break every N videos")
|
|
1001
|
+
ap.add_argument("--chunk-cooldown", type=int, default=None, help="break between chunks (s)")
|
|
1002
|
+
ap.add_argument("--throttle-cooldown", type=int, default=1800, help="break after 5 throttles in a row (s)")
|
|
1003
|
+
ap.add_argument("-d", "--dir", default=None, help="output folder (created if missing)")
|
|
1004
|
+
ap.add_argument("--layout", default=None, help="output layout: single, videos or tree")
|
|
1005
|
+
ap.add_argument("--timestamps", action="store_true", default=None, help="keep [MM:SS] markers in transcripts")
|
|
1006
|
+
ap.add_argument("--name-template", default=None, help='per-video path template, e.g. "{channel}/{title} [{id}]"')
|
|
1007
|
+
ap.add_argument("--profile", default=None, help="config profile name (or YT2MD_PROFILE)")
|
|
1008
|
+
ap.add_argument("--split-words", type=int, default=0, help="auto-split finished file into N-word parts (0=off)")
|
|
1009
|
+
ap.add_argument("--tui", action="store_true", help="interactive mode (short command)")
|
|
1010
|
+
ap.add_argument("--verbose", action="store_true", help="scrolling log lines instead of the live dashboard")
|
|
1011
|
+
ap.add_argument("--dry-run", action="store_true", help="list + estimate only, download nothing")
|
|
1012
|
+
ap.add_argument("--fresh", action="store_true", help="discard previous progress, start over")
|
|
1013
|
+
ap.add_argument("--self-test", action="store_true")
|
|
1014
|
+
a = ap.parse_args()
|
|
1015
|
+
if a.self_test:
|
|
1016
|
+
_self_test()
|
|
1017
|
+
return
|
|
1018
|
+
if a.tui or not a.urls:
|
|
1019
|
+
try:
|
|
1020
|
+
tui()
|
|
1021
|
+
except (KeyboardInterrupt, EOFError):
|
|
1022
|
+
print("\nExit.")
|
|
1023
|
+
return
|
|
1024
|
+
profile = a.profile or os.environ.get("YT2MD_PROFILE") or None
|
|
1025
|
+
cfg = resolve_config({"outdir": a.dir, "layout": a.layout, "lang": a.lang,
|
|
1026
|
+
"chunk": a.chunk, "timestamps": a.timestamps,
|
|
1027
|
+
"chunk_cooldown_min": (a.chunk_cooldown // 60
|
|
1028
|
+
if a.chunk_cooldown is not None else None),
|
|
1029
|
+
"template": a.name_template}, profile)
|
|
1030
|
+
if a.dry_run:
|
|
1031
|
+
cmd_dryrun(a.urls, a.max, cfg["lang"])
|
|
1032
|
+
return
|
|
1033
|
+
run_job(a.urls, a.out, cfg["lang"], a.max, a.sleep, a.fresh, cfg["chunk"],
|
|
1034
|
+
cfg["chunk_cooldown_min"] * 60, a.throttle_cooldown, outdir=cfg["outdir"],
|
|
1035
|
+
ts=cfg["timestamps"], split_words=a.split_words, verbose=a.verbose,
|
|
1036
|
+
layout=cfg["layout"], template=cfg["template"])
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
if __name__ == "__main__":
|
|
1040
|
+
main()
|