pypodcastde 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pypodcastde-0.1.1/.gitignore +30 -0
- pypodcastde-0.1.1/CHANGELOG.md +17 -0
- pypodcastde-0.1.1/PKG-INFO +88 -0
- pypodcastde-0.1.1/README.md +76 -0
- pypodcastde-0.1.1/locations_example.txt +9 -0
- pypodcastde-0.1.1/pypodcastde/__init__.py +1 -0
- pypodcastde-0.1.1/pypodcastde/__main__.py +5 -0
- pypodcastde-0.1.1/pypodcastde/catalogue.py +50 -0
- pypodcastde-0.1.1/pypodcastde/checkpod.py +235 -0
- pypodcastde-0.1.1/pypodcastde/cli.py +87 -0
- pypodcastde-0.1.1/pypodcastde/config.py +73 -0
- pypodcastde-0.1.1/pypodcastde/modes.py +27 -0
- pypodcastde-0.1.1/pypodcastde/nico.py +170 -0
- pypodcastde-0.1.1/pypodcastde/reporting.py +87 -0
- pypodcastde-0.1.1/pypodcastde/video.py +74 -0
- pypodcastde-0.1.1/pyproject.toml +53 -0
- pypodcastde-0.1.1/tests/test_modes.py +93 -0
- pypodcastde-0.1.1/tests/test_package_contract.py +34 -0
- pypodcastde-0.1.1/tests/test_reporting.py +35 -0
- pypodcastde-0.1.1/tests/test_workflow.py +65 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
.idea/
|
|
3
|
+
__pycache__/
|
|
4
|
+
*.py[cod]
|
|
5
|
+
.ruff_cache/
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
*.egg-info/
|
|
9
|
+
/podcasts/
|
|
10
|
+
.env
|
|
11
|
+
.env.*
|
|
12
|
+
!.env.example
|
|
13
|
+
.coverage
|
|
14
|
+
.coverage.*
|
|
15
|
+
coverage.xml
|
|
16
|
+
junit.xml
|
|
17
|
+
.tox/
|
|
18
|
+
.nox/
|
|
19
|
+
*.whl
|
|
20
|
+
*.part
|
|
21
|
+
.video-archive
|
|
22
|
+
htmlcov/
|
|
23
|
+
.pytest_cache/
|
|
24
|
+
.mypy_cache/
|
|
25
|
+
/config/
|
|
26
|
+
locations_local.txt
|
|
27
|
+
locations.txt
|
|
28
|
+
/compose.yaml
|
|
29
|
+
.DS_Store
|
|
30
|
+
Thumbs.db
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.1.1](https://github.com/gieses/pypodcastde/releases/tag/v0.1.1) - 2026-11-21
|
|
4
|
+
|
|
5
|
+
- Docker now installs the runtime dependencies from `uv.lock`. Download behavior is unchanged.
|
|
6
|
+
- Added `make publish-check` to check release files without uploading them, `make publish-test` for TestPyPI, and `make publish` for PyPI.
|
|
7
|
+
- CI now checks the pre-commit hooks and tests the installed wheel outside the source tree, including creation of the private locations file.
|
|
8
|
+
- Updated the README with a project logo, links to ARD Sounds and the ARD Mediathek, and shorter setup instructions.
|
|
9
|
+
|
|
10
|
+
## [0.1.0](https://github.com/gieses/pypodcastde/releases/tag/v0.1.0) - 2026-11-21
|
|
11
|
+
|
|
12
|
+
First release.
|
|
13
|
+
|
|
14
|
+
- Download ARD audio shows and save video soundtracks as MP3s. The example locations file includes eight German children's shows.
|
|
15
|
+
- Run `pypodcastde mine` to download missing episodes or `pypodcastde schedule` to check every 24 hours. Add `--try` to preview a run or `--force` to replace existing downloads.
|
|
16
|
+
- Keep folder paths in a private locations file. Use local folders, Windows shares, or a mounted share on Linux.
|
|
17
|
+
- Run on a server with Docker Compose. The image includes FFmpeg and ffprobe. Each crawl logs which shows succeeded and which failed.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pypodcastde
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Update a local library of ARD podcasts and video soundtracks
|
|
5
|
+
Project-URL: Repository, https://github.com/gieses/pypodcastde
|
|
6
|
+
Project-URL: Issues, https://github.com/gieses/pypodcastde/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/gieses/pypodcastde/blob/main/CHANGELOG.md
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Requires-Dist: loguru>=0.7.3
|
|
10
|
+
Requires-Dist: yt-dlp>=2026.3.17
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
<img src="assets/logo.svg" alt="A smiling cassette in teal headphones with floating musical notes" width="150" height="150">
|
|
14
|
+
|
|
15
|
+
# pypodcastde
|
|
16
|
+
|
|
17
|
+
[](https://github.com/gieses/pypodcastde/actions/workflows/ci.yml?query=branch%3Amain)
|
|
18
|
+
[](pyproject.toml)
|
|
19
|
+
[](pyproject.toml)
|
|
20
|
+
[](compose.example.yaml)
|
|
21
|
+
|
|
22
|
+
One of the best things about German public broadcasting is how seriously it takes kids’ questions. On [ARD Sounds](https://www.ardsounds.de/), *Lachlabor* tests them with real experiments, while *Ohrenbär* has stories for the quieter end of the day. Over in the [ARD Mediathek](https://www.ardmediathek.de/), Checkerin Marina solves puzzles on screen.
|
|
23
|
+
|
|
24
|
+
pypodcastde keeps those favourites in one folder, ready for a car ride or bedtime. Podcasts keep their original audio format; for video shows, it saves the soundtrack as an MP3. This is an independent tool, not an ARD app.
|
|
25
|
+
|
|
26
|
+
## Your first download
|
|
27
|
+
|
|
28
|
+
You'll need Python 3.11+ and [uv](https://docs.astral.sh/uv/). For video shows, install `ffmpeg` and `ffprobe` too. From the project folder:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv sync --locked
|
|
32
|
+
uv run pypodcastde init-config # Once. Won't overwrite your settings.
|
|
33
|
+
# Edit locations_local.txt to pick shows and folders.
|
|
34
|
+
uv run pypodcastde mine --try # See what would download.
|
|
35
|
+
uv run pypodcastde mine # Download it.
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The first run can fetch a show's whole back catalogue. Later runs skip episodes you already have.
|
|
39
|
+
|
|
40
|
+
### Shows and folders
|
|
41
|
+
|
|
42
|
+
[`locations_example.txt`](locations_example.txt) lists eight shows. Edit your **private copy**, `locations_local.txt`. Keep the header and separate the show name, folder, and URL with semicolons. Delete rows for shows you don't want.
|
|
43
|
+
|
|
44
|
+
| Where you store episodes | Example folder for one show |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| Beside the settings file (Windows or Linux) | `podcasts/Lachlabor` |
|
|
47
|
+
| Windows drive or share | `D:\Podcasts\Lachlabor` or `\\server\share\Podcasts\Lachlabor` |
|
|
48
|
+
| Linux drive or mounted share | `/srv/podcasts/Lachlabor` |
|
|
49
|
+
|
|
50
|
+
The CLI looks for the private file in the current directory. Use `--config PATH` if you keep it elsewhere. To move the whole library without editing each row, use `--output-root DIR`: each show keeps its last folder name under `DIR`. Git ignores the private file and the default `podcasts/` folder.
|
|
51
|
+
|
|
52
|
+
## Daily downloads with Docker
|
|
53
|
+
|
|
54
|
+
On a Linux server with Docker Compose, the image handles FFmpeg for you:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
mkdir -p config
|
|
58
|
+
cp -n locations_example.txt config/locations_local.txt
|
|
59
|
+
cp -n compose.example.yaml compose.yaml
|
|
60
|
+
# Edit compose.yaml: point /srv/podcasts at your writable library folder.
|
|
61
|
+
# If it's on a NAS, mount it before starting Docker.
|
|
62
|
+
docker compose up --build -d
|
|
63
|
+
docker compose logs -f crawler
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The crawler runs immediately, then every 24 hours. It rereads `config/locations_local.txt` each time. Each run ends with a count of successful and failed shows. `Ctrl+C` leaves the log viewer; `docker compose stop` stops the crawler. Downloads stay on the host.
|
|
67
|
+
|
|
68
|
+
## Updating
|
|
69
|
+
|
|
70
|
+
Use `mine --force --try` to preview a full re-download. Drop `--try` to replace existing episodes. Don't run two crawlers against the same folder at once.
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
git pull --ff-only
|
|
74
|
+
uv sync --locked # For a local install
|
|
75
|
+
# Or, for Docker:
|
|
76
|
+
docker compose build --pull && docker compose up -d
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
See the [changelog](CHANGELOG.md) for release notes. If you're working on the code, `make check` runs lint and tests; `make help` lists the other commands.
|
|
80
|
+
|
|
81
|
+
Run `make hooks` once to enable pre-commit checks, or `make pre-commit` to check the whole tree. Use `uv add` (or `uv add --dev`) for dependency changes and commit `uv.lock`. CI and Docker use the locked versions.
|
|
82
|
+
|
|
83
|
+
### Publish a release
|
|
84
|
+
|
|
85
|
+
Run `make publish-check` to build the current version and check it without uploading.
|
|
86
|
+
Use `make publish-test` for TestPyPI or `make publish` for the real PyPI.
|
|
87
|
+
For uploads, set `UV_PUBLISH_TOKEN` through your shell or secret manager using a token
|
|
88
|
+
for that index. Never commit a token. PyPI won't replace an existing release file.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
<img src="assets/logo.svg" alt="A smiling cassette in teal headphones with floating musical notes" width="150" height="150">
|
|
2
|
+
|
|
3
|
+
# pypodcastde
|
|
4
|
+
|
|
5
|
+
[](https://github.com/gieses/pypodcastde/actions/workflows/ci.yml?query=branch%3Amain)
|
|
6
|
+
[](pyproject.toml)
|
|
7
|
+
[](pyproject.toml)
|
|
8
|
+
[](compose.example.yaml)
|
|
9
|
+
|
|
10
|
+
One of the best things about German public broadcasting is how seriously it takes kids’ questions. On [ARD Sounds](https://www.ardsounds.de/), *Lachlabor* tests them with real experiments, while *Ohrenbär* has stories for the quieter end of the day. Over in the [ARD Mediathek](https://www.ardmediathek.de/), Checkerin Marina solves puzzles on screen.
|
|
11
|
+
|
|
12
|
+
pypodcastde keeps those favourites in one folder, ready for a car ride or bedtime. Podcasts keep their original audio format; for video shows, it saves the soundtrack as an MP3. This is an independent tool, not an ARD app.
|
|
13
|
+
|
|
14
|
+
## Your first download
|
|
15
|
+
|
|
16
|
+
You'll need Python 3.11+ and [uv](https://docs.astral.sh/uv/). For video shows, install `ffmpeg` and `ffprobe` too. From the project folder:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
uv sync --locked
|
|
20
|
+
uv run pypodcastde init-config # Once. Won't overwrite your settings.
|
|
21
|
+
# Edit locations_local.txt to pick shows and folders.
|
|
22
|
+
uv run pypodcastde mine --try # See what would download.
|
|
23
|
+
uv run pypodcastde mine # Download it.
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The first run can fetch a show's whole back catalogue. Later runs skip episodes you already have.
|
|
27
|
+
|
|
28
|
+
### Shows and folders
|
|
29
|
+
|
|
30
|
+
[`locations_example.txt`](locations_example.txt) lists eight shows. Edit your **private copy**, `locations_local.txt`. Keep the header and separate the show name, folder, and URL with semicolons. Delete rows for shows you don't want.
|
|
31
|
+
|
|
32
|
+
| Where you store episodes | Example folder for one show |
|
|
33
|
+
| --- | --- |
|
|
34
|
+
| Beside the settings file (Windows or Linux) | `podcasts/Lachlabor` |
|
|
35
|
+
| Windows drive or share | `D:\Podcasts\Lachlabor` or `\\server\share\Podcasts\Lachlabor` |
|
|
36
|
+
| Linux drive or mounted share | `/srv/podcasts/Lachlabor` |
|
|
37
|
+
|
|
38
|
+
The CLI looks for the private file in the current directory. Use `--config PATH` if you keep it elsewhere. To move the whole library without editing each row, use `--output-root DIR`: each show keeps its last folder name under `DIR`. Git ignores the private file and the default `podcasts/` folder.
|
|
39
|
+
|
|
40
|
+
## Daily downloads with Docker
|
|
41
|
+
|
|
42
|
+
On a Linux server with Docker Compose, the image handles FFmpeg for you:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
mkdir -p config
|
|
46
|
+
cp -n locations_example.txt config/locations_local.txt
|
|
47
|
+
cp -n compose.example.yaml compose.yaml
|
|
48
|
+
# Edit compose.yaml: point /srv/podcasts at your writable library folder.
|
|
49
|
+
# If it's on a NAS, mount it before starting Docker.
|
|
50
|
+
docker compose up --build -d
|
|
51
|
+
docker compose logs -f crawler
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The crawler runs immediately, then every 24 hours. It rereads `config/locations_local.txt` each time. Each run ends with a count of successful and failed shows. `Ctrl+C` leaves the log viewer; `docker compose stop` stops the crawler. Downloads stay on the host.
|
|
55
|
+
|
|
56
|
+
## Updating
|
|
57
|
+
|
|
58
|
+
Use `mine --force --try` to preview a full re-download. Drop `--try` to replace existing episodes. Don't run two crawlers against the same folder at once.
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
git pull --ff-only
|
|
62
|
+
uv sync --locked # For a local install
|
|
63
|
+
# Or, for Docker:
|
|
64
|
+
docker compose build --pull && docker compose up -d
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
See the [changelog](CHANGELOG.md) for release notes. If you're working on the code, `make check` runs lint and tests; `make help` lists the other commands.
|
|
68
|
+
|
|
69
|
+
Run `make hooks` once to enable pre-commit checks, or `make pre-commit` to check the whole tree. Use `uv add` (or `uv add --dev`) for dependency changes and commit `uv.lock`. CI and Docker use the locked versions.
|
|
70
|
+
|
|
71
|
+
### Publish a release
|
|
72
|
+
|
|
73
|
+
Run `make publish-check` to build the current version and check it without uploading.
|
|
74
|
+
Use `make publish-test` for TestPyPI or `make publish` for the real PyPI.
|
|
75
|
+
For uploads, set `UV_PUBLISH_TOKEN` through your shell or secret manager using a token
|
|
76
|
+
for that index. Never commit a token. PyPI won't replace an existing release file.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
Podcast Name;Location;URL
|
|
2
|
+
Checker Tobi;podcasts/Checker_Tobi;https://www.ardsounds.de/sendung/checkpod-der-podcast-mit-checker-tobi/urn:ard:show:2af809299201a921/
|
|
3
|
+
Checkerin Marina;podcasts/Checkerin_Marina;https://www.ardmediathek.de/sendung/check-in-marinas-raetsel-abenteuer/Y3JpZDovL2JyLmRlL2Jyb2FkY2FzdFNlcmllcy9TMjAyNVdPMDEzNDAxQTA
|
|
4
|
+
Alle Gegen Nico;podcasts/Alle-gegen-Nico;https://www.ardsounds.de/sendung/alle-gegen-nico-zockt-um-die-quizkrone/urn:ard:show:0305ba2e345adbb5/
|
|
5
|
+
Lachlabor;podcasts/Lachlabor;https://www.ardsounds.de/sendung/lachlabor/urn:ard:show:cfb9386906103b4a/
|
|
6
|
+
Anna und die wilden Tiere;podcasts/Anna_und_die_wilden_Tiere;https://www.ardsounds.de/sendung/anna-und-die-wilden-tiere-der-podcast/urn:ard:show:8530c5d73f93f021/
|
|
7
|
+
Die Maus zum Hören;podcasts/Die_Maus_zum_Hoeren;https://www.ardsounds.de/sendung/die-maus-zum-hoeren/urn:ard:show:b4a27641ebfcb11b/
|
|
8
|
+
Kakadu;podcasts/Kakadu;https://www.ardsounds.de/sendung/kakadu-der-kinderpodcast-deutschlandfunk-kultur/urn:ard:show:30c22e3f2fc00769/
|
|
9
|
+
Ohrenbär;podcasts/Ohrenbaer;https://www.ardsounds.de/sendung/ohrenbaer/urn:ard:show:7d4e6af8e64fee8e/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Download podcasts and convert video episodes into MP3s."""
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Types for the fields requested by the ARD show catalogue query."""
|
|
2
|
+
|
|
3
|
+
from typing import NotRequired, TypedDict
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Audio(TypedDict):
|
|
7
|
+
"""One playable audio variant and its optional direct-download URL."""
|
|
8
|
+
|
|
9
|
+
url: str
|
|
10
|
+
downloadUrl: NotRequired[str | None]
|
|
11
|
+
mimeType: NotRequired[str | None]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Episode(TypedDict):
|
|
15
|
+
"""Episode metadata used to select, name, and index a download."""
|
|
16
|
+
|
|
17
|
+
id: str
|
|
18
|
+
title: str
|
|
19
|
+
publishDate: str
|
|
20
|
+
duration: int | None
|
|
21
|
+
summary: NotRequired[str | None]
|
|
22
|
+
audios: list[Audio] | None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class PageInfo(TypedDict):
|
|
26
|
+
"""Cursor and continuation flag for an ARD result page."""
|
|
27
|
+
|
|
28
|
+
hasNextPage: bool
|
|
29
|
+
endCursor: str | None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Items(TypedDict):
|
|
33
|
+
"""A page of episodes and the size of the complete catalogue."""
|
|
34
|
+
|
|
35
|
+
totalCount: int
|
|
36
|
+
nodes: list[Episode]
|
|
37
|
+
pageInfo: PageInfo
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Show(TypedDict):
|
|
41
|
+
"""Show metadata returned for one catalogue request."""
|
|
42
|
+
|
|
43
|
+
title: str
|
|
44
|
+
items: Items
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class CatalogueData(TypedDict):
|
|
48
|
+
"""GraphQL data payload; unknown show IDs return a null show."""
|
|
49
|
+
|
|
50
|
+
show: Show | None
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Download audio from ARD Sounds. When run directly, download CheckPod."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
import http.client
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
import urllib.request
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from loguru import logger
|
|
14
|
+
|
|
15
|
+
from .catalogue import Audio, CatalogueData, Episode
|
|
16
|
+
from .modes import add_mode_options
|
|
17
|
+
from .reporting import configure_logging, report_run
|
|
18
|
+
|
|
19
|
+
API_URL = "https://api.ardaudiothek.de/graphql"
|
|
20
|
+
SHOW_ID = "urn:ard:show:2af809299201a921"
|
|
21
|
+
SHOW_PREFIX = "CheckPod"
|
|
22
|
+
OUT_DIR = Path("podcasts") / "Checker_Tobi"
|
|
23
|
+
USER_AGENT = "Mozilla/5.0 (CheckPod crawler)"
|
|
24
|
+
PAGE_SIZE = 50
|
|
25
|
+
|
|
26
|
+
QUERY = """
|
|
27
|
+
query($id: ID!, $first: Int!, $after: Cursor) {
|
|
28
|
+
show(id: $id) {
|
|
29
|
+
title
|
|
30
|
+
items(first: $first, after: $after, orderBy: PUBLISH_DATE_DESC) {
|
|
31
|
+
totalCount
|
|
32
|
+
pageInfo { hasNextPage endCursor }
|
|
33
|
+
nodes {
|
|
34
|
+
id title publishDate duration summary
|
|
35
|
+
audios { url downloadUrl mimeType }
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def graphql(variables: dict[str, str | int | None]) -> CatalogueData:
|
|
44
|
+
"""Fetch one catalogue page, raising RuntimeError on GraphQL errors."""
|
|
45
|
+
body = json.dumps({"query": QUERY, "variables": variables}).encode()
|
|
46
|
+
req = urllib.request.Request(
|
|
47
|
+
API_URL,
|
|
48
|
+
data=body,
|
|
49
|
+
headers={"Content-Type": "application/json", "User-Agent": USER_AGENT},
|
|
50
|
+
)
|
|
51
|
+
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
52
|
+
data = json.load(resp)
|
|
53
|
+
if data.get("errors"):
|
|
54
|
+
raise RuntimeError(data["errors"])
|
|
55
|
+
return data["data"]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def fetch_episodes(show_id: str = SHOW_ID) -> list[Episode]:
|
|
59
|
+
"""Collect all pages newest first, rejecting unknown shows and repeated cursors."""
|
|
60
|
+
episodes, cursor = [], None
|
|
61
|
+
cursors = set()
|
|
62
|
+
while True:
|
|
63
|
+
show = graphql({"id": show_id, "first": PAGE_SIZE, "after": cursor})["show"]
|
|
64
|
+
if not show:
|
|
65
|
+
raise ValueError(f"ARD show not found: {show_id}")
|
|
66
|
+
items = show["items"]
|
|
67
|
+
episodes.extend(items["nodes"])
|
|
68
|
+
if not items["pageInfo"]["hasNextPage"]:
|
|
69
|
+
return episodes
|
|
70
|
+
cursor = items["pageInfo"]["endCursor"]
|
|
71
|
+
if not cursor or cursor in cursors:
|
|
72
|
+
raise ValueError("ARD returned a missing or repeated pagination cursor")
|
|
73
|
+
cursors.add(cursor)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def clean_title(title: str, prefix: str = SHOW_PREFIX) -> str:
|
|
77
|
+
"""Remove the show prefix and characters unsuitable for an episode filename."""
|
|
78
|
+
title = re.sub(rf"^\s*{re.escape(prefix)}\s*[:\-–]?\s*", "", title, flags=re.IGNORECASE)
|
|
79
|
+
title = re.sub(r"\s*\|\s*", " - ", title)
|
|
80
|
+
title = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "", title)
|
|
81
|
+
title = re.sub(r"\s+", " ", title).strip(" .")
|
|
82
|
+
return title[:150] or "Unbenannt"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def audio_candidates(audios: list[Audio] | None) -> list[Audio]:
|
|
86
|
+
"""All audio variants, MP3 first; later ones are fallbacks if a URL is dead."""
|
|
87
|
+
return sorted(audios or [], key=lambda a: "mp3" not in (a.get("mimeType") or ""))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def extension(audio: Audio) -> str:
|
|
91
|
+
"""Choose a filename suffix from the advertised media type and URL."""
|
|
92
|
+
url = (audio.get("downloadUrl") or audio["url"]).split("?")[0]
|
|
93
|
+
suffix = Path(url).suffix.lower()
|
|
94
|
+
if "mp3" in (audio.get("mimeType") or ""):
|
|
95
|
+
return ".mp3"
|
|
96
|
+
return ".m4a" if suffix in (".mp4", ".m4a", "") else suffix
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def download(url: str, target: Path, retries: int = 3) -> None:
|
|
100
|
+
"""Stream audio to a temporary file and replace the target after size verification."""
|
|
101
|
+
tmp = target.with_suffix(target.suffix + ".part")
|
|
102
|
+
for attempt in range(1, retries + 1):
|
|
103
|
+
try:
|
|
104
|
+
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
105
|
+
with urllib.request.urlopen(req, timeout=120) as resp, open(tmp, "wb") as f:
|
|
106
|
+
expected_size = resp.headers.get("Content-Length")
|
|
107
|
+
received = 0
|
|
108
|
+
while chunk := resp.read(1 << 16):
|
|
109
|
+
f.write(chunk)
|
|
110
|
+
received += len(chunk)
|
|
111
|
+
if received == 0:
|
|
112
|
+
raise OSError("Empty audio response")
|
|
113
|
+
if expected_size is not None and received != int(expected_size):
|
|
114
|
+
raise OSError(
|
|
115
|
+
f"Incomplete audio response: expected {expected_size} bytes, got {received}"
|
|
116
|
+
)
|
|
117
|
+
tmp.replace(target)
|
|
118
|
+
return
|
|
119
|
+
except (OSError, ValueError, http.client.HTTPException) as exc:
|
|
120
|
+
try:
|
|
121
|
+
tmp.unlink(missing_ok=True)
|
|
122
|
+
except OSError as cleanup_exc:
|
|
123
|
+
logger.warning("Could not remove {}: {}", tmp, cleanup_exc)
|
|
124
|
+
if attempt == retries or getattr(exc, "code", None) in (403, 404, 410):
|
|
125
|
+
raise
|
|
126
|
+
logger.warning("Retry {}/{} for {}: {}", attempt, retries - 1, target.name, exc)
|
|
127
|
+
time.sleep(2 * attempt)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def main() -> int:
|
|
131
|
+
"""Run the standalone CheckPod crawler with optional episode and output limits."""
|
|
132
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
133
|
+
parser.add_argument("-o", "--out", type=Path, default=OUT_DIR, help="output directory")
|
|
134
|
+
parser.add_argument("-n", "--limit", type=int, help="only the N newest episodes")
|
|
135
|
+
add_mode_options(parser)
|
|
136
|
+
args = parser.parse_args()
|
|
137
|
+
configure_logging()
|
|
138
|
+
with report_run(args.dry_run, args.force) as summary:
|
|
139
|
+
summary.crawl(
|
|
140
|
+
SHOW_PREFIX,
|
|
141
|
+
lambda: mine(
|
|
142
|
+
SHOW_ID, SHOW_PREFIX, args.out, args.dry_run, args.limit, force=args.force
|
|
143
|
+
),
|
|
144
|
+
)
|
|
145
|
+
return summary.status
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def mine(
|
|
149
|
+
show_id: str,
|
|
150
|
+
prefix: str,
|
|
151
|
+
out: Path,
|
|
152
|
+
dry_run: bool,
|
|
153
|
+
limit: int | None = None,
|
|
154
|
+
*,
|
|
155
|
+
force: bool = False,
|
|
156
|
+
) -> int:
|
|
157
|
+
"""Preview or download the selected episodes; return nonzero for failed downloads."""
|
|
158
|
+
if not dry_run:
|
|
159
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
160
|
+
episodes = fetch_episodes(show_id)
|
|
161
|
+
logger.info("Found {} episodes", len(episodes))
|
|
162
|
+
if limit:
|
|
163
|
+
episodes = episodes[:limit]
|
|
164
|
+
|
|
165
|
+
used_names: set[str] = set()
|
|
166
|
+
metadata, failed = [], 0
|
|
167
|
+
downloaded = planned = skipped = unavailable = 0
|
|
168
|
+
for ep in sorted(episodes, key=lambda e: e["publishDate"]):
|
|
169
|
+
candidates = audio_candidates(ep["audios"])
|
|
170
|
+
if not candidates:
|
|
171
|
+
unavailable += 1
|
|
172
|
+
logger.warning("No audio available: {}", ep["title"])
|
|
173
|
+
continue
|
|
174
|
+
name = f"{prefix} - {ep['publishDate'][:10]} - {clean_title(ep['title'], prefix)}"
|
|
175
|
+
if name.lower() in used_names:
|
|
176
|
+
name += f" ({hashlib.sha256(ep['id'].encode()).hexdigest()[:12]})"
|
|
177
|
+
used_names.add(name.lower())
|
|
178
|
+
|
|
179
|
+
entry = {
|
|
180
|
+
"file": None,
|
|
181
|
+
"id": ep["id"],
|
|
182
|
+
"title": ep["title"],
|
|
183
|
+
"publishDate": ep["publishDate"],
|
|
184
|
+
"duration": ep["duration"],
|
|
185
|
+
"summary": ep.get("summary"),
|
|
186
|
+
}
|
|
187
|
+
existing = [out / (name + ext) for ext in {extension(a) for a in candidates}]
|
|
188
|
+
existing = [p for p in existing if p.is_file() and p.stat().st_size]
|
|
189
|
+
if existing and not force:
|
|
190
|
+
skipped += 1
|
|
191
|
+
entry["file"] = existing[0].name
|
|
192
|
+
metadata.append(entry)
|
|
193
|
+
continue
|
|
194
|
+
|
|
195
|
+
logger.info("{}: {}", "Would download" if dry_run else "Downloading", name)
|
|
196
|
+
if dry_run:
|
|
197
|
+
planned += 1
|
|
198
|
+
continue
|
|
199
|
+
last_exc = None
|
|
200
|
+
for audio in candidates:
|
|
201
|
+
target = out / (name + extension(audio))
|
|
202
|
+
try:
|
|
203
|
+
download(audio.get("downloadUrl") or audio["url"], target)
|
|
204
|
+
entry["file"] = target.name
|
|
205
|
+
break
|
|
206
|
+
except (OSError, ValueError, http.client.HTTPException) as exc:
|
|
207
|
+
last_exc = exc
|
|
208
|
+
if entry["file"]:
|
|
209
|
+
downloaded += 1
|
|
210
|
+
metadata.append(entry)
|
|
211
|
+
else:
|
|
212
|
+
failed += 1
|
|
213
|
+
if existing:
|
|
214
|
+
entry["file"] = existing[0].name
|
|
215
|
+
metadata.append(entry)
|
|
216
|
+
logger.error("Could not download {}: {}", name, last_exc)
|
|
217
|
+
|
|
218
|
+
if not dry_run:
|
|
219
|
+
manifest = out / "episodes.json"
|
|
220
|
+
temporary = manifest.with_suffix(".json.part")
|
|
221
|
+
temporary.write_text(json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
222
|
+
temporary.replace(manifest)
|
|
223
|
+
logger.info(
|
|
224
|
+
"Episodes | downloaded: {} | planned: {} | existing: {} | unavailable: {} | failed: {}",
|
|
225
|
+
downloaded,
|
|
226
|
+
planned,
|
|
227
|
+
skipped,
|
|
228
|
+
unavailable,
|
|
229
|
+
failed,
|
|
230
|
+
)
|
|
231
|
+
return 1 if failed else 0
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
if __name__ == "__main__":
|
|
235
|
+
sys.exit(main())
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Download the shows in a locations file, once or every 24 hours."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import csv
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from loguru import logger
|
|
10
|
+
|
|
11
|
+
from . import checkpod, nico, video
|
|
12
|
+
from .config import Podcast, default_config, example_config, load
|
|
13
|
+
from .modes import add_mode_options
|
|
14
|
+
from .reporting import configure_logging, report_run
|
|
15
|
+
|
|
16
|
+
DAY = 24 * 60 * 60
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def crawl(podcast: Podcast, dry_run: bool, force: bool) -> int:
|
|
20
|
+
"""Run the downloader appropriate for the source and return its exit status."""
|
|
21
|
+
logger.info("Destination: {}", podcast.destination)
|
|
22
|
+
if podcast.show_id == nico.SHOW_ID:
|
|
23
|
+
return nico.mine(podcast.destination, dry_run, force=force)
|
|
24
|
+
if podcast.show_id:
|
|
25
|
+
prefix = checkpod.SHOW_PREFIX if podcast.show_id == checkpod.SHOW_ID else podcast.name
|
|
26
|
+
return checkpod.mine(podcast.show_id, prefix, podcast.destination, dry_run, force=force)
|
|
27
|
+
return video.mine(podcast.url, podcast.destination, dry_run, force=force)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def update(path: Path, root: Path | None, dry_run: bool, force: bool = False) -> int:
|
|
31
|
+
"""Read the show list and continue with the next show if a download fails."""
|
|
32
|
+
with report_run(dry_run, force) as summary:
|
|
33
|
+
try:
|
|
34
|
+
podcasts = load(path, root)
|
|
35
|
+
except (OSError, ValueError, csv.Error) as exc:
|
|
36
|
+
logger.error("Cannot load {}: {}", path, exc)
|
|
37
|
+
if isinstance(exc, FileNotFoundError):
|
|
38
|
+
logger.info(
|
|
39
|
+
"Create a private table with 'pypodcastde init-config' or pass --config PATH"
|
|
40
|
+
)
|
|
41
|
+
summary.config_error = True
|
|
42
|
+
else:
|
|
43
|
+
for podcast in podcasts:
|
|
44
|
+
summary.crawl(podcast.name, lambda p=podcast: crawl(p, dry_run, force))
|
|
45
|
+
return summary.status
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def main(argv: list[str] | None = None) -> int:
|
|
49
|
+
"""Run the requested CLI command; return 130 if a crawl or wait is interrupted."""
|
|
50
|
+
parser = argparse.ArgumentParser(description="Update local podcast directories")
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
"command",
|
|
53
|
+
choices=["mine", "schedule", "init-config"],
|
|
54
|
+
help="update once, schedule daily, or create a private locations table",
|
|
55
|
+
)
|
|
56
|
+
parser.add_argument(
|
|
57
|
+
"--config", type=Path, default=default_config(), help="semicolon-separated locations table"
|
|
58
|
+
)
|
|
59
|
+
parser.add_argument(
|
|
60
|
+
"--output-root",
|
|
61
|
+
type=Path,
|
|
62
|
+
default=os.getenv("PYPODCASTDE_OUTPUT_ROOT"),
|
|
63
|
+
help="place each destination directory under this root",
|
|
64
|
+
)
|
|
65
|
+
add_mode_options(parser)
|
|
66
|
+
args = parser.parse_args(argv)
|
|
67
|
+
configure_logging()
|
|
68
|
+
if args.command == "init-config":
|
|
69
|
+
try:
|
|
70
|
+
template = example_config().read_bytes()
|
|
71
|
+
with args.config.open("xb") as stream:
|
|
72
|
+
stream.write(template)
|
|
73
|
+
except OSError as exc:
|
|
74
|
+
logger.error("Cannot create {}: {}", args.config, exc)
|
|
75
|
+
return 1
|
|
76
|
+
logger.success("Created {}. Edit the locations before crawling.", args.config)
|
|
77
|
+
return 0
|
|
78
|
+
try:
|
|
79
|
+
while True:
|
|
80
|
+
start = time.monotonic()
|
|
81
|
+
status = update(args.config, args.output_root, args.dry_run, args.force)
|
|
82
|
+
if args.command == "mine":
|
|
83
|
+
return status
|
|
84
|
+
logger.info("Next crawl scheduled 24 hours from the start of this run")
|
|
85
|
+
time.sleep(max(0, DAY - (time.monotonic() - start)))
|
|
86
|
+
except KeyboardInterrupt:
|
|
87
|
+
return 130
|