focus-feed 1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. focus_feed-1.0/.github/workflows/publish-to-pypi.yml +53 -0
  2. focus_feed-1.0/.gitignore +67 -0
  3. focus_feed-1.0/.pre-commit-config.yaml +22 -0
  4. focus_feed-1.0/AGENTS.md +58 -0
  5. focus_feed-1.0/PKG-INFO +18 -0
  6. focus_feed-1.0/README.md +286 -0
  7. focus_feed-1.0/artifacts/design.md +109 -0
  8. focus_feed-1.0/artifacts/proposal.md +37 -0
  9. focus_feed-1.0/artifacts/specs.md +102 -0
  10. focus_feed-1.0/artifacts/tasks.md +1590 -0
  11. focus_feed-1.0/example.env +29 -0
  12. focus_feed-1.0/pyproject.toml +136 -0
  13. focus_feed-1.0/src/focus_feed/__init__.py +3 -0
  14. focus_feed-1.0/src/focus_feed/cli.py +2390 -0
  15. focus_feed-1.0/src/focus_feed/config.py +157 -0
  16. focus_feed-1.0/src/focus_feed/errors.py +392 -0
  17. focus_feed-1.0/src/focus_feed/feeds.py +1524 -0
  18. focus_feed-1.0/src/focus_feed/metadata.py +589 -0
  19. focus_feed-1.0/src/focus_feed/producer.py +1504 -0
  20. focus_feed-1.0/src/focus_feed/py.typed +0 -0
  21. focus_feed-1.0/src/focus_feed/scanner.py +595 -0
  22. focus_feed-1.0/src/focus_feed/services/__init__.py +1 -0
  23. focus_feed-1.0/src/focus_feed/services/extractors.py +514 -0
  24. focus_feed-1.0/src/focus_feed/services/remote_transcriber.py +371 -0
  25. focus_feed-1.0/src/focus_feed/services/summarizer.py +458 -0
  26. focus_feed-1.0/src/focus_feed/services/transcriber.py +470 -0
  27. focus_feed-1.0/tests/__init__.py +0 -0
  28. focus_feed-1.0/tests/conftest.py +76 -0
  29. focus_feed-1.0/tests/test_cli.py +4716 -0
  30. focus_feed-1.0/tests/test_config.py +539 -0
  31. focus_feed-1.0/tests/test_errors.py +631 -0
  32. focus_feed-1.0/tests/test_extractors.py +1324 -0
  33. focus_feed-1.0/tests/test_feeds.py +4566 -0
  34. focus_feed-1.0/tests/test_integration.py +637 -0
  35. focus_feed-1.0/tests/test_metadata.py +1603 -0
  36. focus_feed-1.0/tests/test_producer.py +2584 -0
  37. focus_feed-1.0/tests/test_remote_transcriber.py +336 -0
  38. focus_feed-1.0/tests/test_scanner.py +1037 -0
  39. focus_feed-1.0/tests/test_summarizer.py +1088 -0
  40. focus_feed-1.0/tests/test_transcriber.py +731 -0
  41. focus_feed-1.0/uv.lock +1288 -0
@@ -0,0 +1,53 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ build:
12
+ name: Build distributions
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - name: Check out repository
16
+ uses: actions/checkout@v7
17
+
18
+ - name: Set up Python
19
+ uses: actions/setup-python@v7
20
+ with:
21
+ python-version: "3.x"
22
+
23
+ - name: Install build frontend
24
+ run: python -m pip install --upgrade build
25
+
26
+ - name: Build source distribution and wheel
27
+ run: python -m build
28
+
29
+ - name: Store distributions
30
+ uses: actions/upload-artifact@v7
31
+ with:
32
+ name: pypi-distributions
33
+ path: dist/
34
+ if-no-files-found: error
35
+
36
+ publish:
37
+ name: Publish distributions to PyPI
38
+ needs: build
39
+ runs-on: ubuntu-latest
40
+ environment:
41
+ name: pypi
42
+ url: https://pypi.org/p/focus-feed
43
+ permissions:
44
+ id-token: write
45
+ steps:
46
+ - name: Retrieve distributions
47
+ uses: actions/download-artifact@v8
48
+ with:
49
+ name: pypi-distributions
50
+ path: dist/
51
+
52
+ - name: Publish package distributions to PyPI
53
+ uses: pypa/gh-action-pypi-publish@v1.14.3
@@ -0,0 +1,67 @@
1
+ data/
2
+ __pycache__/
3
+ scripts/
4
+ .env
5
+ *.log
6
+
7
+ # Python bytecode and native extensions
8
+ *.py[cod]
9
+ *$py.class
10
+ *.so
11
+
12
+ # Distribution / packaging
13
+ .Python
14
+ build/
15
+ develop-eggs/
16
+ dist/
17
+ downloads/
18
+ eggs/
19
+ .eggs/
20
+ lib/
21
+ lib64/
22
+ parts/
23
+ sdist/
24
+ var/
25
+ wheels/
26
+ *.egg-info/
27
+ .installed.cfg
28
+ *.egg
29
+ pip-wheel-metadata/
30
+ share/python-wheels/
31
+
32
+ # Virtual environments
33
+ .venv/
34
+ venv/
35
+ env/
36
+ ENV/
37
+
38
+ # Test and type-check caches
39
+ .pytest_cache/
40
+ .mypy_cache/
41
+ .ruff_cache/
42
+ *.pyc
43
+
44
+ # Coverage
45
+ .coverage
46
+ .coverage.*
47
+ htmlcov/
48
+ .tox/
49
+ .nox/
50
+ coverage.xml
51
+ *.cover
52
+
53
+ # Jupyter and IDE files
54
+ .ipynb_checkpoints
55
+ .idea/
56
+ .vscode/
57
+ *.swp
58
+ *.swo
59
+ *~
60
+ .cursor/
61
+
62
+ # OS files
63
+ .DS_Store
64
+ Thumbs.db
65
+
66
+ # uv cache
67
+ uv-cache/
@@ -0,0 +1,22 @@
1
+ repos:
2
+ - repo: https://github.com/astral-sh/ruff-pre-commit
3
+ rev: v0.8.0
4
+ hooks:
5
+ - id: ruff
6
+ args: [--fix]
7
+ - id: ruff-format
8
+
9
+ - repo: https://github.com/pre-commit/mirrors-mypy
10
+ rev: v1.13.0
11
+ hooks:
12
+ - id: mypy
13
+ files: ^src/
14
+ additional_dependencies:
15
+ - pydantic-settings
16
+ - pydantic
17
+ - openai
18
+ - feedgen
19
+ - watchdog
20
+ - faster-whisper
21
+ - pypdf
22
+ - beautifulsoup4
@@ -0,0 +1,58 @@
1
+ # AGENTS.md - Focus Feed
2
+
3
+ Purpose: Transform education content (PDF/HTML/MP4) into RSS feeds.
4
+
5
+ Command: `uv run focus-feed <command>` from the workspace root (directory containing `src/`, `tests/`, and `data/`).
6
+
7
+ ## Core
8
+ - MP4: Requires `transcribe` → `summarize` → feed-eligible.
9
+ - PDF/HTML: Requires `extract` → `summarize` → feed-eligible.
10
+ - Feeds: Generated only where `feed.json` exists. Requires feed-eligible files.
11
+
12
+ ## Key Commands
13
+ - `process` / `watch`: Process files batch/incrementally.
14
+ - `transcribe`, `summarize`, `extract`, `title`, `feeds`, `validate`, `create-feed`, `cleanup`.
15
+ - Flags: `--force`, `--dry-run`.
16
+
17
+ ## Metadata Schema
18
+ Each file gets a `[filename].json` with:
19
+ ```json
20
+ {
21
+ "title": {"filename": "", "extracted": "", "generated": ""},
22
+ "description": "", "summary": "", "content": "",
23
+ "filetype": "mp4|pdf|html", "hash": "sha256...",
24
+ "date": 1710000000.123, "publication_date": 1710000000.0,
25
+ "duration": 45.5, "creation_time": "ISO-8601",
26
+ "transcribe": {"status": "complete"},
27
+ "summarize": {"status": "complete"},
28
+ "extract": {"status": "complete"},
29
+ "generate_title": {"status": "complete"},
30
+ "error": ""
31
+ }
32
+ ```
33
+
34
+ Feed Eligibility: MP4 = transcribe & summarize complete. PDF/HTML = extract & summarize complete.
35
+
36
+ Title Rules: `generated` > `extracted` > `filename`. Parent folder name prepended.
37
+
38
+ ## Config (Env Vars)
39
+ Required: `BASE_URL`, `LLM_ENDPOINT`, `LLM_API_KEY`, `LLM_MODEL`, `PROMPT_ARTICLE`, `PROMPT_PODCAST`, `PROMPT_TITLE`.
40
+
41
+ Optional: `PROMPT_ARTICLE_REFLECTION`, `PROMPT_PODCAST_REFLECTION`, `TRANSCRIPTION_ENDPOINT` (remote overrides local), `TRANSCRIPTION_API_KEY`, `TRANSCRIPTION_REMOTE_MODEL`, `TRANSCRIPTION_MODEL`, `TRANSCRIPTION_DEVICE`, `TRANSCRIPTION_COMPUTE_TYPE`.
42
+
43
+ ## Artifacts
44
+ - `artifacts/proposal.md` - Project proposal and scope.
45
+ - `artifacts/specs.md` - Functional/non-functional specs.
46
+ - `artifacts/design.md` - System architecture and data flow.
47
+ - `artifacts/tasks.md` - Implementation task list.
48
+
49
+ ## Development
50
+ ```bash
51
+ uv sync
52
+ uv run pytest
53
+ uv run ruff check
54
+ uv run ruff format
55
+ uv run mypy .
56
+ ```
57
+
58
+ Run commands from workspace root.
@@ -0,0 +1,18 @@
1
+ Metadata-Version: 2.5
2
+ Name: focus-feed
3
+ Version: 1.0
4
+ Summary: A local RSS/Atom feed generator for audio, HTML and PDF files
5
+ Requires-Python: >=3.14
6
+ Requires-Dist: av>=11.0
7
+ Requires-Dist: beautifulsoup4>=4.12.0
8
+ Requires-Dist: click>=8.5.0
9
+ Requires-Dist: feedgen>=1.0.0
10
+ Requires-Dist: markdown>=3.5.0
11
+ Requires-Dist: openai>=2.48.0
12
+ Requires-Dist: pydantic-settings>=2.15.0
13
+ Requires-Dist: pydantic>=2.12.0
14
+ Requires-Dist: pypdf>=4.0.0
15
+ Requires-Dist: python-dotenv>=1.0.0
16
+ Requires-Dist: watchdog>=3.0.0
17
+ Provides-Extra: local-transcription
18
+ Requires-Dist: faster-whisper>=1.0.0; extra == 'local-transcription'
@@ -0,0 +1,286 @@
1
+ # Focus Feed
2
+
3
+ **Transform your education content into mobile-friendly RSS feeds for podcast and article consumption.**
4
+
5
+ ## Project Summary
6
+
7
+ Turn disorganized PDF, HTML, and MP4 audio files into RSS feeds for mobile apps.
8
+
9
+ - Podcast feed - For MP4 audio (Antennapod and other podcast apps)
10
+ - Articles feed - For PDF/HTML documents (any RSS reader)
11
+ - Per-folder feeds - Organize with `feed.json` in subfolders
12
+ - Discovery - Auto-generated `index.html` lists all available feeds
13
+
14
+ All processing uses your configured endpoints. Optional local transcription available.
15
+
16
+ **Documentation:**
17
+ - [Proposal](./artifacts/proposal.md) - Problem and scope
18
+ - [Specifications](./artifacts/specs.md) - Detailed requirements
19
+ - [Design Document](./artifacts/design.md) - Architecture
20
+
21
+ ---
22
+
23
+ ## Quick Start
24
+
25
+ 1. **Configure** via environment variables or `.env` file
26
+
27
+ 2. **Process your content** (targeting specific files or directories):
28
+ ```bash
29
+ uv run focus-feed process ./videos/lecture.mp4
30
+ uv run focus-feed process ./documents/ ./videos/
31
+ ```
32
+
33
+ 3. **Access your feeds** at `{BASE_URL}/index.html`
34
+
35
+ ---
36
+ To process all files, pass your data directory:
37
+ ```bash
38
+ uv run focus-feed process ./data
39
+ ```
40
+
41
+ ---
42
+
43
+ ## Configuration
44
+
45
+ ### Required Environment Variables
46
+
47
+ - `BASE_URL` - Root URL where data directory is served (e.g., `https://example.com`)
48
+ - `LLM_ENDPOINT` - OpenAI-compatible API endpoint for summarization (can be remote or local)
49
+ - `LLM_API_KEY` - API key for LLM endpoint
50
+ - `LLM_MODEL` - Model identifier for summarization
51
+ - `PROMPT_ARTICLE` - Article summarization prompt (PDF/HTML)
52
+ - `PROMPT_PODCAST` - Podcast summarization prompt (MP4)
53
+ - `PROMPT_TITLE` - Title generation prompt
54
+
55
+ ### Optional Environment Variables
56
+
57
+ - `PROMPT_ARTICLE_REFLECTION` - Article summarization with reflection questions (default: `PROMPT_ARTICLE`)
58
+ - `PROMPT_PODCAST_REFLECTION` - Podcast summarization with reflection questions (default: `PROMPT_PODCAST`)
59
+ - `TRANSCRIPTION_ENDPOINT` - Remote transcription API endpoint (enables remote mode)
60
+ - `TRANSCRIPTION_API_KEY` - Remote transcription API key
61
+ - `TRANSCRIPTION_REMOTE_MODEL` - Remote transcription model override
62
+ - `TRANSCRIPTION_MODEL` - Local transcription model (default: `large-v3-turbo`)
63
+ - `TRANSCRIPTION_DEVICE` - Transcription device (cuda/cpu) (default: `cuda`)
64
+ - `TRANSCRIPTION_COMPUTE_TYPE` - Compute type for transcription (default: `float16`)
65
+
66
+ ### Example `.env` File
67
+
68
+ ```bash
69
+ # Required
70
+ BASE_URL=https://example.com
71
+ LLM_ENDPOINT=https://llm.example.com/v1
72
+ LLM_API_KEY=your-api-key
73
+ LLM_MODEL=mistralai/Mistral-7B-Instruct
74
+
75
+ # Prompts
76
+ PROMPT_ARTICLE="Summarize key takeaways from this document concisely in a few paragraphs."
77
+ PROMPT_PODCAST="Summarize key takeaways from this podcast transcript concisely in a few paragraphs."
78
+ PROMPT_TITLE="Generate a concise, descriptive title (5-10 words) for the given content."
79
+
80
+ # Optional
81
+ PROMPT_ARTICLE_REFLECTION="Summarize with reflection questions: ..."
82
+ PROMPT_PODCAST_REFLECTION="Summarize with reflection questions: ..."
83
+
84
+ # Transcription (default: local faster-whisper on GPU)
85
+ TRANSCRIPTION_MODEL=large-v3-turbo
86
+ TRANSCRIPTION_DEVICE=cuda
87
+ TRANSCRIPTION_COMPUTE_TYPE=float16
88
+
89
+ # Remote transcription (optional - overrides local when set)
90
+ # TRANSCRIPTION_ENDPOINT=https://transcription.example.com/v1
91
+ # TRANSCRIPTION_API_KEY=your-transcription-api-key
92
+ ```
93
+
94
+ **Note:** When `TRANSCRIPTION_ENDPOINT` is configured, remote transcription mode is used instead of local faster-whisper.
95
+
96
+ ---
97
+
98
+ ## Processing Modes
99
+
100
+ ### Batch Mode
101
+
102
+ Process all files at once:
103
+ ```bash
104
+ uv run focus-feed process ./data
105
+ ```
106
+
107
+ ### Incremental Mode
108
+
109
+ Watch for new/updated files:
110
+ ```bash
111
+ uv run focus-feed watch ./data
112
+ ```
113
+
114
+ ---
115
+
116
+ ## Granular Processing
117
+
118
+ Target specific steps or files by passing files and directories as positional arguments:
119
+
120
+ - `transcribe` - Transcribe MP4 files to markdown
121
+ - `summarize` - Generate summaries from transcripts or document content
122
+ - `extract` - Extract metadata from PDF/HTML files
123
+ - `title` - Generate titles for files that lack proper titles
124
+ - `feeds` - Generate RSS feeds and index.html
125
+ - `validate` - Validate all metadata files
126
+ - `create-feed` - Create a template `feed.json` in a directory
127
+ - `cleanup` - Remove unknown keys from JSON metadata files
128
+
129
+ ### Common Flags
130
+
131
+ - `--force` - Force re-processing even if complete
132
+ - `--dry-run` - Preview without making changes (transcribe, summarize, extract)
133
+ - `--config-file <path>` - Use custom `.env` file
134
+
135
+ ### Examples
136
+
137
+ ```bash
138
+ # Process a single file
139
+ uv run focus-feed process ./videos/lecture.mp4
140
+
141
+ # Force re-transcription of a specific file
142
+ uv run focus-feed transcribe ./videos/lecture.mp4 --force
143
+
144
+ # Summarize a PDF
145
+ uv run focus-feed summarize ./documents/report.pdf
146
+
147
+ # Process all files in a directory
148
+ uv run focus-feed process ./documents/
149
+
150
+ # Process multiple files and directories at once
151
+ uv run focus-feed process ./videos/lecture.mp4 ./documents/report.pdf ./course1/
152
+
153
+ # See what would be transcribed
154
+ uv run focus-feed transcribe --dry-run
155
+ ```
156
+
157
+ ---
158
+ ## File Organization
159
+
160
+ ### Directory Structure
161
+
162
+ ```
163
+ data/
164
+ ├── index.html # Auto-generated: lists all feeds
165
+ ├── feed.json # REQUIRED for root-level feeds
166
+ ├── podcast.xml # Generated if MP4 files present
167
+ ├── articles.xml # Generated if PDF/HTML files present
168
+ └── Course1/
169
+ ├── feed.json # REQUIRED for per-folder feeds
170
+ ├── podcast.xml # Course1 audio feed
171
+ ├── articles.xml # Course1 articles feed
172
+ ├── lecture.mp4
173
+ ├── lecture.mp4.json # Auto-generated metadata
174
+ └── reflection.md # Optional: reflection questions
175
+ ```
176
+
177
+ ### feed.json Format
178
+
179
+ ```json
180
+ {
181
+ "title": "Course 1: Introduction to AI",
182
+ "description": "All materials for Course 1",
183
+ "language": "en-us",
184
+ "author": "Course Author"
185
+ }
186
+ ```
187
+
188
+ **Required:** `title`
189
+ **Optional:** `description`, `language`, `author`, `link`, `image`
190
+
191
+ ### reflection.md (Optional)
192
+
193
+ Place a `reflection.md` file in any folder to focus summarization on specific questions:
194
+
195
+ ```markdown
196
+ # Reflection Questions
197
+
198
+ 1. What are the key frameworks or mental models presented?
199
+ 2. What analytical tools can help develop critical thinking?
200
+ 3. How does this content relate to systems thinking?
201
+ ```
202
+
203
+ **Note:** Works with both MP4 and PDF/HTML files. Completely optional.
204
+
205
+ ---
206
+
207
+ ## Feed Generation Rules
208
+
209
+ - **Podcast feed** (`podcast.xml`): Generated only if `feed.json` exists AND at least one MP4 file is processed
210
+ - **Articles feed** (`articles.xml`): Generated only if `feed.json` exists AND at least one PDF/HTML file is processed
211
+ - **Root-level feeds**: Require `feed.json` in the data directory root
212
+ - **Per-folder feeds**: Require `feed.json` in the subfolder
213
+ - **Index page** (`index.html`): Auto-generated at root, lists all feeds from all folders
214
+
215
+ ---
216
+ ## Supported File Types
217
+
218
+ - **MP4 (audio)** - Transcribed + summarized, Podcast feed, `description=""`, `summary`=key takeaways, `content`=transcript
219
+ - **PDF** - Metadata + content extracted, Articles feed, `description`=metadata, `content`=text, `summary`=null (unless summarized)
220
+ - **HTML** - Metadata + content extracted, Articles feed, `description`=metadata, `content`=text, `summary`=null (unless summarized)
221
+
222
+ ---
223
+
224
+ ## Requirements
225
+
226
+ ### Software
227
+ - Python 3.14+
228
+ - uv - For dependency management
229
+ - LLM Endpoint - OpenAI-compatible API for summarization (can be remote or local)
230
+ - Optional: `ffprobe` - For audio metadata extraction (falls back to file metadata)
231
+
232
+ ### Optional: Local Transcription
233
+ - Nvidia GPU with CUDA - Required for efficient local transcription
234
+ - CUDA Toolkit - Must be installed
235
+ - Models: faster-whisper models (downloaded automatically, ~2-6 GB)
236
+
237
+ ### Models
238
+ - Summarization: Any OpenAI-compatible model with 128K+ context window recommended
239
+
240
+ ---
241
+
242
+ ## Optional Dependencies
243
+
244
+ ### Local Transcription (GPU-accelerated)
245
+
246
+ ```bash
247
+ uv sync --extra local-transcription
248
+ ```
249
+
250
+ Installs `faster-whisper` and `av` for local GPU-accelerated transcription.
251
+
252
+ ---
253
+
254
+ ## Troubleshooting
255
+
256
+ ### Transcription Issues
257
+
258
+ - **Large files take time**: No time constraints; acceptable for personal use
259
+ - **CUDA out of memory**: Use smaller model (`medium`, `small`, `base`) or `int8` compute type
260
+ - **Model not found**: Verify model name or installation
261
+
262
+ ### Remote Transcription Issues
263
+
264
+ - **Audio exceeds maximum duration**: Configure vLLM server with `VLLM_MAX_AUDIO_DECODE_DURATION_S=3600` (or higher)
265
+ - **Connection errors**: Verify `TRANSCRIPTION_ENDPOINT` and `TRANSCRIPTION_API_KEY`
266
+
267
+ ### Feed Issues
268
+
269
+ - **Missing feeds**: Ensure `feed.json` exists and files are processed (check `.json` metadata)
270
+ - **No root feeds**: Root-level `feed.json` is required
271
+ - **Invalid characters**: Control characters in PDF/HTML are automatically removed
272
+
273
+ ### File Processing Issues
274
+
275
+ - **Special characters**: Warnings are generated; filenames are preserved
276
+ - **Orphaned JSON**: Delete `.json` file to re-process
277
+ - **Force re-processing**: Use `--force` flag
278
+
279
+ ---
280
+
281
+ ## Project Artifacts
282
+
283
+ - [Proposal](./artifacts/proposal.md) - Problem and scope
284
+ - [Specifications](./artifacts/specs.md) - Detailed requirements
285
+ - [Design Document](./artifacts/design.md) - Architecture
286
+ - [Tasks](./artifacts/tasks.md) - Implementation plan
@@ -0,0 +1,109 @@
1
+ # Focus Feed - Design
2
+
3
+ **Runner:** All commands run via `uv run focus-feed <command>` from project root.
4
+
5
+ ## 1. Overview
6
+ Standalone app that processes MP4, PDF, HTML files and generates private RSS feeds for personal consumption. Local-first by default; remote transcription optional.
7
+
8
+ **Goals**
9
+ - Privacy-first, local processing
10
+ - Idempotent, skips completed files
11
+ - Mobile-ready RSS feeds
12
+ - Per-folder feeds via `feed.json`
13
+ - Auto root `index.html` discovery
14
+
15
+ ## 2. Architecture
16
+
17
+ **Components**
18
+ - **Scanner & Watcher (`watchdog`):** Recursively discovers files, debounced change detection, periodic rescan fallback for unreliable events
19
+ - **Producer Pipeline:** Queue manager that decides skip/process/resume based on JSON step maps
20
+ - Transcription Service: MP4 → markdown via faster-whisper local CUDA or remote OpenAI-compatible API
21
+ - Summarization Service: LLM key takeaways from transcript/extracted text, with reflection.md injection
22
+ - Extraction Service: PDF/HTML metadata+plaintext via `pypdf`/`beautifulsoup4`; audio metadata via optional `ffprobe` (falls back to file metadata)
23
+ - **Feed Generator (`feedgen`):** Aggregates eligible metadata per feed.json, builds RSS items with enclosure links, pubDate logic, and GUIDs; writes podcast.xml / articles.xml
24
+ - **Index Generator:** Builds root index.html with links/titles/descriptions from all feed.json configs
25
+ - **CLI (`click`) + Config (`pydantic-settings` + `.env`)**
26
+
27
+ **Data Flow**
28
+ 1. Scanner discovers file recursively
29
+ 2. If JSON exists, check step statuses. Skip if all applicable steps complete; otherwise resume incomplete steps
30
+ 3. Dispatch by type:
31
+ - MP4: load reflection.md if present → transcribe to markdown → store in content → summarize with reflection prompt → extract audio metadata via optional ffprobe → generate_title via LLM → build title dict → apply parent folder prefix
32
+ - PDF/HTML: extract metadata & plaintext → sanitize control chars → store description/content → generate_title via LLM → build title dict → apply folder prefix
33
+ 4. Write/update JSON with step maps, hash, timestamps
34
+ 5. Feed Generator scans for feed.json, collects feed-eligible items from subtree, sorts by folder/filename, converts markdown→HTML with XSS protection, writes podcast.xml / articles.xml
35
+ 6. Index Generator updates root index.html with all feeds
36
+
37
+ ## 3. Interfaces & Contracts
38
+
39
+ **CLI**
40
+ `uv run focus-feed <process|watch|feeds|validate|transcribe|summarize|extract|title|create-feed|cleanup>`
41
+ Flags: `--config-file`, `--force`, `--dry-run`
42
+
43
+ **Environment**
44
+ Required: BASE_URL, LLM_ENDPOINT, LLM_API_KEY, LLM_MODEL, PROMPT_ARTICLE, PROMPT_PODCAST, PROMPT_TITLE
45
+ Optional: PROMPT_ARTICLE_REFLECTION, PROMPT_PODCAST_REFLECTION, TRANSCRIPTION_ENDPOINT, TRANSCRIPTION_API_KEY, TRANSCRIPTION_MODEL, TRANSCRIPTION_DEVICE, TRANSCRIPTION_COMPUTE_TYPE, TRANSCRIPTION_REMOTE_MODEL
46
+
47
+ **JSON Metadata**
48
+ Per file `<name>.json` with title dict `{filename,extracted,generated}`, description, summary, content, filetype, dates ms, size, hash, step maps `transcribe/summarize/extract/generate_title` with status/timing, error.
49
+ Title precedence: generated > extracted > filename. Parent folder name auto-prepended.
50
+
51
+ **Feed Eligibility**
52
+ MP4: transcribe.complete AND summarize.complete
53
+ PDF/HTML: extract.complete AND summarize.complete
54
+
55
+ Note: Nested feeds include all content from their entire subtree; inner feeds are subsets of outer feeds.
56
+
57
+ **feed.json**
58
+ Required for feeds. Fields: title required, description, language, author, link, image optional. Aggregates subtree.
59
+
60
+ **reflection.md**
61
+ Optional per folder. Questions prepended to LLM summarization prompt. Uses reflection prompts if set.
62
+
63
+ ## 4. Technology Stack
64
+
65
+ **Core deps**
66
+ feedgen, watchdog, openai, pydantic-settings, pydantic, click, pypdf, beautifulsoup4, av, markdown>=3.5.0, python-dotenv
67
+
68
+ **Transcription**
69
+ - Local default: faster-whisper with CUDA GPU, model large-v3-turbo float16
70
+ - Remote optional: OpenAI-compatible API via TRANSCRIPTION_ENDPOINT. Nvidia GPU not required for remote mode.
71
+
72
+ **Summarization**
73
+ OpenAI-compatible endpoint. Prompts per filetype. Context 128K+ recommended.
74
+
75
+ **Extraction**
76
+ PDF: pypdf; HTML: beautifulsoup4; Audio metadata: optional ffprobe with file-metadata fallback. Control characters removed for XML safety.
77
+
78
+ ## 5. Key Decisions
79
+ - Title as dict with provenance for fallback and display
80
+ - LLM title generation for MP4s with sparse metadata
81
+ - Auto folder name prefix for context in readers
82
+ - Regenerate feeds/index after each processing
83
+ - Persistent error state in JSON, manual fix or --force
84
+ - Full plaintext extraction for richer feeds
85
+ - Markdown→HTML with XSS protection
86
+ - Per-folder feeds via feed.json only
87
+ - Optional reflection.md for focused summarization
88
+ - Millisecond precision timestamps
89
+ - Conditional generation: feed.json + eligible files required
90
+
91
+ ## 6. Risks & Mitigations
92
+ Transcription quality → use established models
93
+ LLM inconsistency → tested prompts, configurable
94
+ Watcher misses events → periodic rescan
95
+ Special chars → escape output, warn
96
+ Large audio → acceptable for personal use
97
+ GPU memory → float16, smaller models
98
+ API unavailable → clear errors, retry
99
+ Control chars → auto remove
100
+ Missing feeds → document feed.json requirement
101
+
102
+ ## 7. Assumptions & Dependencies
103
+ - `data/` directory exists and is writable at the workspace root, alongside `src/` and `tests/`
104
+ - Existing web server serves data directory
105
+ - Python 3.14+ and uv for dependency management
106
+ - LLM endpoint available (can be remote or local)
107
+ - Nvidia GPU optional for local transcription; required for best performance in local mode, not needed for remote transcription
108
+ - No ffprobe required by default; audio metadata uses optional ffprobe and falls back to file metadata when unavailable
109
+ - Remote transcription: vLLM server requires `VLLM_MAX_AUDIO_DECODE_DURATION_S` configuration for longer audio files
@@ -0,0 +1,37 @@
1
+ # Education Content Feed System - Proposal
2
+
3
+ ## Problem
4
+ Education files (PDF, HTML, MP4) in `data/` are disconnected. Users cannot consume them efficiently across devices via standard apps.
5
+
6
+ ## User Intent
7
+ Make content accessible via private RSS feeds hosted on an existing internal web server:
8
+ - Podcast feed for Antennapod and podcast apps
9
+ - Articles feed for RSS readers
10
+ Per-folder feeds via `feed.json`, with auto-generated root `index.html` for discovery.
11
+
12
+ ## Scope
13
+
14
+ ### In Scope
15
+ - **Input:** PDF, HTML, MP4 in `data/` directory (at workspace root) and its subfolders
16
+ - **Processing:** Batch and incremental watch mode
17
+ - MP4: transcribe to markdown, LLM key takeaway summary
18
+ - PDF/HTML: extract metadata + content; summarization required for feed eligibility
19
+ - Track steps: `transcribe`, `summarize`, `extract`, `generate_title`
20
+ - **Metadata:** `<filename>.json` per file with title dict `{filename,extracted,generated}`, description, summary, content, dates (ms), hash, size, duration/creation_time, step maps `transcribe/summarize/extract/generate_title`
21
+ - MP4: description blank, summary = takeaways, content = transcript
22
+ - PDF/HTML: description = extracted metadata, content = plaintext, summary null unless summarized
23
+ - **Feeds:** `podcast.xml` for MP4, `articles.xml` for PDF/HTML, per-folder only when `feed.json` exists
24
+ - **Features:** `reflection.md` injection, auto folder-name title prefix, markdown→HTML conversion, control character sanitization
25
+
26
+ ### Out of Scope
27
+ - AI agents for metadata gap filling
28
+ - Cloud services by default; remote transcription optional
29
+ - Web server deployment / public distribution
30
+
31
+ ## Success Criteria
32
+ - Content consumable via standard RSS/podcast apps
33
+ - Automatic transcription + summarization for MP4, extraction for PDF/HTML
34
+ - Local-first processing, incremental skips completed files
35
+ - Warnings for orphaned JSONs and special characters
36
+ - Millisecond-precision timestamps, feed eligibility from step statuses
37
+ - Root `index.html` lists all feeds