focus-feed 1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- focus_feed-1.0/.github/workflows/publish-to-pypi.yml +53 -0
- focus_feed-1.0/.gitignore +67 -0
- focus_feed-1.0/.pre-commit-config.yaml +22 -0
- focus_feed-1.0/AGENTS.md +58 -0
- focus_feed-1.0/PKG-INFO +18 -0
- focus_feed-1.0/README.md +286 -0
- focus_feed-1.0/artifacts/design.md +109 -0
- focus_feed-1.0/artifacts/proposal.md +37 -0
- focus_feed-1.0/artifacts/specs.md +102 -0
- focus_feed-1.0/artifacts/tasks.md +1590 -0
- focus_feed-1.0/example.env +29 -0
- focus_feed-1.0/pyproject.toml +136 -0
- focus_feed-1.0/src/focus_feed/__init__.py +3 -0
- focus_feed-1.0/src/focus_feed/cli.py +2390 -0
- focus_feed-1.0/src/focus_feed/config.py +157 -0
- focus_feed-1.0/src/focus_feed/errors.py +392 -0
- focus_feed-1.0/src/focus_feed/feeds.py +1524 -0
- focus_feed-1.0/src/focus_feed/metadata.py +589 -0
- focus_feed-1.0/src/focus_feed/producer.py +1504 -0
- focus_feed-1.0/src/focus_feed/py.typed +0 -0
- focus_feed-1.0/src/focus_feed/scanner.py +595 -0
- focus_feed-1.0/src/focus_feed/services/__init__.py +1 -0
- focus_feed-1.0/src/focus_feed/services/extractors.py +514 -0
- focus_feed-1.0/src/focus_feed/services/remote_transcriber.py +371 -0
- focus_feed-1.0/src/focus_feed/services/summarizer.py +458 -0
- focus_feed-1.0/src/focus_feed/services/transcriber.py +470 -0
- focus_feed-1.0/tests/__init__.py +0 -0
- focus_feed-1.0/tests/conftest.py +76 -0
- focus_feed-1.0/tests/test_cli.py +4716 -0
- focus_feed-1.0/tests/test_config.py +539 -0
- focus_feed-1.0/tests/test_errors.py +631 -0
- focus_feed-1.0/tests/test_extractors.py +1324 -0
- focus_feed-1.0/tests/test_feeds.py +4566 -0
- focus_feed-1.0/tests/test_integration.py +637 -0
- focus_feed-1.0/tests/test_metadata.py +1603 -0
- focus_feed-1.0/tests/test_producer.py +2584 -0
- focus_feed-1.0/tests/test_remote_transcriber.py +336 -0
- focus_feed-1.0/tests/test_scanner.py +1037 -0
- focus_feed-1.0/tests/test_summarizer.py +1088 -0
- focus_feed-1.0/tests/test_transcriber.py +731 -0
- focus_feed-1.0/uv.lock +1288 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
name: Build distributions
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- name: Check out repository
|
|
16
|
+
uses: actions/checkout@v7
|
|
17
|
+
|
|
18
|
+
- name: Set up Python
|
|
19
|
+
uses: actions/setup-python@v7
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.x"
|
|
22
|
+
|
|
23
|
+
- name: Install build frontend
|
|
24
|
+
run: python -m pip install --upgrade build
|
|
25
|
+
|
|
26
|
+
- name: Build source distribution and wheel
|
|
27
|
+
run: python -m build
|
|
28
|
+
|
|
29
|
+
- name: Store distributions
|
|
30
|
+
uses: actions/upload-artifact@v7
|
|
31
|
+
with:
|
|
32
|
+
name: pypi-distributions
|
|
33
|
+
path: dist/
|
|
34
|
+
if-no-files-found: error
|
|
35
|
+
|
|
36
|
+
publish:
|
|
37
|
+
name: Publish distributions to PyPI
|
|
38
|
+
needs: build
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
environment:
|
|
41
|
+
name: pypi
|
|
42
|
+
url: https://pypi.org/p/focus-feed
|
|
43
|
+
permissions:
|
|
44
|
+
id-token: write
|
|
45
|
+
steps:
|
|
46
|
+
- name: Retrieve distributions
|
|
47
|
+
uses: actions/download-artifact@v8
|
|
48
|
+
with:
|
|
49
|
+
name: pypi-distributions
|
|
50
|
+
path: dist/
|
|
51
|
+
|
|
52
|
+
- name: Publish package distributions to PyPI
|
|
53
|
+
uses: pypa/gh-action-pypi-publish@v1.14.3
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
data/
|
|
2
|
+
__pycache__/
|
|
3
|
+
scripts/
|
|
4
|
+
.env
|
|
5
|
+
*.log
|
|
6
|
+
|
|
7
|
+
# Python bytecode and native extensions
|
|
8
|
+
*.py[cod]
|
|
9
|
+
*$py.class
|
|
10
|
+
*.so
|
|
11
|
+
|
|
12
|
+
# Distribution / packaging
|
|
13
|
+
.Python
|
|
14
|
+
build/
|
|
15
|
+
develop-eggs/
|
|
16
|
+
dist/
|
|
17
|
+
downloads/
|
|
18
|
+
eggs/
|
|
19
|
+
.eggs/
|
|
20
|
+
lib/
|
|
21
|
+
lib64/
|
|
22
|
+
parts/
|
|
23
|
+
sdist/
|
|
24
|
+
var/
|
|
25
|
+
wheels/
|
|
26
|
+
*.egg-info/
|
|
27
|
+
.installed.cfg
|
|
28
|
+
*.egg
|
|
29
|
+
pip-wheel-metadata/
|
|
30
|
+
share/python-wheels/
|
|
31
|
+
|
|
32
|
+
# Virtual environments
|
|
33
|
+
.venv/
|
|
34
|
+
venv/
|
|
35
|
+
env/
|
|
36
|
+
ENV/
|
|
37
|
+
|
|
38
|
+
# Test and type-check caches
|
|
39
|
+
.pytest_cache/
|
|
40
|
+
.mypy_cache/
|
|
41
|
+
.ruff_cache/
|
|
42
|
+
*.pyc
|
|
43
|
+
|
|
44
|
+
# Coverage
|
|
45
|
+
.coverage
|
|
46
|
+
.coverage.*
|
|
47
|
+
htmlcov/
|
|
48
|
+
.tox/
|
|
49
|
+
.nox/
|
|
50
|
+
coverage.xml
|
|
51
|
+
*.cover
|
|
52
|
+
|
|
53
|
+
# Jupyter and IDE files
|
|
54
|
+
.ipynb_checkpoints
|
|
55
|
+
.idea/
|
|
56
|
+
.vscode/
|
|
57
|
+
*.swp
|
|
58
|
+
*.swo
|
|
59
|
+
*~
|
|
60
|
+
.cursor/
|
|
61
|
+
|
|
62
|
+
# OS files
|
|
63
|
+
.DS_Store
|
|
64
|
+
Thumbs.db
|
|
65
|
+
|
|
66
|
+
# uv cache
|
|
67
|
+
uv-cache/
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
+
rev: v0.8.0
|
|
4
|
+
hooks:
|
|
5
|
+
- id: ruff
|
|
6
|
+
args: [--fix]
|
|
7
|
+
- id: ruff-format
|
|
8
|
+
|
|
9
|
+
- repo: https://github.com/pre-commit/mirrors-mypy
|
|
10
|
+
rev: v1.13.0
|
|
11
|
+
hooks:
|
|
12
|
+
- id: mypy
|
|
13
|
+
files: ^src/
|
|
14
|
+
additional_dependencies:
|
|
15
|
+
- pydantic-settings
|
|
16
|
+
- pydantic
|
|
17
|
+
- openai
|
|
18
|
+
- feedgen
|
|
19
|
+
- watchdog
|
|
20
|
+
- faster-whisper
|
|
21
|
+
- pypdf
|
|
22
|
+
- beautifulsoup4
|
focus_feed-1.0/AGENTS.md
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# AGENTS.md - Focus Feed
|
|
2
|
+
|
|
3
|
+
Purpose: Transform education content (PDF/HTML/MP4) into RSS feeds.
|
|
4
|
+
|
|
5
|
+
Command: `uv run focus-feed <command>` from the workspace root (directory containing `src/`, `tests/`, and `data/`).
|
|
6
|
+
|
|
7
|
+
## Core
|
|
8
|
+
- MP4: Requires `transcribe` → `summarize` → feed-eligible.
|
|
9
|
+
- PDF/HTML: Requires `extract` → `summarize` → feed-eligible.
|
|
10
|
+
- Feeds: Generated only where `feed.json` exists. Requires feed-eligible files.
|
|
11
|
+
|
|
12
|
+
## Key Commands
|
|
13
|
+
- `process` / `watch`: Process files batch/incrementally.
|
|
14
|
+
- `transcribe`, `summarize`, `extract`, `title`, `feeds`, `validate`, `create-feed`, `cleanup`.
|
|
15
|
+
- Flags: `--force`, `--dry-run`.
|
|
16
|
+
|
|
17
|
+
## Metadata Schema
|
|
18
|
+
Each file gets a `[filename].json` with:
|
|
19
|
+
```json
|
|
20
|
+
{
|
|
21
|
+
"title": {"filename": "", "extracted": "", "generated": ""},
|
|
22
|
+
"description": "", "summary": "", "content": "",
|
|
23
|
+
"filetype": "mp4|pdf|html", "hash": "sha256...",
|
|
24
|
+
"date": 1710000000.123, "publication_date": 1710000000.0,
|
|
25
|
+
"duration": 45.5, "creation_time": "ISO-8601",
|
|
26
|
+
"transcribe": {"status": "complete"},
|
|
27
|
+
"summarize": {"status": "complete"},
|
|
28
|
+
"extract": {"status": "complete"},
|
|
29
|
+
"generate_title": {"status": "complete"},
|
|
30
|
+
"error": ""
|
|
31
|
+
}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Feed Eligibility: MP4 = transcribe & summarize complete. PDF/HTML = extract & summarize complete.
|
|
35
|
+
|
|
36
|
+
Title Rules: `generated` > `extracted` > `filename`. Parent folder name prepended.
|
|
37
|
+
|
|
38
|
+
## Config (Env Vars)
|
|
39
|
+
Required: `BASE_URL`, `LLM_ENDPOINT`, `LLM_API_KEY`, `LLM_MODEL`, `PROMPT_ARTICLE`, `PROMPT_PODCAST`, `PROMPT_TITLE`.
|
|
40
|
+
|
|
41
|
+
Optional: `PROMPT_ARTICLE_REFLECTION`, `PROMPT_PODCAST_REFLECTION`, `TRANSCRIPTION_ENDPOINT` (remote overrides local), `TRANSCRIPTION_API_KEY`, `TRANSCRIPTION_REMOTE_MODEL`, `TRANSCRIPTION_MODEL`, `TRANSCRIPTION_DEVICE`, `TRANSCRIPTION_COMPUTE_TYPE`.
|
|
42
|
+
|
|
43
|
+
## Artifacts
|
|
44
|
+
- `artifacts/proposal.md` - Project proposal and scope.
|
|
45
|
+
- `artifacts/specs.md` - Functional/non-functional specs.
|
|
46
|
+
- `artifacts/design.md` - System architecture and data flow.
|
|
47
|
+
- `artifacts/tasks.md` - Implementation task list.
|
|
48
|
+
|
|
49
|
+
## Development
|
|
50
|
+
```bash
|
|
51
|
+
uv sync
|
|
52
|
+
uv run pytest
|
|
53
|
+
uv run ruff check
|
|
54
|
+
uv run ruff format
|
|
55
|
+
uv run mypy .
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Run commands from workspace root.
|
focus_feed-1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: focus-feed
|
|
3
|
+
Version: 1.0
|
|
4
|
+
Summary: A local RSS/Atom feed generator for audio, HTML and PDF files
|
|
5
|
+
Requires-Python: >=3.14
|
|
6
|
+
Requires-Dist: av>=11.0
|
|
7
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
8
|
+
Requires-Dist: click>=8.5.0
|
|
9
|
+
Requires-Dist: feedgen>=1.0.0
|
|
10
|
+
Requires-Dist: markdown>=3.5.0
|
|
11
|
+
Requires-Dist: openai>=2.48.0
|
|
12
|
+
Requires-Dist: pydantic-settings>=2.15.0
|
|
13
|
+
Requires-Dist: pydantic>=2.12.0
|
|
14
|
+
Requires-Dist: pypdf>=4.0.0
|
|
15
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
16
|
+
Requires-Dist: watchdog>=3.0.0
|
|
17
|
+
Provides-Extra: local-transcription
|
|
18
|
+
Requires-Dist: faster-whisper>=1.0.0; extra == 'local-transcription'
|
focus_feed-1.0/README.md
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
# Focus Feed
|
|
2
|
+
|
|
3
|
+
**Transform your education content into mobile-friendly RSS feeds for podcast and article consumption.**
|
|
4
|
+
|
|
5
|
+
## Project Summary
|
|
6
|
+
|
|
7
|
+
Turn disorganized PDF, HTML, and MP4 audio files into RSS feeds for mobile apps.
|
|
8
|
+
|
|
9
|
+
- Podcast feed - For MP4 audio (Antennapod and other podcast apps)
|
|
10
|
+
- Articles feed - For PDF/HTML documents (any RSS reader)
|
|
11
|
+
- Per-folder feeds - Organize with `feed.json` in subfolders
|
|
12
|
+
- Discovery - Auto-generated `index.html` lists all available feeds
|
|
13
|
+
|
|
14
|
+
All processing uses your configured endpoints. Optional local transcription available.
|
|
15
|
+
|
|
16
|
+
**Documentation:**
|
|
17
|
+
- [Proposal](./artifacts/proposal.md) - Problem and scope
|
|
18
|
+
- [Specifications](./artifacts/specs.md) - Detailed requirements
|
|
19
|
+
- [Design Document](./artifacts/design.md) - Architecture
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## Quick Start
|
|
24
|
+
|
|
25
|
+
1. **Configure** via environment variables or `.env` file
|
|
26
|
+
|
|
27
|
+
2. **Process your content** (targeting specific files or directories):
|
|
28
|
+
```bash
|
|
29
|
+
uv run focus-feed process ./videos/lecture.mp4
|
|
30
|
+
uv run focus-feed process ./documents/ ./videos/
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
3. **Access your feeds** at `{BASE_URL}/index.html`
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
To process all files, pass your data directory:
|
|
37
|
+
```bash
|
|
38
|
+
uv run focus-feed process ./data
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Configuration
|
|
44
|
+
|
|
45
|
+
### Required Environment Variables
|
|
46
|
+
|
|
47
|
+
- `BASE_URL` - Root URL where data directory is served (e.g., `https://example.com`)
|
|
48
|
+
- `LLM_ENDPOINT` - OpenAI-compatible API endpoint for summarization (can be remote or local)
|
|
49
|
+
- `LLM_API_KEY` - API key for LLM endpoint
|
|
50
|
+
- `LLM_MODEL` - Model identifier for summarization
|
|
51
|
+
- `PROMPT_ARTICLE` - Article summarization prompt (PDF/HTML)
|
|
52
|
+
- `PROMPT_PODCAST` - Podcast summarization prompt (MP4)
|
|
53
|
+
- `PROMPT_TITLE` - Title generation prompt
|
|
54
|
+
|
|
55
|
+
### Optional Environment Variables
|
|
56
|
+
|
|
57
|
+
- `PROMPT_ARTICLE_REFLECTION` - Article summarization with reflection questions (default: `PROMPT_ARTICLE`)
|
|
58
|
+
- `PROMPT_PODCAST_REFLECTION` - Podcast summarization with reflection questions (default: `PROMPT_PODCAST`)
|
|
59
|
+
- `TRANSCRIPTION_ENDPOINT` - Remote transcription API endpoint (enables remote mode)
|
|
60
|
+
- `TRANSCRIPTION_API_KEY` - Remote transcription API key
|
|
61
|
+
- `TRANSCRIPTION_REMOTE_MODEL` - Remote transcription model override
|
|
62
|
+
- `TRANSCRIPTION_MODEL` - Local transcription model (default: `large-v3-turbo`)
|
|
63
|
+
- `TRANSCRIPTION_DEVICE` - Transcription device (cuda/cpu) (default: `cuda`)
|
|
64
|
+
- `TRANSCRIPTION_COMPUTE_TYPE` - Compute type for transcription (default: `float16`)
|
|
65
|
+
|
|
66
|
+
### Example `.env` File
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
# Required
|
|
70
|
+
BASE_URL=https://example.com
|
|
71
|
+
LLM_ENDPOINT=https://llm.example.com/v1
|
|
72
|
+
LLM_API_KEY=your-api-key
|
|
73
|
+
LLM_MODEL=mistralai/Mistral-7B-Instruct
|
|
74
|
+
|
|
75
|
+
# Prompts
|
|
76
|
+
PROMPT_ARTICLE="Summarize key takeaways from this document concisely in a few paragraphs."
|
|
77
|
+
PROMPT_PODCAST="Summarize key takeaways from this podcast transcript concisely in a few paragraphs."
|
|
78
|
+
PROMPT_TITLE="Generate a concise, descriptive title (5-10 words) for the given content."
|
|
79
|
+
|
|
80
|
+
# Optional
|
|
81
|
+
PROMPT_ARTICLE_REFLECTION="Summarize with reflection questions: ..."
|
|
82
|
+
PROMPT_PODCAST_REFLECTION="Summarize with reflection questions: ..."
|
|
83
|
+
|
|
84
|
+
# Transcription (default: local faster-whisper on GPU)
|
|
85
|
+
TRANSCRIPTION_MODEL=large-v3-turbo
|
|
86
|
+
TRANSCRIPTION_DEVICE=cuda
|
|
87
|
+
TRANSCRIPTION_COMPUTE_TYPE=float16
|
|
88
|
+
|
|
89
|
+
# Remote transcription (optional - overrides local when set)
|
|
90
|
+
# TRANSCRIPTION_ENDPOINT=https://transcription.example.com/v1
|
|
91
|
+
# TRANSCRIPTION_API_KEY=your-transcription-api-key
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
**Note:** When `TRANSCRIPTION_ENDPOINT` is configured, remote transcription mode is used instead of local faster-whisper.
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## Processing Modes
|
|
99
|
+
|
|
100
|
+
### Batch Mode
|
|
101
|
+
|
|
102
|
+
Process all files at once:
|
|
103
|
+
```bash
|
|
104
|
+
uv run focus-feed process ./data
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### Incremental Mode
|
|
108
|
+
|
|
109
|
+
Watch for new/updated files:
|
|
110
|
+
```bash
|
|
111
|
+
uv run focus-feed watch ./data
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Granular Processing
|
|
117
|
+
|
|
118
|
+
Target specific steps or files by passing files and directories as positional arguments:
|
|
119
|
+
|
|
120
|
+
- `transcribe` - Transcribe MP4 files to markdown
|
|
121
|
+
- `summarize` - Generate summaries from transcripts or document content
|
|
122
|
+
- `extract` - Extract metadata from PDF/HTML files
|
|
123
|
+
- `title` - Generate titles for files that lack proper titles
|
|
124
|
+
- `feeds` - Generate RSS feeds and index.html
|
|
125
|
+
- `validate` - Validate all metadata files
|
|
126
|
+
- `create-feed` - Create a template `feed.json` in a directory
|
|
127
|
+
- `cleanup` - Remove unknown keys from JSON metadata files
|
|
128
|
+
|
|
129
|
+
### Common Flags
|
|
130
|
+
|
|
131
|
+
- `--force` - Force re-processing even if complete
|
|
132
|
+
- `--dry-run` - Preview without making changes (transcribe, summarize, extract)
|
|
133
|
+
- `--config-file <path>` - Use custom `.env` file
|
|
134
|
+
|
|
135
|
+
### Examples
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
# Process a single file
|
|
139
|
+
uv run focus-feed process ./videos/lecture.mp4
|
|
140
|
+
|
|
141
|
+
# Force re-transcription of a specific file
|
|
142
|
+
uv run focus-feed transcribe ./videos/lecture.mp4 --force
|
|
143
|
+
|
|
144
|
+
# Summarize a PDF
|
|
145
|
+
uv run focus-feed summarize ./documents/report.pdf
|
|
146
|
+
|
|
147
|
+
# Process all files in a directory
|
|
148
|
+
uv run focus-feed process ./documents/
|
|
149
|
+
|
|
150
|
+
# Process multiple files and directories at once
|
|
151
|
+
uv run focus-feed process ./videos/lecture.mp4 ./documents/report.pdf ./course1/
|
|
152
|
+
|
|
153
|
+
# See what would be transcribed
|
|
154
|
+
uv run focus-feed transcribe --dry-run
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
## File Organization
|
|
159
|
+
|
|
160
|
+
### Directory Structure
|
|
161
|
+
|
|
162
|
+
```
|
|
163
|
+
data/
|
|
164
|
+
├── index.html # Auto-generated: lists all feeds
|
|
165
|
+
├── feed.json # REQUIRED for root-level feeds
|
|
166
|
+
├── podcast.xml # Generated if MP4 files present
|
|
167
|
+
├── articles.xml # Generated if PDF/HTML files present
|
|
168
|
+
└── Course1/
|
|
169
|
+
├── feed.json # REQUIRED for per-folder feeds
|
|
170
|
+
├── podcast.xml # Course1 audio feed
|
|
171
|
+
├── articles.xml # Course1 articles feed
|
|
172
|
+
├── lecture.mp4
|
|
173
|
+
├── lecture.mp4.json # Auto-generated metadata
|
|
174
|
+
└── reflection.md # Optional: reflection questions
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### feed.json Format
|
|
178
|
+
|
|
179
|
+
```json
|
|
180
|
+
{
|
|
181
|
+
"title": "Course 1: Introduction to AI",
|
|
182
|
+
"description": "All materials for Course 1",
|
|
183
|
+
"language": "en-us",
|
|
184
|
+
"author": "Course Author"
|
|
185
|
+
}
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
**Required:** `title`
|
|
189
|
+
**Optional:** `description`, `language`, `author`, `link`, `image`
|
|
190
|
+
|
|
191
|
+
### reflection.md (Optional)
|
|
192
|
+
|
|
193
|
+
Place a `reflection.md` file in any folder to focus summarization on specific questions:
|
|
194
|
+
|
|
195
|
+
```markdown
|
|
196
|
+
# Reflection Questions
|
|
197
|
+
|
|
198
|
+
1. What are the key frameworks or mental models presented?
|
|
199
|
+
2. What analytical tools can help develop critical thinking?
|
|
200
|
+
3. How does this content relate to systems thinking?
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
**Note:** Works with both MP4 and PDF/HTML files. Completely optional.
|
|
204
|
+
|
|
205
|
+
---
|
|
206
|
+
|
|
207
|
+
## Feed Generation Rules
|
|
208
|
+
|
|
209
|
+
- **Podcast feed** (`podcast.xml`): Generated only if `feed.json` exists AND at least one MP4 file is processed
|
|
210
|
+
- **Articles feed** (`articles.xml`): Generated only if `feed.json` exists AND at least one PDF/HTML file is processed
|
|
211
|
+
- **Root-level feeds**: Require `feed.json` in the data directory root
|
|
212
|
+
- **Per-folder feeds**: Require `feed.json` in the subfolder
|
|
213
|
+
- **Index page** (`index.html`): Auto-generated at root, lists all feeds from all folders
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
## Supported File Types
|
|
217
|
+
|
|
218
|
+
- **MP4 (audio)** - Transcribed + summarized, Podcast feed, `description=""`, `summary`=key takeaways, `content`=transcript
|
|
219
|
+
- **PDF** - Metadata + content extracted, Articles feed, `description`=metadata, `content`=text, `summary`=null (unless summarized)
|
|
220
|
+
- **HTML** - Metadata + content extracted, Articles feed, `description`=metadata, `content`=text, `summary`=null (unless summarized)
|
|
221
|
+
|
|
222
|
+
---
|
|
223
|
+
|
|
224
|
+
## Requirements
|
|
225
|
+
|
|
226
|
+
### Software
|
|
227
|
+
- Python 3.14+
|
|
228
|
+
- uv - For dependency management
|
|
229
|
+
- LLM Endpoint - OpenAI-compatible API for summarization (can be remote or local)
|
|
230
|
+
- Optional: `ffprobe` - For audio metadata extraction (falls back to file metadata)
|
|
231
|
+
|
|
232
|
+
### Optional: Local Transcription
|
|
233
|
+
- Nvidia GPU with CUDA - Required for efficient local transcription
|
|
234
|
+
- CUDA Toolkit - Must be installed
|
|
235
|
+
- Models: faster-whisper models (downloaded automatically, ~2-6 GB)
|
|
236
|
+
|
|
237
|
+
### Models
|
|
238
|
+
- Summarization: Any OpenAI-compatible model with 128K+ context window recommended
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## Optional Dependencies
|
|
243
|
+
|
|
244
|
+
### Local Transcription (GPU-accelerated)
|
|
245
|
+
|
|
246
|
+
```bash
|
|
247
|
+
uv sync --extra local-transcription
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Installs `faster-whisper` and `av` for local GPU-accelerated transcription.
|
|
251
|
+
|
|
252
|
+
---
|
|
253
|
+
|
|
254
|
+
## Troubleshooting
|
|
255
|
+
|
|
256
|
+
### Transcription Issues
|
|
257
|
+
|
|
258
|
+
- **Large files take time**: No time constraints; acceptable for personal use
|
|
259
|
+
- **CUDA out of memory**: Use smaller model (`medium`, `small`, `base`) or `int8` compute type
|
|
260
|
+
- **Model not found**: Verify model name or installation
|
|
261
|
+
|
|
262
|
+
### Remote Transcription Issues
|
|
263
|
+
|
|
264
|
+
- **Audio exceeds maximum duration**: Configure vLLM server with `VLLM_MAX_AUDIO_DECODE_DURATION_S=3600` (or higher)
|
|
265
|
+
- **Connection errors**: Verify `TRANSCRIPTION_ENDPOINT` and `TRANSCRIPTION_API_KEY`
|
|
266
|
+
|
|
267
|
+
### Feed Issues
|
|
268
|
+
|
|
269
|
+
- **Missing feeds**: Ensure `feed.json` exists and files are processed (check `.json` metadata)
|
|
270
|
+
- **No root feeds**: Root-level `feed.json` is required
|
|
271
|
+
- **Invalid characters**: Control characters in PDF/HTML are automatically removed
|
|
272
|
+
|
|
273
|
+
### File Processing Issues
|
|
274
|
+
|
|
275
|
+
- **Special characters**: Warnings are generated; filenames are preserved
|
|
276
|
+
- **Orphaned JSON**: Delete `.json` file to re-process
|
|
277
|
+
- **Force re-processing**: Use `--force` flag
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
281
|
+
## Project Artifacts
|
|
282
|
+
|
|
283
|
+
- [Proposal](./artifacts/proposal.md) - Problem and scope
|
|
284
|
+
- [Specifications](./artifacts/specs.md) - Detailed requirements
|
|
285
|
+
- [Design Document](./artifacts/design.md) - Architecture
|
|
286
|
+
- [Tasks](./artifacts/tasks.md) - Implementation plan
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# Focus Feed - Design
|
|
2
|
+
|
|
3
|
+
**Runner:** All commands run via `uv run focus-feed <command>` from project root.
|
|
4
|
+
|
|
5
|
+
## 1. Overview
|
|
6
|
+
Standalone app that processes MP4, PDF, HTML files and generates private RSS feeds for personal consumption. Local-first by default; remote transcription optional.
|
|
7
|
+
|
|
8
|
+
**Goals**
|
|
9
|
+
- Privacy-first, local processing
|
|
10
|
+
- Idempotent, skips completed files
|
|
11
|
+
- Mobile-ready RSS feeds
|
|
12
|
+
- Per-folder feeds via `feed.json`
|
|
13
|
+
- Auto root `index.html` discovery
|
|
14
|
+
|
|
15
|
+
## 2. Architecture
|
|
16
|
+
|
|
17
|
+
**Components**
|
|
18
|
+
- **Scanner & Watcher (`watchdog`):** Recursively discovers files, debounced change detection, periodic rescan fallback for unreliable events
|
|
19
|
+
- **Producer Pipeline:** Queue manager that decides skip/process/resume based on JSON step maps
|
|
20
|
+
- Transcription Service: MP4 → markdown via faster-whisper local CUDA or remote OpenAI-compatible API
|
|
21
|
+
- Summarization Service: LLM key takeaways from transcript/extracted text, with reflection.md injection
|
|
22
|
+
- Extraction Service: PDF/HTML metadata+plaintext via `pypdf`/`beautifulsoup4`; audio metadata via optional `ffprobe` (falls back to file metadata)
|
|
23
|
+
- **Feed Generator (`feedgen`):** Aggregates eligible metadata per feed.json, builds RSS items with enclosure links, pubDate logic, and GUIDs; writes podcast.xml / articles.xml
|
|
24
|
+
- **Index Generator:** Builds root index.html with links/titles/descriptions from all feed.json configs
|
|
25
|
+
- **CLI (`click`) + Config (`pydantic-settings` + `.env`)**
|
|
26
|
+
|
|
27
|
+
**Data Flow**
|
|
28
|
+
1. Scanner discovers file recursively
|
|
29
|
+
2. If JSON exists, check step statuses. Skip if all applicable steps complete; otherwise resume incomplete steps
|
|
30
|
+
3. Dispatch by type:
|
|
31
|
+
- MP4: load reflection.md if present → transcribe to markdown → store in content → summarize with reflection prompt → extract audio metadata via optional ffprobe → generate_title via LLM → build title dict → apply parent folder prefix
|
|
32
|
+
- PDF/HTML: extract metadata & plaintext → sanitize control chars → store description/content → generate_title via LLM → build title dict → apply folder prefix
|
|
33
|
+
4. Write/update JSON with step maps, hash, timestamps
|
|
34
|
+
5. Feed Generator scans for feed.json, collects feed-eligible items from subtree, sorts by folder/filename, converts markdown→HTML with XSS protection, writes podcast.xml / articles.xml
|
|
35
|
+
6. Index Generator updates root index.html with all feeds
|
|
36
|
+
|
|
37
|
+
## 3. Interfaces & Contracts
|
|
38
|
+
|
|
39
|
+
**CLI**
|
|
40
|
+
`uv run focus-feed <process|watch|feeds|validate|transcribe|summarize|extract|title|create-feed|cleanup>`
|
|
41
|
+
Flags: `--config-file`, `--force`, `--dry-run`
|
|
42
|
+
|
|
43
|
+
**Environment**
|
|
44
|
+
Required: BASE_URL, LLM_ENDPOINT, LLM_API_KEY, LLM_MODEL, PROMPT_ARTICLE, PROMPT_PODCAST, PROMPT_TITLE
|
|
45
|
+
Optional: PROMPT_ARTICLE_REFLECTION, PROMPT_PODCAST_REFLECTION, TRANSCRIPTION_ENDPOINT, TRANSCRIPTION_API_KEY, TRANSCRIPTION_MODEL, TRANSCRIPTION_DEVICE, TRANSCRIPTION_COMPUTE_TYPE, TRANSCRIPTION_REMOTE_MODEL
|
|
46
|
+
|
|
47
|
+
**JSON Metadata**
|
|
48
|
+
Per file `<name>.json` with title dict `{filename,extracted,generated}`, description, summary, content, filetype, dates ms, size, hash, step maps `transcribe/summarize/extract/generate_title` with status/timing, error.
|
|
49
|
+
Title precedence: generated > extracted > filename. Parent folder name auto-prepended.
|
|
50
|
+
|
|
51
|
+
**Feed Eligibility**
|
|
52
|
+
MP4: transcribe.complete AND summarize.complete
|
|
53
|
+
PDF/HTML: extract.complete AND summarize.complete
|
|
54
|
+
|
|
55
|
+
Note: Nested feeds include all content from their entire subtree; inner feeds are subsets of outer feeds.
|
|
56
|
+
|
|
57
|
+
**feed.json**
|
|
58
|
+
Required for feeds. Fields: title required, description, language, author, link, image optional. Aggregates subtree.
|
|
59
|
+
|
|
60
|
+
**reflection.md**
|
|
61
|
+
Optional per folder. Questions prepended to LLM summarization prompt. Uses reflection prompts if set.
|
|
62
|
+
|
|
63
|
+
## 4. Technology Stack
|
|
64
|
+
|
|
65
|
+
**Core deps**
|
|
66
|
+
feedgen, watchdog, openai, pydantic-settings, pydantic, click, pypdf, beautifulsoup4, av, markdown>=3.5.0, python-dotenv
|
|
67
|
+
|
|
68
|
+
**Transcription**
|
|
69
|
+
- Local default: faster-whisper with CUDA GPU, model large-v3-turbo float16
|
|
70
|
+
- Remote optional: OpenAI-compatible API via TRANSCRIPTION_ENDPOINT. Nvidia GPU not required for remote mode.
|
|
71
|
+
|
|
72
|
+
**Summarization**
|
|
73
|
+
OpenAI-compatible endpoint. Prompts per filetype. Context 128K+ recommended.
|
|
74
|
+
|
|
75
|
+
**Extraction**
|
|
76
|
+
PDF: pypdf; HTML: beautifulsoup4; Audio metadata: optional ffprobe with file-metadata fallback. Control characters removed for XML safety.
|
|
77
|
+
|
|
78
|
+
## 5. Key Decisions
|
|
79
|
+
- Title as dict with provenance for fallback and display
|
|
80
|
+
- LLM title generation for MP4s with sparse metadata
|
|
81
|
+
- Auto folder name prefix for context in readers
|
|
82
|
+
- Regenerate feeds/index after each processing
|
|
83
|
+
- Persistent error state in JSON, manual fix or --force
|
|
84
|
+
- Full plaintext extraction for richer feeds
|
|
85
|
+
- Markdown→HTML with XSS protection
|
|
86
|
+
- Per-folder feeds via feed.json only
|
|
87
|
+
- Optional reflection.md for focused summarization
|
|
88
|
+
- Millisecond precision timestamps
|
|
89
|
+
- Conditional generation: feed.json + eligible files required
|
|
90
|
+
|
|
91
|
+
## 6. Risks & Mitigations
|
|
92
|
+
Transcription quality → use established models
|
|
93
|
+
LLM inconsistency → tested prompts, configurable
|
|
94
|
+
Watcher misses events → periodic rescan
|
|
95
|
+
Special chars → escape output, warn
|
|
96
|
+
Large audio → acceptable for personal use
|
|
97
|
+
GPU memory → float16, smaller models
|
|
98
|
+
API unavailable → clear errors, retry
|
|
99
|
+
Control chars → auto remove
|
|
100
|
+
Missing feeds → document feed.json requirement
|
|
101
|
+
|
|
102
|
+
## 7. Assumptions & Dependencies
|
|
103
|
+
- `data/` directory exists and is writable at the workspace root, alongside `src/` and `tests/`
|
|
104
|
+
- Existing web server serves data directory
|
|
105
|
+
- Python 3.14+ and uv for dependency management
|
|
106
|
+
- LLM endpoint available (can be remote or local)
|
|
107
|
+
- Nvidia GPU optional for local transcription; required for best performance in local mode, not needed for remote transcription
|
|
108
|
+
- No ffprobe required by default; audio metadata uses optional ffprobe and falls back to file metadata when unavailable
|
|
109
|
+
- Remote transcription: vLLM server requires `VLLM_MAX_AUDIO_DECODE_DURATION_S` configuration for longer audio files
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Education Content Feed System - Proposal
|
|
2
|
+
|
|
3
|
+
## Problem
|
|
4
|
+
Education files (PDF, HTML, MP4) in `data/` are disconnected. Users cannot consume them efficiently across devices via standard apps.
|
|
5
|
+
|
|
6
|
+
## User Intent
|
|
7
|
+
Make content accessible via private RSS feeds hosted on an existing internal web server:
|
|
8
|
+
- Podcast feed for Antennapod and podcast apps
|
|
9
|
+
- Articles feed for RSS readers
|
|
10
|
+
Per-folder feeds via `feed.json`, with auto-generated root `index.html` for discovery.
|
|
11
|
+
|
|
12
|
+
## Scope
|
|
13
|
+
|
|
14
|
+
### In Scope
|
|
15
|
+
- **Input:** PDF, HTML, MP4 in `data/` directory (at workspace root) and its subfolders
|
|
16
|
+
- **Processing:** Batch and incremental watch mode
|
|
17
|
+
- MP4: transcribe to markdown, LLM key takeaway summary
|
|
18
|
+
- PDF/HTML: extract metadata + content; summarization required for feed eligibility
|
|
19
|
+
- Track steps: `transcribe`, `summarize`, `extract`, `generate_title`
|
|
20
|
+
- **Metadata:** `<filename>.json` per file with title dict `{filename,extracted,generated}`, description, summary, content, dates (ms), hash, size, duration/creation_time, step maps `transcribe/summarize/extract/generate_title`
|
|
21
|
+
- MP4: description blank, summary = takeaways, content = transcript
|
|
22
|
+
- PDF/HTML: description = extracted metadata, content = plaintext, summary null unless summarized
|
|
23
|
+
- **Feeds:** `podcast.xml` for MP4, `articles.xml` for PDF/HTML, per-folder only when `feed.json` exists
|
|
24
|
+
- **Features:** `reflection.md` injection, auto folder-name title prefix, markdown→HTML conversion, control character sanitization
|
|
25
|
+
|
|
26
|
+
### Out of Scope
|
|
27
|
+
- AI agents for metadata gap filling
|
|
28
|
+
- Cloud services by default; remote transcription optional
|
|
29
|
+
- Web server deployment / public distribution
|
|
30
|
+
|
|
31
|
+
## Success Criteria
|
|
32
|
+
- Content consumable via standard RSS/podcast apps
|
|
33
|
+
- Automatic transcription + summarization for MP4, extraction for PDF/HTML
|
|
34
|
+
- Local-first processing, incremental skips completed files
|
|
35
|
+
- Warnings for orphaned JSONs and special characters
|
|
36
|
+
- Millisecond-precision timestamps, feed eligibility from step statuses
|
|
37
|
+
- Root `index.html` lists all feeds
|