content-core 2.1.0__tar.gz → 2.2.1__tar.gz

{content_core-2.1.0 → content_core-2.2.1}/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "content-core",
-  "version": "2.1.0",
+  "version": "2.2.1",
   "description": "Extract text content from URLs, documents, YouTube videos, Reddit posts, and audio/video files — via CLI (uvx) or MCP.",
   "author": {
     "name": "Luis Novo",

{content_core-2.1.0 → content_core-2.2.1}/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "content-core",
-  "version": "2.1.0",
+  "version": "2.2.1",
   "description": "Extract text content from URLs, documents, YouTube videos, Reddit posts, and audio/video files — via CLI (uvx) or MCP.",
   "skills": "./skills/"
 }

{content_core-2.1.0 → content_core-2.2.1}/.github/workflows/claude-code-review.yml

@@ -12,11 +12,11 @@ on:
 
 jobs:
   claude-review:
-    # Optional: Filter by PR author
-    # if: |
-    #   github.event.pull_request.user.login == 'external-contributor' ||
-    #   github.event.pull_request.user.login == 'new-developer' ||
-    #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
+    # Fork PRs get no repository secrets, so the review could only fail there.
+    # (pull_request_target would expose the secrets to untrusted code.)
+    # To also filter by PR author, extend this expression with `&&` -- a job
+    # takes a single `if:`.
+    if: github.event.pull_request.head.repo.full_name == github.repository
 
     runs-on: ubuntu-latest
     permissions:

{content_core-2.1.0 → content_core-2.2.1}/AGENTS.md

@@ -92,6 +92,7 @@ src/content_core/
 │   │   ├── docx.py          # python-docx
 │   │   ├── pptx.py          # python-pptx
 │   │   ├── xlsx.py          # openpyxl
+│   │   ├── odf.py           # ODT/ODS/ODP via odfpy
 │   │   └── docling.py       # Optional Docling integration
 │   └── media/               # Audio/video processing
 │       ├── __init__.py      # Video→audio pipeline
@@ -129,7 +130,7 @@ from content_core import ContentCoreConfig
 config = ContentCoreConfig(url_engine="firecrawl", audio_concurrency=5)
 ```
 
-Key settings: `url_engine`, `document_engine`, `audio_provider`, `audio_model`, `firecrawl_api_url`, `youtube_languages`, `llm_provider`, `llm_model`, `docling_ocr`, `docling_formulas`, `docling_vision`
+Key settings: `url_engine`, `document_engine`, `audio_provider`, `audio_model`, `firecrawl_api_url`, `youtube_languages`, `youtube_cookies_file`, `youtube_proxy`, `llm_provider`, `llm_model`, `docling_ocr`, `docling_formulas`, `docling_vision`
 
 Docling enrichment flags (`docling_ocr`, `docling_formulas`, `docling_vision`) control OCR, formula extraction, and image/chart processing when `document_engine="docling"`. These are also exposed as CLI flags (`--formulas`, `--pictures`, `--no-ocr`) and MCP parameters.
 
@@ -166,7 +167,7 @@ When you change a specific processor or module, run only the relevant tests for
 | `processors/url/youtube.py` | `uv run pytest -k "youtube"` |
 | `processors/url/reddit.py` | `uv run pytest -k "reddit"` |
 | `processors/document/pdf.py` | `uv run pytest -k "pdf"` |
-| `processors/document/docx.py` or `pptx.py` or `xlsx.py` | `uv run pytest -k "office"` |
+| `processors/document/docx.py` or `pptx.py` or `xlsx.py` or `odf.py` | `uv run pytest -k "office"` |
 | `processors/document/docling.py` | `uv run pytest -k "docling"` |
 | `processors/text.py` | `uv run pytest -k "text_processing or html_file"` |
 | `processors/media/audio.py` | `uv run pytest -k "audio or media_pipeline"` |
@@ -191,7 +192,7 @@ tests/
 │   ├── test_pdf_extraction.py     # PDF text cleaning, extraction with mocked pdfplumber
 │   ├── test_pdf_helpers.py        # Formula detection, table conversion helpers
 │   ├── test_epub_extraction.py    # EPUB extraction with mocked fast-ebook
-│   ├── test_office_extraction.py  # DOCX/PPTX/XLSX routing and extraction
+│   ├── test_office_extraction.py  # DOCX/PPTX/XLSX/ODT/ODS/ODP routing and extraction
 │   ├── test_docling_extraction.py # Docling output formats with mocked converter
 │   ├── test_text_processing.py    # HTML detection, markdown conversion, <title>/<head> handling
 │   ├── test_html_file_extraction.py # Local .html file → markdown via text processor
@@ -206,7 +207,7 @@ tests/
 │   └── test_file_detector*.py     # MIME detection, performance, edge cases
 │
 ├── integration/       # Local files, no network (~22 tests)
-│   ├── test_extraction.py   # Real file extraction (PDF, DOCX, PPTX, XLSX, EPUB, HTML, etc.)
+│   ├── test_extraction.py   # Real file extraction (PDF, DOCX, PPTX, XLSX, ODT/ODS/ODP, EPUB, HTML, etc.)
 │   └── test_cli_v2.py       # CLI subcommands via CliRunner with real extraction
 │
 └── e2e/

{content_core-2.1.0 → content_core-2.2.1}/CHANGELOG.md

@@ -7,6 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.2.1] - 2026-11-03
+
+### Security
+- URL fetches made directly by content-core (the `simple` engine, the MIME-type probe, remote file downloads and Reddit posts) refuse link-local and cloud-metadata addresses, including on every redirect hop and for hostnames that resolve to them. Private and loopback addresses are still allowed. A refused URL raises `InvalidInputError` (`BlockedAddressError`).
+
+## [2.2.0] - 2026-10-03
+
+### Added
+- OpenDocument extraction for `.odt`, `.ods` and `.odp` through the office route, using `odfpy` (new core dependency). Text documents become markdown paragraphs, headings, lists and tables; spreadsheets become one markdown table per sheet; presentations become one block per slide. Detection reads the ZIP's `mimetype` member, so a wrong or missing extension still works (#43).
+- YouTube cookie and proxy support for the transcript path (#46). `youtube_cookies_file` (`CCORE_YOUTUBE_COOKIES_FILE`) loads a Netscape `cookies.txt` into the `youtube-transcript-api` session, unblocking IP-flagged networks; a missing or unreadable file raises `ConfigurationError`. `youtube_proxy` (`CCORE_YOUTUBE_PROXY`) routes both `youtube-transcript-api` and the `pytubefix` fallback through a proxy URL — use a residential proxy, as datacenter proxies are blocked. With neither set, behavior is unchanged. Cookie values are never logged.
+- `audio_segment_minutes` setting (`CCORE_AUDIO_SEGMENT_MINUTES`) controls the length of the segments long audio is split into before transcription. The default stays 10 minutes; `0` sends the file whole, for self-hosted STT endpoints without an upload size limit (#94).
+
+### Changed
+- **URL extraction failures now raise instead of returning empty output** (#60). `content=""` now only means the source was genuinely empty. A named `url_engine` that fails raises `NetworkError` (connection/timeout/DNS), `NotFoundError` (HTTP 404/410 from the page, `simple` engine), `ExternalServiceError` (engine API or HTTP errors) or `ConfigurationError` (`crawl4ai` not installed). `auto` still falls through firecrawl → jina → crawl4ai → simple and raises the last error only when every engine failed. A malformed URL raises `InvalidInputError`. The `simple` engine no longer returns `"Failed to extract content: ..."` or `"No content found"` as page content. Remote-file downloads raise the same typed errors, and a download that cannot be saved locally raises `FileOperationError`.
+- File extraction failures raise typed errors (#60): corrupted PDF, EPUB or unreadable text files raise `FileOperationError` (chained to the original cause) instead of bare `Exception`/`RuntimeError`; a missing file still raises `FileNotFoundError`. ffmpeg/ffprobe failures in audio and video processing raise `FileOperationError`: video extraction no longer returns empty content when ffprobe or ffmpeg fails. Speech-to-text provider failures (after retries) raise `ExternalServiceError`, and an STT model that cannot be created raises `ConfigurationError`.
+- Retry classification now treats `ConfigurationError`, `UnsupportedTypeException`, `InvalidInputError` and `FileOperationError` as permanent and `NetworkError` as transient (#60).
+- YouTube extraction raises instead of returning empty content when no transcript can be fetched (#60): `NoTranscriptFound` when neither youtube-transcript-api nor pytubefix finds a transcript, `ExternalServiceError` (chained to the provider error) when YouTube blocks or fails the request on both paths, e.g. `IpBlocked`, and `InvalidInputError` for a YouTube URL without a video ID. Provider errors from `get_best_transcript` and `extract_transcript_pytubefix` now propagate after retries instead of being swallowed (`extract_transcript_pytubefix` still returns `(None, None)` for a video without captions); a failure of the first path still falls back to the second.
+- An explicit `audio_provider`/`audio_model` that cannot be created now raises `ConfigurationError` instead of silently transcribing with the default STT model, per "Explicit choice is a requirement" (#99).
+- youtube-transcript-api failures are retried by type: blocked requests (`RequestBlocked`/`IpBlocked`) are retried, so a rotating proxy can get a new IP; other transcript errors that name a state of the video (unavailable, transcripts disabled, age-restricted...) are not (#99).
+
+### Fixed
+- `youtube_proxy` now also applies to the YouTube video title lookup, which previously used only env-var proxies (#99).
+- `requests`, imported directly for the YouTube cookie session, is declared as a dependency instead of arriving transitively (#99).
+- A failed Crawl4AI crawl (navigation timeout, blocked page) now raises `ExternalServiceError` instead of failing with a pydantic `ValidationError` on `content=None`, and `auto` falls through to the next engine. Applies to both the local browser and the Docker API mode (#101).
+
+### Security
+- Raised minimum versions to releases without known vulnerabilities: `aiohttp>=3.14.3`, `pillow>=12.3.0`, `click>=8.3.3`, `pydantic-settings>=2.14.2`, and in the extras `docling>=2.94.0`, `crawl4ai>=0.9.0` and `langchain-core>=1.3.3`. Upgrading content-core upgrades these packages in an existing environment. The development lock was refreshed; it now also exercises `fastmcp` 4 and `mcp` 2, which fresh installs already resolve to (#102).
+
 ## [2.1.0] - 2026-09-06
 
 ### Added

{content_core-2.1.0 → content_core-2.2.1}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.5
 Name: content-core
-Version: 2.1.0
+Version: 2.2.1
 Summary: Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: ai-prompter>=0.4.0
-Requires-Dist: aiohttp>=3.13
+Requires-Dist: aiohttp>=3.14.3
 Requires-Dist: bs4>=0.0.2
-Requires-Dist: click>=8.3.0
+Requires-Dist: click>=8.3.3
 Requires-Dist: esperanto>=2.26.0
 Requires-Dist: fast-ebook>=0.2.0
 Requires-Dist: fastmcp>=3.0.0
@@ -17,28 +17,30 @@ Requires-Dist: jinja2>=3.1.6
 Requires-Dist: langdetect>=1.0.9
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.0
+Requires-Dist: odfpy>=1.4.1
 Requires-Dist: openpyxl>=3.1.5
 Requires-Dist: pandas>=2.2.3
 Requires-Dist: pdfplumber>=0.11.0
-Requires-Dist: pillow>=12.0.0
-Requires-Dist: pydantic-settings>=2.12.0
+Requires-Dist: pillow>=12.3.0
+Requires-Dist: pydantic-settings>=2.14.2
 Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.2.0
 Requires-Dist: python-pptx>=1.0.2
 Requires-Dist: pytubefix>=10.0.0
 Requires-Dist: readability-lxml>=0.8.4.1
+Requires-Dist: requests>=2.31.0
 Requires-Dist: tenacity>=9.0.0
 Requires-Dist: tomli>=2.0; python_version < '3.11'
 Requires-Dist: validators>=0.34.0
 Requires-Dist: youtube-transcript-api>=1.2.0
 Provides-Extra: crawl4ai
-Requires-Dist: crawl4ai>=0.7.0; extra == 'crawl4ai'
+Requires-Dist: crawl4ai>=0.9.0; extra == 'crawl4ai'
 Provides-Extra: docling
-Requires-Dist: docling>=2.86.0; extra == 'docling'
+Requires-Dist: docling>=2.94.0; extra == 'docling'
 Requires-Dist: peft>=0.18.0; extra == 'docling'
 Requires-Dist: transformers<5,>=4.50; extra == 'docling'
 Provides-Extra: langchain
-Requires-Dist: langchain-core>=0.1.0; extra == 'langchain'
+Requires-Dist: langchain-core>=1.3.3; extra == 'langchain'
 Description-Content-Type: text/markdown
 
 # Content Core
@@ -59,7 +61,7 @@ Extract, process, and summarize content from URLs, files, and text through a uni
 | Category | Formats |
 |----------|---------|
 | Web | URLs, HTML pages, YouTube videos, Reddit posts |
-| Documents | PDF, DOCX, PPTX, XLSX, EPUB, HTML, Markdown, plain text |
+| Documents | PDF, DOCX, PPTX, XLSX, ODT, ODS, ODP, EPUB, HTML, Markdown, plain text |
 | Media | MP3, WAV, M4A, FLAC, OGG (audio); MP4, AVI, MOV, MKV (video) |
 
 ## Quick Start
@@ -269,6 +271,7 @@ Content Core uses `ContentCoreConfig` powered by pydantic-settings. Settings are
 | `CCORE_URL_ENGINE` | URL extraction engine (`auto`, `simple`, `firecrawl`, `jina`, `crawl4ai`) | `auto` |
 | `CCORE_DOCUMENT_ENGINE` | Document extraction engine (`auto`, `simple`, `docling`) — `docling` raises `ConfigurationError` if the extra is not installed; `auto` falls back silently | `auto` |
 | `CCORE_AUDIO_CONCURRENCY` | Concurrent audio transcriptions (1-10) | `3` |
+| `CCORE_AUDIO_SEGMENT_MINUTES` | Length of the segments long audio is split into; `0` sends the file whole | `10` |
 | `CRAWL4AI_API_URL` | Crawl4AI Docker API URL (omit for local browser mode) | - |
 | `CRAWL4AI_API_TOKEN` | Bearer token for the Crawl4AI Docker API (required by Crawl4AI >= 0.9.0) | - |
 | `FIRECRAWL_API_URL` | Custom Firecrawl API URL for self-hosted instances or Firecrawl-compatible backends (e.g. fastCRW) | - |
@@ -280,6 +283,8 @@ Content Core uses `ContentCoreConfig` powered by pydantic-settings. Settings are
 | `CCORE_STT_MODEL` | Speech-to-text model | - |
 | `CCORE_STT_TIMEOUT` | Speech-to-text timeout in seconds | - |
 | `CCORE_YOUTUBE_LANGUAGES` | Preferred YouTube transcript languages | - |
+| `CCORE_YOUTUBE_COOKIES_FILE` | Path to a Netscape `cookies.txt` for YouTube transcripts (missing file raises `ConfigurationError`) | - |
+| `CCORE_YOUTUBE_PROXY` | Proxy URL for YouTube transcripts, e.g. `http://user:pass@host:port` | - |
 
 API keys for external services are set via their standard environment variables (e.g., `OPENAI_API_KEY`, `FIRECRAWL_API_KEY`, `JINA_API_KEY`).
 
@@ -287,6 +292,10 @@ API keys for external services are set via their standard environment variables
 
 Content Core reads standard `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY` environment variables automatically. No additional configuration is needed.
 
+### YouTube on Blocked Networks
+
+If YouTube extraction fails with `IpBlocked`/`RequestBlocked` on a flagged IP (cloud hosts, VPNs), either point `CCORE_YOUTUBE_COOKIES_FILE` at a `cookies.txt` exported from a logged-in browser (re-export it when the cookies expire), or set `CCORE_YOUTUBE_PROXY` to a **residential** proxy — datacenter proxies get blocked or CAPTCHA'd. See [docs/usage.md](docs/usage.md#youtube-on-blocked-networks).
+
 ## Optional Dependencies
 
 ```bash

{content_core-2.1.0 → content_core-2.2.1}/README.md

@@ -16,7 +16,7 @@ Extract, process, and summarize content from URLs, files, and text through a uni
 | Category | Formats |
 |----------|---------|
 | Web | URLs, HTML pages, YouTube videos, Reddit posts |
-| Documents | PDF, DOCX, PPTX, XLSX, EPUB, HTML, Markdown, plain text |
+| Documents | PDF, DOCX, PPTX, XLSX, ODT, ODS, ODP, EPUB, HTML, Markdown, plain text |
 | Media | MP3, WAV, M4A, FLAC, OGG (audio); MP4, AVI, MOV, MKV (video) |
 
 ## Quick Start
@@ -226,6 +226,7 @@ Content Core uses `ContentCoreConfig` powered by pydantic-settings. Settings are
 | `CCORE_URL_ENGINE` | URL extraction engine (`auto`, `simple`, `firecrawl`, `jina`, `crawl4ai`) | `auto` |
 | `CCORE_DOCUMENT_ENGINE` | Document extraction engine (`auto`, `simple`, `docling`) — `docling` raises `ConfigurationError` if the extra is not installed; `auto` falls back silently | `auto` |
 | `CCORE_AUDIO_CONCURRENCY` | Concurrent audio transcriptions (1-10) | `3` |
+| `CCORE_AUDIO_SEGMENT_MINUTES` | Length of the segments long audio is split into; `0` sends the file whole | `10` |
 | `CRAWL4AI_API_URL` | Crawl4AI Docker API URL (omit for local browser mode) | - |
 | `CRAWL4AI_API_TOKEN` | Bearer token for the Crawl4AI Docker API (required by Crawl4AI >= 0.9.0) | - |
 | `FIRECRAWL_API_URL` | Custom Firecrawl API URL for self-hosted instances or Firecrawl-compatible backends (e.g. fastCRW) | - |
@@ -237,6 +238,8 @@ Content Core uses `ContentCoreConfig` powered by pydantic-settings. Settings are
 | `CCORE_STT_MODEL` | Speech-to-text model | - |
 | `CCORE_STT_TIMEOUT` | Speech-to-text timeout in seconds | - |
 | `CCORE_YOUTUBE_LANGUAGES` | Preferred YouTube transcript languages | - |
+| `CCORE_YOUTUBE_COOKIES_FILE` | Path to a Netscape `cookies.txt` for YouTube transcripts (missing file raises `ConfigurationError`) | - |
+| `CCORE_YOUTUBE_PROXY` | Proxy URL for YouTube transcripts, e.g. `http://user:pass@host:port` | - |
 
 API keys for external services are set via their standard environment variables (e.g., `OPENAI_API_KEY`, `FIRECRAWL_API_KEY`, `JINA_API_KEY`).
 
@@ -244,6 +247,10 @@ API keys for external services are set via their standard environment variables
 
 Content Core reads standard `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY` environment variables automatically. No additional configuration is needed.
 
+### YouTube on Blocked Networks
+
+If YouTube extraction fails with `IpBlocked`/`RequestBlocked` on a flagged IP (cloud hosts, VPNs), either point `CCORE_YOUTUBE_COOKIES_FILE` at a `cookies.txt` exported from a logged-in browser (re-export it when the cookies expire), or set `CCORE_YOUTUBE_PROXY` to a **residential** proxy — datacenter proxies get blocked or CAPTCHA'd. See [docs/usage.md](docs/usage.md#youtube-on-blocked-networks).
+
 ## Optional Dependencies
 
 ```bash

{content_core-2.1.0 → content_core-2.2.1}/docs/cli.md

@@ -154,6 +154,7 @@ content-core config delete llm_provider
 |-----|-------------|---------|
 | `audio_concurrency` | Parallel transcription limit (1-10) | `3` |
 | `audio_model` | Override STT model | — |
+| `audio_segment_minutes` | Audio segment length in minutes; `0` disables splitting | `10` |
 | `crawl4ai_api_url` | Crawl4AI Docker API URL (omit for local mode) | — |
 | `crawl4ai_api_token` | Bearer token for the Crawl4AI Docker API (required by Crawl4AI >= 0.9.0) | — |
 | `audio_provider` | Override STT provider | — |

{content_core-2.1.0 → content_core-2.2.1}/docs/processors.md

@@ -106,6 +106,14 @@ The `document_engine` setting controls document processing. When set to `auto` (
 - File: `document/xlsx.py`
 - Extracts data from Excel spreadsheets
 
+### OpenDocument (ODT/ODS/ODP)
+
+- File: `document/odf.py`
+- Uses odfpy for LibreOffice/OpenOffice formats
+- ODT: paragraphs, headings (by outline level), lists and tables as markdown
+- ODS: one markdown table per sheet, preceded by `# Sheet: <sheet name>` (as for XLSX)
+- ODP: one block per slide (`# Slide N` followed by its text frames, as for PPTX)
+
 ### Docling (Optional)
 
 - File: `document/docling.py`
@@ -160,6 +168,7 @@ Located in `src/content_core/processors/media/`.
 - Detection methods:
   - Binary signature matching for PDF, images, audio, video, and archives
   - ZIP structure inspection for Office formats (DOCX, XLSX, PPTX) and EPUB
+  - OpenDocument (ODT, ODS, ODP) identified by the ZIP's `mimetype` member
   - Content analysis for text-based formats (HTML, XML, JSON, YAML, CSV, Markdown)
 - Reads only the first 512 bytes for binary signatures and 1024 bytes for text content analysis
 - Works regardless of file extension

{content_core-2.1.0 → content_core-2.2.1}/docs/usage.md

@@ -67,6 +67,9 @@ result = await content_core.extract_content(file_path="slides.pptx")
 # Excel
 result = await content_core.extract_content(file_path="data.xlsx")
 
+# OpenDocument (LibreOffice): .odt, .ods, .odp
+result = await content_core.extract_content(file_path="report.odt")
+
 # Use Docling engine for richer parsing
 from content_core import ContentCoreConfig
 config = ContentCoreConfig(document_engine="docling", docling_output_format="html")
@@ -95,6 +98,55 @@ result = await content_core.extract_content(url="https://www.youtube.com/watch?v
 print(result.content)  # Video transcript
 ```
 
+#### YouTube on blocked networks
+
+On IP-flagged networks (cloud hosts, some ISPs, VPNs) YouTube refuses transcript
+requests even for public videos with captions — the extraction fails with
+`IpBlocked`/`RequestBlocked` from `youtube-transcript-api`. Two settings help;
+both are off by default, and with neither set the behavior is unchanged.
+
+**Browser cookies** (`youtube_cookies_file` / `CCORE_YOUTUBE_COOKIES_FILE`) —
+path to a Netscape-format `cookies.txt` exported from a browser that is logged
+in to YouTube (use a "Get cookies.txt"-style browser extension, or let yt-dlp
+dump its cookie jar: `yt-dlp --cookies-from-browser chrome --cookies cookies.txt
+--skip-download <any YouTube URL>`). This unblocks the transcript path on the
+same IP.
+
+- Cookies are credentials: keep the file private. Content Core logs only the
+  file path (at debug level), never cookie values.
+- Session cookies expire. When extraction starts failing again, re-export the
+  file.
+- A missing, unreadable or malformed file raises `ConfigurationError` — Content
+  Core does not silently fall back to anonymous requests.
+- Cookies apply to the `youtube-transcript-api` path only; the `pytubefix`
+  fallback has no cookies-file support.
+
+**Proxy** (`youtube_proxy` / `CCORE_YOUTUBE_PROXY`) — a proxy URL such as
+`http://user:pass@host:port`, used by every YouTube request: the
+`youtube-transcript-api` transcript, the `pytubefix` fallback and the video
+title lookup. The standard `HTTP_PROXY`/`HTTPS_PROXY` variables keep working
+too; this setting scopes the proxy to YouTube.
+
+Use a **residential** proxy — preferably a rotating one (e.g. Webshare
+"Residential"), so a blocked IP is swapped on the next retry. In testing
+against a blocked network, **datacenter proxies failed** (`IpBlocked`,
+`RequestBlocked` or a Google CAPTCHA), while rotating and sticky residential
+proxies returned the full transcript. See the test matrix in
+[issue #46](https://github.com/lfnovo/content-core/issues/46).
+
+```python
+config = ContentCoreConfig(
+    youtube_cookies_file="~/secrets/youtube-cookies.txt",
+    youtube_proxy="http://user:pass@p.webshare.io:80",
+)
+result = await content_core.extract_content(url="https://youtu.be/dQw4w9WgXcQ", config=config)
+```
+
+```bash
+CCORE_YOUTUBE_COOKIES_FILE=/path/to/cookies.txt
+CCORE_YOUTUBE_PROXY=http://user:pass@host:port
+```
+
 ### Local HTML Files
 
 ```python
@@ -177,6 +229,8 @@ CCORE_STT_PROVIDER=openai
 CCORE_STT_MODEL=whisper-1
 CCORE_STT_TIMEOUT=3600
 CCORE_YOUTUBE_LANGUAGES=en,pt
+CCORE_YOUTUBE_COOKIES_FILE=/path/to/cookies.txt
+CCORE_YOUTUBE_PROXY=http://user:pass@host:port
 CCORE_FIRECRAWL_PROXY=auto
 CCORE_FIRECRAWL_WAIT_FOR=3000
 FIRECRAWL_API_URL=http://localhost:3002
@@ -217,12 +271,12 @@ FIRECRAWL_API_KEY=anykey
 
 ### Document Engines
 
-The `document_engine` setting controls how files (PDF, DOCX, PPTX, XLSX, HTML) are processed:
+The `document_engine` setting controls how files (PDF, DOCX, PPTX, XLSX, ODT, ODS, ODP, HTML) are processed:
 
 | Engine | Description | Requirements |
 |--------|-------------|-------------|
 | `auto` (default) | Tries Docling first, falls back to simple | Depends on installed extras |
-| `simple` | pdfplumber for PDF, fast-ebook for EPUB, python-docx/openpyxl/python-pptx for Office, markdownify for HTML | None (included) |
+| `simple` | pdfplumber for PDF, fast-ebook for EPUB, python-docx/openpyxl/python-pptx for Office, odfpy for OpenDocument, markdownify for HTML | None (included) |
 | `docling` | Docling library for rich document parsing | `pip install content-core[docling]` |
 
 `auto` is a preference: it uses Docling when installed and silently falls back to `simple` otherwise. `docling` is a requirement: if the extra is not installed, extraction raises `ConfigurationError` instead of silently producing simple-engine output. Install it with `pip install content-core[docling]`, or set `CCORE_DOCUMENT_ENGINE=simple` (or `auto`) to proceed without it.
@@ -305,6 +359,27 @@ config = ContentCoreConfig(audio_concurrency=5)
 
 Higher values speed up processing of long files but may hit API rate limits.
 
+### Segment Length
+
+Long audio is split into 10-minute segments, a length that fits the upload
+size limits of the cloud STT APIs (OpenAI rejects uploads over 25 MB). A
+self-hosted endpoint has no such limit, and every cut costs it the context
+that falls across it. Set a longer segment, or `0` to send the file whole:
+
+```bash
+CCORE_AUDIO_SEGMENT_MINUTES=0
+```
+
+Or in code:
+
+```python
+config = ContentCoreConfig(audio_segment_minutes=0)
+```
+
+Keep the default when using a cloud provider: with splitting off or a long
+segment, a long file can exceed the provider's upload limit and the
+transcription fails.
+
 ### Custom STT Models
 
 Override the speech-to-text provider and model per call:

{content_core-2.1.0 → content_core-2.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "content-core"
-version = "2.1.0"
+version = "2.2.1"
 description = "Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server"
 readme = "README.md"
 homepage = "https://github.com/lfnovo/content-core"
@@ -9,7 +9,7 @@ authors = [
 ]
 requires-python = ">=3.10"
 dependencies = [
-    "aiohttp>=3.13",
+    "aiohttp>=3.14.3",
     "bs4>=0.0.2",
     "esperanto>=2.26.0",
     "jinja2>=3.1.6",
@@ -27,20 +27,22 @@ dependencies = [
     "ai-prompter>=0.4.0",
     "readability-lxml>=0.8.4.1",
     "firecrawl-py>=4.13.0",
-    "pillow>=12.0.0",
+    "pillow>=12.3.0",
     "pytubefix>=10.0.0",
     "fastmcp>=3.0.0",
     "tenacity>=9.0.0",
     "markdownify>=0.14.0",
-    "pydantic-settings>=2.12.0",
+    "pydantic-settings>=2.14.2",
     "tomli>=2.0; python_version < '3.11'",
-    "click>=8.3.0",
+    "click>=8.3.3",
+    "odfpy>=1.4.1",
+    "requests>=2.31.0",
 ]
 
 [project.optional-dependencies]
-docling = ["docling>=2.86.0", "peft>=0.18.0", "transformers>=4.50,<5"]
-crawl4ai = ["crawl4ai>=0.7.0"]
-langchain = ["langchain-core>=0.1.0"]
+docling = ["docling>=2.94.0", "peft>=0.18.0", "transformers>=4.50,<5"]
+crawl4ai = ["crawl4ai>=0.9.0"]
+langchain = ["langchain-core>=1.3.3"]
 
 [project.scripts]
 content-core = "content_core.cli:cli"
@@ -60,8 +62,8 @@ package-dir = {"content_core" = "src/content_core"}
 
 [dependency-groups]
 dev = [
-    "crawl4ai>=0.8.0",
-    "docling>=2.86.0",
+    "crawl4ai>=0.9.0",
+    "docling>=2.94.0",
     "transformers>=4.50,<5",
     "ipykernel>=7.0.0",
     "ipywidgets>=8.0.0",

{content_core-2.1.0 → content_core-2.2.1}/src/content_core/cli.py

@@ -109,6 +109,7 @@ def config():
     \b
     Available keys:
       audio_concurrency    Parallel transcription limit (1-10, default: 3)
+      audio_segment_minutes  Audio segment length in minutes, 0 disables splitting (default: 10)
       crawl4ai_api_url     Crawl4AI Docker API URL (default: none, uses local mode)
       crawl4ai_api_token   Bearer token for the Crawl4AI Docker API (required by Crawl4AI >= 0.9.0)
       audio_model          Override STT model

{content_core-2.1.0 → content_core-2.2.1}/src/content_core/common/exceptions.py

@@ -1,15 +1,8 @@
 """The public exception taxonomy for ``extract_content``.
 
 Per the raise/degrade boundary in ``ARCHITECTURE.md``, total failure raises
-one of these; degradation only exists within a single source.
-
-The taxonomy is complete, its raise sites are not yet: ``NotFoundError``,
-``NetworkError``, ``ExternalServiceError`` and ``FileOperationError`` have
-no raise site in this release -- the failures they name still escape as
-untyped exceptions (an ``aiohttp.ClientError`` out of a download, say)
-until the migration in #60 lands. ``ContentCoreError`` catches every typed
-failure, so it is the right single handler -- but it does not yet cover the
-whole library.
+one of these; degradation only exists within a single source. Every one
+derives from ``ContentCoreError``, the single handler for typed failures.
 """
 
 

content_core-2.2.1/src/content_core/common/http.py

@@ -0,0 +1,131 @@
+"""HTTP sessions for fetching user-supplied URLs.
+
+Callers (Open Notebook, the MCP server) pass URLs they did not choose, and the
+fetch runs from inside their network. ``guarded_session()`` refuses to connect
+to link-local addresses (169.254.0.0/16, fe80::/10, which includes cloud
+metadata services such as 169.254.169.254) and to AWS's IPv6 metadata address,
+on the first request and on every redirect hop:
+
+- hostnames are checked by the connector's resolver, on the addresses the
+  connection will actually use (no re-resolution between check and connect);
+- IP literals skip the resolver, so they are checked before the request and
+  on each redirect ``Location``.
+
+Private and loopback addresses stay allowed: self-hosted deployments
+legitimately fetch from their own network. When the session goes through an
+HTTP(S) proxy (``trust_env``), the target hostname is resolved by the proxy,
+so only IP literals are checked locally.
+"""
+
+import ipaddress
+import socket
+from types import SimpleNamespace
+from typing import Any
+
+import aiohttp
+from aiohttp.abc import AbstractResolver, ResolveResult
+from yarl import URL
+
+from content_core.common.exceptions import InvalidInputError
+
+# AWS IMDSv6 is a Unique Local Address, not link-local.
+_AWS_IMDS_V6 = ipaddress.ip_address("fd00:ec2::254")
+
+IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
+
+
+class BlockedAddressError(InvalidInputError):
+    """Raised when a URL, or a redirect it leads to, targets a blocked address."""
+
+
+def is_blocked_address(ip: IPAddress) -> bool:
+    """True for link-local and cloud-metadata addresses."""
+    if ip.is_link_local:
+        return True
+    if isinstance(ip, ipaddress.IPv6Address):
+        mapped = ip.ipv4_mapped
+        if mapped is not None and mapped.is_link_local:
+            return True
+        # Compare without scope id so fd00:ec2::254%eth0 is caught too.
+        if int(ip) == int(_AWS_IMDS_V6):
+            return True
+    return False
+
+
+def _parse_ip(host: str) -> IPAddress | None:
+    try:
+        return ipaddress.ip_address(host.strip("[]").split("%", 1)[0])
+    except ValueError:
+        return None
+
+
+def check_url_host(url: str | URL) -> None:
+    """Raise BlockedAddressError if ``url``'s host is a blocked IP literal."""
+    host = URL(str(url)).host or ""
+    ip = _parse_ip(host)
+    if ip is not None and is_blocked_address(ip):
+        raise BlockedAddressError(
+            f"Refusing to fetch {url}: link-local and cloud-metadata "
+            "addresses are not allowed"
+        )
+
+
+class GuardedResolver(AbstractResolver):
+    """Resolver that drops blocked addresses and fails if none remain."""
+
+    def __init__(self) -> None:
+        self._resolver = aiohttp.DefaultResolver()
+
+    async def resolve(
+        self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET
+    ) -> list[ResolveResult]:
+        results = await self._resolver.resolve(host, port, family)
+        allowed = [
+            r
+            for r in results
+            if not ((ip := _parse_ip(r["host"])) and is_blocked_address(ip))
+        ]
+        if not allowed:
+            raise BlockedAddressError(
+                f"Refusing to fetch from {host}: it resolves to a link-local or "
+                "cloud-metadata address"
+            )
+        return allowed
+
+    async def close(self) -> None:
+        await self._resolver.close()
+
+
+async def _on_request_redirect(
+    session: aiohttp.ClientSession,
+    ctx: SimpleNamespace,
+    params: aiohttp.TraceRequestRedirectParams,
+) -> None:
+    location = params.response.headers.get("Location") or params.response.headers.get(
+        "URI"
+    )
+    if location:
+        check_url_host(params.response.url.join(URL(location)))
+
+
+def guarded_session(**kwargs: Any) -> aiohttp.ClientSession:
+    """A ClientSession that refuses blocked addresses on every hop.
+
+    Use it for any request whose URL comes from the caller. The initial URL's
+    IP literal is checked by ``guarded_request_url``; pass the URL through it
+    before the request.
+    """
+    trace = aiohttp.TraceConfig()
+    trace.on_request_redirect.append(_on_request_redirect)
+    kwargs.setdefault("trust_env", True)
+    return aiohttp.ClientSession(
+        connector=aiohttp.TCPConnector(resolver=GuardedResolver()),
+        trace_configs=[trace],
+        **kwargs,
+    )
+
+
+def guarded_request_url(url: str) -> str:
+    """Check ``url``'s host before the first request; returns it unchanged."""
+    check_url_host(url)
+    return url