content-core 2.0.2__tar.gz → 2.0.4__tar.gz

{content_core-2.0.2 → content_core-2.0.4}/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.0.4] - 2026-07-12
+
+### Added
+- `check_file_support(file_path, config)` public API for cheap pre-flight validation of whether a file can be extracted, without running extraction; returns a `FileSupport` verdict
+
+### Fixed
+- YouTube `youtube.com/live/<id>` and `youtube.com/shorts/<id>` URLs were not recognized, causing title and transcript extraction to fail
+
+## [2.0.3] - 2026-04-13
+
+### Added
+- `--version` flag to CLI (`content-core --version`)
+
 ## [2.0.2] - 2026-04-13
 
 ### Fixed

{content_core-2.0.2 → content_core-2.0.4}/CLAUDE.md

@@ -21,6 +21,38 @@ content-core mcp
 content-core config list|set|delete
 ```
 
+## For Automated Agents
+
+If you are an automated coding agent (harny, Claude Code in headless mode, etc.) working on this repo:
+
+### Install command
+
+```bash
+uv sync --group dev
+```
+
+### Validator command
+
+```bash
+uv run pytest tests/unit tests/integration -v
+```
+
+This is the gate. It returns clean pass/fail in ~15s, requires no credentials or network, and matches what CI enforces on `main` (CI runs only `tests/unit`; integration tests are also safe locally — no network/API keys). For targeted feedback during iteration, use `uv run pytest -k "<keyword>"` per the table in the Testing section below.
+
+### Do NOT use as gates
+
+- **`ruff check`** — available via `make ruff` but NOT enforced in CI. Including it as a validator gate would surface unaudited pre-existing findings unrelated to your task.
+- **`mypy`** — not enforced in CI. No baseline configured. Same risk.
+- **`make test-e2e` / `make test-e2e-heavy`** — require network access, API keys for LLM/STT providers, and large model downloads. Will fail in a sandboxed worktree.
+
+If you believe a task genuinely requires lint or type cleanup, do it as a *separate task* with explicit scope, not as a side effect of an unrelated change.
+
+### Scope guidance
+
+- Single processor changes (`processors/url/*.py`, `processors/document/*.py`, `processors/media/*.py`) are the natural unit of work — keep changes confined to one file plus its matching test in `tests/unit/`.
+- Avoid touching `extraction.py` (orchestrator) or `config.py` unless the task explicitly targets routing or configuration.
+- New processors should follow the `Processor` protocol in `src/content_core/processors/protocol.py`.
+
 ## Codebase Structure
 
 ```

{content_core-2.0.2 → content_core-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 2.0.2
+Version: 2.0.4
 Summary: Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE

{content_core-2.0.2 → content_core-2.0.4}/SKILL.md

@@ -152,8 +152,9 @@ If summarization fails with an API key error, fall back to `extract_content` and
 
 ## Guidelines
 
-- Prefer the MCP tools if available — they are async and more efficient than CLI calls
-- If MCP is not available, use the CLI via Bash with `uvx content-core`
+- For small/medium content (articles, short pages): prefer MCP tools if available — they are async and more efficient
+- For large content (long documents, full books, lengthy transcripts): prefer the CLI via Bash, redirecting output to a file (`uvx content-core extract "URL" > output.md`). This avoids flooding the agent's context window with large payloads. Read only the relevant sections from the file as needed.
+- If MCP is not available, always use the CLI via Bash with `uvx content-core`
 - For URLs: extraction works without any API key
 - For audio/video: requires `OPENAI_API_KEY` (or another STT provider key)
 - For summarization: requires an LLM API key

{content_core-2.0.2 → content_core-2.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "content-core"
-version = "2.0.2"
+version = "2.0.4"
 description = "Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server"
 readme = "README.md"
 homepage = "https://github.com/lfnovo/content-core"

{content_core-2.0.2 → content_core-2.0.4}/src/content_core/__init__.py

@@ -5,9 +5,9 @@ load_dotenv()
 
 from content_core.config import ContentCoreConfig
 from content_core.content.summary import summarize
-from content_core.extraction import extract_content
+from content_core.extraction import check_file_support, extract_content
 from content_core.logging import configure_logging
-from content_core.common.state import ExtractionInput, ExtractionOutput
+from content_core.common.state import ExtractionInput, ExtractionOutput, FileSupport
 
 # Convenience alias
 extract = extract_content
@@ -18,8 +18,10 @@ configure_logging(debug=False)
 __all__ = [
     "extract_content",
     "extract",
+    "check_file_support",
     "summarize",
     "ContentCoreConfig",
     "ExtractionInput",
     "ExtractionOutput",
+    "FileSupport",
 ]

{content_core-2.0.2 → content_core-2.0.4}/src/content_core/cli.py

@@ -7,7 +7,13 @@ import click
 from content_core.logging import configure_logging
 
 
+def _get_version():
+    from importlib.metadata import version
+    return version("content-core")
+
+
 @click.group()
+@click.version_option(package_name="content-core")
 @click.option("--debug", is_flag=True, help="Enable debug logging")
 def cli(debug):
     """Content Core — Extract and summarize content from any source."""

{content_core-2.0.2 → content_core-2.0.4}/src/content_core/common/state.py

@@ -29,7 +29,24 @@ class ExtractionOutput(BaseModel):
     metadata: dict = Field(default_factory=dict)
 
 
+class FileSupport(BaseModel):
+    """Verdict from a pre-flight file-support check.
+
+    Returned by ``check_file_support`` so callers can validate an upload
+    cheaply (identification + routing only, no extraction) before committing
+    to a full extraction job.
+    """
+
+    supported: bool
+    file_path: str
+    identified_type: str = ""  # MIME type detected for the file
+    document_engine: str = ""  # engine the verdict was computed for
+    processor: Optional[str] = None  # processor that would handle it, if supported
+    reason: Optional[str] = None  # human-readable explanation when unsupported
+
+
 __all__ = [
     "ExtractionInput",
     "ExtractionOutput",
+    "FileSupport",
 ]

{content_core-2.0.2 → content_core-2.0.4}/src/content_core/extraction.py

@@ -11,7 +11,7 @@ from content_core.common.exceptions import InvalidInputError, UnsupportedTypeExc
 from content_core.common.retry import retry_download
 from content_core.config import ContentCoreConfig, get_default_config
 from content_core.logging import logger
-from content_core.common.state import ExtractionOutput
+from content_core.common.state import ExtractionOutput, FileSupport
 
 # Import processor v2 functions
 from content_core.processors.media.audio import transcribe_audio
@@ -105,6 +105,90 @@ async def _extract_url(url: str, cfg: ContentCoreConfig) -> ExtractionOutput:
     return await extract_from_url(url, cfg)
 
 
+def _route_for_mime(mime: str, cfg: ContentCoreConfig) -> str | None:
+    """Return the processor that would handle a file of this MIME type.
+
+    Single source of truth for file-support routing, shared by ``_extract_file``
+    (actual extraction) and ``check_file_support`` (pre-flight validation), so
+    the pre-flight answer can never disagree with what extraction really does.
+
+    Returns the processor name ("docling", "pdf", "epub", "office", "video",
+    "audio", "text") or ``None`` if the type is unsupported.
+    """
+    engine = cfg.document_engine
+    if engine == "docling" or (
+        engine == "auto" and DOCLING_AVAILABLE and mime in DOCLING_SUPPORTED
+    ):
+        if DOCLING_AVAILABLE and extract_docling is not None:
+            return "docling"
+
+    if mime in SUPPORTED_PDF_TYPES:
+        return "pdf"
+    if mime in SUPPORTED_EPUB_TYPES:
+        return "epub"
+    if mime in SUPPORTED_OFFICE_TYPES:
+        return "office"
+    if mime.startswith("video/"):
+        return "video"
+    if mime.startswith("audio/"):
+        return "audio"
+    if mime == "text/plain":
+        return "text"
+    return None
+
+
+async def check_file_support(
+    file_path: str, config: ContentCoreConfig | None = None
+) -> FileSupport:
+    """Check whether content-core can extract a given file, without extracting.
+
+    Runs the same identification + routing that ``extract_content`` uses, but
+    stops before extraction, so the answer carries the same authority as a real
+    run at a fraction of the cost (it only reads the file header). "Unsupported"
+    is returned as a verdict rather than raised, since it is an expected outcome
+    at ingestion time.
+
+    Args:
+        file_path: Local file path to validate.
+        config: Optional config override. The ``document_engine`` setting is
+            honored, since supported types vary by engine.
+
+    Returns:
+        FileSupport verdict with ``supported``, the ``identified_type`` (MIME),
+        the ``processor`` that would handle it (when supported), and a ``reason``
+        when it would not.
+    """
+    from content_core.content.identification import get_file_type
+
+    cfg = config or get_default_config()
+
+    # Identification can fail outright for unrecognizable files -- that is itself
+    # an "unsupported" verdict (extract_content would raise here too), not an
+    # error the caller should have to catch.
+    try:
+        mime = await get_file_type(file_path)
+    except UnsupportedTypeException as exc:
+        return FileSupport(
+            supported=False,
+            file_path=file_path,
+            identified_type="",
+            document_engine=cfg.document_engine,
+            processor=None,
+            reason=str(exc),
+        )
+
+    processor = _route_for_mime(mime, cfg)
+    supported = processor is not None
+    return FileSupport(
+        supported=supported,
+        file_path=file_path,
+        identified_type=mime,
+        document_engine=cfg.document_engine,
+        processor=processor,
+        reason=None if supported else f"Unsupported file type: {mime}",
+    )
+
+
 async def _extract_file(
     path: str, cfg: ContentCoreConfig, delete_after: bool = False
 ) -> ExtractionOutput:
@@ -116,32 +200,30 @@ async def _extract_file(
 
     used_docling = False
     try:
+        route = _route_for_mime(mime, cfg)
+
         # Docling routing (if enabled and supported)
-        engine = cfg.document_engine
-        if engine == "docling" or (
-            engine == "auto" and DOCLING_AVAILABLE and mime in DOCLING_SUPPORTED
-        ):
-            if DOCLING_AVAILABLE and extract_docling is not None:
-                used_docling = True
-                result = await extract_docling(path, cfg)
-                if not result.title:
-                    result.title = os.path.basename(path)
-                result.identified_type = mime
-                result.source_type = "file"
-                return result
+        if route == "docling":
+            used_docling = True
+            result = await extract_docling(path, cfg)
+            if not result.title:
+                result.title = os.path.basename(path)
+            result.identified_type = mime
+            result.source_type = "file"
+            return result
 
         # Standard processors
-        if mime in SUPPORTED_PDF_TYPES:
+        if route == "pdf":
             result = await extract_pdf_file(path, cfg)
-        elif mime in SUPPORTED_EPUB_TYPES:
+        elif route == "epub":
             result = await extract_epub_file(path, cfg)
-        elif mime in SUPPORTED_OFFICE_TYPES:
+        elif route == "office":
             result = await extract_office(path, mime, cfg)
-        elif mime.startswith("video/"):
+        elif route == "video":
             result = await extract_video(path, cfg)
-        elif mime.startswith("audio/"):
+        elif route == "audio":
             result = await transcribe_audio(path, cfg)
-        elif mime == "text/plain":
+        elif route == "text":
             result = await extract_text_file(path, cfg)
         else:
             raise UnsupportedTypeException(f"Unsupported file type: {mime}")

{content_core-2.0.2 → content_core-2.0.4}/src/content_core/processors/url/youtube.py

@@ -60,6 +60,8 @@ async def _extract_youtube_id(url):
         r"(?:"  # Group start
         r"/embed/"  # Embed URL
         r"|/v/"  # Older video URL
+        r"|/live/"  # Livestream URL (active or ended)
+        r"|/shorts/"  # Shorts URL
         r"|/watch\?v="  # Standard watch URL
         r"|/watch\?.+&v="  # Other watch URL
         r")"  # Group end

{content_core-2.0.2 → content_core-2.0.4}/tests/unit/test_routing.py

@@ -7,8 +7,8 @@ import pytest
 
 from content_core.common.exceptions import InvalidInputError, UnsupportedTypeException
 from content_core.config import ContentCoreConfig
-from content_core.extraction import extract_content
-from content_core.common.state import ExtractionOutput
+from content_core.extraction import check_file_support, extract_content
+from content_core.common.state import ExtractionOutput, FileSupport
 
 
 def _make_output(**kwargs) -> ExtractionOutput:
@@ -324,3 +324,86 @@ async def test_docling_flags_warning_without_engine():
 
         mock_logger.warning.assert_called_once()
         assert "docling" in mock_logger.warning.call_args[0][0].lower()
+
+
+# ---------------------------------------------------------------------------
+# 14. check_file_support pre-flight verdict
+# ---------------------------------------------------------------------------
+@pytest.mark.asyncio
+async def test_check_file_support_supported():
+    cfg = ContentCoreConfig(document_engine="simple")
+    with patch(
+        "content_core.content.identification.get_file_type",
+        new_callable=AsyncMock,
+        return_value="application/pdf",
+    ):
+        result = await check_file_support("/tmp/test.pdf", config=cfg)
+    assert isinstance(result, FileSupport)
+    assert result.supported is True
+    assert result.identified_type == "application/pdf"
+    assert result.processor == "pdf"
+    assert result.reason is None
+    assert result.document_engine == "simple"
+    assert result.file_path == "/tmp/test.pdf"
+
+
+@pytest.mark.asyncio
+async def test_check_file_support_unsupported():
+    cfg = ContentCoreConfig(document_engine="simple")
+    with patch(
+        "content_core.content.identification.get_file_type",
+        new_callable=AsyncMock,
+        return_value="application/x-unknown-binary",
+    ):
+        result = await check_file_support("/tmp/test.bin", config=cfg)
+    assert result.supported is False
+    assert result.processor is None
+    assert result.reason is not None
+    assert "application/x-unknown-binary" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_check_file_support_unidentifiable_returns_verdict():
+    """A file whose type can't be determined is a verdict, not a raised error."""
+    cfg = ContentCoreConfig(document_engine="simple")
+    with patch(
+        "content_core.content.identification.get_file_type",
+        new_callable=AsyncMock,
+        side_effect=UnsupportedTypeException("Unable to determine file type for: x"),
+    ):
+        result = await check_file_support("/tmp/mystery.xyz", config=cfg)
+    assert result.supported is False
+    assert result.processor is None
+    assert result.identified_type == ""
+    assert "determine file type" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_check_file_support_does_not_extract():
+    """The pre-flight check must never invoke a real extractor."""
+    cfg = ContentCoreConfig(document_engine="simple")
+    with patch(
+        "content_core.content.identification.get_file_type",
+        new_callable=AsyncMock,
+        return_value="application/pdf",
+    ), patch(
+        "content_core.extraction.extract_pdf_file", new_callable=AsyncMock
+    ) as mock_pdf:
+        await check_file_support("/tmp/test.pdf", config=cfg)
+        mock_pdf.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_check_file_support_agrees_with_extraction():
+    """The verdict must never disagree with what extract_content actually does."""
+    cfg = ContentCoreConfig(document_engine="simple")
+    with patch(
+        "content_core.content.identification.get_file_type",
+        new_callable=AsyncMock,
+        return_value="application/x-unknown-binary",
+    ):
+        verdict = await check_file_support("/tmp/test.bin", config=cfg)
+        assert verdict.supported is False
+        # extraction of the same type raises, confirming the verdict
+        with pytest.raises(UnsupportedTypeException):
+            await extract_content(file_path="/tmp/test.bin", config=cfg)

{content_core-2.0.2 → content_core-2.0.4}/tests/unit/test_youtube_parsing.py

@@ -25,6 +25,18 @@ class TestExtractYoutubeId:
         )
         assert result == "dQw4w9WgXcQ"
 
+    async def test_live_url(self):
+        result = await _extract_youtube_id(
+            "https://www.youtube.com/live/dQw4w9WgXcQ"
+        )
+        assert result == "dQw4w9WgXcQ"
+
+    async def test_shorts_url(self):
+        result = await _extract_youtube_id(
+            "https://www.youtube.com/shorts/dQw4w9WgXcQ"
+        )
+        assert result == "dQw4w9WgXcQ"
+
     async def test_url_with_extra_params(self):
         result = await _extract_youtube_id(
             "https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=120"

{content_core-2.0.2 → content_core-2.0.4}/uv.lock

@@ -775,7 +775,7 @@ wheels = [
 
 [[package]]
 name = "content-core"
-version = "2.0.2"
+version = "2.0.4"
 source = { editable = "." }
 dependencies = [
     { name = "ai-prompter" },