arcus-provider-runtime 0.3.1__py3-none-any.whl

arcus/provider_runtime/__init__.py

@@ -0,0 +1,25 @@
+"""arcus provider-runtime public API."""
+
+from .factory import Factory, register_defaults
+from .provider_interface import ExtractionContext, Provider
+from .registry import ProviderRegistry
+from .types import (
+    EXIT_CODES,
+    DetectionResult,
+    ExtractionResult,
+    Segment,
+    SourceMetadata,
+)
+
+__all__ = [
+    "EXIT_CODES",
+    "DetectionResult",
+    "ExtractionContext",
+    "ExtractionResult",
+    "Factory",
+    "Provider",
+    "ProviderRegistry",
+    "Segment",
+    "SourceMetadata",
+    "register_defaults",
+]

arcus/provider_runtime/factory.py

@@ -0,0 +1,174 @@
+"""Factory: end-to-end orchestration of detection → cache check → extract → write."""
+
+from __future__ import annotations
+
+import re
+import tempfile
+import traceback
+from pathlib import Path
+
+from .log import EventLogger, now_iso
+from .provider_interface import ExtractionContext, Provider
+from .registry import ProviderRegistry
+from .types import EXIT_CODES
+from .writer import cache_hit_exists, write_failure_stub, write_success
+
+
+class Factory:
+    """Owns a ProviderRegistry; provides `.run(input, out_dir)` for one-shot extraction."""
+
+    def __init__(self, registry: ProviderRegistry) -> None:
+        self.registry = registry
+
+    def detect(self, raw_input: str):
+        return self.registry.detect(raw_input)
+
+    def run(
+        self,
+        raw_input: str,
+        *,
+        out_dir: Path,
+        force: bool = False,
+        json_log: bool = False,
+        keep_intermediates: bool = False,
+        notebook_tag: str | None = None,
+    ) -> int:
+        """Detect → cache check → extract → write outputs. Returns exit code."""
+        logger = EventLogger(out_dir, json_log_stderr=json_log)
+        logger.emit({"ts": now_iso(), "raw": raw_input, "status": "started"})
+
+        match = self.registry.detect(raw_input)
+        if match is None:
+            logger.emit({"ts": now_iso(), "raw": raw_input, "status": "failed",
+                         "error": "no provider matched"})
+            return EXIT_CODES["EXTRACTORS_EXHAUSTED"]
+
+        provider, detection = match
+        logger.emit({
+            "ts": now_iso(),
+            "kind": provider.kind,
+            "source_id": detection.source_id,
+            "event": "detected",
+        })
+
+        # Cache check uses the provider's predicted slug. If the on-disk
+        # file's frontmatter `source_id` matches this detection's source_id,
+        # we trust the file and short-circuit. Disambiguated forms
+        # (`<slug>--<8char>.md`) are checked too — see cache_hit_exists.
+        try:
+            predicted_slug = provider.predict_slug(detection)
+        except Exception as e:
+            # predict_slug failed (e.g., metadata fetch hit network error).
+            # Fall through to extraction — the real extract() call will
+            # surface the same error in a structured way.
+            logger.emit({
+                "ts": now_iso(),
+                "kind": provider.kind,
+                "source_id": detection.source_id,
+                "event": "predict_slug_failed",
+                "error": str(e),
+            })
+            predicted_slug = None
+
+        if (
+            not force
+            and predicted_slug is not None
+            and cache_hit_exists(out_dir, predicted_slug, detection.source_id)
+        ):
+            logger.emit({
+                "ts": now_iso(),
+                "kind": provider.kind,
+                "source_id": detection.source_id,
+                "status": "cache_hit",
+                "slug": predicted_slug,
+            })
+            return EXIT_CODES["SUCCESS"]
+
+        # Sanitize source_id for use as a tempdir prefix — URLs and local
+        # paths both contain '/' which mkdtemp interprets as a path separator.
+        safe_prefix = re.sub(r"[^A-Za-z0-9._-]", "_", detection.source_id)[:40]
+        with tempfile.TemporaryDirectory(prefix=f"arcus-{safe_prefix}-") as tmp:
+            context = ExtractionContext(
+                out_dir=out_dir,
+                work_dir=Path(tmp),
+                notebook_tag=notebook_tag,
+                keep_intermediates=keep_intermediates,
+                factory=self,
+            )
+            try:
+                result = provider.extract(detection, context)
+            except Exception as e:  # provider-level uncaught — never crash the CLI
+                tb = traceback.format_exc()
+                logger.emit({
+                    "ts": now_iso(),
+                    "kind": provider.kind,
+                    "source_id": detection.source_id,
+                    "status": "failed",
+                    "error": f"unhandled: {e}",
+                    "traceback": tb,
+                })
+                write_failure_stub(
+                    out_dir,
+                    slug=detection.source_id,
+                    source=detection.raw,
+                    source_id=detection.source_id,
+                    kind=provider.kind,
+                    title=None,
+                    exit_code=EXIT_CODES["PROVIDER_PRIMARY_FAILED"],
+                    extractor_attempted=[provider.kind],
+                    error=str(e),
+                )
+                return EXIT_CODES["PROVIDER_PRIMARY_FAILED"]
+
+        if result.status == "success":
+            write_success(out_dir, result.metadata.slug, result)
+            logger.emit({
+                "ts": now_iso(),
+                "kind": provider.kind,
+                "source_id": detection.source_id,
+                "status": "success",
+                "slug": result.metadata.slug,
+            })
+            return EXIT_CODES["SUCCESS"]
+
+        # status == "failed"
+        write_failure_stub(
+            out_dir,
+            slug=result.metadata.slug,
+            source=detection.raw,
+            source_id=detection.source_id,
+            kind=provider.kind,
+            title=result.metadata.title or None,
+            exit_code=result.exit_code or EXIT_CODES["PROVIDER_PRIMARY_FAILED"],
+            extractor_attempted=[provider.kind],
+            error=result.error or "unknown failure",
+        )
+        logger.emit({
+            "ts": now_iso(),
+            "kind": provider.kind,
+            "source_id": detection.source_id,
+            "status": "failed",
+            "error": result.error,
+        })
+        return result.exit_code or EXIT_CODES["PROVIDER_PRIMARY_FAILED"]
+
+
+def register_defaults(registry: ProviderRegistry) -> None:
+    """Register the v1 providers.
+
+    Dispatch order matters — first-match-wins:
+      1. YouTube (specific host pattern)
+      2. PDF (specific file suffix; must run before HTML so .pdf URLs
+         don't get caught by HTML's broad scheme match)
+      3. Docs (specific file suffixes — docx/xlsx/pptx/epub; same reason
+         as PDF — register before HTML)
+      4. HTML (catch-all for any other http(s) URL)
+    """
+    from .providers.docs.docs import DocsProvider
+    from .providers.html.html import HtmlProvider
+    from .providers.pdf.pdf import PdfProvider
+    from .providers.youtube.youtube import YouTubeProvider
+    registry.register(YouTubeProvider())
+    registry.register(PdfProvider())
+    registry.register(DocsProvider())
+    registry.register(HtmlProvider())

arcus/provider_runtime/log.py

@@ -0,0 +1,46 @@
+"""NDJSON event logger.
+
+Every arcus invocation appends structured events to
+`<out_dir>/.log/extract-log.ndjson`. Consumers parse this for
+audit logging. Optionally mirrors to stderr (`--json-log` flag)
+so subprocess callers can read events without touching disk.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def now_iso() -> str:
+    """UTC ISO-8601 timestamp like '2026-05-17T19:42:01.123456+00:00'."""
+    return datetime.now(timezone.utc).isoformat()
+
+
+class EventLogger:
+    """Append-only NDJSON event logger."""
+
+    def __init__(self, out_dir: Path, *, json_log_stderr: bool = False) -> None:
+        self.out_dir = out_dir
+        self.json_log_stderr = json_log_stderr
+        self._log_dir = out_dir / ".log"
+        self._log_file = self._log_dir / "extract-log.ndjson"
+        self._dir_ready = False
+
+    def emit(self, event: dict[str, Any]) -> None:
+        """Append a single event. Creates the .log directory lazily."""
+        line = json.dumps(event, ensure_ascii=False) + "\n"
+
+        if self.json_log_stderr:
+            sys.stderr.write(line)
+            sys.stderr.flush()
+
+        if not self._dir_ready:
+            self._log_dir.mkdir(parents=True, exist_ok=True)
+            self._dir_ready = True
+
+        with self._log_file.open("a", encoding="utf-8") as f:
+            f.write(line)

arcus/provider_runtime/provider_interface.py

@@ -0,0 +1,86 @@
+"""Provider Protocol — every content-extraction provider implements this shape.
+
+This file is documentation by code. Concrete providers live under
+`providers/<kind>/`. The Protocol is `@runtime_checkable` so the factory
+can `isinstance(p, Provider)` for sanity in tests, though duck-typing is
+the contract; the Protocol is not enforced beyond static type checking.
+
+Lifecycle:
+  1. Factory.detect(input) walks registered providers calling .matches(input).
+     First non-None DetectionResult wins.
+  2. Caller passes detection to provider.extract(detection, context).
+  3. Provider returns ExtractionResult; caller writes via shared writer.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from .types import DetectionResult, ExtractionResult
+
+
+@dataclass
+class ExtractionContext:
+    """Context passed to provider.extract().
+
+    arcus is a single-source download/extraction layer — see
+    feedback-arcus-pure-download-layer. Providers don't recurse; the
+    `factory` field is reserved/unused by current providers.
+    """
+
+    out_dir: Path
+    work_dir: Path
+    notebook_tag: str | None = None
+    keep_intermediates: bool = False
+    # Reserved; no current provider recurses (per the pure-download-layer rule).
+    factory: "Factory | None" = None  # forward-ref to break import cycle
+
+
+@runtime_checkable
+class Provider(Protocol):
+    """Single content-extraction provider."""
+
+    kind: str
+    """Stable identifier — e.g. 'youtube', 'html', 'pdf', 'athena_topic'."""
+
+    def matches(self, raw_input: str) -> DetectionResult | None:
+        """Pure: return parsed detection if this provider handles the input.
+
+        No network, no file IO. Detection uses string shape only.
+        """
+        ...
+
+    def predict_slug(self, detection: DetectionResult) -> str:
+        """Return the bare (pre-disambiguation) slug that `extract()` will use.
+
+        The factory calls this before extraction so it can cache-check the
+        right filename. The slug MUST match what `extract()` will eventually
+        assign to `ExtractionResult.metadata.slug` (modulo collision
+        disambiguation, which the writer handles per-file).
+
+        Providers MAY fetch metadata here. Heavy fetches should cache
+        internally so `extract()` doesn't repeat the work.
+        """
+        ...
+
+    def extract(
+        self,
+        detection: DetectionResult,
+        context: ExtractionContext,
+    ) -> ExtractionResult:
+        """Fetch + transform the content. Network IO + filesystem allowed.
+
+        Returns ExtractionResult with status='success' or 'failed'.
+        Single-source only — providers do not recurse or aggregate.
+        """
+        ...
+
+
+# `ExtractionContext.factory` is typed as the string `"Factory | None"`.
+# We deliberately do NOT import Factory at module load time — `from __future__
+# import annotations` keeps every annotation as a string, and dataclasses
+# never resolves it. Composite providers that need the live class can
+# `from .factory import Factory` lazily inside a method body without
+# creating an import cycle.