corpus-forge 0.1.0b2__py3-none-any.whl

corpus_forge/__init__.py

@@ -0,0 +1,3 @@
+"""Corpus Forge — chat with your data, build a living trainable corpus."""
+
+__version__ = "0.1.0b2"

corpus_forge/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for python -m corpus_forge"""
+
+from .cli import app
+
+if __name__ == "__main__":
+    app()

corpus_forge/_http.py

@@ -0,0 +1,193 @@
+"""Shared HTTP transport for remote model-backend clients.
+
+Every remote model integration in corpus-forge (VLM, Whisper, code
+enricher, LLM classifier, multi-modal embedder) speaks to a JSON HTTP
+endpoint and maps the same set of failure modes onto a family-specific
+error triad — ``<Family>UnavailableError`` / ``TimeoutError`` /
+``ResponseError``.
+
+This module owns the mapping in one place. Each family declares the
+triad with an :class:`HttpErrors` bundle and calls :func:`request_json`,
+which:
+
+- catches the standard ``requests`` exception ladder (``Timeout`` /
+  ``ConnectionError`` / ``RequestException``) and raises the matching
+  family-typed error;
+- optionally promotes ``401``/``403`` to the family's "unavailable"
+  bucket (API-key rejection is a configuration failure, not a flake);
+- treats non-2xx HTTP, malformed JSON, non-object JSON, and missing
+  required keys as response errors with a truncated body snippet in the
+  message.
+
+Tests mock ``requests.post`` / ``requests.get`` directly. This module
+calls them by name (not via ``requests.request``) so existing
+``patch("requests.post", ...)`` contracts continue to work.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Literal
+
+__all__ = ["HttpErrors", "bearer_headers", "request_json"]
+
+# Snippet length for response bodies inside error messages.  Long enough
+# to be informative, short enough to keep audit logs scannable.
+_BODY_SNIPPET = 200
+
+Method = Literal["GET", "POST"]
+
+
+@dataclass(frozen=True)
+class HttpErrors:
+    """The three discriminable HTTP-transport error classes for a family.
+
+    Declared once at module scope per family (e.g.
+    ``_ERR = HttpErrors(VLMUnavailableError, VLMTimeoutError,
+    VLMResponseError)``) and threaded through :func:`request_json` so
+    the shared transport raises the right family-typed exception.
+    """
+
+    unavailable: type[BaseException]
+    timeout: type[BaseException]
+    response: type[BaseException]
+
+
+def bearer_headers(
+    api_key: str | None, *, extra: Mapping[str, str] | None = None
+) -> dict[str, str]:
+    """Build a header dict with optional ``Authorization: Bearer <key>``.
+
+    Returns an empty dict (plus any ``extra`` overrides) when ``api_key``
+    is falsy — matches the "open hosted Ollama" case where the header is
+    omitted entirely.
+    """
+    headers: dict[str, str] = {}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    if extra:
+        headers.update(extra)
+    return headers
+
+
+def _snippet(text: str | None) -> str:
+    return (text or "")[:_BODY_SNIPPET]
+
+
+def request_json(
+    method: Method,
+    url: str,
+    *,
+    timeout_s: float,
+    errors: HttpErrors,
+    label: str,
+    base_url: str | None = None,
+    json_body: Mapping[str, Any] | None = None,
+    files: Mapping[str, Any] | None = None,
+    data: Mapping[str, Any] | None = None,
+    headers: Mapping[str, str] | None = None,
+    api_key: str | None = None,
+    required_keys: Sequence[str] = (),
+    auth_to_unavailable: bool = True,
+    health_check: bool = False,
+) -> dict[str, Any]:
+    """Issue an HTTP request and return the parsed JSON object.
+
+    Args:
+        method: ``"GET"`` or ``"POST"``. POST is dispatched to
+            ``requests.post``; GET to ``requests.get`` — by name so test
+            ``patch("requests.post", ...)`` contracts survive.
+        url: Fully composed request URL.
+        timeout_s: Per-request HTTP budget.
+        errors: Family-specific :class:`HttpErrors` triad.
+        label: Human-readable name used in error messages
+            (e.g. ``"Ollama generate"``, ``"Mistral OCR"``).
+        base_url: Base URL shown in "Cannot connect to <label> at
+            <base_url>" — defaults to ``url`` when omitted.
+        json_body: Mapping to serialise as the JSON request body.
+        files: Multipart upload files (POST only).
+        data: Multipart form-data fields (POST only).
+        headers: Extra request headers. ``Authorization`` is set
+            automatically when ``api_key`` is provided.
+        api_key: Bearer token. ``None`` / empty omits the header.
+        required_keys: Top-level keys that MUST appear in the parsed
+            JSON; missing keys raise ``errors.response`` (or
+            ``errors.unavailable`` when ``health_check=True``).
+        auth_to_unavailable: When True (default), 401/403 responses are
+            raised as ``errors.unavailable`` ("API key rejected"). Set
+            False for endpoints without auth (local Ollama daemons).
+        health_check: Probe-mode toggle. When True, every non-success
+            failure — Timeout, non-2xx, malformed JSON, missing required
+            key — is raised as ``errors.unavailable`` ("not reachable" /
+            "unhealthy"). Use this for warmup probes; leave False for
+            body calls where Timeout vs Response is a meaningful
+            distinction for retry/back-off callers.
+
+    Returns:
+        The parsed top-level JSON object (always a ``dict``).
+
+    Raises:
+        errors.unavailable: connect refused / DNS failure / 401 / 403 /
+            generic ``RequestException``, or — with ``health_check=True``
+            — any other non-success.
+        errors.timeout: ``requests.Timeout`` on a body call
+            (``health_check=False``).
+        errors.response: non-2xx HTTP, malformed JSON, non-object JSON,
+            or a missing ``required_keys`` entry (``health_check=False``).
+    """
+    import requests  # noqa: PLC0415 — lazy: every model backend keeps `requests` optional
+
+    request_headers = bearer_headers(api_key, extra=headers)
+
+    kwargs: dict[str, Any] = {"headers": request_headers, "timeout": timeout_s}
+    if json_body is not None:
+        kwargs["json"] = dict(json_body)
+    if files is not None:
+        kwargs["files"] = dict(files)
+    if data is not None:
+        kwargs["data"] = dict(data)
+
+    base = base_url if base_url is not None else url
+    fn = requests.post if method == "POST" else requests.get
+
+    # In health-check mode, response/timeout failures collapse to the
+    # unavailable bucket. We pick the response-error class once and the
+    # body-validation branches reuse it.
+    body_error = errors.unavailable if health_check else errors.response
+
+    try:
+        resp = fn(url, **kwargs)
+    except requests.Timeout as exc:
+        if health_check:
+            raise errors.unavailable(
+                f"{label} at {base} did not respond within {timeout_s}s — is it reachable?"
+            ) from exc
+        raise errors.timeout(f"{label} exceeded {timeout_s}s budget at {url}") from exc
+    except requests.ConnectionError as exc:
+        raise errors.unavailable(f"Cannot connect to {label} at {base}: {exc}") from exc
+    except requests.RequestException as exc:
+        raise errors.unavailable(f"{label} request failed: {exc}") from exc
+
+    if auth_to_unavailable and resp.status_code in (401, 403):
+        raise errors.unavailable(
+            f"{label} API key rejected (HTTP {resp.status_code}): {_snippet(resp.text)}"
+        )
+    if not resp.ok:
+        raise body_error(f"HTTP {resp.status_code}: {_snippet(resp.text)}")
+
+    try:
+        payload = resp.json()
+    except ValueError as exc:
+        raise body_error(f"Malformed JSON from {label}: {_snippet(resp.text)}") from exc
+
+    if not isinstance(payload, dict):
+        raise body_error(f"{label} returned non-object JSON: {str(payload)[:_BODY_SNIPPET]}")
+
+    for key in required_keys:
+        if key not in payload:
+            raise body_error(
+                f"{label} response missing {key!r} key: {str(payload)[:_BODY_SNIPPET]}"
+            )
+
+    return payload

corpus_forge/_ml_device.py

@@ -0,0 +1,57 @@
+"""Shared device-detection helper for sentence-transformers-style backends.
+
+Four call sites used to roll the same MPS → CUDA → CPU heuristic by
+hand: :class:`SentenceTransformersEmbedder`, :class:`ClipLocalEmbedder`,
+:class:`LocalWhisper`, and :class:`CrossEncoderReranker`. They now all
+call :func:`detect_device`.
+
+The single subtlety is :class:`LocalWhisper`: ``faster-whisper`` does
+not yet support the MPS backend, so it disables the MPS branch via
+``prefer_mps=False``.
+
+``torch`` is imported lazily so this module is safe to import in
+environments where the ML stack isn't installed (it returns ``"cpu"``
+unconditionally in that case).
+"""
+
+from __future__ import annotations
+
+__all__ = ["detect_device", "resolve_device"]
+
+_AUTO = "auto"
+
+
+def detect_device(*, prefer_mps: bool = True) -> str:
+    """Pick the best available concrete device.
+
+    Args:
+        prefer_mps: When True (default), Apple Silicon's Metal backend
+            is preferred when available. Set False for libraries that
+            don't yet support MPS (faster-whisper).
+
+    Returns:
+        ``"mps"`` (when ``prefer_mps`` and available), ``"cuda"``, or
+        ``"cpu"``. Falls back to ``"cpu"`` when ``torch`` isn't
+        importable so callers can still run on hosts without the ML
+        stack installed.
+    """
+    try:
+        import torch  # noqa: PLC0415
+    except ImportError:
+        return "cpu"
+    if prefer_mps and torch.backends.mps.is_available():
+        return "mps"
+    if torch.cuda.is_available():
+        return "cuda"
+    return "cpu"
+
+
+def resolve_device(device: str, *, prefer_mps: bool = True) -> str:
+    """Translate the ``"auto"`` sentinel into a concrete device.
+
+    Any other value is returned unchanged so callers can pass through
+    user-specified ``"cpu"`` / ``"cuda"`` / ``"mps"`` strings.
+    """
+    if device == _AUTO:
+        return detect_device(prefer_mps=prefer_mps)
+    return device

corpus_forge/alembic/env.py

@@ -0,0 +1,120 @@
+"""Alembic migration environment.
+
+Dialect-aware configuration:
+- SQLite  → render_as_batch=True (no DDL transactional ALTER support)
+- Postgres → version_table_schema="corpus"
+
+All operator-facing messages go through the ``alembic.runtime.migration``
+logger (stderr).  No print() calls.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+log = logging.getLogger("alembic.runtime.migration")
+
+
+def _get_url() -> str:
+    """Resolve the database URL from env var or alembic.ini config.
+
+    Priority:
+    1. ``DATABASE_URL`` environment variable (honoured by the legacy migrator)
+    2. ``CORPUS_FORGE_DATABASE_URL`` environment variable
+    3. ``sqlalchemy.url`` from the Alembic Config object
+    """
+    url = os.environ.get("DATABASE_URL") or os.environ.get("CORPUS_FORGE_DATABASE_URL")
+    if url:
+        return url
+    cfg_url: str = context.config.get_main_option("sqlalchemy.url", default="")
+    return cfg_url
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL and not an Engine; calls to
+    context.execute() emit the given string to the script output.
+    """
+    url = _get_url()
+
+    # Determine whether this looks like a Postgres URL to set schema.
+    is_postgres = url.startswith(("postgresql", "postgres"))
+
+    configure_kwargs: dict = {
+        "url": url,
+        "target_metadata": None,
+        "literal_binds": True,
+        "dialect_opts": {"paramstyle": "named"},
+        "version_table": "alembic_version",
+    }
+    if is_postgres:
+        configure_kwargs["version_table_schema"] = "corpus"
+
+    with context.begin_transaction():
+        context.configure(**configure_kwargs)
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode (with an Engine/Connection)."""
+    creator = context.config.attributes.get("creator")
+    if creator is not None:
+        # In-memory SQLite: use the backend's shared-cache factory so Alembic
+        # operates on the same in-memory database as the SQLiteBackend instance.
+        connectable = engine_from_config(
+            {},
+            prefix="sqlalchemy.",
+            poolclass=pool.NullPool,
+            creator=creator,
+            # SQLite dialect is inferred from the creator connection.
+            url="sqlite+pysqlite://",
+        )
+    else:
+        connectable = engine_from_config(
+            context.config.get_section(context.config.config_ini_section, {}),
+            prefix="sqlalchemy.",
+            poolclass=pool.NullPool,
+            url=_get_url() or None,
+        )
+
+    with connectable.connect() as connection:
+        dialect_name: str = connection.dialect.name
+
+        if dialect_name == "sqlite":
+            context.configure(
+                connection=connection,
+                render_as_batch=True,
+                version_table="alembic_version",
+                target_metadata=None,
+            )
+        else:
+            # Postgres (and any other dialect) — use corpus schema for version table.
+            context.configure(
+                connection=connection,
+                version_table="alembic_version",
+                version_table_schema="corpus",
+                target_metadata=None,
+            )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+# Module body: only execute when invoked via Alembic CLI / programmatic runner.
+# Guarded so that a plain ``import corpus_forge.alembic.env`` (e.g. in tests)
+# does not attempt to run migrations without a configured Alembic context.
+try:
+    _offline = context.is_offline_mode()
+except Exception:
+    # Not running under Alembic's migration framework — plain import, skip.
+    pass
+else:
+    if _offline:
+        run_migrations_offline()
+    else:
+        run_migrations_online()

corpus_forge/alembic/script.py.mako

@@ -0,0 +1,30 @@
+"""${message}  # noqa: D
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+
+from __future__ import annotations
+
+from typing import Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, tuple[str, ...], None] = ${repr(branch_labels)}
+depends_on: Union[str, tuple[str, ...], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    """Apply forward migrations."""
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    """Apply reverse migrations."""
+    ${downgrades if downgrades else "pass"}