smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/models/windows.py

@@ -0,0 +1,116 @@
+"""Dynamic context-window discovery (D26 layer 1).
+
+Four of the six wires publish their window; asking costs one cheap metadata GET,
+so the probe runs at most once per run, lazily, and only when the static table's
+budget already looks too small. A failed probe is never fatal: the conservative
+table stays the floor, and reduce's bisection is the backstop when everything lies.
+OpenAI and Anthropic don't expose window size via API; ``SMARTPIPE_CONTEXT_TOKENS``
+covers them (and overrides everything else).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from smartpipe.core.jsontools import as_items, as_record
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    import httpx
+
+    from smartpipe.models.base import ModelRef
+
+__all__ = ["probe_context_window"]
+
+_PROBE_TIMEOUT_S = 5.0
+
+
+async def probe_context_window(
+    ref: ModelRef, *, client: httpx.AsyncClient, env: Mapping[str, str]
+) -> int | None:
+    """The provider's own answer to "how big is this model's window?", or None."""
+    try:
+        match ref.provider:
+            case "ollama":
+                return await _ollama(ref.name, client, env)
+            case "mistral":
+                return await _mistral(ref.name, client, env)
+            case "openrouter":
+                return await _openrouter(ref.name, client, env)
+            case "gemini":
+                return await _gemini(ref.name, client, env)
+            case _:  # openai/anthropic publish no window via API
+                return None
+    except Exception:
+        return None
+
+
+async def _ollama(name: str, client: httpx.AsyncClient, env: Mapping[str, str]) -> int | None:
+    from smartpipe.models.ollama import resolve_host
+
+    response = await client.post(
+        f"{resolve_host(env)}/api/show", json={"model": name}, timeout=_PROBE_TIMEOUT_S
+    )
+    response.raise_for_status()
+    record = as_record(response.json())
+    info = as_record(record.get("model_info")) if record is not None else None
+    if info is None:
+        return None
+    # the key is architecture-prefixed: "llama.context_length", "qwen3.context_length", …
+    lengths = (value for key, value in info.items() if key.endswith(".context_length"))
+    first = next(lengths, None)
+    return first if isinstance(first, int) else None
+
+
+async def _mistral(name: str, client: httpx.AsyncClient, env: Mapping[str, str]) -> int | None:
+    from smartpipe.models.openai_compat import MISTRAL_WIRE, resolve_base_url
+
+    key = env.get(MISTRAL_WIRE.key_env, "").strip()
+    if not key:
+        return None
+    response = await client.get(
+        f"{resolve_base_url(env, MISTRAL_WIRE)}/v1/models/{name}",
+        headers={"Authorization": f"Bearer {key}"},
+        timeout=_PROBE_TIMEOUT_S,
+    )
+    response.raise_for_status()
+    record = as_record(response.json())
+    value = record.get("max_context_length") if record is not None else None
+    return value if isinstance(value, int) else None
+
+
+async def _openrouter(name: str, client: httpx.AsyncClient, env: Mapping[str, str]) -> int | None:
+    from smartpipe.models.openai_compat import OPENROUTER_WIRE, resolve_base_url
+
+    response = await client.get(
+        f"{resolve_base_url(env, OPENROUTER_WIRE)}/v1/models", timeout=_PROBE_TIMEOUT_S
+    )
+    response.raise_for_status()
+    record = as_record(response.json())
+    rows = as_items(record.get("data")) if record is not None else None
+    if rows is None:
+        return None
+    for row in rows:
+        entry = as_record(row)
+        if entry is not None and entry.get("id") == name:
+            value = entry.get("context_length")
+            return value if isinstance(value, int) else None
+    return None
+
+
+async def _gemini(name: str, client: httpx.AsyncClient, env: Mapping[str, str]) -> int | None:
+    from smartpipe.models.openai_compat import GEMINI_WIRE, resolve_base_url
+
+    key = env.get(GEMINI_WIRE.key_env, "").strip()
+    if not key:
+        return None
+    # the native endpoint lives one path segment above the OpenAI-compat root
+    native = resolve_base_url(env, GEMINI_WIRE).removesuffix("/openai")
+    response = await client.get(
+        f"{native}/models/{name}", headers={"x-goog-api-key": key}, timeout=_PROBE_TIMEOUT_S
+    )
+    response.raise_for_status()
+    record = as_record(response.json())
+    value = record.get("inputTokenLimit") if record is not None else None
+    return value if isinstance(value, int) else None

smartpipe/parsing/__init__.py

@@ -0,0 +1,10 @@
+"""File parsing: detect a file's kind, extract its text (spec §3.1, D08).
+
+The user never names a parser. ``detect`` sniffs the kind (pure, never raises);
+``extract`` turns it into text, lazy-importing the optional markitdown bridge only
+when a document actually needs it.
+"""
+
+from __future__ import annotations
+
+__all__: list[str] = []

smartpipe/parsing/detect.py

@@ -0,0 +1,178 @@
+"""Detect a file's kind (spec §3.1, stage-07 contract) — pure, never raises.
+
+Extension first, magic bytes as a backstop. Detection only *classifies*; extraction
+(and any missing-dependency handling) lives in ``extract``.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import TYPE_CHECKING, Literal
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+__all__ = ["FileKind", "Route", "audio_mime", "detect_kind", "route", "video_mime"]
+
+Route = Literal["text", "doc", "audio", "video", "image", "skip"]
+
+
+class FileKind(Enum):
+    TEXT = "text"
+    MARKDOWN = "markdown"
+    CSV = "csv"
+    JSON = "json"
+    PDF = "pdf"
+    DOCX = "docx"
+    XLSX = "xlsx"
+    PPTX = "pptx"
+    HTML = "html"
+    EPUB = "epub"
+    AUDIO = "audio"
+    VIDEO = "video"
+    IMAGE = "image"
+    UNKNOWN_BINARY = "unknown-binary"
+
+
+_BY_EXTENSION: dict[str, FileKind] = {
+    ".txt": FileKind.TEXT,
+    ".text": FileKind.TEXT,
+    ".log": FileKind.TEXT,
+    ".md": FileKind.MARKDOWN,
+    ".markdown": FileKind.MARKDOWN,
+    ".csv": FileKind.CSV,
+    ".tsv": FileKind.CSV,
+    ".json": FileKind.JSON,
+    ".jsonl": FileKind.JSON,
+    ".ndjson": FileKind.JSON,
+    ".pdf": FileKind.PDF,
+    ".docx": FileKind.DOCX,
+    ".xlsx": FileKind.XLSX,
+    ".pptx": FileKind.PPTX,
+    ".html": FileKind.HTML,
+    ".htm": FileKind.HTML,
+    ".epub": FileKind.EPUB,
+    ".mp3": FileKind.AUDIO,
+    ".wav": FileKind.AUDIO,
+    ".flac": FileKind.AUDIO,
+    ".m4a": FileKind.AUDIO,
+    ".ogg": FileKind.AUDIO,
+    ".mp4": FileKind.VIDEO,
+    ".mov": FileKind.VIDEO,
+    ".mkv": FileKind.VIDEO,
+    ".webm": FileKind.VIDEO,
+    ".png": FileKind.IMAGE,
+    ".jpg": FileKind.IMAGE,
+    ".jpeg": FileKind.IMAGE,
+    ".gif": FileKind.IMAGE,
+    ".webp": FileKind.IMAGE,
+}
+
+_ROUTES: dict[FileKind, Route] = {
+    FileKind.TEXT: "text",
+    FileKind.MARKDOWN: "text",
+    FileKind.CSV: "text",
+    FileKind.JSON: "text",
+    FileKind.PDF: "doc",
+    FileKind.DOCX: "doc",
+    FileKind.XLSX: "doc",
+    FileKind.PPTX: "doc",
+    FileKind.HTML: "doc",
+    FileKind.EPUB: "doc",
+    FileKind.AUDIO: "audio",
+    FileKind.VIDEO: "video",
+    FileKind.IMAGE: "image",
+    FileKind.UNKNOWN_BINARY: "skip",
+}
+
+
+_AUDIO_MIME_BY_SUFFIX = {
+    ".mp3": "audio/mpeg",
+    ".wav": "audio/wav",
+    ".flac": "audio/flac",
+    ".m4a": "audio/mp4",
+    ".ogg": "audio/ogg",
+}
+
+
+def audio_mime(path: Path) -> str:
+    """The wire mime for an audio file — suffix-driven, mp3 as the safe default."""
+    return _AUDIO_MIME_BY_SUFFIX.get(path.suffix.lower(), "audio/mpeg")
+
+
+def route(kind: FileKind) -> Route:
+    return _ROUTES[kind]
+
+
+def detect_kind(path: Path, head: bytes) -> FileKind:
+    by_ext = _BY_EXTENSION.get(path.suffix.lower())
+    if by_ext is not None:
+        return by_ext
+    return _sniff(head)
+
+
+def _sniff(head: bytes) -> FileKind:
+    if head.startswith(b"%PDF"):
+        return FileKind.PDF
+    if head.startswith(b"PK\x03\x04"):
+        return _sniff_zip(head)
+    if _is_audio(head):
+        return FileKind.AUDIO
+    if len(head) >= 12 and head[4:8] == b"ftyp":
+        return FileKind.VIDEO  # mp4/mov family
+    if _is_image(head):
+        return FileKind.IMAGE
+    if _is_utf8(head):
+        return FileKind.TEXT
+    return FileKind.UNKNOWN_BINARY
+
+
+def _sniff_zip(head: bytes) -> FileKind:
+    # OOXML/EPUB are zips; the inner directory names appear near the front
+    if b"word/" in head:
+        return FileKind.DOCX
+    if b"xl/" in head:
+        return FileKind.XLSX
+    if b"ppt/" in head:
+        return FileKind.PPTX
+    if b"epub" in head or b"mimetype" in head:
+        return FileKind.EPUB
+    return FileKind.UNKNOWN_BINARY
+
+
+def _is_audio(head: bytes) -> bool:
+    if head.startswith((b"ID3", b"fLaC", b"OggS")):
+        return True
+    if head.startswith(b"\xff\xfb") or head.startswith(b"\xff\xf3"):  # MP3 frame sync
+        return True
+    return head.startswith(b"RIFF") and head[8:12] == b"WAVE"
+
+
+def _is_image(head: bytes) -> bool:
+    if head.startswith((b"\x89PNG\r\n\x1a\n", b"\xff\xd8\xff", b"GIF87a", b"GIF89a")):
+        return True
+    return head.startswith(b"RIFF") and head[8:12] == b"WEBP"
+
+
+def _is_utf8(head: bytes) -> bool:
+    if b"\x00" in head:
+        return False  # NUL bytes never appear in text; a strong binary signal
+    try:
+        head.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        # tolerate an invalid byte only in the last 3 positions — a multibyte char
+        # truncated at the 8 KiB sniff boundary, not mid-stream binary
+        return exc.start >= len(head) - 3
+    return True
+
+
+_VIDEO_MIME_BY_SUFFIX = {
+    ".mp4": "video/mp4",
+    ".mov": "video/quicktime",
+    ".mkv": "video/x-matroska",
+    ".webm": "video/webm",
+}
+
+
+def video_mime(path: Path) -> str:
+    return _VIDEO_MIME_BY_SUFFIX.get(path.suffix.lower(), "video/mp4")