smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/verbs/convert.py

@@ -0,0 +1,251 @@
+"""Modality → text conversion for embedding and text verbs (D27/D33).
+
+One ladder per modality, one cost fence: a LOCAL chat model converts for free,
+automatically; a cloud model converts only behind ``--allow-captions``; whisper
+is audio's always-there fallback. Every conversion is a per-row degraded note.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from smartpipe.core.errors import ItemError
+from smartpipe.engine.schema import validate_and_coerce
+from smartpipe.models.base import AudioData, CompletionRequest, ImageData, VideoData
+
+if TYPE_CHECKING:
+    from smartpipe.io.diagnostics import DegradationLog
+    from smartpipe.io.items import Item
+    from smartpipe.models.base import ChatModel, EmbeddingModel
+    from smartpipe.models.stt import RemoteTranscriber
+
+__all__ = ["IMAGE_NEEDS_CAPTION", "Converter", "embed_video_halves", "make_converter"]
+
+AUDIO_TO_TEXT_SYSTEM = (
+    "You convert audio to text for search indexing. If the audio contains "
+    "speech, transcribe it verbatim. If it does not, describe the sound in one "
+    "or two factual sentences. Reply with only the transcript or description."
+)
+
+IMAGE_TO_TEXT_SYSTEM = (
+    "You describe images for search indexing. Describe the image factually in "
+    "two or three sentences, including any visible text verbatim. Reply with "
+    "only the description."
+)
+
+VIDEO_TO_TEXT_SYSTEM = (
+    "You convert video to text for search indexing. Describe what is shown in "
+    "two or three factual sentences, then transcribe all speech verbatim. "
+    "Reply with only the description and transcript."
+)
+
+IMAGE_NEEDS_CAPTION = (
+    "image needs a description to embed — a local vision model does it free "
+    "(profile 'local'), or opt into paid captions: --allow-captions"
+)
+
+_CONVERT_MAX_TOKENS = 512
+_CAPTION_FRAMES = 4  # the fallback captions a handful, not a filmstrip
+
+_VIDEO_HALVES_SCHEMA: dict[str, object] = {
+    "type": "object",
+    "properties": {
+        "visual": {"type": "string", "description": "what is shown, 2-3 sentences"},
+        "transcript": {"type": "string", "description": "all speech verbatim; empty if none"},
+    },
+    "required": ["visual", "transcript"],
+    "additionalProperties": False,
+}
+
+
+@dataclass(frozen=True, slots=True)
+class Converter:
+    """The per-run conversion policy: which chat model (if any) may convert."""
+
+    chat: ChatModel | None  # None: no model resolvable — lower rungs only
+    allow_paid: bool  # --allow-captions
+    log: DegradationLog
+    stt: RemoteTranscriber | None = None  # the stt-model role (D39/05): verbatim rung 0
+
+    def _meter_paid(self) -> None:
+        from smartpipe.io import metering
+
+        if self.chat is None or self.chat.ref.provider != "ollama":
+            metering.add_conversion()  # cloud conversions bill; local ones don't
+
+    def _model_may_convert(self) -> bool:
+        if self.chat is None:
+            return False
+        return self.chat.ref.provider == "ollama" or self.allow_paid
+
+    def _rung_name(self) -> str:
+        assert self.chat is not None
+        return f"{self.chat.ref.provider}/{self.chat.ref.name}"
+
+    async def audio_to_text(self, audio: AudioData, where: str) -> str:
+        """stt-model rung (verbatim, consent-gated) → LLM rung → whisper →
+        the two-fix skip. A configured transcriber runs FIRST: whoever set it
+        wants verbatim, and LLM hearing paraphrases (D39/05)."""
+        if self.stt is not None and self.allow_paid:
+            try:
+                transcript = await self.stt.transcribe(audio)
+            except ItemError:
+                transcript = ""  # the wire hiccuped — the ladder continues below
+            if transcript:
+                self._meter_paid()
+                self.log.note(
+                    where,
+                    "audio → text",
+                    f"transcribed by {self.stt.ref.provider}/{self.stt.ref.name}",
+                )
+                return transcript
+        if self._model_may_convert():
+            assert self.chat is not None
+            try:
+                text = await self.chat.complete(
+                    CompletionRequest(
+                        system=AUDIO_TO_TEXT_SYSTEM,
+                        user="Convert this audio to text.",
+                        media=(audio,),
+                        max_tokens=_CONVERT_MAX_TOKENS,
+                        presence_penalty=0.5,  # prose call — anti-rambling (D35)
+                        frequency_penalty=0.5,
+                    )
+                )
+            except ItemError:
+                text = None  # the model can't hear — fall through to whisper
+            if text is not None and text.strip():
+                self._meter_paid()
+                self.log.note(where, "audio → text", f"heard by {self._rung_name()}")
+                return text.strip()
+        import asyncio
+        import os
+
+        from smartpipe.parsing.extract import whisper_size
+
+        transcript = await asyncio.to_thread(_whisper_or_skip, audio)
+
+        self.log.note(where, "audio → text", f"whisper {whisper_size(os.environ)}")
+        return transcript
+
+    async def video_halves(self, video: VideoData, where: str) -> tuple[str | None, str | None]:
+        """(visual, speech) — the two halves of a video's meaning (D36).
+
+        Rung 0: ONE watching call returns both via a response schema (gemini
+        native accepts; every other wire refuses pre-send, free). Rung 1: frame
+        captions + the track through the audio ladder. Rung 2: track only."""
+        if self._model_may_convert():
+            assert self.chat is not None
+            try:
+                reply = await self.chat.complete(
+                    CompletionRequest(
+                        system=VIDEO_TO_TEXT_SYSTEM,
+                        user="Convert this video to text.",
+                        media=(video,),
+                        json_schema=_VIDEO_HALVES_SCHEMA,
+                        max_tokens=_CONVERT_MAX_TOKENS,
+                    )
+                )
+                halves = validate_and_coerce(reply, _VIDEO_HALVES_SCHEMA)
+                visual = str(halves.get("visual") or "").strip() or None
+                speech = str(halves.get("transcript") or "").strip() or None
+                if visual or speech:
+                    self._meter_paid()
+                    self.log.note(where, "video → text", f"watched by {self._rung_name()}")
+                    return visual, speech
+            except ItemError:
+                pass  # this wire can't watch — the refusal cost nothing
+        import asyncio
+
+        from smartpipe.parsing.extract import video_to_parts
+
+        parts = await asyncio.to_thread(video_to_parts, video, max_frames=_CAPTION_FRAMES)
+        visual = None
+        if self._model_may_convert() and parts.frames:
+            captions = [await self.image_to_text(frame, where) for frame in parts.frames]
+            visual = "\n".join(
+                f"[scene {position}] {caption}"
+                for position, caption in enumerate(captions, start=1)
+            )
+        speech = None
+        if parts.track is not None:
+            speech = await self.audio_to_text(parts.track, where)
+        if visual is None and speech is None:
+            raise ItemError(
+                "this video has no audio track and no model can describe its "
+                "frames — map can still see them"
+            )
+        if visual is None:
+            self.log.note(
+                where, "video → text", "audio track only; frames dropped — map sees frames"
+            )
+        return visual, speech
+
+    async def image_to_text(self, image: ImageData, where: str) -> str:
+        """LLM rung or nothing — there is no free non-LLM rung for images."""
+        if not self._model_may_convert():
+            raise ItemError(IMAGE_NEEDS_CAPTION)
+        assert self.chat is not None
+        text = await self.chat.complete(
+            CompletionRequest(
+                system=IMAGE_TO_TEXT_SYSTEM,
+                user="Describe this image.",
+                media=(image,),
+                max_tokens=_CONVERT_MAX_TOKENS,
+                presence_penalty=0.5,  # prose call — anti-rambling (D35)
+                frequency_penalty=0.5,
+            )
+        )
+        self._meter_paid()
+        self.log.note(where, "image → text", f"described by {self._rung_name()}")
+        return text.strip()
+
+
+async def embed_video_halves(
+    model: EmbeddingModel,
+    item: Item,
+    video: VideoData,
+    converter: Converter,
+) -> tuple[Item, tuple[float, ...]]:
+    """D36: a video's vector is the FAIR AVERAGE of its two halves — the visual
+    description and the speech transcript embedded separately and mean-pooled
+    50/50, so neither drowns the other. Returns (converted item, vector)."""
+    from dataclasses import replace
+
+    from smartpipe.engine.chunking import mean_pool
+    from smartpipe.io.items import describe_source
+
+    visual, speech = await converter.video_halves(video, describe_source(item.source))
+    texts = [part for part in (visual, speech) if part]
+    vectors = await model.embed(texts)
+    vector = mean_pool(vectors)
+    converted = replace(item, text="\n\n".join(texts), media=())
+    return converted, vector
+
+
+AUDIO_NEEDS_TEXT = (
+    "audio items need text here — local transcription failed, "
+    "and no audio-capable model is configured (try map, or set stt-model)"
+)
+
+
+def _whisper_or_skip(audio: AudioData) -> str:
+    """Whisper with the pinned two-fix skip (self-contained: no common import —
+    verbs.common imports THIS module, and a cycle turns types Unknown)."""
+    from smartpipe.parsing.extract import MissingExtra, transcribe_audio
+
+    try:
+        return transcribe_audio(audio)
+    except MissingExtra as exc:
+        raise ItemError(AUDIO_NEEDS_TEXT) from exc
+
+
+def make_converter(
+    chat: ChatModel | None,
+    *,
+    allow_paid: bool,
+    log: DegradationLog,
+    stt: RemoteTranscriber | None = None,
+) -> Converter:
+    return Converter(chat=chat, allow_paid=allow_paid, log=log, stt=stt)

smartpipe/verbs/diff.py

@@ -0,0 +1,206 @@
+"""The ``diff`` verb: what distinguishes two sets of items (D38/06).
+
+KQL's ``diffpatterns`` for meaning: embed both sides, cluster the union,
+keep the lopsided themes. The post-incident "what changed" (P2), the eval
+regression story (P7), and dataset drift before the GPU bill (P12).
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, replace
+from typing import TYPE_CHECKING
+
+from smartpipe.core.errors import ExitCode, UsageFault
+from smartpipe.engine.chunking import mean_pool
+from smartpipe.engine.clustering import adaptive_threshold, leader_clusters
+from smartpipe.engine.ranking import cosine
+from smartpipe.engine.runner import Done, FailurePolicy
+from smartpipe.io import diagnostics, readers
+from smartpipe.io.inputs import STDIN
+from smartpipe.io.items import ItemSource, describe_source, item_from_line
+from smartpipe.io.writers import RenderMode, WriterConfig, make_writer
+from smartpipe.verbs.cluster import ClusterContext, label_cluster
+from smartpipe.verbs.common import embed_in_batches
+from smartpipe.verbs.convert import make_converter
+from smartpipe.verbs.embed import optional_chat
+
+if TYPE_CHECKING:
+    from pathlib import Path
+    from typing import TextIO
+
+    from smartpipe.io.items import Item
+
+__all__ = ["DiffRequest", "run_diff"]
+
+_MIN_LOPSIDEDNESS = 0.05  # below this a theme is "both sides" — omitted by default
+_EXAMPLES = 3
+
+
+@dataclass(frozen=True, slots=True)
+class DiffRequest:
+    right: Path
+    top: int | None = None
+    show_all: bool = False
+    model_flag: str | None = None  # chat, for theme labels
+    embed_flag: str | None = None
+    concurrency_flag: int | None = None
+    allow_captions: bool = False
+
+
+async def run_diff(
+    request: DiffRequest,
+    context: ClusterContext,
+    *,
+    stdin: TextIO,
+    stdout: TextIO,
+    stop: asyncio.Event | None = None,
+) -> ExitCode:
+    model = await context.embedding_model(request.embed_flag)
+    left_iter, _total = readers.resolve_items(STDIN, stdin, stop=stop)
+    left = [item async for item in left_iter]
+    right = _read_right(request.right)
+    if not left or not right:
+        raise UsageFault("diff needs items on BOTH sides — left is stdin, right is --right FILE")
+    boundary = len(left)
+    diagnostics.note(
+        f"diff: left = stdin ({len(left):,}) · right = {request.right.name} "
+        f"({len(right):,}) · ~{len(left) + len(right):,} embeddings + labels for "
+        "the lopsided themes"
+    )
+
+    log = diagnostics.DegradationLog()
+    converter_chat = await optional_chat(context)
+    converter = make_converter(
+        converter_chat,
+        allow_paid=request.allow_captions,
+        log=log,
+        stt=context.remote_transcriber(converter_chat.ref if converter_chat else None),
+    )
+    union_items: list[Item] = []
+    vectors: list[tuple[float, ...]] = []
+    outcomes = embed_in_batches(
+        model,
+        [*left, *right],
+        failure_policy=FailurePolicy(),
+        stop=stop,
+        log=log,
+        converter=converter,
+    )
+    position = 0
+    kept_positions: list[int] = []
+    async for outcome in outcomes:
+        if isinstance(outcome, Done):
+            embedded, vector = outcome.value
+            union_items.append(embedded)
+            vectors.append(vector)
+            kept_positions.append(position)
+        else:
+            diagnostics.warn(f"excluded: {describe_source(outcome.source)} ({outcome.reason})")
+        position += 1
+    log.finish()
+    if not vectors:
+        return ExitCode.ALL_FAILED
+
+    left_total = sum(1 for kept in kept_positions if kept < boundary)
+    right_total = len(kept_positions) - left_total
+    if left_total == 0 or right_total == 0:
+        raise UsageFault("diff needs embeddable items on both sides")
+
+    clusters = leader_clusters(vectors, threshold=adaptive_threshold(vectors))
+    themes = [
+        _theme(members, kept_positions, boundary, left_total, right_total) for members in clusters
+    ]
+    themes.sort(key=lambda theme: -theme.lopsidedness)
+
+    chat = None
+    lopsided = [theme for theme in themes if theme.side != "both"]
+    shown = lopsided if request.top is None else lopsided[: request.top]
+    if request.show_all:
+        shown = [*shown, *(theme for theme in themes if theme.side == "both")]
+    if shown:
+        if request.model_flag is not None:
+            chat = await context.chat_model(request.model_flag)
+        else:
+            chat = await optional_chat(context)
+
+    writer = make_writer(
+        WriterConfig(mode=RenderMode.NDJSON, color=False, width=80, fields=None), stdout
+    )
+    for number, theme in enumerate(shown, start=1):
+        if chat is not None:
+            texts = [union_items[member].text for member in theme.dominant_members[:8]]
+            label = await label_cluster(chat, texts, fallback=f"cluster {number}")
+        else:
+            label = f"cluster {number}"
+        writer.write_record(
+            {
+                "side": theme.side,
+                "theme": label,
+                "share_left": round(theme.share_left, 2),
+                "share_right": round(theme.share_right, 2),
+                "examples": _examples(theme.dominant_members, union_items, vectors),
+            }
+        )
+    writer.flush()
+    omitted = len(themes) - len(lopsided)
+    if omitted and not request.show_all:
+        diagnostics.note(f"diff: {omitted} shared theme(s) omitted — --all shows them")
+    if chat is None and shown:
+        diagnostics.note("no chat model — themes are unnamed; configure one: smartpipe config")
+    return ExitCode.OK
+
+
+@dataclass(frozen=True, slots=True)
+class _Theme:
+    side: str  # "left" | "right" | "both"
+    share_left: float
+    share_right: float
+    lopsidedness: float
+    dominant_members: tuple[int, ...]  # indices into union_items, dominant side first
+
+
+def _theme(
+    members: list[int],
+    kept_positions: list[int],
+    boundary: int,
+    left_total: int,
+    right_total: int,
+) -> _Theme:
+    left_members = [m for m in members if kept_positions[m] < boundary]
+    right_members = [m for m in members if kept_positions[m] >= boundary]
+    share_left = len(left_members) / left_total
+    share_right = len(right_members) / right_total
+    lopsidedness = abs(share_left - share_right)
+    dominant = left_members if share_left >= share_right else right_members
+    minority = right_members if share_left >= share_right else left_members
+    side = "both"
+    if lopsidedness >= _MIN_LOPSIDEDNESS and len(dominant) >= 2:
+        side = "left" if share_left > share_right else "right"
+    return _Theme(
+        side=side,
+        share_left=share_left,
+        share_right=share_right,
+        lopsidedness=lopsidedness,
+        dominant_members=(*dominant, *minority),
+    )
+
+
+def _examples(
+    members: tuple[int, ...], items: list[Item], vectors: list[tuple[float, ...]]
+) -> list[str]:
+    pool = list(members[: _EXAMPLES * 4])
+    centroid = mean_pool([vectors[member] for member in pool])
+    nearest = sorted(pool, key=lambda member: -cosine(vectors[member], centroid))
+    return [items[member].text[:160] for member in nearest[:_EXAMPLES]]
+
+
+def _read_right(path: Path) -> list[Item]:
+    if not path.exists():
+        raise UsageFault(f"no such file: {path}\n  --right needs a JSONL or plain-lines file.")
+    lines = path.read_text(encoding="utf-8").splitlines(keepends=True)
+    return [
+        replace(item_from_line(line, index), source=ItemSource("file", path.name, index))
+        for index, line in enumerate(lines)
+        if line.strip()
+    ]

smartpipe/verbs/distinct.py

@@ -0,0 +1,164 @@
+"""The ``distinct`` verb: near-duplicate folding (D38/03, KQL ``distinct``).
+
+Exact duplicates fold for free (hashing, before any embedding is spent); the
+rest embed once and leader-cluster. First occurrence wins, input order and
+bytes are preserved, and the receipt states exactly what was folded — the
+training-data decontamination move (P12) and the alert-storm collapser (P6).
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+from smartpipe.core.errors import ExitCode, UsageFault
+from smartpipe.engine.clustering import leader_clusters
+from smartpipe.engine.runner import Done, FailurePolicy
+from smartpipe.io import diagnostics, readers
+from smartpipe.io.inputs import STDIN
+from smartpipe.io.items import describe_source
+from smartpipe.io.writers import RenderMode, WriterConfig, make_writer
+from smartpipe.verbs.common import embed_in_batches
+from smartpipe.verbs.convert import make_converter
+from smartpipe.verbs.embed import optional_chat
+
+if TYPE_CHECKING:
+    from typing import TextIO
+
+    from smartpipe.io.inputs import InputSpec
+    from smartpipe.io.items import Item
+    from smartpipe.models.base import ChatModel, EmbeddingModel, ModelRef
+    from smartpipe.models.stt import RemoteTranscriber
+
+__all__ = ["DistinctRequest", "run_distinct"]
+
+_DEFAULT_THRESHOLD = 0.90
+
+
+@dataclass(frozen=True, slots=True)
+class DistinctRequest:
+    show_groups: bool = False
+    threshold: float = _DEFAULT_THRESHOLD
+    model_flag: str | None = None
+    concurrency_flag: int | None = None
+    allow_captions: bool = False
+    input: InputSpec = STDIN
+
+
+class DistinctContext(Protocol):
+    def remote_transcriber(self, chat_ref: ModelRef | None = None) -> RemoteTranscriber | None: ...
+    async def chat_model(self, flag: str | None = None) -> ChatModel: ...
+    async def embedding_model(self, flag: str | None = None) -> EmbeddingModel: ...
+    def concurrency(self, flag: int | None = None) -> int: ...
+
+
+async def run_distinct(
+    request: DistinctRequest,
+    context: DistinctContext,
+    *,
+    stdin: TextIO,
+    stdout: TextIO,
+    stop: asyncio.Event | None = None,
+) -> ExitCode:
+    if not 0.0 < request.threshold <= 1.0:
+        raise UsageFault("--threshold is a cosine similarity: between 0 and 1")
+    model = await context.embedding_model(request.model_flag)
+    items_iter, _total = readers.resolve_items(request.input, stdin, stop=stop)
+    items = [item async for item in items_iter]
+    if not items:
+        return ExitCode.OK
+
+    # exact fast path: identical text folds for free, before any embedding
+    first_of: dict[str, int] = {}
+    exact_dupes_of: dict[int, list[int]] = {}
+    uniques: list[Item] = []
+    unique_positions: list[int] = []
+    exact_folded = 0
+    for position, item in enumerate(items):
+        key = item.text.strip() if not item.media else f"\x00media:{position}"
+        seen_at = first_of.get(key)
+        if seen_at is not None:
+            exact_dupes_of.setdefault(seen_at, []).append(position)
+            exact_folded += 1
+            continue
+        first_of[key] = position
+        uniques.append(item)
+        unique_positions.append(position)
+
+    log = diagnostics.DegradationLog()
+    converter_chat = await optional_chat(context)
+    converter = make_converter(
+        converter_chat,
+        allow_paid=request.allow_captions,
+        log=log,
+        stt=context.remote_transcriber(converter_chat.ref if converter_chat else None),
+    )
+    vectors: dict[int, tuple[float, ...]] = {}  # original position → vector
+    unexamined: list[int] = []  # embed-skipped: kept, disclosed
+    outcomes = embed_in_batches(
+        model,
+        uniques,
+        failure_policy=FailurePolicy(),
+        stop=stop,
+        log=log,
+        converter=converter,
+    )
+    embed_order: list[int] = []  # positions, in outcome order
+    async for outcome in outcomes:
+        if isinstance(outcome, Done):
+            embedded_item, vector = outcome.value
+            del embedded_item
+            position = unique_positions[len(embed_order) + len(unexamined)]
+            vectors[position] = vector
+            embed_order.append(position)
+        else:  # Skipped: keep the item — never silently drop what we couldn't compare
+            position = unique_positions[len(embed_order) + len(unexamined)]
+            unexamined.append(position)
+            diagnostics.warn(
+                f"kept unexamined: {describe_source(outcome.source)} ({outcome.reason})"
+            )
+    log.finish()
+
+    clusters = leader_clusters([vectors[p] for p in embed_order], threshold=request.threshold)
+    kept: set[int] = set(unexamined)
+    near_dupes_of: dict[int, list[int]] = {}
+    near_folded = 0
+    for members in clusters:
+        leader_position = embed_order[members[0]]
+        kept.add(leader_position)
+        followers = [embed_order[m] for m in members[1:]]
+        near_dupes_of[leader_position] = followers
+        near_folded += len(followers)
+
+    total = len(items)
+    diagnostics.note(
+        f"distinct: kept {len(kept):,} of {total:,} "
+        f"({exact_folded:,} exact + {near_folded:,} near duplicates folded)"
+    )
+
+    if request.show_groups:
+        writer = make_writer(
+            WriterConfig(mode=RenderMode.NDJSON, color=False, width=80, fields=None), stdout
+        )
+        for position in sorted(kept):
+            duplicates = [
+                items[p].text
+                for p in (*near_dupes_of.get(position, ()), *exact_dupes_of.get(position, ()))
+            ]
+            # exact dupes of folded near-dupes belong to the group too
+            for follower in near_dupes_of.get(position, ()):
+                duplicates.extend(items[p].text for p in exact_dupes_of.get(follower, ()))
+            writer.write_record(
+                {
+                    "kept": items[position].text,
+                    "count": 1 + len(duplicates),
+                    "duplicates": duplicates,
+                }
+            )
+        writer.flush()
+        return ExitCode.OK
+
+    for position in sorted(kept):
+        stdout.write(items[position].raw + "\n")
+    return ExitCode.OK