smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/verbs/embed.py

@@ -0,0 +1,166 @@
+"""The ``embed`` verb: turn each item into a vector (spec §3.3).
+
+The only verb that never touches a chat LLM — it uses the embedding model and
+emits one NDJSON record per item: ``{"text", "vector", "source"}``. Output is
+always NDJSON (a vector has no human view), so it feeds ``top_k`` or a file.
+
+Two execution shapes (plan/post-1.0/06, DEFER-3): a finite file corpus is
+embedded in ≤64-text chunks (64x fewer round-trips, poison chunks re-run
+item-by-item); a stream stays one item per call — latency beats throughput
+when lines arrive over time.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.engine.runner import Done, FailurePolicy, run_ordered
+from smartpipe.io import diagnostics, readers, tty
+from smartpipe.io.inputs import STDIN
+from smartpipe.io.items import describe_source
+from smartpipe.io.progress import make_stderr_spinner
+from smartpipe.io.writers import RenderMode, WriterConfig, make_writer
+from smartpipe.models.base import VideoData
+from smartpipe.verbs.common import (
+    embed_in_batches,
+    ensure_text,
+    interrupted_exit_code,
+    outcome_exit_code,
+)
+from smartpipe.verbs.convert import Converter, embed_video_halves, make_converter
+
+if TYPE_CHECKING:
+    from typing import TextIO
+
+    from smartpipe.io.inputs import InputSpec
+    from smartpipe.io.items import Item
+    from smartpipe.models.base import ChatModel, EmbeddingModel, ModelRef
+    from smartpipe.models.stt import RemoteTranscriber
+
+__all__ = ["EmbedContext", "EmbedRequest", "optional_chat", "run_embed"]
+
+
+@dataclass(frozen=True, slots=True)
+class EmbedRequest:
+    model_flag: str | None
+    concurrency_flag: int | None
+    allow_captions: bool = False  # cloud conversions opt-in (D33)
+    input: InputSpec = STDIN
+    fields: tuple[str, ...] | None = None  # --fields: project the {text, vector, source} records
+
+
+class EmbedContext(Protocol):
+    def remote_transcriber(self, chat_ref: ModelRef | None = None) -> RemoteTranscriber | None: ...
+    async def chat_model(self, flag: str | None = None) -> ChatModel: ...
+    async def embedding_model(self, flag: str | None = None) -> EmbeddingModel: ...
+    def concurrency(self, flag: int | None = None) -> int: ...
+
+
+async def run_embed(
+    request: EmbedRequest,
+    context: EmbedContext,
+    *,
+    stdin: TextIO,
+    stdout: TextIO,
+    stop: asyncio.Event | None = None,
+) -> ExitCode:
+    model = await context.embedding_model(request.model_flag)
+    concurrency = context.concurrency(request.concurrency_flag)
+
+    items_iter, total = readers.resolve_items(request.input, stdin, stop=stop)
+    if (total is None or total > 0) and tty.stdout_is_tty():
+        diagnostics.note(
+            "embeddings are large — redirect to a file: smartpipe embed > corpus.embeddings"
+        )
+    writer = make_writer(
+        WriterConfig(mode=RenderMode.NDJSON, color=False, width=80, fields=request.fields), stdout
+    )
+    spinner = make_stderr_spinner()
+    spinner.start(total=total)
+    log = diagnostics.DegradationLog()  # per-row conversion disclosure (D27)
+    converter_chat = await optional_chat(context)
+    converter = make_converter(
+        converter_chat,
+        allow_paid=request.allow_captions,
+        log=log,
+        stt=context.remote_transcriber(converter_chat.ref if converter_chat else None),
+    )
+
+    done = 0
+    skipped = 0
+    if total is not None:
+        # finite --in corpus: chunked calls, run_ordered bypassed on purpose —
+        # batching ≠ per-item workers (order from sequential chunks, isolation
+        # from the per-item fallback inside embed_in_batches)
+        collected = [item async for item in items_iter]
+        outcomes = embed_in_batches(
+            model,
+            collected,
+            failure_policy=FailurePolicy(),
+            stop=stop,
+            log=log,
+            converter=converter,
+        )
+    else:
+        # live stream: one item per call — latency beats throughput
+
+        async def worker(item: Item) -> tuple[Item, tuple[float, ...]]:
+            return await _embed_one(model, item, log, converter)
+
+        outcomes = run_ordered(
+            items_iter,
+            worker,
+            concurrency=concurrency,
+            failure_policy=FailurePolicy(),
+            stop=stop,
+        )
+    try:
+        async for outcome in outcomes:
+            if isinstance(outcome, Done):
+                item, vector = outcome.value
+                writer.write_record(
+                    {
+                        "text": item.text,
+                        "vector": list(vector),
+                        "source": item.source.name,
+                    }
+                )
+                done += 1
+            else:  # Skipped
+                diagnostics.warn(f"skipped: {describe_source(outcome.source)} ({outcome.reason})")
+                skipped += 1
+            spinner.advance()
+    finally:
+        spinner.finish()
+        writer.flush()
+        log.finish()
+    if stop is not None and stop.is_set():
+        diagnostics.interrupted_summary(processed=done, skipped=skipped)
+        return interrupted_exit_code(done=done, skipped=skipped)
+    return outcome_exit_code(done=done, skipped=skipped)
+
+
+async def optional_chat(context: EmbedContext) -> ChatModel | None:
+    """The converter's LLM rung — absent when no chat model is configured;
+    embedding never fails because chat isn't set up (D33)."""
+    try:
+        return await context.chat_model()
+    except Exception:
+        return None
+
+
+async def _embed_one(
+    model: EmbeddingModel,
+    item: Item,
+    log: diagnostics.DegradationLog,
+    converter: Converter,
+) -> tuple[Item, tuple[float, ...]]:
+    video = next((part for part in item.media if isinstance(part, VideoData)), None)
+    if video is not None and converter.chat is not None:
+        return await embed_video_halves(model, item, video, converter)  # 50/50 (D36)
+    item = await ensure_text(item, log=log, converter=converter)  # D33 ladder
+    vectors = await model.embed([item.text])
+    return item, vectors[0]  # the CONVERTED item — its text is what the vector means

smartpipe/verbs/extend.py

@@ -0,0 +1,180 @@
+"""The ``extend`` verb: enrich, don't replace (D38/02, KQL ``extend``).
+
+map's machinery with a merge at the emit edge: the input record's fields
+survive, the extracted fields land beside them. The verb every dataset owner
+(P1/P11/P12/P13) needs so results flow back into their existing pipelines.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Mapping
+from dataclasses import dataclass
+from functools import partial
+from typing import TYPE_CHECKING
+
+from smartpipe.core.errors import ExitCode, ItemError, UsageFault
+from smartpipe.engine.prompts import parse_prompt, plan_map, to_instruction
+from smartpipe.engine.runner import Done, FailurePolicy, run_ordered
+from smartpipe.engine.schema import load_schema
+from smartpipe.io import diagnostics, readers
+from smartpipe.io.inputs import STDIN
+from smartpipe.io.items import describe_source
+from smartpipe.io.progress import make_stderr_spinner
+from smartpipe.verbs.common import (
+    WindowGate,
+    interrupted_exit_code,
+    outcome_exit_code,
+    resolve_schema,
+)
+from smartpipe.verbs.map import MapContext, map_one
+
+if TYPE_CHECKING:
+    from pathlib import Path
+    from typing import TextIO
+
+    from smartpipe.io.inputs import InputSpec
+    from smartpipe.io.items import Item
+    from smartpipe.io.writers import OutputFormat
+
+__all__ = ["ExtendRequest", "base_fields", "run_extend"]
+
+_PROMPT_OVERHEAD_TOKENS = 500  # instruction + wrapper + reply headroom (map's)
+
+EXTEND_NEEDS_FIELDS = (
+    "extend adds fields — name them in braces or pass --schema\n"
+    '  Example: smartpipe extend "Add {sentiment enum(pos, neg, neutral)}"\n'
+    "  Plain-text transformation? That's map."
+)
+
+# media transport keys (D27/D32): consumed by the model, poison to re-emit
+_TRANSPORT_KEYS = frozenset({"audio_b64", "image_b64", "video_b64", "parts", "mime"})
+
+
+@dataclass(frozen=True, slots=True)
+class ExtendRequest:
+    prompt: str
+    schema_path: Path | None
+    model_flag: str | None
+    output: OutputFormat
+    concurrency_flag: int | None
+    input: InputSpec = STDIN
+    fields: tuple[str, ...] | None = None
+    schema_dsl: str | None = None
+    tally_field: str | None = None
+    explode_field: str | None = None
+    frame_every: float | None = None  # D43
+    max_frames: int | None = None  # D43
+
+
+async def run_extend(
+    request: ExtendRequest,
+    context: MapContext,
+    *,
+    stdin: TextIO,
+    stdout: TextIO,
+    stop: asyncio.Event | None = None,
+) -> ExitCode:
+    tokens = parse_prompt(request.prompt, allow_descriptions=True)
+    schema = resolve_schema(request.schema_path, request.schema_dsl, loader=load_schema)
+    plan = plan_map(tokens, schema=schema)
+    if plan.mode != "structured":
+        raise UsageFault(EXTEND_NEEDS_FIELDS)  # exit 64, zero model calls
+    instruction = to_instruction(tokens)
+    items_iter, total = readers.resolve_items(request.input, stdin, stop=stop)
+    model = await context.chat_model(request.model_flag)
+    writer = context.writer(request.output, structured=True, stdout=stdout, fields=request.fields)
+    concurrency = context.concurrency(request.concurrency_flag)
+
+    tally = None
+    if request.tally_field is not None:
+        from smartpipe.engine.tally import Tally
+
+        tally = Tally(request.tally_field)
+
+    spinner = make_stderr_spinner()
+    spinner.start(total=total)
+
+    log = diagnostics.DegradationLog()
+    gate = WindowGate(
+        provider=model.ref.provider,
+        model_name=model.ref.name,
+        overhead=_PROMPT_OVERHEAD_TOKENS,
+        window=partial(context.context_window, model.ref),
+    )
+
+    async def worker(item: Item) -> tuple[Item, Mapping[str, object]]:
+        budget = await gate.budget_for_oversized(item.text)
+        if budget is not None:
+            raise ItemError(gate.refusal(item.text, budget))  # D26: no silent chunking
+        result = await map_one(
+            model,
+            plan,
+            instruction,
+            item,
+            log,
+            frame_every=request.frame_every,
+            max_frames=request.max_frames,
+        )
+        assert isinstance(result, Mapping)  # structured mode: validated against the schema
+        return item, result
+
+    done = 0
+    skipped = 0
+    overwritten: set[str] = set()  # disclosed once per field
+    outcomes = run_ordered(
+        items_iter,
+        worker,
+        concurrency=concurrency,
+        failure_policy=FailurePolicy(),
+        stop=stop,
+    )
+    try:
+        async for outcome in outcomes:
+            if isinstance(outcome, Done):
+                item, extracted = outcome.value
+                base = base_fields(item)
+                for collision in sorted(base.keys() & extracted.keys()):
+                    if collision not in overwritten:
+                        overwritten.add(collision)
+                        diagnostics.note(f"overwriting '{collision}' on incoming records")
+                merged: dict[str, object] = {**base, **extracted}
+                for row in _rows(merged, request.explode_field):
+                    writer.write_record(row)
+                    if tally is not None:
+                        tally.add(row)
+                        spinner.extra = tally.live_segment()
+                done += 1
+            else:  # Skipped
+                diagnostics.warn(f"skipped: {describe_source(outcome.source)} ({outcome.reason})")
+                skipped += 1
+            spinner.advance()
+    finally:
+        spinner.finish()
+        writer.flush()
+        log.finish()
+    if tally is not None and tally.counts:
+        diagnostics.note(tally.final_line())
+    if stop is not None and stop.is_set():
+        diagnostics.interrupted_summary(processed=done, skipped=skipped)
+        return interrupted_exit_code(done=done, skipped=skipped)
+    return outcome_exit_code(done=done, skipped=skipped)
+
+
+def base_fields(item: Item) -> dict[str, object]:
+    """The record to enrich: its own fields, or {"text": …} for plain lines.
+    Media-transport payloads are dropped — consumed by the model, poison to
+    re-emit (megabytes of b64 in every output row)."""
+    if item.data is None:
+        return {"text": item.text}
+    if not item.media:
+        return dict(item.data)
+    return {key: value for key, value in item.data.items() if key not in _TRANSPORT_KEYS}
+
+
+def _rows(merged: Mapping[str, object], explode_field: str | None) -> list[Mapping[str, object]]:
+    if explode_field is None:
+        return [merged]
+    from smartpipe.engine.tally import explode_record
+
+    return list(explode_record(merged, explode_field))

smartpipe/verbs/filter.py

@@ -0,0 +1,191 @@
+"""The ``filter`` verb: semantic grep (spec §3.2).
+
+Judges each item against a natural-language condition and emits the items that
+match — byte-for-byte unchanged, in input order, a strict subset of the input.
+``--not`` inverts. Zero matches is success (exit 0), unlike grep: an empty result
+is a valid result.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, replace
+from functools import partial
+from typing import TYPE_CHECKING, Protocol
+
+from smartpipe.cli import screens
+from smartpipe.core.errors import ExitCode, ItemError, UsageFault
+from smartpipe.engine.chunking import split_text
+from smartpipe.engine.prompts import (
+    JUDGE_SCHEMA,
+    build_filter_request,
+    build_repair_request,
+    has_brace,
+    interpolate_fields,
+    parse_prompt,
+    reject_comma_groups,
+)
+from smartpipe.engine.runner import Done, FailurePolicy, run_ordered
+from smartpipe.engine.schema import validate_and_coerce
+from smartpipe.io import diagnostics, readers
+from smartpipe.io.inputs import STDIN
+from smartpipe.io.items import describe_source
+from smartpipe.io.progress import make_stderr_spinner
+from smartpipe.io.writers import OutputFormat
+from smartpipe.verbs.common import (
+    WindowGate,
+    ensure_text,
+    interrupted_exit_code,
+    outcome_exit_code,
+    prepend,
+)
+from smartpipe.verbs.convert import Converter, make_converter
+
+if TYPE_CHECKING:
+    from typing import TextIO
+
+    from smartpipe.engine.prompts import Token
+    from smartpipe.io.inputs import InputSpec
+    from smartpipe.io.items import Item
+    from smartpipe.io.writers import ResultWriter
+    from smartpipe.models.base import ChatModel, ModelRef
+    from smartpipe.models.stt import RemoteTranscriber
+
+__all__ = ["FilterContext", "FilterRequest", "run_filter"]
+
+_PROMPT_OVERHEAD_TOKENS = 500  # condition + judge wrapper headroom
+
+
+@dataclass(frozen=True, slots=True)
+class FilterRequest:
+    condition: str
+    invert: bool
+    model_flag: str | None
+    concurrency_flag: int | None
+    input: InputSpec = STDIN
+    allow_captions: bool = False  # cloud conversions opt-in (D33)
+
+
+class FilterContext(Protocol):
+    def remote_transcriber(self, chat_ref: ModelRef | None = None) -> RemoteTranscriber | None: ...
+    async def chat_model(self, flag: str | None = None) -> ChatModel: ...
+    async def context_window(self, ref: ModelRef) -> int | None: ...
+    def concurrency(self, flag: int | None = None) -> int: ...
+    def writer(
+        self, output_flag: OutputFormat, *, structured: bool, stdout: TextIO
+    ) -> ResultWriter: ...
+
+
+async def run_filter(
+    request: FilterRequest,
+    context: FilterContext,
+    *,
+    stdin: TextIO,
+    stdout: TextIO,
+    stop: asyncio.Event | None = None,
+) -> ExitCode:
+    tokens = parse_prompt(request.condition)  # UsageFault on bad grammar
+    reject_comma_groups(tokens)  # UsageFault: comma-braces are map-only
+    items_iter, total = readers.resolve_items(request.input, stdin, stop=stop)
+    model = await context.chat_model(request.model_flag)
+    writer = context.writer(OutputFormat.AUTO, structured=False, stdout=stdout)
+    concurrency = context.concurrency(request.concurrency_flag)
+
+    # First-item brace check (streaming can't see "all items" up front): the common
+    # mistake — braces over a plain-text pipe — still fails fast, before any model
+    # call; a mixed stream after a JSON first line skips per item instead.
+    first = await anext(items_iter, None)
+    if first is None:
+        if stop is not None and stop.is_set():
+            diagnostics.interrupted_summary(processed=0, skipped=0)
+            return interrupted_exit_code(done=0, skipped=0)
+        return ExitCode.OK
+    if has_brace(tokens) and first.data is None:
+        raise UsageFault(screens.FIELD_REF_ON_PLAIN_INPUT)  # exit 64, zero model calls
+    items_iter = prepend(first, items_iter)
+
+    spinner = make_stderr_spinner()
+    spinner.start(total=total)
+
+    log = diagnostics.DegradationLog()  # per-row conversion disclosure (D27)
+    converter = make_converter(
+        model, allow_paid=request.allow_captions, log=log, stt=context.remote_transcriber(model.ref)
+    )
+    gate = WindowGate(
+        provider=model.ref.provider,
+        model_name=model.ref.name,
+        overhead=_PROMPT_OVERHEAD_TOKENS,
+        window=partial(context.context_window, model.ref),
+    )
+
+    async def worker(item: Item) -> tuple[Item, bool]:
+        budget = await gate.budget_for_oversized(item.text)
+        if budget is None:
+            return item, await _judge(model, tokens, item, log, converter)
+        # D26: judge the chunks — any match keeps the whole item (--not inverts after)
+        for chunk in split_text(item.text, budget):
+            if await _judge(model, tokens, replace(item, text=chunk), log, converter):
+                return item, True
+        return item, False
+
+    judged = 0
+    matches = 0
+    skipped = 0
+    outcomes = run_ordered(
+        items_iter,
+        worker,
+        concurrency=concurrency,
+        failure_policy=FailurePolicy(),
+        stop=stop,
+    )
+    try:
+        async for outcome in outcomes:
+            if isinstance(outcome, Done):
+                judged += 1
+                item, matched = outcome.value
+                if matched:
+                    matches += 1
+                    spinner.matched = matches  # the status line's "N matched" segment
+                if matched is not request.invert:  # kept (or, with --not, dropped)
+                    _emit_match(writer, item)
+            else:  # Skipped
+                diagnostics.warn(f"skipped: {describe_source(outcome.source)} ({outcome.reason})")
+                skipped += 1
+            spinner.advance()
+    finally:
+        spinner.finish()
+        writer.flush()
+        log.finish()
+    if stop is not None and stop.is_set():
+        diagnostics.interrupted_summary(processed=judged, skipped=skipped)
+        return interrupted_exit_code(done=judged, skipped=skipped)
+    return outcome_exit_code(done=judged, skipped=skipped)
+
+
+def _emit_match(writer: ResultWriter, item: Item) -> None:
+    # In file mode the useful output is the filename, not the extracted document text
+    # (rank/keep files → get paths back, the Unix behavior — spec §8 / stage-07).
+    if item.source.kind == "file":
+        writer.write_text(item.source.name)
+    else:
+        writer.write_passthrough(item)
+
+
+async def _judge(
+    model: ChatModel,
+    tokens: tuple[Token, ...],
+    item: Item,
+    log: diagnostics.DegradationLog,
+    converter: Converter,
+) -> bool:
+    item = await ensure_text(item, log=log, converter=converter)  # D33 ladder
+    condition = interpolate_fields(tokens, item.data)  # ItemError → skip-and-warn
+    request = build_filter_request(condition, item.text)
+    reply = await model.complete(request)
+    try:
+        verdict = validate_and_coerce(reply, JUDGE_SCHEMA)
+    except ItemError as first_error:
+        repair = build_repair_request(request, bad_reply=reply, error=str(first_error))
+        repaired = await model.complete(repair)
+        verdict = validate_and_coerce(repaired, JUDGE_SCHEMA)  # second failure → Skipped
+    return bool(verdict["match"])

smartpipe/verbs/getschema.py

@@ -0,0 +1,135 @@
+"""The ``getschema`` verb: what's in this stream (D38/09, KQL ``getschema``).
+
+Everyone's first 30 seconds with a new file, answered for free: fields,
+types, coverage, an example each — and what to try next.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.io import diagnostics, tty
+from smartpipe.io.items import item_from_line
+
+if TYPE_CHECKING:
+    from typing import TextIO
+
+__all__ = ["GetSchemaRequest", "run_getschema"]
+
+_SCAN_CAP = 10_000
+_EXAMPLE_WIDTH = 24
+
+
+@dataclass(frozen=True, slots=True)
+class GetSchemaRequest:
+    scan_all: bool = False
+
+
+@dataclass(slots=True)
+class _FieldState:
+    types: set[str] = field(default_factory=set[str])
+    non_null: int = 0
+    example: str | None = None
+
+
+def run_getschema(request: GetSchemaRequest, *, stdin: TextIO, stdout: TextIO) -> ExitCode:
+    fields: dict[str, _FieldState] = {}
+    scanned = 0
+    plain_lengths: list[int] = []
+    capped = False
+    for index, line in enumerate(stdin):
+        if not line.strip():
+            continue
+        if not request.scan_all and scanned >= _SCAN_CAP:
+            capped = True
+            break
+        scanned += 1
+        item = item_from_line(line, index)
+        if item.data is None:
+            plain_lengths.append(len(item.text))
+            continue
+        for name, value in item.data.items():
+            state = fields.setdefault(name, _FieldState())
+            state.types.add(_type_name(value))
+            if value is not None:
+                state.non_null += 1
+                if state.example is None:
+                    state.example = _render_example(value)
+
+    if not fields:
+        lengths = sorted(plain_lengths)
+        median = lengths[len(lengths) // 2] if lengths else 0
+        stdout.write(
+            f"plain text lines (no fields) — {scanned:,} lines · median {median:,} chars\n"
+        )
+        diagnostics.note('try: smartpipe map "Extract {label}" · smartpipe cluster')
+        return ExitCode.OK
+
+    rows = [
+        {
+            "field": name,
+            "type": "|".join(sorted(state.types - {"null"}) or ["null"]),
+            "coverage": f"{round(100 * state.non_null / scanned)}%",
+            "example": state.example if state.example is not None else "",
+        }
+        for name, state in fields.items()
+    ]
+    if tty.stdout_is_tty():  # pragma: no cover — piped in tests; the table is trivial
+        widths = {
+            key: max(len(key), *(len(str(row[key])) for row in rows))
+            for key in ("field", "type", "coverage")
+        }
+        from smartpipe.cli.screens import heading
+
+        header = (
+            f"{'field'.ljust(widths['field'])}  {'type'.ljust(widths['type'])}  "
+            f"{'coverage'.ljust(widths['coverage'])}  example"
+        )
+        stdout.write(heading(header) + "\n")
+        for row in rows:
+            stdout.write(
+                f"{str(row['field']).ljust(widths['field'])}  "
+                f"{str(row['type']).ljust(widths['type'])}  "
+                f"{str(row['coverage']).ljust(widths['coverage'])}  {row['example']}\n"
+            )
+    else:
+        for row in rows:
+            stdout.write(json.dumps(row, separators=(",", ":")) + "\n")
+    if capped:
+        diagnostics.note(f"getschema: first {_SCAN_CAP:,} rows — --all scans everything")
+    best = max(
+        (name for name in fields if name != "text"),
+        key=lambda name: fields[name].non_null,
+        default=None,
+    )
+    if best is not None:
+        diagnostics.note(f"try: smartpipe chart {best} · smartpipe where '{best} …'")
+    return ExitCode.OK
+
+
+def _type_name(value: object) -> str:
+    match value:
+        case None:
+            return "null"
+        case bool():
+            return "boolean"
+        case int():
+            return "integer"
+        case float():
+            return "number"
+        case str():
+            return "string"
+        case list():
+            return "array"
+        case _:
+            return "object"
+
+
+def _render_example(value: object) -> str:
+    rendered = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+    if len(rendered) > _EXAMPLE_WIDTH:
+        return rendered[: _EXAMPLE_WIDTH - 1] + "…"
+    return rendered