smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/core/errors.py

@@ -0,0 +1,57 @@
+"""Exit codes and the error taxonomy.
+
+Contract: plan/decisions.md D12 and plan/architecture.md "Error taxonomy".
+"""
+
+from __future__ import annotations
+
+from enum import IntEnum
+
+__all__ = [
+    "ExitCode",
+    "ItemError",
+    "SempipeError",
+    "SetupFault",
+    "TooManyFailures",
+    "UsageFault",
+]
+
+
+class ExitCode(IntEnum):
+    OK = 0
+    PARTIAL = 1
+    SETUP = 2
+    ALL_FAILED = 3
+    USAGE = 64
+    BUG = 70
+    INTERRUPTED = 130
+    PIPE_CLOSED = 141  # 128 + SIGPIPE: downstream closed the pipe (| head) — die silently
+
+
+class SempipeError(Exception):
+    """Base of all *expected* smartpipe failures. Never raised directly."""
+
+
+class UsageFault(SempipeError):
+    """Bad flags/arguments/grammar misuse → exit 64."""
+
+
+class SetupFault(SempipeError):
+    """No model, unreachable endpoint, missing key/extra, broken config → exit 2.
+
+    The message may be a full multi-line screen from plan/ux.md.
+    """
+
+
+class ItemError(SempipeError):
+    """One item failed; the runner turns this into a skip-and-warn, never a crash."""
+
+
+class TooManyFailures(SempipeError):
+    """The failure policy tripped (>50 % of items failed) → exit 3."""
+
+    def __init__(self, failed: int, total: int, last_reason: str) -> None:
+        self.failed = failed
+        self.total = total
+        self.last_reason = last_reason
+        super().__init__(f"stopping — {failed} of {total} items failed the same way")

smartpipe/core/jsontools.py

@@ -0,0 +1,56 @@
+"""Typed access into untrusted JSON (provider replies), without ``cast``.
+
+Each helper narrows ``object`` to a concrete shape or returns ``None`` — the
+caller decides whether a wrong shape is an ``ItemError`` or a bug.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, TypeGuard
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+
+__all__ = ["as_float_vector", "as_items", "as_record", "as_str", "record_at"]
+
+
+def _is_record(value: object) -> TypeGuard[Mapping[str, object]]:
+    return isinstance(value, dict)  # json.loads keys are str by contract
+
+
+def _is_items(value: object) -> TypeGuard[Sequence[object]]:
+    return isinstance(value, list)
+
+
+def as_record(value: object) -> Mapping[str, object] | None:
+    return value if _is_record(value) else None
+
+
+def as_items(value: object) -> Sequence[object] | None:
+    return value if _is_items(value) else None
+
+
+def as_str(value: object) -> str | None:
+    return value if isinstance(value, str) else None
+
+
+def as_float_vector(value: object) -> tuple[float, ...] | None:
+    items = as_items(value)
+    if items is None:
+        return None
+    vector: list[float] = []
+    for element in items:
+        if isinstance(element, bool) or not isinstance(element, int | float):
+            return None
+        vector.append(float(element))
+    return tuple(vector)
+
+
+def record_at(value: object, *keys: str) -> Mapping[str, object] | None:
+    """Walk nested objects: ``record_at(data, "message")`` → the message record."""
+    current = as_record(value)
+    for key in keys:
+        if current is None:
+            return None
+        current = as_record(current.get(key))
+    return current

smartpipe/engine/__init__.py

@@ -0,0 +1,9 @@
+"""The pure functional core: prompts, schema, chunking, ranking, ordering.
+
+Nothing in this package does I/O, reads the clock, or touches the environment —
+every input is a parameter. That is what makes it exhaustively testable.
+"""
+
+from __future__ import annotations
+
+__all__: list[str] = []

smartpipe/engine/aggregate.py

@@ -0,0 +1,234 @@
+"""The ``summarize`` grammar and fold (D38/07) — KQL's micro-syntax, pure.
+
+`count(), avg(total), p95(total) by region` parses once into a plan; records
+fold into per-group state in one pass. Percentile aggregations keep each
+group's numeric values (the one memory note); everything else streams.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Literal, assert_never
+
+from smartpipe.core.errors import UsageFault
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+__all__ = [
+    "SUMMARIZE_MENU",
+    "Aggregation",
+    "BinKey",
+    "GroupState",
+    "SummarizePlan",
+    "finish",
+    "fold",
+    "group_key",
+    "parse_summarize",
+]
+
+SUMMARIZE_MENU = (
+    "error: can't parse the summarize expression\n"
+    "  Shape:        AGG[, AGG…] [by FIELD[, FIELD…]]\n"
+    "  Aggregations: count() · sum(f) · avg(f) · min(f) · max(f)\n"
+    "                p50(f) · p90(f) · p95(f) · p99(f) · dcount(f)\n"
+    "  Example:      smartpipe summarize 'count(), avg(total), p95(total) by region'"
+)
+
+_FUNCTIONS = ("count", "sum", "avg", "min", "max", "p50", "p90", "p95", "p99", "dcount")
+_Fn = Literal["count", "sum", "avg", "min", "max", "p50", "p90", "p95", "p99", "dcount"]
+
+
+@dataclass(frozen=True, slots=True)
+class Aggregation:
+    fn: _Fn
+    field: str | None  # None only for count()
+    name: str  # KQL's output naming: count, avg_total, p95_total, dcount_user
+
+
+@dataclass(frozen=True, slots=True)
+class BinKey:
+    field: str
+    bucket_seconds: int
+    name: str  # "<field>_bin"
+
+
+@dataclass(frozen=True, slots=True)
+class SummarizePlan:
+    aggregations: tuple[Aggregation, ...]
+    by: tuple[str | BinKey, ...]
+
+    @property
+    def by_names(self) -> tuple[str, ...]:
+        return tuple(key if isinstance(key, str) else key.name for key in self.by)
+
+
+def parse_summarize(text: str) -> SummarizePlan:
+    head, _, tail = text.partition(" by ")
+    by = (
+        tuple(_parse_by(name.strip()) for name in _split_bins(tail) if name.strip()) if tail else ()
+    )
+    if tail and not by:
+        raise UsageFault(SUMMARIZE_MENU + "\n  ('by' needs at least one field)")
+    aggregations = tuple(_parse_agg(part.strip()) for part in head.split(",") if part.strip())
+    if not aggregations:
+        raise UsageFault(SUMMARIZE_MENU + "\n  (name at least one aggregation)")
+    return SummarizePlan(aggregations, by)
+
+
+def _split_bins(tail: str) -> list[str]:
+    """Split by-keys on commas outside parens — bin(ts, 1h) survives whole."""
+    parts: list[str] = []
+    depth = 0
+    current = ""
+    for char in tail:
+        if char == "(":
+            depth += 1
+        elif char == ")":
+            depth -= 1
+        if char == "," and depth == 0:
+            parts.append(current)
+            current = ""
+        else:
+            current += char
+    parts.append(current)
+    return parts
+
+
+def _parse_by(token: str) -> str | BinKey:
+    if not token.startswith("bin(") or not token.endswith(")"):
+        return token
+    inner = token[4:-1]
+    field_name, comma, bucket_text = inner.partition(",")
+    field_name = field_name.strip()
+    if not comma or not field_name:
+        raise UsageFault(SUMMARIZE_MENU + "\n  (bin needs bin(field, bucket) — e.g. bin(ts, 1h))")
+    from smartpipe.engine.timebin import parse_bucket
+
+    return BinKey(field_name, parse_bucket(bucket_text.strip()), f"{field_name}_bin")
+
+
+def _parse_agg(token: str) -> Aggregation:
+    name, paren, rest = token.partition("(")
+    name = name.strip()
+    if not paren or not rest.endswith(")"):
+        raise UsageFault(SUMMARIZE_MENU + f"\n  (stuck at: {token!r})")
+    if name not in _FUNCTIONS:
+        raise UsageFault(SUMMARIZE_MENU + f"\n  ({name!r} isn't an aggregation)")
+    inner = rest[:-1].strip()
+    if name == "count":
+        if inner:
+            raise UsageFault(SUMMARIZE_MENU + "\n  (count() takes no field)")
+        return Aggregation("count", None, "count")
+    if not inner:
+        raise UsageFault(SUMMARIZE_MENU + f"\n  ({name}() needs a field)")
+    fn: _Fn = name  # type: ignore[assignment]  # narrowed by the membership check
+    return Aggregation(fn, inner, f"{name}_{inner}")
+
+
+@dataclass(slots=True)
+class GroupState:
+    count: int = 0
+    sums: dict[str, float] = field(default_factory=dict[str, float])
+    mins: dict[str, float] = field(default_factory=dict[str, float])
+    maxes: dict[str, float] = field(default_factory=dict[str, float])
+    numeric_counts: dict[str, int] = field(default_factory=dict[str, int])
+    values: dict[str, list[float]] = field(default_factory=dict[str, "list[float]"])
+    distinct: dict[str, set[str]] = field(default_factory=dict[str, "set[str]"])
+    skipped_non_numeric: dict[str, int] = field(default_factory=dict[str, int])
+
+
+def fold(plan: SummarizePlan, state: GroupState, record: Mapping[str, object]) -> None:
+    state.count += 1
+    for aggregation in plan.aggregations:
+        if aggregation.field is None:
+            continue
+        value = record.get(aggregation.field)
+        if aggregation.fn == "dcount":
+            if value is not None:
+                state.distinct.setdefault(aggregation.field, set()).add(str(value))
+            continue
+        number = _numeric(value)
+        if number is None:
+            if value is not None:
+                state.skipped_non_numeric[aggregation.field] = (
+                    state.skipped_non_numeric.get(aggregation.field, 0) + 1
+                )
+            continue
+        state.numeric_counts[aggregation.field] = state.numeric_counts.get(aggregation.field, 0) + 1
+        state.sums[aggregation.field] = state.sums.get(aggregation.field, 0.0) + number
+        held = state.mins.get(aggregation.field)
+        state.mins[aggregation.field] = number if held is None else min(held, number)
+        held = state.maxes.get(aggregation.field)
+        state.maxes[aggregation.field] = number if held is None else max(held, number)
+        if aggregation.fn in ("p50", "p90", "p95", "p99"):
+            state.values.setdefault(aggregation.field, []).append(number)
+
+
+def finish(plan: SummarizePlan, key: tuple[object, ...], state: GroupState) -> dict[str, object]:
+    row: dict[str, object] = dict(zip(plan.by_names, key, strict=True))
+    for aggregation in plan.aggregations:
+        row[aggregation.name] = _value(aggregation, state)
+    return row
+
+
+def _value(aggregation: Aggregation, state: GroupState) -> object:
+    field_name = aggregation.field
+    match aggregation.fn:
+        case "count":
+            return state.count
+        case "dcount":
+            assert field_name is not None
+            return len(state.distinct.get(field_name, set()))
+        case "sum":
+            assert field_name is not None
+            return _round(state.sums.get(field_name)) if field_name in state.sums else None
+        case "avg":
+            assert field_name is not None
+            seen = state.numeric_counts.get(field_name, 0)
+            return _round(state.sums[field_name] / seen) if seen else None
+        case "min":
+            assert field_name is not None
+            return _round(state.mins.get(field_name))
+        case "max":
+            assert field_name is not None
+            return _round(state.maxes.get(field_name))
+        case "p50" | "p90" | "p95" | "p99":
+            assert field_name is not None
+            values = sorted(state.values.get(field_name, ()))
+            if not values:
+                return None
+            quantile = int(aggregation.fn[1:]) / 100
+            rank = min(len(values) - 1, max(0, round(quantile * (len(values) - 1))))
+            return _round(values[rank])
+        case _ as unreachable:  # pragma: no cover — pyright proves exhaustiveness
+            assert_never(unreachable)
+
+
+def _round(value: float | None) -> float | None:
+    if value is None:
+        return None
+    return round(value, 4)
+
+
+def _numeric(value: object) -> float | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)):
+        return float(value)
+    return None
+
+
+def group_key(plan: SummarizePlan, record: Mapping[str, object]) -> tuple[object, ...]:
+    """The record's group key: plain fields pass through; bin() keys become
+    UTC bucket labels (unparseable timestamps group under null, visibly)."""
+    from smartpipe.engine.timebin import bucket_label, parse_timestamp
+
+    parts: list[object] = []
+    for key in plan.by:
+        if isinstance(key, str):
+            parts.append(record.get(key))
+            continue
+        epoch = parse_timestamp(record.get(key.field))
+        parts.append(None if epoch is None else bucket_label(epoch, key.bucket_seconds))
+    return tuple(parts)

smartpipe/engine/blocking.py

@@ -0,0 +1,44 @@
+"""Candidate selection for ``join`` (D21) — the *block* of embed-block-judge.
+
+Pure math over an in-memory index: brute-force cosine, same envelope as
+``top_k`` (no vector database, no ANN — recorded non-goals; the corpus lives in
+the pipe). ``candidates`` is the recall knob's implementation: everything the
+judge never sees is a match the user never gets, so ties and ordering are pinned
+and property-tested.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from smartpipe.engine.ranking import rank, select
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+__all__ = ["RightIndex", "build_index", "candidates"]
+
+
+@dataclass(frozen=True, slots=True)
+class RightIndex:
+    """The build side, embedded once: position ↔ the caller's right-row payloads."""
+
+    vectors: tuple[tuple[float, ...], ...]
+
+    def __len__(self) -> int:
+        return len(self.vectors)
+
+
+def build_index(vectors: Sequence[Sequence[float]]) -> RightIndex:
+    return RightIndex(vectors=tuple(tuple(vector) for vector in vectors))
+
+
+def candidates(
+    query: Sequence[float], index: RightIndex, *, k: int, threshold: float | None
+) -> tuple[tuple[int, float], ...]:
+    """The top-``k`` right positions for one left vector, best-first, ties by
+    position; ``threshold`` filters before ``k`` (a floor, then a width)."""
+    if k < 1:
+        raise ValueError(f"k must be >= 1, got {k}")
+    return select(rank(query, index.vectors), k=k, threshold=threshold)

smartpipe/engine/chart.py

@@ -0,0 +1,143 @@
+"""Bar-chart rendering for ``smartpipe chart`` — pure, no plotting dependency.
+
+Terminal bars from block characters; ``--save`` writes a hand-rolled SVG (text,
+so it costs no dependency and converts to anything). Counts in, pictures out.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+__all__ = ["CHARTS_EXTRA_SCREEN", "render_bars", "render_svg", "render_svg_panels"]
+
+_BLOCK = "▇"
+_BAR_WIDTH = 40  # cells for the longest bar; the rest scale
+
+
+def render_bars(counts: Sequence[tuple[str, int]], *, width: int = _BAR_WIDTH) -> str:
+    """Horizontal unicode bars, widest value = ``width`` cells, labels aligned."""
+    if not counts:
+        return "(nothing to chart)"
+    label_width = max(len(label) for label, _ in counts)
+    top = max(count for _, count in counts) or 1
+    lines: list[str] = []
+    for label, count in counts:
+        cells = max(1, round(count / top * width)) if count else 0
+        lines.append(f"{label.ljust(label_width)}  {_BLOCK * cells} {count:,}")
+    return "\n".join(lines)
+
+
+_ROW_HEIGHT = 28
+_LABEL_SPACE = 180
+_CHART_WIDTH = 640
+
+CHARTS_EXTRA_SCREEN = (
+    "error: saving charts needs an optional dependency\n"
+    "  svgwrite ships with smartpipe — reinstall smartpipe"
+)
+
+
+def render_svg(counts: Sequence[tuple[str, int]], *, title: str | None = None) -> str:
+    """An SVG bar chart via svgwrite (the ``[charts]`` extra) — a library, not
+    bespoke markup (owner's call: no hand-rolled SVG to maintain)."""
+    try:
+        import svgwrite
+    except ImportError as exc:
+        from smartpipe.core.errors import SetupFault
+
+        raise SetupFault(CHARTS_EXTRA_SCREEN) from exc
+
+    rows = list(counts) or [("(nothing)", 0)]
+    top = max(count for _, count in rows) or 1
+    header = 36 if title else 8
+    height = header + len(rows) * _ROW_HEIGHT + 8
+    drawing = svgwrite.Drawing(size=(_CHART_WIDTH, height))
+    drawing.add(drawing.rect(insert=(0, 0), size=(_CHART_WIDTH, height), fill="white"))
+    text_style = {"font_family": "system-ui, sans-serif", "font_size": "13px"}
+    if title:
+        drawing.add(
+            drawing.text(
+                title,
+                insert=(12, 24),
+                font_weight="bold",
+                font_size="16px",
+                font_family="system-ui, sans-serif",
+            )
+        )
+    bar_space = _CHART_WIDTH - _LABEL_SPACE - 80
+    for row, (label, count) in enumerate(rows):
+        y = header + row * _ROW_HEIGHT
+        bar = max(2, round(count / top * bar_space)) if count else 0
+        drawing.add(
+            drawing.text(label, insert=(_LABEL_SPACE - 8, y + 18), text_anchor="end", **text_style)
+        )
+        drawing.add(
+            drawing.rect(insert=(_LABEL_SPACE, y + 6), size=(bar, 16), fill="#4477aa", rx=2)
+        )
+        drawing.add(
+            drawing.text(f"{count:,}", insert=(_LABEL_SPACE + bar + 6, y + 18), **text_style)
+        )
+    return drawing.tostring()
+
+
+def render_svg_panels(
+    panels: Sequence[tuple[str, Sequence[tuple[str, int]]]], *, title: str | None = None
+) -> str:
+    """Several bar panels stacked in one SVG (chart --facet). Same dependency
+    rules as render_svg: svgwrite behind the [charts] extra."""
+    try:
+        import svgwrite
+    except ImportError as exc:
+        from smartpipe.core.errors import SetupFault
+
+        raise SetupFault(CHARTS_EXTRA_SCREEN) from exc
+
+    header = 36 if title else 8
+    heights = [24 + max(1, len(counts)) * _ROW_HEIGHT + 8 for _name, counts in panels]
+    total_height = header + sum(heights)
+    drawing = svgwrite.Drawing(size=(_CHART_WIDTH, total_height))
+    drawing.add(drawing.rect(insert=(0, 0), size=(_CHART_WIDTH, total_height), fill="white"))
+    text_style = {"font_family": "system-ui, sans-serif", "font_size": "13px"}
+    if title:
+        drawing.add(
+            drawing.text(
+                title,
+                insert=(12, 24),
+                font_weight="bold",
+                font_size="16px",
+                font_family="system-ui, sans-serif",
+            )
+        )
+    offset = header
+    bar_space = _CHART_WIDTH - _LABEL_SPACE - 80
+    for (name, counts), panel_height in zip(panels, heights, strict=True):
+        drawing.add(
+            drawing.text(
+                name,
+                insert=(12, offset + 16),
+                font_weight="bold",
+                font_size="13px",
+                font_family="system-ui, sans-serif",
+            )
+        )
+        rows = list(counts) or [("(nothing)", 0)]
+        top = max(count for _label, count in rows) or 1
+        for row, (label, count) in enumerate(rows):
+            y = offset + 24 + row * _ROW_HEIGHT
+            bar = max(2, round(count / top * bar_space)) if count else 0
+            drawing.add(
+                drawing.text(
+                    label, insert=(_LABEL_SPACE - 8, y + 18), text_anchor="end", **text_style
+                )
+            )
+            drawing.add(
+                drawing.rect(insert=(_LABEL_SPACE, y + 6), size=(bar, 16), fill="#4477aa", rx=2)
+            )
+            drawing.add(
+                drawing.text(f"{count:,}", insert=(_LABEL_SPACE + bar + 6, y + 18), **text_style)
+            )
+        offset += panel_height
+    return drawing.tostring()