smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/engine/chunking.py

@@ -0,0 +1,161 @@
+"""Chunking math for the recursive ``reduce`` (spec §3.5) — pure.
+
+The reduce tree "just works": when the input exceeds what the model can hold,
+smartpipe splits it into chunks, summarizes each, and recurses on the summaries —
+no flags, no strategy selection. This module owns the arithmetic (token estimate,
+context budget, item-boundary splitting); the verb drives the recursion, since
+each level's size isn't known until the model produces the notes.
+
+Token estimation is deliberately crude (≈4 chars/token) and used with a safety
+factor — being conservative just means more levels, never a truncated call.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+__all__ = [
+    "budget_for",
+    "chunk_indices",
+    "estimate_tokens",
+    "fits_in_one",
+    "halve",
+    "is_context_overflow",
+    "mean_pool",
+    "split_text",
+]
+
+# Conservative per-provider input windows; ollama is deliberately small because we
+# can't cheaply know a local model's context length (a smaller budget is always safe).
+_CONTEXT: dict[str, int] = {
+    "ollama": 8000,
+    "openai": 128000,
+    "anthropic": 200000,
+    "mistral": 128000,
+    "gemini": 128000,  # conservative: flash models carry ≥1M, but budget for the floor
+    "openrouter": 32000,  # unknowable per-model — the safe floor for routed models
+}
+_SAFETY = 0.6
+_CHARS_PER_TOKEN = 4
+
+
+def estimate_tokens(text: str) -> int:
+    return math.ceil(len(text) / _CHARS_PER_TOKEN) if text else 0
+
+
+def budget_for(provider: str, *, prompt_overhead: int, window: int | None = None) -> int:
+    """Token budget for one call. ``window`` (from a live probe or the
+    SMARTPIPE_CONTEXT_TOKENS override) beats the static table when present."""
+    resolved = window if window is not None else _CONTEXT.get(provider, _CONTEXT["ollama"])
+    return int(resolved * _SAFETY) - prompt_overhead
+
+
+_OVERFLOW_MARKERS = (
+    "context_length",
+    "context length",
+    "context window",
+    "maximum context",
+    "too long",
+    "too large",
+    "input length",
+    "prompt is too long",
+    "exceeds the limit",
+)
+
+
+def is_context_overflow(message: str) -> bool:
+    """Does this per-item error text look like a context-window overflow?
+
+    The classifier that lets reduce self-correct (D26): estimates are hints,
+    the wire's own rejection is ground truth — matched loosely on the phrases
+    the five wired providers actually use."""
+    lowered = message.lower()
+    return any(marker in lowered for marker in _OVERFLOW_MARKERS)
+
+
+def halve(chunk: tuple[int, ...]) -> tuple[tuple[int, ...], tuple[int, ...]]:
+    """Split a chunk of item indexes into two non-empty halves (len must be ≥ 2)."""
+    middle = len(chunk) // 2
+    return chunk[:middle], chunk[middle:]
+
+
+def fits_in_one(sizes: Sequence[int], budget: int) -> bool:
+    return sum(sizes) <= budget
+
+
+def chunk_indices(sizes: Sequence[int], budget: int) -> tuple[tuple[int, ...], ...]:
+    """Group consecutive item indexes into chunks whose total estimate fits the
+    budget. An item is never split; one that alone exceeds the budget gets its own
+    (over-budget) chunk rather than being dropped."""
+    chunks: list[tuple[int, ...]] = []
+    current: list[int] = []
+    current_size = 0
+    for index, size in enumerate(sizes):
+        if current and current_size + size > budget:
+            chunks.append(tuple(current))
+            current = []
+            current_size = 0
+        current.append(index)
+        current_size += size
+    if current:
+        chunks.append(tuple(current))
+    return tuple(chunks)
+
+
+def split_text(text: str, budget: int) -> tuple[str, ...]:
+    """Break one oversized text into ≤-budget chunks (D26 layer 3).
+
+    Paragraph boundaries first, then lines, then a hard character cut — and the
+    chunks concatenate back to the original text exactly (nothing added, nothing
+    lost; separators travel with their preceding piece)."""
+    if estimate_tokens(text) <= budget:
+        return (text,)
+    pieces = _carrying_separators(text, r"\n\n+")
+    if len(pieces) == 1:
+        pieces = _carrying_separators(text, r"\n")
+    chunks: list[str] = []
+    current = ""
+    for piece in pieces:
+        for part in _hard_cut(piece, budget):
+            if current and estimate_tokens(current + part) > budget:
+                chunks.append(current)
+                current = ""
+            current += part
+    if current:
+        chunks.append(current)
+    return tuple(chunks)
+
+
+def _carrying_separators(text: str, pattern: str) -> list[str]:
+    """Split by a separator regex, attaching each separator to the piece before
+    it, so ``"".join(result) == text``."""
+    import re
+
+    tokens = re.split(f"({pattern})", text)
+    pieces: list[str] = []
+    for token in tokens:
+        if pieces and re.fullmatch(pattern, token or " ") and token:
+            pieces[-1] += token
+        elif token:
+            pieces.append(token)
+    return pieces or [text]
+
+
+def _hard_cut(piece: str, budget: int) -> tuple[str, ...]:
+    """A single piece that alone exceeds the budget is cut at character bounds."""
+    limit = max(budget * _CHARS_PER_TOKEN, 1)
+    if len(piece) <= limit:
+        return (piece,)
+    return tuple(piece[i : i + limit] for i in range(0, len(piece), limit))
+
+
+def mean_pool(vectors: Sequence[tuple[float, ...]]) -> tuple[float, ...]:
+    """Component-wise mean of chunk embeddings — the standard whole-document
+    vector when one text had to be embedded in pieces (D26)."""
+    assert vectors, "mean_pool needs at least one vector"
+    count = len(vectors)
+    return tuple(sum(component) / count for component in zip(*vectors, strict=True))

smartpipe/engine/clustering.py

@@ -0,0 +1,94 @@
+"""Vector clustering for distinct/outliers/cluster/diff (D38) — pure, stdlib.
+
+Hand-rolled on purpose (frozen dependency budget: no sklearn, ever). Leader
+clustering is greedy and input-order stable: deterministic re-runs are a
+feature — the answer must not change under the user.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from smartpipe.engine.ranking import cosine
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+__all__ = ["adaptive_threshold", "knn_mean_distance", "leader_clusters", "merge_to_k"]
+
+
+def leader_clusters(vectors: Sequence[tuple[float, ...]], *, threshold: float) -> list[list[int]]:
+    """Greedy leader clustering: each vector joins the FIRST cluster whose
+    leader (its first member) is ≥ ``threshold`` cosine-similar, else founds
+    its own. Returns clusters as index lists, in founding order."""
+    clusters: list[list[int]] = []
+    for index, vector in enumerate(vectors):
+        for members in clusters:
+            if cosine(vectors[members[0]], vector) >= threshold:
+                members.append(index)
+                break
+        else:
+            clusters.append([index])
+    return clusters
+
+
+def knn_mean_distance(vectors: Sequence[tuple[float, ...]], *, k: int) -> list[float]:
+    """Mean cosine DISTANCE (1 - similarity) to each vector's k nearest
+    neighbors — the weirdness score for ``outliers``. Robust on multi-cluster
+    corpora where centroid distance lies. O(n²): fine at corpus sizes where a
+    human will read the answer."""
+    n = len(vectors)
+    neighbors = min(k, n - 1)
+    if neighbors <= 0:
+        return [0.0] * n
+    scores: list[float] = []
+    for index, vector in enumerate(vectors):
+        distances = sorted(
+            1.0 - cosine(vector, other)
+            for position, other in enumerate(vectors)
+            if position != index
+        )
+        scores.append(sum(distances[:neighbors]) / neighbors)
+    return scores
+
+
+def merge_to_k(
+    vectors: Sequence[tuple[float, ...]], clusters: list[list[int]], *, k: int
+) -> list[list[int]]:
+    """Force exactly ``k`` clusters: repeatedly merge the SMALLEST into the
+    cluster whose leader is most similar to its leader (ties: earliest).
+    Deterministic; returns a new list, founding order preserved."""
+    merged = [list(members) for members in clusters]
+    while len(merged) > max(1, k):
+        smallest = min(range(len(merged)), key=lambda index: (len(merged[index]), index))
+        leader = vectors[merged[smallest][0]]
+        best = max(
+            (index for index in range(len(merged)) if index != smallest),
+            key=lambda index: (cosine(vectors[merged[index][0]], leader), -index),
+        )
+        merged[best].extend(merged[smallest])
+        del merged[smallest]
+    return merged
+
+
+_PAIR_BUDGET = 20_000  # pairwise sample cap — clustering stays interactive
+
+
+def adaptive_threshold(vectors: Sequence[tuple[float, ...]]) -> float:
+    """A grouping threshold derived from the corpus itself: median pairwise
+    similarity (the cross-theme background) plus 35% of the gap to the 95th
+    percentile (the same-theme tail). Fixed thresholds can't serve every
+    embedder's geometry — gemini's same-theme pairs sit near 0.7 where
+    synthetic unit vectors sit near 1.0 (measured, D38/05). Deterministic
+    given the corpus; pairs are stride-sampled past the budget."""
+    n = len(vectors)
+    if n < 3:
+        return 0.99  # nothing to group — only near-identity folds
+    pairs = [(i, j) for i in range(n) for j in range(i + 1, n)]
+    if len(pairs) > _PAIR_BUDGET:
+        stride = len(pairs) // _PAIR_BUDGET + 1
+        pairs = pairs[::stride]
+    similarities = sorted(cosine(vectors[i], vectors[j]) for i, j in pairs)
+    median = similarities[len(similarities) // 2]
+    p95 = similarities[min(len(similarities) - 1, int(len(similarities) * 0.95))]
+    return median + 0.35 * (p95 - median)

smartpipe/engine/predicate.py

@@ -0,0 +1,330 @@
+"""The ``where`` predicate grammar (D38/01): a closed, KQL-flavored menu.
+
+Pure: parse once into a small AST, evaluate per item with zero I/O. Missing
+fields evaluate false (KQL behavior — streams keep flowing) but are tallied so
+the verb can disclose them at the end; silence must never lie.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Literal, assert_never
+
+from smartpipe.core.errors import UsageFault
+
+if TYPE_CHECKING:
+    from smartpipe.io.items import Item
+
+__all__ = [
+    "WHERE_MENU",
+    "Compare",
+    "Contains",
+    "FieldTally",
+    "Has",
+    "Matches",
+    "Predicate",
+    "evaluate",
+    "parse_predicate",
+]
+
+WHERE_MENU = (
+    "error: can't parse the where predicate\n"
+    '  Operators: FIELD has "word" · FIELD contains "text" · FIELD matches /re/\n'
+    '             FIELD == VALUE · != · > · >= · < · <=   (numbers or "strings")\n'
+    "  Combine:   and · or · not · ( )     FIELD is a record field, or text\n"
+    '  Example:   text has "ERROR" and not text contains "retry"\n'
+    "  Semantic condition instead? smartpipe filter judges with a model."
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Has:
+    field: str
+    word: str
+
+
+@dataclass(frozen=True, slots=True)
+class Contains:
+    field: str
+    text: str
+
+
+@dataclass(frozen=True, slots=True)
+class Matches:
+    field: str
+    pattern: re.Pattern[str]
+
+
+@dataclass(frozen=True, slots=True)
+class Compare:
+    field: str
+    op: Literal["==", "!=", ">", ">=", "<", "<="]
+    value: str | float
+
+
+@dataclass(frozen=True, slots=True)
+class And:
+    left: Predicate
+    right: Predicate
+
+
+@dataclass(frozen=True, slots=True)
+class Or:
+    left: Predicate
+    right: Predicate
+
+
+@dataclass(frozen=True, slots=True)
+class Not:
+    inner: Predicate
+
+
+Predicate = Has | Contains | Matches | Compare | And | Or | Not
+
+
+@dataclass(slots=True)
+class FieldTally:
+    """Mutable disclosure state owned by the verb: what silence would hide."""
+
+    missing: Counter[str] = field(default_factory=Counter[str])
+    non_numeric: Counter[str] = field(default_factory=Counter[str])
+
+
+# --- tokenizer --------------------------------------------------------------------
+
+_TOKEN = re.compile(
+    r"""\s*(?:
+        (?P<lparen>\()
+      | (?P<rparen>\))
+      | (?P<op>==|!=|>=|<=|>|<)
+      | (?P<string>"(?P<sbody>[^"]*)")
+      | (?P<sopen>")
+      | (?P<regex>/(?P<rbody>(?:\\/|[^/])*)/)
+      | (?P<number>-?\d+(?:\.\d+)?)
+      | (?P<word>[A-Za-z_][A-Za-z0-9_.]*)
+    )""",
+    re.VERBOSE,
+)
+
+_KEYWORDS = frozenset({"and", "or", "not", "has", "contains", "matches"})
+
+
+def _tokenize(text: str) -> list[tuple[str, str]]:
+    tokens: list[tuple[str, str]] = []
+    position = 0
+    while position < len(text):
+        match = _TOKEN.match(text, position)
+        if match is None:
+            remainder = text[position:].strip()
+            if not remainder:
+                break
+            raise UsageFault(WHERE_MENU + f"\n  (stuck at: {remainder[:30]!r})")
+        if match.lastgroup is None or match.end() == position:
+            break
+        kind = match.lastgroup
+        if kind == "sopen":
+            raise UsageFault(WHERE_MENU + "\n  (unclosed string literal)")
+        if kind == "string":
+            tokens.append(("string", match.group("sbody")))
+        elif kind == "regex":
+            tokens.append(("regex", match.group("rbody").replace("\\/", "/")))
+        elif kind == "word":
+            word = match.group("word")
+            tokens.append((word, word) if word in _KEYWORDS else ("field", word))
+        else:
+            tokens.append((kind, match.group(kind) or match.group(0).strip()))
+        position = match.end()
+    return tokens
+
+
+# --- recursive descent: or > and > not > leaf --------------------------------------
+
+
+class _Parser:
+    def __init__(self, tokens: list[tuple[str, str]]) -> None:
+        self.tokens = tokens
+        self.position = 0
+
+    def peek(self) -> tuple[str, str] | None:
+        return self.tokens[self.position] if self.position < len(self.tokens) else None
+
+    def take(self) -> tuple[str, str]:
+        token = self.peek()
+        if token is None:
+            raise UsageFault(WHERE_MENU + "\n  (predicate ends mid-expression)")
+        self.position += 1
+        return token
+
+    def parse_or(self) -> Predicate:
+        node = self.parse_and()
+        while (token := self.peek()) is not None and token[0] == "or":
+            self.take()
+            node = Or(node, self.parse_and())
+        return node
+
+    def parse_and(self) -> Predicate:
+        node = self.parse_not()
+        while (token := self.peek()) is not None and token[0] == "and":
+            self.take()
+            node = And(node, self.parse_not())
+        return node
+
+    def parse_not(self) -> Predicate:
+        if (token := self.peek()) is not None and token[0] == "not":
+            self.take()
+            return Not(self.parse_not())
+        return self.parse_leaf()
+
+    def parse_leaf(self) -> Predicate:
+        kind, value = self.take()
+        if kind == "lparen":
+            inner = self.parse_or()
+            closing = self.peek()
+            if closing is None or closing[0] != "rparen":
+                raise UsageFault(WHERE_MENU + "\n  (missing closing ')')")
+            self.take()
+            return inner
+        if kind != "field":
+            raise UsageFault(WHERE_MENU + f"\n  (expected a field name, got {value!r})")
+        return self.parse_operator(value)
+
+    def parse_operator(self, field_name: str) -> Predicate:
+        kind, value = self.take()
+        match kind:
+            case "has" | "contains":
+                text_kind, text_value = self.take()
+                if text_kind != "string":
+                    raise UsageFault(WHERE_MENU + f'\n  ({kind} needs a "quoted string")')
+                return (
+                    Has(field_name, text_value)
+                    if kind == "has"
+                    else Contains(field_name, text_value)
+                )
+            case "matches":
+                regex_kind, regex_value = self.take()
+                if regex_kind != "regex":
+                    raise UsageFault(WHERE_MENU + "\n  (matches needs /a regex/)")
+                try:
+                    return Matches(field_name, re.compile(regex_value))
+                except re.error as exc:
+                    raise UsageFault(WHERE_MENU + f"\n  (bad regex: {exc})") from exc
+            case "op":
+                literal_kind, literal_value = self.take()
+                if literal_kind == "number":
+                    return Compare(field_name, _as_op(value), float(literal_value))
+                if literal_kind == "string":
+                    return Compare(field_name, _as_op(value), literal_value)
+                if literal_kind == "field" and literal_value in ("true", "false"):
+                    # booleans compare as their JSON spelling
+                    return Compare(field_name, _as_op(value), literal_value)
+                raise UsageFault(WHERE_MENU + f'\n  ({value} needs a number or "string")')
+            case _:
+                raise UsageFault(WHERE_MENU + f"\n  (expected an operator after {field_name!r})")
+
+
+def _as_op(text: str) -> Literal["==", "!=", ">", ">=", "<", "<="]:
+    match text:
+        case "==" | "!=" | ">" | ">=" | "<" | "<=":
+            return text
+        case _ as unreachable:  # pragma: no cover — the tokenizer only emits the six
+            raise AssertionError(unreachable)
+
+
+def parse_predicate(text: str) -> Predicate:
+    parser = _Parser(_tokenize(text))
+    node = parser.parse_or()
+    leftover = parser.peek()
+    if leftover is not None:
+        raise UsageFault(WHERE_MENU + f"\n  (unexpected trailing {leftover[1]!r})")
+    return node
+
+
+# --- evaluation --------------------------------------------------------------------
+
+
+def evaluate(predicate: Predicate, item: Item, tally: FieldTally) -> bool:
+    match predicate:
+        case And(left, right):
+            return evaluate(left, item, tally) and evaluate(right, item, tally)
+        case Or(left, right):
+            return evaluate(left, item, tally) or evaluate(right, item, tally)
+        case Not(inner):
+            return not evaluate(inner, item, tally)
+        case Has(field_name, word):
+            value = _resolve(field_name, item, tally)
+            if value is None:
+                return False
+            boundary = rf"(?<![A-Za-z0-9_]){re.escape(word)}(?![A-Za-z0-9_])"
+            return re.search(boundary, _as_text(value), re.IGNORECASE) is not None
+        case Contains(field_name, needle):
+            value = _resolve(field_name, item, tally)
+            return value is not None and needle.lower() in _as_text(value).lower()
+        case Matches(field_name, pattern):
+            value = _resolve(field_name, item, tally)
+            return value is not None and pattern.search(_as_text(value)) is not None
+        case Compare(field_name, op, expected):
+            value = _resolve(field_name, item, tally)
+            if value is None:
+                return False
+            return _compare(field_name, value, op, expected, tally)
+        case _ as unreachable:  # pragma: no cover — pyright proves exhaustiveness
+            assert_never(unreachable)
+
+
+def _resolve(field_name: str, item: Item, tally: FieldTally) -> object | None:
+    if field_name == "text":
+        return item.text
+    value = item.data.get(field_name) if item.data is not None else None
+    if value is None:
+        tally.missing[field_name] += 1
+    return value
+
+
+def _as_text(value: object) -> str:
+    return value if isinstance(value, str) else str(value)
+
+
+def _numeric(value: object) -> float | None:
+    if isinstance(value, bool):  # bool is int; keep booleans out of arithmetic
+        return None
+    if isinstance(value, (int, float)):
+        return float(value)
+    if isinstance(value, str):
+        try:
+            return float(value)
+        except ValueError:
+            return None
+    return None
+
+
+def _compare(
+    field_name: str,
+    value: object,
+    op: Literal["==", "!=", ">", ">=", "<", "<="],
+    expected: str | float,
+    tally: FieldTally,
+) -> bool:
+    ours = _numeric(value)
+    theirs = _numeric(expected)
+    if ours is not None and theirs is not None:
+        match op:
+            case "==":
+                return ours == theirs
+            case "!=":
+                return ours != theirs
+            case ">":
+                return ours > theirs
+            case ">=":
+                return ours >= theirs
+            case "<":
+                return ours < theirs
+            case "<=":
+                return ours <= theirs
+    if op in ("==", "!="):
+        # string equality — booleans/None compare as their JSON spelling
+        rendered = _as_text(value) if not isinstance(value, bool) else str(value).lower()
+        return (rendered == _as_text(expected)) is (op == "==")
+    tally.non_numeric[field_name] += 1  # ordered compare on a non-number: no match
+    return False