smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/verbs/chart.py

@@ -0,0 +1,153 @@
+"""The ``chart`` verb: NDJSON in, a bar chart out. Free — no model calls.
+
+The chart IS the result, so it goes to stdout; ``--save`` additionally writes a
+dependency-free SVG. Counts a field across records (or tallies whole lines),
+which makes it the natural tail for the tools upstream:
+
+    … | smartpipe map "Extract {label}" | smartpipe chart label
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+from smartpipe.core.errors import ExitCode, UsageFault
+from smartpipe.engine.chart import render_bars, render_svg, render_svg_panels
+from smartpipe.io import diagnostics
+from smartpipe.io.items import item_from_line
+
+if TYPE_CHECKING:
+    from pathlib import Path
+    from typing import TextIO
+
+__all__ = ["ChartRequest", "run_chart"]
+
+_DEFAULT_TOP = 20
+
+
+@dataclass(frozen=True, slots=True)
+class ChartRequest:
+    field: str | None = None  # None: tally whole lines
+    top: int | None = None
+    save: Path | None = None
+    title: str | None = None
+    facets: tuple[str, ...] = ()  # --facet a,b,c: several panels, one pass
+    by_time: str | None = None  # --by-time FIELD:BUCKET — chronological bars (D38/13)
+
+
+class ChartContext(Protocol):
+    """chart needs nothing from the container — the Protocol keeps the shape."""
+
+
+def run_chart(request: ChartRequest, *, stdin: TextIO, stdout: TextIO) -> ExitCode:
+    if request.facets and request.field is not None:
+        raise UsageFault("--facet replaces the FIELD argument — pass one or the other")
+    if request.by_time is not None and (request.facets or request.field is not None):
+        raise UsageFault("--by-time replaces FIELD/--facet — pass one of the three")
+    if request.by_time is not None:
+        return _run_by_time(request, stdin=stdin, stdout=stdout)
+    if request.facets:
+        return _run_facets(request, stdin=stdin, stdout=stdout)
+    counts: Counter[str] = Counter()
+    for index, line in enumerate(stdin):
+        if not line.strip():
+            continue
+        item = item_from_line(line, index)
+        if request.field is not None:
+            value = item.data.get(request.field) if item.data is not None else None
+            counts["(missing)" if value is None else str(value)] += 1
+        else:
+            counts[item.text.strip()] += 1
+    ranked = counts.most_common(request.top or _DEFAULT_TOP)
+    dropped = len(counts) - len(ranked)
+    stdout.write(render_bars(ranked) + "\n")
+    if dropped > 0:
+        diagnostics.note(f"{dropped} more values below the top {len(ranked)} (--top widens)")
+    if request.save is not None:
+        request.save.write_text(
+            render_svg(ranked, title=request.title or request.field), encoding="utf-8"
+        )
+        diagnostics.note(f"chart saved: {request.save} (SVG — opens anywhere, converts to png)")
+    return ExitCode.OK
+
+
+def _run_facets(request: ChartRequest, *, stdin: TextIO, stdout: TextIO) -> ExitCode:
+    tallies: dict[str, Counter[str]] = {facet: Counter() for facet in request.facets}
+    for index, line in enumerate(stdin):
+        if not line.strip():
+            continue
+        item = item_from_line(line, index)
+        for facet, counter in tallies.items():
+            value = item.data.get(facet) if item.data is not None else None
+            counter["(missing)" if value is None else str(value)] += 1
+    limit = request.top or _DEFAULT_TOP
+    panels: list[tuple[str, list[tuple[str, int]]]] = []
+    for facet, counter in tallies.items():
+        ranked = counter.most_common(limit)
+        panels.append((facet, ranked))
+        dropped = len(counter) - len(ranked)
+        if dropped > 0:
+            diagnostics.note(f"{facet}: {dropped} more values below the top {len(ranked)}")
+    rule_width = 46
+    from smartpipe.cli.screens import heading
+
+    sections = [
+        heading(f"── {facet} {'─' * max(1, rule_width - len(facet) - 4)}")
+        + "\n"
+        + render_bars(ranked)
+        for facet, ranked in panels
+    ]
+    stdout.write("\n".join(sections) + "\n")
+    if request.save is not None:
+        request.save.write_text(render_svg_panels(panels, title=request.title), encoding="utf-8")
+        diagnostics.note(f"chart saved: {request.save} (SVG — opens anywhere, converts to png)")
+    return ExitCode.OK
+
+
+def _run_by_time(request: ChartRequest, *, stdin: TextIO, stdout: TextIO) -> ExitCode:
+    from smartpipe.engine.timebin import bucket_label, parse_bucket, parse_timestamp
+
+    assert request.by_time is not None
+    field, colon, bucket_text = request.by_time.partition(":")
+    if not colon or not field.strip():
+        raise UsageFault(
+            "--by-time takes FIELD:BUCKET — e.g. --by-time ts:1h\n"
+            "  Buckets: 1m · 5m · 15m · 1h · 6h · 1d"
+        )
+    bucket = parse_bucket(bucket_text)
+    field = field.strip()
+    counts: Counter[int] = Counter()
+    unparseable = 0
+    for index, line in enumerate(stdin):
+        if not line.strip():
+            continue
+        item = item_from_line(line, index)
+        value = item.data.get(field) if item.data is not None else None
+        epoch = parse_timestamp(value)
+        if epoch is None:
+            unparseable += 1
+            continue
+        counts[int(epoch // bucket) * bucket] += 1
+    if not counts:
+        stdout.write("(nothing to chart)\n")
+    else:
+        # chronological, zero-filled: gaps in a time series are signal
+        first, last = min(counts), max(counts)
+        rows = [
+            (bucket_label(float(moment), bucket), counts.get(moment, 0))
+            for moment in range(first, last + bucket, bucket)
+        ]
+        stdout.write(render_bars(rows) + "\n")
+        if request.save is not None:
+            request.save.write_text(
+                render_svg(rows, title=request.title or field), encoding="utf-8"
+            )
+            diagnostics.note(f"chart saved: {request.save} (SVG — opens anywhere, converts to png)")
+    if unparseable:
+        diagnostics.note(
+            f"{unparseable:,} rows with unparseable '{field}' — "
+            "ISO-8601 or epoch only; preprocess with jq/date"
+        )
+    return ExitCode.OK

smartpipe/verbs/cluster.py

@@ -0,0 +1,220 @@
+"""The ``cluster`` verb: group by meaning, label each group (D38/05).
+
+KQL's ``autocluster``/``reduce by`` done semantically: N embeddings + one
+label call per cluster — never N chat calls. The Monday slide (P3), the
+phishing lure families (P6), and the qualitative codebook (P10) as one verb.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+from smartpipe.core.errors import ExitCode, ItemError, UsageFault
+from smartpipe.engine.chunking import mean_pool
+from smartpipe.engine.clustering import adaptive_threshold, leader_clusters, merge_to_k
+from smartpipe.engine.ranking import cosine
+from smartpipe.engine.runner import Done, FailurePolicy
+from smartpipe.engine.schema import validate_and_coerce
+from smartpipe.io import diagnostics, readers
+from smartpipe.io.inputs import STDIN
+from smartpipe.io.items import describe_source
+from smartpipe.io.writers import RenderMode, WriterConfig, make_writer
+from smartpipe.models.base import ChatModel, CompletionRequest
+from smartpipe.verbs.common import embed_in_batches
+from smartpipe.verbs.convert import make_converter
+from smartpipe.verbs.embed import optional_chat
+
+if TYPE_CHECKING:
+    from typing import TextIO
+
+    from smartpipe.io.inputs import InputSpec
+    from smartpipe.io.items import Item
+    from smartpipe.models.base import EmbeddingModel, ModelRef
+    from smartpipe.models.stt import RemoteTranscriber
+
+__all__ = ["ClusterRequest", "label_cluster", "run_cluster"]
+
+_EXAMPLES = 3
+
+_LABEL_SCHEMA: dict[str, object] = {
+    "type": "object",
+    "properties": {
+        "label": {"type": "string", "description": "3-6 words naming what unites the items"}
+    },
+    "required": ["label"],
+    "additionalProperties": False,
+}
+
+_LABEL_SYSTEM = (
+    "You name clusters of similar items. Reply with a short, specific label "
+    "(3-6 words) capturing what unites them — content, not form."
+)
+
+
+@dataclass(frozen=True, slots=True)
+class ClusterRequest:
+    k: int | None = None
+    top: int | None = None
+    explode: str | None = None  # "members" is the only value
+    model_flag: str | None = None  # chat, for labels
+    embed_flag: str | None = None
+    concurrency_flag: int | None = None
+    allow_captions: bool = False
+    input: InputSpec = STDIN
+
+
+class ClusterContext(Protocol):
+    def remote_transcriber(self, chat_ref: ModelRef | None = None) -> RemoteTranscriber | None: ...
+    async def chat_model(self, flag: str | None = None) -> ChatModel: ...
+    async def embedding_model(self, flag: str | None = None) -> EmbeddingModel: ...
+    def concurrency(self, flag: int | None = None) -> int: ...
+
+
+async def run_cluster(
+    request: ClusterRequest,
+    context: ClusterContext,
+    *,
+    stdin: TextIO,
+    stdout: TextIO,
+    stop: asyncio.Event | None = None,
+) -> ExitCode:
+    if request.explode is not None and request.explode != "members":
+        raise UsageFault("--explode takes exactly 'members' (one row per input item)")
+    if request.k is not None and request.k < 1:
+        raise UsageFault("--k needs a positive cluster count")
+    model = await context.embedding_model(request.embed_flag)
+    items_iter, _total = readers.resolve_items(request.input, stdin, stop=stop)
+    items = [item async for item in items_iter]
+    if not items:
+        return ExitCode.OK
+    diagnostics.note(
+        f"cluster: ~{len(items):,} embeddings + one label call per cluster (typically < 20)"
+    )
+
+    log = diagnostics.DegradationLog()
+    converter_chat = await optional_chat(context)
+    converter = make_converter(
+        converter_chat,
+        allow_paid=request.allow_captions,
+        log=log,
+        stt=context.remote_transcriber(converter_chat.ref if converter_chat else None),
+    )
+    clustered_items: list[Item] = []
+    vectors: list[tuple[float, ...]] = []
+    outcomes = embed_in_batches(
+        model,
+        items,
+        failure_policy=FailurePolicy(),
+        stop=stop,
+        log=log,
+        converter=converter,
+    )
+    async for outcome in outcomes:
+        if isinstance(outcome, Done):
+            embedded, vector = outcome.value
+            clustered_items.append(embedded)
+            vectors.append(vector)
+        else:
+            diagnostics.warn(f"excluded: {describe_source(outcome.source)} ({outcome.reason})")
+    log.finish()
+    if not vectors:
+        return ExitCode.ALL_FAILED
+
+    # the threshold adapts to the embedder's geometry (measured: gemini's
+    # same-theme pairs sit near 0.7; a fixed bar can't serve every model)
+    clusters = leader_clusters(vectors, threshold=adaptive_threshold(vectors))
+    if request.k is not None and len(clusters) > request.k:
+        clusters = merge_to_k(vectors, clusters, k=request.k)
+    clusters = sorted(clusters, key=len, reverse=True)
+
+    labels = await _label_clusters(context, request, clusters, clustered_items)
+
+    writer = make_writer(
+        WriterConfig(mode=RenderMode.NDJSON, color=False, width=80, fields=None), stdout
+    )
+    if request.explode == "members":
+        member_label: dict[int, str] = {}
+        for members, label in zip(clusters, labels, strict=True):
+            for member in members:
+                member_label[member] = label
+        for position, item in enumerate(clustered_items):
+            record: dict[str, object]
+            if item.data is not None:
+                record = {key: value for key, value in item.data.items() if key != "vector"}
+            else:
+                record = {"text": item.raw}
+            record["cluster"] = member_label[position]
+            writer.write_record(record)
+        writer.flush()
+        return ExitCode.OK
+
+    total = len(clustered_items)
+    shown = clusters if request.top is None else clusters[: request.top]
+    for members, label in zip(shown, labels[: len(shown)], strict=True):
+        writer.write_record(
+            {
+                "cluster": label,
+                "size": len(members),
+                "share": round(len(members) / total, 2),
+                "examples": _examples(members, clustered_items, vectors),
+            }
+        )
+    folded = clusters[len(shown) :]
+    if folded:
+        size = sum(len(members) for members in folded)
+        writer.write_record(
+            {"cluster": "(other)", "size": size, "share": round(size / total, 2), "examples": []}
+        )
+    writer.flush()
+    return ExitCode.OK
+
+
+async def _label_clusters(
+    context: ClusterContext,
+    request: ClusterRequest,
+    clusters: list[list[int]],
+    items: list[Item],
+) -> list[str]:
+    if request.model_flag is not None:
+        chat: ChatModel | None = await context.chat_model(request.model_flag)
+    else:
+        chat = await optional_chat(context)
+    if chat is None:
+        diagnostics.note(
+            "no chat model — clusters are unnamed (cluster 1, 2, …); "
+            "configure one for real labels: smartpipe config"
+        )
+        return [f"cluster {number}" for number in range(1, len(clusters) + 1)]
+    labels: list[str] = []
+    for number, members in enumerate(clusters, start=1):
+        texts = [items[member].text for member in members[:8]]
+        labels.append(await label_cluster(chat, texts, fallback=f"cluster {number}"))
+    return labels
+
+
+async def label_cluster(chat: ChatModel, texts: list[str], *, fallback: str) -> str:
+    """One temperature-0 label call for a group of similar texts (shared by
+    cluster and diff). Failures keep the fallback, disclosed."""
+    quotes = "\n".join(f"- {text[:200]}" for text in texts)
+    try:
+        reply = await chat.complete(
+            CompletionRequest(
+                system=_LABEL_SYSTEM,
+                user=f"Items in this cluster:\n{quotes}",
+                json_schema=_LABEL_SCHEMA,
+                max_tokens=64,
+            )
+        )
+        verdict = validate_and_coerce(reply, _LABEL_SCHEMA)
+        return str(verdict["label"]).strip() or fallback
+    except ItemError as exc:
+        diagnostics.warn(f"{fallback} label failed ({exc}) — kept numbered")
+        return fallback
+
+
+def _examples(members: list[int], items: list[Item], vectors: list[tuple[float, ...]]) -> list[str]:
+    centroid = mean_pool([vectors[member] for member in members])
+    nearest = sorted(members, key=lambda member: -cosine(vectors[member], centroid))
+    return [items[member].text[:160] for member in nearest[:_EXAMPLES]]