agent-skill-description-optimizer 0.2.0__py3-none-any.whl

skill_optimizer/interpreter.py

@@ -0,0 +1,171 @@
+"""Pure trigger-decision logic over a ``claude -p`` stream-json event sequence.
+
+Separated from subprocess transport so it can be tested exhaustively against
+synthetic and recorded event streams. The decision is made from tool-call *intent*,
+so it does not depend on tool execution.
+"""
+
+import json
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+# Tool names that, when invoked, may reference the injected candidate command.
+_SKILL_TOOLS = ("Skill", "Read")
+
+
+def _scan_assistant(message: Mapping[str, Any], cmd_name: str) -> bool | None:
+    """Decide from a non-streaming ``assistant`` message.
+
+    Args:
+        message: The assistant message object.
+        cmd_name: The injected command name to look for.
+
+    Returns:
+        ``True``/``False`` from the first ``tool_use`` item, or ``None`` if the
+        message contains no tool call.
+    """
+    tool_use = next(
+        (c for c in message.get("content", []) if c.get("type") == "tool_use"),
+        None,
+    )
+    if tool_use is None:
+        return None
+    if tool_use.get("name") not in _SKILL_TOOLS:
+        return False
+    return cmd_name in json.dumps(tool_use.get("input", {}))
+
+
+@dataclass(slots=True)
+class _ScanState:
+    """Mutable state for scanning a streamed tool call.
+
+    Attributes:
+        cmd_name: The injected command name to detect.
+        pending: Name of the in-progress skill tool, or ``None`` if none is open.
+        acc: Accumulated ``input_json_delta`` text for the pending tool.
+    """
+
+    cmd_name: str
+    pending: str | None = None
+    acc: str = ""
+
+    def feed_stream_event(self, stream: Mapping[str, Any]) -> bool | None:
+        """Advance the scan with one ``stream_event`` payload.
+
+        Args:
+            stream: The inner ``event`` object of a ``stream_event``.
+
+        Returns:
+            ``True``/``False`` once decided, or ``None`` to keep scanning.
+        """
+        stream_type = stream.get("type", "")
+        if stream_type == "content_block_start":
+            return self._on_block_start(stream.get("content_block", {}))
+        if stream_type == "content_block_delta":
+            return self._on_delta(stream.get("delta", {}))
+        if stream_type in ("content_block_stop", "message_stop"):
+            if self.pending is not None:
+                return self.cmd_name in self.acc
+            if stream_type == "message_stop":
+                return False
+        return None
+
+    def _on_block_start(self, block: Mapping[str, Any]) -> bool | None:
+        """Handle a ``content_block_start``.
+
+        Args:
+            block: The ``content_block`` object.
+
+        Returns:
+            ``False`` if a non-skill tool starts first, else ``None``.
+        """
+        if block.get("type") == "tool_use":
+            if block.get("name") in _SKILL_TOOLS:
+                self.pending = block.get("name")
+                self.acc = ""
+            else:
+                return False
+        return None
+
+    def _on_delta(self, delta: Mapping[str, Any]) -> bool | None:
+        """Handle a ``content_block_delta``, accumulating tool input.
+
+        Args:
+            delta: The ``delta`` object.
+
+        Returns:
+            ``True`` once ``cmd_name`` appears in the accumulated input, else
+            ``None``.
+        """
+        if self.pending is not None and delta.get("type") == "input_json_delta":
+            self.acc += delta.get("partial_json", "")
+            if self.cmd_name in self.acc:
+                return True
+        return None
+
+
+def _interpret_events_status(
+    events: Iterable[Mapping[str, Any]], cmd_name: str
+) -> bool | None:
+    """Decide the trigger outcome, distinguishing a decisive result from an
+    exhausted stream.
+
+    Consumes events lazily and returns as soon as the outcome is decided, so a
+    generator backed by a live subprocess can be abandoned early. The decision rules:
+
+    - The first ``tool_use`` block that is **not** ``Skill``/``Read`` means the model
+      went elsewhere first: ``False`` (even if a skill is used later).
+    - A ``Skill``/``Read`` ``tool_use`` whose streamed input contains ``cmd_name``
+      (accumulated across ``input_json_delta`` chunks): ``True``.
+    - A block/message stop while a skill tool is pending resolves to whether
+      ``cmd_name`` was seen in that block's input.
+    - A non-streaming ``assistant`` ``tool_use`` is judged on its full input.
+    - ``message_stop`` with nothing pending, or a ``result`` event: ``False``.
+
+    ``thinking`` and ``text`` content blocks are ignored.
+
+    Unlike :func:`interpret_events`, this returns ``None`` when the stream is exhausted
+    with **no** decisive terminal event (a timeout-truncated, empty, or otherwise
+    incomplete stream), so the caller can treat an unjudgeable probe differently from a
+    genuine non-trigger.
+
+    Args:
+        events: Parsed stream-json events, in order.
+        cmd_name: The injected command name to look for in tool input.
+
+    Returns:
+        ``True`` (decisive trigger), ``False`` (decisive non-trigger), or ``None``
+        (no decisive terminal event was observed).
+    """
+    state = _ScanState(cmd_name)
+    for event in events:
+        event_type = event.get("type")
+        if event_type == "stream_event":
+            decision = state.feed_stream_event(event.get("event", {}))
+        elif event_type == "assistant":
+            decision = _scan_assistant(event.get("message", {}), cmd_name)
+        elif event_type == "result":
+            decision = False
+        else:
+            decision = None
+        if decision is not None:
+            return decision
+    return None
+
+
+def interpret_events(events: Iterable[Mapping[str, Any]], cmd_name: str) -> bool:
+    """Decide whether a stream invokes the injected candidate command.
+
+    A thin, pure-bool wrapper over :func:`_interpret_events_status` that coerces the
+    "no decisive terminal event" case (``None``) to ``False``. See that function for
+    the full decision rules.
+
+    Args:
+        events: Parsed stream-json events, in order.
+        cmd_name: The injected command name to look for in tool input.
+
+    Returns:
+        ``True`` if the model invoked the candidate command, else ``False``.
+    """
+    return _interpret_events_status(events, cmd_name) is True

skill_optimizer/models.py

@@ -0,0 +1,155 @@
+"""Shared types and constants for the skill-description optimizer."""
+
+from dataclasses import dataclass
+from typing import TypedDict
+
+# Convenience aliases so callers can pass ``--models haiku,sonnet,opus``. These are
+# source-pinned to current tier IDs (verified against Anthropic's official model table);
+# any full model id also passes through unchanged. For reproducibility, pass an explicit
+# full ``--model``/``--models`` id rather than relying on these moving-target defaults.
+MODEL_ALIASES: dict[str, str] = {
+    "haiku": "claude-haiku-4-5-20251001",
+    "sonnet": "claude-sonnet-5",
+    "opus": "claude-opus-4-8",
+}
+
+
+class EvalQuery(TypedDict):
+    """A single evaluation query.
+
+    Attributes:
+        query: The natural-language task shown to the model.
+        should_trigger: Whether a correct skill description should fire for it.
+    """
+
+    query: str
+    should_trigger: bool
+
+
+# ``pass`` is a reserved word, so this member needs the functional TypedDict syntax.
+# ``pass`` is ``None`` when every probe for this ``(query, model)`` cell was
+# unjudgeable (timeout/CLI error), so the cell is excluded from accuracy denominators
+# rather than scored as a miss. ``triggers``/``runs`` are the judged-run counts (so a
+# cell with 2 judged + 1 error is representable), and ``errors`` is the unjudged count.
+ModelResult = TypedDict(
+    "ModelResult",
+    {
+        "trigger_rate": float,
+        "pass": bool | None,
+        "triggers": int,
+        "runs": int,
+        "errors": int,
+    },
+)
+"""Per-model outcome for one query: trigger rate, pass/fail/unjudged, and counts."""
+
+
+class PerQuery(TypedDict):
+    """Per-query roll-up across all evaluated models.
+
+    Attributes:
+        index: Original positional index into the full eval set (dedup-safe key used
+            by train/test slicing and the HTML report, never the query text).
+        query: The evaluated query text.
+        should_trigger: Whether a correct description should fire for it.
+        models: Per-model outcomes keyed by model id.
+        all_pass: ``True`` if every judged model passes, ``False`` if at least one
+            judged model fails, ``None`` if every model was unjudged for this query.
+    """
+
+    index: int
+    query: str
+    should_trigger: bool
+    models: dict[str, ModelResult]
+    all_pass: bool | None
+
+
+class ImproverAttempt(TypedDict):
+    """One previously-tried description plus its **train-only** per-query results.
+
+    Held-out (test) results are deliberately excluded so the improver never sees the
+    selection set (blinding), which would otherwise invite overfitting.
+
+    Attributes:
+        description: The description that was tried.
+        train_results: The training-set :data:`PerQuery` roll-ups for that attempt.
+    """
+
+    description: str
+    train_results: list[PerQuery]
+
+
+class ConfusionMatrix(TypedDict):
+    """Trigger confusion matrix over judged runs, with derived ratios.
+
+    Attributes:
+        tp: True positives (should-trigger runs that triggered).
+        fp: False positives (should-not-trigger runs that triggered).
+        tn: True negatives (should-not-trigger runs that stayed silent).
+        fn: False negatives (should-trigger runs that stayed silent).
+        precision: ``tp / (tp + fp)`` (``1.0`` when there are no positives predicted).
+        recall: ``tp / (tp + fn)`` (``1.0`` when there are no positives expected).
+        accuracy: ``(tp + tn) / total`` (``0.0`` when there are no judged runs).
+    """
+
+    tp: int
+    fp: int
+    tn: int
+    fn: int
+    precision: float
+    recall: float
+    accuracy: float
+
+
+class EvalResult(TypedDict):
+    """Aggregated evaluation of one description across queries and models.
+
+    Attributes:
+        description: The description that was evaluated.
+        per_model_accuracy: Accuracy in ``[0, 1]`` keyed by model id; ``None`` for a
+            model with zero judged queries in the (sub)set.
+        mean_accuracy: Mean of the non-``None`` ``per_model_accuracy`` values.
+        min_accuracy: Minimum (weakest-model) non-``None`` accuracy.
+        per_query: One :data:`PerQuery` entry per evaluation query.
+        errors: Total unjudged probe count across all cells.
+        unjudged: Number of unjudged ``query x model`` cells.
+        score_valid: ``False`` when no model had any judged query (accuracy unusable
+            for selection); ``True`` otherwise.
+        confusion: Aggregate confusion matrix over all judged runs.
+        per_model_confusion: Per-model confusion matrix keyed by model id.
+    """
+
+    description: str
+    per_model_accuracy: dict[str, float | None]
+    mean_accuracy: float
+    min_accuracy: float
+    per_query: list[PerQuery]
+    errors: int
+    unjudged: int
+    score_valid: bool
+    confusion: ConfusionMatrix
+    per_model_confusion: dict[str, ConfusionMatrix]
+
+
+@dataclass(frozen=True, slots=True)
+class EvalConfig:
+    """Settings shared across every query in an evaluation run.
+
+    Bundling these avoids threading the same long positional-argument list through
+    ``evaluate`` and its callers.
+
+    Attributes:
+        models: Resolved model ids to evaluate against.
+        repeats: Number of runs per ``(query, model)`` pair.
+        timeout: Per-run wall-clock budget, in seconds.
+        workers: Maximum concurrent ``claude -p`` subprocesses.
+        threshold: Trigger-rate at or above which a query counts as triggered.
+        settings_json: Optional ``--settings`` JSON blob, or ``None`` to omit it.
+    """
+
+    models: tuple[str, ...]
+    repeats: int
+    timeout: int
+    workers: int
+    threshold: float
+    settings_json: str | None = None

skill_optimizer/report.py

@@ -0,0 +1,297 @@
+"""HTML report generation, ported from skill-creator's ``generate_report.py``.
+
+Adapted to this tool's multi-model, tri-state history: query cells are anchored by the
+original positional ``index`` (dedup-safe), render ``pass`` as a tri-state
+(green/red/neutral, never a red cross for an unjudged probe), carry per-model detail in
+the cell ``title`` attribute, and highlight the winning row from the ``is_best`` flag
+(never recomputed from a score).
+"""
+
+import html
+from typing import Any
+
+_CSS = """
+        body {
+            font-family: 'Lora', Georgia, serif;
+            max-width: 100%;
+            margin: 0 auto;
+            padding: 20px;
+            background: #faf9f5;
+            color: #141413;
+        }
+        h1 { font-family: 'Poppins', sans-serif; color: #141413; }
+        .explainer, .summary {
+            background: white;
+            padding: 15px;
+            border-radius: 6px;
+            margin-bottom: 20px;
+            border: 1px solid #e8e6dc;
+        }
+        .explainer { color: #b0aea5; font-size: 0.875rem; line-height: 1.6; }
+        .summary p { margin: 5px 0; }
+        .best { color: #788c5d; font-weight: bold; }
+        .table-container { overflow-x: auto; width: 100%; }
+        table {
+            border-collapse: collapse;
+            background: white;
+            border: 1px solid #e8e6dc;
+            border-radius: 6px;
+            font-size: 12px;
+            min-width: 100%;
+        }
+        th, td {
+            padding: 8px;
+            text-align: left;
+            border: 1px solid #e8e6dc;
+            white-space: normal;
+            word-wrap: break-word;
+        }
+        th {
+            font-family: 'Poppins', sans-serif;
+            background: #141413;
+            color: #faf9f5;
+            font-weight: 500;
+        }
+        th.test-col { background: #6a9bcc; }
+        th.query-col { min-width: 200px; }
+        td.description {
+            font-family: monospace;
+            font-size: 11px;
+            word-wrap: break-word;
+            max-width: 400px;
+        }
+        td.result { text-align: center; font-size: 16px; min-width: 40px; }
+        td.test-result { background: #f0f6fc; }
+        .pass { color: #788c5d; }
+        .fail { color: #c44; }
+        .unjudged { color: #b0aea5; }
+        .rate { font-size: 9px; color: #b0aea5; display: block; }
+        tr:hover { background: #faf9f5; }
+        .score {
+            display: inline-block;
+            padding: 2px 6px;
+            border-radius: 4px;
+            font-weight: bold;
+            font-size: 11px;
+        }
+        .score-good { background: #eef2e8; color: #788c5d; }
+        .score-ok { background: #fef3c7; color: #d97706; }
+        .score-bad { background: #fceaea; color: #c44; }
+        .best-row { background: #f5f8f2; }
+        th.positive-col { border-bottom: 3px solid #788c5d; }
+        th.negative-col { border-bottom: 3px solid #c44; }
+        .legend {
+            font-family: 'Poppins', sans-serif;
+            display: flex;
+            gap: 20px;
+            margin-bottom: 10px;
+            font-size: 13px;
+            align-items: center;
+        }
+        .legend-item { display: flex; align-items: center; gap: 6px; }
+        .legend-swatch { width: 16px; height: 16px; border-radius: 3px; display: inline-block; }
+        .swatch-positive { background: #141413; border-bottom: 3px solid #788c5d; }
+        .swatch-negative { background: #141413; border-bottom: 3px solid #c44; }
+        .swatch-test { background: #6a9bcc; }
+        .swatch-train { background: #141413; }
+"""
+
+
+def _column_headers(results: list[dict[str, Any]], test: bool) -> str:
+    """Render ``<th>`` column headers for a set of query results.
+
+    Args:
+        results: The ordered result entries (from ``history[0]``) to build columns for.
+        test: Whether these are held-out (test) columns (adds the test styling).
+
+    Returns:
+        The concatenated header cells.
+    """
+    parts: list[str] = []
+    for r in results:
+        polarity = "positive-col" if r.get("should_trigger", True) else "negative-col"
+        cls = f"test-col {polarity}" if test else polarity
+        parts.append(
+            f'                <th class="{cls}">{html.escape(r["query"])}</th>\n'
+        )
+    return "".join(parts)
+
+
+def _aggregate_runs(results: list[dict[str, Any]]) -> tuple[int, int]:
+    """Sum correct runs and total runs across all query results.
+
+    Args:
+        results: The result entries to aggregate.
+
+    Returns:
+        A ``(correct, total)`` tuple of run counts.
+    """
+    correct = 0
+    total = 0
+    for r in results:
+        runs = r.get("runs", 0)
+        triggers = r.get("triggers", 0)
+        total += runs
+        correct += triggers if r.get("should_trigger", True) else runs - triggers
+    return correct, total
+
+
+def _score_class(correct: int, total: int) -> str:
+    """Map a correct/total ratio to a CSS score class.
+
+    Args:
+        correct: Correct run count.
+        total: Total run count.
+
+    Returns:
+        The CSS class name.
+    """
+    if total > 0:
+        ratio = correct / total
+        if ratio >= 0.8:
+            return "score-good"
+        if ratio >= 0.5:
+            return "score-ok"
+    return "score-bad"
+
+
+def _result_cell(r: dict[str, Any], test: bool) -> str:
+    """Render one query result cell, tri-state and per-model annotated.
+
+    Args:
+        r: The result entry (may be empty if the column has no matching index).
+        test: Whether this is a held-out (test) cell.
+
+    Returns:
+        The ``<td>`` cell HTML.
+    """
+    did_pass = r.get("pass")
+    triggers = r.get("triggers", 0)
+    runs = r.get("runs", 0)
+    if did_pass is True:
+        icon, css = "✓", "pass"
+    elif did_pass is False:
+        icon, css = "✗", "fail"
+    else:
+        # Unjudged (all probes errored): a neutral marker, never a red cross.
+        icon, css = "–", "unjudged"
+    per_model = r.get("models", {})
+    title = ", ".join(
+        f"{m}: {d.get('triggers', 0)}/{d.get('runs', 0)}" for m, d in per_model.items()
+    )
+    cls = f"result test-result {css}" if test else f"result {css}"
+    return (
+        f'                <td class="{cls}" title="{html.escape(title)}">{icon}'
+        f'<span class="rate">{triggers}/{runs}</span></td>\n'
+    )
+
+
+def _row(
+    h: dict[str, Any],
+    train_cols: list[dict[str, Any]],
+    test_cols: list[dict[str, Any]],
+) -> str:
+    """Render one iteration row.
+
+    Args:
+        h: The history entry for this iteration.
+        train_cols: The ordered training column definitions (from ``history[0]``).
+        test_cols: The ordered held-out column definitions (from ``history[0]``).
+
+    Returns:
+        The ``<tr>`` row HTML.
+    """
+    train_results = h.get("train_results", h.get("results", []))
+    test_results = h.get("test_results", [])
+    # Anchor cells by original positional index, never query text (dedup-safe).
+    train_by_index = {r["index"]: r for r in train_results}
+    test_by_index = {r["index"]: r for r in test_results}
+    train_correct, train_runs = _aggregate_runs(train_results)
+    test_correct, test_runs = _aggregate_runs(test_results)
+    row_class = "best-row" if h.get("is_best") else ""
+    cells = [
+        f'            <tr class="{row_class}">\n',
+        f"                <td>{h.get('iteration', '?')}</td>\n",
+        f'                <td><span class="score {_score_class(train_correct, train_runs)}">'
+        f"{train_correct}/{train_runs}</span></td>\n",
+        f'                <td><span class="score {_score_class(test_correct, test_runs)}">'
+        f"{test_correct}/{test_runs}</span></td>\n",
+        f'                <td class="description">{html.escape(h.get("description", ""))}</td>\n',
+    ]
+    cells.extend(
+        _result_cell(train_by_index.get(c["index"], {}), False) for c in train_cols
+    )
+    cells.extend(
+        _result_cell(test_by_index.get(c["index"], {}), True) for c in test_cols
+    )
+    cells.append("            </tr>\n")
+    return "".join(cells)
+
+
+def generate_html(
+    data: dict[str, Any], auto_refresh: bool = False, skill_name: str = ""
+) -> str:
+    """Generate the HTML optimization report from loop output data.
+
+    Args:
+        data: The report dict (the same superset written to stdout / ``results.json``).
+        auto_refresh: When ``True``, add a 5-second meta-refresh (for the live report).
+        skill_name: Skill name for the report title.
+
+    Returns:
+        The rendered HTML document.
+    """
+    history: list[dict[str, Any]] = data.get("history", [])
+    title_prefix = html.escape(f"{skill_name} — ") if skill_name else ""
+    first = history[0] if history else {}
+    train_cols: list[dict[str, Any]] = first.get(
+        "train_results", first.get("results", [])
+    )
+    test_cols: list[dict[str, Any]] = first.get("test_results", [])
+    refresh_tag = (
+        '    <meta http-equiv="refresh" content="5">\n' if auto_refresh else ""
+    )
+    best_test_score = data.get("best_test_score")
+    parts: list[str] = [
+        '<!DOCTYPE html>\n<html>\n<head>\n    <meta charset="utf-8">\n',
+        refresh_tag,
+        f"    <title>{title_prefix}Skill Description Optimization</title>\n",
+        f"    <style>{_CSS}    </style>\n</head>\n<body>\n",
+        f"    <h1>{title_prefix}Skill Description Optimization</h1>\n",
+        '    <div class="explainer"><strong>Optimizing your skill’s '
+        "description.</strong> Each row is a description attempt. Query columns show "
+        "test cases: a green check means the skill triggered correctly (or correctly "
+        "stayed silent), a red cross means it got it wrong, a grey dash means the probe "
+        "was unjudgeable. The best-performing row is highlighted.</div>\n",
+        '    <div class="summary">\n'
+        f"        <p><strong>Original:</strong> {html.escape(str(data.get('original_description', 'N/A')))}</p>\n"
+        f'        <p class="best"><strong>Best:</strong> {html.escape(str(data.get("best_description", "N/A")))}</p>\n'
+        f"        <p><strong>Best Score:</strong> {data.get('best_score', 'N/A')} "
+        f"{'(test)' if best_test_score else '(train)'}</p>\n"
+        f"        <p><strong>Iterations:</strong> {data.get('iterations_run', 0)} | "
+        f"<strong>Train:</strong> {data.get('train_size', '?')} | "
+        f"<strong>Test:</strong> {data.get('test_size', '?')}</p>\n"
+        "    </div>\n",
+        '    <div class="legend">\n'
+        '        <span style="font-weight:600">Query columns:</span>\n'
+        '        <span class="legend-item"><span class="legend-swatch swatch-positive">'
+        "</span> Should trigger</span>\n"
+        '        <span class="legend-item"><span class="legend-swatch swatch-negative">'
+        "</span> Should NOT trigger</span>\n"
+        '        <span class="legend-item"><span class="legend-swatch swatch-train">'
+        "</span> Train</span>\n"
+        '        <span class="legend-item"><span class="legend-swatch swatch-test">'
+        "</span> Test</span>\n"
+        "    </div>\n",
+        '    <div class="table-container">\n    <table>\n        <thead>\n'
+        "            <tr>\n"
+        "                <th>Iter</th>\n                <th>Train</th>\n"
+        '                <th>Test</th>\n                <th class="query-col">'
+        "Description</th>\n",
+        _column_headers(train_cols, False),
+        _column_headers(test_cols, True),
+        "            </tr>\n        </thead>\n        <tbody>\n",
+    ]
+    parts.extend(_row(h, train_cols, test_cols) for h in history)
+    parts.append("        </tbody>\n    </table>\n    </div>\n</body>\n</html>\n")
+    return "".join(parts)