agent-skill-description-optimizer 0.2.0__py3-none-any.whl

skill_optimizer/selection.py

@@ -0,0 +1,250 @@
+"""Eval-set splitting, model resolution, and candidate-selection helpers."""
+
+import math
+import random
+
+from skill_optimizer.models import MODEL_ALIASES, EvalQuery, EvalResult
+
+
+def stratified_split(
+    eval_set: list[EvalQuery], test_frac: float, seed: int = 42
+) -> tuple[list[int], list[int]]:
+    """Split query *indices* into train/test, preserving the trigger/no-trigger ratio.
+
+    A ``test_frac`` of zero disables the held-out set (skill-creator's ``--holdout 0``
+    semantics): every index becomes a training index, in original order, and no class
+    balancing is required. A positive fraction must be finite and in ``[0, 1)``, each
+    trigger class must hold at least two queries, and the seeded per-class allocation
+    must leave at least one training and one held-out query in each class; otherwise a
+    :class:`ValueError` is raised before any split is produced.
+
+    Each class is shuffled with a fixed-seed local RNG first, so a positive split is
+    reproducible but independent of how the eval file happens to be ordered (otherwise
+    the held-out set is just the first N as written, which clusters similar queries).
+    Returning positional indices (not query sublists) keeps the dedup invariant intact
+    and lets callers slice ``per_query`` by the same indices without matching on text.
+
+    Args:
+        eval_set: All evaluation queries.
+        test_frac: Fraction of each class to hold out for testing; ``0`` disables it.
+        seed: RNG seed for the per-class shuffle.
+
+    Returns:
+        A ``(train_idx, test_idx)`` tuple of original positional indices into
+        ``eval_set``. The two lists are disjoint and together cover every index.
+
+    Raises:
+        ValueError: If ``test_frac`` is non-finite or outside ``[0, 1)``, or (for a
+            positive fraction) a class has fewer than two queries or cannot retain a
+            training query at the requested fraction.
+    """
+    if not math.isfinite(test_frac) or not 0 <= test_frac < 1:
+        raise ValueError("holdout must be finite and satisfy 0 <= holdout < 1")
+    if test_frac == 0:
+        # Zero disables the held-out set: all-train, no-test, original order preserved,
+        # with no class-balance requirement (mirrors skill-creator's --holdout 0).
+        return list(range(len(eval_set))), []
+    positive = [i for i, q in enumerate(eval_set) if q["should_trigger"]]
+    negative = [i for i, q in enumerate(eval_set) if not q["should_trigger"]]
+    for group, label in ((positive, "positive"), (negative, "negative")):
+        if len(group) < 2:
+            raise ValueError(
+                f"{label} class must contain at least 2 queries when holdout > 0"
+            )
+    # Deterministic, reproducible train/test split -- not a security/crypto use, so
+    # the standard (non-cryptographic) RNG is the correct choice here.
+    rng = random.Random(seed)  # noqa: S311 # nosec B311
+    rng.shuffle(positive)
+    rng.shuffle(negative)
+    train: list[int] = []
+    test: list[int] = []
+    for group, label in ((positive, "positive"), (negative, "negative")):
+        n_test = max(1, round(len(group) * test_frac))
+        if n_test >= len(group):
+            raise ValueError(
+                f"{label} class cannot retain a training query at this holdout"
+            )
+        test += group[:n_test]
+        train += group[n_test:]
+    return train, test
+
+
+def resolve_models(spec: str | None) -> list[str]:
+    """Resolve a comma-separated model spec into a list of full model ids.
+
+    Args:
+        spec: Comma-separated aliases/ids (e.g. ``"haiku,sonnet"``), or ``None``.
+
+    Returns:
+        Resolved model ids. Defaults to ``[sonnet]`` when ``spec`` is empty/``None``;
+        a spec of only separators (e.g. ``","``) yields an empty list.
+    """
+    if not spec:
+        return [MODEL_ALIASES["sonnet"]]
+    return [
+        MODEL_ALIASES.get(s.strip(), s.strip()) for s in spec.split(",") if s.strip()
+    ]
+
+
+def _short_model(model: str) -> str:
+    """Return a model's short display name (the segment after the first ``-``).
+
+    Args:
+        model: A full model id (e.g. ``"claude-sonnet-5"``) or bare name.
+
+    Returns:
+        The short name (e.g. ``"sonnet"``), or the full name if it has no ``-``.
+    """
+    return model.split("-")[1] if "-" in model else model
+
+
+def summarize(tag: str, ev: EvalResult) -> str:
+    """Render a one-line summary of an evaluation result.
+
+    Args:
+        tag: Short label for the line (e.g. ``"BASELINE"``).
+        ev: The evaluation result to summarize.
+
+    Returns:
+        A line of the form ``"<tag>: mean=… min=…  (model=acc …)"`` where each model
+        is shown by its short name; a model with no judged query renders as ``n/a``.
+    """
+    per_model = " ".join(
+        f"{_short_model(m)}={'n/a' if a is None else f'{a:.2f}'}"
+        for m, a in ev["per_model_accuracy"].items()
+    )
+    return (
+        f"{tag}: mean={ev['mean_accuracy']:.3f} "
+        f"min={ev['min_accuracy']:.3f}  ({per_model})"
+    )
+
+
+def summarize_verbose(tag: str, ev: EvalResult) -> str:
+    """Render a multi-line, skill-creator-style precision/recall/accuracy summary.
+
+    Leads with the one-line :func:`summarize`, then an aggregate confusion header, a
+    per-model precision/recall/accuracy line, and a PASS/FAIL/N-A line per query. A
+    fully-unjudged model (its ``per_model_accuracy`` is ``None``) renders as ``n/a``,
+    as does the aggregate when every model is unjudged, mirroring :func:`summarize`'s
+    tri-state guard instead of the misleading ``100%/100%/0%`` an all-zeros confusion
+    matrix would otherwise show.
+
+    Args:
+        tag: Short label for the block (e.g. ``"BASELINE"``).
+        ev: The evaluation result to summarize.
+
+    Returns:
+        The multi-line summary.
+    """
+    c = ev["confusion"]
+    judged = c["tp"] + c["fp"] + c["tn"] + c["fn"]
+    agg = (
+        "  aggregate: n/a"
+        if judged == 0
+        else f"  aggregate: {c['tp'] + c['tn']}/{judged} correct  "
+        f"precision={c['precision']:.0%} recall={c['recall']:.0%} "
+        f"accuracy={c['accuracy']:.0%}"
+    )
+    lines = [summarize(tag, ev), agg]
+    acc = ev["per_model_accuracy"]
+    for m, cm in ev["per_model_confusion"].items():
+        if acc[m] is None:
+            lines.append(f"  {_short_model(m)}: n/a")
+        else:
+            lines.append(
+                f"  {_short_model(m)}: precision={cm['precision']:.0%} "
+                f"recall={cm['recall']:.0%} accuracy={cm['accuracy']:.0%}"
+            )
+    for pq in ev["per_query"]:
+        status = (
+            "PASS"
+            if pq["all_pass"] is True
+            else "FAIL"
+            if pq["all_pass"] is False
+            else "N/A "
+        )
+        lines.append(
+            f"  [{status}] expected={pq['should_trigger']}: {pq['query'][:60]}"
+        )
+    return "\n".join(lines)
+
+
+def is_better_candidate(
+    cand_mean: float,
+    cand_min: float,
+    best_mean: float,
+    best_min: float,
+    epsilon: float,
+    cand_chars: int | None = None,
+    best_chars: int | None = None,
+    max_chars: int | None = None,
+    *,
+    score_label: str = "held-out",
+) -> tuple[bool, str]:
+    """Decide whether a candidate beats the incumbent on the selection score.
+
+    The primary criterion is the selection *mean* accuracy (selecting on held-out
+    rather than train is what avoids overfitting the eval set). But on a small
+    selection split a sub-``epsilon`` mean difference is noise, not signal -- so when
+    two descriptions tie within ``epsilon``, break the tie in favor of the higher
+    *min* (weakest-model) accuracy. Set ``epsilon=0`` for strict mean-only selection.
+
+    When ``max_chars`` is set the character budget is a HARD constraint that dominates
+    the score comparison: an over-budget candidate can never win, a legal candidate
+    beats an over-budget incumbent even at an equal-or-slightly-lower mean (the
+    incumbent cannot ship), and among two legal ties within ``epsilon`` the shorter
+    wins. Passing no ``*_chars`` (the 5-argument call) leaves length out entirely and
+    reproduces the pre-budget behavior byte-for-byte.
+
+    Args:
+        cand_mean: Candidate selection mean accuracy.
+        cand_min: Candidate selection min (weakest-model) accuracy.
+        best_mean: Incumbent selection mean accuracy.
+        best_min: Incumbent selection min accuracy.
+        epsilon: Mean-accuracy band within which scores count as tied.
+        cand_chars: Candidate description length, for the budget constraint.
+        best_chars: Incumbent description length, for the budget constraint.
+        max_chars: Hard character ceiling, or ``None`` to ignore length entirely.
+        score_label: Name of the selection score used in the reason strings
+            (``"held-out"`` with a holdout, ``"train"`` when the holdout is disabled).
+
+    Returns:
+        ``(win, reason)`` where ``win`` is ``True`` if the candidate should replace
+        the incumbent, and ``reason`` is a short human-readable explanation.
+    """
+
+    def _over(chars: int | None) -> bool:
+        return max_chars is not None and chars is not None and chars > max_chars
+
+    if _over(cand_chars):
+        return False, f"over char budget ({cand_chars} > {max_chars})"
+    if _over(best_chars):
+        # Incumbent can't ship; any legal candidate replaces it -- ideally with a
+        # better/tied mean, but even a slightly lower one beats unshippable.
+        rel = "non-regressing" if cand_mean >= best_mean - epsilon else "lower"
+        return True, (
+            f"incumbent over budget ({best_chars} > {max_chars}); adopting legal "
+            f"candidate ({cand_chars} chars) with {rel} {score_label} mean "
+            f"({cand_mean:.3f} vs {best_mean:.3f})"
+        )
+    if cand_mean > best_mean + epsilon:
+        return True, f"improved {score_label} mean ({cand_mean:.3f} > {best_mean:.3f})"
+    if epsilon > 0 and abs(cand_mean - best_mean) <= epsilon and cand_min > best_min:
+        return True, (
+            f"{score_label} mean within {epsilon:.3f} "
+            f"({cand_mean:.3f} vs {best_mean:.3f}); "
+            f"better weakest-model min ({cand_min:.3f} > {best_min:.3f})"
+        )
+    if (
+        max_chars is not None
+        and abs(cand_mean - best_mean) <= epsilon
+        and cand_chars is not None
+        and best_chars is not None
+        and cand_chars < best_chars
+    ):
+        return True, (
+            f"{score_label} mean within {epsilon:.3f} "
+            f"({cand_mean:.3f} vs {best_mean:.3f}); "
+            f"shorter description preferred ({cand_chars} < {best_chars} chars)"
+        )
+    return False, f"no {score_label} gain"

skill_optimizer/skill_md.py

@@ -0,0 +1,123 @@
+"""Parsing and writing of a skill's ``SKILL.md`` frontmatter."""
+
+import re
+from pathlib import Path
+
+# YAML block/folded scalar indicators that carry no inline text on the ``key:`` line.
+_BLOCK_SCALAR_INDICATORS = frozenset({"|", ">", "|-", ">-", "|+", ">+", ""})
+
+# Characters kept verbatim when a skill name is reduced to a filesystem path token.
+# Every other character — path separators, whitespace, shell/format metacharacters —
+# collapses to a single ``_``. A shared skill's ``name:`` is attacker-controlled, so it
+# must never reach a filename verbatim.
+_UNSAFE_NAME_CHARS = re.compile(r"[^A-Za-z0-9._-]+")
+
+# Cap on the path-token length so a pathologically long ``name:`` cannot produce an
+# over-long filename that the OS rejects with ``ENAMETOOLONG`` mid-run.
+_MAX_NAME_TOKEN_CHARS = 64
+
+
+def safe_name_token(name: str, *, fallback: str = "skill") -> str:
+    """Reduce a skill name to a filesystem-safe token for use as a path component.
+
+    A shared ``SKILL.md``'s ``name:`` frontmatter is attacker-controlled, so it must
+    never reach a filename verbatim: an absolute path, a ``../`` sequence, or a bare path
+    separator would let a downstream write escape its intended directory. This collapses
+    every run of characters outside ``[A-Za-z0-9._-]`` to a single ``_``, strips leading
+    and trailing separators/dots (so a name that is only dots or separators — e.g. ``..``
+    or ``/`` — can never yield ``.``, ``..``, or an empty component), caps the length,
+    and falls back to a fixed token when nothing usable remains.
+
+    Args:
+        name: The raw, possibly attacker-controlled skill name.
+        fallback: Token returned when ``name`` has no filesystem-safe characters.
+
+    Returns:
+        A token drawn only from ``[A-Za-z0-9._-]``, never empty, never ``.`` or ``..``,
+        and never containing a path separator.
+    """
+    token = _UNSAFE_NAME_CHARS.sub("_", name).strip("._-")[:_MAX_NAME_TOKEN_CHARS]
+    return token.strip("._-") or fallback
+
+
+def parse_skill_md(skill_md: Path) -> tuple[str, str, str]:
+    """Extract the name, description, and body from a ``SKILL.md`` file.
+
+    Handles inline, folded (``>``), and block (``|``) scalar description styles, and
+    collapses internal whitespace in the description to single spaces.
+
+    Args:
+        skill_md: Path to the ``SKILL.md`` file.
+
+    Returns:
+        A ``(name, description, body)`` tuple. ``name`` is empty when absent.
+
+    Raises:
+        ValueError: If the file has no ``---`` delimited YAML frontmatter.
+    """
+    text = skill_md.read_text()
+    match = re.search(r"^---\n(.*?)\n---\n?(.*)$", text, re.DOTALL)
+    if not match:
+        raise ValueError(f"No YAML frontmatter in {skill_md}")
+    frontmatter, body = match[1], match[2]
+
+    name = ""
+    if name_match := re.search(r"^name:\s*(.+)$", frontmatter, re.MULTILINE):
+        name = name_match[1].strip().strip("'\"")
+
+    desc_parts: list[str] = []
+    capturing = False
+    for line in frontmatter.splitlines():
+        if not capturing:
+            if desc_match := re.match(r"^description:\s*(.*)$", line):
+                capturing = True
+                rest = desc_match[1].strip()
+                if rest not in _BLOCK_SCALAR_INDICATORS:
+                    desc_parts.append(rest)
+            continue
+        # Continuation: indented or blank lines belong to the description; the first
+        # non-indented, non-blank line is the next top-level key and ends capture.
+        if re.match(r"^\s+\S", line) or line.strip() == "":
+            desc_parts.append(line.strip())
+        else:
+            break
+
+    description = re.sub(r"\s+", " ", " ".join(p for p in desc_parts if p)).strip()
+    return name, description, body
+
+
+def write_description(skill_md: Path, new_description: str) -> None:
+    """Rewrite the description in ``SKILL.md`` as a literal block scalar (``|``).
+
+    The original file is backed up to ``<name>.bak`` first. The new description is
+    appended after the remaining frontmatter keys (so key order may change).
+
+    Args:
+        skill_md: Path to the ``SKILL.md`` file to modify.
+        new_description: The replacement description text.
+
+    Raises:
+        ValueError: If the file has no ``---`` delimited YAML frontmatter. Note the
+            backup is written before this check, so a failed write still leaves a
+            ``.bak`` behind.
+    """
+    text = skill_md.read_text()
+    skill_md.with_suffix(f"{skill_md.suffix}.bak").write_text(text)
+    match = re.search(r"^---\n(.*?)\n---", text, re.DOTALL)
+    if not match:
+        raise ValueError(f"No frontmatter in {skill_md}")
+    frontmatter = match[1]
+    # Drop the existing description block (``description:`` up to the next top-level
+    # key or end of frontmatter).
+    fm_without_desc = re.sub(
+        r"^description:.*?(?=^\w[\w-]*:|\Z)",
+        "",
+        frontmatter + "\n",
+        flags=re.DOTALL | re.MULTILINE,
+    ).rstrip("\n")
+    folded = "\n  ".join(new_description.split("\n"))
+    new_frontmatter = (
+        fm_without_desc + "\n" if fm_without_desc else ""
+    ) + f"description: |\n  {folded}"
+    new_text = text[: match.start(1)] + new_frontmatter + text[match.end(1) :]
+    skill_md.write_text(new_text)