agentforge-framework 0.2.0__py3-none-any.whl

agentforge_framework/skills/unslop/scripts/calibrate_score.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+"""
+Aggregate a teach-calibration preferences JSONL into per-dimension confidence,
+surface conflicts against a measured stylometric profile, and pick the next
+dimension to play.
+
+Preferences JSONL rows (one per game round):
+    {"pair_id": "...", "dimension": "contractions", "choice": "a"|"b"|"neither",
+     "ts": "...", "a_label": "contracted", "b_label": "expanded"}
+
+`a_label`/`b_label` are optional. When present (calibrate_pairs.py's
+transform_applied names the B pole; A sits at the other pole in the same
+dimension's POLES pair) they let this script report a semantic preferred
+DIRECTION ("expanded", "short", "formal", ...) instead of the bare literal
+"a"/"b" tally, which is what --profile conflict detection needs. Rows without
+labels still count toward n and the confidence interval; they just can't
+resolve to a named pole (direction is reported as "a" or "b").
+
+Usage:
+    python3 calibrate_score.py --preferences prefs.jsonl
+    python3 calibrate_score.py --preferences prefs.jsonl --profile profile.json
+    python3 calibrate_score.py --preferences prefs.jsonl --next
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+
+from calibrate_pairs import DIMENSIONS, POLES  # noqa: E402
+
+MIN_K = 5
+Z = 1.96
+
+# Dimension -> (profile_key, pole_meaning) used only for --profile conflict
+# detection. `low_pole`/`high_pole` say which named pole a LOW vs HIGH measured
+# value corresponds to, so a confident stated preference for the opposite pole
+# of what the measured value implies is flagged.
+_PROFILE_LINKS: dict[str, dict] = {
+    "contractions": {
+        "profile_key": "contraction_rate",
+        "low_pole": "expanded",
+        "high_pole": "contracted",
+        "low_threshold": 0.05,
+        "high_threshold": 0.20,
+    },
+    "sentence_length": {
+        "profile_key": "avg_sentence_length",
+        "low_pole": "short",
+        "high_pole": "long",
+        "low_threshold": 12.0,
+        "high_threshold": 20.0,
+    },
+    "staccato": {
+        "profile_key": "avg_sentence_length",
+        "low_pole": "staccato",
+        "high_pole": "flowing",
+        "low_threshold": 12.0,
+        "high_threshold": 20.0,
+    },
+    "em_dash": {
+        "profile_key": "em_dash_rate",
+        "low_pole": "plain",
+        "high_pole": "dashed",
+        "low_threshold": 0.02,
+        "high_threshold": 0.15,
+    },
+    "connectives": {
+        "profile_key": "formal_connective_rate",
+        "low_pole": "plain",
+        "high_pole": "formal",
+        "low_threshold": 0.10,
+        "high_threshold": 0.40,
+    },
+}
+
+CONFIDENCE_STOP = 0.7
+K_STOP = 9
+
+
+def wilson_lower_bound(successes: int, n: int, z: float = Z) -> float:
+    if n == 0:
+        return 0.0
+    phat = successes / n
+    denom = 1 + z * z / n
+    center = phat + z * z / (2 * n)
+    margin = z * math.sqrt((phat * (1 - phat) + z * z / (4 * n)) / n)
+    return round(max(0.0, (center - margin) / denom), 3)
+
+
+def load_preferences(path: Path) -> list[dict]:
+    rows = []
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if line:
+            rows.append(json.loads(line))
+    return rows
+
+
+def dedup_by_pair_id(rows: list[dict]) -> list[dict]:
+    """Keep one row per `pair_id`: the LATEST by `ts` (ISO 8601, so a plain
+    string comparison orders correctly). A game round replayed twice (e.g. the
+    agent crashed mid-write and re-ran the round) must count once toward `n`
+    and resolve to whatever the user most recently chose, not double-count or
+    let an earlier write win. Rows without a `pair_id` can't be deduped
+    against anything, so each is kept as-is.
+    """
+    by_pair_id: dict[str, dict] = {}
+    passthrough: list[dict] = []
+    for row in rows:
+        pid = row.get("pair_id")
+        if not pid:
+            passthrough.append(row)
+            continue
+        existing = by_pair_id.get(pid)
+        if existing is None or row.get("ts", "") >= existing.get("ts", ""):
+            by_pair_id[pid] = row
+    return list(by_pair_id.values()) + passthrough
+
+
+def aggregate(rows: list[dict]) -> dict[str, dict]:
+    rows = dedup_by_pair_id(rows)
+    result = {dim: {"n": 0, "tally": {}, "neither": 0} for dim in DIMENSIONS}
+
+    for row in rows:
+        dim = row.get("dimension")
+        if dim not in result:
+            continue
+        result[dim]["n"] += 1
+        choice = row.get("choice")
+        if choice == "neither":
+            result[dim]["neither"] += 1
+            continue
+        if choice not in ("a", "b"):
+            continue
+        label = row.get(f"{choice}_label") or choice
+        result[dim]["tally"][label] = result[dim]["tally"].get(label, 0) + 1
+
+    dimensions = {}
+    for dim, data in result.items():
+        n = data["n"]
+        decisive = sum(data["tally"].values())
+        if n < MIN_K or decisive == 0:
+            dimensions[dim] = {
+                "n": n,
+                "status": "insufficient",
+                "preferred": None,
+                "confidence": 0.0,
+            }
+            continue
+        max_count = max(data["tally"].values())
+        top_labels = [label for label, count in data["tally"].items() if count == max_count]
+        if len(top_labels) > 1:
+            # A genuine tie between the top two (or more) tallies: there is no
+            # lean to report, so `preferred` must be null rather than
+            # silently picking whichever label happens to sort last.
+            dimensions[dim] = {
+                "n": n,
+                "status": "tied",
+                "preferred": None,
+                "confidence": 0.0,
+            }
+            continue
+        preferred_label = top_labels[0]
+        confidence = wilson_lower_bound(max_count, decisive)
+        dimensions[dim] = {
+            "n": n,
+            "status": "confident",
+            "preferred": preferred_label,
+            "confidence": confidence,
+        }
+    return dimensions
+
+
+def detect_conflicts(dimensions: dict[str, dict], profile: dict) -> list[dict]:
+    conflicts = []
+    for dim, data in dimensions.items():
+        if data["status"] != "confident" or data["confidence"] < CONFIDENCE_STOP:
+            continue
+        link = _PROFILE_LINKS.get(dim)
+        if not link or link["profile_key"] not in profile:
+            continue
+        measured = profile[link["profile_key"]]
+        preferred = data["preferred"]
+        conflict_pole = None
+        if preferred == link["low_pole"] and measured >= link["high_threshold"]:
+            conflict_pole = link["high_pole"]
+        elif preferred == link["high_pole"] and measured <= link["low_threshold"]:
+            conflict_pole = link["low_pole"]
+        if conflict_pole is None:
+            continue
+        conflicts.append({
+            "dimension": dim,
+            "preferred": preferred,
+            "preferred_confidence": data["confidence"],
+            "preferred_provenance": "stated-preference",
+            "measured_key": link["profile_key"],
+            "measured_value": measured,
+            "measured_provenance": "measured-from-samples",
+            "message": (
+                f"Stated preference for '{preferred}' ({dim}) contradicts "
+                f"{link['profile_key']}={measured} measured from samples, "
+                f"which points toward '{conflict_pole}'."
+            ),
+        })
+    return conflicts
+
+
+def next_dimension(dimensions: dict[str, dict]) -> dict:
+    def sort_key(dim: str) -> tuple:
+        data = dimensions[dim]
+        confidence = data["confidence"] if data["status"] == "confident" else 0.0
+        return (data["n"], confidence, DIMENSIONS.index(dim))
+
+    ordered = sorted(DIMENSIONS, key=sort_key)
+    chosen = ordered[0]
+    data = dimensions[chosen]
+    if data["n"] == min(dimensions[d]["n"] for d in DIMENSIONS):
+        reason = "fewest_observations"
+    else:
+        reason = "lowest_confidence"
+    return {
+        "next_dimension": chosen,
+        "reason": reason,
+        "n": data["n"],
+        "confidence": data["confidence"],
+    }
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--preferences", required=True)
+    parser.add_argument("--profile")
+    parser.add_argument("--next", action="store_true")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str]) -> int:
+    args = parse_args(argv)
+
+    try:
+        rows = load_preferences(Path(args.preferences))
+    except OSError as e:
+        print(json.dumps({"error": f"could not read preferences file: {e}"}))
+        return 2
+
+    dimensions = aggregate(rows)
+
+    if args.next:
+        print(json.dumps(next_dimension(dimensions), indent=2, sort_keys=True))
+        return 0
+
+    output = {"dimensions": dimensions}
+
+    if args.profile:
+        try:
+            profile = json.loads(Path(args.profile).read_text())
+        except OSError as e:
+            print(json.dumps({"error": f"could not read profile file: {e}"}))
+            return 2
+        output["conflicts"] = detect_conflicts(dimensions, profile)
+
+    print(json.dumps(output, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

agentforge_framework/skills/unslop/scripts/check_packs.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""Validate tiered detector pack integrity."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+PACK_DIR = ROOT / "references" / "packs"
+MANIFEST = PACK_DIR / "manifest.json"
+
+JUDGE_ONLY_MACROS = {
+    "macro_both_sidesism",
+    "macro_redemption_arc",
+    "macro_preview_recap",
+    "macro_over_determination",
+    "macro_emotional_flatness",
+}
+
+
+def scanner_categories() -> set[str]:
+    sys.path.insert(0, str(ROOT))
+    from scripts.banned_phrase_scan import BANNED_PHRASES, STRUCTURAL_PATTERNS
+
+    return {v["category"] for v in BANNED_PHRASES.values()} | {
+        v["category"] for v in STRUCTURAL_PATTERNS
+    }
+
+
+def main() -> int:
+    failures: list[str] = []
+    data = json.loads(MANIFEST.read_text())
+    packs: dict[str, list[str]] = data["packs"]
+    category_to_pack: dict[str, str] = {}
+
+    for pack, categories in packs.items():
+        path = PACK_DIR / f"{pack}.md"
+        if not path.exists():
+            failures.append(f"missing pack file: {path}")
+            continue
+        text = path.read_text()
+        line_count = len(text.splitlines())
+        if line_count > 120:
+            failures.append(f"{path} has {line_count} lines; max 120")
+        if "## Emit" not in text:
+            failures.append(f"{path} missing ## Emit section")
+        for category in categories:
+            if category in category_to_pack:
+                failures.append(
+                    f"category {category} mapped to both {category_to_pack[category]} and {pack}"
+                )
+            category_to_pack[category] = pack
+
+    scanner = scanner_categories()
+    missing = scanner - set(category_to_pack)
+    extra_scanner = (set(category_to_pack) - scanner) - JUDGE_ONLY_MACROS
+    if missing:
+        failures.append(f"scanner categories missing from packs: {sorted(missing)}")
+    if extra_scanner:
+        failures.append(f"unknown non-macro categories in manifest: {sorted(extra_scanner)}")
+
+    macro_hits = {m: category_to_pack.get(m) for m in JUDGE_ONLY_MACROS}
+    missing_macros = [m for m, pack in macro_hits.items() if pack is None]
+    if missing_macros:
+        failures.append(f"judge-only macro families missing: {sorted(missing_macros)}")
+    for macro, pack in macro_hits.items():
+        if pack and pack != "pack-structure":
+            failures.append(f"{macro} must live in pack-structure, found {pack}")
+
+    if failures:
+        print("\n".join(failures), file=sys.stderr)
+        return 1
+    print(f"pack integrity ok: {len(packs)} packs, {len(scanner)} scanner categories")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentforge_framework/skills/unslop/scripts/check_suggestions.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Contract gates for co-writer suggestions.
+
+Reads a suggestions file — the JSON emitted by suggest.py, i.e. an object with a
+``document`` string and a ``suggestions`` list — and enforces four blocking
+contracts. Each failure is named so a caller can act on it:
+
+  span-minimality      Every suggested_replacement edits only its own span:
+                       span.text matches document[start:end], the replacement
+                       differs from the span text, and it shares no leading or
+                       trailing whole word with the span (which would mean the
+                       span grabbed unchanged text and could be shrunk — the
+                       signature of a whole-sentence rewrite).
+  replacement-scanner  Every suggested_replacement passes both scanners in
+                       isolation AND introduces no new violation in context.
+  accept-all           Applying every replacement yields a document that passes
+                       both scanners with validate_preservation exit 0 vs the
+                       original.
+  span-overlap         Suggestion spans must not overlap.
+
+Exit 0 when all gates pass, 1 otherwise. Failures (with the offending gate) are
+reported as JSON on stdout.
+
+Usage:
+    python3 scripts/check_suggestions.py suggestions.json
+    python3 scripts/check_suggestions.py < suggestions.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from banned_phrase_scan import scan_for_violations
+from structure_scan import scan as structure_scan
+from validate_preservation import validate_preservation
+
+
+def _leading_shared_words(a: str, b: str) -> int:
+    """Number of identical leading whole words shared by two strings."""
+    n = 0
+    for x, y in zip(a.split(), b.split()):
+        if x == y:
+            n += 1
+        else:
+            break
+    return n
+
+
+def _trailing_shared_words(a: str, b: str) -> int:
+    ra = " ".join(reversed(a.split()))
+    rb = " ".join(reversed(b.split()))
+    return _leading_shared_words(ra, rb)
+
+
+def _line_starts(text: str) -> list[int]:
+    starts = [0]
+    for i, ch in enumerate(text):
+        if ch == "\n":
+            starts.append(i + 1)
+    return starts
+
+
+def violation_spans(text: str) -> list[tuple[int, int]]:
+    """Absolute (start, end) offsets of every banned-phrase/structural violation."""
+    starts = _line_starts(text)
+    out = []
+    for v in scan_for_violations(text):
+        s = starts[v["line_number"] - 1] + v["column"] - 1
+        out.append((s, s + len(v["phrase"])))
+    return out
+
+
+def _scanners_clean(text: str) -> bool:
+    return not scan_for_violations(text) and not structure_scan(text).get("flags")
+
+
+def apply_all(document: str, suggestions: list[dict]) -> str:
+    """Apply every non-null replacement, right-to-left so offsets stay valid."""
+    out = document
+    for s in sorted(suggestions, key=lambda s: s["span"]["start"], reverse=True):
+        rep = s.get("suggested_replacement")
+        if rep is None:
+            continue
+        st, en = s["span"]["start"], s["span"]["end"]
+        out = out[:st] + rep + out[en:]
+    return out
+
+
+def check(document: str, suggestions: list[dict]) -> list[dict]:
+    failures: list[dict] = []
+
+    # span-overlap: spans must be disjoint.
+    order = sorted(range(len(suggestions)),
+                   key=lambda i: (suggestions[i]["span"]["start"], suggestions[i]["span"]["end"]))
+    last_end = None
+    last_i = None
+    for i in order:
+        st, en = suggestions[i]["span"]["start"], suggestions[i]["span"]["end"]
+        if last_end is not None and st < last_end:
+            failures.append({
+                "gate": "span-overlap",
+                "suggestions": [last_i, i],
+                "detail": f"span {st}-{en} overlaps the previous span ending at {last_end}",
+            })
+        last_end, last_i = en, i
+
+    # span-minimality: span accuracy + tight, local replacements.
+    for i, s in enumerate(suggestions):
+        st, en = s["span"]["start"], s["span"]["end"]
+        span_text = document[st:en]
+        if span_text != s["span"]["text"]:
+            failures.append({
+                "gate": "span-minimality",
+                "suggestion": i,
+                "detail": "span.text does not match document[start:end]",
+            })
+        rep = s.get("suggested_replacement")
+        if rep is None:
+            continue
+        if rep == span_text:
+            failures.append({
+                "gate": "span-minimality",
+                "suggestion": i,
+                "detail": "replacement is identical to the span text (no change)",
+            })
+            continue
+        if _leading_shared_words(span_text, rep) > 0 or _trailing_shared_words(span_text, rep) > 0:
+            failures.append({
+                "gate": "span-minimality",
+                "suggestion": i,
+                "detail": "replacement shares leading/trailing whole words with the span; "
+                          "shrink the span so the edit is minimal",
+            })
+
+    # replacement-scanner: each replacement is clean alone and in context.
+    for i, s in enumerate(suggestions):
+        rep = s.get("suggested_replacement")
+        if rep is None:
+            continue
+        if not _scanners_clean(rep):
+            failures.append({
+                "gate": "replacement-scanner",
+                "suggestion": i,
+                "detail": "replacement does not pass both scanners in isolation",
+            })
+        st, en = s["span"]["start"], s["span"]["end"]
+        ctx = document[:st] + rep + document[en:]
+        new_start, new_end = st, st + len(rep)
+        if any(a < new_end and new_start < b for (a, b) in violation_spans(ctx)):
+            failures.append({
+                "gate": "replacement-scanner",
+                "suggestion": i,
+                "detail": "replacement introduces a violation in context",
+            })
+
+    # accept-all: applying everything yields a clean, constraint-preserving doc.
+    unresolved = [i for i, s in enumerate(suggestions) if s.get("suggested_replacement") is None]
+    if unresolved:
+        failures.append({
+            "gate": "accept-all",
+            "detail": f"suggestions {unresolved} have no replacement; cannot accept-all",
+        })
+    applied = apply_all(document, suggestions)
+    if not _scanners_clean(applied):
+        failures.append({
+            "gate": "accept-all",
+            "detail": "the accept-all document still fails a scanner",
+        })
+    preservation = validate_preservation(document, applied)
+    if not preservation["passed"]:
+        failures.append({
+            "gate": "accept-all",
+            "detail": "validate_preservation failed against the original",
+            "missing": preservation["missing"],
+        })
+
+    return failures
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Contract gates for co-writer suggestions.")
+    parser.add_argument(
+        "path", nargs="?", help="Path to a suggestions JSON file (default: read stdin)"
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str]) -> int:
+    args = parse_args(argv)
+    if args.path:
+        path = Path(args.path)
+        if not path.exists():
+            print(f"Missing file: {path}", file=sys.stderr)
+            return 2
+        raw = path.read_text(errors="replace")
+    else:
+        raw = sys.stdin.buffer.read().decode("utf-8", errors="replace")
+
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as e:
+        print(json.dumps({"passed": False, "error": f"invalid JSON: {e}"}, indent=2))
+        return 1
+
+    document = data.get("document")
+    suggestions = data.get("suggestions", [])
+    if not isinstance(document, str):
+        print(json.dumps({"passed": False, "error": "missing 'document' string"}, indent=2))
+        return 1
+
+    failures = check(document, suggestions)
+    result = {
+        "passed": not failures,
+        "failure_gates": sorted({f["gate"] for f in failures}),
+        "failures": failures,
+    }
+    print(json.dumps(result, indent=2))
+    return 0 if not failures else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))