agentforge-framework 0.2.0__py3-none-any.whl

agentforge_framework/skills/unslop/scripts/structure_scan.py

@@ -0,0 +1,322 @@
+#!/usr/bin/env python3
+"""Scan prose for macro-structure AI-writing patterns."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+
+from _lang import (  # noqa: E402
+    ENGLISH_FUNCTION_WORDS,
+    english_function_share,
+    is_probably_english,
+    paragraphs as _prose_paragraphs,
+    words,
+)
+from readability_metrics import split_sentences  # noqa: E402
+
+
+STOPWORDS = {
+    "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of",
+    "with", "by", "from", "is", "are", "was", "were", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did", "will", "would", "could",
+    "should", "may", "might", "must", "can", "it", "its", "this", "that",
+    "these", "those", "we", "you", "they", "i", "he", "she", "as", "than",
+    "if", "then", "so", "not", "no", "yes", "into", "over", "under",
+}
+
+
+CONNECTIVE_OPENERS = re.compile(
+    r"^(however|moreover|furthermore|additionally|in addition|overall|"
+    r"consequently|nevertheless)\b",
+    re.I,
+)
+# Paragraph-initial "Every <noun> <verb-phrase>" template opener ("Every
+# transformation is scored ...", "Every rewrite passes ..."). As a RHYTHM tell
+# this is about REPETITION, so the metric below fires only at 2+ such paragraph
+# openers; a lone "Every child deserves a good school." stays clean. The verb is
+# a copula/auxiliary or a present/past inflection (-s / -ed) so the second word
+# reads as a predicate, not another noun.
+EVERY_OPENER_RE = re.compile(
+    r"^every\s+[a-z][\w'-]*\s+(?:is|are|was|were|be|been|being|has|have|had|"
+    r"do|does|did|can|will|shall|must|should|would|may|might|"
+    r"[a-z]+(?:s|ed))\b",
+    re.I,
+)
+SIGNPOST_RE = re.compile(
+    r"\b(first,|next,|having (?:covered|established)|in (?:this|the following|"
+    r"the next) section|as (?:mentioned|noted) (?:above|earlier)|let us turn|"
+    r"let's turn)\b",
+    re.I,
+)
+CLOSER_RE = re.compile(
+    r",\s+(ensuring|highlighting|reflecting|allowing|enabling|underscoring|"
+    r"showcasing|emphasizing|fostering|driving|paving|reinforcing|solidifying|"
+    r"demonstrating|contributing to)\b[^.!?]*[.!?\"]?$",
+    re.I,
+)
+CODA_START_RE = re.compile(
+    r"^(ultimately,|in the end,|in conclusion|as we've seen|only time will tell|"
+    r"remember,|the future\b)",
+    re.I,
+)
+BOLD_COLON_RE = re.compile(r"^\s*[-*+]?\s*\*\*[^*]{1,40}\*\*\s*:", re.M)
+
+
+def prose_paragraphs(text: str) -> list[str]:
+    return _prose_paragraphs(text, blank_blockquotes=True)
+
+
+def cv(values: list[int]) -> float:
+    if not values:
+        return 0.0
+    mean = sum(values) / len(values)
+    if mean == 0:
+        return 0.0
+    return math.sqrt(sum((v - mean) ** 2 for v in values) / len(values)) / mean
+
+
+def content_bigrams(text: str) -> set[tuple[str, str]]:
+    toks = [w for w in words(text) if len(w) > 3 and w not in STOPWORDS]
+    return set(zip(toks, toks[1:]))
+
+
+def triad_count(text: str) -> int:
+    return len(re.findall(r"\b[A-Za-z][A-Za-z-]+,\s+[A-Za-z][A-Za-z-]+,\s+and\s+[A-Za-z][A-Za-z-]+\b", text))
+
+
+def flag(metric: str, value: float | int, threshold: str, detail: str, suggestion: str) -> dict:
+    return {
+        "metric": metric,
+        "value": value,
+        "threshold": threshold,
+        "severity": "soft",
+        "detail": detail,
+        "suggestion": suggestion,
+    }
+
+
+# Metrics a given --genre suppresses outright (the genre's normal shape trips
+# the metric without being AI-slop). Table form so each suppression is a single
+# lookup instead of an inline `genre != "..."` conditional per metric.
+GENRE_SUPPRESSIONS = {
+    "docs": {"bold_colon_listicle"},
+    "social": {"one_line_staccato"},
+}
+
+
+def scan(text: str, genre: str = "prose") -> dict:
+    paragraphs = prose_paragraphs(text)
+    prose_text = "\n\n".join(paragraphs)
+    sentences = split_sentences(prose_text)
+    sentence_lengths = [len(words(s)) for s in sentences]
+    prose_words = words(prose_text)
+    para_lengths = [len(words(p)) for p in paragraphs]
+    metrics = {
+        "sentence_burstiness": round(cv(sentence_lengths), 3),
+        "summary_sandwich": 0.0,
+        "paragraph_cv": round(cv(para_lengths), 3),
+        "sentence_mean_len": round(sum(sentence_lengths) / len(sentence_lengths), 1) if sentence_lengths else 0,
+        "triad_density": round((triad_count(prose_text) / len(prose_words) * 1000), 3) if prose_words else 0,
+        "em_dash_per_1k": round((text.count("—") / len(prose_words) * 1000), 3) if prose_words else 0,
+        "bold_colon_listicle_count": len(BOLD_COLON_RE.findall(text)),
+        "one_line_staccato_share": 0.0,
+        "connective_paragraph_openers": 0,
+        "every_template_openers": 0,
+        "signpost_density": 0.0,
+        "opener_unique_ratio": 0.0,
+        "top_opener_share": 0.0,
+        "max_consecutive_opener": 0,
+        "participial_closer_share": 0.0,
+        "conclusion_coda": False,
+    }
+    flags = []
+
+    if len(paragraphs) >= 2:
+        first = content_bigrams(paragraphs[0])
+        last = content_bigrams(paragraphs[-1])
+        union = first | last
+        metrics["summary_sandwich"] = round(len(first & last) / len(union), 3) if union else 0.0
+
+    if len(sentences) >= 8 and metrics["sentence_burstiness"] < 0.55:
+        flags.append(flag(
+            "sentence_burstiness",
+            metrics["sentence_burstiness"],
+            "< 0.55 over at least 8 prose sentences",
+            "Sentence lengths are unusually uniform for running prose.",
+            "Vary sentence length and cadence; if this is formal reference prose, review before treating it as blocking.",
+        ))
+
+    if len(paragraphs) >= 3:
+        coda = bool(CODA_START_RE.search(paragraphs[-1]))
+        if not coda:
+            coda = len(content_bigrams(paragraphs[0]) & content_bigrams(paragraphs[-1])) >= 2
+        metrics["conclusion_coda"] = coda
+        if coda:
+            flags.append(flag(
+                "conclusion_coda",
+                1,
+                "last paragraph starts with a stock coda or repeats 2+ first-paragraph content bigrams",
+                "The ending reads like a recap/moral coda instead of a concrete final point.",
+                "Cut the wrap-up or end on a specific fact; if this is an abstract or executive summary, judge the genre before changing it.",
+            ))
+
+    if (metrics["bold_colon_listicle_count"] >= 3
+            and "bold_colon_listicle" not in GENRE_SUPPRESSIONS.get(genre, set())):
+        flags.append(flag(
+            "bold_colon_listicle",
+            metrics["bold_colon_listicle_count"],
+            ">= 3 bold-label colon lines",
+            "The raw Markdown has repeated bold-label listicle formatting.",
+            "Convert to prose or plain bullets; if this is a reference doc, rerun with --genre docs.",
+        ))
+
+    if paragraphs:
+        one_line = 0
+        for p in paragraphs:
+            ps = split_sentences(p)
+            if len(ps) == 1 and len(words(ps[0])) < 12:
+                one_line += 1
+        metrics["one_line_staccato_share"] = round(one_line / len(paragraphs), 3)
+        if (len(paragraphs) >= 6 and metrics["one_line_staccato_share"] > 0.6
+                and "one_line_staccato" not in GENRE_SUPPRESSIONS.get(genre, set())):
+            flags.append(flag(
+                "one_line_staccato",
+                metrics["one_line_staccato_share"],
+                "> 0.60 over at least 6 paragraphs",
+                "Most paragraphs are short single-sentence beats.",
+                "Merge related beats and vary paragraph length; if this is social copy, rerun with --genre social.",
+            ))
+
+    connective = sum(1 for p in paragraphs if CONNECTIVE_OPENERS.search(p))
+    metrics["connective_paragraph_openers"] = connective
+    if connective >= 3 or (len(paragraphs) >= 8 and connective / len(paragraphs) > 0.4):
+        flags.append(flag(
+            "connective_paragraph_openers",
+            connective,
+            ">= 3 paragraphs or > 40% of 8+ paragraphs",
+            "Paragraphs repeatedly open with formal transition words.",
+            "Replace scaffold openers with specific topic sentences; academic prose may justify some connectors.",
+        ))
+
+    every_openers = sum(1 for p in paragraphs if EVERY_OPENER_RE.search(p))
+    metrics["every_template_openers"] = every_openers
+    if every_openers >= 2:
+        flags.append(flag(
+            "every_template_openers",
+            every_openers,
+            ">= 2 paragraphs opening 'Every <noun> <verb>'",
+            "Paragraphs repeatedly open on the 'Every ___ is/does ...' template.",
+            "Vary the paragraph openings; a repeated Every-template is a machine rhythm tell even when each sentence is fine on its own.",
+        ))
+
+    if prose_words:
+        signposts = len(SIGNPOST_RE.findall(prose_text))
+        metrics["signpost_density"] = round(signposts / len(prose_words) * 100, 3)
+        if len(prose_words) >= 150 and metrics["signpost_density"] > 0.6:
+            flags.append(flag(
+                "signpost_density",
+                metrics["signpost_density"],
+                "> 0.6 per 100 prose words, minimum 150 words",
+                "The text over-explains its own structure.",
+                "Remove roadmap language unless the genre is a textbook, legal brief, or long guide.",
+            ))
+
+    if len(sentences) >= 5:
+        openers = []
+        for s in sentences:
+            ws = words(s)
+            if not ws:
+                continue
+            openers.append(ws[0])
+        enumeration = {"the", "a", "an", "section", "chapter", "figure", "table",
+                       "step", "part", "appendix"}
+        counted = [o for o in openers if o not in enumeration]
+        top_count = 0
+        if counted:
+            counts = Counter(counted)
+            top_count = max(counts.values())
+            metrics["opener_unique_ratio"] = round(len(counts) / len(counted), 3)
+            metrics["top_opener_share"] = round(top_count / len(counted), 3)
+        run = max_run = 0
+        prev = None
+        for opener in counted:
+            run = run + 1 if opener == prev else 1
+            prev = opener
+            max_run = max(max_run, run)
+        metrics["max_consecutive_opener"] = max_run
+        top_repeat = metrics["top_opener_share"] > 0.25 and top_count >= 4
+        if metrics["opener_unique_ratio"] < 0.55 or top_repeat or max_run >= 4:
+            flags.append(flag(
+                "opener_repetition",
+                metrics["opener_unique_ratio"],
+                "unique ratio < 0.55, one opener > 25% with 4+ uses, or 4 consecutive identical openers",
+                "Sentence openings repeat in a template-like rhythm.",
+                "Rewrite repeated starts; step-by-step docs may need repeated imperative openers.",
+            ))
+
+    if sentences:
+        closer_count = sum(1 for s in sentences if CLOSER_RE.search(s))
+        metrics["participial_closer_share"] = round(closer_count / len(sentences), 3)
+        if len(sentences) >= 8 and metrics["participial_closer_share"] >= 0.15:
+            flags.append(flag(
+                "participial_closer_share",
+                metrics["participial_closer_share"],
+                ">= 0.15 over at least 8 prose sentences",
+                "Many sentences end with editorial -ing consequence tails.",
+                "Make the consequence concrete or cut the tail; analytical prose may allow an occasional closer.",
+            ))
+
+    return {
+        "flags": flags,
+        "flagged": {f["metric"]: True for f in flags},
+        "metrics": metrics,
+        "genre": genre,
+        "prose_sentences": len(sentences),
+        "prose_paragraphs": len(paragraphs),
+    }
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("path", nargs="?")
+    parser.add_argument("--genre", choices=["prose", "docs", "social"], default="prose")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str]) -> int:
+    args = parse_args(argv)
+    if args.path:
+        path = Path(args.path)
+        if not path.exists():
+            print(f"Missing file: {path}", file=sys.stderr)
+            return 2
+        text = path.read_text(errors="replace")
+    else:
+        text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
+
+    # English-only graceful decline, matching banned_phrase_scan.py.
+    result = scan(text, args.genre)
+
+    # Function-word absence alone is not evidence of a foreign language
+    # (imperative stacks and buzzword lists are English slop with few function
+    # words). Decline only when the heuristic fails AND nothing flagged.
+    if not result.get("flags") and not is_probably_english(text):
+        print(json.dumps({"non_english": True, "violations": [], "flags": []}, indent=2))
+        print("note: input appears non-English; scanner declined (English-only).", file=sys.stderr)
+        return 0
+
+    print(json.dumps(result, indent=2))
+    return 1 if result["flags"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

agentforge_framework/skills/unslop/scripts/suggest.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""Co-writer suggestion mode: emit LSP-style structured edit suggestions.
+
+Detection is cheap and deterministic (banned_phrase_scan + structure_scan).
+Replacement generation is DELEGATED to a stronger model: by default every
+suggestion is emitted with ``"suggested_replacement": null``. A separate
+``--apply-replacements FILE`` mode merges externally-produced replacements back
+in and light-validates them. The blocking contract gates live in
+check_suggestions.py; this script never rewrites the document itself.
+
+Output shape:
+    {
+      "document": "<original text>",
+      "suggestions": [
+        {
+          "span": {"start": N, "end": N, "text": "..."},
+          "severity": "hard" | "soft",
+          "category": "...",
+          "rationale": "...",
+          "suggested_replacement": null,
+          "phrased_as_question": bool
+        },
+        ...
+      ],
+      "counts": {...}
+    }
+
+Soft findings are phrased as questions (register-dependent judgment calls);
+hard findings are stated as direct replacements. Suggestions are emitted in a
+deterministic order (span start, then end, then category) and never overlap.
+
+Usage:
+    python3 scripts/suggest.py document.md
+    python3 scripts/suggest.py < document.md
+    python3 scripts/suggest.py document.md --apply-replacements replacements.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+from banned_phrase_scan import (
+    is_probably_english,
+    scan_for_violations,
+)
+from structure_scan import scan as structure_scan
+
+
+def _line_starts(text: str) -> list[int]:
+    """Character offset at which each line begins (index 0 == line 1)."""
+    starts = [0]
+    for i, ch in enumerate(text):
+        if ch == "\n":
+            starts.append(i + 1)
+    return starts
+
+
+def _offset(line_starts: list[int], line_number: int, column: int) -> int:
+    return line_starts[line_number - 1] + (column - 1)
+
+
+def _rationale(category: str, span_text: str, suggestion: str | None, is_soft: bool) -> str:
+    """Build a reviewer-facing rationale.
+
+    Soft findings are worded as a question because they are judgment calls whose
+    right answer depends on register; hard findings are stated as a direct fix.
+    """
+    if is_soft:
+        hint = f" Consider: {suggestion}." if suggestion else ""
+        return f"Soft tell ({category}): could “{span_text}” be cut or reworded here?{hint}"
+    if suggestion:
+        return f"AI-writing tell ({category}): replace “{span_text}” — {suggestion}"
+    return f"AI-writing tell ({category}): replace “{span_text}”."
+
+
+def build_suggestions(text: str) -> list[dict]:
+    """Detect AI-isms and turn each into a span-anchored suggestion.
+
+    Spans come from banned_phrase_scan (phrases + structural regexes), which are
+    the findings with real character offsets that a replacement can be applied
+    to. macro-structure flags from structure_scan are document-level and are
+    surfaced in ``counts`` instead, since they have no single applyable span.
+    """
+    violations = scan_for_violations(text)
+    line_starts = _line_starts(text)
+    candidates: list[dict] = []
+    for v in violations:
+        start = _offset(line_starts, v["line_number"], v["column"])
+        end = start + len(v["phrase"])
+        span_text = text[start:end]
+        is_soft = v["severity"] == "soft"
+        candidates.append({
+            "span": {"start": start, "end": end, "text": span_text},
+            "severity": v["severity"],
+            "category": v["category"],
+            "rationale": _rationale(v["category"], span_text, v.get("suggestion"), is_soft),
+            "suggested_replacement": None,
+            "phrased_as_question": is_soft,
+        })
+
+    candidates.sort(key=lambda s: (s["span"]["start"], s["span"]["end"], s["category"]))
+
+    # Enforce non-overlap deterministically: keep the earliest-starting span and
+    # drop any later suggestion that overlaps an already-kept one, so the emitted
+    # set always satisfies the span-overlap contract gate.
+    kept: list[dict] = []
+    last_end = -1
+    for s in candidates:
+        if s["span"]["start"] >= last_end:
+            kept.append(s)
+            last_end = s["span"]["end"]
+    return kept
+
+
+def counts_block(suggestions: list[dict], struct: dict) -> dict:
+    by_category: dict[str, int] = {}
+    hard = soft = 0
+    for s in suggestions:
+        by_category[s["category"]] = by_category.get(s["category"], 0) + 1
+        if s["severity"] == "soft":
+            soft += 1
+        else:
+            hard += 1
+    return {
+        "total": len(suggestions),
+        "hard": hard,
+        "soft": soft,
+        "by_category": by_category,
+        "structure_flags": [f["metric"] for f in struct.get("flags", [])],
+    }
+
+
+def apply_replacements(suggestions: list[dict], repl_path: str) -> list[str]:
+    """Merge externally-produced replacements into suggestions and light-validate.
+
+    The replacement file is ``{"replacements": [{"start", "end", "replacement"}]}``.
+    Each replacement is keyed to a suggestion by exact (start, end) span. This is
+    only a light merge/validation pass; the blocking contract lives in
+    check_suggestions.py.
+    """
+    data = json.loads(Path(repl_path).read_text())
+    index = {(r["start"], r["end"]): r["replacement"] for r in data.get("replacements", [])}
+    warnings: list[str] = []
+    matched: set[tuple[int, int]] = set()
+    for s in suggestions:
+        key = (s["span"]["start"], s["span"]["end"])
+        if key not in index:
+            continue
+        rep = index[key]
+        s["suggested_replacement"] = rep
+        matched.add(key)
+        if rep == s["span"]["text"]:
+            warnings.append(f"replacement for span {list(key)} is identical to span text")
+        elif scan_for_violations(rep) or structure_scan(rep).get("flags"):
+            warnings.append(f"replacement for span {list(key)} does not pass the scanners in isolation")
+    for key in index:
+        if key not in matched:
+            warnings.append(f"replacement targets unknown span {list(key)}")
+    return warnings
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("path", nargs="?", help="Document file. Reads stdin when omitted.")
+    parser.add_argument(
+        "--apply-replacements",
+        metavar="FILE",
+        help="Merge externally-produced replacements (JSON) into the suggestions.",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str]) -> int:
+    args = parse_args(argv)
+    if args.path:
+        path = Path(args.path)
+        if not path.exists():
+            print(f"Missing file: {path}", file=sys.stderr)
+            return 2
+        text = path.read_text(errors="replace")
+    else:
+        text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
+
+    # English-only graceful decline, matching the two scanners.
+    if not is_probably_english(text):
+        print(json.dumps({"non_english": True, "document": text, "suggestions": [],
+                          "counts": {"total": 0, "hard": 0, "soft": 0,
+                                     "by_category": {}, "structure_flags": []}},
+                         indent=2))
+        print("note: input appears non-English; co-writer declined (English-only).", file=sys.stderr)
+        return 0
+
+    suggestions = build_suggestions(text)
+    struct = structure_scan(text)
+    out = {
+        "document": text,
+        "suggestions": suggestions,
+        "counts": counts_block(suggestions, struct),
+    }
+    if args.apply_replacements:
+        out["apply_warnings"] = apply_replacements(suggestions, args.apply_replacements)
+    print(json.dumps(out, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))