holt-cli 0.1.0__py3-none-any.whl

holt/agent/entry.py

@@ -0,0 +1,86 @@
+"""Entry points: which open issue an outsider should attempt first.
+
+**Not shipped. Kept as the prototype that showed what not to build.**
+
+It was designed, pre-registered and then measured against the comparators that
+could make it unnecessary (`eval/PATHFINDER-DESIGN.md`, written before any of it
+existed). It did not beat them. The measurement is a module constant here, not a
+line in a document the reader will never open, because a ranking whose evaluation
+lives only in the README is an unsupported ranking with the caveat filed where
+nobody looks.
+
+**Why it failed is more useful than the fact that it did.** This ranker never
+sees the contributor. It produces one ranking for everybody, which means it is
+answering "which issues here are generally approachable" -- exactly what a
+`good first issue` label already encodes. It did not lose because the model is
+weak. It lost because it was solving the label's problem, and a tie with the
+label is the expected outcome of that.
+
+Available behind `holt analyze --entry-points`, off by default, and it still
+prints its own measurement when asked for. The successor question -- which issue
+is a sensible next step *for this person, given what they have already merged
+here* -- is a different question, and one no label answers.
+"""
+
+from __future__ import annotations
+
+from holt.agent.signals import build_threads, compute
+from holt.agent.stages import find_paths
+from holt.issues import open_at_cutoff
+from holt.model import ModelClient
+from holt.types import EvidenceRecord
+
+# The signals the ranker is shown. Kept as a constant because the evaluation and
+# the shipped path must build byte-identical prompts, or the recorded trajectories
+# stop replaying and the published number stops describing what users run.
+RANKER_SIGNALS = ("outsider_merged", "outsider_threads", "median_first_response_hours")
+
+#: Measured on both pools. Regenerate with:
+#:     uv run python eval/pathfinder_harness.py --replay
+#:     uv run python eval/pathfinder_harness.py --replay --pool eval/pool2.json \
+#:         --labels eval/results_labels_pool2.json
+MEASURED = {
+    "repositories": 25,
+    "issues_ranked": 3613,
+    "precision_at_3": {"holt": 0.173, "good_first_issue": 0.187, "recency": 0.160, "random": 0.151},
+    "paired_ci_vs_label": (-0.133, 0.120),
+    "sign_test_p": 0.51,
+    "repos_with_no_labelled_issue": 13,
+}
+
+DISCLAIMER = (
+    "> **This ranking is not measurably better than picking at random.** Measured\n"
+    "> over 25 repositories and 3,613 issues held out before the cutoff:\n"
+    "> precision@3 was **0.173** for this ranking, **0.187** for GitHub's\n"
+    "> `good first issue` label and **0.151** for a random pick — differences well\n"
+    "> inside noise (paired 95% CI [−0.13, +0.12], sign test p = 0.51).\n"
+    ">\n"
+    "> It is printed anyway because **13 of those 25 repositories had no\n"
+    "> beginner-labelled issue at all**, so on half of them there is no free signal\n"
+    "> to lose to. Read it as a reading order, not a recommendation.\n"
+    ">\n"
+    "> Check it yourself: `uv run python eval/pathfinder_harness.py --replay`"
+)
+
+
+def rank(
+    repo: str,
+    issue_records: list[EvidenceRecord],
+    pull_records: list[EvidenceRecord],
+    model: ModelClient,
+) -> list[dict]:
+    """Rank the issues open at the cutoff. `[]` when there are none.
+
+    Called by both `eval/pathfinder_harness.py` and the CLI, so the ranking a
+    user sees is produced by the same code path as the ranking that was scored.
+    """
+    candidates = open_at_cutoff(issue_records)
+    if not candidates:
+        return []
+    signals = compute(build_threads(pull_records)).as_dict()
+    return find_paths(
+        repo,
+        list(candidates.values()),
+        {k: signals[k] for k in RANKER_SIGNALS},
+        model,
+    )

holt/agent/findings.py

@@ -0,0 +1,49 @@
+"""Typed findings: what the stages produce, before anything becomes a verdict."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class Finding:
+    """One field, its value, and the evidence that supports it.
+
+    A finding with no resolvable evidence is dropped by Stage D rather than
+    softened, so `evidence_ids` is what keeps a claim alive.
+    """
+
+    field: str
+    value: Any
+    evidence_ids: tuple[str, ...] = ()
+    note: str = ""
+
+
+@dataclass(slots=True)
+class Findings:
+    items: list[Finding] = field(default_factory=list)
+
+    def add(self, field_name: str, value: Any, evidence_ids=(), note: str = "") -> None:
+        self.items.append(Finding(field_name, value, tuple(evidence_ids), note))
+
+    def drop(self, field_name: str) -> None:
+        """Remove a field entirely.
+
+        Used where two sources of evidence disagree: the standing rule is to
+        drop the contested field rather than pick a side, and a field that is
+        dropped must not survive in the claim list either.
+        """
+        self.items = [i for i in self.items if i.field != field_name]
+
+    def get(self, field_name: str, default: Any = None) -> Any:
+        for item in self.items:
+            if item.field == field_name:
+                return item.value
+        return default
+
+    def __iter__(self):
+        return iter(self.items)
+
+    def __len__(self) -> int:
+        return len(self.items)

holt/agent/landing.py

@@ -0,0 +1,154 @@
+"""Where outsider work actually landed, and where it never did.
+
+Every pull request Holt reads carries its file list. Until now that list was used
+for exactly one thing -- deciding whether a diff was substantial enough to count
+-- and then discarded. This counts it instead.
+
+The output is the sentence a newcomer most needs and cannot get anywhere:
+*in a tree of two hundred thousand files, these three directories are where
+strangers' work has actually been merged, and these are where strangers tried and
+never succeeded.* GitHub does not show it, CONTRIBUTING does not say it, and it is
+not inferable from stars, issue labels or commit frequency.
+
+**It ranks nothing and predicts nothing.** After five capabilities cut for losing
+to a cheap comparator, that is deliberate: there is no ordering to be beaten here,
+only a count of what happened. Arithmetic over evidence already in hand, no model
+call, and nothing that could disagree with the verdict.
+
+Read `attempted but never landed` carefully -- it is a description of this
+sample, not a prohibition. A directory can appear there because outsiders are
+turned away from it, or because only two people ever tried.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass
+
+from holt.agent.signals import Thread, newcomer_threads
+
+# Two path segments. One is too coarse to act on in a monorepo (`pkgs`, `src`);
+# three splits the same area into a dozen near-identical rows.
+DEPTH = 2
+MIN_ATTEMPTS = 2
+TOP_N = 4
+
+# When two segments barely group anything -- a plugin registry where every entry
+# is its own directory produces ninety areas from a hundred pull requests -- the
+# rows read as insight while carrying none. Above this ratio of areas to pull
+# requests, fall back to one segment, which for that registry correctly collapses
+# to a single honest row: everything lands in `plugins`.
+REGROUP_ABOVE = 0.5
+
+
+@dataclass(frozen=True, slots=True)
+class Area:
+    path: str
+    landed: int
+    attempted: int
+
+    @property
+    def rate(self) -> float:
+        return self.landed / self.attempted if self.attempted else 0.0
+
+
+@dataclass(frozen=True, slots=True)
+class Landing:
+    """Where outsiders got in, and where they did not."""
+
+    landed: list[Area]
+    never: list[Area]
+    outsider_threads: int
+    outsider_merges: int
+    depth: int = DEPTH
+
+    def __bool__(self) -> bool:
+        return bool(self.landed or self.never)
+
+
+def area_of(path: str, depth: int = DEPTH) -> str:
+    """`pkgs/by-name/fo/foo/package.nix` -> `pkgs/by-name`. A root file -> `(root)`."""
+    parts = path.split("/")
+    if len(parts) == 1:
+        return "(root)"
+    return "/".join(parts[:depth])
+
+
+def _tally(outsiders: list[Thread], depth: int) -> tuple[Counter, Counter]:
+    landed: Counter = Counter()
+    attempted: Counter = Counter()
+    for thread in outsiders:
+        # Count each area once per pull request. A change touching forty files in
+        # one directory is one attempt at that directory, not forty.
+        for area in sorted({area_of(f, depth) for f in (thread.files or [])}):
+            attempted[area] += 1
+            if thread.merged:
+                landed[area] += 1
+    return landed, attempted
+
+
+def compute(threads: dict[str, Thread]) -> Landing:
+    outsiders = newcomer_threads(threads)
+    depth = DEPTH
+    landed, attempted = _tally(outsiders, depth)
+    if outsiders and len(attempted) > REGROUP_ABOVE * len(outsiders):
+        depth = 1
+        landed, attempted = _tally(outsiders, depth)
+
+    # `Counter.most_common` breaks ties by insertion order, and insertion order
+    # here was set-iteration order, which varies with the process hash seed. Two
+    # replays of the same recording printed different fourth rows -- a report
+    # this project claims is a function of the evidence quietly was not. Ordered
+    # explicitly instead: most merges first, then the better odds, then the
+    # larger sample, then alphabetically, so the order is total.
+    got_in = [
+        Area(a, landed[a], attempted[a])
+        for a in sorted(
+            landed,
+            key=lambda a: (-landed[a], -landed[a] / attempted[a], -attempted[a], a),
+        )[:TOP_N]
+    ]
+    never = [
+        Area(a, 0, attempted[a])
+        for a in sorted(attempted, key=lambda a: (-attempted[a], a))
+        if landed.get(a, 0) == 0 and attempted[a] >= MIN_ATTEMPTS
+    ][:TOP_N]
+
+    return Landing(
+        landed=got_in,
+        never=never,
+        outsider_threads=len(outsiders),
+        outsider_merges=sum(1 for t in outsiders if t.merged),
+        depth=depth,
+    )
+
+
+def render(landing: Landing) -> list[str]:
+    """Markdown lines, or nothing at all when there is nothing to say."""
+    if not landing:
+        return []
+    lines = ["## Where outsider work landed", ""]
+    lines.append(
+        f"Counted over the {landing.outsider_threads} pull requests opened here by "
+        f"people with no prior merge, of which {landing.outsider_merges} were merged. "
+        + ("Paths are cut to their first two segments, so a short path may name a "
+           "file rather than a directory." if landing.depth == 2 else
+           "Paths are cut to their first segment: at two, almost every pull request "
+           "here landed in a directory of its own, which groups nothing.")
+    )
+    lines.append("")
+    for area in landing.landed:
+        lines.append(
+            f"- **`{area.path}`** — {area.landed} merged "
+            f"of {area.attempted} attempted ({area.rate:.0%})"
+        )
+    if landing.never:
+        lines.append("")
+        joined = ", ".join(f"`{a.path}` ({a.attempted})" for a in landing.never)
+        lines.append(
+            f"Outsiders attempted these and none were merged: {joined}. "
+            f"That is what this sample shows, not a rule — a directory can appear "
+            f"here because newcomers are turned away from it, or because only a "
+            f"couple of people ever tried."
+        )
+    return lines

holt/agent/pipeline.py

@@ -0,0 +1,244 @@
+"""A -> B -> C -> D -> verdict -> E.
+
+The ordering that matters: the verdict is computed *before* narration and handed
+to Stage E as an input it cannot alter. If the report and verdict.py could
+disagree, the determinism claim would be worth nothing.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from holt.agent import landing, stages
+from holt.agent.findings import Finding, Findings
+from holt.agent.signals import Signals, build_threads, compute
+from holt.agent.verdict import classify as decide
+from holt.agent.verdict import contested_kind
+from holt.agent.verify import check_quotes, verify
+from holt.evidence.provider import EvidenceProvider
+from holt.model import ModelClient
+from holt.report import VERDICT_HEADLINES, Assessment, Claim, Verdict
+
+MAX_CLAIM_CHARS = 240
+MAX_QUOTE_CHARS = 180
+
+
+def clip(text: str, limit: int) -> str:
+    """Cut on a word boundary. Cutting mid-word reads as a bug, because it is one."""
+    text = " ".join(text.split())
+    if len(text) <= limit:
+        return text
+    head = text[:limit]
+    cut = max(head.rfind(" "), head.rfind(". "))
+    return (head[:cut] if cut > limit // 2 else head).rstrip(" ,;:.") + "…"
+
+
+@dataclass(slots=True)
+class Trace:
+    """What happened, for the demo and the trajectory record."""
+
+    signals: Signals
+    before_verification: int = 0
+    after_verification: int = 0
+    dropped: list[Finding] = field(default_factory=list)
+    # Findings whose id resolved but whose quotation is not in the record.
+    invented: list[Finding] = field(default_factory=list)
+    rules: list[str] = field(default_factory=list)
+
+
+def analyze(
+    repo: str,
+    provider: EvidenceProvider,
+    model: ModelClient | None,
+    contributor_days: int = 7,
+    as_of: datetime | None = None,
+) -> tuple[Assessment, Trace]:
+    if model is None:
+        return analyze_without_model(repo, provider, contributor_days, as_of)
+    records = provider.fetch(repo)
+    threads = build_threads(records)
+    signals = compute(threads)
+
+    findings = Findings()
+    stages.classify(repo, records, threads, model, findings)
+    stages.assess_opportunity(repo, records, model, findings)
+    stages.read_outcomes(repo, threads, model, findings)
+
+    before = len(findings)
+    findings, dropped = verify(findings, provider)
+
+    # `repo_kind` is the only model-derived field that can decide the answer by
+    # itself, and Stage D cannot check it -- an id resolving says nothing about
+    # whether a classification is true. Where the evidence contradicts the
+    # reason the kind rule would give, the field is dropped before it decides
+    # anything and the disagreement is printed. See eval/PREREGISTRATION-4.md.
+    meta = next((r for r in records if r.evidence_id.endswith(":meta")), None)
+    contested = contested_kind(findings, signals, meta.payload if meta else None)
+    if contested:
+        findings.drop("repo_kind")
+
+    verdict, rules = decide(findings, signals, contributor_days)
+    if contested:
+        rules.insert(0, contested)
+    # The narration prompt is deliberately held to the signal fields that existed
+    # when the trajectories were recorded. New signals reach the *verdict*
+    # immediately but only reach the prose on the next re-record, so adding one
+    # does not invalidate every committed trajectory and break replay for a judge.
+    # When a new signal changes the outcome it still reaches the narrator, via
+    # the rule trace.
+    narrated_signals = {
+        k: v for k, v in signals.as_dict().items()
+        if k not in ("reviewed_share", "merge_rate", "merged_files_median",
+                     "merged_dirs_median", "merged_with_files")
+    }
+    narrated = stages.narrate(
+        repo, verdict.value, rules, findings, narrated_signals, model
+    )
+
+    # The evidence list is built from verified findings, not written by the
+    # model. Stage E supplies prose; it cannot introduce a citation.
+    #
+    # The quote check runs here rather than inside Stage D on purpose. A claim
+    # whose id does not resolve is worthless to everyone, narrator included, so
+    # `verify` removes it before anything else runs. A claim whose id resolves
+    # but whose words are not in the record is a different failure: the thread
+    # is real and the outcome may well be right, and what must not reach the
+    # reader is the quotation. Filtering the claim list is exactly that, and it
+    # leaves the narration prompt byte-identical, so every committed trajectory
+    # still replays -- a guarantee that would otherwise cost a re-record of the
+    # frozen benchmark to buy.
+    quoting, invented = check_quotes(findings, records)
+    claims: list[Claim] = []
+    for item in quoting:
+        if item.field == "thread_outcome":
+            outcome = item.value["outcome"].replace("_", " ")
+            quote = (item.value.get("quote") or "").strip()
+            # An empty quote used to render as a pair of quotation marks with
+            # nothing between them, which reads as a bug because it is one.
+            text = (f"{outcome} — “{clip(quote, MAX_QUOTE_CHARS)}”" if quote
+                    else f"{outcome}, nothing said")
+        else:
+            text = f"{item.field.replace('_', ' ')}: {item.value}" + (
+                f" — {clip(item.note, MAX_CLAIM_CHARS)}" if item.note else "")
+        claims.append(Claim(text=text, evidence_id=item.evidence_ids[0]))
+
+    assessment = Assessment(
+        repo=repo,
+        verdict=verdict,
+        summary=narrated["what_the_evidence_shows"],
+        bottom_line=narrated["bottom_line"],
+        limits=narrated["what_could_not_be_determined"],
+        rules=list(rules),
+        contributor_days=contributor_days,
+        as_of=as_of,
+        landing=landing.render(landing.compute(threads)),
+        claims=claims,
+        method="holt (A classify, B opportunity, C outcomes, D verify, deterministic verdict, E narrate)",
+        replayed=model.replayed,
+        models=list(model.usage.models),
+        dropped_claims=len(dropped) + len(invented),
+    )
+    return assessment, Trace(
+        signals=signals,
+        before_verification=before,
+        after_verification=len(findings),
+        dropped=dropped,
+        invented=invented,
+        rules=rules,
+    )
+
+
+# --- degraded mode -----------------------------------------------------------
+#
+# The project measured its own model stages at +0.01 MCC over the arithmetic
+# (Iteration 22). A finding that large about your own architecture should change
+# the architecture, not just the write-up: if the rules decide the verdict, the
+# verdict must be obtainable without a model, and the reader must be told what
+# they lost. That is this function.
+#
+# It is not a second implementation of the verdict. It calls the same `decide`
+# on the same `Signals`, so the two modes cannot disagree about a repository
+# they both have findings for -- a test asserts exactly that over the pool.
+# What it does not do is write: Stages A, B, C and E never run, so there are no
+# thread quotes, no narration, and no `repo_kind`. That absence is the point of
+# `eval/evidence_integrity.py`'s yield column, and it is stated on the report
+# rather than left for the reader to notice.
+
+NO_MODEL_METHOD = (
+    "holt --no-model (deterministic verdict from arithmetic; "
+    "stages A, B, C and E did not run)"
+)
+
+
+def analyze_without_model(
+    repo: str,
+    provider: EvidenceProvider,
+    contributor_days: int = 7,
+    as_of: datetime | None = None,
+) -> tuple[Assessment, Trace]:
+    """The verdict, with no model call anywhere and the cost of that printed."""
+    records = provider.fetch(repo)
+    threads = build_threads(records)
+    signals = compute(threads)
+
+    findings = Findings()
+    # `is_archived` is a structured GitHub field. Stage A was asking a model to
+    # read a boolean the provider already had, which is the clearest single
+    # illustration of why the model stages measured +0.01: some of what they
+    # were doing did not need a model at all. Here it is taken from the record
+    # and cited to it.
+    meta = next((r for r in records if r.evidence_id.endswith(":meta")), None)
+    if meta is not None and meta.payload.get("is_archived"):
+        findings.add("is_archived", True, (meta.evidence_id,),
+                     "GitHub reports this repository as archived")
+
+    verdict, rules = decide(findings, signals, contributor_days)
+
+    s = signals.as_dict()
+    if signals.outsider_threads:
+        summary = (
+            f"{s['outsider_merged']} of {s['outsider_threads']} outsider pull "
+            f"requests merged, by {s['distinct_merged_authors']} distinct people "
+            f"out of {s['distinct_outsider_authors']} who tried"
+        )
+        if s["median_first_response_hours"] is not None:
+            summary += f"; median first response {s['median_first_response_hours']}h"
+        summary += (
+            f"; {s['outsider_ignored']} drew no response at all. "
+            "Counted from the pull request record, not judged."
+        )
+    else:
+        summary = (
+            "No outsider pull requests in the period read, so the arithmetic has "
+            "nothing to count."
+        )
+
+    return Assessment(
+        repo=repo,
+        verdict=verdict,
+        summary=summary,
+        bottom_line=f"{VERDICT_HEADLINES[verdict]}. " + (rules[0] if rules else ""),
+        limits=(
+            "No model ran. This report is the verdict and the rule that produced "
+            "it; the parts a model writes — what specific threads said, who was "
+            "welcoming, what kind of project this is, and the prose explaining "
+            "any of it — are absent, not merely brief. Measured: this mode scores "
+            "MCC +0.60 against the full pipeline's +0.61 in sample, but +0.55 "
+            "against +0.63 out of sample, and writes 0 citable statements against "
+            "its 11.8 (eval/evidence_integrity.py). Run without --no-model for a "
+            "report you can check against the record."
+        ),
+        rules=list(rules),
+        contributor_days=contributor_days,
+        as_of=as_of,
+        landing=landing.render(landing.compute(threads)),
+        claims=[
+            Claim(text=f"{i.field.replace('_', ' ')}: {i.value}", evidence_id=i.evidence_ids[0])
+            for i in findings
+        ],
+        method=NO_MODEL_METHOD,
+        replayed=False,
+        models=[],
+        dropped_claims=0,
+    ), Trace(signals=signals, rules=rules)