holt-cli 0.1.0__py3-none-any.whl

holt/agent/verdict.py

@@ -0,0 +1,226 @@
+"""The only path from findings to a verdict, and it is a plain function.
+
+No model runs here. Three reasons, all of which matter:
+
+* a judge who reruns Holt gets our numbers, not a resample of them
+* scoring across a pool is not polluted by model variance
+* "is this just a wrapper around a prompt?" is answered by a file rather than
+  an argument
+
+Stage E writes prose *around* this decision and is handed the result as an input
+it cannot change. A test asserts the rendered report and this function agree.
+"""
+
+from __future__ import annotations
+
+from holt.agent.findings import Findings
+from holt.agent.signals import Signals
+from holt.report import Verdict
+
+# Kinds where a merged pull request is not a software contribution. Landing work
+# in these is easy and means nothing for the question being asked.
+NON_SOFTWARE_KINDS = {"registry", "awesome_list", "portfolio", "course_material"}
+
+# Kinds where outside contribution is not accepted regardless of activity.
+CLOSED_KINDS = {"mirror"}
+
+# --- contesting the one field that can decide alone -------------------------
+#
+# `repo_kind` is the only model-derived input either of the rules above reads,
+# and Stage D cannot check it: it verifies that a cited id resolves, and a
+# classification is not a quotation. A model that answers `mirror` while citing
+# a real README has made a claim that verifies perfectly and is false -- which
+# is exactly what happened to `pytorch/pytorch` under a 3B local model, and to
+# `aden-hive/hive`, which was called a registry.
+#
+# So the two kinds that can flip a verdict are checked against the evidence they
+# implicitly claim something about. What is contested is the rule's stated
+# *reason*, never what the repository "really is": a catalogue entry is one file
+# in one place, and a mirror does not merge outsiders' pull requests. Where the
+# evidence disagrees the field is dropped rather than overridden -- no verdict
+# is asserted in its place, `classify` falls through to the arithmetic, and the
+# disagreement is printed for the reader.
+#
+# Pre-registered with its predictions in eval/PREREGISTRATION-4.md; thresholds
+# chosen on pool 1 and that fitting disclosed there.
+CATALOGUE_KINDS = {"registry", "awesome_list"}
+
+# `portfolio` and `course_material` are deliberately not contested this way.
+# Their reason is about whose project it is, not the shape of a diff, and a
+# portfolio being real code is not a contradiction.
+MIN_MERGES_FOR_SHAPE = 5
+
+# One directory. Not "few enough files": that criterion was pre-registered,
+# failed out-of-sample, and is gone. `microsoft/winget-pkgs` is a real registry
+# whose every entry is three YAML manifests -- installer, locale, version -- in
+# one package directory, so a median-files test called a correct classification
+# a hallucination on all four of its recordings. What survived is the criterion
+# that did not misfire: a catalogue entry lands in one place, whatever it
+# weighs. The narrowing was chosen after seeing that failure and is disclosed as
+# such in eval/PREREGISTRATION-4.md; it has no untouched holdout behind it.
+CATALOGUE_DIRS_MAX = 2
+
+# How long the contributor has. Everything time-shaped scales from this, because
+# "is this repository worth my time" has no answer independent of how much time
+# you have: a maintainer who replies in five days is fine if you have three
+# months and useless if you have three days.
+DEFAULT_CONTRIBUTOR_DAYS = 7
+
+# Rubber-stamp rejection. Validated out-of-sample on pool 2 -- specificity 0.58
+# to 0.83 with all three pre-registered predictions holding. Thresholds were
+# chosen on pool 1 and that fitting is disclosed in eval/PREREGISTRATION-2.md.
+#
+# Both halves are required. Landing easily alone describes a welcoming project;
+# going unreviewed alone describes a project whose review happens elsewhere,
+# which is what killed the first rejection rule when nixpkgs was withheld. It is
+# the conjunction that describes work being waved through unread.
+RUBBER_STAMP_REVIEWED_MAX = 0.20
+RUBBER_STAMP_MERGE_RATE_MIN = 0.60
+
+# One merge from one person is an anecdote; two people is a pattern.
+MIN_MERGES = 2
+MIN_DISTINCT_AUTHORS = 2
+
+# Below this share of ignored attempts, silence is noise rather than a policy.
+IGNORED_SHARE = 0.7
+
+# ...and below this many attempts there is no share worth speaking of. Four
+# ignored pull requests out of four is not evidence of hostility, it is four
+# data points. tensorflow/tensorflow reaches exactly that shape: 97% of its pull
+# request traffic is automation, leaving a handful of outsider threads. Without
+# this guard the rule turns a thin sample into a confident accusation.
+MIN_ATTEMPTS_FOR_HOSTILE = 8
+
+
+def contested_kind(
+    findings: Findings, signals: Signals, meta: dict | None = None
+) -> str | None:
+    """Why the claimed `repo_kind` disagrees with the evidence, or None.
+
+    Returns the sentence a reader should see, not a boolean: a field being
+    dropped is a thing that happened to their report and it is printed.
+    """
+    kind = findings.get("repo_kind")
+
+    if kind in CATALOGUE_KINDS:
+        if (
+            signals.merged_with_files >= MIN_MERGES_FOR_SHAPE
+            and signals.merged_dirs_median is not None
+            and signals.merged_dirs_median >= CATALOGUE_DIRS_MAX
+        ):
+            return (
+                f"repo_kind={kind} was claimed, but merged work here spans a "
+                f"median of {signals.merged_dirs_median:g} top-level directories "
+                "rather than landing in one place, which is not a catalogue "
+                "entry; the field is dropped and decided nothing"
+            )
+
+    if kind in CLOSED_KINDS:
+        # `is_mirror` alone would not be enough -- GitHub sets it only for
+        # repositories created as mirrors, so a genuine mirror can report
+        # false. The merges are what disprove the claim being made.
+        if (
+            (meta or {}).get("is_mirror") is False
+            and signals.outsider_merged >= MIN_MERGES
+            and signals.distinct_merged_authors >= MIN_DISTINCT_AUTHORS
+        ):
+            return (
+                f"repo_kind={kind} was claimed, but GitHub does not report this "
+                f"repository as a mirror and {signals.outsider_merged} outside "
+                f"pull requests by {signals.distinct_merged_authors} people were "
+                "merged in the period read; the field is dropped and decided nothing"
+            )
+
+    return None
+
+
+def classify(
+    findings: Findings,
+    signals: Signals,
+    contributor_days: int = DEFAULT_CONTRIBUTOR_DAYS,
+) -> tuple[Verdict, list[str]]:
+    """Return a verdict and the rule trace that produced it.
+
+    `contributor_days` is the time the person actually has. Re-running this
+    function with a different budget costs nothing and calls no model, because
+    the findings are already computed -- which is a thing a single prompt cannot
+    do without paying for the whole assessment again.
+    """
+    trace: list[str] = []
+    slow_response_hours = contributor_days * 24.0
+    kind = findings.get("repo_kind")
+
+    if findings.get("is_archived"):
+        trace.append("archived: no longer accepting work")
+        return Verdict.NOT_VIABLE, trace
+
+    if kind in CLOSED_KINDS:
+        trace.append(f"repo_kind={kind}: outside pull requests are not the contribution path")
+        return Verdict.NOT_VIABLE, trace
+
+    if kind in NON_SOFTWARE_KINDS:
+        trace.append(f"repo_kind={kind}: merged work here is not a software contribution")
+        return Verdict.NOT_VIABLE, trace
+
+    if signals.outsider_threads == 0:
+        # "In the period read", not "before the cutoff": the cutoff is an
+        # evaluation device, and this line is printed verbatim to users.
+        trace.append("no outsider attempts in the period read: nothing to judge from")
+        return Verdict.INSUFFICIENT_EVIDENCE, trace
+
+    ignored_share = signals.outsider_ignored / signals.outsider_threads
+    if (
+        signals.outsider_merged == 0
+        and ignored_share > IGNORED_SHARE
+        and signals.outsider_threads >= MIN_ATTEMPTS_FOR_HOSTILE
+    ):
+        trace.append(
+            f"{signals.outsider_ignored}/{signals.outsider_threads} outsider attempts "
+            "drew no response and none merged"
+        )
+        return Verdict.NOT_VIABLE, trace
+
+    slow = (
+        signals.median_first_response_hours is not None
+        and signals.median_first_response_hours > slow_response_hours
+    )
+    if (
+        signals.outsider_merged >= MIN_MERGES
+        and signals.distinct_outsider_authors >= MIN_DISTINCT_AUTHORS
+        and not slow
+    ):
+        trace.append(
+            f"{signals.outsider_merged} first-time merges by "
+            f"{signals.distinct_merged_authors} distinct people, out of "
+            f"{signals.outsider_threads} attempts by "
+            f"{signals.distinct_outsider_authors}; median first response "
+            f"{signals.median_first_response_hours}h"
+        )
+        if (
+            signals.reviewed_share is not None
+            and signals.merge_rate is not None
+            and signals.reviewed_share < RUBBER_STAMP_REVIEWED_MAX
+            and signals.merge_rate > RUBBER_STAMP_MERGE_RATE_MIN
+        ):
+            trace.append(
+                f"but only {signals.reviewed_share:.0%} of merges drew any human "
+                f"reply while {signals.merge_rate:.0%} of attempts landed: work is "
+                "being waved through unread, so a contribution here buys no review"
+            )
+            return Verdict.NOT_VIABLE, trace
+        return Verdict.VIABLE, trace
+
+    if slow:
+        trace.append(
+            f"median first response {signals.median_first_response_hours}h "
+            f"exceeds the {slow_response_hours:.0f}h a {contributor_days}-day "
+            "budget allows"
+        )
+    if signals.outsider_merged == 0 and ignored_share > IGNORED_SHARE:
+        trace.append(
+            f"{signals.outsider_ignored}/{signals.outsider_threads} attempts ignored, "
+            f"but fewer than {MIN_ATTEMPTS_FOR_HOSTILE} attempts is too thin to call hostile"
+        )
+    elif signals.outsider_merged < MIN_MERGES:
+        trace.append(f"only {signals.outsider_merged} outsider merges in the period read")
+    return Verdict.INSUFFICIENT_EVIDENCE, trace

holt/agent/verify.py

@@ -0,0 +1,140 @@
+"""Stage D — the only stage that removes things.
+
+Two mechanical checks, both of which delete rather than soften.
+
+**Resolution.** Every finding carries the evidence ids that support it. This
+resolves each one against the provider. A finding whose evidence does not
+resolve is dropped: the alternative is prose that hedges around a claim nobody
+can check, which is how unsupported statements survive into reports.
+
+**Quotation.** A claim can cite a pull request that exists and still put words
+in its mouth. `eval/evidence_integrity.py` has measured that gap since Iteration
+11 -- it is the difference between an id that resolves and evidence that says
+what the claim says -- and the check here is the same function the metric uses,
+so the number the eval reports and the guarantee the reader gets cannot drift
+apart. A quote that is not in the record takes its claim with it, for the same
+reason: keeping the outcome while deleting the fabricated quote is softening.
+
+No model runs here. Resolution is a lookup and quotation is string matching.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+
+from holt.agent.findings import Finding, Findings
+from holt.evidence.provider import EvidenceProvider
+from holt.types import EvidenceRecord
+
+# A quote counts as present if a long-enough run of its words appears verbatim
+# in the record. Models normalise whitespace and clip mid-sentence; penalising
+# that would measure formatting rather than fidelity.
+SHINGLE = 6
+
+# `stages._render_thread` prefixes every reply with the speaker -- `[AUTHOR]`,
+# `[octocat]` -- and a model quoting that reply routinely copies the prefix.
+# Nobody said those words: they are ours. Stripping the tag before matching
+# stops the guard blaming the model for our own formatting, which is the same
+# lesson NO_REPLIES taught. What is left still has to be in the record, and a
+# "quote" that is nothing *but* our scaffolding quotes nothing at all.
+SPEAKER_TAG = re.compile(r"^\s*\[[^\]]{1,40}\]\s*")
+
+
+def verify(findings: Findings, provider: EvidenceProvider) -> tuple[Findings, list[Finding]]:
+    """Return (surviving findings, dropped findings)."""
+    kept, dropped = Findings(), []
+    for item in findings:
+        if not item.evidence_ids:
+            dropped.append(item)
+            continue
+        resolved = tuple(e for e in item.evidence_ids if provider.resolve(e) is not None)
+        if not resolved:
+            dropped.append(item)
+            continue
+        kept.add(item.field, item.value, resolved, item.note)
+    return kept, dropped
+
+
+# Punctuation is folded away with the whitespace, for the same reason. A model
+# that quotes ``[`nixpkgs-review`]``(...) as ``[`nixpkgs-review`].`` -- closing a
+# markdown link it truncated -- has quoted the thread; rejecting it would be
+# measuring punctuation. What survives folding is the words, and those still
+# have to be the record's words, in the record's order.
+_NON_WORD = re.compile(r"[^\w]+", re.UNICODE)
+
+
+def normalise(text: str) -> str:
+    return " ".join(_NON_WORD.sub(" ", (text or "").lower()).split())
+
+
+def spoken_part(quote: str) -> str:
+    """The words attributed to a person, with our speaker tag removed."""
+    return SPEAKER_TAG.sub("", quote or "", count=1).strip()
+
+
+def quote_supported(quote: str, haystack: str) -> bool:
+    q, h = normalise(quote), normalise(haystack)
+    if not q:
+        return False
+    words = q.split()
+    if len(words) <= SHINGLE:
+        return q in h
+    return any(
+        " ".join(words[i : i + SHINGLE]) in h for i in range(len(words) - SHINGLE + 1)
+    )
+
+
+def spoken_words(records: Iterable[EvidenceRecord]) -> dict[str, str]:
+    """Everything said anywhere on each pull request, keyed by its number.
+
+    Quotes come from reviews and comments, which live in their own records
+    (`#12:review:0`, `#12:comment:1`) rather than in the `:opened` record a
+    Stage C finding cites. Checking a quote against the cited record alone would
+    reject every real quotation, so the haystack is the whole thread.
+    """
+    said: dict[str, list[str]] = {}
+    for record in records:
+        number = _pr_number(record.evidence_id)
+        if number is None:
+            continue
+        payload = record.payload
+        text = f"{payload.get('title') or ''} {payload.get('body') or ''}"
+        if text.strip():
+            said.setdefault(number, []).append(text)
+    return {k: " ".join(v) for k, v in said.items()}
+
+
+def _pr_number(evidence_id: str) -> str | None:
+    if "#" not in evidence_id:
+        return None
+    return evidence_id.split("#")[-1].split(":")[0]
+
+
+def check_quotes(
+    findings: Findings, records: Iterable[EvidenceRecord]
+) -> tuple[list[Finding], list[Finding]]:
+    """Split findings into (quoting the record, putting words in its mouth).
+
+    Findings that quote nothing pass through untouched -- there is nothing to
+    check and nothing to accuse them of.
+    """
+    said = spoken_words(records)
+    supported, invented = [], []
+    for item in findings:
+        raw = item.value.get("quote", "") if isinstance(item.value, dict) else ""
+        if not raw or not str(raw).strip():
+            supported.append(item)
+            continue
+        quote = spoken_part(str(raw))
+        if not quote:
+            # Entirely our own scaffolding. There is no quotation here to check,
+            # and printing `“[octocat]”` to a reader is not evidence of anything.
+            invented.append(item)
+            continue
+        number = _pr_number(item.evidence_ids[0]) if item.evidence_ids else None
+        if number is not None and quote_supported(quote, said.get(number, "")):
+            supported.append(item)
+        else:
+            invented.append(item)
+    return supported, invented

holt/baseline.py

@@ -0,0 +1,89 @@
+"""The baseline solution: one prompt, the README, and the repository metadata.
+
+This is a *solution*, not a comparator -- it has its own entry point
+(`holt analyze --baseline`) and produces the same Assessment the full pipeline
+does. It is what a competent engineer would build in an afternoon, and it is the
+thing the rest of the system has to beat.
+
+It reads through the same EvidenceProvider, so it is bound by the same holdout
+and runs in the same fixture and replay modes. Same task, same cases, same
+evidence, different method.
+"""
+
+from __future__ import annotations
+
+from holt.evidence.provider import EvidenceProvider
+from holt.model import ModelClient
+from holt.report import Assessment, Claim, Verdict
+
+SYSTEM = """You assess whether a GitHub repository is a worthwhile place for an \
+outside developer -- someone with no prior connection to the project -- to spend \
+a week contributing.
+
+Answer with one of three verdicts:
+  viable                 an outsider could realistically land a meaningful change
+  not_viable             an outsider could not, or the work would not be software
+  insufficient_evidence  the material does not support a call either way
+
+Prefer insufficient_evidence to a confident guess."""
+
+SCHEMA = {
+    "type": "object",
+    "properties": {
+        "verdict": {"type": "string", "enum": [v.value for v in Verdict]},
+        "summary": {"type": "string"},
+        "reasons": {"type": "array", "items": {"type": "string"}},
+    },
+    "required": ["verdict", "summary", "reasons"],
+    "additionalProperties": False,
+}
+
+
+def _prompt(repo: str, meta: dict, readme: str | None) -> str:
+    parts = [f"Repository: {repo}", "", "Metadata:"]
+    for key in (
+        "description",
+        "primary_language",
+        "stargazer_count",
+        "pushed_at",
+        "is_archived",
+        "is_fork",
+        "is_mirror",
+        "homepage_url",
+    ):
+        parts.append(f"  {key}: {meta.get(key)!r}")
+    parts += ["", "README:", readme or "(no README found at the cutoff)"]
+    return "\n".join(parts)
+
+
+def assess(repo: str, provider: EvidenceProvider, model: ModelClient) -> Assessment:
+    records = provider.fetch(repo)
+    meta = next(
+        (r.payload for r in records if r.evidence_id.endswith(":meta")),
+        {},
+    )
+    readme = next(
+        (r.payload.get("text") for r in records if r.evidence_id.endswith(":readme")),
+        None,
+    )
+
+    result = model.complete(
+        label="baseline",
+        system=SYSTEM,
+        prompt=_prompt(repo, meta, readme),
+        schema=SCHEMA,
+    )
+
+    # The baseline cites nothing beyond what it was shown. That is the honest
+    # rendering of a method that never looked at a pull request: its reasons are
+    # impressions of a README, and the report should not dress them as evidence.
+    claims = [Claim(text=r, evidence_id=None) for r in result.get("reasons", [])]
+    return Assessment(
+        repo=repo,
+        verdict=Verdict(result["verdict"]),
+        summary=result["summary"],
+        claims=claims,
+        method="baseline (single prompt over README and metadata)",
+        replayed=model.replayed,
+        models=list(model.usage.models),
+    )

holt/baseline_matched.py

@@ -0,0 +1,116 @@
+"""An evidence-matched baseline: one prompt, everything Holt sees.
+
+The original baseline reads a README and repository metadata. Holt reads two
+hundred pull request threads. Comparing them measures mostly "pull request
+history beats a landing page", which is true but is not the claim the
+architecture is making.
+
+This baseline closes that gap. It receives the *same* arithmetic signals Holt
+computes and the *same* twelve-thread digest Stage C reads, and is asked for the
+same three-valued verdict in a single call. What differs is only the
+architecture: one model call decides, instead of typed findings passing through
+verification into a model-free verdict function.
+
+If Holt does not beat this, the pipeline is not earning its complexity, and that
+is worth knowing and publishing.
+"""
+
+from __future__ import annotations
+
+from holt.agent.signals import build_threads, compute
+from holt.agent.stages import _render_thread
+from holt.evidence.provider import EvidenceProvider
+from holt.model import ModelClient
+from holt.report import Assessment, Claim, Verdict
+
+SYSTEM = """You assess whether a GitHub repository is a worthwhile place for an \
+outside developer -- someone with no prior connection to the project -- to spend \
+a week contributing.
+
+You are given the repository's README, its metadata, arithmetic measured from its
+pull request history before the cutoff, and a sample of its pull request threads.
+
+Judge what a merged contribution here actually *is*. A repository where merged
+work means appending an entry to a catalogue -- package manifests, domain
+records, plugin listings -- is easy to contribute to and is not a place to spend
+a week writing software.
+
+Answer with one of three verdicts:
+  viable                 an outsider could realistically land a meaningful change
+  not_viable             an outsider could not, or the work would not be software
+  insufficient_evidence  the material does not support a call either way
+
+Prefer insufficient_evidence to a confident guess. Cite evidence ids you were
+given for the reasons you list."""
+
+SCHEMA = {
+    "type": "object",
+    "properties": {
+        "verdict": {"type": "string", "enum": [v.value for v in Verdict]},
+        "summary": {"type": "string"},
+        "reasons": {"type": "array", "items": {"type": "string"}},
+    },
+    "required": ["verdict", "summary", "reasons"],
+    "additionalProperties": False,
+}
+
+THREAD_SAMPLE = 12
+
+
+RECORDED_SIGNAL_FIELDS = (
+    "total_threads",
+    "outsider_threads",
+    "outsider_merged",
+    "outsider_ignored",
+    "median_first_response_hours",
+    "bot_share",
+    "distinct_outsider_authors",
+    "distinct_merged_authors",
+    "reviewed_share",
+    "merge_rate",
+)
+
+
+def assess(repo: str, provider: EvidenceProvider, model: ModelClient) -> Assessment:
+    records = provider.fetch(repo)
+    meta = next((r.payload for r in records if r.evidence_id.endswith(":meta")), {})
+    readme = next(
+        (r.payload.get("text") for r in records if r.evidence_id.endswith(":readme")), None
+    )
+    threads = build_threads(records)
+    signals = compute(threads)
+
+    talkative = sorted(
+        (t for t in threads.values() if not t.author_is_bot),
+        key=lambda t: (len(t.responses), t.additions + t.deletions),
+        reverse=True,
+    )[:THREAD_SAMPLE]
+
+    parts = [f"Repository: {repo}", "", "Metadata:"]
+    for key in ("description", "primary_language", "stargazer_count", "pushed_at",
+                "is_archived", "is_fork", "is_mirror", "homepage_url"):
+        parts.append(f"  {key}: {meta.get(key)!r}")
+    parts += ["", "Measured from pull request history before the cutoff:"]
+    # The exact ten fields this prompt was recorded with, listed rather than
+    # taken from `as_dict()`. This is the *comparison* method: a signal added to
+    # the pipeline must not silently change what the baseline is shown, or the
+    # ablation stops being the same experiment -- and every recorded
+    # `baseline_matched` call becomes a replay miss the moment anyone adds one.
+    measured = signals.as_dict()
+    parts += [f"  {k}: {measured[k]}" for k in RECORDED_SIGNAL_FIELDS]
+    parts += ["", "README:", (readme or "(none at the cutoff)")[:6000], ""]
+    parts += ["Pull request threads:", ""]
+    parts += [_render_thread(t) for t in talkative]
+
+    result = model.complete(
+        label="baseline_matched", system=SYSTEM, prompt="\n".join(parts), schema=SCHEMA
+    )
+    return Assessment(
+        repo=repo,
+        verdict=Verdict(result["verdict"]),
+        summary=result["summary"],
+        claims=[Claim(text=r, evidence_id=None) for r in result.get("reasons", [])],
+        method="baseline, evidence-matched (one prompt, the same signals and threads Holt reads)",
+        replayed=model.replayed,
+        models=list(model.usage.models),
+    )