psyclaw 0.27.22 → 0.28.0

package/vendor/ars/scripts/test_431_exact_or_bust.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+"""#431 — exact-title-or-bust acceptance regression tests.
+
+The v5 architecture pivot (spec docs/design/2026-06-13-431-title-match-hardening-spec.md
+§0.12): a NON-exact title is never promoted to `matched` on title-similarity +
+year/author alone — that high-overlap-same-author signal is the shared signature
+of an author's own related-but-distinct works (a correction and its original, a
+reply and its target, Part I / Part II, a no-ordinal companion). The only
+disambiguator all four resolvers can compute is the exact normalized title.
+
+These tests pin the behavior the pre-existing client tests did NOT cover: the
+pre-#431 clients returned the highest-ratio candidate regardless of exactness,
+so a high-ratio related work was wrongly `matched`. Each client test below feeds
+a candidate that clears the 0.70 ratio but is NOT an exact normalized title and
+asserts the title fallback now yields no match; the legitimate exact / acronym /
+case variants still match; F3 ordering reaches a correct exact #2 past a
+non-exact #1; an exact-but-generic title with no corroborating ID stays
+unmatched; and the resolver→reducer chain maps a title-keyed miss to
+`unresolvable` (never a false `false`).
+"""
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+if str(REPO_ROOT / "scripts") not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+
+def _json_resp(payload: dict) -> MagicMock:
+    resp = MagicMock()
+    resp.read.return_value = json.dumps(payload).encode("utf-8")
+    resp.__enter__ = MagicMock(return_value=resp)
+    resp.__exit__ = MagicMock(return_value=None)
+    return resp
+
+
+def _atom(*titles: str) -> bytes:
+    entries = "".join(
+        "<entry>"
+        "<id>http://arxiv.org/abs/1706.03762v5</id>"
+        f"<title>{t}</title>"
+        "<published>2017-06-12T00:00:00Z</published>"
+        "</entry>"
+        for t in titles
+    )
+    body = (
+        '<?xml version="1.0" encoding="UTF-8"?>'
+        '<feed xmlns="http://www.w3.org/2005/Atom">'
+        + entries
+        + "</feed>"
+    )
+    return body.encode("utf-8")
+
+
+def _atom_resp(*titles: str) -> MagicMock:
+    resp = MagicMock()
+    resp.read.return_value = _atom(*titles)
+    resp.__enter__ = MagicMock(return_value=resp)
+    resp.__exit__ = MagicMock(return_value=None)
+    return resp
+
+
+# A real R4 false-positive (spec §0.12): same first 4 words, ratio ~0.90, but a
+# DISTINCT work — different topic in the tail. Non-exact ⇒ must NOT match.
+_CITED = "Federated Learning for Mobile Keyboard Prediction"
+_RELATED_WORK = "Federated Learning for Mobile Health Prediction"
+
+
+class CrossrefExactOrBustTest(unittest.TestCase):
+    def _search(self, *cand_titles, year=None):
+        from crossref_client import CrossrefClient
+
+        payload = {"message": {"items": [{"title": [t]} for t in cand_titles]}}
+        with patch("urllib.request.urlopen", return_value=_json_resp(payload)):
+            return CrossrefClient().title_search(_CITED, year=year)
+
+    def test_non_exact_high_ratio_does_not_match(self):
+        self.assertIsNone(self._search(_RELATED_WORK))
+
+    def test_exact_title_matches(self):
+        result = self._search(_CITED)
+        self.assertIsNotNone(result)
+        self.assertEqual(result["title"], [_CITED])
+
+    def test_acronym_variant_matches(self):
+        from crossref_client import CrossrefClient
+
+        payload = {"message": {"items": [
+            {"title": ["RAG: Retrieval Augmented Generation"]}
+        ]}}
+        with patch("urllib.request.urlopen", return_value=_json_resp(payload)):
+            result = CrossrefClient().title_search(
+                "R.A.G.: Retrieval Augmented Generation"
+            )
+        self.assertIsNotNone(result)  # dotted-acronym normalizes byte-equal
+
+    def test_f3_ordering_reaches_exact_second_candidate(self):
+        # #1 is the high-ratio related work (non-exact), #2 is the exact work.
+        # Pre-#431 returned #1 and dropped the real work to unresolvable.
+        result = self._search(_RELATED_WORK, _CITED)
+        self.assertIsNotNone(result)
+        self.assertEqual(result["title"], [_CITED])
+
+    def test_generic_exact_title_no_id_does_not_match(self):
+        # §0.12.2: a bare generic title is never promoted on the title path
+        # (no ID can corroborate here). Even an exact 'Editorial'/'Editorial'
+        # returns no match.
+        from crossref_client import CrossrefClient
+
+        payload = {"message": {"items": [{"title": ["Editorial"]}]}}
+        with patch("urllib.request.urlopen", return_value=_json_resp(payload)):
+            result = CrossrefClient().title_search("Editorial")
+        self.assertIsNone(result)
+
+
+class OpenAlexExactOrBustTest(unittest.TestCase):
+    def _search(self, *cand_titles, year=None):
+        from openalex_client import OpenAlexClient
+
+        payload = {"results": [{"title": t} for t in cand_titles]}
+        with patch("urllib.request.urlopen", return_value=_json_resp(payload)):
+            return OpenAlexClient().title_search(_CITED, year=year)
+
+    def test_non_exact_high_ratio_does_not_match(self):
+        self.assertIsNone(self._search(_RELATED_WORK))
+
+    def test_exact_title_matches(self):
+        result = self._search(_CITED)
+        self.assertIsNotNone(result)
+        self.assertEqual(result["title"], _CITED)
+
+    def test_generic_exact_title_no_id_does_not_match(self):
+        from openalex_client import OpenAlexClient
+
+        payload = {"results": [{"title": "Editorial"}]}
+        with patch("urllib.request.urlopen", return_value=_json_resp(payload)):
+            self.assertIsNone(OpenAlexClient().title_search("Editorial"))
+
+
+class ArxivExactOrBustTest(unittest.TestCase):
+    def _search(self, *cand_titles, year=None):
+        from arxiv_client import ArxivClient
+
+        with patch("urllib.request.urlopen", return_value=_atom_resp(*cand_titles)):
+            return ArxivClient().title_search(_CITED, year=year)
+
+    def test_non_exact_high_ratio_does_not_match(self):
+        self.assertIsNone(self._search(_RELATED_WORK))
+
+    def test_exact_title_matches(self):
+        result = self._search(_CITED)
+        self.assertIsNotNone(result)
+        self.assertEqual(result["title"], _CITED)
+
+    def test_generic_exact_title_no_id_does_not_match(self):
+        from arxiv_client import ArxivClient
+
+        with patch("urllib.request.urlopen", return_value=_atom_resp("Editorial")):
+            self.assertIsNone(ArxivClient().title_search("Editorial"))
+
+
+class SemanticScholarExactOrBustTest(unittest.TestCase):
+    def _search(self, *cand_titles, year=None):
+        import semantic_scholar_client as ssc
+
+        payload = {"data": [{"title": t, "paperId": f"p{i}"}
+                            for i, t in enumerate(cand_titles)]}
+        resp = _json_resp(payload)
+        with patch("urllib.request.urlopen", return_value=resp):
+            client = ssc.SemanticScholarClient(sleep=MagicMock())
+            return client._lookup_by_title(_CITED, year)
+
+    def test_non_exact_high_ratio_does_not_match(self):
+        self.assertIs(self._search(_RELATED_WORK)["matched"], False)
+
+    def test_exact_title_matches(self):
+        self.assertIs(self._search(_CITED)["matched"], True)
+
+    def test_f3_ordering_reaches_exact_second_candidate(self):
+        result = self._search(_RELATED_WORK, _CITED)
+        self.assertIs(result["matched"], True)
+
+    def test_generic_exact_title_no_id_does_not_match(self):
+        self.assertIs(self._search("Editorial")["matched"], False)
+
+
+class ExactNormalizedTitleHelperTest(unittest.TestCase):
+    """Direct pins on the exact_normalized_title helper (#431 §0.12.1). The
+    acronym pre-pass must only ADD matches over the base normalization, never
+    drop one the base form already had — codex round-6 P2 caught that an
+    acronym-only equality check regressed a punctuation-only variant where
+    exactly one side is a contiguous initialism."""
+
+    def test_base_equal_punctuation_variant_matches(self):
+        from _text_similarity import exact_normalized_title
+
+        # base form makes these byte-equal ('d h lawrence'), but the acronym
+        # form does not ('dh' vs 'd h') — must still be exact via base equality.
+        self.assertTrue(exact_normalized_title("D.H. Lawrence", "D. H. Lawrence"))
+        self.assertTrue(exact_normalized_title("U.S. policy", "U. S. policy"))
+        self.assertTrue(
+            exact_normalized_title("U.S.A. policy", "U. S. A. policy")
+        )
+
+    def test_acronym_only_equal_variant_matches(self):
+        from _text_similarity import exact_normalized_title
+
+        # base form differs ('r a g x' vs 'rag x'); acronym collapse makes them
+        # equal — the original #431 acronym carve-out.
+        self.assertTrue(exact_normalized_title("R.A.G.: x", "RAG: x"))
+
+    def test_distinct_initialisms_do_not_match(self):
+        from _text_similarity import exact_normalized_title
+
+        # different initials are a different work — neither form equal.
+        self.assertFalse(
+            exact_normalized_title("D.H. Lawrence", "D.K. Lawrence")
+        )
+
+
+class ResolverVerdictNarrowingTest(unittest.TestCase):
+    """End-to-end (spec §0.5 / §0.12.1): a non-exact title fallback on a
+    title-only entry (no resolvable ID) must reduce to `unresolvable`, NEVER
+    `false`. `false` is reserved for an ID-keyed miss (C-V6(a))."""
+
+    def test_title_only_non_exact_reduces_to_unresolvable(self):
+        import contamination_signals as cs
+        from citation_verification_summary import reduce_lookup_verified
+
+        # No DOI ⇒ queried_by='title'. Client title_search returns None (the v5
+        # non-exact verdict). The resolver flow reports unmatched + title-keyed.
+        client = MagicMock()
+        client.title_search.return_value = None
+        entry = {"citation_key": "x", "title": _CITED}  # no doi
+        unmatched, matched_by, queried_by = cs._resolve_doi_then_title(entry, client)
+        self.assertIs(unmatched, True)
+        self.assertIsNone(matched_by)
+        self.assertEqual(queried_by, "title")
+
+        # The reducer maps a title-keyed unmatched to `unresolvable`, not false.
+        verdict = reduce_lookup_verified({
+            "crossref": {"status": "unmatched", "queried_by": "title"},
+        })
+        self.assertEqual(verdict, "unresolvable")
+
+
+if __name__ == "__main__":
+    unittest.main()

package/vendor/ars/scripts/test__eval_threshold_gate.py

@@ -0,0 +1,125 @@
+"""Tests for scripts/_eval_threshold_gate.py (#184 Delta 3 / Fix 1; #328 per-class gate).
+
+The absolute-threshold gate must fail a PR when ANY declared binding threshold
+regresses — aggregate OR per-class. Manifests declare both (e.g.
+citation_extraction: aggregate accuracy >= 0.90 AND per_class accuracy >= 0.85),
+run_evals stamps ``per_class[].passed`` against the per-class threshold, and the
+gate must honour it. #328: the gate previously inspected only the aggregate, so a
+per-class-fail-but-aggregate-pass report passed the gate when it should block.
+"""
+from __future__ import annotations
+
+from scripts import _eval_threshold_gate as gate
+
+
+def _task(task_name="citation_extraction", agg_metric="accuracy",
+          agg_passed=True, per_class=None, status="measured"):
+    entry = {
+        "task_name": task_name,
+        "status": status,
+        "aggregate_metric": {"metric": agg_metric, "passed": agg_passed},
+    }
+    if per_class is not None:
+        entry["per_class"] = per_class
+    return entry
+
+
+def _report(*tasks):
+    return {"per_task": list(tasks)}
+
+
+# ---------------------------------------------------------------------------
+# Aggregate gate (pre-#328 behaviour, must stay)
+# ---------------------------------------------------------------------------
+def test_aggregate_pass_no_failure():
+    assert gate.failed_tasks(_report(_task(agg_passed=True))) == []
+
+
+def test_aggregate_fail_reported():
+    assert gate.failed_tasks(_report(_task(agg_passed=False))) == [
+        "citation_extraction.aggregate.accuracy"
+    ]
+
+
+def test_no_threshold_not_gated():
+    # aggregate without a ``passed`` key (task declares no threshold) is not gated
+    t = {"task_name": "t", "status": "measured", "aggregate_metric": {"metric": "accuracy"}}
+    assert gate.failed_tasks(_report(t)) == []
+
+
+def test_pending_task_never_gated():
+    assert gate.failed_tasks(_report(_task(status="pending", agg_passed=False))) == []
+
+
+def test_unknown_nonmeasured_status_never_gated():
+    # symmetric with the lift gate: a future non-measured status is skipped, not gated
+    assert gate.failed_tasks(_report(_task(status="error", agg_passed=False))) == []
+
+
+def test_missing_status_treated_as_measured():
+    # back-compat: a task with no status key is gated (pre-status reports stay valid)
+    t = {"task_name": "t", "aggregate_metric": {"metric": "accuracy", "passed": False}}
+    assert gate.failed_tasks(_report(t)) == ["t.aggregate.accuracy"]
+
+
+# ---------------------------------------------------------------------------
+# Per-class gate (#328 — the hole)
+# ---------------------------------------------------------------------------
+def test_per_class_fail_with_aggregate_pass_is_reported():
+    """The #328 regression case: aggregate passes, a binding per-class fails.
+
+    e.g. citation_extraction.false.accuracy drops below 0.85 while the aggregate
+    stays >= 0.90. The gate MUST surface it.
+    """
+    report = _report(_task(
+        agg_passed=True,
+        per_class=[
+            {"class_name": "true", "metric": "accuracy", "passed": True},
+            {"class_name": "false", "metric": "accuracy", "passed": False},
+            {"class_name": "unresolvable", "metric": "accuracy", "passed": True},
+        ],
+    ))
+    assert gate.failed_tasks(report) == ["citation_extraction.false.accuracy"]
+
+
+def test_per_class_key_shape_matches_lift_gate():
+    """Per-class failures use the ``<task>.<class>.<metric>`` shape (matches the
+    lift gate's key shape, not the aggregate's ``<task>.aggregate.<metric>``)."""
+    report = _report(_task(
+        task_name="rq_framing_patterns", agg_metric="balanced_accuracy",
+        agg_passed=True,
+        per_class=[
+            {"class_name": "fnr", "metric": "fnr", "passed": False},
+            {"class_name": "fpr", "metric": "fpr", "passed": True},
+        ],
+    ))
+    assert gate.failed_tasks(report) == ["rq_framing_patterns.fnr.fnr"]
+
+
+def test_both_aggregate_and_per_class_fail_reported():
+    report = _report(_task(
+        agg_passed=False,
+        per_class=[{"class_name": "false", "metric": "accuracy", "passed": False}],
+    ))
+    assert gate.failed_tasks(report) == [
+        "citation_extraction.aggregate.accuracy",
+        "citation_extraction.false.accuracy",
+    ]
+
+
+def test_per_class_without_threshold_not_gated():
+    # a per_class row that carries no ``passed`` key (no per-class threshold) is
+    # not gated — only ``passed is False`` blocks
+    report = _report(_task(
+        agg_passed=True,
+        per_class=[{"class_name": "true", "metric": "accuracy"}],
+    ))
+    assert gate.failed_tasks(report) == []
+
+
+def test_per_class_on_pending_task_never_gated():
+    report = _report(_task(
+        status="pending", agg_passed=True,
+        per_class=[{"class_name": "false", "metric": "accuracy", "passed": False}],
+    ))
+    assert gate.failed_tasks(report) == []

package/vendor/ars/scripts/test__markdown_lint_util.py

@@ -0,0 +1,129 @@
+"""Direct tests for the shared markdown grammar (#771).
+
+The consumer lints' mutation suites pin their invariants end-to-end; this
+suite pins the grammar itself once, so a grammar change is reasoned about
+here rather than across three fixture repos.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from _markdown_lint_util import (
+    RelativeLinkFailure,
+    anchor_resolves,
+    extract_link_targets,
+    github_slug,
+    heading_slugs,
+    links_to,
+    relative_link_failure,
+    strip_non_rendering,
+)
+
+
+@pytest.mark.parametrize(
+    ("md_text", "expected"),
+    [
+        ("[label](docs/a.md)", ["docs/a.md"]),
+        # Titled link: the target is captured, the title is not.
+        ('[label](docs/a.md "the title")', ["docs/a.md"]),
+        # An image renders no anchor.
+        ("![alt](image.png)", []),
+        # A link inside backticks renders literally.
+        ("`[label](docs/a.md)`", []),
+        # A link inside a fence renders literally.
+        ("```\n[label](docs/a.md)\n```", []),
+        # A four-backtick fence is not closed by a ``` example line.
+        ("````\n```\n[label](docs/a.md)\n```\n````", []),
+        # A type-2 HTML comment block swallows its lines.
+        ("<!--\n[label](docs/a.md)\n-->", []),
+        # An inline comment span is raw HTML; surrounding links render.
+        ("<span><!-- x --></span> [label](docs/a.md)", ["docs/a.md"]),
+    ],
+)
+def test_extract_link_targets(md_text: str, expected: list[str]) -> None:
+    assert extract_link_targets(md_text) == expected
+
+
+def test_heading_slugs_keeps_backticked_words() -> None:
+    # GitHub slugs keep a heading's backticked words, so span stripping
+    # must not apply to heading extraction.
+    assert heading_slugs("## The `foo` flag") == {"the-foo-flag"}
+
+
+def test_heading_inside_fence_is_not_a_heading() -> None:
+    assert heading_slugs("```\n## fenced\n```") == set()
+
+
+def test_anchor_resolves_exact_generated_id() -> None:
+    text = "# Mixed Case\n"
+    assert anchor_resolves(text, "mixed-case")
+    assert not anchor_resolves(text, "Mixed-Case")
+
+
+def test_github_slug_consecutive_hyphens() -> None:
+    assert github_slug("CLI / IDE") == "cli--ide"
+
+
+def test_strip_non_rendering_keeps_code_spans() -> None:
+    assert strip_non_rendering("keep `this span`") == "keep `this span`"
+
+
+def test_links_to_resolves_destination(tmp_path: Path) -> None:
+    target = tmp_path / "docs" / "PAGE.md"
+    target.parent.mkdir()
+    target.write_text("# t", encoding="utf-8")
+    assert links_to("[x](docs/PAGE.md)", tmp_path, target.resolve())
+    # Label naming the file does not count when the target points elsewhere.
+    assert not links_to("[docs/PAGE.md](docs/OTHER.md)", tmp_path, target.resolve())
+    # Image form renders no anchor.
+    assert not links_to("![x](docs/PAGE.md)", tmp_path, target.resolve())
+
+
+@pytest.fixture()
+def link_repo(tmp_path: Path) -> tuple[Path, Path]:
+    source = tmp_path / "docs" / "SOURCE.md"
+    source.parent.mkdir()
+    source.write_text("# Source Heading\n", encoding="utf-8")
+    (tmp_path / "docs" / "TARGET.md").write_text(
+        "# Mixed Case\n", encoding="utf-8"
+    )
+    (tmp_path / "docs" / "data.json").write_text("{}\n", encoding="utf-8")
+    return tmp_path, source
+
+
+@pytest.mark.parametrize(
+    ("target", "expected"),
+    [
+        ("TARGET.md#mixed-case", None),
+        ("#source-heading", None),
+        ("#", RelativeLinkFailure.ANCHOR_NOT_FOUND),
+        ("TARGET.md#", None),
+        ("MISSING.md", RelativeLinkFailure.DOES_NOT_EXIST),
+        ("../../outside.md", RelativeLinkFailure.ESCAPES_REPOSITORY),
+        (
+            "data.json#mixed-case",
+            RelativeLinkFailure.ANCHOR_ON_NON_MARKDOWN,
+        ),
+        ("TARGET.md#missing", RelativeLinkFailure.ANCHOR_NOT_FOUND),
+    ],
+)
+def test_relative_link_failure_ladder(
+    link_repo: tuple[Path, Path],
+    target: str,
+    expected: RelativeLinkFailure | None,
+) -> None:
+    root, source = link_repo
+    assert relative_link_failure(root, source, target) is expected
+
+
+def test_relative_link_fragment_is_an_exact_id_not_slug_input(
+    link_repo: tuple[Path, Path],
+) -> None:
+    root, source = link_repo
+    # Slugifying this fragment would turn it into the real `mixed-case` id
+    # and silently accept a link whose authored fragment is case-wrong.
+    assert relative_link_failure(
+        root, source, "TARGET.md#Mixed-Case"
+    ) is RelativeLinkFailure.ANCHOR_NOT_FOUND