psyclaw 0.27.22 → 0.28.0

package/vendor/ars/scripts/test_block_parser.py

@@ -0,0 +1,259 @@
+"""Tests for the shared block parser (#89 Item 7 Slice A, spec §8.1).
+
+Covers: segmentation per supported block class, marker attachment, the
+§3.1 malformed-state rejections (duplicate IDs, orphan marker, marker
+stack, marker-in-fence-as-content), the unsupported-construct rejections
+by name (setext underline, raw-HTML opener, footnote definition), and
+the §3.2 hash normalization rules.
+
+Run standalone:
+    python -m unittest scripts/test_block_parser.py -v
+"""
+from __future__ import annotations
+
+import unittest
+
+from scripts._block_parser import (
+    BlockParseError,
+    base_draft_hash,
+    block_hash,
+    normalize_block_text,
+    parse_document,
+    segment_fragment,
+    split_lines_keepends,
+)
+
+
+def _kinds(text: str) -> list[str]:
+    return [b.kind for b in parse_document(text).blocks]
+
+
+class TestSegmentation(unittest.TestCase):
+    def test_blank_separated_paragraphs(self):
+        doc = parse_document("Para one.\n\nPara two.\n")
+        self.assertEqual([b.kind for b in doc.blocks], ["text", "text"])
+        self.assertEqual(doc.blocks[0].first_line, "Para one.")
+
+    def test_multiline_paragraph_is_one_block(self):
+        doc = parse_document("Line one\nline two\nline three.\n\nNext.\n")
+        self.assertEqual(len(doc.blocks), 2)
+
+    def test_atx_heading_is_single_line_block(self):
+        doc = parse_document("# Title\nBody right under heading.\n")
+        self.assertEqual([b.kind for b in doc.blocks], ["heading", "text"])
+
+    def test_heading_interrupts_text_run(self):
+        doc = parse_document("Intro text.\n## Section\n")
+        self.assertEqual([b.kind for b in doc.blocks], ["text", "heading"])
+
+    def test_fenced_code_never_split(self):
+        text = "```python\n\ncode line\n\n# not a heading\n```\n"
+        doc = parse_document(text)
+        self.assertEqual([b.kind for b in doc.blocks], ["fence"])
+
+    def test_tilde_fence(self):
+        doc = parse_document("~~~\nx\n~~~\n")
+        self.assertEqual([b.kind for b in doc.blocks], ["fence"])
+
+    def test_unterminated_fence_rejected(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("```\ncode\n")
+        self.assertEqual(ctx.exception.kind, "unterminated_fence")
+
+    def test_table_run_is_one_block(self):
+        text = "| a | b |\n|---|---|\n| 1 | 2 |\n"
+        doc = parse_document(text)
+        self.assertEqual([b.kind for b in doc.blocks], ["table"])
+
+    def test_list_run_with_indented_continuation(self):
+        text = "- item one\n  continued\n- item two\n"
+        doc = parse_document(text)
+        self.assertEqual([b.kind for b in doc.blocks], ["list"])
+
+    def test_loose_list_blank_inside_run(self):
+        text = "- item one\n\n- item two\n\nNot a list.\n"
+        doc = parse_document(text)
+        self.assertEqual([b.kind for b in doc.blocks], ["list", "text"])
+
+    def test_ordered_list(self):
+        doc = parse_document("1. one\n2. two\n")
+        self.assertEqual([b.kind for b in doc.blocks], ["list"])
+
+    def test_blockquote_run(self):
+        doc = parse_document("> quoted\n> more\n\ntext\n")
+        self.assertEqual([b.kind for b in doc.blocks], ["blockquote", "text"])
+
+    def test_frontmatter_skipped_never_labeled(self):
+        text = "---\ntitle: x\n---\n\nBody.\n"
+        doc = parse_document(text)
+        self.assertIsNotNone(doc.frontmatter_span)
+        self.assertEqual(len(doc.blocks), 1)
+        self.assertEqual(doc.text[doc.frontmatter_span[0] : doc.frontmatter_span[1]], "---\ntitle: x\n---\n")
+
+    def test_unterminated_frontmatter_rejected(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("---\ntitle: x\n")
+        self.assertEqual(ctx.exception.kind, "unterminated_frontmatter")
+
+    def test_spans_reconstruct_document(self):
+        text = "---\nfm: 1\n---\n\n# H\n\nPara.\n\n- l1\n- l2\n"
+        doc = parse_document(text)
+        rebuilt = doc.text[doc.frontmatter_span[0] : doc.frontmatter_span[1]]
+        pos = doc.frontmatter_span[1]
+        for b in doc.blocks:
+            start = b.full_start
+            rebuilt += doc.text[pos:start] + doc.text[start : b.span[1]]
+            pos = b.span[1]
+        rebuilt += doc.text[pos:]
+        self.assertEqual(rebuilt, text)
+
+    def test_no_trailing_newline_final_block(self):
+        doc = parse_document("Para one.\n\nPara two no EOL")
+        self.assertEqual(len(doc.blocks), 2)
+        self.assertEqual(doc.text[doc.blocks[1].span[0] : doc.blocks[1].span[1]], "Para two no EOL")
+
+
+class TestMarkers(unittest.TestCase):
+    def test_marker_attaches_to_following_block(self):
+        doc = parse_document("<!--block:B0001-->\nPara.\n")
+        self.assertEqual(doc.blocks[0].block_id, "B0001")
+        self.assertIsNotNone(doc.blocks[0].marker_span)
+
+    def test_unlabeled_block_has_no_id(self):
+        doc = parse_document("Para.\n")
+        self.assertIsNone(doc.blocks[0].block_id)
+
+    def test_orphan_marker_before_blank_rejected(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("<!--block:B0001-->\n\nPara.\n")
+        self.assertEqual(ctx.exception.kind, "orphan_marker")
+        self.assertTrue(ctx.exception.hint)
+
+    def test_orphan_marker_at_eof_rejected(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("Para.\n\n<!--block:B0002-->\n")
+        self.assertEqual(ctx.exception.kind, "orphan_marker")
+
+    def test_marker_stack_rejected(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("<!--block:B0001-->\n<!--block:B0002-->\nPara.\n")
+        self.assertEqual(ctx.exception.kind, "marker_stack")
+
+    def test_duplicate_block_ids_rejected(self):
+        text = "<!--block:B0001-->\nOne.\n\n<!--block:B0001-->\nTwo.\n"
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document(text)
+        self.assertEqual(ctx.exception.kind, "duplicate_block_id")
+
+    def test_marker_inside_fence_is_content(self):
+        text = "```\n<!--block:B0009-->\n```\n\n<!--block:B0001-->\nPara.\n"
+        doc = parse_document(text)
+        self.assertEqual([b.block_id for b in doc.blocks], [None, "B0001"])
+        self.assertIn("<!--block:B0009-->", doc.blocks[0].normalized_text)
+
+    def test_next_fresh_id_is_max_plus_one(self):
+        text = "<!--block:B0007-->\nOne.\n\n<!--block:B0003-->\nTwo.\n"
+        doc = parse_document(text)
+        self.assertEqual(doc.next_fresh_id_num(), 8)
+
+    def test_marker_grammar_requires_exact_shape(self):
+        # A near-miss marker line (missing the B prefix) is not a marker;
+        # it is text content and merges with the adjacent text run.
+        doc = parse_document("<!--block:0001-->\nPara.\n")
+        self.assertEqual(len(doc.blocks), 1)
+        self.assertIsNone(doc.blocks[0].block_id)
+        self.assertIn("<!--block:0001-->", doc.blocks[0].normalized_text)
+
+
+class TestUnsupportedConstructs(unittest.TestCase):
+    def test_setext_underline_rejected_by_name(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("Heading text\n===\n")
+        self.assertEqual(ctx.exception.kind, "unsupported_construct:setext_underline")
+
+    def test_setext_dashes_rejected(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("Heading text\n---\n")
+        self.assertEqual(ctx.exception.kind, "unsupported_construct:setext_underline")
+
+    def test_thematic_break_after_blank_is_plain_text(self):
+        doc = parse_document("Para.\n\n---\n\nMore.\n")
+        self.assertEqual(len(doc.blocks), 3)
+
+    def test_raw_html_opener_rejected_by_name(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("<div>\ncontent\n</div>\n")
+        self.assertEqual(ctx.exception.kind, "unsupported_construct:raw_html_block")
+
+    def test_footnote_definition_rejected_by_name(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            parse_document("[^1]: a footnote definition\n")
+        self.assertEqual(ctx.exception.kind, "unsupported_construct:footnote_definition")
+
+    def test_inline_ref_marker_is_not_html_opener(self):
+        # v3.7.1 citation markers start with <!-- which the raw-HTML
+        # detector (^</?[A-Za-z]) must not catch.
+        doc = parse_document("Claim text.<!--ref:smith2024-->\n")
+        self.assertEqual([b.kind for b in doc.blocks], ["text"])
+
+
+class TestFragmentMode(unittest.TestCase):
+    def test_fragment_segments_multi_paragraph(self):
+        spans = segment_fragment("Para one.\n\nPara two.")
+        self.assertEqual(len(spans), 2)
+
+    def test_fragment_rejects_embedded_marker(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            segment_fragment("<!--block:B0001-->\nPara.\n")
+        self.assertEqual(ctx.exception.kind, "marker_in_fragment")
+
+    def test_fragment_rejects_empty(self):
+        with self.assertRaises(BlockParseError) as ctx:
+            segment_fragment("   \n\n")
+        self.assertEqual(ctx.exception.kind, "empty_fragment")
+
+    def test_fragment_has_no_frontmatter_detection(self):
+        # A leading --- in a fragment is not frontmatter; --- after
+        # nothing is a one-line text block (thematic-break shape).
+        spans = segment_fragment("---\n\nPara.\n")
+        self.assertEqual(len(spans), 2)
+
+    def test_fragment_rejects_unsupported_constructs(self):
+        with self.assertRaises(BlockParseError):
+            segment_fragment("text\n===\n")
+
+
+class TestNormalizationAndHashes(unittest.TestCase):
+    def test_crlf_normalized_for_hash_only(self):
+        self.assertEqual(
+            normalize_block_text(["line one\r\n", "line two\r\n"]),
+            "line one\nline two",
+        )
+
+    def test_blank_edges_stripped_intraline_whitespace_kept(self):
+        self.assertEqual(
+            normalize_block_text(["\n", "a  b  \n", "\n"]),
+            "a  b  ",
+        )
+
+    def test_hash_is_12_hex(self):
+        h = block_hash("some text")
+        self.assertRegex(h, r"^[0-9a-f]{12}$")
+
+    def test_base_draft_hash_is_12_hex_over_bytes(self):
+        h = base_draft_hash(b"raw bytes")
+        self.assertRegex(h, r"^[0-9a-f]{12}$")
+
+    def test_crlf_and_lf_blocks_hash_equal(self):
+        a = parse_document("Para text.\r\n").blocks[0]
+        b = parse_document("Para text.\n").blocks[0]
+        self.assertEqual(a.norm_hash, b.norm_hash)
+
+    def test_split_lines_keepends_no_unicode_linebreak_split(self):
+        # U+2028 must NOT split lines (str.splitlines would).
+        lines = split_lines_keepends("a
b\nc\n")
+        self.assertEqual(lines, ["a
b\n", "c\n"])
+
+
+if __name__ == "__main__":
+    unittest.main()

package/vendor/ars/scripts/test_bootstrap_timeline_yaml.py

@@ -0,0 +1,148 @@
+"""Tests for bootstrap_timeline_yaml.py (opt-in Crossref + pdftotext)."""
+from __future__ import annotations
+
+import importlib.util
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = REPO_ROOT / "scripts/bootstrap_timeline_yaml.py"
+
+
+def _load_script_module():
+    spec = importlib.util.spec_from_file_location("bootstrap_timeline_yaml", SCRIPT)
+    assert spec is not None
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_bootstrap_dry_run_emits_skeleton(tmp_path):
+    """Dry-run produces a timeline.yaml skeleton without calling Crossref/pdftotext."""
+    corpus = tmp_path / "corpus.yaml"
+    corpus.write_text(yaml.safe_dump({
+        "literature_corpus": [
+            {"citation_key": "alpha", "title": "Alpha", "authors": [{"family": "Smith"}],
+             "year": 2024, "source_pointer": "doi:10.1/alpha", "doi": "10.1/alpha"},
+        ]
+    }))
+    out = tmp_path / "timeline.yaml"
+    result = subprocess.run(
+        [sys.executable, str(SCRIPT),
+         "--corpus", str(corpus),
+         "--output", str(out),
+         "--dry-run"],
+        capture_output=True, text=True,
+    )
+    assert result.returncode == 0, f"stderr={result.stderr}"
+    assert out.exists()
+    data = yaml.safe_load(out.read_text())
+    assert data["schema_version"] == "1.0"
+    sources = data["sources"]
+    assert len(sources) == 1
+    assert sources[0]["citation_key"] == "alpha"
+    # Dry run: published_date.method should NOT be crossref_lookup (no API call)
+    # Acceptable: either falls back to corpus year (method: adapter_metadata) OR absent
+    pd = sources[0].get("published_date")
+    if pd is not None:
+        assert pd.get("provenance", {}).get("method") != "crossref_lookup"
+
+
+def test_bootstrap_validates_against_timeline_schema(tmp_path):
+    """Generated timeline.yaml must validate against timeline.schema.json."""
+    import jsonschema
+    corpus = tmp_path / "corpus.yaml"
+    corpus.write_text(yaml.safe_dump({
+        "literature_corpus": [
+            {"citation_key": "alpha", "title": "Alpha", "authors": [{"family": "Smith"}],
+             "year": 2024, "source_pointer": "doi:10.1/alpha", "doi": "10.1/alpha"},
+        ]
+    }))
+    out = tmp_path / "timeline.yaml"
+    result = subprocess.run(
+        [sys.executable, str(SCRIPT),
+         "--corpus", str(corpus),
+         "--output", str(out),
+         "--dry-run"],
+        capture_output=True, text=True,
+    )
+    assert result.returncode == 0
+    data = yaml.safe_load(out.read_text())
+    schema = json.loads((REPO_ROOT / "shared/contracts/passport/timeline.schema.json").read_text())
+    jsonschema.validate(data, schema)  # should not raise
+
+
+def test_crossref_lookup_quotes_doi_path(monkeypatch):
+    module = _load_script_module()
+    captured = []
+
+    class FakeResponse:
+        status_code = 200
+
+        def json(self):
+            return {"message": {"title": ["ok"]}}
+
+    class FakeRequests:
+        @staticmethod
+        def get(url, timeout):
+            captured.append((url, timeout))
+            return FakeResponse()
+
+    monkeypatch.setattr(module, "requests", FakeRequests)
+
+    result = module._crossref_lookup("10.1000/foo?bar=baz", dry_run=False)
+
+    assert result == {"title": ["ok"]}
+    assert captured == [
+        ("https://api.crossref.org/works/10.1000%2Ffoo%3Fbar%3Dbaz", 10)
+    ]
+
+
+def test_bootstrap_entry_locator_matches_queried_url(monkeypatch):
+    # Codex follow-up to #310: the lookup encodes the DOI, so the recorded
+    # source_locator must point at the same encoded URL — not the raw DOI —
+    # or provenance references a resource that was never queried.
+    module = _load_script_module()
+
+    monkeypatch.setattr(
+        module,
+        "_crossref_lookup",
+        lambda doi, dry_run: {"issued": {"date-parts": [[2024, 3]]}},
+    )
+
+    out = module._bootstrap_entry(
+        {"citation_key": "foo2024", "doi": "10.1000/foo?bar=baz"},
+        dry_run=False,
+    )
+
+    assert out["published_date"]["provenance"]["source_locator"] == (
+        "https://api.crossref.org/works/10.1000%2Ffoo%3Fbar%3Dbaz"
+    )
+
+
+def test_pdftotext_uses_resolved_executable(monkeypatch, tmp_path):
+    module = _load_script_module()
+    pdf = tmp_path / "paper.pdf"
+    pdf.touch()
+    calls = []
+
+    class FakeCompleted:
+        returncode = 0
+        stdout = "\nFirst line\nSecond line\n"
+
+    monkeypatch.setattr(module.shutil, "which", lambda name: "/usr/local/bin/pdftotext")
+    monkeypatch.setattr(
+        module.subprocess,
+        "run",
+        lambda args, **kwargs: calls.append((args, kwargs)) or FakeCompleted(),
+    )
+
+    result = module._pdftotext_first_line(pdf, dry_run=False)
+
+    assert result == "First line"
+    assert calls[0][0][0] == "/usr/local/bin/pdftotext"