logseq-md 0.1.0__tar.gz → 0.2.0__tar.gz

{logseq_md-0.1.0 → logseq_md-0.2.0}/.pre-commit-config.yaml

@@ -158,6 +158,17 @@ repos:
     pass_filenames: false
     always_run: true
 
+  # Every fixture directory carries leak evidence that still describes it. The
+  # leak check needs a real graph and runs on a maintainer's machine; this is
+  # the offline half, and it runs on every commit because a fixture edited
+  # after its check is exactly the change nobody thinks to re-check.
+  - id: fixture-evidence
+    name: Fixture leak evidence matches the committed fixtures
+    entry: uv run --script .scripts/check_fixture_evidence.py
+    language: system
+    pass_filenames: false
+    always_run: true
+
   # No symlinks in the tree. A symlink is committed as a mode-120000 blob
   # holding a path, and a clone on Windows without developer mode writes that
   # path out as an ordinary text file: the link silently becomes a one-line

logseq_md-0.2.0/.scripts/check_fixture_evidence.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""Check that every fixture directory carries evidence its leak check still describes.
+
+Usage:
+  check_fixture_evidence.py [path ...]
+
+The leak check (`.scripts/check_fixture_leaks.py`) needs a real graph, so it
+runs on a maintainer's machine and records `tests/evidence/fixtures-<topic>.json`.
+This is the offline half, run in CI and as a pre-commit hook. It cannot tell
+that the check ran; it can tell that the recording still describes the fixtures
+that are committed. It fails when:
+
+- a fixture directory has no evidence file;
+- a recorded hash no longer matches a fixture, or the leak script itself;
+- the recorded file set differs from the directory's;
+- the recorded outcome is not `pass`.
+
+Paths are accepted and ignored, because a directory is checked as a whole.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+FIXTURES = "tests/fixtures"
+EVIDENCE = "tests/evidence"
+LEAK_SCRIPT = ".scripts/check_fixture_leaks.py"
+SCHEMA_VERSION = 1
+
+
+def sha256_of(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def check_topic(topic: str, directory: Path, root: Path) -> list[str]:
+    evidence_path = root / EVIDENCE / f"fixtures-{topic}.json"
+    relative = evidence_path.relative_to(root).as_posix()
+    record = f"run `uv run --script {LEAK_SCRIPT} <graph> --record`"
+    if not evidence_path.is_file():
+        return [f"{topic}: no evidence at {relative}; {record}"]
+    try:
+        evidence = json.loads(evidence_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as error:
+        return [f"{relative}: not valid JSON ({error})"]
+    if not isinstance(evidence, dict):
+        return [f"{relative}: not a JSON object"]
+
+    problems: list[str] = []
+    if evidence.get("schema_version") != SCHEMA_VERSION:
+        problems.append(f"{relative}: schema_version is {evidence.get('schema_version')!r}, expected {SCHEMA_VERSION}")
+    if evidence.get("script_sha256") != sha256_of(root / LEAK_SCRIPT):
+        problems.append(f"{relative}: {LEAK_SCRIPT} changed since the evidence was recorded; {record}")
+
+    current = {path.relative_to(root).as_posix() for path in directory.rglob("*") if path.is_file()}
+    entries = evidence.get("files")
+    if not isinstance(entries, list) or not all(isinstance(e, dict) and "path" in e and "sha256" in e for e in entries):
+        return [*problems, f"{relative}: `files` must be a list of {{path, sha256}}"]
+    recorded = {entry["path"]: entry["sha256"] for entry in entries}
+    problems += [
+        f"{relative}: {path} is not covered by the evidence; {record}" for path in sorted(current - recorded.keys())
+    ]
+    problems += [
+        f"{relative}: covers {path}, which no longer exists; {record}" for path in sorted(recorded.keys() - current)
+    ]
+    problems += [
+        f"{relative}: {path} changed since the evidence was recorded; {record}"
+        for path in sorted(current & recorded.keys())
+        if sha256_of(root / path) != recorded[path]
+    ]
+
+    if evidence.get("outcome") != "pass":
+        problems.append(f"{relative}: recorded outcome is {evidence.get('outcome')!r}, not 'pass'")
+    return problems
+
+
+def check_all(root: Path = REPO_ROOT) -> list[str]:
+    fixtures = root / FIXTURES
+    if not fixtures.is_dir():
+        return []
+    problems: list[str] = []
+    for directory in sorted(fixtures.iterdir()):
+        if directory.is_dir():
+            problems += check_topic(directory.name, directory, root)
+    return problems
+
+
+def main(argv: list[str]) -> int:
+    problems = check_all()
+    for problem in problems:
+        print(f"error: {problem}", file=sys.stderr)
+    return 1 if problems else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

logseq_md-0.2.0/.scripts/check_fixture_leaks.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""Check that no fixture line was copied out of a real graph, and record that it was checked.
+
+Usage:
+  check_fixture_leaks.py <graph> [--record]
+
+A fixture is written by somebody who has just read a private graph, which is how
+a real line ends up in one. This compares every line of every file under
+`tests/fixtures/<topic>/` with every line of every markdown file in the graph,
+and fails on any match that is more than structure: a bullet on its own, a `---` rule,
+whitespace, an `id::` or `collapsed::` line, a fence marker or `:END:`.
+
+Lines are compared as bytes with surrounding whitespace removed, and once more
+with a leading bullet removed, so a line copied at a different depth or lifted
+out of a block still matches. That catches a copied line and not a paraphrase;
+reading the fixture diff stays a reviewer's job.
+
+With `--record`, it writes `tests/evidence/fixtures-<topic>.json` per fixture
+directory: the hash of every fixture file in it, the hash of this script, the
+outcome and how many graph files were scanned. **No path, name or line from the
+graph is written**, and the graph is opened with `"rb"` and nothing else. A test
+reads this source and fails if graph-reading code gains a write call; the only
+writing is in `write_evidence`, which touches the repository.
+
+`.scripts/check_fixture_evidence.py` is the offline half that runs in CI.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+FIXTURES = "tests/fixtures"
+EVIDENCE = "tests/evidence"
+SCHEMA_VERSION = 1
+
+LINE_BREAK = re.compile(rb"\r\n|\n|\r")
+BULLET = re.compile(rb"^-[ \t]+")
+STRUCTURE = re.compile(rb"^(-|-{3,}|:END:|(`{3,}|~{3,})[\w+-]*|(id|collapsed)::.*)?$", re.IGNORECASE)
+
+
+def normalised(line: bytes) -> list[bytes]:
+    """The forms of a line worth comparing: stripped, and stripped of its bullet."""
+    stripped = line.strip(b" \t\xef\xbb\xbf")
+    forms = [stripped]
+    unbulleted = BULLET.sub(b"", stripped)
+    if unbulleted != stripped:
+        forms.append(unbulleted)
+    return [form for form in forms if not STRUCTURE.match(form)]
+
+
+def graph_lines(graph: Path) -> tuple[set[bytes], int]:
+    """Every comparable line form in the graph's markdown, and how many files held them."""
+    seen: set[bytes] = set()
+    scanned = 0
+    for path in sorted(graph.rglob("*.md")):
+        if not path.is_file():
+            continue
+        with path.open("rb") as handle:
+            data = handle.read()
+        scanned += 1
+        for line in LINE_BREAK.split(data):
+            seen.update(normalised(line))
+    return seen, scanned
+
+
+def fixture_topics(root: Path) -> dict[str, list[Path]]:
+    """Each fixture directory, mapped to every file in it."""
+    fixtures = root / FIXTURES
+    if not fixtures.is_dir():
+        return {}
+    return {
+        directory.name: sorted(path for path in directory.rglob("*") if path.is_file())
+        for directory in sorted(fixtures.iterdir())
+        if directory.is_dir()
+    }
+
+
+def leaks(files: list[Path], seen: set[bytes], root: Path) -> list[str]:
+    """Each fixture line found in the graph, named by fixture path and line number only."""
+    found: list[str] = []
+    for path in files:
+        for number, line in enumerate(LINE_BREAK.split(path.read_bytes()), start=1):
+            if any(form in seen for form in normalised(line)):
+                found.append(f"{path.relative_to(root).as_posix()}:{number}")
+    return found
+
+
+def sha256_of(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def evidence_for(topic: str, files: list[Path], outcome: str, scanned: int, root: Path) -> dict:
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "topic": topic,
+        "files": [{"path": path.relative_to(root).as_posix(), "sha256": sha256_of(path)} for path in files],
+        "script_sha256": sha256_of(Path(__file__).resolve()),
+        "outcome": outcome,
+        "graph_files_scanned": scanned,
+        "recorded_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+    }
+
+
+def write_evidence(evidence: dict, root: Path) -> Path:
+    """The one place this script writes, and it writes into the repository."""
+    target = root / EVIDENCE / f"fixtures-{evidence['topic']}.json"
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_bytes((json.dumps(evidence, indent=2, sort_keys=True) + "\n").encode("utf-8"))
+    return target
+
+
+def check(graph: Path, root: Path = REPO_ROOT, *, record: bool = False) -> int:
+    seen, scanned = graph_lines(graph)
+    if scanned == 0:
+        print(f"error: no markdown under {graph}; is it a graph?", file=sys.stderr)
+        return 2
+    failed = False
+    for topic, files in fixture_topics(root).items():
+        found = leaks(files, seen, root)
+        outcome = "fail" if found else "pass"
+        failed |= bool(found)
+        for location in found:
+            print(f"leak: {location} occurs verbatim in the graph", file=sys.stderr)
+        print(f"{topic}: {outcome} ({len(files)} fixture files, {scanned} graph files)")
+        if record:
+            print(
+                f"  recorded {write_evidence(evidence_for(topic, files, outcome, scanned, root), root).relative_to(root).as_posix()}"
+            )
+    return 1 if failed else 0
+
+
+def main(argv: list[str]) -> int:
+    record = "--record" in argv
+    paths = [arg for arg in argv if arg != "--record"]
+    if len(paths) != 1:
+        print(__doc__, file=sys.stderr)
+        return 2
+    return check(Path(paths[0]), record=record)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))