ctxverify 0.1.0__py3-none-any.whl

ctxverify/adapters/obsidian.py

@@ -0,0 +1,253 @@
+"""Export an Obsidian-style vault into a context pack.
+
+The mapping is intentionally flat: one note becomes one claim, with the note
+itself as the evidence. That is not as rich as a compiled pack, but it is
+honest, and it means a vault can be checked on day one instead of after a
+compiler is written.
+
+What the adapter does preserve is the part that is expensive to get right:
+the link structure of the vault, resolved the way Obsidian resolves it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ..frontmatter import as_list, as_text, parse_frontmatter, split_frontmatter
+from ..model import CLAIM_STATUSES, Claim, Entity, Evidence, Manifest, Pack, Source
+from ..resolve import LinkIndex, strip_wikilink
+from ..writer import evidence_record
+
+SKIP_DIRS = {".git", ".obsidian", ".trash", "node_modules", ".venv", "__pycache__"}
+
+HEADING_RE = re.compile(r"^#{1,6}\s+(.+?)\s*$", re.MULTILINE)
+WIKILINK_RE = re.compile(r"\[\[([^\[\]]+?)\]\]")
+CODE_FENCE_RE = re.compile(r"```.*?```", re.DOTALL)
+#: A note id that looks like a card number: "C-001", "P-012".
+NUMBER_LIKE = re.compile(r"^[A-Za-z]{1,6}-\d{1,6}$")
+
+
+def _iter_notes(vault: Path):
+    for path in sorted(vault.rglob("*.md")):
+        parts = set(path.relative_to(vault).parts[:-1])
+        if parts & SKIP_DIRS:
+            continue
+        if any(part.startswith(".") for part in parts):
+            continue
+        yield path
+
+
+def _slug(path: Path) -> str:
+    stem = path.stem
+    cleaned = re.sub(r"[^\w\-一-鿿]+", "-", stem).strip("-")
+    return cleaned or "note"
+
+
+def _first_paragraph(body: str) -> str:
+    text = CODE_FENCE_RE.sub("", body)
+    for block in re.split(r"\n\s*\n", text):
+        stripped = block.strip()
+        if not stripped:
+            continue
+        if stripped.startswith("#"):
+            heading = HEADING_RE.search(stripped)
+            if heading:
+                return heading.group(1).strip()
+            continue
+        if stripped.startswith(("```", "|", "-", "*", ">", "<!--")):
+            continue
+        return re.sub(r"\s+", " ", stripped)[:600]
+    return ""
+
+
+def export_obsidian(
+    vault: str | Path,
+    pack_id: str | None = None,
+    include: list[str] | None = None,
+    exclude: list[str] | None = None,
+) -> Pack:
+    """Read ``vault`` and return a :class:`Pack`."""
+    root = Path(vault).expanduser().resolve()
+    if not root.exists():
+        raise FileNotFoundError(f"vault not found: {root}")
+
+    include = include or []
+    exclude = exclude or []
+
+    notes = []
+    for path in _iter_notes(root):
+        rel = path.relative_to(root).as_posix()
+        if include and not any(pattern in rel for pattern in include):
+            continue
+        if exclude and any(pattern in rel for pattern in exclude):
+            continue
+        notes.append((path, rel))
+
+    # Obsidian resolves a link to a note's alias as well as to its filename.
+    # Reading them is the difference between reporting a real broken link and
+    # reporting 1,254 false ones.
+    aliases: dict[str, str] = {}
+    note_frontmatter: dict[str, dict] = {}
+    for path, rel in notes:
+        text = path.read_text(encoding="utf-8", errors="replace")
+        front = parse_frontmatter(text)
+        note_frontmatter[rel] = front
+        for alias in as_list(front.get("aliases")) + as_list(front.get("alias")):
+            alias = alias.strip()
+            if alias:
+                aliases.setdefault(alias, rel)
+                aliases.setdefault(alias.lower(), rel)
+
+    index = LinkIndex(paths=[rel for _, rel in notes], aliases=aliases)
+
+    claims: list[Claim] = []
+    evidence: list[Evidence] = []
+    entities: list[Entity] = []
+    sources: list[Source] = []
+    entity_ids: dict[str, str] = {}
+
+    for number, (path, rel) in enumerate(notes, start=1):
+        text = path.read_text(encoding="utf-8", errors="replace")
+        front = note_frontmatter.get(rel) or parse_frontmatter(text)
+        _, body = split_frontmatter(text)
+
+        source_id = f"src-{number:03d}"
+        claim_id = f"clm-{number:03d}"
+        evidence_id = f"ev-{number:03d}"
+
+        title = as_text(front.get("title")).strip() or path.stem
+        summary = (
+            as_text(front.get("summary")).strip()
+            or as_text(front.get("description")).strip()
+            or as_text(front.get("abstract")).strip()
+            or _first_paragraph(body)
+        )
+
+        raw_status = as_text(front.get("status")).strip().lower()
+        status = raw_status if raw_status in CLAIM_STATUSES else "asserted"
+
+        confidence = front.get("confidence")
+        if not isinstance(confidence, (int, float)) or isinstance(confidence, bool):
+            confidence = None
+
+        topics = as_list(front.get("topics")) or as_list(front.get("domain")) or as_list(
+            front.get("tags")
+        )
+
+        links = []
+        for raw in WIKILINK_RE.findall(body):
+            target = strip_wikilink(raw)
+            if not target:
+                continue
+            resolved = index.resolve(raw)
+            links.append({"target": raw, "resolved": resolved})
+
+        claim_entities: list[str] = []
+        for name in as_list(front.get("entities")):
+            key = name.strip().lower()
+            if not key:
+                continue
+            if key not in entity_ids:
+                entity_id = f"ent-{len(entity_ids) + 1:03d}"
+                entity_ids[key] = entity_id
+                entities.append(Entity(id=entity_id, name=name.strip()))
+            claim_entities.append(entity_ids[key])
+
+        created = (
+            as_text(front.get("created")).strip()
+            or as_text(front.get("createdAt")).strip()
+            or as_text(front.get("date")).strip()
+            or None
+        )
+        updated = (
+            as_text(front.get("updated")).strip()
+            or as_text(front.get("updatedAt")).strip()
+            or as_text(front.get("modified")).strip()
+            or None
+        )
+
+        claims.append(
+            Claim(
+                id=claim_id,
+                text=summary or title,
+                status=status,
+                confidence=float(confidence) if confidence is not None else None,
+                evidence=[evidence_id],
+                entities=claim_entities,
+                topics=topics,
+                links=links,
+                created_at=created,
+                updated_at=updated,
+            )
+        )
+
+        quote = summary or title
+        record = evidence_record(
+            evidence_id,
+            claim_id,
+            source_id,
+            quote,
+            locator={"path": rel, "line_start": 1, "line_end": 1},
+            retrieved_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        )
+        evidence.append(Evidence.from_dict(record))
+
+        # A corpus that keeps several numbering spaces side by side declares
+        # them: a source in namespace "billing" may carry C-001 without
+        # colliding with the global C-001.
+        extra: dict = {}
+        aliases = as_list(front.get("aliases"))
+        if aliases:
+            extra["aliases"] = aliases
+        namespace = as_text(front.get("namespace")).strip() or as_text(
+            front.get("编号空间")
+        ).strip()
+        if namespace:
+            extra["namespace"] = namespace
+        # A note's own number. It is kept under a distinct key so that it can
+        # never be mistaken for the pack's source id.
+        number = as_text(front.get("number")).strip() or as_text(front.get("编号")).strip()
+        if not number:
+            note_id = as_text(front.get("id")).strip()
+            if note_id and NUMBER_LIKE.match(note_id):
+                number = note_id
+        if number:
+            extra["number"] = number
+
+        sources.append(
+            Source(
+                id=source_id,
+                path=rel,
+                title=title,
+                kind="markdown",
+                sha256=hashlib.sha256(text.encode("utf-8")).hexdigest(),
+                modified_at=datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat(
+                    timespec="seconds"
+                ),
+                extra=extra,
+            )
+        )
+
+    manifest = Manifest(
+        pack_id=pack_id or root.name,
+        title=f"{root.name} (Obsidian export)",
+        description=f"Context pack exported from the Obsidian vault at {root}.",
+        generated_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        generator={"name": "ctxverify-adapter-obsidian", "version": "0.1.0"},
+        sources=sources,
+    )
+
+    return Pack(
+        root=str(root),
+        manifest=manifest,
+        claims=claims,
+        evidence=evidence,
+        entities=entities,
+        relations=[],
+        contradictions=[],
+        freshness={},
+        freshness_policy={"default_ttl_days": 180},
+    )

ctxverify/checks/__init__.py

@@ -0,0 +1,203 @@
+"""Check registry and runner."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+import time
+from typing import Any, Callable, Iterable
+
+from ..model import Pack
+from ..report import Finding, Report, sort_findings
+from ..resolve import LinkIndex, build_index
+from ..rules import ERROR, RULES_BY_ID, RULE_GROUPS
+
+CheckFn = Callable[[Pack, "CheckContext"], Iterable[Finding]]
+
+
+@dataclass
+class CheckContext:
+    """Everything a check may need beyond the pack itself."""
+
+    now: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    options: dict[str, Any] = field(default_factory=dict)
+    index: LinkIndex | None = None
+    entities: dict[str, str] = field(default_factory=dict)
+
+    def option(self, key: str, default: Any = None) -> Any:
+        return self.options.get(key, default)
+
+    def resolve_entity(self, name: str) -> str | None:
+        """Resolve an entity name or alias to an entity id."""
+        return self.entities.get(name) or self.entities.get(name.strip().lower())
+
+
+_CHECKS: list[tuple[str, CheckFn]] = []
+
+
+def register(name: str) -> Callable[[CheckFn], CheckFn]:
+    def decorator(fn: CheckFn) -> CheckFn:
+        _CHECKS.append((name, fn))
+        return fn
+
+    return decorator
+
+
+def check_names() -> list[str]:
+    return [name for name, _ in _CHECKS]
+
+
+def run_checks(pack: Pack, options: dict[str, Any] | None = None) -> Report:
+    """Run every registered check against ``pack`` and return the report."""
+    options = dict(options or {})
+    now = options.pop("now", None) or datetime.now(timezone.utc)
+    started = time.perf_counter()
+
+    # Entity names and aliases are a separate namespace from file paths.
+    # They resolve entity references, not links to files, so they must not
+    # make a link to a missing file look resolvable.
+    entity_alias_map: dict[str, str] = {}
+    for entity in pack.entities:
+        if entity.name:
+            entity_alias_map.setdefault(entity.name, entity.id)
+            entity_alias_map.setdefault(entity.id, entity.id)
+        for alias in entity.aliases:
+            entity_alias_map.setdefault(alias, entity.id)
+
+    # File aliases live on the source record: a note can be linked by a name
+    # other than its path, exactly as Obsidian resolves an alias.
+    file_aliases: dict[str, str] = {}
+    for source in pack.manifest.sources:
+        for alias in source.extra.get("aliases") or []:
+            alias = str(alias).strip()
+            if alias:
+                file_aliases.setdefault(alias, source.path)
+
+    index = build_index([s.path for s in pack.manifest.sources], file_aliases)
+
+    ctx = CheckContext(now=now, options=options, index=index, entities=entity_alias_map)
+    ctx.options["invalid_ttls"] = set(getattr(pack, "invalid_ttls", set()) or set())
+
+    findings: list[Finding] = list(getattr(pack, "load_findings", []) or [])
+    enabled_groups, enabled_rule_ids = _resolve_selection(options.get("rules"))
+    disabled_groups = _explicit_groups(options.get("exclude_rules"))
+    _, disabled_rule_ids = _resolve_selection(options.get("exclude_rules"))
+    for name, fn in _CHECKS:
+        if enabled_groups is not None and name not in enabled_groups:
+            continue
+        if name in disabled_groups:
+            continue
+        try:
+            findings.extend(fn(pack, ctx))
+        except Exception as exc:  # a broken check must not hide the rest
+            # This is a bug in ctxverify, not a defect in the pack. It gets
+            # its own rule id so that the two are never confused, and it is
+            # never filtered away: a tool that crashed still has to say so.
+            findings.append(
+                Finding(
+                    rule="ctxverify.internal",
+                    severity=ERROR,
+                    message=(
+                        f"internal error while running check {name}: "
+                        f"{type(exc).__name__}: {exc}"
+                    ),
+                    location=pack.root,
+                    hint="this is a bug in ctxverify, not a problem with the pack; please report it",
+                )
+            )
+
+    # A selection may name a single rule id. Selecting `ids.duplicate` must
+    # report `ids.duplicate` and not its neighbour `ids.collision`.
+    # Structural findings are never filtered away: a pack that cannot be
+    # judged cannot be judged "clean" by narrowing the run.
+    if enabled_rule_ids:
+        findings = [
+            f
+            for f in findings
+            if f.rule in enabled_rule_ids
+            or f.rule in ("pack.structure", "ctxverify.internal")
+        ]
+    if disabled_rule_ids:
+        findings = [f for f in findings if f.rule not in disabled_rule_ids]
+
+    findings = sort_findings(findings)
+    report = Report(
+        pack=pack.root,
+        findings=findings,
+        stats=pack_stats(pack),
+        checked_at=now.astimezone(timezone.utc).isoformat(timespec="seconds"),
+        duration_ms=int((time.perf_counter() - started) * 1000),
+        tool_version=tool_version(),
+    )
+    return report
+
+
+def _resolve_selection(selection: Any) -> tuple[set[str] | None, set[str]]:
+    """Turn a user's selection into check groups and rule ids.
+
+    A selection may name a check group (``identity``) or a rule id
+    (``ids.duplicate``). A rule id resolves to the group that implements it,
+    so the check still runs; the rule ids are returned separately so that the
+    findings can be narrowed to exactly the rules that were named.
+    """
+    if not selection:
+        return None, set()
+    groups: set[str] = set()
+    rule_ids: set[str] = set()
+    for item in selection:
+        key = str(item).strip()
+        if not key:
+            continue
+        if key in RULES_BY_ID:
+            groups.add(RULE_GROUPS.get(key, key))
+            rule_ids.add(key)
+            continue
+        groups.add(key)
+    return (groups or None), rule_ids
+
+
+def _explicit_groups(selection: Any) -> set[str]:
+    """The check groups named directly, ignoring rule ids.
+
+    Excluding a group silences every rule in it. Excluding a single rule id
+    must not: the check still runs and the other rules it produces are still
+    reported.
+    """
+    if not selection:
+        return set()
+    groups: set[str] = set()
+    for item in selection:
+        key = str(item).strip()
+        if key and key not in RULES_BY_ID:
+            groups.add(key)
+    return groups
+
+
+def pack_stats(pack: Pack) -> dict[str, Any]:
+    return {
+        "claims": len(pack.claims),
+        "evidence": len(pack.evidence),
+        "entities": len(pack.entities),
+        "relations": len(pack.relations),
+        "contradictions": len(pack.contradictions),
+        "sources": len(pack.manifest.sources),
+        "freshness_entries": len(pack.freshness),
+    }
+
+
+def tool_version() -> str:
+    from ..version import __version__
+
+    return __version__
+
+
+# Importing the modules registers their checks.
+from . import actions as _actions  # noqa: E402,F401
+from . import consistency as _consistency  # noqa: E402,F401
+from . import duplicates as _duplicates  # noqa: E402,F401
+from . import freshness as _freshness  # noqa: E402,F401
+from . import hygiene as _hygiene  # noqa: E402,F401
+from . import identity as _identity  # noqa: E402,F401
+from . import links as _links  # noqa: E402,F401
+from . import orphans as _orphans  # noqa: E402,F401
+from . import provenance as _provenance  # noqa: E402,F401