sessionwiki 2.0.0__py3-none-any.whl

sessionwiki/audit.py

@@ -0,0 +1,174 @@
+"""Pre-publish privacy audit for raw/, wiki/, and site/.
+
+The converter redacts known secrets at ingest time, but generated pages
+and tool output can still contain a username, a home path, or a token
+that appeared after redaction. This command is a read-only second look:
+
+    sessionwiki audit
+    sessionwiki audit --path site --json
+    sessionwiki audit --fail-on-findings
+
+It never edits files. Exit code is 1 only with ``--fail-on-findings``.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any, Iterable, Optional, Sequence
+
+from sessionwiki import REPO_ROOT
+from sessionwiki.config import resolve_config
+from sessionwiki.convert import _DEFAULT_TOKEN_PATTERNS
+
+DEFAULT_RELATIVE_PATHS = ("raw", "wiki", "site")
+TEXT_EXTENSIONS = {
+    ".md", ".txt", ".json", ".jsonl", ".html", ".htm", ".xml",
+    ".yml", ".yaml", ".toml", ".css", ".js", ".log", ".csv",
+}
+SKIP_DIRS = {".git", "__pycache__", ".venv", "node_modules", ".pytest_cache", ".ruff_cache"}
+
+
+def _iter_files(paths: Sequence[Path]) -> Iterable[Path]:
+    for base in paths:
+        path = Path(base)
+        if path.is_file():
+            yield path
+            continue
+        if not path.is_dir():
+            continue
+        for child in path.rglob("*"):
+            if any(part in SKIP_DIRS for part in child.parts):
+                continue
+            if child.is_file() and child.suffix.lower() in TEXT_EXTENSIONS:
+                yield child
+
+
+def _preview(line: str, start: int, end: int, limit: int = 180) -> str:
+    masked = line[:start] + "<MATCH>" + line[end:]
+    return " ".join(masked.split())[:limit]
+
+
+def _compile_user_patterns(patterns: Sequence[str]) -> list[re.Pattern[str]]:
+    compiled: list[re.Pattern[str]] = []
+    for raw in patterns:
+        try:
+            compiled.append(re.compile(raw))
+        except re.error:
+            continue
+    return compiled
+
+
+def audit_paths(
+    paths: Sequence[Path],
+    *,
+    username: Optional[str] = None,
+    extra_patterns: Sequence[str] = (),
+    max_findings: int = 200,
+) -> dict[str, Any]:
+    """Scan text files and return findings. Does not modify anything."""
+    findings: list[dict[str, Any]] = []
+    files_scanned = 0
+    secret_patterns = [
+        (pattern, "secret") for pattern in _DEFAULT_TOKEN_PATTERNS
+    ]
+    secret_patterns.extend(
+        (pattern, "pattern") for pattern in _compile_user_patterns(extra_patterns)
+    )
+    user_re = re.compile(re.escape(username), re.IGNORECASE) if username else None
+
+    for file_path in _iter_files(paths):
+        if len(findings) >= max_findings:
+            break
+        try:
+            raw = file_path.read_bytes()
+        except OSError:
+            continue
+        if b"\x00" in raw[:4096]:
+            continue
+        text = raw.decode("utf-8", errors="replace")
+        files_scanned += 1
+        for line_number, line in enumerate(text.splitlines(), 1):
+            if user_re is not None:
+                for match in user_re.finditer(line):
+                    findings.append(
+                        {
+                            "path": str(file_path),
+                            "line": line_number,
+                            "kind": "username",
+                            "preview": _preview(line, match.start(), match.end()),
+                        }
+                    )
+                    if len(findings) >= max_findings:
+                        break
+            for pattern, kind in secret_patterns:
+                for match in pattern.finditer(line):
+                    findings.append(
+                        {
+                            "path": str(file_path),
+                            "line": line_number,
+                            "kind": kind,
+                            "preview": _preview(line, match.start(), match.end()),
+                        }
+                    )
+                    if len(findings) >= max_findings:
+                        break
+            if len(findings) >= max_findings:
+                break
+
+    counts: dict[str, int] = {}
+    for finding in findings:
+        counts[finding["kind"]] = counts.get(finding["kind"], 0) + 1
+    return {
+        "files_scanned": files_scanned,
+        "username": username or "",
+        "findings": findings,
+        "counts": counts,
+        "truncated": len(findings) >= max_findings,
+    }
+
+
+def cmd_audit(args: Any) -> int:
+    """Print the privacy report for the selected paths."""
+    configured = resolve_config()
+    redaction = configured.get("redaction") if isinstance(configured, dict) else None
+    redaction = redaction if isinstance(redaction, dict) else {}
+    username = (
+        getattr(args, "username", None)
+        or redaction.get("real_username")
+        or os.environ.get("USER")
+        or os.environ.get("USERNAME")
+        or ""
+    )
+    paths = getattr(args, "path", None)
+    if not paths:
+        paths = [REPO_ROOT / relative for relative in DEFAULT_RELATIVE_PATHS]
+
+    report = audit_paths(
+        paths,
+        username=username,
+        extra_patterns=redaction.get("extra_patterns") or [],
+    )
+
+    if getattr(args, "json", False):
+        print(json.dumps(report, ensure_ascii=False, indent=2))
+    else:
+        print(f"audit: scanned {report['files_scanned']} file(s)")
+        print(f"audit: {len(report['findings'])} finding(s)")
+        for kind, count in sorted(report["counts"].items()):
+            print(f"  {kind}: {count}")
+        for finding in report["findings"][:50]:
+            print(
+                f"  {finding['path']}:{finding['line']} "
+                f"[{finding['kind']}] {finding['preview']}"
+            )
+        if report["truncated"]:
+            print("  … findings truncated")
+        if report["findings"]:
+            print("audit: review before publishing raw/, wiki/, or site/")
+
+    if report["findings"] and getattr(args, "fail_on_findings", False):
+        return 1
+    return 0

sessionwiki/backlinks.py

@@ -0,0 +1,224 @@
+"""Backlink injection: every wiki page that's linked-to gets a managed
+``## Referenced by`` section listing its referrers (#328).
+
+Today a source page links out to ``[[Pratiyush]]`` but the
+``entities/Pratiyush.md`` page doesn't know about it — 95% of pages
+end up as graph orphans (575 / 596 sources with zero inbound links).
+
+This module rebuilds the reverse-reference index
+(:func:`sessionwiki.references.build_index`) and injects a managed
+section into each target page bounded by HTML-comment sentinels so
+rerun is idempotent:
+
+    <!-- BACKLINKS:START -->
+    ## Referenced by
+
+    - [[session-slug-1]] — Session: title 1 (2026-04-19)
+    - [[session-slug-2]] — Session: title 2 (2026-04-18)
+    - …
+    <!-- BACKLINKS:END -->
+
+Characteristics:
+
+* Stdlib-only.
+* Safe on every page — we only write inside the sentinels; everything
+  else is preserved verbatim.
+* Skips pages that are already inside a ``wiki/archive/`` subtree.
+* Caps each list at 50 entries so model entity pages don't balloon
+  (the full list stays reachable via the graph + ``sessionwiki references
+  <slug>``).
+* Sorts by date descending when the referrer has a ``date:`` field,
+  otherwise alphabetically by title.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from sessionwiki import REPO_ROOT
+
+
+_START = "<!-- BACKLINKS:START -->"
+_END = "<!-- BACKLINKS:END -->"
+_BACKLINK_BLOCK_RE = re.compile(
+    re.escape(_START) + r".*?" + re.escape(_END) + r"\s*",
+    re.DOTALL,
+)
+DEFAULT_MAX_ENTRIES = 50
+
+
+@dataclass
+class BacklinkEntry:
+    """One row in a ``## Referenced by`` list."""
+
+    slug: str
+    title: str
+    date: str  # may be empty
+
+    def sort_key(self) -> tuple[str, str]:
+        # Newest-first when date is ISO-8601; alphabetical by title otherwise.
+        return (self.date or "", self.title.lower())
+
+
+def _parse_frontmatter(text: str) -> tuple[dict[str, str], str]:
+    m = re.match(r"^---\n(.*?)\n---\n(.*)$", text, re.DOTALL)
+    if not m:
+        return {}, text
+    fm: dict[str, str] = {}
+    for line in m.group(1).splitlines():
+        mm = re.match(r"^([a-zA-Z_][\w-]*):\s*(.*)$", line)
+        if mm:
+            fm[mm.group(1)] = mm.group(2).strip().strip("'\"")
+    return fm, m.group(2)
+
+
+def _collect_pages(wiki_dir: Path) -> dict[str, dict[str, Any]]:
+    """Gather every wiki page into ``{slug: {path, meta, body}}``.
+
+    Skips ``wiki/archive/`` (deprecated) and ``_context.md`` stubs —
+    neither wants a backlinks block.
+    """
+    out: dict[str, dict[str, Any]] = {}
+    if not wiki_dir.is_dir():
+        return out
+    for p in sorted(wiki_dir.rglob("*.md")):
+        if "archive" in p.parts:
+            continue
+        if p.name == "_context.md":
+            continue
+        try:
+            text = p.read_text(encoding="utf-8")
+        except OSError:
+            continue
+        meta, body = _parse_frontmatter(text)
+        out[p.stem] = {"path": p, "meta": meta, "body": body, "text": text}
+    return out
+
+
+# Match `[[wikilink]]` or `[[wikilink|display]]` targets.
+_WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]*)?\]\]")
+
+
+def build_reverse_index(pages: dict[str, dict[str, Any]]) -> dict[str, list[BacklinkEntry]]:
+    """Return ``{target_slug: [BacklinkEntry(referrer)]}``.
+
+    An entry is produced for every ``[[Target]]`` occurrence in any
+    page's body (even if the Target doesn't exist — the graph's
+    broken-link checker handles that separately).  Self-references are
+    skipped (a page never appears in its own backlinks list).
+    """
+    rev: dict[str, list[BacklinkEntry]] = {}
+    for slug, page in pages.items():
+        body = page["body"]
+        for target in set(_WIKILINK_RE.findall(body)):
+            t = target.split("#")[0].strip()
+            if not t or t == slug:
+                continue
+            rev.setdefault(t, []).append(
+                BacklinkEntry(
+                    slug=slug,
+                    title=page["meta"].get("title", slug),
+                    date=page["meta"].get("date", ""),
+                )
+            )
+    return rev
+
+
+def _render_block(entries: list[BacklinkEntry], *, max_entries: int) -> str:
+    """Render the sentinel-bounded ``## Referenced by`` block."""
+    ordered = sorted(entries, key=lambda e: e.sort_key(), reverse=True)
+    truncated = len(ordered) > max_entries
+    ordered = ordered[:max_entries]
+    lines = [_START, "", "## Referenced by", ""]
+    for e in ordered:
+        suffix = f" — {e.title}" if e.title and e.title != e.slug else ""
+        if e.date:
+            suffix = f"{suffix} ({e.date})" if suffix else f" ({e.date})"
+        lines.append(f"- [[{e.slug}]]{suffix}")
+    if truncated:
+        lines.append(f"")
+        lines.append(
+            f"*…and {len(entries) - max_entries} more referrer(s) — "
+            f"run `sessionwiki references <slug>` for the full list.*"
+        )
+    lines.append("")
+    lines.append(_END)
+    return "\n".join(lines) + "\n"
+
+
+def inject_block(text: str, block: str) -> str:
+    """Replace or append the backlinks block in ``text``.
+
+    Finds the existing sentinel-bounded section and swaps it; appends
+    to the end of the file with a leading blank line when the sentinels
+    aren't present.  Idempotent.
+    """
+    stripped = text.rstrip()
+    if _START in text and _END in text:
+        new_text = _BACKLINK_BLOCK_RE.sub(block, text, count=1)
+        # Normalise trailing newlines.
+        if not new_text.endswith("\n"):
+            new_text += "\n"
+        return new_text
+    return stripped + "\n\n" + block
+
+
+def remove_block(text: str) -> str:
+    """Strip any existing backlinks block (used by ``--prune``)."""
+    if _START not in text:
+        return text
+    return _BACKLINK_BLOCK_RE.sub("", text).rstrip() + "\n"
+
+
+def inject_all(
+    wiki_dir: Optional[Path] = None,
+    *,
+    max_entries: int = DEFAULT_MAX_ENTRIES,
+    dry_run: bool = False,
+) -> dict[str, int]:
+    """Walk ``wiki_dir`` and inject backlinks into every target page.
+
+    Returns ``{slug: num_backlinks}`` for every page that got an
+    (added, updated, or unchanged) block.
+    """
+    root = wiki_dir or (REPO_ROOT / "wiki")
+    pages = _collect_pages(root)
+    rev = build_reverse_index(pages)
+    results: dict[str, int] = {}
+    for target_slug, entries in rev.items():
+        target = pages.get(target_slug)
+        if not target:
+            # Broken wikilink — skip.
+            continue
+        block = _render_block(entries, max_entries=max_entries)
+        new_text = inject_block(target["text"], block)
+        if new_text != target["text"] and not dry_run:
+            target["path"].write_text(new_text, encoding="utf-8")
+        results[target_slug] = len(entries)
+    return results
+
+
+def prune_all(
+    wiki_dir: Optional[Path] = None, *, dry_run: bool = False
+) -> int:
+    """Strip backlink blocks from every page. Returns count touched."""
+    root = wiki_dir or (REPO_ROOT / "wiki")
+    count = 0
+    for p in root.rglob("*.md"):
+        if "archive" in p.parts:
+            continue
+        try:
+            text = p.read_text(encoding="utf-8")
+        except OSError:
+            continue
+        if _START not in text:
+            continue
+        new_text = remove_block(text)
+        if new_text != text:
+            count += 1
+            if not dry_run:
+                p.write_text(new_text, encoding="utf-8")
+    return count