codetrail-cli 1.4.1__py3-none-any.whl

codetrail/repo/signal.py

@@ -0,0 +1,65 @@
+"""The "you're behind" signal: what landed since the last update, from git alone (design section 8.5).
+
+It fetches into the mirror without rebuilding `source/`, under the target's lock taken without waiting: while an
+update runs, the signal says so instead. Areas are the top-level folders of visible changed paths; excluded paths
+are never counted.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from codetrail.config import Paths, check_containment, load_target
+from codetrail.database import connect
+from codetrail.facts.store import FactStore
+from codetrail.lock import TargetBusy, target_lock
+from codetrail.repo.history import changed_paths, commit_count, merges_between
+from codetrail.repo.mirror import read_file_at, refresh_mirror
+from codetrail.repo.refresh import TARGET_IGNORE_FILE, ignore_lines
+from codetrail.repo.rules import ExclusionRules
+from codetrail.repo.source import SourceManifest
+
+
+@dataclass(frozen=True)
+class Signal:
+    updated_commit: str | None = None
+    head: str | None = None
+    merges: int = 0
+    commits: int = 0
+    areas: list[str] = field(default_factory=list)
+    updating: bool = False
+
+
+def behind(paths: Paths, name: str) -> Signal:
+    target = load_target(paths, name)
+    check_containment(paths, target.repository)
+    data = paths.target_data(name)
+    try:
+        with target_lock(paths, name):
+            mirror = data / "mirror.git"
+            head = refresh_mirror(mirror, target.repository, target.branch)
+            snapshot = _latest_snapshot_commit(paths, name)
+            if snapshot is None:
+                return Signal(head=head)
+            rules = ExclusionRules(ignore_lines(paths, name, read_file_at(mirror, head, TARGET_IGNORE_FILE)))
+            manifest = SourceManifest.load(data / "source.json")
+            flagged = manifest.excluded_paths() if manifest else set()
+            visible = [path for path in changed_paths(mirror, snapshot, head)
+                       if rules.reason(path) is None and path not in flagged]  # fmt: skip
+            areas = sorted({path.split("/", 1)[0] for path in visible if "/" in path})
+            return Signal(snapshot, head, merges_between(mirror, snapshot, head), commit_count(mirror, snapshot, head),
+                          areas)  # fmt: skip
+    except TargetBusy:
+        return Signal(updating=True)
+
+
+def _latest_snapshot_commit(paths: Paths, name: str) -> str | None:
+    database = paths.target_data(name) / "codetrail.db"
+    if not database.exists():
+        return None
+    connection = connect(database)
+    try:
+        snapshot = FactStore(connection).latest_snapshot()
+    finally:
+        connection.close()
+    return snapshot.commit if snapshot else None

codetrail/repo/source.py

@@ -0,0 +1,166 @@
+"""The materialized sources: only the files Codetrail may see, at one commit (design sections 3.2 to 3.4).
+
+`source/` is rebuilt in full on every refresh: written into `source.next/`, scanned with gitleaks, cleared of what it
+flags, then swapped into place. If anything fails, `source.next/` is removed and `source/` stays as it was.
+Symlinks, submodules and unsafe paths are never written.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from collections.abc import Callable, Mapping
+from dataclasses import asdict, dataclass, field
+from pathlib import Path, PurePosixPath
+
+from codetrail.errors import CodetrailError
+from codetrail.repo.mirror import list_tree, read_blobs
+from codetrail.repo.rules import ExclusionRules, Reason, on_disk_key
+from codetrail.repo.secrets import SecretScanner
+
+REGULAR_FILE_MODES = {"100644", "100755"}
+# A full SHA-1 or SHA-256 object name, the only commit a manifest may name: never an option, a ref or a range.
+FULL_COMMIT = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}")
+
+
+@dataclass(frozen=True)
+class Excluded:
+    path: str
+    reason: Reason
+    rule: str | None = None
+
+
+@dataclass(frozen=True)
+class SourceManifest:
+    commit: str
+    files: dict[str, str]  # path -> blob
+    excluded: list[Excluded] = field(default_factory=list)
+
+    def save(self, path: Path) -> None:
+        path.write_text(json.dumps(asdict(self), indent=1, sort_keys=True), encoding="utf-8")
+
+    @classmethod
+    def load(cls, path: Path) -> SourceManifest | None:
+        if not path.exists():
+            return None
+        data = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(data["commit"], str) or not FULL_COMMIT.fullmatch(data["commit"]):
+            raise CodetrailError(f"{path} names no valid commit; run codetrail update again.")
+        excluded = [Excluded(item["path"], Reason(item["reason"]), item["rule"]) for item in data["excluded"]]
+        return cls(commit=data["commit"], files=data["files"], excluded=excluded)
+
+    def excluded_paths(self) -> set[str]:
+        return {item.path for item in self.excluded}
+
+
+def safe_path(path: str) -> bool:
+    """A relative path with no empty, `.`, `..` or `.git` parts (in any case), which stays inside its folder."""
+    if not path or path.startswith("/") or "\\" in path:
+        return False
+    if any("\udc80" <= character <= "\udcff" for character in path):  # bytes that aren't UTF-8 (surrogateescape)
+        return False
+    parts = path.split("/")
+    return all(part and part not in (".", "..") and on_disk_key(part) != ".git" for part in parts)
+
+
+def allowed_reader(source: Path, files: Mapping[str, str], max_bytes: int) -> Callable[[str], bytes | None]:
+    """A reader of allowed files only: a listed path, inside `source/`, a plain file within the size cap.
+
+    The system pass (design 17.3) and the documentation metrics (design 18.2) read files only through it.
+    """
+    root = source.resolve()
+
+    def read(path: str) -> bytes | None:
+        if path not in files:
+            return None
+        file = source / PurePosixPath(path)
+        try:
+            if file.is_symlink() or not file.resolve().is_relative_to(root) or not file.is_file():
+                return None
+            return file.read_bytes() if file.stat().st_size <= max_bytes else None
+        except OSError:
+            return None
+
+    return read
+
+
+def build_source(
+    mirror: Path, commit: str, rules: ExclusionRules, scanner: SecretScanner, data_dir: Path
+) -> SourceManifest:
+    source, next_source = data_dir / "source", data_dir / "source.next"
+    shutil.rmtree(next_source, ignore_errors=True)
+    next_source.mkdir(parents=True)
+    try:
+        files: dict[str, str] = {}
+        excluded: list[Excluded] = []
+        entries = list_tree(mirror, commit)
+        colliding = _colliding_paths([entry.path for entry in entries])
+        for entry in entries:
+            reason = Reason.UNSAFE_PATH if entry.path in colliding else _reason(entry.mode, entry.path, rules)
+            if reason is None:
+                files[entry.path] = entry.blob
+            else:
+                excluded.append(Excluded(entry.path, reason))
+        paths_by_blob: dict[str, list[str]] = {}
+        for path, blob in files.items():
+            paths_by_blob.setdefault(blob, []).append(path)
+        for blob, content in read_blobs(mirror, list(paths_by_blob)):
+            for path in paths_by_blob[blob]:
+                destination = next_source / PurePosixPath(path)
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                destination.write_bytes(content)
+        flagged: dict[str, str] = {}
+        for finding in scanner.scan_directory(next_source):
+            flagged.setdefault(finding.path, finding.rule)
+        for path, rule in sorted(flagged.items()):
+            (next_source / PurePosixPath(path)).unlink()
+            del files[path]
+            excluded.append(Excluded(path, Reason.GITLEAKS, rule))
+        manifest = SourceManifest(
+            commit=commit, files=dict(sorted(files.items())), excluded=sorted(excluded, key=lambda item: item.path)
+        )
+        _swap(next_source, source)
+        manifest.save(data_dir / "source.json")
+        return manifest
+    except BaseException:
+        shutil.rmtree(next_source, ignore_errors=True)
+        raise
+
+
+def _colliding_paths(paths: list[str]) -> set[str]:
+    """Paths that would share a file or folder on a case-insensitive, normalizing file system.
+
+    Each path and each of its parent folders is a name on disk; two different spellings of one name collide, and
+    every path under either spelling is excluded (A.txt with a.txt, Docs/a.md with docs/b.md, Notes with notes/x).
+    """
+    spellings: dict[str, set[str]] = {}
+    for path in paths:
+        parts = path.split("/")
+        for end in range(1, len(parts) + 1):
+            prefix = "/".join(parts[:end])
+            spellings.setdefault(on_disk_key(prefix), set()).add(prefix)
+    clashing = {key for key, names in spellings.items() if len(names) > 1}
+    colliding = set()
+    for path in paths:
+        parts = path.split("/")
+        if any(on_disk_key("/".join(parts[:end])) in clashing for end in range(1, len(parts) + 1)):
+            colliding.add(path)
+    return colliding
+
+
+def _reason(mode: str, path: str, rules: ExclusionRules) -> Reason | None:
+    if not safe_path(path):
+        return Reason.UNSAFE_PATH
+    if mode not in REGULAR_FILE_MODES:
+        return Reason.NOT_A_FILE
+    return rules.reason(path)
+
+
+def _swap(next_source: Path, source: Path) -> None:
+    old = source.with_name("source.old")
+    shutil.rmtree(old, ignore_errors=True)
+    if source.exists():
+        source.rename(old)
+    next_source.rename(source)
+    shutil.rmtree(old, ignore_errors=True)

codetrail/search.py

@@ -0,0 +1,178 @@
+"""Search over the guide: an in-memory SQLite FTS5 index of pages, saved answers, decisions and facts (design 16.3).
+
+Only what the reader can already see in the guide is indexed: each page's title and body (never its front matter,
+which holds the checks' rubrics), and the names and kinds of facts. Source files are never read. Queries never reach
+FTS5's query language: every word is quoted as a literal, the last one matches as a prefix, and the expression is
+always a bound parameter. Snippets are plain text, split into segments that say whether they matched.
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+import threading
+import unicodedata
+from collections.abc import Iterable
+from dataclasses import dataclass
+from urllib.parse import quote
+
+from markdown_it import MarkdownIt
+
+from codetrail.facts import EntityKind
+from codetrail.facts.store import FactStore
+from codetrail.generate.validate import DIAGRAM, FACT_LINK, MARKER
+from codetrail.guide import GuideRepository
+
+WORD = re.compile(r"\w+")
+SNIPPET_CHARS = 160
+PARSER = MarkdownIt("commonmark").enable("table")
+# Title, headings and text, weighted for bm25; the other columns are stored, not searched.
+SCHEMA = (
+    "CREATE VIRTUAL TABLE documents USING fts5(title, headings, text, id UNINDEXED, kind UNINDEXED, "
+    "link UNINDEXED, tokenize = 'unicode61 remove_diacritics 2')"
+)
+
+
+@dataclass(frozen=True)
+class Document:
+    id: str  # a page id, "facts/<fact id>" or "decisions/<fact id>"
+    kind: str  # "area", "concept", "path", "digest", "answer", "decision" or "fact"
+    title: str
+    headings: str
+    text: str
+    link: str
+
+
+@dataclass(frozen=True)
+class Result:
+    document: Document
+    snippet: list[tuple[str, bool]]  # plain-text segments, and whether each one matched
+
+
+def match_expression(query: str, max_chars: int) -> str | None:
+    """The query as an FTS5 expression of quoted words, the last a prefix; None when no word is left."""
+    words = [word for word in query[:max_chars].split() if WORD.search(word)]
+    if not words:
+        return None
+    quoted = ['"' + word.replace('"', '""') + '"' for word in words]
+    return " ".join(quoted) + "*"
+
+
+def _fold(text: str) -> str:
+    """Lower case without accents, the way the index's tokenizer compares words."""
+    return "".join(char for char in unicodedata.normalize("NFKD", text.casefold()) if not unicodedata.combining(char))
+
+
+def _snippet(text: str, query: str, max_chars: int) -> list[tuple[str, bool]]:
+    terms = [_fold(word) for word in WORD.findall(query[:max_chars])]
+    if not text or not terms:
+        return [(text[:SNIPPET_CHARS], False)] if text else []
+
+    def matches(word: str) -> bool:
+        folded = _fold(word)
+        return folded in terms[:-1] or folded.startswith(terms[-1])
+
+    hits = [match for match in WORD.finditer(text) if matches(match.group())]
+    start = max(0, hits[0].start() - SNIPPET_CHARS // 3) if hits else 0
+    if start > 0:  # begin at a word, not inside one
+        space = text.find(" ", start)
+        start = space + 1 if 0 <= space < (hits[0].start() if hits else len(text)) else start
+    end = min(len(text), start + SNIPPET_CHARS)
+    if end < len(text):
+        space = text.rfind(" ", start, end)
+        end = space if space > (hits[0].end() if hits else start) else end
+    segments: list[tuple[str, bool]] = [("…", False)] if start > 0 else []
+    position = start
+    for hit in hits:
+        if hit.start() < start or hit.end() > end:
+            continue
+        if hit.start() > position:
+            segments.append((text[position : hit.start()], False))
+        segments.append((hit.group(), True))
+        position = hit.end()
+    if position < end:
+        segments.append((text[position:end], False))
+    if end < len(text):
+        segments.append(("…", False))
+    return segments
+
+
+class SearchIndex:
+    """One in-memory index, shared by the page's requests; every use holds the lock."""
+
+    def __init__(self) -> None:
+        self._connection = sqlite3.connect(":memory:", check_same_thread=False)
+        self._connection.execute(SCHEMA)
+        self._lock = threading.Lock()
+
+    def rebuild(self, documents: Iterable[Document]) -> None:
+        rows = [_row(document) for document in documents]
+        with self._lock, self._connection:
+            self._connection.execute("DELETE FROM documents")
+            self._connection.executemany("INSERT INTO documents VALUES (?, ?, ?, ?, ?, ?)", rows)
+
+    def add(self, document: Document) -> None:
+        with self._lock, self._connection:
+            self._connection.execute("DELETE FROM documents WHERE id = ?", (document.id,))
+            self._connection.execute("INSERT INTO documents VALUES (?, ?, ?, ?, ?, ?)", _row(document))
+
+    def search(self, query: str, max_chars: int, limit: int) -> list[Result]:
+        expression = match_expression(query, max_chars)
+        if expression is None:
+            return []
+        with self._lock:
+            try:
+                rows = self._connection.execute(
+                    "SELECT title, headings, text, id, kind, link FROM documents WHERE documents MATCH ?"
+                    " ORDER BY bm25(documents, 10.0, 4.0, 1.0) LIMIT ?",
+                    (expression, limit),
+                ).fetchall()
+            except sqlite3.OperationalError:  # every word is quoted, so this is FTS5 refusing an odd token
+                return []
+        results = []
+        for title, headings, text, identifier, kind, link in rows:
+            document = Document(identifier, kind, title, headings, text, link)
+            results.append(Result(document, _snippet(text, query, max_chars)))
+        return results
+
+
+def _row(document: Document) -> tuple[str, str, str, str, str, str]:
+    return (document.title, document.headings, document.text, document.id, document.kind, document.link)
+
+
+def _plain(body: str) -> tuple[str, str]:
+    """A page body's headings and its text, without Markdown, rationale markers or diagram placeholders."""
+    lines = [line for line in body.splitlines() if not MARKER.match(line) and not DIAGRAM.match(line.strip())]
+    source = FACT_LINK.sub(lambda match: match.group(1), "\n".join(lines))
+    headings: list[str] = []
+    pieces: list[str] = []
+    in_heading = False
+    for token in PARSER.parse(source):
+        if token.type == "heading_open":
+            in_heading = True
+        elif token.type == "heading_close":
+            in_heading = False
+        elif token.type == "inline":
+            words = "".join(child.content for child in token.children or [] if child.type in ("text", "code_inline"))
+            (headings if in_heading else pieces).append(words)
+        elif token.type in ("fence", "code_block"):
+            pieces.append(token.content.strip())
+    return "\n".join(headings), " ".join(piece for piece in pieces if piece)
+
+
+def guide_documents(guide: GuideRepository, store: FactStore | None) -> list[Document]:
+    """Everything search covers: the guide's pages (title and body only), decisions, and facts by name and kind."""
+    documents = []
+    for page in guide.pages() if guide.root.exists() else []:
+        headings, text = _plain(page.body)
+        documents.append(Document(page.id, page.kind, page.title, headings, text, f"/pages/{page.id}"))
+    for entity in store.entities() if store is not None else []:
+        link = f"/facts/{quote(entity.id, safe='')}"
+        if entity.kind == EntityKind.DECISION:
+            title = f"{entity.attributes.get('number', '')}. {entity.attributes.get('title', '')}".strip(". ")
+            text = str(entity.attributes.get("status", ""))
+            documents.append(Document(f"decisions/{entity.id}", "decision", title, "", text, link))
+        else:
+            text = f"{entity.kind} {entity.attributes.get('name', '')}".strip()
+            documents.append(Document(f"facts/{entity.id}", "fact", entity.id, "", text, link))
+    return documents

codetrail/server.py

@@ -0,0 +1,45 @@
+"""`codetrail serve`: the page on 127.0.0.1 only, opened through a single-use sign-in link (design section 7.4)."""
+
+from __future__ import annotations
+
+import webbrowser
+from dataclasses import dataclass
+
+import uvicorn
+from starlette.types import ASGIApp
+
+from codetrail.config import Paths, check_containment, load_global, load_target
+from codetrail.lock import target_in_use
+from codetrail.web.app import create_app
+from codetrail.web.security import SessionState
+
+HOST = "127.0.0.1"  # never configurable
+
+
+@dataclass(frozen=True)
+class Server:
+    app: ASGIApp
+    host: str
+    port: int
+    url: str
+
+
+def prepare_server(paths: Paths, name: str) -> Server:
+    target = load_target(paths, name)
+    check_containment(paths, target.repository)
+    settings = load_global(paths)
+    session = SessionState(settings.server.login_code_ttl_seconds, session_minutes=settings.server.session_minutes)
+    app = create_app(paths, name, session, settings)
+    port = settings.server.port
+    return Server(app, HOST, port, f"http://{HOST}:{port}/login?code={session.issue_login_code()}")
+
+
+def serve(paths: Paths, name: str, open_browser: bool) -> None:
+    with target_in_use(paths, name):  # for as long as it serves, so the target isn't removed under it
+        server = prepare_server(paths, name)
+        print(f"Codetrail is serving {name} at http://{server.host}:{server.port}/", flush=True)
+        print(f"Sign in (this link works once, for a short time): {server.url}", flush=True)
+        if open_browser:
+            webbrowser.open(server.url)
+        # No access log: the sign-in link's code would be written to it.
+        uvicorn.run(server.app, host=server.host, port=server.port, log_level="warning", access_log=False)