codetrail-cli 1.4.1__py3-none-any.whl

codetrail/facts/store.py

@@ -0,0 +1,209 @@
+"""The fact store: each fact version is one row, valid from `first_seen` to `last_seen` (design section 4.2; ADR 0003).
+
+An update writes only what changed: new rows for new and changed facts, a closed range for removed ones, and fresh
+sources in place for unchanged facts. The diff for a snapshot is a query on the ranges that open or close at it.
+Table and column names come only from this module's constants; values are always parameters.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from collections.abc import Iterable, Sequence
+from datetime import UTC, datetime
+from typing import Any
+
+from codetrail.facts import Entity, EntityKind, FactDiff, Relation, RelationKind, Snapshot, Source
+
+ENTITY_KEY = ("id",)
+RELATION_KEY = ("source_id", "kind", "target_id")
+
+
+def _sources_json(sources: tuple[Source, ...]) -> str:
+    return json.dumps([[source.path, source.start_line, source.end_line] for source in sources])
+
+
+def _sources(text: str) -> tuple[Source, ...]:
+    return tuple(Source(path, start, end) for path, start, end in json.loads(text))
+
+
+class FactStore:
+    def __init__(self, connection: sqlite3.Connection) -> None:
+        self.connection = connection
+
+    def record(
+        self, commit: str, entities: Iterable[Entity], relations: Iterable[Relation]
+    ) -> tuple[Snapshot, FactDiff]:
+        """Records the facts found at `commit` as a new snapshot and returns it with its diff."""
+        connection = self.connection
+        connection.execute("BEGIN")
+        try:
+            taken_at = datetime.now(UTC).isoformat(timespec="seconds")
+            cursor = connection.execute(
+                "INSERT INTO snapshots (commit_sha, taken_at) VALUES (?, ?)", (commit, taken_at)
+            )
+            snapshot = Snapshot(int(cursor.lastrowid or 0), commit, taken_at)
+            previous = self.previous_snapshot(snapshot)
+            closing = previous.id if previous else None
+            entity_rows: dict[tuple[str, ...], tuple[str, dict[str, Any]]] = {
+                (entity.id,): (entity.hash, {"kind": str(entity.kind), "attributes": _json(entity.attributes),
+                                             "sources": _sources_json(entity.sources)})
+                for entity in entities
+            }  # fmt: skip
+            relation_rows: dict[tuple[str, ...], tuple[str, dict[str, Any]]] = {
+                relation.key: (relation.hash, {"attributes": _json(relation.attributes),
+                                               "sources": _sources_json(relation.sources)})
+                for relation in relations
+            }  # fmt: skip
+            self._write("entities", ENTITY_KEY, entity_rows, snapshot.id, closing)
+            self._write("relations", RELATION_KEY, relation_rows, snapshot.id, closing)
+            connection.execute("COMMIT")
+        except BaseException:
+            if connection.in_transaction:  # SQLite ends it itself after some errors
+                connection.execute("ROLLBACK")
+            raise
+        return snapshot, self.diff(snapshot)
+
+    def _write(
+        self,
+        table: str,
+        key_columns: Sequence[str],
+        rows: dict[tuple[str, ...], tuple[str, dict[str, Any]]],
+        snapshot_id: int,
+        closing: int | None,
+    ) -> None:
+        keys = ", ".join(key_columns)
+        current = {
+            tuple(row[column] for column in key_columns): (row["rowid"], row["hash"])
+            for row in self.connection.execute(f"SELECT rowid, hash, {keys} FROM {table} WHERE last_seen IS NULL")
+        }
+        for key, (fact_hash, values) in rows.items():
+            existing = current.pop(key, None)
+            if existing is not None and existing[1] == fact_hash:
+                self.connection.execute(
+                    f"UPDATE {table} SET sources = ? WHERE rowid = ?",
+                    (values["sources"], existing[0]),
+                )
+                continue
+            if existing is not None:
+                self.connection.execute(f"UPDATE {table} SET last_seen = ? WHERE rowid = ?", (closing, existing[0]))
+            columns = (*key_columns, *values, "hash", "first_seen")
+            self.connection.execute(
+                f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({', '.join('?' for _ in columns)})",
+                (*key, *values.values(), fact_hash, snapshot_id),
+            )
+        for rowid, _hash in current.values():  # facts no longer found
+            self.connection.execute(f"UPDATE {table} SET last_seen = ? WHERE rowid = ?", (closing, rowid))
+
+    def latest_snapshot(self) -> Snapshot | None:
+        row = self.connection.execute("SELECT * FROM snapshots ORDER BY id DESC LIMIT 1").fetchone()
+        return Snapshot(row["id"], row["commit_sha"], row["taken_at"]) if row else None
+
+    def first_snapshot(self, commit: str) -> Snapshot | None:
+        """The earliest snapshot taken at the commit (a newer Codetrail can take more at the same one)."""
+        row = self.connection.execute(
+            "SELECT * FROM snapshots WHERE commit_sha = ? ORDER BY id LIMIT 1", (commit,)
+        ).fetchone()
+        return Snapshot(row["id"], row["commit_sha"], row["taken_at"]) if row else None
+
+    def previous_snapshot(self, snapshot: Snapshot) -> Snapshot | None:
+        row = self.connection.execute(
+            "SELECT * FROM snapshots WHERE id < ? ORDER BY id DESC LIMIT 1", (snapshot.id,)
+        ).fetchone()
+        return Snapshot(row["id"], row["commit_sha"], row["taken_at"]) if row else None
+
+    def diff(self, snapshot: Snapshot) -> FactDiff:
+        previous = self.previous_snapshot(snapshot)
+        closed_at = previous.id if previous else -1
+        entities = self._opened_and_closed("entities", ENTITY_KEY, snapshot.id, closed_at)
+        relations = self._opened_and_closed("relations", RELATION_KEY, snapshot.id, closed_at)
+        return FactDiff(
+            added_entities=sorted(key[0] for key in entities[0] - entities[1]),
+            changed_entities=sorted(key[0] for key in entities[0] & entities[1]),
+            removed_entities=sorted(key[0] for key in entities[1] - entities[0]),
+            added_relations=sorted(_relation_key(key) for key in relations[0] - relations[1]),
+            changed_relations=sorted(_relation_key(key) for key in relations[0] & relations[1]),
+            removed_relations=sorted(_relation_key(key) for key in relations[1] - relations[0]),
+        )
+
+    def _opened_and_closed(
+        self, table: str, key_columns: Sequence[str], opened_at: int, closed_at: int
+    ) -> tuple[set[tuple[str, ...]], set[tuple[str, ...]]]:
+        keys = ", ".join(key_columns)
+        opened = {tuple(row) for row in self.connection.execute(
+            f"SELECT {keys} FROM {table} WHERE first_seen = ?", (opened_at,))}  # fmt: skip
+        closed = {tuple(row) for row in self.connection.execute(
+            f"SELECT {keys} FROM {table} WHERE last_seen = ?", (closed_at,))}  # fmt: skip
+        return opened, closed
+
+    def snapshots(self) -> list[Snapshot]:
+        """Every snapshot, oldest first."""
+        rows = self.connection.execute("SELECT * FROM snapshots ORDER BY id")
+        return [Snapshot(row["id"], row["commit_sha"], row["taken_at"]) for row in rows]
+
+    def entity_counts(self, snapshot_ids: Sequence[int]) -> dict[int, dict[str, int]]:
+        """Each snapshot's entities by kind, from the validity ranges (design section 19.1)."""
+        counts: dict[int, dict[str, int]] = {snapshot_id: {} for snapshot_id in snapshot_ids}
+        marks = ", ".join("?" for _ in snapshot_ids)
+        rows = self.connection.execute(
+            "SELECT s.id AS snapshot, e.kind AS kind, COUNT(*) AS count FROM snapshots s JOIN entities e"
+            " ON e.first_seen <= s.id AND (e.last_seen IS NULL OR e.last_seen >= s.id)"
+            f" WHERE s.id IN ({marks}) GROUP BY s.id, e.kind", tuple(snapshot_ids))  # fmt: skip
+        for row in rows:
+            counts[row["snapshot"]][row["kind"]] = row["count"]
+        return counts
+
+    def entity_spans(self, kind: EntityKind) -> list[tuple[str, int, int | None]]:
+        """Each entity of the kind ever seen: its first snapshot, and its last unless it is still current."""
+        rows = self.connection.execute(
+            "SELECT id, MIN(first_seen) AS first, CASE WHEN SUM(last_seen IS NULL) > 0 THEN NULL"
+            " ELSE MAX(last_seen) END AS last FROM entities WHERE kind = ? GROUP BY id ORDER BY id",
+            (str(kind),),
+        )
+        return [(row["id"], row["first"], row["last"]) for row in rows]
+
+    def entities(self, kind: EntityKind | None = None) -> list[Entity]:
+        query = "SELECT * FROM entities WHERE last_seen IS NULL" + (" AND kind = ?" if kind else "") + " ORDER BY id"
+        return [_entity(row) for row in self.connection.execute(query, (str(kind),) if kind else ())]
+
+    def relations(self, kind: RelationKind | None = None) -> list[Relation]:
+        query = "SELECT * FROM relations WHERE last_seen IS NULL" + (" AND kind = ?" if kind else "")
+        query += " ORDER BY source_id, kind, target_id"
+        return [_relation(row) for row in self.connection.execute(query, (str(kind),) if kind else ())]
+
+    def entities_at(self, snapshot_id: int) -> list[Entity]:
+        """Every entity as it was at the snapshot."""
+        query = ("SELECT * FROM entities WHERE first_seen <= ? AND (last_seen IS NULL OR last_seen >= ?)"
+                 " ORDER BY id")  # fmt: skip
+        return [_entity(row) for row in self.connection.execute(query, (snapshot_id, snapshot_id))]
+
+    def relations_at(self, snapshot_id: int) -> list[Relation]:
+        """Every relation as it was at the snapshot."""
+        query = ("SELECT * FROM relations WHERE first_seen <= ? AND (last_seen IS NULL OR last_seen >= ?)"
+                 " ORDER BY source_id, kind, target_id")  # fmt: skip
+        return [_relation(row) for row in self.connection.execute(query, (snapshot_id, snapshot_id))]
+
+    def entity(self, entity_id: str) -> Entity | None:
+        row = self.connection.execute(
+            "SELECT * FROM entities WHERE id = ? AND last_seen IS NULL", (entity_id,)
+        ).fetchone()
+        return _entity(row) if row else None
+
+
+def _json(attributes: Any) -> str:
+    return json.dumps(dict(attributes), sort_keys=True, ensure_ascii=False)
+
+
+def _relation_key(key: tuple[str, ...]) -> tuple[str, str, str]:
+    return (key[0], key[1], key[2])
+
+
+def _entity(row: sqlite3.Row) -> Entity:
+    return Entity(row["id"], EntityKind(row["kind"]), json.loads(row["attributes"]), _sources(row["sources"]))
+
+
+def _relation(row: sqlite3.Row) -> Relation:
+    return Relation(
+        row["source_id"], RelationKind(row["kind"]), row["target_id"], json.loads(row["attributes"]),
+        _sources(row["sources"]),
+    )  # fmt: skip

codetrail/generate/__init__.py

@@ -0,0 +1 @@
+"""Generation: the outline, the pages, their validation and the digest (design section 6)."""

codetrail/generate/failures.py

@@ -0,0 +1,31 @@
+"""Pages whose drafts failed validation, remembered so an update doesn't pay for them again unchanged (design 6.5).
+
+A page is remembered with the hash of its scope when it failed; while the hash is the same, updates skip it. A
+provider error isn't remembered, since it says nothing about the page.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from datetime import UTC, datetime
+
+
+class FailedPages:
+    def __init__(self, connection: sqlite3.Connection) -> None:
+        self.connection = connection
+
+    def scope_hash(self, page_id: str) -> str | None:
+        """The scope hash the page failed with, or None when it hasn't failed."""
+        row = self.connection.execute("SELECT scope_hash FROM page_failures WHERE page_id = ?", (page_id,)).fetchone()
+        return str(row["scope_hash"]) if row else None
+
+    def record(self, page_id: str, scope_hash: str) -> None:
+        now = datetime.now(UTC).isoformat(timespec="seconds")
+        self.connection.execute(
+            "INSERT INTO page_failures (page_id, scope_hash, failed_at) VALUES (?, ?, ?) ON CONFLICT (page_id)"
+            " DO UPDATE SET scope_hash = excluded.scope_hash, failed_at = excluded.failed_at",
+            (page_id, scope_hash, now),
+        )
+
+    def forget(self, page_id: str) -> None:
+        self.connection.execute("DELETE FROM page_failures WHERE page_id = ?", (page_id,))

codetrail/generate/outline.py

@@ -0,0 +1,159 @@
+"""The outline: every page's id, kind, title and scope (design section 6.2).
+
+Claude proposes it; Codetrail keeps only entries it can check: a valid id for the kind, a scope that holds visible
+files, and fact ids that exist. The founder may edit `outline.yaml`; Claude only adds to it.
+"""
+
+from __future__ import annotations
+
+import unicodedata
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+from codetrail.facts import EntityKind
+from codetrail.facts.store import FactStore
+from codetrail.guide import PAGE_ID
+from codetrail.repo.source import SourceManifest
+
+KIND_FOLDERS = {"area": "areas", "concept": "concepts"}
+
+
+CONTROL_CATEGORIES = ("Cc", "Cf", "Cs", "Zl", "Zp")  # characters a terminal acts on, or that hide text
+
+
+@dataclass(frozen=True)
+class OutlineEntry:
+    id: str
+    kind: str
+    title: str
+    scope_paths: list[str]
+    scope_kinds: list[str] = field(default_factory=list)
+    facts: list[str] = field(default_factory=list)
+
+
+def in_scope(path: str, scope_paths: Iterable[str]) -> bool:
+    for scope in scope_paths:
+        scope = scope.strip("/")
+        if not scope or path == scope or path.startswith(scope + "/"):
+            return True
+    return False
+
+
+def validate_outline(
+    raw: Sequence[Mapping[str, Any]], store: FactStore, manifest: SourceManifest, existing: Sequence[OutlineEntry] = ()
+) -> tuple[list[OutlineEntry], list[str]]:
+    """The entries that pass, and a problem for each one dropped or trimmed."""
+    entries: list[OutlineEntry] = []
+    problems: list[str] = []
+    seen = {entry.id for entry in existing}
+    kinds = {str(kind) for kind in EntityKind}
+    for item in raw:
+        page_id = str(item.get("id", "")).strip().lower()
+        kind = str(item.get("kind", ""))
+        if (
+            kind not in KIND_FOLDERS
+            or not PAGE_ID.fullmatch(page_id)
+            or not page_id.startswith(KIND_FOLDERS[kind] + "/")
+        ):
+            problems.append(f"{page_id!r}: the id must be {KIND_FOLDERS.get(kind, 'areas or concepts')}/<slug>.")
+            continue
+        if page_id in seen:
+            problems.append(f"{page_id}: the id is already used.")
+            continue
+        scope = [str(path).strip("/") for path in item.get("scope_paths", []) if isinstance(path, str)]
+        scope = [path for path in scope if any(in_scope(file, [path]) for file in manifest.files)]
+        if not scope:
+            problems.append(f"{page_id}: none of its scope paths holds a visible file.")
+            continue
+        facts = [str(fact) for fact in item.get("facts", []) if isinstance(fact, str)]
+        known = [fact for fact in facts if store.entity(fact) is not None]
+        if len(known) != len(facts):
+            problems.append(f"{page_id}: dropped facts that don't exist: {sorted(set(facts) - set(known))}.")
+        scope_kinds = [str(kind) for kind in item.get("scope_kinds", []) if str(kind) in kinds]
+        title = str(item.get("title", page_id)).strip()
+        if any(unicodedata.category(character) in CONTROL_CATEGORIES for character in title):
+            problems.append(f"{page_id}: the title holds a control character.")  # a terminal would act on it
+            continue
+        entries.append(OutlineEntry(page_id, kind, title, scope, scope_kinds, known))
+        seen.add(page_id)
+    return entries, problems
+
+
+def outline_entries(data: Mapping[str, Any] | None) -> list[OutlineEntry]:
+    """The entries of a stored outline, skipping any that are malformed (the founder may have edited it)."""
+    entries = []
+    for item in (data or {}).get("pages", []) or []:
+        if isinstance(item, Mapping) and PAGE_ID.fullmatch(str(item.get("id", ""))):
+            entries.append(
+                OutlineEntry(
+                    str(item["id"]),
+                    str(item.get("kind", "")),
+                    str(item.get("title", item["id"])),
+                    [str(path) for path in item.get("scope_paths", []) or []],
+                    [str(kind) for kind in item.get("scope_kinds", []) or []],
+                    [str(fact) for fact in item.get("facts", []) or []],
+                )
+            )
+    return entries
+
+
+@dataclass(frozen=True)
+class OutlinePath:
+    id: str
+    title: str
+    goal: str
+    steps: list[str]
+
+
+def outline_data(entries: Iterable[OutlineEntry], paths: Iterable[OutlinePath] = ()) -> dict[str, Any]:
+    return {"pages": [asdict(entry) for entry in entries], "paths": [asdict(path) for path in paths]}
+
+
+def validate_paths(
+    raw: Sequence[Mapping[str, Any]], page_ids: set[str], existing: Sequence[OutlinePath] = ()
+) -> tuple[list[OutlinePath], list[str]]:
+    """Paths whose id is valid and that keep at least one step naming an existing page."""
+    paths: list[OutlinePath] = []
+    problems: list[str] = []
+    seen = {path.id for path in existing}
+    for item in raw:
+        path_id = str(item.get("id", "")).strip().lower()
+        if not path_id.startswith("paths/") or not PAGE_ID.fullmatch(path_id) or path_id in seen:
+            problems.append(f"{path_id!r}: a path id must be a new paths/<slug>.")
+            continue
+        steps = [str(step) for step in item.get("steps", []) if isinstance(step, str)]
+        kept = [step for step in steps if step in page_ids]
+        if len(kept) != len(steps):
+            problems.append(f"{path_id}: dropped steps that aren't pages: {sorted(set(steps) - set(kept))}.")
+        if not kept:
+            problems.append(f"{path_id}: no step names an existing page.")
+            continue
+        paths.append(OutlinePath(path_id, str(item.get("title", path_id)), str(item.get("goal", "")), kept))
+        seen.add(path_id)
+    return paths, problems
+
+
+def outline_paths(data: Mapping[str, Any] | None, page_ids: set[str]) -> list[OutlinePath]:
+    """The stored paths, each keeping only steps that are still pages."""
+    paths = []
+    for item in (data or {}).get("paths", []) or []:
+        path_id = str(item.get("id", "")) if isinstance(item, Mapping) else ""
+        if isinstance(item, Mapping) and path_id.startswith("paths/") and PAGE_ID.fullmatch(path_id):
+            steps = [str(step) for step in item.get("steps", []) or [] if str(step) in page_ids]
+            if steps:
+                paths.append(OutlinePath(str(item["id"]), str(item.get("title", "")), str(item.get("goal", "")), steps))
+    return paths
+
+
+def uncovered_facts(store: FactStore, entries: Sequence[OutlineEntry]) -> list[str]:
+    """Projects, modules and decisions that no page's scope covers."""
+    named = {fact for entry in entries for fact in entry.facts}
+    uncovered = []
+    for entity in store.entities():
+        if entity.kind is EntityKind.PACKAGE or entity.id in named:
+            continue
+        paths = [source.path for source in entity.sources]
+        if not any(in_scope(path, entry.scope_paths) for path in paths for entry in entries):
+            uncovered.append(entity.id)
+    return sorted(uncovered)

codetrail/generate/revise.py

@@ -0,0 +1,72 @@
+"""Revising a page: the sections the assistant rewrote, put into the page in place of the old ones (design 6.3).
+
+A section is a heading and everything under it up to the next heading of the same or a higher level, so replacing
+`## How it works` replaces its `###` subsections too. Lines inside code fences are never headings. A section whose
+heading the page doesn't have is added at the end; one with an empty body is removed.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping, Sequence
+
+from codetrail.assistant.prompts import FULLWIDTH_AT
+
+HEADING = re.compile(r"(#{1,6})\s+\S")
+FENCE = re.compile(r"(```|~~~)")
+
+
+def apply_sections(body: str, sections: Sequence[Mapping[str, object]]) -> str:
+    lines = body.rstrip("\n").split("\n")
+    for section in sections:
+        heading = str(section.get("heading", "")).strip().replace(FULLWIDTH_AT, "@")  # the prompt showed it fullwidth
+        if not HEADING.match(heading):
+            continue
+        text = _without_heading(str(section.get("body", "")), heading)
+        new = [heading, "", *text.split("\n")] if text else []
+        start, end = _section(lines, heading)
+        if start is None:
+            if new:
+                lines += ["", *new]
+            continue
+        if new and end < len(lines) and lines[end].strip():  # keep a blank line before the next heading
+            new.append("")
+        lines[start:end] = new
+        if not new and 0 < start < len(lines) and not lines[start - 1].strip() and not lines[start].strip():
+            del lines[start]  # the blank lines on both sides of a removed section become one
+        while lines and not lines[-1].strip():
+            lines.pop()
+    return "\n".join(lines) + "\n"
+
+
+def _without_heading(text: str, heading: str) -> str:
+    """The section's text, without the heading line a model may repeat at its top."""
+    text = text.strip("\n")
+    first, _, rest = text.partition("\n")
+    return rest.strip("\n") if first.strip() == heading else text
+
+
+def _section(lines: list[str], heading: str) -> tuple[int | None, int]:
+    """Where the section under the heading starts and ends (the line after it), outside code fences."""
+    start: int | None = None
+    level = 0
+    fenced = False
+    for index, line in enumerate(lines):
+        if FENCE.match(line.strip()):
+            fenced = not fenced
+            continue
+        match = None if fenced else HEADING.match(line)
+        if match is None:
+            continue
+        if start is not None and len(match.group(1)) <= level:
+            return start, _trim_blank(lines, start, index)
+        if start is None and line.strip() == heading:
+            start, level = index, len(match.group(1))
+    return start, len(lines)
+
+
+def _trim_blank(lines: list[str], start: int, end: int) -> int:
+    """The section's end, before the blank lines that separate it from the next heading."""
+    while end > start + 1 and not lines[end - 1].strip():
+        end -= 1
+    return end