witful 0.1.3__py3-none-any.whl

witful/__init__.py

@@ -0,0 +1,5 @@
+"""
+Curate portable preferences, sourced teachings, and observed reuse through Witful.
+"""
+
+from __future__ import annotations

witful/checkpoints/__init__.py

@@ -0,0 +1,5 @@
+"""
+Compare explicit project boundaries using immutable snapshots of documented evidence.
+"""
+
+from __future__ import annotations

witful/checkpoints/cli.py

@@ -0,0 +1,116 @@
+"""
+Record, compare, mirror, and render explicit project checkpoints.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from jsonschema import ValidationError
+
+from witful.checkpoints.diff import compare
+from witful.checkpoints.journal import read, record
+from witful.checkpoints.models import Checkpoint
+from witful.checkpoints.plotting import build
+from witful.checkpoints.storage import SQLiteCheckpointStore, open_store
+from witful.files import replace_file
+from witful.knowledge.contracts import as_data, schema
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+__all__ = ["main"]
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """
+    Execute one bounded operation, keeping remote writes explicit and plots out of recording.
+
+    Args:
+        argv (Sequence[str] | None): Explicit arguments or process command line.
+
+    Returns:
+        int: Zero on success; parser exit two for invalid data, references, or unavailable storage.
+    """
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo", type=Path, default=Path.cwd())
+    parser.add_argument("--namespace", default=os.environ.get("WITFUL_DATABASE_NAMESPACE", "default"))
+    parser.add_argument("--database", type=Path, help="Local SQLite mirror, default .witful/checkpoints.sqlite3")
+    commands = parser.add_subparsers(dest="command", required=True)
+    capture = commands.add_parser("record", help="Snapshot reviewed evidence at a project's committed HEAD")
+    capture.add_argument("--project", type=Path, required=True)
+    capture.add_argument("--project-id", required=True, help="Stable chosen alias; remote URLs and machine paths are never inferred")
+    commands.add_parser("check", help="Validate the journal without creating data or plots")
+    commands.add_parser("schemas", help="Generate the authoritative checkpoint schema")
+    commands.add_parser("sync", help="Seed the SQLite mirror, or explicitly selected WITFUL_DATABASE_URL")
+    difference = commands.add_parser("diff", help="Compare two snapshots, defaulting to the last two checkpoints")
+    difference.add_argument("--before")
+    difference.add_argument("--after")
+    difference.add_argument("--from-database", action="store_true")
+    render = commands.add_parser("build", help="Compute all differences and refresh recent-project plots")
+    render.add_argument("--from-database", action="store_true")
+    render.add_argument("--output-dir", type=Path, default=Path("docs/assets"))
+    render.add_argument("--limit", type=int, default=8)
+    args = parser.parse_args(argv)
+    root = args.repo.resolve()
+    database = args.database or root / ".witful/checkpoints.sqlite3"
+
+    try:
+        if args.command == "schemas":
+            path = root / "knowledge/schemas/checkpoint.schema.json"
+            path.parent.mkdir(parents=True, exist_ok=True)
+            content = (json.dumps(schema(Checkpoint), indent=2, sort_keys=True) + "\n").encode()
+            replace_file(path, content, expected=path.read_bytes() if path.exists() else None)
+            return 0
+
+        if args.command == "record":
+            event = record(root, args.project, args.project_id)
+
+            # Journal persistence precedes the disposable local mirror; an interrupted sync is safe to retry.
+            SQLiteCheckpointStore(database, args.namespace).seed(read(root / "knowledge/checkpoints.jsonl"))
+            print(event.id)
+            return 0
+
+        stored = args.command in {"build", "diff"} and args.from_database
+        events = (
+            open_store(database, os.environ.get("WITFUL_DATABASE_URL"), args.namespace).load()
+            if stored
+            else read(root / "knowledge/checkpoints.jsonl")
+        )
+
+        if args.command == "check":
+            print(f"Validated {len(events)} project checkpoints")
+        elif args.command == "sync":
+            print(
+                json.dumps({"added_checkpoints": open_store(database, os.environ.get("WITFUL_DATABASE_URL"), args.namespace).seed(events)})
+            )
+        elif args.command == "diff":
+            if len(events) < 2 and (not args.before or not args.after):
+                raise ValueError("Record two checkpoints or supply two existing checkpoint IDs")
+
+            indexed = {event.id: event for event in events}
+            before = indexed.get(args.before) if args.before else events[-2]
+            after = indexed.get(args.after) if args.after else events[-1]
+
+            if before is None or after is None:
+                raise ValueError("Unknown checkpoint ID")
+
+            print(json.dumps([as_data(item) for item in compare(before, after)], indent=2))
+        else:
+            output = args.output_dir if args.output_dir.is_absolute() else root / args.output_dir
+
+            for path in build(events, output, args.limit):
+                print(path)
+    except (OSError, ValueError, RuntimeError, ValidationError, subprocess.TimeoutExpired):
+        parser.exit(
+            2,
+            "Checkpoint operation failed; validate source records, project HEAD, requested IDs, and storage access. "
+            "No source text or credentials were logged.\n",
+        )
+
+    return 0

witful/checkpoints/diff.py

@@ -0,0 +1,122 @@
+"""
+Compare checkpoint evidence using stable IDs rather than subtracting aggregate counts.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from attrs import frozen
+
+from witful.checkpoints.models import SkillState
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from witful.checkpoints.models import Checkpoint, Fact
+
+__all__ = ["Change", "SkillDiff", "compare", "visits"]
+
+
+@frozen
+class Change:
+    """
+    Distinguish new evidence from revisions and withdrawals.
+
+    Attributes:
+        added (tuple[str, ...]): Keys present only in the later snapshot.
+        removed (tuple[str, ...]): Keys present only in the earlier snapshot.
+        revised (tuple[str, ...]): Shared keys with changed content fingerprints.
+    """
+
+    added: tuple[str, ...] = ()
+    removed: tuple[str, ...] = ()
+    revised: tuple[str, ...] = ()
+
+
+@frozen
+class SkillDiff:
+    """
+    Describe one changed skill without interpreting its counts as competence.
+
+    Attributes:
+        id (str): Stable skill ID.
+        name (str): Most recent label, or the earlier one for a removed skill.
+        status (str): Added, removed, renamed, or retained skill.
+        lessons (Change): Documented teaching changes; classification changes remain observable.
+        uses (Change): Actual task/context observations added, removed, or revised.
+    """
+
+    id: str
+    name: str
+    status: str
+    lessons: Change
+    uses: Change
+
+
+def _change(before: tuple[Fact, ...], after: tuple[Fact, ...]) -> Change:
+    """
+    Derive exact set differences and shared-key revisions.
+
+    Args:
+        before (tuple[Fact, ...]): Earlier validated facts.
+        after (tuple[Fact, ...]): Later validated facts.
+
+    Returns:
+        Change: Canonical keys grouped by their observed change type.
+    """
+    left = {fact.id: fact.digest for fact in before}
+    right = {fact.id: fact.digest for fact in after}
+    return Change(
+        tuple(sorted(right.keys() - left.keys())),
+        tuple(sorted(left.keys() - right.keys())),
+        tuple(sorted(key for key in left.keys() & right.keys() if left[key] != right[key])),
+    )
+
+
+def compare(before: Checkpoint, after: Checkpoint) -> tuple[SkillDiff, ...]:
+    """
+    Compare any two stored snapshots without loading current lessons or application records.
+
+    Args:
+        before (Checkpoint): Earlier or explicitly chosen baseline.
+        after (Checkpoint): Comparison target.
+
+    Returns:
+        tuple[SkillDiff, ...]: Only changed skills, sorted by stable ID; identical evidence yields no differences.
+    """
+    left = {state.id: state for state in before.skills}
+    right = {state.id: state for state in after.skills}
+    result: list[SkillDiff] = []
+
+    for key in sorted(left.keys() | right.keys()):
+        prior = left.get(key, SkillState(key, right[key].name)) if key not in left else left[key]
+        current = right.get(key, SkillState(key, prior.name))
+        status = "added" if key not in left else "removed" if key not in right else "renamed" if prior.name != current.name else "retained"
+        item = SkillDiff(key, current.name, status, _change(prior.lessons, current.lessons), _change(prior.uses, current.uses))
+
+        if status != "retained" or item.lessons != Change() or item.uses != Change():
+            result.append(item)
+
+    return tuple(result)
+
+
+def visits(events: Sequence[Checkpoint]) -> tuple[Checkpoint, ...]:
+    """
+    Collapse consecutive commits in one project to the final state of that visit.
+
+    Args:
+        events (Sequence[Checkpoint]): Validated journal in append order.
+
+    Returns:
+        tuple[Checkpoint, ...]: Visit endpoints in chronological order; returning to a project creates another visit.
+    """
+    result: list[Checkpoint] = []
+
+    for event in events:
+        if result and result[-1].project == event.project:
+            result[-1] = event
+        else:
+            result.append(event)
+
+    return tuple(result)

witful/checkpoints/journal.py

@@ -0,0 +1,154 @@
+"""
+Append explicit Git boundaries with a validated hash chain and idempotent retry semantics.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import json
+import re
+import subprocess
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from attrs import evolve
+
+from witful.checkpoints.models import Checkpoint
+from witful.checkpoints.snapshot import digest, snapshot
+from witful.files import replace_file
+from witful.knowledge.applications import read_applications
+from witful.knowledge.compiler import compile_identity
+from witful.knowledge.contracts import as_data, structure
+from witful.knowledge.models import Catalog
+from witful.lessons import read_journal
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from pathlib import Path
+
+__all__ = ["checkpoint_id", "read", "record", "validate"]
+_HASH = re.compile(r"[0-9a-f]{64}")
+
+
+def checkpoint_id(event: Checkpoint) -> str:
+    """
+    Bind project metadata and all snapshot fields to an immutable identifier.
+
+    Args:
+        event (Checkpoint): Record whose current ID is ignored.
+
+    Returns:
+        str: Canonical content digest.
+    """
+    value = as_data(event)
+    value.pop("id")
+    return digest(value)
+
+
+def validate(events: Sequence[Checkpoint]) -> None:
+    """
+    Reject broken chronology, altered content, duplicate facts, and ambiguous states.
+
+    Args:
+        events (Sequence[Checkpoint]): Complete ordered history.
+
+    Returns:
+        None: The hash chain and every snapshot meet the checkpoint contract.
+
+    Raises:
+        ValueError: A record violates a structural or chronological invariant.
+    """
+    previous: str | None = None
+    seen: set[str] = set()
+
+    for event in events:
+        if event.id != checkpoint_id(event) or event.id in seen or event.previous != previous:
+            raise ValueError("Invalid checkpoint content or history chain")
+        if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._/-]{0,127}", event.project) or ".." in event.project.split("/"):
+            raise ValueError("Project aliases require 1-128 letters, digits, dots, slashes, underscores, or hyphens")
+        if not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", event.revision):
+            raise ValueError("Checkpoint revision must be a full Git commit ID")
+        if datetime.fromisoformat(event.recorded_at).tzinfo is None:
+            raise ValueError("Checkpoint recording time must include a timezone")
+        identifiers = [state.id for state in event.skills]
+
+        if identifiers != sorted(set(identifiers)) or any(not state.id.strip() or not state.name.strip() for state in event.skills):
+            raise ValueError("Checkpoint skills require sorted unique IDs and nonempty names")
+
+        for state in event.skills:
+            for facts in (state.lessons, state.uses):
+                keys = [fact.id for fact in facts]
+
+                if keys != sorted(set(keys)) or any(not fact.id.strip() or not _HASH.fullmatch(fact.digest) for fact in facts):
+                    raise ValueError("Checkpoint facts require unique sorted keys and SHA-256 digests")
+
+        previous = event.id
+        seen.add(event.id)
+
+
+def read(path: Path) -> tuple[Checkpoint, ...]:
+    """
+    Read an empty or existing source journal without creating a database.
+
+    Args:
+        path (Path): Collection-owned checkpoint JSONL file.
+
+    Returns:
+        tuple[Checkpoint, ...]: Complete validated history; absence means no recorded boundaries.
+    """
+    events = (
+        tuple(structure(json.loads(line), Checkpoint) for line in path.read_text().splitlines() if line.strip()) if path.exists() else ()
+    )
+    validate(events)
+    return events
+
+
+def record(root: Path, project: Path, alias: str) -> Checkpoint:
+    """
+    Append one explicit project boundary using current reviewed collection inputs.
+
+    Args:
+        root (Path): Collection containing lessons and application journals.
+        project (Path): Existing Git checkout to identify, without reading its source or remote configuration.
+        alias (str): Stable project identifier selected by the caller.
+
+    Returns:
+        Checkpoint: Newly appended record or the last record for an identical immediate retry.
+
+    Raises:
+        ValueError: Git has no committed HEAD or the source evidence is invalid.
+        RuntimeError: A source journal changes concurrently with the atomic replacement.
+    """
+    result = subprocess.run(
+        ["git", "-C", str(project.resolve()), "rev-parse", "--verify", "HEAD^{commit}"],
+        capture_output=True,
+        text=True,
+        check=False,
+        timeout=10,
+    )
+
+    if result.returncode:
+        raise ValueError("Checkpoint project must be an existing Git checkout with a committed HEAD")
+
+    # Reuse the collection's writer lock so application updates cannot race snapshot compilation.
+    with (root / ".applications.lock").open("a") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        path = root / "knowledge/checkpoints.jsonl"
+        original = path.read_bytes() if path.exists() else None
+        events = read(path)
+        catalog = structure(json.loads((root / "knowledge/catalog.json").read_text()), Catalog)
+        identity = compile_identity(read_journal(root / "lessons.jsonl"), catalog)
+        skills = snapshot(identity, read_applications(root / "knowledge/applications.jsonl"))
+        revision = result.stdout.strip()
+
+        # Returning to a project after another visit is a new boundary, even at the same commit.
+        if events and (events[-1].project, events[-1].revision, events[-1].skills) == (alias, revision, skills):
+            return events[-1]
+
+        event = Checkpoint("", events[-1].id if events else None, alias, revision, datetime.now(UTC).isoformat(), skills)
+        event = evolve(event, id=checkpoint_id(event))
+        validate((*events, event))
+        content = (original or b"").rstrip(b"\n")
+        content += (b"\n" if content else b"") + json.dumps(as_data(event), sort_keys=True).encode() + b"\n"
+        replace_file(path, content, expected=original)
+        return event

witful/checkpoints/models.py

@@ -0,0 +1,67 @@
+"""
+Define portable checkpoint snapshots independently of Git paths and storage engines.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from attrs import frozen
+
+__all__ = ["Checkpoint", "Fact", "SkillState"]
+
+
+@frozen
+class Fact:
+    """
+    Identify a documented item and its revision without copying its text into every checkpoint.
+
+    Attributes:
+        id (str): Stable item key; application keys hash their task and context.
+        digest (str): SHA-256 of the corresponding canonical source object.
+    """
+
+    id: str
+    digest: str
+
+
+@frozen
+class SkillState:
+    """
+    Snapshot one skill's current teachings and explicitly recorded applications.
+
+    Attributes:
+        id (str): Stable skill ID.
+        name (str): Reviewed display name at this boundary.
+        lessons (tuple[Fact, ...]): Active lessons, including their lemma annotations in the digest.
+        uses (tuple[Fact, ...]): Latest observation for each task/context reaching this skill.
+    """
+
+    id: str
+    name: str
+    lessons: tuple[Fact, ...] = ()
+    uses: tuple[Fact, ...] = ()
+
+
+@frozen
+class Checkpoint:
+    """
+    Mark a project visit or commit without asserting that the project caused the observed changes.
+
+    Attributes:
+        id (str): SHA-256 of the complete record except this field.
+        previous (str | None): Prior checkpoint ID, giving an explicit chronology.
+        project (str): User-selected stable public or private alias, never an inferred remote URL.
+        revision (str): Exact Git commit at recording time.
+        recorded_at (str): Timezone-aware recording time, independent of Git author dates.
+        skills (tuple[SkillState, ...]): Complete canonical state, sufficient to recompute comparisons.
+        version (Literal[1]): Supported source schema version.
+    """
+
+    id: str
+    previous: str | None
+    project: str
+    revision: str
+    recorded_at: str
+    skills: tuple[SkillState, ...] = ()
+    version: Literal[1] = 1

witful/checkpoints/plotting.py

@@ -0,0 +1,146 @@
+"""
+Render recent project visits and cumulative comparisons without implying proficiency scores.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+import matplotlib as mpl
+from matplotlib.figure import Figure
+
+from witful.checkpoints.diff import compare, visits
+from witful.checkpoints.snapshot import digest
+from witful.files import replace_file
+from witful.knowledge.contracts import as_data
+from witful.knowledge.plotting import save_figure
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from pathlib import Path
+
+    from witful.checkpoints.models import Checkpoint
+
+__all__ = ["build"]
+
+
+def _figure(pairs: Sequence[tuple[Checkpoint, Checkpoint]], title: str, question: str) -> Figure:
+    """
+    Show sparse signed additions, withdrawals, and revisions for each changed skill.
+
+    Args:
+        pairs (Sequence[tuple[Checkpoint, Checkpoint]]): Baseline/target pairs in newest-first display order.
+        title (str): Short plot title.
+        question (str): Question answered by this view.
+
+    Returns:
+        Figure: Annotated matrix with zero-change cells muted and no inferred initial growth.
+    """
+    differences = [compare(before, after) for before, after in pairs]
+    names = {item.id: item.name for group in reversed(differences) for item in group}
+    keys = sorted(names, key=lambda key: (names[key].casefold(), key))
+    figure = Figure(figsize=(max(10, min(30, 5 + 2 * len(pairs))), max(4.5, 2.7 + len(keys) * 0.65)), layout="constrained")
+    figure.suptitle(title + "\n" + question, fontsize=16)
+    axis = figure.subplots()
+
+    if not keys or not pairs:
+        message = (
+            "No changed skill evidence between these boundaries" if pairs else "Record a second project visit to compare with the baseline"
+        )
+        axis.text(0.5, 0.5, message, transform=axis.transAxes, ha="center", va="center", color="#586774")
+        axis.set_axis_off()
+        return figure
+
+    # Cell brightness describes change volume; text retains direction and revisions separately.
+    cells = [[0 for _ in pairs] for _ in keys]
+    annotations: dict[tuple[int, int], str] = {}
+
+    for column, group in enumerate(differences):
+        for item in group:
+            row = keys.index(item.id)
+            learning, use = item.lessons, item.uses
+            size = sum(len(values) for change in (learning, use) for values in (change.added, change.removed, change.revised))
+            cells[row][column] = max(size, int(item.status != "retained"))
+            annotations[row, column] = (
+                f"L +{len(learning.added)} / -{len(learning.removed)} / ~{len(learning.revised)}\n"
+                f"U +{len(use.added)} / -{len(use.removed)} / ~{len(use.revised)}"
+                + (f"\n{item.status}" if item.status != "retained" else "")
+            )
+
+    maximum = max(max(row) for row in cells)
+    axis.imshow(cells, cmap="Blues", vmin=0, vmax=max(2, maximum * 2), aspect="auto")
+    axis.set_yticks(range(len(keys)), [names[key] for key in keys], fontsize=10)
+    axis.set_xticks(
+        range(len(pairs)),
+        [f"{after.project} @ {after.revision[:7]}\nvs {before.project} @ {before.revision[:7]}" for before, after in pairs],
+        fontsize=9,
+        rotation=20,
+        ha="right",
+    )
+    axis.tick_params(length=0)
+
+    for (row, column), text in annotations.items():
+        axis.text(column, row, text, ha="center", va="center", fontsize=9, color="#193448")
+
+    axis.set_xlabel("L = documented lessons; U = observed uses; + added / - removed / ~ revised. Counts are not mastery.", labelpad=14)
+    return figure
+
+
+def build(events: Sequence[Checkpoint], destination: Path, limit: int = 8) -> tuple[Path, ...]:
+    """
+    Export full comparisons and bounded plots ordered by recorded project visits.
+
+    Args:
+        events (Sequence[Checkpoint]): Validated complete checkpoint history.
+        destination (Path): Generated artifact directory.
+        limit (int): Maximum recent comparisons shown in each plot, from one through twenty.
+
+    Returns:
+        tuple[Path, ...]: Complete JSON export and two SVG/PNG views.
+
+    Raises:
+        ValueError: The plot limit is outside the documented range.
+    """
+    if not 1 <= limit <= 20:
+        raise ValueError("Checkpoint plot limit must be between one and twenty")
+
+    endpoints = visits(events)
+    adjacent = list(reversed(list(zip(endpoints[:-1], endpoints[1:], strict=True))))
+    latest = [(before, endpoints[-1]) for before in reversed(endpoints[:-1])] if endpoints else []
+    commits = list(zip(events[:-1], events[1:], strict=True))
+    destination.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "checkpoints": [as_data(event) for event in events],
+        "commit_diffs": [
+            {"before": before.id, "after": after.id, "skills": [as_data(item) for item in compare(before, after)]}
+            for before, after in commits
+        ],
+        "visit_endpoints": [event.id for event in endpoints],
+        "latest_vs_previous_visits": [
+            {"before": before.id, "after": after.id, "skills": [as_data(item) for item in compare(before, after)]}
+            for before, after in latest
+        ],
+    }
+    path = destination / "checkpoints.json"
+    replace_file(
+        path, (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode(), expected=path.read_bytes() if path.exists() else None
+    )
+    outputs = [path]
+
+    with mpl.rc_context(mpl.rcParamsDefault):
+        mpl.rcParams.update({"font.family": "DejaVu Sans", "svg.fonttype": "none", "svg.hashsalt": digest(payload)})
+
+        for name, pairs, title, question in (
+            ("checkpoint-transitions", adjacent, "Project chapters", "What changed between consecutive project visits?"),
+            (
+                "checkpoint-comparisons",
+                latest,
+                "Current chapter in context",
+                "How does the latest recorded state differ from earlier project visits?",
+            ),
+        ):
+            figure = _figure(pairs[:limit], title, question)
+            outputs.extend(save_figure(figure, destination, name))
+
+    return tuple(outputs)