devmemory-cli 0.1.0.dev0__py3-none-any.whl

devmemory/services/versions.py

@@ -0,0 +1,287 @@
+"""Development-version services: create from an event, read, list, diff, search."""
+
+from __future__ import annotations
+
+import json
+import uuid
+from datetime import UTC, datetime
+
+from pydantic import BaseModel
+
+from devmemory.domain.enums import AssociationMethod, FeatureStatus, VersionStatus
+from devmemory.domain.errors import DevMemoryError
+from devmemory.domain.models import (
+    ChangedFile,
+    DevelopmentEvent,
+    DevelopmentVersion,
+    DiffStat,
+    Regression,
+)
+from devmemory.logging import get_logger
+from devmemory.services.context import ProjectContext
+from devmemory.storage.repositories import FeatureRepository
+from devmemory.storage.versions import VersionRepository
+
+_log = get_logger(__name__)
+
+
+class VersionExistsError(DevMemoryError):
+    """A version already exists for this commit."""
+
+    exit_code = 8
+
+    def __init__(self, version: DevelopmentVersion) -> None:
+        super().__init__(
+            f"{version.version_id} already records commit {version.git_commit[:12]}.",
+            hint="Nothing to do. Use `--force` to supersede it (a later phase).",
+        )
+        self.version = version
+
+
+class VersionNotFoundError(DevMemoryError):
+    exit_code = 9
+
+
+class VersionDiff(BaseModel):
+    from_version_id: str
+    to_version_id: str
+    from_commit: str
+    to_commit: str
+    from_number: int
+    to_number: int
+    files: list[ChangedFile]
+    stat: DiffStat
+    diff_text: str
+    metric_changes: dict[str, dict[str, float | None]]
+    test_changes: dict[str, int | None]
+    status_from: VersionStatus
+    status_to: VersionStatus
+    feature_from: str | None = None
+    feature_to: str | None = None
+    checkpoint_from: str | None = None
+    checkpoint_to: str | None = None
+
+
+def create_version_from_event(
+    ctx: ProjectContext,
+    event: DevelopmentEvent,
+    *,
+    force: bool = False,
+    regressions: list[Regression] | None = None,
+) -> DevelopmentVersion:
+    """Persist a normalized event as a new :class:`DevelopmentVersion`.
+
+    The checkpoint pipeline assembles ``event`` (status, tests, metrics) and the
+    detected ``regressions`` before calling in; the analysis layer is applied
+    afterwards via :func:`~devmemory.services.versions.set_analysis`.
+    """
+    repo = VersionRepository(ctx.db)
+
+    existing = repo.find_by_commit(event.project_id, event.commit.sha)
+    if existing is not None and not force:
+        raise VersionExistsError(existing)
+
+    if existing is not None:
+        number = existing.version_number
+        version_id = existing.version_id
+    else:
+        number = repo.next_version_number(event.project_id)
+        version_id = f"v{number}"
+
+    feature_id = _resolve_feature(ctx, event)
+
+    checkpoint = event.checkpoint
+    version = DevelopmentVersion(
+        version_id=version_id,
+        version_number=number,
+        project_id=event.project_id,
+        intent=event.intent or (checkpoint.intent if checkpoint else None) or event.commit.subject,
+        agent=event.agent or (checkpoint.agent if checkpoint else None),
+        model=event.model or (checkpoint.model if checkpoint else None),
+        git_commit=event.commit.sha,
+        parent_commit=event.parent_commit or event.commit.parent,
+        branch=event.branch,
+        feature_id=feature_id,
+        status=event.status or VersionStatus.NEEDS_REVIEW,
+        files_changed=len(event.changed_files),
+        lines_added=event.diff_stat.additions,
+        lines_removed=event.diff_stat.deletions,
+        changed_files=event.changed_files,
+        primary_checkpoint=checkpoint,
+        checkpoint_ids=[checkpoint.checkpoint_id] if checkpoint else [],
+        entire_association_method=(
+            checkpoint.association_method if checkpoint else AssociationMethod.NONE
+        ),
+        entire_association_confidence=(checkpoint.association_confidence if checkpoint else 0.0),
+        tests=event.tests,
+        metrics=event.metrics,
+        regressions=[r.model_copy(update={"version_id": version_id}) for r in (regressions or [])],
+        environment=event.environment,
+        run_id=event.run_id,
+        created_at=datetime.now(UTC),
+        committed_at=event.commit.committed_at,
+    )
+
+    replaced = existing is not None
+    stored = (
+        repo.replace(version, source_event=event)
+        if replaced
+        else repo.create(version, source_event=event)
+    )
+
+    _record_event(
+        ctx,
+        version_id=stored.version_id,
+        event_type="version.replaced" if replaced else "version.created",
+        payload={
+            "commit": stored.git_commit,
+            "status": stored.status.value,
+            "checkpoint": stored.primary_checkpoint.checkpoint_id
+            if stored.primary_checkpoint
+            else None,
+            "regressions": len(stored.regressions),
+        },
+    )
+    _log.info(
+        "version.replaced" if replaced else "version.created",
+        version=stored.version_id,
+        commit=stored.git_commit[:12],
+        status=stored.status.value,
+        checkpoint=stored.primary_checkpoint.checkpoint_id if stored.primary_checkpoint else None,
+    )
+    return stored
+
+
+def get_version(ctx: ProjectContext, ref: str) -> DevelopmentVersion:
+    repo = VersionRepository(ctx.db)
+    project = _project_id(ctx)
+    version = repo.resolve(project, ref)
+    if version is None:
+        raise VersionNotFoundError(
+            f"No version matches {ref!r}.",
+            hint="Run `devmemory history` to list versions.",
+        )
+    return version
+
+
+def list_versions(
+    ctx: ProjectContext, *, limit: int = 100, offset: int = 0, ascending: bool = True
+) -> list[DevelopmentVersion]:
+    return VersionRepository(ctx.db).page(
+        _project_id(ctx), limit=limit, offset=offset, ascending=ascending
+    )
+
+
+def version_diff(ctx: ProjectContext, from_ref: str, to_ref: str) -> VersionDiff:
+    a = get_version(ctx, from_ref)
+    b = get_version(ctx, to_ref)
+
+    files = ctx.git.changed_files(a.git_commit, b.git_commit)
+    diff_text = ctx.git.diff_text(a.git_commit, b.git_commit)
+
+    metrics_a = {m.name: m for m in a.metrics}
+    metrics_b = {m.name: m for m in b.metrics}
+    metric_changes: dict[str, dict[str, float | None]] = {}
+    for name in sorted(set(metrics_a) | set(metrics_b)):
+        before = metrics_a[name].after if name in metrics_a else None
+        after = metrics_b[name].after if name in metrics_b else None
+        metric_changes[name] = {
+            "before": before,
+            "after": after,
+            "delta": (after - before) if before is not None and after is not None else None,
+        }
+
+    return VersionDiff(
+        from_version_id=a.version_id,
+        to_version_id=b.version_id,
+        from_commit=a.git_commit,
+        to_commit=b.git_commit,
+        from_number=a.version_number,
+        to_number=b.version_number,
+        files=files,
+        stat=DiffStat.from_files(files),
+        diff_text=diff_text,
+        metric_changes=metric_changes,
+        test_changes={
+            "passed": _delta(
+                a.tests.passed if a.tests else None, b.tests.passed if b.tests else None
+            ),
+            "failed": _delta(
+                a.tests.failed if a.tests else None, b.tests.failed if b.tests else None
+            ),
+        },
+        status_from=a.status,
+        status_to=b.status,
+        feature_from=a.feature_id.split(":", 1)[-1] if a.feature_id else None,
+        feature_to=b.feature_id.split(":", 1)[-1] if b.feature_id else None,
+        checkpoint_from=a.primary_checkpoint.checkpoint_id if a.primary_checkpoint else None,
+        checkpoint_to=b.primary_checkpoint.checkpoint_id if b.primary_checkpoint else None,
+    )
+
+
+def search_versions(
+    ctx: ProjectContext, query: str, *, limit: int = 50
+) -> list[DevelopmentVersion]:
+    repo = VersionRepository(ctx.db)
+    project = _project_id(ctx)
+    ids = repo.search_ids(project, query, limit=limit)
+    return [v for vid in ids if (v := repo.get(vid)) is not None]
+
+
+# --- helpers -----------------------------------------------------------------------
+
+
+def _resolve_feature(ctx: ProjectContext, event: DevelopmentEvent) -> str | None:
+    if not event.feature:
+        return None
+    feature = FeatureRepository(ctx.db).upsert(
+        event.project_id,
+        event.feature,
+        status=FeatureStatus.IN_PROGRESS,
+        derived_from=event.feature_derived_from,
+    )
+    return feature.feature_id
+
+
+def _record_event(
+    ctx: ProjectContext,
+    *,
+    version_id: str | None,
+    event_type: str,
+    payload: dict[str, object],
+) -> None:
+    with ctx.db.transaction() as conn:
+        conn.execute(
+            "INSERT INTO events (event_id, project_id, version_id, type, source, payload_json, "
+            "created_at) VALUES (?, ?, ?, ?, 'devmemory', ?, ?)",
+            (
+                uuid.uuid4().hex,
+                _project_id(ctx),
+                version_id,
+                event_type,
+                json.dumps(payload),
+                datetime.now(UTC).isoformat(),
+            ),
+        )
+
+
+def _project_id(ctx: ProjectContext) -> str:
+    return ctx.config.project_id
+
+
+def _delta(a: int | None, b: int | None) -> int | None:
+    if a is None or b is None:
+        return None
+    return b - a
+
+
+__all__ = [
+    "VersionDiff",
+    "VersionExistsError",
+    "VersionNotFoundError",
+    "create_version_from_event",
+    "get_version",
+    "list_versions",
+    "search_versions",
+    "version_diff",
+]

devmemory/storage/__init__.py

@@ -0,0 +1,9 @@
+"""Local persistence: a single SQLite database under ``.devmemory/metadata.db``.
+
+Only this package imports ``sqlite3``. Services talk to repositories; repositories
+talk to :class:`~devmemory.storage.db.Database`.
+"""
+
+from devmemory.storage.db import Database
+
+__all__ = ["Database"]

devmemory/storage/artifacts.py

@@ -0,0 +1,113 @@
+"""Project snapshots: a compressed archive of the source tree at a version's commit.
+
+Built from ``git archive`` (so it is exactly the committed state - no working-tree
+noise), then repacked through Python's ``tarfile`` to apply the configured
+exclusions and record a content hash. Git remains the authoritative history; the
+snapshot is a reproducibility convenience.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import tarfile
+from datetime import UTC, datetime
+from pathlib import Path
+
+from devmemory.adapters.git import GitAdapter
+from devmemory.domain.errors import StorageError
+from devmemory.domain.models import Artifact
+from devmemory.logging import get_logger
+
+_log = get_logger(__name__)
+
+_DEFAULT_EXCLUDE = (".git", ".devmemory", ".venv", "venv", "node_modules", "__pycache__")
+
+
+class ArtifactStore:
+    def __init__(self, artifacts_dir: Path, git: GitAdapter) -> None:
+        self._dir = artifacts_dir
+        self._git = git
+
+    def create_snapshot(
+        self,
+        *,
+        version_id: str,
+        commit_sha: str,
+        exclude: list[str] | None = None,
+    ) -> Artifact:
+        self._dir.mkdir(parents=True, exist_ok=True)
+        patterns = tuple(exclude) if exclude is not None else _DEFAULT_EXCLUDE
+        raw = self._git.archive_tar(commit_sha)
+
+        out_path = self._dir / f"{version_id}_{commit_sha[:7]}.tar.gz"
+        digest = hashlib.sha256()
+        kept = 0
+        try:
+            with (
+                tarfile.open(fileobj=io.BytesIO(raw), mode="r:") as src,
+                tarfile.open(out_path, mode="w:gz") as dst,
+            ):
+                for member in src.getmembers():
+                    if _excluded(member.name, patterns):
+                        continue
+                    extracted = src.extractfile(member) if member.isfile() else None
+                    data = extracted.read() if extracted is not None else b""
+                    if member.isfile():
+                        digest.update(member.name.encode())
+                        digest.update(data)
+                    dst.addfile(member, io.BytesIO(data) if member.isfile() else None)
+                    kept += 1
+        except (tarfile.TarError, OSError) as exc:
+            raise StorageError(f"could not build snapshot for {version_id}: {exc}") from exc
+
+        size = out_path.stat().st_size
+        _log.info(
+            "artifact.created", version=version_id, path=str(out_path), files=kept, bytes=size
+        )
+        return Artifact(
+            artifact_id=_artifact_id(version_id, commit_sha),
+            version_id=version_id,
+            path=str(out_path.relative_to(self._dir.parent.parent))
+            if _is_relative(out_path, self._dir.parent.parent)
+            else str(out_path),
+            type="project_snapshot",
+            size_bytes=size,
+            sha256=digest.hexdigest(),
+            created_at=datetime.now(UTC),
+        )
+
+    def extract(self, artifact: Artifact, dest: Path) -> None:
+        archive = self._resolve(artifact)
+        if not archive.is_file():
+            raise StorageError(f"artifact archive missing: {archive}")
+        dest.mkdir(parents=True, exist_ok=True)
+        try:
+            with tarfile.open(archive, mode="r:gz") as tf:
+                tf.extractall(dest, filter="data")
+        except (tarfile.TarError, OSError) as exc:
+            raise StorageError(f"could not extract {archive}: {exc}") from exc
+
+    def _resolve(self, artifact: Artifact) -> Path:
+        p = Path(artifact.path)
+        return p if p.is_absolute() else self._dir.parent.parent / p
+
+
+def _excluded(name: str, patterns: tuple[str, ...]) -> bool:
+    parts = name.replace("\\", "/").split("/")
+    return any(p in parts for p in patterns)
+
+
+def _artifact_id(version_id: str, commit_sha: str) -> str:
+    return hashlib.sha256(f"{version_id}:{commit_sha}".encode()).hexdigest()[:16]
+
+
+def _is_relative(path: Path, base: Path) -> bool:
+    try:
+        path.relative_to(base)
+    except ValueError:
+        return False
+    return True
+
+
+__all__ = ["ArtifactStore"]

devmemory/storage/db.py

@@ -0,0 +1,205 @@
+"""SQLite connection management and a forward-only schema migration runner.
+
+Migrations are plain ``.sql`` files in ``migrations/`` named ``NNNN_slug.sql``.
+They are applied in numeric order inside a transaction, and each is recorded in
+``schema_migrations`` so re-running ``migrate()`` is a no-op.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import re
+import sqlite3
+import threading
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from devmemory.domain.errors import MigrationError
+from devmemory.logging import get_logger
+
+_log = get_logger(__name__)
+
+MIGRATIONS_DIR = Path(__file__).parent / "migrations"
+_MIGRATION_RE = re.compile(r"^(\d{4})_([a-z0-9_]+)\.sql$")
+
+
+@dataclass(frozen=True, slots=True)
+class Migration:
+    version: int
+    name: str
+    path: Path
+
+    @property
+    def sql(self) -> str:
+        return self.path.read_text(encoding="utf-8")
+
+
+def discover_migrations(directory: Path = MIGRATIONS_DIR) -> list[Migration]:
+    """Return every migration file, ordered by version, validating the sequence."""
+    migrations: list[Migration] = []
+    for path in sorted(directory.glob("*.sql")):
+        match = _MIGRATION_RE.match(path.name)
+        if match is None:
+            raise MigrationError(
+                f"Migration file {path.name!r} does not match NNNN_slug.sql.",
+            )
+        migrations.append(Migration(int(match.group(1)), match.group(2), path))
+
+    for expected, migration in enumerate(migrations, start=1):
+        if migration.version != expected:
+            raise MigrationError(
+                f"Migration numbering gap: expected {expected:04d}, found "
+                f"{migration.version:04d} ({migration.name}).",
+            )
+    return migrations
+
+
+class Database:
+    """Access to a project's metadata database.
+
+    A single connection, shared. Every read and write goes through the helpers
+    below, each guarded by a re-entrant lock, so the web server (uvicorn's
+    threadpool) and the CLI both use it safely. ``check_same_thread=False`` is set
+    for the server; access is still fully serialized by ``_lock``.
+    """
+
+    def __init__(
+        self,
+        path: Path,
+        *,
+        migrations_dir: Path = MIGRATIONS_DIR,
+        check_same_thread: bool = True,
+    ) -> None:
+        self.path = path
+        self._migrations_dir = migrations_dir
+        self._check_same_thread = check_same_thread
+        self._conn: sqlite3.Connection | None = None
+        self._lock = threading.RLock()
+
+    # -- connection ------------------------------------------------------------
+
+    @property
+    def connection(self) -> sqlite3.Connection:
+        if self._conn is None:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            conn = sqlite3.connect(
+                self.path,
+                isolation_level=None,  # autocommit; transactions are explicit
+                timeout=30.0,
+                check_same_thread=self._check_same_thread,
+            )
+            conn.row_factory = sqlite3.Row
+            conn.execute("PRAGMA journal_mode = WAL")
+            conn.execute("PRAGMA foreign_keys = ON")
+            conn.execute("PRAGMA busy_timeout = 30000")
+            self._conn = conn
+        return self._conn
+
+    def close(self) -> None:
+        with self._lock:
+            if self._conn is not None:
+                with contextlib.suppress(sqlite3.Error):
+                    self._conn.close()
+                self._conn = None
+
+    def __enter__(self) -> Database:
+        return self
+
+    def __exit__(self, *_exc: object) -> None:
+        self.close()
+
+    # -- guarded access -----------------------------------------------------
+
+    def query(self, sql: str, params: tuple[object, ...] = ()) -> list[sqlite3.Row]:
+        with self._lock:
+            rows: list[sqlite3.Row] = self.connection.execute(sql, params).fetchall()
+            return rows
+
+    def query_one(self, sql: str, params: tuple[object, ...] = ()) -> sqlite3.Row | None:
+        with self._lock:
+            row: sqlite3.Row | None = self.connection.execute(sql, params).fetchone()
+            return row
+
+    def execute(self, sql: str, params: tuple[object, ...] = ()) -> None:
+        with self._lock:
+            self.connection.execute(sql, params)
+
+    @contextmanager
+    def transaction(self) -> Iterator[sqlite3.Connection]:
+        """Run a block inside ``BEGIN``/``COMMIT``, rolling back on any exception."""
+        with self._lock:
+            conn = self.connection
+            conn.execute("BEGIN")
+            try:
+                yield conn
+            except BaseException:
+                conn.execute("ROLLBACK")
+                raise
+            else:
+                conn.execute("COMMIT")
+
+    # -- migrations ----------------------------------------------------------
+
+    def _ensure_migrations_table(self) -> None:
+        self.execute(
+            """
+            CREATE TABLE IF NOT EXISTS schema_migrations (
+                version    INTEGER PRIMARY KEY,
+                name       TEXT NOT NULL,
+                applied_at TEXT NOT NULL
+            )
+            """
+        )
+
+    def applied_versions(self) -> set[int]:
+        self._ensure_migrations_table()
+        return {int(row["version"]) for row in self.query("SELECT version FROM schema_migrations")}
+
+    def migrate(self) -> list[Migration]:
+        """Apply every pending migration atomically. Returns the ones applied, in order.
+
+        Each migration's DDL and its ``schema_migrations`` row are committed together
+        inside a single ``BEGIN``/``COMMIT`` so a failure leaves no partial schema.
+        ``executescript`` is used (not per-statement ``execute``) so multi-statement
+        DDL - triggers, ``CREATE VIRTUAL TABLE``, FTS shadow tables - works.
+        """
+        with self._lock:
+            self._ensure_migrations_table()
+            applied = self.applied_versions()
+            pending = [
+                m for m in discover_migrations(self._migrations_dir) if m.version not in applied
+            ]
+            conn = self.connection
+            return self._apply_pending(conn, pending)
+
+    def _apply_pending(self, conn: sqlite3.Connection, pending: list[Migration]) -> list[Migration]:
+        for migration in pending:
+            _log.info("migration.apply", version=migration.version, name=migration.name)
+            applied_at = datetime.now(UTC).isoformat()
+            script = (
+                "BEGIN;\n"
+                f"{migration.sql.strip().rstrip(';')};\n"
+                "INSERT INTO schema_migrations (version, name, applied_at) "
+                f"VALUES ({migration.version}, '{migration.name}', '{applied_at}');\n"
+                "COMMIT;\n"
+            )
+            try:
+                conn.executescript(script)
+            except sqlite3.Error as exc:
+                with contextlib.suppress(sqlite3.Error):
+                    conn.executescript("ROLLBACK;")
+                raise MigrationError(
+                    f"Migration {migration.version:04d}_{migration.name} failed: {exc}",
+                ) from exc
+        return pending
+
+    def schema_version(self) -> int:
+        """Highest applied migration version, or 0 if none."""
+        versions = self.applied_versions()
+        return max(versions) if versions else 0
+
+
+__all__ = ["Database", "Migration", "discover_migrations"]

devmemory/storage/graph_impacts.py

@@ -0,0 +1,63 @@
+"""Persistence for graph change-impact results (Phase 12).
+
+Kept out of the main version transaction: impact analysis is optional, slow, and
+often backfilled after the fact.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+
+from devmemory.adapters.graph import GraphImpact
+from devmemory.storage.db import Database
+
+
+class GraphImpactRepository:
+    def __init__(self, db: Database) -> None:
+        self._db = db
+
+    def set(self, version_id: str, impact: GraphImpact) -> None:
+        generated = impact.generated_at or datetime.now(UTC).isoformat()
+        stored = impact.model_copy(update={"version_id": version_id, "generated_at": generated})
+        self._db.execute(
+            """
+            INSERT INTO graph_impacts
+                (version_id, base_commit, head_commit, entity_count, max_dependents,
+                 generated_at, payload)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(version_id) DO UPDATE SET
+                base_commit    = excluded.base_commit,
+                head_commit    = excluded.head_commit,
+                entity_count   = excluded.entity_count,
+                max_dependents = excluded.max_dependents,
+                generated_at   = excluded.generated_at,
+                payload        = excluded.payload
+            """,
+            (
+                version_id,
+                stored.base_commit,
+                stored.head_commit,
+                stored.entity_count,
+                stored.max_dependents,
+                generated,
+                stored.model_dump_json(),
+            ),
+        )
+
+    def get(self, version_id: str) -> GraphImpact | None:
+        row = self._db.query_one(
+            "SELECT payload FROM graph_impacts WHERE version_id = ?", (version_id,)
+        )
+        if row is None:
+            return None
+        try:
+            return GraphImpact.model_validate(json.loads(row["payload"]))
+        except (json.JSONDecodeError, ValueError):
+            return None
+
+    def delete(self, version_id: str) -> None:
+        self._db.execute("DELETE FROM graph_impacts WHERE version_id = ?", (version_id,))
+
+
+__all__ = ["GraphImpactRepository"]

devmemory/storage/migrations/0001_init.sql

@@ -0,0 +1,15 @@
+-- 0001_init: the project record.
+--
+-- One row per repository DevMemory tracks. The rich schema (versions,
+-- changed_files, tests, metrics, features, regressions, entire_checkpoints,
+-- analysis, artifacts, FTS) arrives in a later migration alongside the
+-- persistence layer that fills it.
+
+CREATE TABLE projects (
+    project_id          TEXT PRIMARY KEY,
+    name                TEXT NOT NULL,
+    repo_path           TEXT NOT NULL,
+    created_at          TEXT NOT NULL,
+    updated_at          TEXT NOT NULL,
+    current_version_id  INTEGER
+);