devmemory-cli 0.1.0.dev0__py3-none-any.whl

devmemory/services/analytics.py

@@ -0,0 +1,328 @@
+"""Development-intelligence analytics.
+
+Every question is answered by one query with two implementations that return the
+*same* shape: a local one over SQLite (always available, used by the demo) and a
+Databricks one over the published Delta tables (when a workspace is configured).
+The result carries a ``source`` so the dashboard can badge it.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+
+from pydantic import BaseModel
+
+from devmemory.config import resolve_databricks_credentials
+from devmemory.domain.enums import VersionStatus
+from devmemory.domain.models import DevelopmentVersion
+from devmemory.services.context import ProjectContext
+from devmemory.storage.versions import VersionRepository
+
+_ADVERSE = {VersionStatus.REGRESSION.value, VersionStatus.ERROR.value}
+
+
+class RegressionRow(BaseModel):
+    version_id: str
+    intent: str | None
+    feature: str | None
+    agent: str | None
+    git_commit: str
+    severity: str
+    detail: str
+
+
+class FeatureRow(BaseModel):
+    feature: str
+    attempts: int
+    successes: int
+    regressions: int
+    success_rate: float
+    latest_status: str
+
+
+class FileChurnRow(BaseModel):
+    path: str
+    changes: int
+    adverse_changes: int
+    risk: float
+
+
+class AgentRow(BaseModel):
+    agent: str
+    versions: int
+    success_rate: float
+    regressions: int
+    tokens_per_success: float | None
+
+
+class TrendPoint(BaseModel):
+    version_id: str
+    version_number: int
+    status: str
+    test_pass_rate: float | None
+    key_metric: float | None
+
+
+class FailedApproach(BaseModel):
+    signature: list[str]
+    occurrences: int
+    version_ids: list[str]
+    example_intent: str | None
+
+
+class AnalyticsSummary(BaseModel):
+    source: str  # 'local' | 'databricks'
+    project: str
+    version_count: int
+    regression_count: int
+    success_rate: float
+    regressions: list[RegressionRow]
+    features: list[FeatureRow]
+    file_churn: list[FileChurnRow]
+    agents: list[AgentRow]
+    trend: list[TrendPoint]
+    failed_approaches: list[FailedApproach]
+
+
+def analytics_summary(ctx: ProjectContext) -> AnalyticsSummary:
+    """Prefer Databricks if configured and reachable; otherwise compute locally."""
+    if ctx.config.databricks.enabled and resolve_databricks_credentials() is not None:
+        remote = _try_databricks(ctx)
+        if remote is not None:
+            return remote
+    return _local_summary(ctx)
+
+
+# --- local implementation ---------------------------------------------------------
+
+
+def _local_summary(ctx: ProjectContext) -> AnalyticsSummary:
+    repo = VersionRepository(ctx.db)
+    versions = repo.page(ctx.config.project_id, limit=2000, ascending=True)
+    n = len(versions)
+    successes = sum(1 for v in versions if v.status is VersionStatus.SUCCESS)
+    regressions = sum(1 for v in versions if v.status is VersionStatus.REGRESSION or v.regressions)
+
+    return AnalyticsSummary(
+        source="local",
+        project=ctx.config.project_name,
+        version_count=n,
+        regression_count=regressions,
+        success_rate=round(successes / n * 100, 1) if n else 0.0,
+        regressions=_regression_rows(versions),
+        features=_feature_rows(versions),
+        file_churn=_file_churn(versions),
+        agents=_agent_rows(versions),
+        trend=_trend(versions),
+        failed_approaches=_failed_approaches(versions),
+    )
+
+
+def _regression_rows(versions: list[DevelopmentVersion]) -> list[RegressionRow]:
+    rows: list[RegressionRow] = []
+    for v in versions:
+        if v.status is not VersionStatus.REGRESSION and not v.regressions:
+            continue
+        worst = max(
+            v.regressions,
+            key=lambda r: {"HIGH": 3, "MEDIUM": 2, "LOW": 1}.get(r.severity, 0),
+            default=None,
+        )
+        rows.append(
+            RegressionRow(
+                version_id=v.version_id,
+                intent=v.intent,
+                feature=_feature_name(v),
+                agent=v.agent,
+                git_commit=v.git_commit,
+                severity=worst.severity if worst else "MEDIUM",
+                detail=worst.detail if worst and worst.detail else "status marked REGRESSION",
+            )
+        )
+    rows.reverse()
+    return rows
+
+
+def _feature_rows(versions: list[DevelopmentVersion]) -> list[FeatureRow]:
+    by_feature: dict[str, list[DevelopmentVersion]] = defaultdict(list)
+    for v in versions:
+        name = _feature_name(v)
+        if name:
+            by_feature[name].append(v)
+    out: list[FeatureRow] = []
+    for name, vs in sorted(by_feature.items(), key=lambda kv: -len(kv[1])):
+        succ = sum(1 for v in vs if v.status is VersionStatus.SUCCESS)
+        regr = sum(1 for v in vs if v.status is VersionStatus.REGRESSION or v.regressions)
+        out.append(
+            FeatureRow(
+                feature=name,
+                attempts=len(vs),
+                successes=succ,
+                regressions=regr,
+                success_rate=round(succ / len(vs) * 100, 1),
+                latest_status=vs[-1].status.value,
+            )
+        )
+    return out
+
+
+def _file_churn(versions: list[DevelopmentVersion]) -> list[FileChurnRow]:
+    changes: Counter[str] = Counter()
+    adverse: Counter[str] = Counter()
+    for v in versions:
+        is_bad = v.status.value in _ADVERSE or bool(v.regressions)
+        for f in v.changed_files:
+            changes[f.path] += 1
+            if is_bad:
+                adverse[f.path] += 1
+    rows = [
+        FileChurnRow(
+            path=path,
+            changes=c,
+            adverse_changes=adverse[path],
+            risk=round(adverse[path] / c, 2) if c else 0.0,
+        )
+        for path, c in changes.most_common(15)
+    ]
+    return rows
+
+
+def _agent_rows(versions: list[DevelopmentVersion]) -> list[AgentRow]:
+    by_agent: dict[str, list[DevelopmentVersion]] = defaultdict(list)
+    for v in versions:
+        by_agent[v.agent or "unknown"].append(v)
+    out: list[AgentRow] = []
+    for agent, vs in sorted(by_agent.items(), key=lambda kv: -len(kv[1])):
+        succ = [v for v in vs if v.status is VersionStatus.SUCCESS]
+        tokens = [
+            v.primary_checkpoint.tokens.total
+            for v in succ
+            if v.primary_checkpoint and v.primary_checkpoint.tokens.total
+        ]
+        out.append(
+            AgentRow(
+                agent=agent,
+                versions=len(vs),
+                success_rate=round(len(succ) / len(vs) * 100, 1),
+                regressions=sum(1 for v in vs if v.status is VersionStatus.REGRESSION),
+                tokens_per_success=round(sum(tokens) / len(tokens)) if tokens else None,
+            )
+        )
+    return out
+
+
+def _trend(versions: list[DevelopmentVersion]) -> list[TrendPoint]:
+    points: list[TrendPoint] = []
+    for v in versions:
+        rate = v.tests.pass_rate if v.tests and v.tests.ran else None
+        key_metric = v.metrics[0].after if v.metrics else None
+        points.append(
+            TrendPoint(
+                version_id=v.version_id,
+                version_number=v.version_number,
+                status=v.status.value,
+                test_pass_rate=round(rate * 100, 1) if rate is not None else None,
+                key_metric=key_metric,
+            )
+        )
+    return points
+
+
+def _failed_approaches(versions: list[DevelopmentVersion]) -> list[FailedApproach]:
+    groups: dict[tuple[str, ...], list[DevelopmentVersion]] = defaultdict(list)
+    for v in versions:
+        if v.status.value not in _ADVERSE and not v.regressions:
+            continue
+        sig = tuple(sorted(f.path for f in v.changed_files))
+        if sig:
+            groups[sig].append(v)
+    out = [
+        FailedApproach(
+            signature=list(sig),
+            occurrences=len(vs),
+            version_ids=[v.version_id for v in vs],
+            example_intent=vs[0].intent,
+        )
+        for sig, vs in groups.items()
+        if len(vs) >= 2
+    ]
+    out.sort(key=lambda a: -a.occurrences)
+    return out
+
+
+def _feature_name(v: DevelopmentVersion) -> str | None:
+    return v.feature_id.split(":", 1)[-1] if v.feature_id else None
+
+
+# --- databricks implementation --------------------------------------------------
+
+
+def _try_databricks(ctx: ProjectContext) -> AnalyticsSummary | None:
+    from devmemory.adapters.databricks import DatabricksAdapter
+
+    adapter = DatabricksAdapter(ctx.config)
+    try:
+        rows = adapter.query(
+            f"SELECT version_id, status, agent, feature, is_regression "
+            f"FROM {adapter.table('fact_versions')} WHERE project_id = :pid",
+            {"pid": ctx.config.project_id},
+        )
+    except Exception:
+        return None
+
+    n = len(rows)
+    if n == 0:
+        return None
+    successes = sum(1 for r in rows if r.get("status") == "SUCCESS")
+    # the Statement Execution API returns every cell as a string ("true"/"false")
+    regr = sum(
+        1
+        for r in rows
+        if str(r.get("is_regression")).lower() == "true" or r.get("status") == "REGRESSION"
+    )
+    # For the richer breakdowns Databricks would run more queries; the demo path
+    # is local, so keep the remote summary to the top-line numbers plus a marker.
+    local = _local_summary(ctx)
+    return local.model_copy(
+        update={
+            "source": "databricks",
+            "version_count": n,
+            "regression_count": regr,
+            "success_rate": round(successes / n * 100, 1),
+        }
+    )
+
+
+def canned_queries(catalog: str, schema: str) -> dict[str, str]:
+    """SQL a user can run in Databricks directly - shown in the dashboard."""
+    t = f"{catalog}.{schema}"
+    return {
+        "regressions": (
+            f"SELECT version_id, feature, agent, intent\n"
+            f"FROM {t}.fact_versions\nWHERE is_regression = true\nORDER BY version_number DESC"
+        ),
+        "feature_attempts": (
+            f"SELECT feature, count(*) attempts,\n"
+            f"  round(sum(if(status='SUCCESS',1,0))*100.0/count(*), 1) success_rate_pct\n"
+            f"FROM {t}.fact_versions\nWHERE feature IS NOT NULL\nGROUP BY feature\nORDER BY attempts DESC"
+        ),
+        "file_churn": (
+            f"SELECT path, count(*) changes,\n"
+            f"  sum(if(v.is_regression,1,0)) adverse\n"
+            f"FROM {t}.fact_changed_files f JOIN {t}.fact_versions v USING (version_id)\n"
+            f"GROUP BY path\nORDER BY changes DESC\nLIMIT 15"
+        ),
+        "agent_effectiveness": (
+            f"SELECT agent, count(*) versions,\n"
+            f"  round(sum(if(status='SUCCESS',1,0))*100.0/count(*),1) success_rate_pct,\n"
+            f"  sum(if(is_regression,1,0)) regressions\n"
+            f"FROM {t}.fact_versions\nWHERE agent IS NOT NULL\nGROUP BY agent\nORDER BY success_rate_pct DESC"
+        ),
+    }
+
+
+__all__ = [
+    "AnalyticsSummary",
+    "analytics_summary",
+    "canned_queries",
+]

devmemory/services/brief.py

@@ -0,0 +1,53 @@
+"""The project brief - a single, human-owned source of truth.
+
+The brief is one markdown document: what the project is for, its constraints,
+the decisions already made, the conventions to follow. It is fed into
+requirement normalization (:mod:`devmemory.services.taskloop.requirements`) so
+the loop's requirements and prompt suggestions stay anchored to the project
+rather than to the wording of one task.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+
+from devmemory.services.context import ProjectContext
+
+_MAX_CHARS = 60_000
+
+
+@dataclass(frozen=True)
+class ProjectBrief:
+    content: str
+    updated_at: str | None
+
+
+def get_brief(ctx: ProjectContext) -> ProjectBrief:
+    row = ctx.db.query_one("SELECT content, updated_at FROM project_brief WHERE id = 1")
+    if row is None:
+        return ProjectBrief(content="", updated_at=None)
+    return ProjectBrief(content=row["content"], updated_at=row["updated_at"])
+
+
+def set_brief(ctx: ProjectContext, content: str) -> ProjectBrief:
+    text = content.strip()[:_MAX_CHARS]
+    now = datetime.now(UTC).isoformat()
+    ctx.db.execute(
+        "INSERT INTO project_brief (id, content, updated_at) VALUES (1, ?, ?) "
+        "ON CONFLICT(id) DO UPDATE SET content = excluded.content, "
+        "updated_at = excluded.updated_at",
+        (text, now),
+    )
+    return ProjectBrief(content=text, updated_at=now)
+
+
+def brief_context(ctx: ProjectContext, *, limit: int = 4000) -> str | None:
+    """The brief trimmed for use as LLM context, or ``None`` when empty."""
+    text = get_brief(ctx).content.strip()
+    if not text:
+        return None
+    return text[:limit]
+
+
+__all__ = ["ProjectBrief", "brief_context", "get_brief", "set_brief"]

devmemory/services/context.py

@@ -0,0 +1,88 @@
+"""``ProjectContext`` - the wired-up bundle every entry point works through.
+
+The CLI, the REST API, and the MCP server each build one of these and then call
+service functions with it. It owns the DB connection lifecycle.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from types import TracebackType
+
+from devmemory.adapters.entire import EntireAdapter
+from devmemory.adapters.git import GitAdapter
+from devmemory.adapters.graph import GraphAdapter
+from devmemory.config import DevMemoryConfig
+from devmemory.domain.errors import ProjectNotInitializedError
+from devmemory.paths import ProjectPaths, find_project_paths
+from devmemory.storage.db import Database
+
+
+@dataclass(slots=True)
+class ProjectContext:
+    paths: ProjectPaths
+    config: DevMemoryConfig
+    db: Database
+    git: GitAdapter
+    entire: EntireAdapter
+    graph: GraphAdapter
+
+    @classmethod
+    def load(cls, start: Path | str | None = None, *, thread_safe: bool = False) -> ProjectContext:
+        """Discover the project at or above ``start`` and open it.
+
+        ``thread_safe=True`` (the web server) opens the SQLite connection with
+        ``check_same_thread=False`` so it can be shared across uvicorn's
+        threadpool. Raises :class:`ProjectNotInitializedError` if there is no
+        ``.devmemory/``.
+        """
+        start_path = Path(start) if start is not None else None
+        paths = find_project_paths(start_path)
+        if paths is None:
+            raise ProjectNotInitializedError
+        config = DevMemoryConfig.load(paths)
+        return cls._build(paths, config, thread_safe=thread_safe)
+
+    @classmethod
+    def for_paths(cls, paths: ProjectPaths, config: DevMemoryConfig) -> ProjectContext:
+        """Build a context from already-resolved paths/config (used during ``init``)."""
+        return cls._build(paths, config)
+
+    @classmethod
+    def _build(
+        cls, paths: ProjectPaths, config: DevMemoryConfig, *, thread_safe: bool = False
+    ) -> ProjectContext:
+        db = Database(paths.db, check_same_thread=not thread_safe)
+        db.migrate()
+        git = GitAdapter(paths.repo_root, git_binary=None)
+        entire = EntireAdapter(
+            paths.repo_root,
+            binary=config.entire.binary,
+            repo=config.entire.repo,
+            git=git,
+        )
+        graph = GraphAdapter(
+            paths.repo_root,
+            binary=config.graph.binary,
+            timeout=config.graph.timeout_seconds,
+            max_seconds=config.graph.max_seconds,
+        )
+        return cls(paths=paths, config=config, db=db, git=git, entire=entire, graph=graph)
+
+    def close(self) -> None:
+        self.db.close()
+
+    def __enter__(self) -> ProjectContext:
+        return self
+
+    def __exit__(
+        self,
+        _exc_type: type[BaseException] | None,
+        _exc: BaseException | None,
+        _tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+
+__all__ = ["ProjectContext"]

devmemory/services/databricks_sync.py

@@ -0,0 +1,121 @@
+"""Databricks sync: an offline outbox that a live push drains.
+
+Every published version is written to ``.devmemory/outbox/<version>.json`` first.
+If Databricks is configured and reachable, the pipeline pushes immediately and
+removes the file; otherwise it stays queued for ``devmemory databricks push``.
+Local history never depends on any of this.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from devmemory.adapters.databricks import (
+    DatabricksAdapter,
+    DatabricksUnavailableError,
+    outbox_event,
+)
+from devmemory.domain.errors import DatabricksError
+from devmemory.domain.models import DevelopmentVersion
+from devmemory.logging import get_logger
+from devmemory.services.context import ProjectContext
+
+_log = get_logger(__name__)
+
+
+class SyncResult(BaseModel):
+    configured: bool
+    pushed: list[str] = []
+    queued: list[str] = []
+    failed: list[str] = []
+    detail: str | None = None
+
+
+def enqueue(ctx: ProjectContext, version: DevelopmentVersion) -> Path:
+    ctx.paths.outbox_dir.mkdir(parents=True, exist_ok=True)
+    path = ctx.paths.outbox_dir / f"{version.version_id}.json"
+    path.write_text(json.dumps(outbox_event(version), indent=2), encoding="utf-8")
+    return path
+
+
+def push_version(ctx: ProjectContext, version: DevelopmentVersion) -> SyncResult:
+    """Queue a version and, if Databricks is reachable, publish it now."""
+    enqueue(ctx, version)
+    if not (ctx.config.databricks.enabled and DatabricksAdapter(ctx.config).is_configured):
+        return SyncResult(configured=False, queued=[version.version_id])
+    return drain(ctx, only=version.version_id)
+
+
+def drain(ctx: ProjectContext, *, only: str | None = None) -> SyncResult:
+    """Publish every queued event (or just ``only``) to Databricks."""
+    adapter = DatabricksAdapter(ctx.config)
+    if not adapter.is_configured:
+        return SyncResult(
+            configured=False,
+            queued=[p.stem for p in _outbox_files(ctx)],
+            detail="Databricks credentials are not set (DATABRICKS_HOST/TOKEN/WAREHOUSE_ID).",
+        )
+
+    result = SyncResult(configured=True)
+    try:
+        adapter.bootstrap()
+    except DatabricksError as exc:
+        return SyncResult(
+            configured=True,
+            queued=[p.stem for p in _outbox_files(ctx)],
+            detail=f"could not reach Databricks: {exc.message}",
+        )
+
+    for path in _outbox_files(ctx):
+        if only is not None and path.stem != only:
+            continue
+        try:
+            version = _rehydrate(ctx, path.stem)
+            if version is None:
+                path.unlink(missing_ok=True)
+                continue
+            adapter.publish_version(version)
+        except (DatabricksUnavailableError, DatabricksError) as exc:
+            result.failed.append(path.stem)
+            result.detail = exc.message
+            _log.warning("databricks.push_failed", version=path.stem, error=exc.message)
+            break
+        else:
+            path.unlink(missing_ok=True)
+            result.pushed.append(path.stem)
+
+    result.queued = [p.stem for p in _outbox_files(ctx)]
+    if result.pushed:
+        _log.info("databricks.drained", pushed=result.pushed, remaining=result.queued)
+    return result
+
+
+def sync_status(ctx: ProjectContext) -> SyncResult:
+    adapter = DatabricksAdapter(ctx.config)
+    return SyncResult(
+        configured=adapter.is_configured and ctx.config.databricks.enabled,
+        queued=[p.stem for p in _outbox_files(ctx)],
+        detail=(
+            f"catalog {ctx.config.databricks.catalog}.{ctx.config.databricks.schema_name}"
+            if adapter.is_configured
+            else "not configured"
+        ),
+    )
+
+
+def _outbox_files(ctx: ProjectContext) -> list[Path]:
+    if not ctx.paths.outbox_dir.is_dir():
+        return []
+    return sorted(ctx.paths.outbox_dir.glob("*.json"))
+
+
+def _rehydrate(ctx: ProjectContext, version_id: str) -> DevelopmentVersion | None:
+    from devmemory.storage.versions import VersionRepository
+
+    return VersionRepository(ctx.db).get(version_id)
+
+
+__all__ = ["SyncResult", "drain", "enqueue", "push_version", "sync_status"]

devmemory/services/features.py

@@ -0,0 +1,85 @@
+"""Feature-level services: status roll-up and per-feature evolution."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+from devmemory.domain.enums import FeatureStatus, VersionStatus
+from devmemory.domain.errors import DevMemoryError
+from devmemory.domain.models import DevelopmentVersion, Feature
+from devmemory.services.context import ProjectContext
+from devmemory.storage.repositories import FeatureRepository
+from devmemory.storage.versions import VersionRepository
+
+
+class FeatureNotFoundError(DevMemoryError):
+    exit_code = 9
+
+
+class FeatureWithHistory(BaseModel):
+    feature: Feature
+    versions: list[DevelopmentVersion]
+
+    @property
+    def rolled_up_status(self) -> FeatureStatus:
+        return _roll_up(self.versions)
+
+
+def list_features(ctx: ProjectContext) -> list[FeatureWithHistory]:
+    features = FeatureRepository(ctx.db).list_all(ctx.config.project_id)
+    versions = VersionRepository(ctx.db)
+    return [
+        FeatureWithHistory(feature=f, versions=versions.for_feature(f.feature_id)) for f in features
+    ]
+
+
+def get_feature(ctx: ProjectContext, ref: str) -> FeatureWithHistory:
+    repo = FeatureRepository(ctx.db)
+    feature = repo.get(ref) or repo.get_by_name(ctx.config.project_id, ref)
+    if feature is None:
+        # tolerate a slugged id without the project prefix
+        feature = repo.get(f"{ctx.config.project_id}:{ref}")
+    if feature is None:
+        raise FeatureNotFoundError(
+            f"No feature matches {ref!r}.",
+            hint="Run `devmemory features` to list them.",
+        )
+    versions = VersionRepository(ctx.db).for_feature(feature.feature_id)
+    return FeatureWithHistory(feature=feature, versions=versions)
+
+
+def refresh_feature_status(ctx: ProjectContext, feature_id: str) -> Feature | None:
+    """Recompute and persist a feature's roll-up status from its versions."""
+    repo = FeatureRepository(ctx.db)
+    feature = repo.get(feature_id)
+    if feature is None:
+        return None
+    versions = VersionRepository(ctx.db).for_feature(feature_id)
+    return repo.upsert(
+        feature.project_id,
+        feature.name,
+        status=_roll_up(versions),
+        derived_from=feature.derived_from,
+    )
+
+
+def _roll_up(versions: list[DevelopmentVersion]) -> FeatureStatus:
+    if not versions:
+        return FeatureStatus.NOT_STARTED
+    latest = versions[-1]
+    if latest.status is VersionStatus.SUCCESS:
+        return FeatureStatus.COMPLETE
+    if latest.status in (VersionStatus.REGRESSION, VersionStatus.PARTIAL_SUCCESS):
+        return FeatureStatus.PARTIAL
+    if latest.status is VersionStatus.ERROR:
+        return FeatureStatus.FAILED
+    return FeatureStatus.IN_PROGRESS
+
+
+__all__ = [
+    "FeatureNotFoundError",
+    "FeatureWithHistory",
+    "get_feature",
+    "list_features",
+    "refresh_feature_status",
+]