devmemory-cli 0.1.0.dev0__py3-none-any.whl

devmemory/adapters/metrics.py

@@ -0,0 +1,150 @@
+"""Metrics adapter: collect arbitrary project metrics for a version.
+
+Metrics come from a JSON file the project writes, or the JSON stdout of a
+command. Two shapes are accepted per metric:
+
+    {"accuracy": 93.4}                       -> after only
+    {"accuracy": {"before": 89.2, "after": 93.4, "unit": "%"}}
+
+The previous version's ``after`` fills any missing ``before`` (done by the
+pipeline, not here). Direction comes from config, then a name heuristic.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+from devmemory.domain.enums import MetricDirection
+from devmemory.domain.errors import CollectionError
+from devmemory.domain.models import Metric
+from devmemory.logging import get_logger
+
+_log = get_logger(__name__)
+
+_LOWER_IS_BETTER = (
+    "latency",
+    "duration",
+    "time_ms",
+    "response_time",
+    "loss",
+    "val_loss",
+    "train_loss",
+    "error_rate",
+    "errors",
+    "failures",
+    "cost",
+    "memory",
+    "mem_mb",
+    "p50",
+    "p90",
+    "p95",
+    "p99",
+    "size_bytes",
+    "bundle_size",
+)
+
+
+class MetricsAdapter:
+    def __init__(self, repo_path: Path | str) -> None:
+        self._cwd = Path(repo_path).resolve()
+
+    def collect(
+        self,
+        *,
+        file: str | None = None,
+        command: str | None = None,
+        directions: dict[str, MetricDirection] | None = None,
+    ) -> list[Metric]:
+        raw = self._load(file=file, command=command)
+        if not raw:
+            return []
+        directions = directions or {}
+        metrics: list[Metric] = []
+        for name, value in raw.items():
+            metric = _to_metric(str(name), value)
+            if metric is None:
+                continue
+            metric.direction = directions.get(metric.name, guess_direction(metric.name))
+            metrics.append(metric)
+        _log.info("metrics.collected", count=len(metrics), names=[m.name for m in metrics])
+        return metrics
+
+    def _load(self, *, file: str | None, command: str | None) -> dict[str, object]:
+        if file:
+            path = (self._cwd / file).resolve()
+            if not path.is_file():
+                raise CollectionError(f"metrics file not found: {path}")
+            return _as_dict(path.read_text(encoding="utf-8"), source=str(path))
+        if command:
+            try:
+                proc = subprocess.run(  # noqa: S602 - user-configured command
+                    command,
+                    cwd=self._cwd,
+                    shell=True,
+                    capture_output=True,
+                    text=True,
+                    timeout=300,
+                )
+            except (OSError, subprocess.SubprocessError) as exc:
+                raise CollectionError(f"metrics command failed: {command}: {exc}") from exc
+            if proc.returncode != 0:
+                raise CollectionError(
+                    f"metrics command exited {proc.returncode}: {proc.stderr.strip()[:200]}"
+                )
+            return _as_dict(proc.stdout, source=f"`{command}`")
+        return {}
+
+
+def guess_direction(name: str) -> MetricDirection:
+    lowered = name.lower().replace("-", "_")
+    if any(hint in lowered for hint in _LOWER_IS_BETTER):
+        return MetricDirection.LOWER_IS_BETTER
+    return MetricDirection.HIGHER_IS_BETTER
+
+
+def _as_dict(text: str, *, source: str) -> dict[str, object]:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise CollectionError(f"metrics from {source} are not valid JSON: {exc}") from exc
+    if isinstance(data, dict) and "metrics" in data and isinstance(data["metrics"], dict):
+        data = data["metrics"]
+    if not isinstance(data, dict):
+        raise CollectionError(f"metrics from {source} must be a JSON object")
+    return data
+
+
+def _to_metric(name: str, value: object) -> Metric | None:
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return Metric(name=name, after=float(value))
+    if isinstance(value, dict):
+        after = _num(value.get("after")) or _num(value.get("value"))
+        before = _num(value.get("before"))
+        if after is None and before is None:
+            return None
+        return Metric(
+            name=name,
+            before=before,
+            after=after,
+            unit=str(value["unit"]) if value.get("unit") is not None else None,
+            metadata={
+                k: v for k, v in value.items() if k not in ("before", "after", "value", "unit")
+            },
+        )
+    return None
+
+
+def _num(value: object) -> float | None:
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return float(value)
+    if isinstance(value, str):
+        try:
+            return float(value)
+        except ValueError:
+            return None
+    return None
+
+
+__all__ = ["MetricsAdapter", "guess_direction"]

devmemory/adapters/tests.py

@@ -0,0 +1,227 @@
+"""Test adapter: run the project's configured test command and normalize the result.
+
+DevMemory does not care which framework you use. It runs a shell command and
+parses what comes back - a JUnit XML report if one is configured, otherwise the
+stdout of pytest / go test / cargo test / a generic "N passed, M failed" line.
+The *outcome* is a fact; whether that outcome is good is decided elsewhere.
+"""
+
+from __future__ import annotations
+
+import re
+import shlex
+import subprocess
+import time
+from pathlib import Path
+from xml.etree import ElementTree
+
+from devmemory.domain.errors import CollectionError
+from devmemory.domain.models import TestOutcome
+from devmemory.logging import get_logger
+
+_log = get_logger(__name__)
+
+_OUTPUT_CAP = 20_000
+
+# pytest summary: "3 failed, 128 passed, 2 skipped, 1 error in 4.20s"
+_PYTEST_TOKEN = re.compile(r"(\d+)\s+(passed|failed|skipped|error|errors|xfailed|xpassed)")
+_PYTEST_FAIL_LINE = re.compile(r"^(?:FAILED|ERROR)\s+(\S+)", re.MULTILINE)
+
+# go test: "ok  pkg  0.1s" / "--- FAIL: TestFoo"
+_GO_FAIL = re.compile(r"^--- FAIL:\s+(\S+)", re.MULTILINE)
+_GO_PASS = re.compile(r"^--- PASS:\s+(\S+)", re.MULTILINE)
+
+# generic: "12 passed", "3 failing", "Tests: 5 failed, 40 passed"
+_GENERIC = re.compile(
+    r"(?P<n>\d+)\s+(?P<kind>passed|passing|failed|failing|skipped|pending|errors?)",
+    re.IGNORECASE,
+)
+
+
+class TestAdapter:
+    __test__ = False  # this runs tests; it is not itself a pytest test case
+
+    def __init__(self, repo_path: Path | str) -> None:
+        self._cwd = Path(repo_path).resolve()
+
+    def run(
+        self,
+        command: str,
+        *,
+        parser: str = "auto",
+        junit_xml: str | None = None,
+        timeout: int = 900,
+    ) -> TestOutcome:
+        """Execute ``command`` and return a normalized :class:`TestOutcome`."""
+        start = time.perf_counter()
+        try:
+            proc = subprocess.run(  # noqa: S602 - user-configured command, run in their repo
+                command,
+                cwd=self._cwd,
+                shell=True,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=timeout,
+            )
+        except subprocess.TimeoutExpired as exc:
+            raise CollectionError(
+                f"test command timed out after {timeout}s: {command}",
+            ) from exc
+        except OSError as exc:
+            raise CollectionError(f"could not run test command {command!r}: {exc}") from exc
+
+        duration = round(time.perf_counter() - start, 2)
+        combined = f"{proc.stdout}\n{proc.stderr}"
+
+        if junit_xml:
+            outcome = self._from_junit(self._cwd / junit_xml)
+        else:
+            outcome = self._parse(combined, parser)
+
+        outcome.command = command
+        outcome.exit_code = proc.returncode
+        outcome.duration_seconds = duration
+        outcome.output = _tail(combined, _OUTPUT_CAP)
+
+        # A command that exited non-zero with no parsed failures still failed.
+        if proc.returncode != 0 and outcome.failed == 0 and outcome.errors == 0:
+            outcome.errors = max(outcome.errors, 1)
+
+        _log.info(
+            "tests.collected",
+            command=command,
+            passed=outcome.passed,
+            failed=outcome.failed,
+            exit_code=proc.returncode,
+        )
+        return outcome
+
+    # -- parsing --------------------------------------------------------
+
+    def _parse(self, text: str, parser: str) -> TestOutcome:
+        chosen = parser
+        if chosen == "auto":
+            chosen = self._sniff(text)
+        if chosen == "pytest":
+            return self._parse_pytest(text)
+        if chosen == "go":
+            return self._parse_go(text)
+        if chosen == "junitxml":
+            raise CollectionError("parser=junitxml requires tests.junit_xml to be set")
+        return self._parse_generic(text)
+
+    @staticmethod
+    def _sniff(text: str) -> str:
+        head = text[:4000].lower()
+        if "=== test session starts ===" in head or "\npassed" in head or "pytest" in head:
+            return "pytest"
+        if "--- fail:" in head or "--- pass:" in head or "\nok  " in head:
+            return "go"
+        return "generic"
+
+    def _parse_pytest(self, text: str) -> TestOutcome:
+        # Take the last summary line so a mid-run "1 failed" doesn't shadow the total.
+        summary_line = ""
+        for line in text.splitlines():
+            if _PYTEST_TOKEN.search(line) and (
+                "passed" in line or "failed" in line or "error" in line
+            ):
+                summary_line = line
+        counts = {"passed": 0, "failed": 0, "skipped": 0, "errors": 0}
+        for count, kind in _PYTEST_TOKEN.findall(summary_line or text):
+            key = "errors" if kind.startswith("error") else kind
+            if key in counts:
+                counts[key] = max(counts[key], int(count))
+        failing = _PYTEST_FAIL_LINE.findall(text)
+        total = sum(counts.values())
+        return TestOutcome(
+            framework="pytest",
+            total=total,
+            passed=counts["passed"],
+            failed=counts["failed"],
+            skipped=counts["skipped"],
+            errors=counts["errors"],
+            failing=failing[:50],
+        )
+
+    def _parse_go(self, text: str) -> TestOutcome:
+        failing = _GO_FAIL.findall(text)
+        passing = _GO_PASS.findall(text)
+        return TestOutcome(
+            framework="go",
+            total=len(failing) + len(passing),
+            passed=len(passing),
+            failed=len(failing),
+            failing=failing[:50],
+        )
+
+    def _parse_generic(self, text: str) -> TestOutcome:
+        counts = {"passed": 0, "failed": 0, "skipped": 0, "errors": 0}
+        for m in _GENERIC.finditer(text):
+            kind = m.group("kind").lower()
+            n = int(m.group("n"))
+            if kind in ("passed", "passing"):
+                counts["passed"] = max(counts["passed"], n)
+            elif kind in ("failed", "failing"):
+                counts["failed"] = max(counts["failed"], n)
+            elif kind in ("skipped", "pending"):
+                counts["skipped"] = max(counts["skipped"], n)
+            elif kind.startswith("error"):
+                counts["errors"] = max(counts["errors"], n)
+        return TestOutcome(
+            framework="generic",
+            total=sum(counts.values()),
+            passed=counts["passed"],
+            failed=counts["failed"],
+            skipped=counts["skipped"],
+            errors=counts["errors"],
+        )
+
+    def _from_junit(self, path: Path) -> TestOutcome:
+        if not path.is_file():
+            raise CollectionError(f"JUnit report not found: {path}")
+        try:
+            root = ElementTree.parse(path).getroot()  # noqa: S314 - our own CI's report
+        except ElementTree.ParseError as exc:
+            raise CollectionError(f"could not parse JUnit report {path}: {exc}") from exc
+
+        suites = [root] if root.tag == "testsuite" else root.findall(".//testsuite")
+        total = failures = errors = skipped = 0
+        failing: list[str] = []
+        for suite in suites:
+            total += int(suite.get("tests", 0))
+            failures += int(suite.get("failures", 0))
+            errors += int(suite.get("errors", 0))
+            skipped += int(suite.get("skipped", 0))
+            for case in suite.findall("testcase"):
+                if case.find("failure") is not None or case.find("error") is not None:
+                    name = case.get("name", "")
+                    classname = case.get("classname", "")
+                    failing.append(f"{classname}::{name}" if classname else name)
+        return TestOutcome(
+            framework="junit",
+            total=total,
+            passed=total - failures - errors - skipped,
+            failed=failures,
+            errors=errors,
+            skipped=skipped,
+            failing=failing[:50],
+        )
+
+
+def _tail(text: str, cap: int) -> str:
+    text = text.strip()
+    return text if len(text) <= cap else "…(truncated)…\n" + text[-cap:]
+
+
+def split_command(command: str) -> list[str]:
+    """POSIX-ish split for display; execution always uses the shell."""
+    try:
+        return shlex.split(command, posix=True)
+    except ValueError:
+        return command.split()
+
+
+__all__ = ["TestAdapter", "split_command"]

devmemory/analysis/__init__.py

@@ -0,0 +1,19 @@
+"""AI analysis layer.
+
+Analysis is *interpretation*, stored separately from facts and never able to
+overwrite them. Providers are tried in a fallback chain; ``rules`` (deterministic,
+offline) is always the tail.
+"""
+
+from __future__ import annotations
+
+from devmemory.analysis.base import AnalysisInput, AnalysisProvider, fact_guard
+from devmemory.analysis.chain import build_providers, run_analysis
+
+__all__ = [
+    "AnalysisInput",
+    "AnalysisProvider",
+    "build_providers",
+    "fact_guard",
+    "run_analysis",
+]

devmemory/analysis/base.py

@@ -0,0 +1,128 @@
+"""The analysis contract: a normalized input, a provider interface, a fact guard.
+
+Analysis is *interpretation*. It is stored in its own table and can only ever
+populate the interpretive fields (``summary``, ``reasoning``, ``recommendation``,
+``warnings``, ``risk``). It can never touch a fact - Git line counts, test
+results, metric values, the Entire checkpoint - because those live on the
+``DevelopmentVersion`` and are never handed to a provider for writing.
+"""
+
+from __future__ import annotations
+
+import abc
+
+from pydantic import BaseModel, Field
+
+from devmemory.domain.models import Analysis
+from devmemory.logging import redact_secrets
+
+_RISK_LEVELS = ("low", "medium", "high")
+_MAX_SUMMARY = 1200
+_MAX_FIELD = 2000
+
+
+class MetricDelta(BaseModel):
+    name: str
+    before: float | None
+    after: float | None
+    direction: str
+    improved: bool
+    worsened: bool
+
+
+class AttemptRef(BaseModel):
+    version_id: str
+    status: str
+    result: str
+    matched_on: list[str]
+
+
+class ImpactRef(BaseModel):
+    entity: str
+    change_type: str
+    dependents: int
+
+
+class AnalysisInput(BaseModel):
+    """Everything a provider is allowed to see. Normalized facts, no raw source
+    unless ``diff_excerpt`` was explicitly enabled in config."""
+
+    version_id: str
+    intent: str | None
+    feature: str | None
+    agent: str | None
+    model: str | None
+    status: str
+    is_adverse: bool
+    files_changed: int
+    lines_added: int
+    lines_removed: int
+    changed_paths: list[str] = Field(default_factory=list)
+    test_summary: str | None = None
+    tests_passed: int | None = None
+    tests_failed: int | None = None
+    metric_deltas: list[MetricDelta] = Field(default_factory=list)
+    regressions: list[str] = Field(default_factory=list)
+    previous_attempts: list[AttemptRef] = Field(default_factory=list)
+    impact_hotspots: list[ImpactRef] = Field(default_factory=list)
+    diff_excerpt: str | None = None
+
+
+class AnalysisProvider(abc.ABC):
+    """One way to turn an :class:`AnalysisInput` into an :class:`Analysis`."""
+
+    #: stable identifier, also stored on the Analysis row
+    name: str = "base"
+
+    @abc.abstractmethod
+    def analyze(self, data: AnalysisInput) -> Analysis | None:
+        """Return an Analysis, or ``None`` to fall through to the next provider."""
+
+
+def fact_guard(analysis: Analysis, data: AnalysisInput, *, provider: str) -> Analysis:
+    """Sanitize a provider's Analysis so it cannot misrepresent the facts.
+
+    - ``provider`` / ``model`` are set by us, never by the model.
+    - ``risk`` is clamped to low|medium|high and can never sit *below* what the
+      recorded status implies (an adverse version is at least ``medium``).
+    - all free text is secret-scrubbed and length-capped.
+    """
+    risk = (analysis.risk or "").strip().lower()
+    if risk not in _RISK_LEVELS:
+        risk = "medium" if data.is_adverse else "low"
+
+    warnings = [redact_secrets(w)[:_MAX_FIELD] for w in analysis.warnings if w.strip()]
+    if data.is_adverse and risk == "low":
+        risk = "medium"
+        warnings.append(
+            "Risk raised to 'medium': this version is recorded as "
+            f"{data.status} with {len(data.regressions)} regression(s)."
+        )
+
+    return analysis.model_copy(
+        update={
+            "summary": redact_secrets(analysis.summary)[:_MAX_SUMMARY],
+            "reasoning": _clip(analysis.reasoning),
+            "recommendation": _clip(analysis.recommendation),
+            "warnings": warnings,
+            "risk": risk,
+            "provider": provider,
+            "model": analysis.model,
+        }
+    )
+
+
+def _clip(text: str | None) -> str | None:
+    if not text or not text.strip():
+        return None
+    return redact_secrets(text)[:_MAX_FIELD]
+
+
+__all__ = [
+    "AnalysisInput",
+    "AnalysisProvider",
+    "AttemptRef",
+    "ImpactRef",
+    "MetricDelta",
+    "fact_guard",
+]

devmemory/analysis/chain.py

@@ -0,0 +1,53 @@
+"""The provider fallback chain.
+
+Try each configured provider in order; the first that returns an Analysis wins,
+after passing through :func:`fact_guard`. ``rules`` is always appended as the
+guaranteed tail, so :func:`run_analysis` never raises and never returns ``None``.
+"""
+
+from __future__ import annotations
+
+from devmemory.analysis.base import AnalysisInput, AnalysisProvider, fact_guard
+from devmemory.analysis.llm import LLMProvider
+from devmemory.analysis.rules import RulesProvider
+from devmemory.domain.models import Analysis
+from devmemory.logging import get_logger
+
+_log = get_logger(__name__)
+
+_LLM_PROVIDERS = {"anthropic", "openai", "gemini"}
+
+
+def build_providers(names: list[str], *, model: str | None = None) -> list[AnalysisProvider]:
+    providers: list[AnalysisProvider] = []
+    seen: set[str] = set()
+    for raw in names:
+        name = raw.strip().lower()
+        if not name or name in seen:
+            continue
+        seen.add(name)
+        if name == "rules":
+            providers.append(RulesProvider())
+        elif name in _LLM_PROVIDERS:
+            providers.append(LLMProvider(name, model))
+        else:
+            _log.warning("analysis.unknown_provider", provider=name)
+    if not any(isinstance(p, RulesProvider) for p in providers):
+        providers.append(RulesProvider())  # guaranteed tail
+    return providers
+
+
+def run_analysis(data: AnalysisInput, providers: list[AnalysisProvider]) -> Analysis:
+    for provider in providers:
+        try:
+            result = provider.analyze(data)
+        except Exception as exc:
+            _log.warning("analysis.provider_error", provider=provider.name, error=str(exc))
+            continue
+        if result is not None:
+            return fact_guard(result, data, provider=provider.name)
+    # build_providers guarantees a RulesProvider tail, so this is unreachable
+    return fact_guard(RulesProvider().analyze(data), data, provider="rules")
+
+
+__all__ = ["build_providers", "run_analysis"]