gdmcode 0.1.0__py3-none-any.whl

src/memory/conventions.py

@@ -0,0 +1,495 @@
+"""Convention extractor — scans project files to learn style rules.
+
+Conventions are stored in ``gdm.db`` and injected into the system prompt
+for every write operation so the agent follows the project's existing style.
+
+Detection is purely pattern-based (no LLM calls).  The extractor re-runs
+at most once per ``_CACHE_TTL_HOURS`` unless ``force=True`` is passed.
+"""
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from src.memory.db import GdmDatabase
+
+__all__ = ["Convention", "ConventionExtractor", "ConventionDrift", "DriftReport"]
+
+log = logging.getLogger(__name__)
+
+# ── Convention keys ──────────────────────────────────────────────────────────
+
+_KEY_NAMING_STYLE: str = "naming_style"
+_KEY_IMPORT_STYLE: str = "import_style"
+_KEY_TEST_STRUCTURE: str = "test_structure"
+_KEY_ASYNC_PATTERN: str = "async_pattern"
+_KEY_TYPE_ANNOTATIONS: str = "type_annotations"
+_KEY_DOCSTRING_FORMAT: str = "docstring_format"
+
+# ── Tuning constants ─────────────────────────────────────────────────────────
+
+_TOP_FILES_LIMIT: int = 40
+_NAMING_SAMPLE_LIMIT: int = 20
+_CACHE_TTL_HOURS: int = 24
+_MAX_FILE_READ_BYTES: int = 8_192
+_MIN_DEF_COUNT: int = 3  # minimum defs to make a confident naming judgement
+
+# ── Excluded directory names ─────────────────────────────────────────────────
+
+_EXCLUDED_DIRS: frozenset[str] = frozenset(
+    {".git", "__pycache__", "node_modules", ".venv", "venv", ".mypy_cache", "dist", "build"}
+)
+
+# ── Compiled regexes ─────────────────────────────────────────────────────────
+
+_SNAKE_DEF_RE = re.compile(r"\bdef\s+[a-z][a-z0-9_]+\s*\(")
+_PASCAL_DEF_RE = re.compile(r"\bdef\s+[A-Z][a-zA-Z0-9]+\s*\(")
+_ABS_IMPORT_RE = re.compile(r"^from\s+src\.", re.MULTILINE)
+_REL_IMPORT_RE = re.compile(r"^from\s+\.", re.MULTILINE)
+_ASYNC_DEF_RE = re.compile(r"\basync\s+def\s+")
+_ALL_DEF_RE = re.compile(r"\bdef\s+\w+")
+_RETURN_ANNOT_RE = re.compile(r"\)\s*->")
+_GOOGLE_DOC_RE = re.compile(r"\bArgs:\s*\n|\bReturns:\s*\n")
+_NUMPY_DOC_RE = re.compile(r"\bParameters\s*\n\s*-{3,}")
+
+# ── Header template ───────────────────────────────────────────────────────────
+
+_CONVENTIONS_HEADER: str = (
+    "## Project Conventions\n\n"
+    "The following conventions were detected in this codebase. "
+    "ALL code you write MUST follow these exactly:\n"
+)
+
+
+# ── Data model ────────────────────────────────────────────────────────────────
+
+
+@dataclass
+class Convention:
+    """A single detected project convention."""
+
+    key: str
+    value: str
+    confidence: float = field(default=1.0)
+
+
+@dataclass
+class ConventionDrift:
+    """A single detected deviation from a project convention."""
+
+    convention: str
+    expected: str
+    found: str
+    line_number: int
+    confidence: float
+
+
+@dataclass
+class DriftReport:
+    """Result of a convention drift check on a single file."""
+
+    drifts: list[ConventionDrift]
+    has_high_confidence_drift: bool
+
+
+# ── Main class ────────────────────────────────────────────────────────────────
+
+
+class ConventionExtractor:
+    """Extracts coding conventions from project source files.
+
+    Scans the top-40 most recently modified files and infers style rules
+    by pattern matching.  Results are stored in ``gdm.db`` conventions table.
+
+    Usage::
+
+        extractor = ConventionExtractor(db, project_root=root)
+        conventions = extractor.extract(project_root)
+        block = extractor.build_conventions_block()  # for system prompt injection
+    """
+
+    def __init__(
+        self,
+        db: GdmDatabase,
+        project_id: str | None = None,
+        *,
+        project_root: Path | None = None,
+    ) -> None:
+        self._db = db
+        self._project_root = project_root
+        if project_id is not None:
+            self._project_id = project_id
+        elif project_root is not None:
+            self._project_id = str(project_root)
+        else:
+            self._project_id = "default"
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def extract(self, project_root: Path, *, force: bool = False) -> list[Convention]:
+        """Scan project and extract conventions.  Cached — skips if fresh.
+
+        Args:
+            project_root: root directory of the project to scan.
+            force: if ``True``, re-scan even if cached data is recent.
+
+        Returns:
+            Detected :class:`Convention` list (deduplicated by key).
+        """
+        if not force and self._is_cache_fresh():
+            log.debug("Conventions cache is fresh for %s, skipping scan", self._project_id)
+            return self._load_from_cache()
+
+        conventions: list[Convention] = []
+        conventions.extend(self._scan_python_files(project_root))
+        conventions.extend(self._scan_typescript_files(project_root))
+
+        test_conv = self._detect_test_structure(project_root)
+        if test_conv is not None:
+            conventions.append(test_conv)
+
+        # Deduplicate — last detected value for a key wins.
+        by_key = {c.key: c for c in conventions}
+        unique = list(by_key.values())
+        self._upsert_conventions(unique)
+        log.info("Extracted %d conventions for project %s", len(unique), self._project_id)
+        return unique
+
+    def build_conventions_block(self) -> str:
+        """Build a system-prompt-ready conventions block from cached data.
+
+        Returns:
+            Formatted string ready for injection into a system prompt,
+            or an empty string if no conventions have been detected yet.
+        """
+        rows = self._db.get_conventions(self._project_id)
+        if not rows:
+            return ""
+        lines: list[str] = []
+        for row in rows:
+            lines.append(f"- {row['key']}: {row['value']}")
+        return _CONVENTIONS_HEADER + "\n".join(lines) + "\n"
+
+    def get_convention(self, key: str) -> str | None:
+        """Return the value for a specific convention key, or ``None``.
+
+        Args:
+            key: one of the ``_KEY_*`` constants (e.g. ``"naming_style"``).
+
+        Returns:
+            The stored convention value, or ``None`` if not yet detected.
+        """
+        row = self._db.execute_one(
+            "SELECT value FROM conventions WHERE project_id = ? AND key = ?",
+            (self._project_id, key),
+        )
+        return str(row["value"]) if row is not None else None
+
+    # ------------------------------------------------------------------
+    # Drift detection
+    # ------------------------------------------------------------------
+
+    def _extract_error_handling(self, files: list[Path]) -> str:
+        """Returns 'try_except' | 'result_type' | 'mixed'"""
+        try_count = 0
+        result_count = 0
+        for f in files[:50]:
+            try:
+                content = f.read_text(errors="ignore")
+            except OSError:
+                continue
+            try_count += len(re.findall(r'\btry\b\s*:', content))
+            result_count += len(re.findall(r'\b(?:Ok|Err|Result)\s*[\[\(]', content))
+        if try_count > result_count * 3:
+            return "try_except"
+        if result_count > try_count * 3:
+            return "result_type"
+        return "mixed"
+
+    def _is_cache_stale(self) -> bool:
+        try:
+            cached_at = self._db.get_convention_cached_at()
+        except Exception:
+            return True
+        if not cached_at:
+            return True
+        if (datetime.now() - cached_at).total_seconds() > 86400:
+            return True
+        try:
+            recent_changes = sum(
+                1 for f in self._project_root.rglob("*.py")
+                if f.stat().st_mtime > cached_at.timestamp()
+            )
+            return recent_changes > 100
+        except (OSError, AttributeError):
+            return False
+
+    def check_drift(self, file_content: str, conventions: dict) -> DriftReport:
+        """Check file content for deviations from the given conventions dict.
+
+        Args:
+            file_content: source text to inspect.
+            conventions: mapping of convention key → expected value string.
+
+        Returns:
+            :class:`DriftReport` with all high-confidence drifts found.
+        """
+        drifts: list[ConventionDrift] = []
+
+        if conventions.get("naming_style") == "snake_case":
+            for m in re.finditer(r'\bdef\s+([a-z][a-zA-Z0-9]*[A-Z][a-zA-Z0-9]*)\b', file_content):
+                line_no = file_content[:m.start()].count('\n') + 1
+                drifts.append(ConventionDrift("naming_style", "snake_case", "camelCase", line_no, 0.9))
+
+        if conventions.get("error_handling") == "result_type":
+            for m in re.finditer(r'\btry\s*:', file_content):
+                line_no = file_content[:m.start()].count('\n') + 1
+                drifts.append(ConventionDrift("error_handling", "result_type", "try_except", line_no, 0.85))
+
+        if conventions.get("import_style") == "absolute":
+            for m in re.finditer(r'^from\s+\.', file_content, re.MULTILINE):
+                line_no = file_content[:m.start()].count('\n') + 1
+                drifts.append(ConventionDrift("import_style", "absolute", "relative", line_no, 0.95))
+
+        high = [d for d in drifts if d.confidence > 0.8]
+        return DriftReport(drifts=high, has_high_confidence_drift=bool(high))
+
+    # ------------------------------------------------------------------
+    # File-type scanners
+    # ------------------------------------------------------------------
+
+    def _scan_python_files(self, root: Path) -> list[Convention]:
+        """Extract Python-specific conventions from .py files."""
+        files = self._get_recent_files(root, ["*.py"])
+        if not files:
+            return []
+
+        result: list[Convention] = []
+        for detector in (
+            self._detect_naming_style,
+            self._detect_import_style,
+            self._detect_async_pattern,
+            self._detect_type_annotations,
+            self._detect_docstring_format,
+        ):
+            conv = detector(files)
+            if conv is not None:
+                result.append(conv)
+        return result
+
+    def _scan_typescript_files(self, root: Path) -> list[Convention]:
+        """Extract TypeScript-specific conventions from .ts/.tsx files."""
+        files = self._get_recent_files(root, ["*.ts", "*.tsx"])
+        if not files:
+            return []
+
+        result: list[Convention] = []
+        async_conv = self._detect_async_pattern(files)
+        if async_conv is not None:
+            result.append(async_conv)
+        return result
+
+    # ------------------------------------------------------------------
+    # Detectors (pattern-based, no LLM)
+    # ------------------------------------------------------------------
+
+    def _detect_naming_style(self, files: list[Path]) -> Convention | None:
+        """Detect function/class naming style by sampling function definitions."""
+        snake_count = 0
+        pascal_count = 0
+        for path in files[:_NAMING_SAMPLE_LIMIT]:
+            text = self._read_file_text(path)
+            snake_count += len(_SNAKE_DEF_RE.findall(text))
+            pascal_count += len(_PASCAL_DEF_RE.findall(text))
+
+        total = snake_count + pascal_count
+        if total < _MIN_DEF_COUNT:
+            return None
+
+        ratio = max(snake_count, pascal_count) / total
+        if ratio < 0.65:
+            # Too ambiguous to assert a single style
+            return Convention(
+                key=_KEY_NAMING_STYLE,
+                value="mixed naming styles detected",
+                confidence=round(1.0 - ratio, 2),
+            )
+        if snake_count >= pascal_count:
+            value = "snake_case functions, PascalCase classes, SCREAMING_SNAKE constants"
+        else:
+            value = "PascalCase functions and classes, SCREAMING_SNAKE constants"
+        return Convention(key=_KEY_NAMING_STYLE, value=value, confidence=round(ratio, 2))
+
+    def _detect_import_style(self, files: list[Path]) -> Convention | None:
+        """Detect import style: absolute (``from src.``) vs relative (``from .``)."""
+        abs_count = 0
+        rel_count = 0
+        for path in files:
+            text = self._read_file_text(path)
+            abs_count += len(_ABS_IMPORT_RE.findall(text))
+            rel_count += len(_REL_IMPORT_RE.findall(text))
+
+        total = abs_count + rel_count
+        if total == 0:
+            return None
+
+        if abs_count >= rel_count:
+            value = "absolute imports (from src. prefix)"
+        else:
+            value = "relative imports (from . prefix)"
+        confidence = min(1.0, max(abs_count, rel_count) / total)
+        return Convention(key=_KEY_IMPORT_STYLE, value=value, confidence=confidence)
+
+    def _detect_test_structure(self, root: Path) -> Convention | None:
+        """Detect test framework from test files and config presence."""
+        tests_dir = root / "tests"
+        if (root / "jest.config.js").exists() or (root / "jest.config.ts").exists():
+            return Convention(key=_KEY_TEST_STRUCTURE, value="jest", confidence=1.0)
+        if (root / "vitest.config.ts").exists() or (root / "vitest.config.js").exists():
+            return Convention(key=_KEY_TEST_STRUCTURE, value="vitest", confidence=1.0)
+        if tests_dir.exists() and (tests_dir / "conftest.py").exists():
+            return Convention(key=_KEY_TEST_STRUCTURE, value="pytest with conftest.py", confidence=1.0)
+        if tests_dir.exists() and any(tests_dir.rglob("test_*.py")):
+            return Convention(key=_KEY_TEST_STRUCTURE, value="pytest", confidence=0.9)
+        return None
+
+    def _detect_async_pattern(self, files: list[Path]) -> Convention | None:
+        """Detect async/await usage ratio across the file sample."""
+        async_count = 0
+        total_count = 0
+        for path in files:
+            text = self._read_file_text(path)
+            async_count += len(_ASYNC_DEF_RE.findall(text))
+            total_count += len(_ALL_DEF_RE.findall(text))
+
+        if total_count < _MIN_DEF_COUNT:
+            return None
+
+        ratio = async_count / total_count
+        if ratio > 0.5:
+            value = f"primarily async/await ({ratio:.0%} of functions are async)"
+        elif ratio > 0.1:
+            value = f"mixed sync/async ({ratio:.0%} of functions are async)"
+        else:
+            value = "primarily synchronous (minimal async usage)"
+        return Convention(key=_KEY_ASYNC_PATTERN, value=value, confidence=0.9)
+
+    def _detect_type_annotations(self, files: list[Path]) -> Convention | None:
+        """Check ratio of function definitions with ``->`` return annotations."""
+        annotated = 0
+        total = 0
+        for path in files:
+            text = self._read_file_text(path)
+            total += len(_ALL_DEF_RE.findall(text))
+            annotated += len(_RETURN_ANNOT_RE.findall(text))
+
+        if total < _MIN_DEF_COUNT:
+            return None
+
+        ratio = min(1.0, annotated / total)
+        if ratio > 0.8:
+            value = f"required — {ratio:.0%} of functions carry return annotations"
+        elif ratio > 0.4:
+            value = f"partial — {ratio:.0%} of functions carry return annotations"
+        else:
+            value = "optional/minimal type annotations"
+        return Convention(key=_KEY_TYPE_ANNOTATIONS, value=value, confidence=0.85)
+
+    def _detect_docstring_format(self, files: list[Path]) -> Convention | None:
+        """Detect Google or NumPy docstring style from a file sample."""
+        google_count = 0
+        numpy_count = 0
+        for path in files[:_NAMING_SAMPLE_LIMIT]:
+            text = self._read_file_text(path)
+            if _GOOGLE_DOC_RE.search(text):
+                google_count += 1
+            if _NUMPY_DOC_RE.search(text):
+                numpy_count += 1
+
+        if google_count == 0 and numpy_count == 0:
+            return None
+
+        if google_count >= numpy_count:
+            value = "Google style (Args:, Returns: sections)"
+            confidence = min(1.0, google_count / 5)
+        else:
+            value = "NumPy style (Parameters/dashes sections)"
+            confidence = min(1.0, numpy_count / 5)
+        return Convention(key=_KEY_DOCSTRING_FORMAT, value=value, confidence=confidence)
+
+    # ------------------------------------------------------------------
+    # Storage helpers
+    # ------------------------------------------------------------------
+
+    def _upsert_conventions(self, conventions: list[Convention]) -> None:
+        """Persist detected conventions to the DB (upsert by project+key).
+
+        Each upsert is committed individually.  The ``upsert_convention``
+        helper on :class:`GdmDatabase` handles its own commit, so wrapping
+        in an explicit transaction would conflict.
+        """
+        for conv in conventions:
+            self._db.upsert_convention(
+                self._project_id, conv.key, conv.value, conv.confidence
+            )
+
+    # ------------------------------------------------------------------
+    # Cache helpers
+    # ------------------------------------------------------------------
+
+    def _is_cache_fresh(self) -> bool:
+        """Return ``True`` if the newest convention row is within the TTL."""
+        row = self._db.execute_one(
+            "SELECT last_updated FROM conventions WHERE project_id = ? "
+            "ORDER BY last_updated DESC LIMIT 1",
+            (self._project_id,),
+        )
+        if row is None:
+            return False
+        last_updated = datetime.fromisoformat(str(row["last_updated"]))
+        if last_updated.tzinfo is None:
+            last_updated = last_updated.replace(tzinfo=timezone.utc)
+        return datetime.now(timezone.utc) - last_updated < timedelta(hours=_CACHE_TTL_HOURS)
+
+    def _load_from_cache(self) -> list[Convention]:
+        """Build a :class:`Convention` list from DB rows."""
+        return [
+            Convention(key=str(r["key"]), value=str(r["value"]), confidence=float(r["confidence"]))
+            for r in self._db.get_conventions(self._project_id)
+        ]
+
+    # ------------------------------------------------------------------
+    # File system helpers
+    # ------------------------------------------------------------------
+
+    def _get_recent_files(self, root: Path, patterns: list[str]) -> list[Path]:
+        """Return up to ``_TOP_FILES_LIMIT`` most recently modified matching files."""
+        candidates: list[Path] = []
+        for pattern in patterns:
+            for path in root.rglob(pattern):
+                if not any(part in _EXCLUDED_DIRS for part in path.parts):
+                    candidates.append(path)
+
+        def _safe_mtime(p: Path) -> float:
+            try:
+                return p.stat().st_mtime
+            except OSError:
+                return 0.0
+
+        candidates.sort(key=_safe_mtime, reverse=True)
+        return candidates[:_TOP_FILES_LIMIT]
+
+    @staticmethod
+    def _read_file_text(path: Path) -> str:
+        """Read up to ``_MAX_FILE_READ_BYTES`` bytes from a file as UTF-8 text."""
+        try:
+            raw = path.read_bytes()[:_MAX_FILE_READ_BYTES]
+            return raw.decode("utf-8", errors="replace")
+        except OSError as exc:
+            log.debug("Cannot read %s: %s", path, exc)
+            return ""