gdmcode 0.1.0__py3-none-any.whl

src/memory/document_index.py

@@ -0,0 +1,205 @@
+"""DocumentIndex — SQLite FTS5 full-text index for local documents.
+
+The index DB lives at ~/.config/gdm/document_index.db by default.
+Pass db_path= in tests to use a temp database.
+"""
+from __future__ import annotations
+import logging, sqlite3, threading, time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+log = logging.getLogger(__name__)
+
+__all__ = ["DocumentIndex", "IndexedChunk", "SearchResult"]
+
+INDEX_DB = Path.home() / ".config" / "gdm" / "document_index.db"
+CHUNK_MAX_WORDS = 500
+
+
+@dataclass
+class IndexedChunk:
+    doc_id: int
+    chunk_index: int
+    text: str
+    source_label: str  # e.g. "Sheet: Revenue" or "Page 3"
+
+
+@dataclass
+class SearchResult:
+    file_path: str
+    source_label: str
+    snippet: str
+    score: float
+    chunk_index: int
+
+
+class DocumentIndex:
+    def __init__(self, db_path: Path = None):
+        self._path = db_path or INDEX_DB
+        self._lock = threading.Lock()
+        self._conn: Optional[sqlite3.Connection] = None
+        self._init_db()
+
+    def _get_conn(self) -> sqlite3.Connection:
+        if self._conn is None:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            self._conn = sqlite3.connect(str(self._path), check_same_thread=False)
+            self._conn.row_factory = sqlite3.Row
+        return self._conn
+
+    def _init_db(self) -> None:
+        with self._lock:
+            conn = self._get_conn()
+            conn.executescript("""
+                CREATE TABLE IF NOT EXISTS indexed_documents (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    file_path TEXT UNIQUE NOT NULL,
+                    format TEXT NOT NULL,
+                    file_mtime REAL NOT NULL,
+                    chunk_count INTEGER NOT NULL DEFAULT 0,
+                    indexed_at REAL NOT NULL
+                );
+                CREATE VIRTUAL TABLE IF NOT EXISTS document_chunks USING fts5(
+                    file_path UNINDEXED,
+                    source_label UNINDEXED,
+                    chunk_index UNINDEXED,
+                    text,
+                    tokenize='porter unicode61'
+                );
+            """)
+            conn.commit()
+
+    def index_document(self, path: Path | str, force: bool = False) -> int:
+        """Index document at path. Returns number of chunks indexed.
+        Returns 0 if file mtime unchanged (unless force=True).
+        Returns -1 on error (file not found, unreadable, etc.)."""
+        from src.tools.document_reader import DocumentReader
+        path = Path(path)
+        try:
+            mtime = path.stat().st_mtime
+        except OSError as exc:
+            log.warning("Cannot stat %s: %s", path, exc)
+            return -1
+
+        with self._lock:
+            conn = self._get_conn()
+            existing = conn.execute(
+                "SELECT file_mtime FROM indexed_documents WHERE file_path=?",
+                (str(path),)
+            ).fetchone()
+            if existing and not force and abs(existing["file_mtime"] - mtime) < 0.001:
+                return 0  # up to date
+
+        content = DocumentReader().read(path)
+        if not content.success:
+            log.warning("Cannot index %s: %s", path, content.error)
+            return -1
+
+        chunks = self._chunk(content)
+        with self._lock:
+            conn = self._get_conn()
+            conn.execute(
+                "DELETE FROM document_chunks WHERE file_path=?", (str(path),)
+            )
+            conn.execute(
+                "DELETE FROM indexed_documents WHERE file_path=?", (str(path),)
+            )
+            conn.executemany(
+                "INSERT INTO document_chunks (file_path, source_label, chunk_index, text) VALUES (?,?,?,?)",
+                [(str(path), c.source_label, c.chunk_index, c.text) for c in chunks]
+            )
+            conn.execute(
+                "INSERT INTO indexed_documents (file_path, format, file_mtime, chunk_count, indexed_at) VALUES (?,?,?,?,?)",
+                (str(path), content.format, mtime, len(chunks), time.time())
+            )
+            conn.commit()
+        return len(chunks)
+
+    def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+        """Full-text search across all indexed documents."""
+        try:
+            with self._lock:
+                conn = self._get_conn()
+                rows = conn.execute(
+                    """SELECT file_path, source_label, chunk_index,
+                              snippet(document_chunks, 3, '[', ']', '...', 20) as snip,
+                              rank
+                       FROM document_chunks
+                       WHERE document_chunks MATCH ?
+                       ORDER BY rank
+                       LIMIT ?""",
+                    (query, limit)
+                ).fetchall()
+        except sqlite3.OperationalError as exc:
+            log.warning("FTS5 search error: %s", exc)
+            return []
+        results = []
+        for row in rows:
+            results.append(SearchResult(
+                file_path=row["file_path"],
+                source_label=row["source_label"],
+                snippet=row["snip"],
+                score=abs(row["rank"]),
+                chunk_index=row["chunk_index"],
+            ))
+        return results
+
+    def list_indexed(self) -> list[dict]:
+        with self._lock:
+            rows = self._get_conn().execute(
+                "SELECT file_path, format, chunk_count, indexed_at FROM indexed_documents ORDER BY indexed_at DESC"
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+    def remove(self, path: Path | str) -> None:
+        path_str = str(Path(path))
+        with self._lock:
+            conn = self._get_conn()
+            conn.execute("DELETE FROM document_chunks WHERE file_path=?", (path_str,))
+            conn.execute("DELETE FROM indexed_documents WHERE file_path=?", (path_str,))
+            conn.commit()
+
+    def _chunk(self, content) -> list[IndexedChunk]:
+        chunks: list[IndexedChunk] = []
+        # Spreadsheets: one chunk per sheet
+        if content.sheets:
+            for sheet in content.sheets:
+                text = sheet.to_text()
+                if text.strip():
+                    chunks.append(IndexedChunk(
+                        doc_id=0, chunk_index=len(chunks),
+                        text=text[:8000],
+                        source_label=f"Sheet: {sheet.name}",
+                    ))
+            return chunks
+        # Text/PDF/DOCX: split on paragraphs, group into ~500 word chunks
+        paragraphs = [p.strip() for p in content.text.split("\n\n") if p.strip()]
+        current_words = 0
+        current_parts: list[str] = []
+        page_hint = 1
+        for para in paragraphs:
+            words = len(para.split())
+            if current_words + words > CHUNK_MAX_WORDS and current_parts:
+                chunks.append(IndexedChunk(
+                    doc_id=0, chunk_index=len(chunks),
+                    text="\n\n".join(current_parts),
+                    source_label=f"Page {page_hint}",
+                ))
+                page_hint += 1
+                current_parts = []
+                current_words = 0
+            current_parts.append(para)
+            current_words += words
+        if current_parts:
+            chunks.append(IndexedChunk(
+                doc_id=0, chunk_index=len(chunks),
+                text="\n\n".join(current_parts),
+                source_label=f"Page {page_hint}",
+            ))
+        return chunks
+
+    def close(self) -> None:
+        if self._conn:
+            self._conn.close()
+            self._conn = None

src/memory/file_cache.py

@@ -0,0 +1,128 @@
+"""mtime-based file freshness cache — anti-hallucination core.
+
+Tracks when each file was last read and its mtime at that time.
+Used to detect stale file content in context and force re-reads.
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from src.memory.db import GdmDatabase
+
+__all__ = ["CacheEntry", "FileCache"]
+
+log = logging.getLogger(__name__)
+
+_PURGE_BATCH_SIZE: int = 500
+
+
+@dataclass
+class CacheEntry:
+    """A single file-cache record."""
+
+    path: Path
+    mtime: float
+    summary: str | None
+    last_read_at: str  # ISO timestamp
+
+
+class FileCache:
+    """mtime-based freshness cache for file reads.
+
+    Tracks when each file was last read and its mtime at that time.
+    Used to detect stale file content in context and force re-reads.
+
+    Usage::
+
+        cache = FileCache(db, project_id)
+        cache.mark_read(Path("src/auth.py"))
+        is_fresh = cache.is_fresh(Path("src/auth.py"))  # False if file changed
+    """
+
+    def __init__(self, db: GdmDatabase, project_id: str) -> None:
+        self._db = db
+        self._project_id = project_id
+
+    @staticmethod
+    def _normalize(path: Path) -> Path:
+        """Return resolved absolute path for consistent DB storage."""
+        return path.resolve()
+
+    def mark_read(self, path: Path, summary: str | None = None) -> None:
+        """Record that this file was just read. Stores current mtime."""
+        normalized = self._normalize(path)
+        try:
+            mtime = normalized.stat().st_mtime
+        except OSError:
+            log.warning("mark_read: cannot stat %s", normalized)
+            return
+        self._db.upsert_file_cache(self._project_id, str(normalized), mtime, summary)
+
+    def is_fresh(self, path: Path) -> bool:
+        """True if file mtime matches cached mtime (file unchanged since last read).
+
+        Returns True if never read (unknown = assumed fresh).
+        Returns False if file no longer exists OR mtime changed.
+        """
+        normalized = self._normalize(path)
+        row = self._db.get_file_cache(self._project_id, str(normalized))
+        if row is None:
+            return True  # never read — assumed fresh per spec
+        try:
+            current_mtime = normalized.stat().st_mtime
+        except OSError:
+            return False
+        return float(row["mtime"]) == current_mtime
+
+    def get_stale_paths(self, paths: list[Path]) -> list[Path]:
+        """Return subset of paths whose content may be stale."""
+        return [p for p in paths if not self.is_fresh(p)]
+
+    def invalidate(self, path: Path) -> None:
+        """Explicitly invalidate cache for a path (e.g., after write)."""
+        normalized = self._normalize(path)
+        self._db.execute(
+            "DELETE FROM file_cache WHERE project_id = ? AND path = ?",
+            (self._project_id, str(normalized)),
+        )
+
+    def get_entry(self, path: Path) -> CacheEntry | None:
+        """Get cache entry for a path. Returns None if not tracked."""
+        normalized = self._normalize(path)
+        row = self._db.get_file_cache(self._project_id, str(normalized))
+        if row is None:
+            return None
+        return CacheEntry(
+            path=normalized,
+            mtime=float(row["mtime"]),
+            summary=row["summary"],
+            last_read_at=row["last_read_at"],
+        )
+
+    def purge_deleted(self) -> int:
+        """Remove entries for files that no longer exist. Returns count purged."""
+        rows = self._db.execute_all(
+            "SELECT path FROM file_cache WHERE project_id = ?",
+            (self._project_id,),
+        )
+        stale = [row["path"] for row in rows if not Path(row["path"]).exists()]
+        if not stale:
+            return 0
+        return self._delete_batch(stale)
+
+    def _delete_batch(self, paths: list[str]) -> int:
+        """Delete file_cache rows for given path strings in batches."""
+        deleted = 0
+        for i in range(0, len(paths), _PURGE_BATCH_SIZE):
+            batch = paths[i : i + _PURGE_BATCH_SIZE]
+            placeholders = ",".join("?" * len(batch))
+            self._db.execute(
+                f"DELETE FROM file_cache WHERE project_id = ? AND path IN ({placeholders})",
+                (self._project_id, *tuple(batch)),
+            )
+            deleted += len(batch)
+        return deleted

src/memory/project_scanner.py

@@ -0,0 +1,178 @@
+"""Initial project record builder — runs once on first invocation.
+
+Updates on /memory refresh. Detects tech stack from marker files.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import sqlite3
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from src.memory.db import GdmDatabase
+
+__all__ = ["ProjectRecord", "ProjectScanner", "estimate_token_count"]
+
+log = logging.getLogger(__name__)
+
+_REFRESH_DAYS: int = 7
+
+_TECH_MARKERS: dict[str, str] = {
+    "pyproject.toml": "python",
+    "requirements.txt": "python",
+    "setup.py": "python",
+    "package.json": "nodejs",
+    "tsconfig.json": "typescript",
+    "Cargo.toml": "rust",
+    "go.mod": "golang",
+    "pom.xml": "java",
+    "build.gradle": "java",
+    "Dockerfile": "docker",
+    "docker-compose.yml": "docker",
+    ".terraform": "terraform",
+    "pubspec.yaml": "dart",
+    "mix.exs": "elixir",
+    "Gemfile": "ruby",
+    "composer.json": "php",
+}
+
+
+@dataclass
+class ProjectRecord:
+    """Lightweight descriptor for a scanned project."""
+
+    project_id: str
+    root_path: Path
+    name: str
+    tech_stack: list[str]  # e.g. ["python", "typescript", "docker"]
+
+
+class ProjectScanner:
+    """Scans project directory and builds/updates the project record in gdm.db.
+
+    Detects tech stack from marker files. Fast — no file content reading.
+
+    Usage::
+
+        scanner = ProjectScanner(db)
+        record = scanner.ensure_project(root_path)  # creates or updates
+    """
+
+    def __init__(self, db: GdmDatabase) -> None:
+        self._db = db
+
+    def ensure_project(self, root: Path) -> ProjectRecord:
+        """Get existing project record or scan and create a new one.
+
+        Always updates last_seen. Updates tech_stack if 7+ days old.
+        Returns ProjectRecord.
+        """
+        normalized = root.resolve()
+        row = self._db.execute_one(
+            "SELECT project_id, root_path, name, tech_stack, last_seen "
+            "FROM projects WHERE root_path = ?",
+            (str(normalized),),
+        )
+        if row is None:
+            return self.scan(root)
+        return self._refresh_record(row, normalized)
+
+    def scan(self, root: Path) -> ProjectRecord:
+        """Full scan — detect tech stack, count files. Creates DB record."""
+        normalized = root.resolve()
+        project_id = self._generate_project_id(normalized)
+        name = normalized.name
+        tech_stack = self._detect_tech_stack(normalized)
+        self._db.execute(
+            "INSERT INTO projects (project_id, root_path, name, tech_stack) "
+            "VALUES (?, ?, ?, ?) "
+            "ON CONFLICT(root_path) DO UPDATE SET "
+            "name = excluded.name, "
+            "tech_stack = excluded.tech_stack, "
+            "last_seen = datetime('now')",
+            (project_id, str(normalized), name, json.dumps(tech_stack)),
+        )
+        row = self._db.execute_one(
+            "SELECT project_id FROM projects WHERE root_path = ?",
+            (str(normalized),),
+        )
+        actual_id = row["project_id"] if row else project_id
+        log.info("Scanned project '%s' id=%s stack=%s", name, actual_id, tech_stack)
+        return ProjectRecord(
+            project_id=actual_id,
+            root_path=normalized,
+            name=name,
+            tech_stack=tech_stack,
+        )
+
+    def get_project_id(self, root: Path) -> str | None:
+        """Get project_id for a root path. Returns None if not in DB."""
+        normalized = root.resolve()
+        row = self._db.execute_one(
+            "SELECT project_id FROM projects WHERE root_path = ?",
+            (str(normalized),),
+        )
+        return row["project_id"] if row is not None else None
+
+    def _refresh_record(self, row: sqlite3.Row, root: Path) -> ProjectRecord:
+        """Update last_seen and optionally refresh tech_stack if stale."""
+        last_seen = datetime.fromisoformat(row["last_seen"])
+        now = datetime.now(timezone.utc).replace(tzinfo=None)  # naive UTC, matches SQLite
+        days_old = (now - last_seen).days
+        tech_stack = json.loads(row["tech_stack"])
+        if days_old >= _REFRESH_DAYS:
+            tech_stack = self._detect_tech_stack(root)
+            self._db.execute(
+                "UPDATE projects SET tech_stack = ?, last_seen = datetime('now') "
+                "WHERE project_id = ?",
+                (json.dumps(tech_stack), row["project_id"]),
+            )
+        else:
+            self._db.execute(
+                "UPDATE projects SET last_seen = datetime('now') WHERE project_id = ?",
+                (row["project_id"],),
+            )
+        return ProjectRecord(
+            project_id=row["project_id"],
+            root_path=root,
+            name=row["name"],
+            tech_stack=tech_stack,
+        )
+
+    def _detect_tech_stack(self, root: Path) -> list[str]:
+        """Detect technology stack from marker files. Returns sorted list."""
+        detected: set[str] = set()
+        for marker, tech in _TECH_MARKERS.items():
+            if (root / marker).exists():
+                detected.add(tech)
+        return sorted(detected)
+
+    def _generate_project_id(self, root: Path) -> str:
+        """Generate stable project ID from root path hash."""
+        return hashlib.sha256(str(root).encode()).hexdigest()[:16]
+
+
+def estimate_token_count(root: Path, tech_stack: list[str] | None = None) -> int:
+    """Estimate total token count for a project directory.
+
+    Uses :class:`~src.agent.context_budget.WholeCodebaseMode` internally for
+    deterministic file inclusion and a fast char-based approximation.
+
+    Args:
+        root: project root directory.
+        tech_stack: list of detected technologies (e.g. ``["python", "nodejs"]``).
+            Pass ``None`` or ``[]`` to scan all text files.
+
+    Returns:
+        Estimated token count (always ≥ 1).
+    """
+    from src.agent.context_budget import ContextBudget, WholeCodebaseMode
+
+    budget = ContextBudget()
+    wcm = WholeCodebaseMode(budget)
+    return wcm.estimate_token_count(root, tech_stack or [])