multi-forge 0.2.0__py3-none-any.whl

forge/search/index_state.py

@@ -0,0 +1,295 @@
+"""Search index state for incremental transcript indexing.
+
+Tracks which transcript files have been indexed via mtime/size fingerprints,
+enabling the search system to skip already-indexed files and detect changes.
+
+Two-layer architecture:
+- IndexState (dataclass): pure in-memory operations (needs_reindex, mark_indexed, prune)
+- IndexStateStore: persistence + locking (read, write, update with file_lock_for_target)
+
+State file location: <project_root>/.forge/search-index/state.json
+
+Follows the BackendRegistry/BackendRegistryStore pattern from forge.backend.registry.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Callable
+
+from forge.core.state import (
+    SchemaVersionError,
+    atomic_write_json,
+    file_lock_for_target,
+    now_iso,
+)
+
+from .exceptions import IndexStateCorruptedError
+
+# Directory and file names
+SEARCH_INDEX_DIR = "search-index"
+STATE_FILENAME = "state.json"
+
+# Schema version — reject anything else (no migration, per coding-standards.md)
+INDEX_STATE_VERSION = 1
+
+# Lock timeouts
+CLI_LOCK_TIMEOUT_S = 5.0
+HANDLER_LOCK_TIMEOUT_S = 1.0
+
+
+def get_project_index_state_path(forge_root: Path) -> Path:
+    """Return the index state path for a Forge project.
+
+    Path: <forge_root>/.forge/search-index/state.json
+    """
+    return forge_root / ".forge" / SEARCH_INDEX_DIR / STATE_FILENAME
+
+
+def _require_absolute(path: Path) -> None:
+    """Raise ValueError if path is not absolute."""
+    if not path.is_absolute():
+        raise ValueError(f"path must be absolute, got: {path}")
+
+
+# --- Data layer ---
+
+
+@dataclass
+class IndexedFileEntry:
+    """Tracking metadata for a single indexed transcript file."""
+
+    mtime: float
+    size: int
+    indexed_at: str
+
+
+@dataclass
+class IndexState:
+    """In-memory representation of the search index state.
+
+    Pure operations — no disk I/O. Persistence is handled by IndexStateStore.
+    """
+
+    schema_version: int = INDEX_STATE_VERSION
+    updated_at: str = ""
+    indexed_files: dict[str, IndexedFileEntry] = field(default_factory=dict)
+
+    def needs_reindex(self, path: Path) -> bool:
+        """Check if a file needs (re)indexing based on mtime/size.
+
+        Returns True if the file is new or has changed since last indexing.
+        Returns False if the file is unchanged OR if the file does not exist on disk.
+
+        Missing-file semantics:
+        - No entry + file missing → False (nothing to do)
+        - Entry exists + file deleted → False (prune_missing() handles cleanup)
+
+        Raises:
+            ValueError: If path is not absolute.
+        """
+        _require_absolute(path)
+
+        try:
+            stat = os.stat(path)
+        except OSError:
+            return False
+
+        key = str(path)
+        entry = self.indexed_files.get(key)
+        if entry is None:
+            return True
+
+        return entry.mtime != stat.st_mtime or entry.size != stat.st_size
+
+    def mark_indexed(self, path: Path) -> None:
+        """Record that a file has been indexed with its current mtime/size.
+
+        Creates or updates the entry for the given path using the file's
+        current stat() values and the current timestamp.
+
+        Raises:
+            ValueError: If path is not absolute.
+            FileNotFoundError: If path does not exist on disk.
+        """
+        _require_absolute(path)
+
+        try:
+            stat = os.stat(path)
+        except FileNotFoundError:
+            raise
+        except OSError as e:
+            raise FileNotFoundError(str(path)) from e
+
+        self.indexed_files[str(path)] = IndexedFileEntry(
+            mtime=stat.st_mtime,
+            size=stat.st_size,
+            indexed_at=now_iso(),
+        )
+
+    def prune_missing(self) -> list[str]:
+        """Remove entries for files that no longer exist on disk.
+
+        Returns:
+            List of path strings that were removed.
+        """
+        to_remove = [key for key in self.indexed_files if not Path(key).is_file()]
+        for key in to_remove:
+            del self.indexed_files[key]
+        return to_remove
+
+
+# --- Persistence layer ---
+
+
+class IndexStateStore:
+    """Manage per-project search index state.
+
+    Store location: <project_root>/.forge/search-index/state.json
+    Tracks which transcript files have been indexed for incremental updates.
+    Uses atomic writes and advisory file locking for concurrent safety.
+
+    Error handling:
+    - Missing file: returns empty state (self-healing)
+    - Corrupted file: raises IndexStateCorruptedError
+    - Wrong schema version: raises SchemaVersionError
+    """
+
+    def __init__(
+        self,
+        forge_root: Path | None = None,
+        *,
+        state_path: Path | None = None,
+    ) -> None:
+        if state_path:
+            self._state_path = state_path  # Explicit override (tests)
+        elif forge_root:
+            self._state_path = get_project_index_state_path(forge_root)
+        else:
+            raise ValueError("Either forge_root or state_path required")
+
+    @property
+    def state_path(self) -> Path:
+        return self._state_path
+
+    def exists(self) -> bool:
+        return self._state_path.is_file()
+
+    def read(self) -> IndexState:
+        """Read the index state from disk.
+
+        Returns empty IndexState if the file does not exist (self-healing).
+
+        Raises:
+            IndexStateCorruptedError: If the file contains invalid JSON or structure.
+            SchemaVersionError: If the schema version doesn't match INDEX_STATE_VERSION.
+        """
+        if not self.exists():
+            return IndexState()
+
+        path_str = str(self._state_path)
+
+        try:
+            with open(self._state_path, encoding="utf-8") as f:
+                data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise IndexStateCorruptedError(path_str, f"invalid JSON: {e}") from e
+        except OSError as e:
+            raise IndexStateCorruptedError(path_str, f"read error: {e}") from e
+
+        if not isinstance(data, dict):
+            raise IndexStateCorruptedError(path_str, f"expected JSON object, got {type(data).__name__}")
+
+        version = data.get("schema_version")
+        if version is None:
+            raise IndexStateCorruptedError(path_str, "missing schema_version")
+        if version != INDEX_STATE_VERSION:
+            raise SchemaVersionError(path_str, INDEX_STATE_VERSION, version)
+
+        # Deserialize indexed_files: dict[str, dict] → dict[str, IndexedFileEntry]
+        indexed_files: dict[str, IndexedFileEntry] = {}
+        raw_files = data.get("indexed_files", {})
+        if isinstance(raw_files, dict):
+            for key, val in raw_files.items():
+                if isinstance(val, dict):
+                    try:
+                        indexed_files[key] = IndexedFileEntry(
+                            mtime=float(val["mtime"]),
+                            size=int(val["size"]),
+                            indexed_at=str(val.get("indexed_at", "")),
+                        )
+                    except (KeyError, TypeError, ValueError):
+                        # Skip malformed entries rather than failing the whole read
+                        continue
+
+        return IndexState(
+            schema_version=version,
+            updated_at=data.get("updated_at", ""),
+            indexed_files=indexed_files,
+        )
+
+    def write(self, state: IndexState) -> None:
+        """Write the index state atomically.
+
+        Sets state.updated_at to the current timestamp before writing.
+        Creates parent directories if needed.
+        """
+        state.updated_at = now_iso()
+        data = asdict(state)
+        atomic_write_json(self._state_path, data)
+
+    def update(self, *, timeout_s: float, mutate: Callable[[IndexState], None]) -> IndexState:
+        """Locked read-modify-write cycle.
+
+        Acquires an advisory file lock, reads the state, calls mutate(state),
+        then writes the updated state. Exceptions from mutate propagate
+        (not swallowed).
+
+        Args:
+            timeout_s: Maximum time to wait for the lock.
+            mutate: Callable that modifies the IndexState in-place.
+
+        Returns:
+            The updated IndexState after mutation and write.
+        """
+        with file_lock_for_target(target_path=self._state_path, timeout_s=timeout_s):
+            state = self.read()
+            mutate(state)
+            self.write(state)
+            return state
+
+    # -- Convenience wrappers --
+
+    def mark_indexed(self, path: Path, *, timeout_s: float = HANDLER_LOCK_TIMEOUT_S) -> None:
+        """Mark a file as indexed (locked read-modify-write).
+
+        Convenience wrapper around update() that calls state.mark_indexed(path).
+
+        Raises:
+            ValueError: If path is not absolute.
+            FileNotFoundError: If path does not exist on disk.
+        """
+
+        def _mutate(state: IndexState) -> None:
+            state.mark_indexed(path)
+
+        self.update(timeout_s=timeout_s, mutate=_mutate)
+
+    def prune_missing(self, *, timeout_s: float = CLI_LOCK_TIMEOUT_S) -> list[str]:
+        """Remove entries for deleted files (locked read-modify-write).
+
+        Convenience wrapper around update() that calls state.prune_missing().
+
+        Returns:
+            List of path strings that were removed.
+        """
+        removed: list[str] = []
+
+        def _mutate(state: IndexState) -> None:
+            removed.extend(state.prune_missing())
+
+        self.update(timeout_s=timeout_s, mutate=_mutate)
+        return removed

forge/search/store.py

@@ -0,0 +1,215 @@
+"""Document metadata store for search-indexed transcripts (v2).
+
+Persists SearchDocumentMeta objects (metadata only — no content, no tokens)
+at <project_root>/.forge/search-index/documents.json. Content and BM25 index
+data are stored in separate files (content.json, bm25_index.json).
+
+Each project has its own store — no cross-project mixing in a single file.
+
+Follows the IndexStateStore pattern: versioned JSON, atomic writes, file locking,
+self-healing on missing file.
+
+Uses dacite for deserialization (consistent with BackendRegistryStore).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+import dacite
+
+from forge.core.state import (
+    SchemaVersionError,
+    atomic_write_json,
+    file_lock_for_target,
+    now_iso,
+)
+
+from .exceptions import SearchDocumentStoreCorruptedError
+from .extractor import SearchDocumentMeta
+from .index_state import SEARCH_INDEX_DIR
+
+logger = logging.getLogger(__name__)
+
+# File and schema constants
+DOCUMENTS_FILENAME = "documents.json"
+DOCUMENT_STORE_VERSION = 1
+
+# Lock timeouts
+STORE_LOCK_TIMEOUT_S = 5.0
+HANDLER_STORE_LOCK_TIMEOUT_S = 1.0
+
+
+def get_project_documents_store_path(forge_root: Path) -> Path:
+    """Return the document store path for a Forge project.
+
+    Path: <forge_root>/.forge/search-index/documents.json
+    """
+    return forge_root / ".forge" / SEARCH_INDEX_DIR / DOCUMENTS_FILENAME
+
+
+class SearchDocumentStore:
+    """Manage per-project search document metadata store.
+
+    Store location: <forge_root>/.forge/search-index/documents.json
+    Documents are keyed by transcript_path (absolute path string).
+
+    V2 schema: metadata only (no content, no tokens). Content and BM25
+    index data are stored in separate files.
+
+    Error handling:
+    - Missing file: returns empty list (self-healing)
+    - Corrupted file: raises SearchDocumentStoreCorruptedError
+    - Wrong schema version: raises SchemaVersionError (v1 triggers rebuild)
+    """
+
+    def __init__(
+        self,
+        forge_root: Path | None = None,
+        *,
+        store_path: Path | None = None,
+    ) -> None:
+        if store_path:
+            self._store_path = store_path  # Explicit override (tests)
+        elif forge_root:
+            self._store_path = get_project_documents_store_path(forge_root)
+        else:
+            raise ValueError("Either forge_root or store_path required")
+
+    @property
+    def store_path(self) -> Path:
+        return self._store_path
+
+    def exists(self) -> bool:
+        return self._store_path.is_file()
+
+    def read(self) -> list[SearchDocumentMeta]:
+        """Read all document metadata from disk.
+
+        Returns empty list if the file does not exist (self-healing).
+
+        Raises:
+            SearchDocumentStoreCorruptedError: If the file contains invalid JSON.
+            SchemaVersionError: If the schema version doesn't match (v1 → rebuild).
+        """
+        if not self.exists():
+            return []
+
+        path_str = str(self._store_path)
+
+        try:
+            with open(self._store_path, encoding="utf-8") as f:
+                data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise SearchDocumentStoreCorruptedError(path_str, f"invalid JSON: {e}") from e
+        except OSError as e:
+            raise SearchDocumentStoreCorruptedError(path_str, f"read error: {e}") from e
+
+        if not isinstance(data, dict):
+            raise SearchDocumentStoreCorruptedError(path_str, f"expected JSON object, got {type(data).__name__}")
+
+        version = data.get("schema_version")
+        if version is None:
+            raise SearchDocumentStoreCorruptedError(path_str, "missing schema_version")
+        if version != DOCUMENT_STORE_VERSION:
+            raise SchemaVersionError(path_str, DOCUMENT_STORE_VERSION, version)
+
+        raw_docs = data.get("documents", [])
+        if not isinstance(raw_docs, list):
+            logger.warning(
+                "Document store %s has non-list 'documents' field (got %s), treating as empty",
+                path_str,
+                type(raw_docs).__name__,
+            )
+            return []
+
+        documents: list[SearchDocumentMeta] = []
+        path_str = str(self._store_path)
+        for i, raw in enumerate(raw_docs):
+            if not isinstance(raw, dict):
+                raise SearchDocumentStoreCorruptedError(path_str, f"entry {i} is {type(raw).__name__}, expected dict")
+            try:
+                doc = dacite.from_dict(
+                    data_class=SearchDocumentMeta,
+                    data=raw,
+                    config=dacite.Config(strict=True),
+                )
+                documents.append(doc)
+            except (dacite.DaciteError, KeyError, TypeError) as e:
+                raise SearchDocumentStoreCorruptedError(path_str, f"entry {i} deserialization error: {e}") from e
+
+        return documents
+
+    def write(self, documents: list[SearchDocumentMeta]) -> None:
+        """Write documents atomically.
+
+        Creates parent directories if needed.
+        """
+        data: dict[str, Any] = {
+            "schema_version": DOCUMENT_STORE_VERSION,
+            "updated_at": now_iso(),
+            "documents": [doc.to_dict() for doc in documents],
+        }
+        atomic_write_json(self._store_path, data)
+
+    def replace_all(
+        self,
+        documents: list[SearchDocumentMeta],
+        *,
+        timeout_s: float = STORE_LOCK_TIMEOUT_S,
+    ) -> None:
+        """Replace all documents under lock (for rebuild-index)."""
+        with file_lock_for_target(target_path=self._store_path, timeout_s=timeout_s):
+            self.write(documents)
+
+    def add(
+        self,
+        doc: SearchDocumentMeta,
+        *,
+        timeout_s: float = HANDLER_STORE_LOCK_TIMEOUT_S,
+    ) -> None:
+        """Add or replace a document (locked read-modify-write).
+
+        Documents are keyed by transcript_path. Idempotent: if a document
+        with the same transcript_path already exists, it is replaced.
+        """
+        with file_lock_for_target(target_path=self._store_path, timeout_s=timeout_s):
+            docs = self.read()
+            docs = [d for d in docs if d.transcript_path != doc.transcript_path]
+            docs.append(doc)
+            self.write(docs)
+
+    def prune_missing(self, *, timeout_s: float = STORE_LOCK_TIMEOUT_S) -> list[str]:
+        """Remove documents whose transcript_path no longer exists on disk.
+
+        Locked read-modify-write. Returns list of removed transcript_path strings.
+        Skips write if nothing was pruned.
+        """
+        with file_lock_for_target(target_path=self._store_path, timeout_s=timeout_s):
+            docs = self.read()
+            kept: list[SearchDocumentMeta] = []
+            removed: list[str] = []
+            for d in docs:
+                if Path(d.transcript_path).is_file():
+                    kept.append(d)
+                else:
+                    removed.append(d.transcript_path)
+            if removed:
+                self.write(kept)
+            return removed
+
+    def remove(self, transcript_path: str, *, timeout_s: float = STORE_LOCK_TIMEOUT_S) -> bool:
+        """Remove a document by transcript_path (locked read-modify-write).
+
+        Returns True if a document was found and removed, False otherwise.
+        """
+        with file_lock_for_target(target_path=self._store_path, timeout_s=timeout_s):
+            docs = self.read()
+            filtered = [d for d in docs if d.transcript_path != transcript_path]
+            if len(filtered) == len(docs):
+                return False
+            self.write(filtered)
+            return True

forge/search/tokenizer.py

@@ -0,0 +1,24 @@
+"""Shared tokenization for BM25 indexing and search.
+
+Used by both extractor (token caching at extraction time) and engine
+(query tokenization + snippet anchoring at search time).
+"""
+
+from __future__ import annotations
+
+import re
+
+# Matches word-like tokens (letters, digits, underscores).
+# Case-insensitive so _best_snippet() can iterate raw mixed-case content
+# and match against lowercased query tokens.
+TOKEN_RE = re.compile(r"[a-zA-Z0-9_]+")
+
+MIN_TOKEN_LENGTH = 2
+
+
+def tokenize(text: str) -> list[str]:
+    """Simple word tokenizer for BM25.
+
+    Lowercase, extract alphanumeric+underscore tokens, filter short tokens.
+    """
+    return [m for m in TOKEN_RE.findall(text.lower()) if len(m) >= MIN_TOKEN_LENGTH]

forge/session/__init__.py

@@ -0,0 +1,130 @@
+"""Forge Session - Named session management for Claude Code.
+
+This module provides the essential public API for session management:
+
+- SessionState: Core data structure for session state (intent + confirmed)
+- SessionStore: Read/write .forge/sessions/<name>/forge.session.json
+- IndexStore: Read/write ~/.forge/sessions/index.json
+- SessionManager: High-level session orchestration
+
+For specialized access, import from submodules directly:
+
+- forge.session.models: All data models (SessionIntent, SessionConfirmed, Worktree, etc.)
+- forge.session.effective: Effective config computation (apply_overrides, get_effective_value)
+- forge.session.overrides: Override operations (validate_key, parse_value, expand_wildcard)
+- forge.session.hooks: Hook integration (handle_session_start, HookResult, etc.)
+- forge.session.exceptions: Full exception hierarchy
+- forge.session.store: Store constants (MANIFEST_FILENAME, etc.)
+- forge.session.index: Index constants (INDEX_DIR, etc.)
+- forge.session.validation: Name validation constants (MIN/MAX_NAME_LENGTH)
+- forge.session.config: Config constants (VALID_PROXY_TEMPLATES)
+
+Quick Start:
+    from forge.session import (
+        create_session_state,
+        SessionStore,
+        IndexStore,
+    )
+
+    # Create a new session state
+    state = create_session_state(
+        "my-session",
+        proxy_template="litellm-gemini",
+        proxy_base_url="http://localhost:8084",
+    )
+
+    # Write state to worktree (per-session directory)
+    store = SessionStore("/path/to/worktree", "my-session")
+    store.write(state)
+
+    # Add to global index
+    index = IndexStore()
+    index.add_from_state(state, "/path/to/project")
+"""
+
+from __future__ import annotations
+
+# Config
+from .active import (
+    ActiveSessionEntry,
+    ActiveSessionIndex,
+    ActiveSessionStore,
+    run_with_active_session,
+    track_active_session,
+)
+from .config import (
+    DEFAULT_PROXY_BASE_URL,
+    DEFAULT_PROXY_TEMPLATE,
+    LAUNCH_MODE_HOST,
+    LAUNCH_MODE_SIDECAR,
+    SIDECAR_RUNTIME_BASE_URL,
+)
+
+# Effective config
+from .effective import compute_effective_intent
+
+# Exceptions (base + common operational)
+from .exceptions import (
+    ForgeSessionError,
+    InvalidSessionNameError,
+    SessionExistsError,
+    SessionNotFoundError,
+)
+
+# Index
+from .index import IndexStore
+
+# Manager
+from .manager import SessionManager
+
+# Models
+from .models import (
+    SCHEMA_VERSION,
+    SessionIndexEntry,
+    SessionState,
+    create_session_state,
+)
+
+# Overrides
+from .overrides import clear_overrides, delete_override, set_override
+
+# Store
+from .store import SessionStore
+
+# Validation
+from .validation import validate_name
+
+__all__ = [
+    # Core types
+    "SessionState",
+    "SessionIndexEntry",
+    "create_session_state",
+    "SCHEMA_VERSION",
+    "ActiveSessionEntry",
+    "ActiveSessionIndex",
+    # Stores
+    "SessionStore",
+    "IndexStore",
+    "ActiveSessionStore",
+    # Manager
+    "SessionManager",
+    # Operations
+    "compute_effective_intent",
+    "set_override",
+    "delete_override",
+    "clear_overrides",
+    "run_with_active_session",
+    "track_active_session",
+    "validate_name",
+    # Config
+    "DEFAULT_PROXY_TEMPLATE",
+    "DEFAULT_PROXY_BASE_URL",
+    "LAUNCH_MODE_HOST",
+    "LAUNCH_MODE_SIDECAR",
+    "SIDECAR_RUNTIME_BASE_URL",
+    # Exceptions (base + common operational)
+    "ForgeSessionError",
+    "SessionNotFoundError",
+    "SessionExistsError",
+    "InvalidSessionNameError",
+]