continuum-toolkit 1.0.0__py3-none-any.whl

storage/recovery.py

@@ -0,0 +1,98 @@
+"""
+Project Continuum - State Corruption Detection & Safe Auto-Recovery
+===================================================================
+Milestone 7 - Phase 17: Git Hooks & Persistent State Management.
+Validates active state JSON integrity, detects schema or byte-level corruption,
+and safely restores from the most recent valid history snapshot.
+"""
+
+from pathlib import Path
+from typing import Optional, Tuple
+
+from core.schema import validate_canonical_state_dict
+from core.serializer import CanonicalStateSerializer, StateSerializationError, StateValidationError
+from core.state_models import CanonicalProjectState
+from storage.manager import ContinuumStorageManager
+from storage.models import RecoveryResult
+
+
+class StateRecoveryManager:
+    """
+    Guarantees state recoverability:
+    1. Audits active state for corruption.
+    2. Recovers cleanly from the latest uncorrupted history snapshot if damaged.
+    """
+
+    def __init__(self, storage_manager: ContinuumStorageManager):
+        self._storage = storage_manager
+
+    def verify_active_state_integrity(self) -> Tuple[bool, Optional[str]]:
+        """
+        Verifies if active state file is valid and compliant with schema.
+        Returns (is_valid, error_reason).
+        """
+        if not self._storage.state_exists():
+            return False, "Active state file does not exist."
+
+        try:
+            state = self._storage.load_state(validate=True)
+            return True, None
+        except (StateSerializationError, StateValidationError, Exception) as e:
+            return False, str(e)
+
+    def recover_corrupted_state(self) -> RecoveryResult:
+        """
+        Attempts to recover from state corruption by finding the most recent
+        valid snapshot in .continuum/history/ and restoring it to state.json.
+        """
+        is_valid, error_reason = self.verify_active_state_integrity()
+        if is_valid:
+            return RecoveryResult(
+                success=True,
+                recovered_from=str(self._storage.active_state_file),
+                state_restored=False,
+                details={"status": "Active state is healthy; no recovery needed."}
+            )
+
+        # Active state is corrupted: inspect history snapshots
+        snapshots = self._storage.list_history_snapshots()
+        if not snapshots:
+            # Check backup directory
+            backup_file = self._storage.backup_dir / "state.json.bak"
+            if backup_file.exists():
+                try:
+                    state = CanonicalStateSerializer.load_from_file(backup_file, validate=True)
+                    self._storage.save_state(state, create_history_snapshot=False)
+                    return RecoveryResult(
+                        success=True,
+                        recovered_from=str(backup_file),
+                        state_restored=True,
+                        details={"restored_from": "backup"}
+                    )
+                except Exception:
+                    pass
+
+            return RecoveryResult(
+                success=False,
+                error_message=f"Active state corrupted ({error_reason}), and no valid history snapshots exist."
+            )
+
+        # Search snapshots for the newest valid one
+        for snapshot_path in snapshots:
+            try:
+                state = CanonicalStateSerializer.load_from_file(snapshot_path, validate=True)
+                # Found valid snapshot! Restore as active state
+                self._storage.save_state(state, create_history_snapshot=False)
+                return RecoveryResult(
+                    success=True,
+                    recovered_from=str(snapshot_path),
+                    state_restored=True,
+                    details={"snapshot_name": snapshot_path.name, "corrupted_reason": error_reason}
+                )
+            except Exception:
+                continue
+
+        return RecoveryResult(
+            success=False,
+            error_message=f"All {len(snapshots)} history snapshots were tested and none were valid."
+        )

watcher/__init__.py

@@ -0,0 +1,17 @@
+"""
+Project Continuum - Watcher Package Exports
+===========================================
+Milestone 7 - Phase 16: Filesystem Observation & Incremental State Updates.
+"""
+
+from watcher.models import ChangeType, FileChangeEvent, IncrementalUpdateResult
+from watcher.detector import WorkspaceChangeDetector
+from watcher.updater import IncrementalStateUpdater
+
+__all__ = [
+    "ChangeType",
+    "FileChangeEvent",
+    "IncrementalUpdateResult",
+    "WorkspaceChangeDetector",
+    "IncrementalStateUpdater",
+]

watcher/detector.py

@@ -0,0 +1,136 @@
+"""
+Project Continuum - Workspace Change Detector
+==============================================
+Milestone 7 - Phase 16: Filesystem Observation & Incremental State Updates.
+Computes filesystem delta sets (creations, modifications, deletions) against
+a cached baseline snapshot with built-in ignore filtering.
+"""
+
+import hashlib
+import os
+from pathlib import Path
+import time
+from typing import Dict, List, Optional, Set, Tuple
+
+from extractors.base import DEFAULT_IGNORE_DIRS
+from watcher.models import ChangeType, FileChangeEvent
+
+
+class WorkspaceChangeDetector:
+    """
+    Tracks workspace file timestamps and SHA-256 hashes to detect modifications,
+    creations, and deletions with high efficiency.
+    """
+
+    def __init__(
+        self,
+        workspace_root: str,
+        ignore_dirs: Optional[Set[str]] = None,
+        max_file_size_mb: float = 10.0
+    ):
+        self.workspace_root = Path(workspace_root).resolve()
+        self.ignore_dirs = ignore_dirs or DEFAULT_IGNORE_DIRS
+        self.max_file_size_bytes = int(max_file_size_mb * 1024 * 1024)
+        self._baseline_snapshot: Dict[str, Tuple[float, int, str]] = {}  # rel_path -> (mtime, size, hash)
+
+    def compute_current_snapshot(self) -> Dict[str, Tuple[float, int, str]]:
+        """
+        Walks the workspace and constructs a map of rel_path -> (mtime, size, sha256).
+        """
+        snapshot: Dict[str, Tuple[float, int, str]] = {}
+        if not self.workspace_root.exists():
+            return snapshot
+
+        for root, dirs, files in os.walk(self.workspace_root):
+            # Prune ignored directories
+            dirs[:] = [d for d in dirs if d not in self.ignore_dirs and not d.startswith(".")]
+
+            for file in files:
+                full_path = Path(root) / file
+                try:
+                    stat = full_path.stat()
+                    if stat.st_size > self.max_file_size_bytes:
+                        continue
+
+                    rel_path = str(full_path.relative_to(self.workspace_root)).replace("\\", "/")
+                    mtime = stat.st_mtime
+                    size = stat.st_size
+
+                    # Hash only if size or mtime differs from cached snapshot or on first pass
+                    cached = self._baseline_snapshot.get(rel_path)
+                    if cached and cached[0] == mtime and cached[1] == size:
+                        f_hash = cached[2]
+                    else:
+                        f_hash = self._hash_file(full_path)
+
+                    snapshot[rel_path] = (mtime, size, f_hash)
+                except (OSError, PermissionError):
+                    continue
+
+        return snapshot
+
+    def initialize_baseline(self) -> int:
+        """
+        Takes initial baseline snapshot of the workspace.
+        Returns number of tracked files.
+        """
+        self._baseline_snapshot = self.compute_current_snapshot()
+        return len(self._baseline_snapshot)
+
+    def detect_changes(self) -> List[FileChangeEvent]:
+        """
+        Compares current filesystem state against baseline snapshot and returns
+        the list of FileChangeEvents, updating the baseline in-place.
+        """
+        current_snapshot = self.compute_current_snapshot()
+        changes: List[FileChangeEvent] = []
+
+        # 1. Check for creations and modifications
+        for rel_path, (mtime, size, f_hash) in current_snapshot.items():
+            if rel_path not in self._baseline_snapshot:
+                changes.append(
+                    FileChangeEvent(
+                        file_path=rel_path,
+                        change_type=ChangeType.CREATED,
+                        file_hash=f_hash,
+                        file_size_bytes=size
+                    )
+                )
+            else:
+                base_mtime, base_size, base_hash = self._baseline_snapshot[rel_path]
+                if f_hash != base_hash:
+                    changes.append(
+                        FileChangeEvent(
+                            file_path=rel_path,
+                            change_type=ChangeType.MODIFIED,
+                            file_hash=f_hash,
+                            file_size_bytes=size
+                        )
+                    )
+
+        # 2. Check for deletions
+        for rel_path, (base_mtime, base_size, base_hash) in self._baseline_snapshot.items():
+            if rel_path not in current_snapshot:
+                changes.append(
+                    FileChangeEvent(
+                        file_path=rel_path,
+                        change_type=ChangeType.DELETED,
+                        file_hash=base_hash,
+                        file_size_bytes=base_size
+                    )
+                )
+
+        # Update baseline to current snapshot
+        self._baseline_snapshot = current_snapshot
+        return changes
+
+    def _hash_file(self, path: Path) -> str:
+        """Calculates SHA-256 hash of file content."""
+        hasher = hashlib.sha256()
+        try:
+            with open(path, "rb") as f:
+                while chunk := f.read(65536):
+                    hasher.update(chunk)
+            return hasher.hexdigest()
+        except (OSError, PermissionError):
+            return ""

watcher/models.py

@@ -0,0 +1,62 @@
+"""
+Project Continuum - Filesystem Observation & Incremental State Models
+======================================================================
+Milestone 7 - Phase 16: Filesystem Observation & Incremental State Updates.
+Defines change event structures and incremental update result containers.
+"""
+
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Dict, List, Optional, Set
+
+
+class ChangeType(str, Enum):
+    """Types of filesystem change events."""
+    CREATED = "CREATED"
+    MODIFIED = "MODIFIED"
+    DELETED = "DELETED"
+
+
+@dataclass
+class FileChangeEvent:
+    """Represents a discrete filesystem modification event."""
+    file_path: str
+    change_type: ChangeType
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    file_hash: Optional[str] = None
+    file_size_bytes: int = 0
+
+    def to_dict(self) -> Dict[str, Any]:
+        d = asdict(self)
+        d["change_type"] = self.change_type.value
+        return d
+
+
+@dataclass
+class IncrementalUpdateResult:
+    """
+    Summary of an incremental state update cycle.
+    """
+    workspace_root: str
+    changes_processed: List[FileChangeEvent] = field(default_factory=list)
+    re_extracted_symbols_count: int = 0
+    removed_symbols_count: int = 0
+    invalidated_node_ids: List[str] = field(default_factory=list)
+    propagated_stale_node_ids: List[str] = field(default_factory=list)
+    full_scan_fallback_triggered: bool = False
+    duration_ms: float = 0.0
+    updated_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "workspace_root": self.workspace_root,
+            "changes_processed": [c.to_dict() for c in self.changes_processed],
+            "re_extracted_symbols_count": self.re_extracted_symbols_count,
+            "removed_symbols_count": self.removed_symbols_count,
+            "invalidated_node_ids": self.invalidated_node_ids,
+            "propagated_stale_node_ids": self.propagated_stale_node_ids,
+            "full_scan_fallback_triggered": self.full_scan_fallback_triggered,
+            "duration_ms": self.duration_ms,
+            "updated_at": self.updated_at
+        }

watcher/updater.py

@@ -0,0 +1,163 @@
+"""
+Project Continuum - Incremental State Updater
+==============================================
+Milestone 7 - Phase 16: Filesystem Observation & Incremental State Updates.
+Applies detected workspace changes to CanonicalProjectState and the Canonical State Graph,
+re-extracting only affected evidence and propagating invalidations.
+"""
+
+from datetime import datetime, timezone
+from pathlib import Path
+import time
+from typing import Any, Dict, List, Optional, Set
+
+from core.enums import NodeType, RelationType, Status
+from core.state_models import (
+    AstSymbol,
+    CanonicalProjectState,
+    GraphNode,
+    ProjectState,
+)
+from extractors.parsers.python_parser import PythonParser
+from extractors.parsers.js_ts_parser import JavaScriptTypeScriptParser
+from extractors.parsers.comment_parser import CommentMarkerParser
+from graph.manager import StateGraphManager
+from watcher.models import ChangeType, FileChangeEvent, IncrementalUpdateResult
+
+
+class IncrementalStateUpdater:
+    """
+    Applies fine-grained file changes to a CanonicalProjectState, updating
+    symbols, files, and propagating invalidations through the StateGraph.
+    """
+
+    MAX_CHANGES_BEFORE_FALLBACK_FULL_SCAN = 50
+
+    def __init__(self, workspace_root: str):
+        self.workspace_root = Path(workspace_root).resolve()
+        self._python_parser = PythonParser()
+        self._js_ts_parser = JavaScriptTypeScriptParser()
+
+    def apply_changes(
+        self,
+        canonical_state: CanonicalProjectState,
+        changes: List[FileChangeEvent],
+        graph_manager: Optional[StateGraphManager] = None
+    ) -> IncrementalUpdateResult:
+        """
+        Updates canonical_state in-place based on the list of file change events.
+        """
+        start_time = time.time()
+        p_state = canonical_state.project_state
+
+        # Fallback to full scan if too many simultaneous modifications occur
+        if len(changes) > self.MAX_CHANGES_BEFORE_FALLBACK_FULL_SCAN:
+            duration = (time.time() - start_time) * 1000.0
+            return IncrementalUpdateResult(
+                workspace_root=str(self.workspace_root),
+                changes_processed=changes,
+                full_scan_fallback_triggered=True,
+                duration_ms=duration
+            )
+
+        re_extracted_symbols = 0
+        removed_symbols = 0
+        invalidated_nodes: Set[str] = set()
+        propagated_stale_nodes: Set[str] = set()
+
+        for change in changes:
+            rel_path = change.file_path.replace("\\", "/")
+            full_path = self.workspace_root / rel_path
+
+            if change.change_type == ChangeType.DELETED:
+                # 1. Remove file from files list
+                if rel_path in p_state.files:
+                    p_state.files.remove(rel_path)
+
+                # 2. Remove all symbols belonging to this file
+                old_sym_count = len(p_state.symbols)
+                p_state.symbols = [s for s in p_state.symbols if s.file_path != rel_path]
+                removed_symbols += (old_sym_count - len(p_state.symbols))
+
+                # 3. Find associated graph node and mark STALE
+                node_id = self._find_node_for_file(canonical_state, rel_path)
+                if node_id:
+                    invalidated_nodes.add(node_id)
+
+            elif change.change_type in (ChangeType.CREATED, ChangeType.MODIFIED):
+                # 1. Ensure file is in files list
+                if rel_path not in p_state.files:
+                    p_state.files.append(rel_path)
+
+                # 2. Remove previous symbols for this file
+                old_sym_count = len(p_state.symbols)
+                p_state.symbols = [s for s in p_state.symbols if s.file_path != rel_path]
+                removed_symbols += (old_sym_count - len(p_state.symbols))
+
+                # 3. Re-extract symbols from file
+                new_symbols = self._extract_file_symbols(full_path, rel_path)
+                p_state.symbols.extend(new_symbols)
+                re_extracted_symbols += len(new_symbols)
+
+                # 4. Find associated graph node
+                node_id = self._find_node_for_file(canonical_state, rel_path)
+                if node_id:
+                    invalidated_nodes.add(node_id)
+
+        # Invalidate graph nodes and propagate STALE downstream if graph_manager provided
+        if graph_manager:
+            for n_id in invalidated_nodes:
+                if n_id in canonical_state.graph_nodes:
+                    node = canonical_state.graph_nodes[n_id]
+                    node.status = Status.STALE
+                    affected = graph_manager.propagate_invalidation(n_id)
+                    propagated_stale_nodes.update(affected)
+
+        canonical_state.updated_at = datetime.now(timezone.utc).isoformat()
+        p_state.last_scanned_at = datetime.now(timezone.utc).isoformat()
+
+        duration = (time.time() - start_time) * 1000.0
+        return IncrementalUpdateResult(
+            workspace_root=str(self.workspace_root),
+            changes_processed=changes,
+            re_extracted_symbols_count=re_extracted_symbols,
+            removed_symbols_count=removed_symbols,
+            invalidated_node_ids=sorted(list(invalidated_nodes)),
+            propagated_stale_node_ids=sorted(list(propagated_stale_nodes)),
+            full_scan_fallback_triggered=False,
+            duration_ms=duration
+        )
+
+    def _find_node_for_file(self, state: CanonicalProjectState, rel_path: str) -> Optional[str]:
+        """Finds a graph node that directly references the given file."""
+        file_stem = Path(rel_path).stem.lower()
+        for node_id, node in state.graph_nodes.items():
+            if node_id.lower() == file_stem or node.name.lower() == file_stem:
+                return node_id
+            if node.metadata.get("file_path") == rel_path:
+                return node_id
+        return None
+
+    def _extract_file_symbols(self, full_path: Path, rel_path: str) -> List[AstSymbol]:
+        """Extracts AST symbols for a single file."""
+        if not full_path.exists():
+            return []
+
+        ext = full_path.suffix.lower()
+        try:
+            content = full_path.read_text(encoding="utf-8", errors="replace")
+        except Exception:
+            return []
+
+        symbols: List[AstSymbol] = []
+        if ext == ".py":
+            res = self._python_parser.parse_source(str(full_path), content)
+            symbols.extend(res.symbols)
+        elif ext in (".js", ".ts", ".jsx", ".tsx"):
+            res = self._js_ts_parser.parse_source(str(full_path), content)
+            symbols.extend(res.symbols)
+
+        # Standardize relative paths
+        for s in symbols:
+            s.file_path = rel_path
+        return symbols