continuum-toolkit 1.0.0__py3-none-any.whl

daemon/service.py

@@ -0,0 +1,170 @@
+"""
+Project Continuum - Daemon Service & Workspace Observer
+========================================================
+Milestone 7 - Phase 18: Continuum Daemon & CLI.
+Implements the continuous background monitoring service that synchronizes
+workspace changes into persistent Canonical State.
+"""
+
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+import threading
+import time
+from typing import Any, Dict, Optional
+
+from core.serializer import CanonicalStateSerializer
+from core.state_models import CanonicalProjectState
+from storage.manager import ContinuumStorageManager
+from storage.recovery import StateRecoveryManager
+from watcher.detector import WorkspaceChangeDetector
+from watcher.updater import IncrementalStateUpdater
+from graph.manager import StateGraphManager
+
+
+@dataclass
+class DaemonStatus:
+    is_running: bool
+    pid: Optional[int] = None
+    workspace_root: str = ""
+    started_at: Optional[str] = None
+    last_tick_at: Optional[str] = None
+    cycles_completed: int = 0
+    total_changes_processed: int = 0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+class ContinuumDaemon:
+    """
+    Background daemon service monitoring workspace filesystem events,
+    applying incremental state updates, and persisting to .continuum/.
+    """
+
+    def __init__(
+        self,
+        workspace_root: str,
+        poll_interval_seconds: float = 1.0
+    ):
+        self.workspace_root = Path(workspace_root).resolve()
+        self.poll_interval = poll_interval_seconds
+        self._storage = ContinuumStorageManager(str(self.workspace_root))
+        self._recovery = StateRecoveryManager(self._storage)
+        self._detector = WorkspaceChangeDetector(str(self.workspace_root))
+        self._updater = IncrementalStateUpdater(str(self.workspace_root))
+
+        self._pid_file = self._storage.continuum_dir / "daemon.pid"
+        self._status_file = self._storage.continuum_dir / "daemon.json"
+        self._stop_event = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+
+        self._cycles = 0
+        self._total_changes = 0
+        self._started_at: Optional[str] = None
+
+    def start(self, run_in_background: bool = False) -> DaemonStatus:
+        """Starts daemon monitoring loop."""
+        self._storage.initialize_storage()
+        self._detector.initialize_baseline()
+        self._stop_event.clear()
+        self._started_at = datetime.now(timezone.utc).isoformat()
+
+        # Write PID file
+        self._pid_file.write_text(str(os.getpid()), encoding="utf-8")
+
+        if run_in_background:
+            self._thread = threading.Thread(target=self._run_loop, daemon=True)
+            self._thread.start()
+        else:
+            self._run_loop()
+
+        return self.get_status()
+
+    def stop(self) -> DaemonStatus:
+        """Stops the daemon gracefully."""
+        self._stop_event.set()
+        if self._thread and self._thread.is_alive():
+            self._thread.join(timeout=3.0)
+
+        if self._pid_file.exists():
+            try:
+                self._pid_file.unlink()
+            except OSError:
+                pass
+
+        return self.get_status()
+
+    def run_once(self) -> Dict[str, Any]:
+        """
+        Executes a single check and synchronization cycle.
+        """
+        self._storage.initialize_storage()
+
+        # Load or initialize state
+        if self._storage.state_exists():
+            try:
+                state = self._storage.load_state(validate=True)
+            except Exception:
+                rec_res = self._recovery.recover_corrupted_state()
+                if rec_res.success:
+                    state = self._storage.load_state(validate=True)
+                else:
+                    state = CanonicalProjectState(project_id="proj_auto_init")
+        else:
+            state = CanonicalProjectState(project_id="proj_auto_init")
+
+        # Detect changes
+        changes = self._detector.detect_changes()
+        result_info = {"changes_detected": len(changes), "updated": False}
+
+        if changes:
+            graph_manager = StateGraphManager(state)
+            update_res = self._updater.apply_changes(state, changes, graph_manager)
+            self._storage.save_state(state, create_history_snapshot=True)
+            self._total_changes += len(changes)
+            result_info["updated"] = True
+            result_info["details"] = update_res.to_dict()
+
+        self._cycles += 1
+        self._write_status_file()
+        return result_info
+
+    def get_status(self) -> DaemonStatus:
+        """Returns current daemon operational status."""
+        is_running = not self._stop_event.is_set() and (self._thread.is_alive() if self._thread else False)
+        pid = None
+        if self._pid_file.exists():
+            try:
+                pid = int(self._pid_file.read_text().strip())
+            except Exception:
+                pass
+
+        return DaemonStatus(
+            is_running=is_running,
+            pid=pid,
+            workspace_root=str(self.workspace_root),
+            started_at=self._started_at,
+            last_tick_at=datetime.now(timezone.utc).isoformat(),
+            cycles_completed=self._cycles,
+            total_changes_processed=self._total_changes
+        )
+
+    def _run_loop(self) -> None:
+        """Internal worker loop."""
+        while not self._stop_event.is_set():
+            try:
+                self.run_once()
+            except Exception:
+                pass
+            time.sleep(self.poll_interval)
+
+    def _write_status_file(self) -> None:
+        """Persists daemon heartbeat to .continuum/daemon.json."""
+        status = self.get_status()
+        try:
+            self._status_file.write_text(json.dumps(status.to_dict(), indent=2), encoding="utf-8")
+        except Exception:
+            pass

extractors/__init__.py

@@ -0,0 +1,51 @@
+"""
+Project Continuum - Extractors Subsystem Package Initialization
+"""
+
+from extractors.base import BaseEvidenceExtractor, DEFAULT_IGNORE_DIRS, CONFIG_FILE_NAMES, DOC_EXTENSIONS
+from extractors.workspace_extractor import WorkspaceEvidenceExtractor
+from extractors.config_extractor import ConfigEvidenceExtractor, LOCKFILE_NAMES
+from extractors.git_extractor import GitEvidenceExtractor
+from extractors.verification_extractor import VerificationEvidenceExtractor
+from extractors.conversation_extractor import ConversationEvidenceExtractor
+from extractors.conversation.models import ConversationTranscript, ConversationTurn, TurnRole
+from extractors.conversation.analyzers import TranscriptAnalyzer, TranscriptAnalysisResult
+from extractors.verification.runners import VerificationRunner, TestOutputParser, ExecutionResult, ParsedTestSummary
+from extractors.config.secret_sanitizer import SecretSanitizer
+from extractors.config.parsers import ManifestParser
+from extractors.parsers.base import ILanguageParser, ParserResult, CommentMarker, ImportStatement, ParseError
+from extractors.parsers.comment_parser import CommentMarkerParser
+from extractors.parsers.python_parser import PythonParser
+from extractors.parsers.js_ts_parser import JavaScriptTypeScriptParser
+
+__all__ = [
+    "BaseEvidenceExtractor",
+    "WorkspaceEvidenceExtractor",
+    "ConfigEvidenceExtractor",
+    "GitEvidenceExtractor",
+    "VerificationEvidenceExtractor",
+    "ConversationEvidenceExtractor",
+    "ConversationTranscript",
+    "ConversationTurn",
+    "TurnRole",
+    "TranscriptAnalyzer",
+    "TranscriptAnalysisResult",
+    "VerificationRunner",
+    "TestOutputParser",
+    "ExecutionResult",
+    "ParsedTestSummary",
+    "SecretSanitizer",
+    "ManifestParser",
+    "DEFAULT_IGNORE_DIRS",
+    "CONFIG_FILE_NAMES",
+    "DOC_EXTENSIONS",
+    "LOCKFILE_NAMES",
+    "ILanguageParser",
+    "ParserResult",
+    "CommentMarker",
+    "ImportStatement",
+    "ParseError",
+    "CommentMarkerParser",
+    "PythonParser",
+    "JavaScriptTypeScriptParser",
+]

extractors/base.py

@@ -0,0 +1,117 @@
+"""
+Project Continuum - Base Evidence Extractor & Classification
+============================================================
+Defines the BaseEvidenceExtractor and workspace classification logic.
+"""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+from core.enums import EvidenceType, EvidenceLevel
+from core.evidence import Evidence, EvidenceProvenance
+from core.interfaces import IEvidenceExtractor
+
+
+# Standard directories to ignore during workspace scanning
+DEFAULT_IGNORE_DIRS: Set[str] = {
+    ".git",
+    ".hg",
+    ".svn",
+    "node_modules",
+    "__pycache__",
+    ".pytest_cache",
+    ".venv",
+    "venv",
+    "env",
+    ".idea",
+    ".vscode",
+    "dist",
+    "build",
+    ".egg-info",
+    "target",
+    ".dart_tool",
+    "coverage",
+    ".continuum"
+}
+
+# Standard configuration filenames
+CONFIG_FILE_NAMES: Set[str] = {
+    "package.json",
+    "pyproject.toml",
+    "setup.py",
+    "setup.cfg",
+    "Cargo.toml",
+    "go.mod",
+    "pom.xml",
+    "build.gradle",
+    "pubspec.yaml",
+    "Dockerfile",
+    "docker-compose.yml",
+    "docker-compose.yaml",
+    "Makefile",
+    "CMakeLists.txt",
+    "tsconfig.json"
+}
+
+# Documentation extensions
+DOC_EXTENSIONS: Set[str] = {
+    ".md",
+    ".markdown",
+    ".rst",
+    ".txt",
+    ".adoc"
+}
+
+
+class BaseEvidenceExtractor(ABC):
+    """
+    Abstract base class providing standard helper methods for all evidence extractors.
+    """
+
+    @property
+    @abstractmethod
+    def extractor_name(self) -> str:
+        """Unique identifier for this extractor."""
+        pass
+
+    @property
+    @abstractmethod
+    def supported_evidence_types(self) -> List[EvidenceType]:
+        """Evidence types this extractor produces."""
+        pass
+
+    @abstractmethod
+    def can_extract(self, target_path_or_input: str) -> bool:
+        """Determines if this extractor is applicable to the target."""
+        pass
+
+    @abstractmethod
+    def extract(self, target_path_or_input: str, context: Optional[Dict[str, Any]] = None) -> List[Evidence]:
+        """Harvests verifiable evidence from the target."""
+        pass
+
+    def create_evidence(
+        self,
+        evidence_type: EvidenceType,
+        summary: str,
+        raw_payload: Dict[str, Any],
+        source_uri: str,
+        locator: str,
+        metadata: Optional[Dict[str, Any]] = None
+    ) -> Evidence:
+        """Helper to create standardized, verified Evidence records with provenance."""
+        level = EvidenceLevel.get_level_for_type(evidence_type)
+        prov = EvidenceProvenance(
+            extractor_name=self.extractor_name,
+            source_uri=source_uri,
+            locator=locator
+        )
+        return Evidence(
+            type=evidence_type,
+            level=level,
+            summary=summary,
+            raw_payload=raw_payload,
+            provenance=prov,
+            metadata=metadata or {}
+        )

extractors/config/parsers.py

@@ -0,0 +1,288 @@
+"""
+Project Continuum - Configuration and Manifest Parsers
+======================================================
+Parsers for package.json, pyproject.toml, Cargo.toml, go.mod,
+pubspec.yaml, Dockerfile, docker-compose.yml, and environment templates.
+"""
+
+import json
+import re
+from typing import Any, Dict, List, Optional
+
+from core.state_models import ManifestInfo
+from extractors.config.secret_sanitizer import SecretSanitizer
+
+try:
+    import tomllib  # Python 3.11+ standard library
+except ImportError:
+    try:
+        import tomli as tomllib  # Python 3.10 with tomli
+    except ImportError:
+        tomllib = None  # Fallback handled via regex
+
+
+class ManifestParser:
+    """
+    Parses various software package manifests and project configuration files.
+    """
+
+    @staticmethod
+    def parse_package_json(file_path: str, content: str) -> ManifestInfo:
+        try:
+            data = json.loads(content)
+        except Exception:
+            return ManifestInfo(manifest_type="package.json", file_path=file_path)
+
+        dependencies = {str(k): str(v) for k, v in data.get("dependencies", {}).items()}
+        dev_dependencies = {str(k): str(v) for k, v in data.get("devDependencies", {}).items()}
+        scripts = {str(k): str(v) for k, v in data.get("scripts", {}).items()}
+
+        return ManifestInfo(
+            manifest_type="package.json",
+            file_path=file_path,
+            project_name=data.get("name"),
+            version=data.get("version"),
+            dependencies=SecretSanitizer.sanitize_dict(dependencies),
+            dev_dependencies=SecretSanitizer.sanitize_dict(dev_dependencies),
+            scripts=scripts
+        )
+
+    @staticmethod
+    def parse_pyproject_toml(file_path: str, content: str) -> ManifestInfo:
+        project_name = None
+        version = None
+        dependencies: Dict[str, str] = {}
+        dev_dependencies: Dict[str, str] = {}
+        scripts: Dict[str, str] = {}
+
+        if tomllib:
+            try:
+                data = tomllib.loads(content)
+                proj = data.get("project", {})
+                project_name = proj.get("name")
+                version = proj.get("version")
+
+                # Project dependencies
+                for dep in proj.get("dependencies", []):
+                    parts = re.split(r"[><=~^! ]+", dep, maxsplit=1)
+                    name = parts[0].strip()
+                    ver = dep[len(name):].strip() if len(parts) > 1 else "*"
+                    dependencies[name] = ver or "*"
+
+                # Optional/dev dependencies
+                for group, deps in proj.get("optional-dependencies", {}).items():
+                    for dep in deps:
+                        parts = re.split(r"[><=~^! ]+", dep, maxsplit=1)
+                        name = parts[0].strip()
+                        ver = dep[len(name):].strip() if len(parts) > 1 else "*"
+                        dev_dependencies[f"{group}:{name}"] = ver or "*"
+
+                # Scripts
+                scripts = {str(k): str(v) for k, v in proj.get("scripts", {}).items()}
+
+                # Poetry fallback if present
+                poetry = data.get("tool", {}).get("poetry", {})
+                if poetry:
+                    project_name = project_name or poetry.get("name")
+                    version = version or poetry.get("version")
+                    for k, v in poetry.get("dependencies", {}).items():
+                        if k != "python":
+                            dependencies[k] = str(v)
+                    for k, v in poetry.get("group", {}).get("dev", {}).get("dependencies", {}).items():
+                        dev_dependencies[k] = str(v)
+
+            except Exception:
+                pass
+        else:
+            # Pure Python fallback parser for Python 3.10 without tomli
+            name_match = re.search(r'name\s*=\s*["\']([^"\']+)["\']', content)
+            if name_match:
+                project_name = name_match.group(1)
+            ver_match = re.search(r'version\s*=\s*["\']([^"\']+)["\']', content)
+            if ver_match:
+                version = ver_match.group(1)
+            deps_match = re.search(r'dependencies\s*=\s*\[(.*?)\]', content, re.DOTALL)
+            if deps_match:
+                for dep in re.findall(r'["\']([^"\']+)["\']', deps_match.group(1)):
+                    parts = re.split(r"[><=~^! ]+", dep, maxsplit=1)
+                    name = parts[0].strip()
+                    ver = dep[len(name):].strip() if len(parts) > 1 else "*"
+                    dependencies[name] = ver or "*"
+
+        return ManifestInfo(
+            manifest_type="pyproject.toml",
+            file_path=file_path,
+            project_name=project_name,
+            version=version,
+            dependencies=dependencies,
+            dev_dependencies=dev_dependencies,
+            scripts=scripts
+        )
+
+    @staticmethod
+    def parse_cargo_toml(file_path: str, content: str) -> ManifestInfo:
+        project_name = None
+        version = None
+        dependencies: Dict[str, str] = {}
+        dev_dependencies: Dict[str, str] = {}
+
+        if tomllib:
+            try:
+                data = tomllib.loads(content)
+                pkg = data.get("package", {})
+                project_name = pkg.get("name")
+                version = pkg.get("version")
+
+                for k, v in data.get("dependencies", {}).items():
+                    dependencies[k] = str(v.get("version", "*")) if isinstance(v, dict) else str(v)
+
+                for k, v in data.get("dev-dependencies", {}).items():
+                    dev_dependencies[k] = str(v.get("version", "*")) if isinstance(v, dict) else str(v)
+            except Exception:
+                pass
+        else:
+            # Pure Python fallback parser for Python 3.10 without tomli
+            name_match = re.search(r'name\s*=\s*["\']([^"\']+)["\']', content)
+            if name_match:
+                project_name = name_match.group(1)
+            ver_match = re.search(r'version\s*=\s*["\']([^"\']+)["\']', content)
+            if ver_match:
+                version = ver_match.group(1)
+            in_deps = False
+            for line in content.splitlines():
+                stripped = line.strip()
+                if stripped.startswith("["):
+                    in_deps = (stripped == "[dependencies]")
+                    continue
+                if in_deps and "=" in stripped:
+                    k, v = stripped.split("=", 1)
+                    k = k.strip()
+                    v_match = re.search(r'["\']([^"\']+)["\']', v)
+                    dependencies[k] = v_match.group(1) if v_match else "*"
+
+        return ManifestInfo(
+            manifest_type="Cargo.toml",
+            file_path=file_path,
+            project_name=project_name,
+            version=version,
+            dependencies=dependencies,
+            dev_dependencies=dev_dependencies,
+            scripts={}
+        )
+
+    @staticmethod
+    def parse_go_mod(file_path: str, content: str) -> ManifestInfo:
+        module_name = None
+        go_version = None
+        dependencies: Dict[str, str] = {}
+
+        lines = content.splitlines()
+        for line in lines:
+            line = line.strip()
+            if line.startswith("module "):
+                module_name = line.replace("module ", "").strip()
+            elif line.startswith("go "):
+                go_version = line.replace("go ", "").strip()
+            elif not line.startswith("//") and not line.startswith("require") and not line.startswith(")") and not line.startswith("("):
+                parts = line.split()
+                if len(parts) >= 2:
+                    dependencies[parts[0]] = parts[1]
+
+        scripts = {"go_version": go_version} if go_version else {}
+
+        return ManifestInfo(
+            manifest_type="go.mod",
+            file_path=file_path,
+            project_name=module_name,
+            version=go_version,
+            dependencies=dependencies,
+            dev_dependencies={},
+            scripts=scripts
+        )
+
+    @staticmethod
+    def parse_pubspec_yaml(file_path: str, content: str) -> ManifestInfo:
+        project_name = None
+        version = None
+        dependencies: Dict[str, str] = {}
+        dev_dependencies: Dict[str, str] = {}
+
+        current_section = None
+        for line in content.splitlines():
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#"):
+                continue
+
+            if not line.startswith(" ") and ":" in stripped:
+                key, val = stripped.split(":", 1)
+                key = key.strip()
+                val = val.strip()
+                if key == "name":
+                    project_name = val
+                elif key == "version":
+                    version = val
+                elif key in {"dependencies", "dev_dependencies"}:
+                    current_section = key
+                else:
+                    current_section = None
+            elif current_section and line.startswith("  ") and ":" in stripped:
+                key, val = stripped.split(":", 1)
+                k = key.strip()
+                v = val.strip() or "*"
+                if current_section == "dependencies" and k != "flutter":
+                    dependencies[k] = v
+                elif current_section == "dev_dependencies":
+                    dev_dependencies[k] = v
+
+        return ManifestInfo(
+            manifest_type="pubspec.yaml",
+            file_path=file_path,
+            project_name=project_name,
+            version=version,
+            dependencies=dependencies,
+            dev_dependencies=dev_dependencies,
+            scripts={}
+        )
+
+    @staticmethod
+    def parse_dockerfile(file_path: str, content: str) -> Dict[str, Any]:
+        base_images = []
+        exposed_ports = []
+        entrypoints = []
+        cmd = None
+
+        for line in content.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("FROM "):
+                base_images.append(stripped.replace("FROM ", "").strip())
+            elif stripped.startswith("EXPOSE "):
+                ports = stripped.replace("EXPOSE ", "").split()
+                exposed_ports.extend(ports)
+            elif stripped.startswith("ENTRYPOINT "):
+                entrypoints.append(stripped.replace("ENTRYPOINT ", "").strip())
+            elif stripped.startswith("CMD "):
+                cmd = stripped.replace("CMD ", "").strip()
+
+        return {
+            "file_path": file_path,
+            "base_images": base_images,
+            "exposed_ports": exposed_ports,
+            "entrypoint": entrypoints[0] if entrypoints else None,
+            "cmd": cmd
+        }
+
+    @staticmethod
+    def parse_env_template(file_path: str, content: str) -> Dict[str, Any]:
+        """Extracts expected environment variable keys while redacting all values."""
+        variables: List[str] = []
+        for line in content.splitlines():
+            stripped = line.strip()
+            if stripped and not stripped.startswith("#") and "=" in stripped:
+                key, _ = stripped.split("=", 1)
+                variables.append(key.strip())
+
+        return {
+            "file_path": file_path,
+            "variable_keys": variables,
+            "sanitized": True
+        }