continuum-toolkit 1.0.0__py3-none-any.whl

extractors/verification_extractor.py

@@ -0,0 +1,263 @@
+"""
+Project Continuum - Test, Build & Verification Evidence Extractor
+=================================================================
+Milestone 2 - Phase 4: Test, Build & Verification Evidence Collection.
+Discovers available test frameworks, explicitly executes configured verification
+commands, records exit codes, durations, parsed outcomes, and registers
+highest-priority Level 1 canonical Evidence.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from core.enums import EvidenceType, EvidenceLevel, Status
+from core.evidence import Evidence
+from core.state_models import (
+    CanonicalProjectState,
+    ManifestInfo,
+    ProjectState,
+    TestResult,
+)
+from extractors.base import BaseEvidenceExtractor
+from extractors.verification.runners import (
+    ExecutionResult,
+    ParsedTestSummary,
+    TestOutputParser,
+    VerificationRunner,
+)
+
+
+class VerificationEvidenceExtractor(BaseEvidenceExtractor):
+    """
+    Extractor responsible for Phase 4: Verification, Test, and Build Evidence.
+    Adheres strictly to the rule: Never run scripts automatically; only discover
+    configurations or run when explicitly requested.
+    """
+
+    @property
+    def extractor_name(self) -> str:
+        return "VerificationEvidenceExtractor"
+
+    @property
+    def supported_evidence_types(self) -> List[EvidenceType]:
+        return [
+            EvidenceType.TEST_RUN,
+            EvidenceType.BUILD_LOG,
+            EvidenceType.RUNTIME_LOG,
+        ]
+
+    def can_extract(self, target_path_or_input: str) -> bool:
+        path = Path(target_path_or_input)
+        return path.exists()
+
+    def discover_verification_configs(self, root_path: Path, manifests: Optional[List[ManifestInfo]] = None) -> List[Dict[str, Any]]:
+        """
+        Scans project directory and manifests for available test and build commands
+        WITHOUT automatically executing any of them.
+        """
+        discovered: List[Dict[str, Any]] = []
+
+        # 1. Check Python tools
+        if (root_path / "pyproject.toml").exists() or (root_path / "pytest.ini").exists() or (root_path / "tests").exists():
+            discovered.append({
+                "framework": "pytest",
+                "type": "test",
+                "recommended_command": "pytest",
+                "config_file": "pytest.ini" if (root_path / "pytest.ini").exists() else "pyproject.toml"
+            })
+            discovered.append({
+                "framework": "unittest",
+                "type": "test",
+                "recommended_command": "py -m unittest discover",
+                "config_file": None
+            })
+
+        # 2. Check JavaScript / TypeScript (package.json scripts)
+        pkg_json = root_path / "package.json"
+        if pkg_json.exists():
+            try:
+                import json
+                data = json.loads(pkg_json.read_text(encoding="utf-8"))
+                scripts = data.get("scripts", {})
+                if "test" in scripts:
+                    discovered.append({
+                        "framework": "npm_test",
+                        "type": "test",
+                        "recommended_command": "npm test",
+                        "script_body": scripts["test"]
+                    })
+                if "build" in scripts:
+                    discovered.append({
+                        "framework": "npm_build",
+                        "type": "build",
+                        "recommended_command": "npm run build",
+                        "script_body": scripts["build"]
+                    })
+            except Exception:
+                pass
+
+        # 3. Check Rust
+        if (root_path / "Cargo.toml").exists():
+            discovered.append({
+                "framework": "cargo_test",
+                "type": "test",
+                "recommended_command": "cargo test",
+                "config_file": "Cargo.toml"
+            })
+            discovered.append({
+                "framework": "cargo_build",
+                "type": "build",
+                "recommended_command": "cargo build",
+                "config_file": "Cargo.toml"
+            })
+
+        # 4. Check Go
+        if (root_path / "go.mod").exists():
+            discovered.append({
+                "framework": "go_test",
+                "type": "test",
+                "recommended_command": "go test ./...",
+                "config_file": "go.mod"
+            })
+            discovered.append({
+                "framework": "go_build",
+                "type": "build",
+                "recommended_command": "go build ./...",
+                "config_file": "go.mod"
+            })
+
+        return discovered
+
+    def run_verification(
+        self,
+        root_path: Path | str,
+        command: str | List[str],
+        suite_name: str = "default_suite",
+        is_build: bool = False,
+        timeout_seconds: int = 60
+    ) -> Tuple[ExecutionResult, ParsedTestSummary, Evidence]:
+        """
+        Explicitly executes a verification command and produces Level 1 canonical Evidence.
+        """
+        root = Path(root_path).resolve()
+        exec_result = VerificationRunner.run_command(command, cwd=root, timeout_seconds=timeout_seconds)
+
+        parsed_summary = TestOutputParser.parse(
+            stdout=exec_result.stdout,
+            stderr=exec_result.stderr,
+            exit_code=exec_result.exit_code,
+            duration_ms=exec_result.duration_ms
+        )
+
+        status = Status.VERIFIED if exec_result.is_success else Status.FAILED
+        evidence_type = EvidenceType.BUILD_LOG if is_build else EvidenceType.TEST_RUN
+
+        summary_text = (
+            f"{'Build' if is_build else 'Test'} suite '{suite_name}' {status.value} (exit code {exec_result.exit_code}) "
+            f"in {exec_result.duration_ms:.2f}ms. "
+            f"Results: {parsed_summary.passed} passed, {parsed_summary.failed} failed, {parsed_summary.errors} errors."
+        )
+
+        raw_payload = {
+            "suite_name": suite_name,
+            "is_build": is_build,
+            "command": exec_result.command,
+            "status": status.value,
+            "exit_code": exec_result.exit_code,
+            "duration_ms": exec_result.duration_ms,
+            "timed_out": exec_result.timed_out,
+            "error": exec_result.error,
+            "summary": {
+                "framework": parsed_summary.framework,
+                "total": parsed_summary.total,
+                "passed": parsed_summary.passed,
+                "failed": parsed_summary.failed,
+                "errors": parsed_summary.errors,
+                "skipped": parsed_summary.skipped,
+            },
+            "failure_details": parsed_summary.failure_details,
+            "stdout_snippet": exec_result.stdout[-1500:] if exec_result.stdout else "",
+            "stderr_snippet": exec_result.stderr[-1500:] if exec_result.stderr else ""
+        }
+
+        evidence = self.create_evidence(
+            evidence_type=evidence_type,
+            summary=summary_text,
+            raw_payload=raw_payload,
+            source_uri=str(root),
+            locator=f"{'build' if is_build else 'test'}:{suite_name}"
+        )
+
+        return exec_result, parsed_summary, evidence
+
+    def extract(self, target_path_or_input: str, context: Optional[Dict[str, Any]] = None) -> List[Evidence]:
+        """
+        Standard extract method: If context contains 'explicit_commands', executes them.
+        Otherwise, extracts discovery metadata only.
+        """
+        root_path = Path(target_path_or_input).resolve()
+        evidence_list: List[Evidence] = []
+
+        if context and "commands" in context:
+            for cmd_info in context["commands"]:
+                cmd = cmd_info.get("command")
+                suite = cmd_info.get("suite", "verification")
+                is_build = cmd_info.get("is_build", False)
+                if cmd:
+                    _, _, ev = self.run_verification(root_path, cmd, suite_name=suite, is_build=is_build)
+                    evidence_list.append(ev)
+        else:
+            # Only record discovery of available configurations
+            configs = self.discover_verification_configs(root_path)
+            evidence_list.append(self.create_evidence(
+                evidence_type=EvidenceType.TEST_RUN,
+                summary=f"Discovered {len(configs)} available verification and build configurations (auto-execution disabled)",
+                raw_payload={"discovered_configs": configs},
+                source_uri=str(root_path),
+                locator="verification:discovery"
+            ))
+
+        return evidence_list
+
+    def populate_project_state(
+        self,
+        root_dir: str,
+        canonical_state: CanonicalProjectState,
+        evidence_list: Optional[List[Evidence]] = None
+    ) -> ProjectState:
+        """
+        Registers verification evidence and updates ProjectState.test_results and build_status.
+        """
+        root_path = Path(root_dir).resolve()
+        ev_list = evidence_list or self.extract(str(root_path))
+
+        test_results: List[TestResult] = []
+        build_status = canonical_state.project_state.build_status
+
+        for ev in ev_list:
+            canonical_state.add_evidence(ev)
+
+            if ev.type == EvidenceType.TEST_RUN and "exit_code" in ev.raw_payload:
+                suite = ev.raw_payload.get("suite_name", "test_suite")
+                status = Status(ev.raw_payload.get("status", Status.FAILED.value))
+                tr = TestResult(
+                    test_id=f"tr_{ev.id}",
+                    name=suite,
+                    suite=suite,
+                    status=status,
+                    exit_code=ev.raw_payload.get("exit_code", 1),
+                    duration_ms=ev.raw_payload.get("duration_ms", 0.0),
+                    output_snippet=ev.raw_payload.get("stdout_snippet", ""),
+                    error_message=ev.raw_payload.get("error") or "\n".join(ev.raw_payload.get("failure_details", [])),
+                    evidence_id=ev.id
+                )
+                test_results.append(tr)
+            elif ev.type == EvidenceType.BUILD_LOG and "exit_code" in ev.raw_payload:
+                build_status = Status(ev.raw_payload.get("status", Status.FAILED.value))
+
+        ps = canonical_state.project_state
+        ps.test_results.extend(test_results)
+        ps.build_status = build_status
+        ps.evidence_ids.extend([ev.id for ev in ev_list])
+
+        return ps

extractors/workspace_extractor.py

@@ -0,0 +1,221 @@
+"""
+Project Continuum - Workspace & Source Code Evidence Extractor
+==============================================================
+Milestone 2 - Phase 1: Physical Workspace & Source Code Inspection.
+Scans the project directory, classifies files, detects programming languages,
+runs language parsers to extract AST symbols and TODO markers, handles malformed
+files gracefully, and produces canonical Evidence records.
+"""
+
+from collections import Counter
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+from core.enums import EvidenceType, Status
+from core.evidence import Evidence
+from core.state_models import (
+    AstSymbol,
+    CanonicalProjectState,
+    ProjectState,
+)
+from extractors.base import (
+    CONFIG_FILE_NAMES,
+    DEFAULT_IGNORE_DIRS,
+    DOC_EXTENSIONS,
+    BaseEvidenceExtractor,
+)
+from extractors.parsers.base import ILanguageParser, ParserResult
+from extractors.parsers.js_ts_parser import JavaScriptTypeScriptParser
+from extractors.parsers.python_parser import PythonParser
+
+
+class WorkspaceEvidenceExtractor(BaseEvidenceExtractor):
+    """
+    Primary extractor for Phase 1.
+    Performs physical filesystem scanning, language detection, and symbol parsing.
+    """
+
+    def __init__(self, custom_parsers: Optional[List[ILanguageParser]] = None):
+        # Register default language parsers
+        self.parsers: Dict[str, ILanguageParser] = {}
+        default_parsers = [PythonParser(), JavaScriptTypeScriptParser()]
+        if custom_parsers:
+            default_parsers.extend(custom_parsers)
+
+        for parser in default_parsers:
+            for ext in parser.supported_extensions:
+                self.parsers[ext.lower()] = parser
+
+    @property
+    def extractor_name(self) -> str:
+        return "WorkspaceEvidenceExtractor"
+
+    @property
+    def supported_evidence_types(self) -> List[EvidenceType]:
+        return [
+            EvidenceType.SOURCE_CODE,
+            EvidenceType.AST_SYMBOL,
+            EvidenceType.DOCUMENTATION,
+            EvidenceType.CONFIG_FILE
+        ]
+
+    def can_extract(self, target_path_or_input: str) -> bool:
+        path = Path(target_path_or_input)
+        return path.exists() and path.is_dir()
+
+    def extract(self, target_path_or_input: str, context: Optional[Dict[str, Any]] = None) -> List[Evidence]:
+        """
+        Scans workspace directory, parses files, and returns a list of canonical Evidence records.
+        """
+        root_path = Path(target_path_or_input).resolve()
+        evidence_list: List[Evidence] = []
+
+        scanned_files, lang_counts = self._scan_directory(root_path)
+
+        # 1. Workspace Structure Evidence
+        structure_payload = {
+            "root_path": str(root_path),
+            "total_files": len(scanned_files),
+            "languages": dict(lang_counts),
+            "file_list": [str(p.relative_to(root_path).as_posix()) for p in scanned_files]
+        }
+        evidence_list.append(self.create_evidence(
+            evidence_type=EvidenceType.SOURCE_CODE,
+            summary=f"Workspace structure scanned: {len(scanned_files)} files across {len(lang_counts)} languages",
+            raw_payload=structure_payload,
+            source_uri=str(root_path),
+            locator="workspace:root"
+        ))
+
+        # 2. Parse Source Code Files & Generate Symbol Evidence
+        for file_path in scanned_files:
+            ext = file_path.suffix.lower()
+            rel_path = file_path.relative_to(root_path).as_posix()
+
+            if ext in self.parsers:
+                parser = self.parsers[ext]
+                try:
+                    content = file_path.read_text(encoding="utf-8", errors="replace")
+                except Exception as e:
+                    continue
+
+                parse_result = parser.parse_source(rel_path, content)
+
+                # Record AST Symbol evidence
+                if parse_result.symbols:
+                    evidence_list.append(self.create_evidence(
+                        evidence_type=EvidenceType.AST_SYMBOL,
+                        summary=f"Extracted {len(parse_result.symbols)} symbols from {rel_path}",
+                        raw_payload={
+                            "file_path": rel_path,
+                            "language": parse_result.language,
+                            "symbols": [s.to_dict() for s in parse_result.symbols],
+                            "imports": [i.to_dict() for i in parse_result.imports],
+                            "exported_symbols": parse_result.exported_symbols,
+                            "markers": [m.to_dict() for m in parse_result.markers],
+                            "errors": [e.to_dict() for e in parse_result.errors]
+                        },
+                        source_uri=rel_path,
+                        locator=f"file:{rel_path}"
+                    ))
+            # Record Documentation Evidence
+            elif ext in DOC_EXTENSIONS:
+                try:
+                    content = file_path.read_text(encoding="utf-8", errors="replace")
+                    evidence_list.append(self.create_evidence(
+                        evidence_type=EvidenceType.DOCUMENTATION,
+                        summary=f"Documentation file {rel_path} ({len(content.splitlines())} lines)",
+                        raw_payload={
+                            "file_path": rel_path,
+                            "content": content
+                        },
+                        source_uri=rel_path,
+                        locator=f"doc:{rel_path}"
+                    ))
+                except Exception:
+                    pass
+
+        return evidence_list
+
+    def populate_project_state(self, root_dir: str, canonical_state: CanonicalProjectState) -> ProjectState:
+        """
+        High-level convenience method: scans workspace, updates `canonical_state.project_state`,
+        and registers all harvested evidence into `canonical_state.evidence_pool`.
+        """
+        root_path = Path(root_dir).resolve()
+        evidence_list = self.extract(str(root_path))
+
+        all_symbols: List[AstSymbol] = []
+        all_markers: List[Dict[str, Any]] = []
+        scanned_files: List[str] = []
+        lang_counter = Counter()
+
+        for ev in evidence_list:
+            canonical_state.add_evidence(ev)
+
+            if ev.type == EvidenceType.SOURCE_CODE:
+                scanned_files = ev.raw_payload.get("file_list", [])
+                for lang, count in ev.raw_payload.get("languages", {}).items():
+                    lang_counter[lang] += count
+            elif ev.type == EvidenceType.AST_SYMBOL:
+                for s_dict in ev.raw_payload.get("symbols", []):
+                    sym = AstSymbol.from_dict(s_dict)
+                    sym.evidence_id = ev.id
+                    all_symbols.append(sym)
+                all_markers.extend(ev.raw_payload.get("markers", []))
+
+        # Update ProjectState
+        ps = canonical_state.project_state
+        ps.root_path = str(root_path)
+        ps.files = scanned_files
+        ps.detected_languages = [lang for lang, _ in lang_counter.most_common()]
+        ps.symbols = all_symbols
+        ps.todo_markers = all_markers
+        ps.evidence_ids = [ev.id for ev in evidence_list]
+
+        return ps
+
+    def _scan_directory(self, root_path: Path) -> tuple[List[Path], Counter]:
+        """Recursively traverses directory ignoring standard irrelevant folders."""
+        scanned_files: List[Path] = []
+        lang_counter = Counter()
+
+        for path in sorted(root_path.rglob("*")):
+            if path.is_file():
+                # Check if any parent part is in ignored list
+                parts = set(path.relative_to(root_path).parts)
+                if any(ignored in parts for ignored in DEFAULT_IGNORE_DIRS):
+                    continue
+
+                scanned_files.append(path)
+                ext = path.suffix.lower()
+                lang = self._classify_extension(ext, path.name)
+                if lang:
+                    lang_counter[lang] += 1
+
+        return scanned_files, lang_counter
+
+    def _classify_extension(self, ext: str, filename: str) -> Optional[str]:
+        if filename in CONFIG_FILE_NAMES:
+            return "Configuration"
+        if ext in {".py", ".pyw"}:
+            return "Python"
+        if ext in {".js", ".jsx", ".mjs", ".cjs"}:
+            return "JavaScript"
+        if ext in {".ts", ".tsx"}:
+            return "TypeScript"
+        if ext in {".go"}:
+            return "Go"
+        if ext in {".rs"}:
+            return "Rust"
+        if ext in {".java"}:
+            return "Java"
+        if ext in {".c", ".h"}:
+            return "C"
+        if ext in {".cpp", ".hpp", ".cc"}:
+            return "C++"
+        if ext in {".dart"}:
+            return "Dart"
+        if ext in DOC_EXTENSIONS:
+            return "Documentation"
+        return None

graph/__init__.py

@@ -0,0 +1,24 @@
+"""
+Project Continuum - State Graph Subsystem Package
+=================================================
+Milestone 4 - Phases 9, 10 & 11.
+"""
+
+from graph.models import GraphStats, GraphValidationResult, PropagationResult
+from graph.snapshot import GraphSnapshot
+from graph.diff import GraphDiff, NodeDelta
+from graph.query import GraphQueryEngine
+from graph.propagator import DependencyPropagator
+from graph.manager import StateGraphManager
+
+__all__ = [
+    "GraphStats",
+    "GraphValidationResult",
+    "PropagationResult",
+    "GraphSnapshot",
+    "GraphDiff",
+    "NodeDelta",
+    "GraphQueryEngine",
+    "DependencyPropagator",
+    "StateGraphManager",
+]

graph/diff.py

@@ -0,0 +1,109 @@
+"""
+Project Continuum - Graph Comparison & Semantic Diff Engine
+===========================================================
+Milestone 4 - Phase 11: Graph Persistence, Querying & Visualization.
+Compares two graph snapshots or StateGraphManager instances to compute fine-grained diffs.
+"""
+
+from dataclasses import dataclass, field, asdict
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from graph.snapshot import GraphSnapshot
+
+
+@dataclass
+class NodeDelta:
+    """Detailed attribute changes for a modified graph node."""
+    old_status: Optional[str] = None
+    new_status: Optional[str] = None
+    old_confidence: Optional[float] = None
+    new_confidence: Optional[float] = None
+    old_evidence_count: int = 0
+    new_evidence_count: int = 0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class GraphDiff:
+    """
+    Semantic difference between two graph snapshots (Baseline -> Target).
+    """
+    baseline_id: str
+    target_id: str
+    added_nodes: List[str] = field(default_factory=list)
+    removed_nodes: List[str] = field(default_factory=list)
+    modified_nodes: Dict[str, NodeDelta] = field(default_factory=dict)
+    added_edges: List[Dict[str, str]] = field(default_factory=list)
+    removed_edges: List[Dict[str, str]] = field(default_factory=list)
+    has_changes: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "baseline_id": self.baseline_id,
+            "target_id": self.target_id,
+            "has_changes": self.has_changes,
+            "added_nodes": self.added_nodes,
+            "removed_nodes": self.removed_nodes,
+            "modified_nodes": {k: v.to_dict() for k, v in self.modified_nodes.items()},
+            "added_edges": self.added_edges,
+            "removed_edges": self.removed_edges,
+        }
+
+    @classmethod
+    def compare(cls, snap_a: GraphSnapshot, snap_b: GraphSnapshot) -> "GraphDiff":
+        """
+        Computes the semantic delta from snap_a to snap_b.
+        """
+        nodes_a = {n["id"]: n for n in snap_a.nodes}
+        nodes_b = {n["id"]: n for n in snap_b.nodes}
+
+        ids_a = set(nodes_a.keys())
+        ids_b = set(nodes_b.keys())
+
+        added_node_ids = sorted(list(ids_b - ids_a))
+        removed_node_ids = sorted(list(ids_a - ids_b))
+        common_node_ids = ids_a & ids_b
+
+        modified_nodes: Dict[str, NodeDelta] = {}
+        for nid in sorted(list(common_node_ids)):
+            na = nodes_a[nid]
+            nb = nodes_b[nid]
+
+            stat_changed = na.get("status") != nb.get("status")
+            conf_changed = na.get("confidence_score") != nb.get("confidence_score")
+            ev_changed = len(na.get("evidence_ids", [])) != len(nb.get("evidence_ids", []))
+
+            if stat_changed or conf_changed or ev_changed:
+                modified_nodes[nid] = NodeDelta(
+                    old_status=na.get("status"),
+                    new_status=nb.get("status"),
+                    old_confidence=na.get("confidence_score"),
+                    new_confidence=nb.get("confidence_score"),
+                    old_evidence_count=len(na.get("evidence_ids", [])),
+                    new_evidence_count=len(nb.get("evidence_ids", []))
+                )
+
+        # Compare edges
+        edges_a = {(e["source_id"], e["target_id"], e["relation"]) for e in snap_a.edges}
+        edges_b = {(e["source_id"], e["target_id"], e["relation"]) for e in snap_b.edges}
+
+        added_edges_raw = edges_b - edges_a
+        removed_edges_raw = edges_a - edges_b
+
+        added_edges = [{"source_id": s, "target_id": t, "relation": r} for s, t, r in sorted(list(added_edges_raw))]
+        removed_edges = [{"source_id": s, "target_id": t, "relation": r} for s, t, r in sorted(list(removed_edges_raw))]
+
+        has_changes = bool(added_node_ids or removed_node_ids or modified_nodes or added_edges or removed_edges)
+
+        return cls(
+            baseline_id=snap_a.snapshot_id,
+            target_id=snap_b.snapshot_id,
+            added_nodes=added_node_ids,
+            removed_nodes=removed_node_ids,
+            modified_nodes=modified_nodes,
+            added_edges=added_edges,
+            removed_edges=removed_edges,
+            has_changes=has_changes
+        )