core-runtime 0.1.1__py3-none-any.whl

body/introspection/graph_analysis_service.py

@@ -0,0 +1,66 @@
+# src/body/introspection/graph_analysis_service.py
+
+"""
+Provides a service for finding semantic clusters of symbols in the codebase
+using K-Means clustering on their vector embeddings.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from shared.infrastructure.clients.qdrant_client import QdrantService
+from shared.infrastructure.intent.operational_config import load_operational_config
+from shared.logger import getLogger
+
+
+try:
+    from sklearn.cluster import KMeans
+except ImportError:
+    KMeans = None
+logger = getLogger(__name__)
+
+_CFG = load_operational_config().misc
+
+
+# ID: ae8922bb-df0c-4edb-a34f-a7114d70faab
+async def find_semantic_clusters(
+    qdrant_service: QdrantService,
+    n_clusters: int = _CFG.clustering_default_n_clusters,
+) -> list[list[str]]:
+    """
+    Finds clusters of semantically similar code symbols using K-Means clustering.
+    """
+    if KMeans is None:
+        logger.error(
+            "scikit-learn is not installed. Cannot perform clustering. Please run 'poetry install --with dev'."
+        )
+        return []
+    logger.info("Finding %s semantic clusters using K-Means...", n_clusters)
+    try:
+        all_points = await qdrant_service.get_all_vectors()
+        if not all_points:
+            logger.warning("No vectors found in Qdrant. Cannot perform clustering.")
+            return []
+        vectors = []
+        symbol_keys = []
+        for point in all_points:
+            if point.payload and "chunk_id" in point.payload and point.vector:
+                symbol_keys.append(point.payload["chunk_id"])
+                vectors.append(point.vector)
+        if not vectors:
+            logger.warning("No valid vectors with symbol payloads found.")
+            return []
+        logger.info("Clustering {len(vectors)} vectors into %s domains...", n_clusters)
+        vector_array = np.array(vectors, dtype=np.float32)
+        kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init="auto")
+        labels = kmeans.fit_predict(vector_array)
+        clusters: list[list[str]] = [[] for _ in range(n_clusters)]
+        for i, label in enumerate(labels):
+            clusters[label].append(symbol_keys[i])
+        logger.info("Found %s semantic clusters.", len(clusters))
+        clusters.sort(key=len, reverse=True)
+        return [c for c in clusters if c]
+    except Exception as e:
+        logger.error("Failed to find semantic clusters: %s", e, exc_info=True)
+        return []

body/introspection/knowledge_graph_service.py

@@ -0,0 +1,170 @@
+# src/body/introspection/knowledge_graph_service.py
+
+"""
+Knowledge Graph Builder - Sensory-Aware logic service.
+UPDATED: Domain heuristics adapted for layered architecture (Wave 3).
+"""
+
+from __future__ import annotations
+
+import ast
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from shared.ast_utility import (
+    FunctionCallVisitor,
+    calculate_structural_hash,
+    extract_base_classes,
+    extract_docstring,
+    extract_parameters,
+    parse_metadata_comment,
+)
+from shared.logger import getLogger
+
+
+if TYPE_CHECKING:
+    from shared.infrastructure.context.limb_workspace import LimbWorkspace
+
+logger = getLogger(__name__)
+
+
+# ID: 322f2e00-3996-4c30-8bc6-ec9291be777c
+class KnowledgeGraphBuilder:
+    """
+    Scan source code to build a comprehensive in-memory knowledge graph.
+    """
+
+    def __init__(self, root_path: Path, workspace: LimbWorkspace | None = None) -> None:
+        self.root_path = Path(root_path).resolve()
+        self.src_dir = self.root_path / "src"
+        self.workspace = workspace
+
+        self.symbols: dict[str, dict[str, Any]] = {}
+        self.domain_map = self._load_domain_map()
+        self.entry_point_patterns = self._load_entry_point_patterns()
+
+    def _load_domain_map(self) -> dict[str, str]:
+        """
+        Loads the architectural domain map.
+        Source structure file has been removed — returns empty map.
+        Callers fall back to heuristic domain detection.
+        """
+        return {}
+
+    def _load_entry_point_patterns(self) -> list[dict[str, Any]]:
+        """
+        Loads entry point patterns.
+        Patterns file has been removed — returns empty list.
+        """
+        return []
+
+    # ID: 75c969e0-5c7c-4f58-9a46-62815947d77a
+    def build(self) -> dict[str, Any]:
+        """
+        Execute the full scan and return the in-memory graph.
+        """
+        mode = "SHADOW" if self.workspace else "STANDARD"
+        logger.info("Building knowledge graph (%s mode) for: %s", mode, self.root_path)
+
+        self.symbols = {}
+
+        if self.workspace:
+            files_to_scan = self.workspace.list_files(directory="src", pattern="*.py")
+            for rel_path in files_to_scan:
+                self._scan_file_v2(rel_path)
+        else:
+            if not self.src_dir.exists():
+                logger.warning("Source directory not found: %s", self.src_dir)
+                return {"metadata": {}, "symbols": {}}
+
+            for py_file in self.src_dir.rglob("*.py"):
+                rel_path = str(py_file.relative_to(self.root_path))
+                self._scan_file_v2(rel_path)
+
+        return {
+            "metadata": {
+                "generated_at": datetime.now(UTC).isoformat(),
+                "repo_root": str(self.root_path),
+                "symbol_count": len(self.symbols),
+                "mode": mode,
+            },
+            "symbols": self.symbols,
+        }
+
+    def _scan_file_v2(self, rel_path: str) -> None:
+        """Scan a file using the sensory overlay."""
+        try:
+            if self.workspace:
+                content = self.workspace.read_text(rel_path)
+            else:
+                content = (self.root_path / rel_path).read_text(encoding="utf-8")
+
+            tree = ast.parse(content, filename=rel_path)
+            source_lines = content.splitlines()
+
+            for node in ast.walk(tree):
+                if isinstance(
+                    node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
+                ):
+                    self._process_symbol(node, Path(rel_path), source_lines)
+
+        except Exception as exc:
+            logger.error("Failed to process file %s: %s", rel_path, exc)
+
+    def _determine_domain(self, file_path: Path) -> str:
+        """Determine the architectural domain of a file."""
+        abs_file_path = (self.root_path / file_path).resolve()
+
+        parts = file_path.parts
+        if len(parts) >= 3 and parts[0] == "src":
+            return parts[2]
+
+        for domain_path, domain_name in self.domain_map.items():
+            if str(abs_file_path).startswith(str(Path(domain_path).resolve())):
+                return domain_name
+
+        return "unknown"
+
+    def _process_symbol(
+        self, node: ast.AST, rel_path: Path, source_lines: list[str]
+    ) -> None:
+        """Extract metadata for a symbol."""
+        if not hasattr(node, "name"):
+            return
+
+        symbol_path_key = f"{rel_path.as_posix()}::{node.name}"
+        metadata = parse_metadata_comment(node, source_lines) or {}
+        docstring = (extract_docstring(node) or "").strip()
+
+        call_visitor = FunctionCallVisitor()
+        try:
+            call_visitor.visit(node)
+        except Exception:
+            pass
+
+        symbol_data = {
+            "uuid": symbol_path_key,
+            "key": metadata.get("capability"),
+            "symbol_path": symbol_path_key,
+            "name": node.name,
+            "type": type(node).__name__,
+            "file_path": rel_path.as_posix(),
+            "domain": self._determine_domain(rel_path),
+            "is_public": not node.name.startswith("_"),
+            "title": node.name.replace("_", " ").title(),
+            "description": docstring.split("\n")[0] if docstring else None,
+            "docstring": docstring,
+            "calls": sorted(set(getattr(call_visitor, "calls", []))),
+            "line_number": node.lineno,
+            "end_line_number": getattr(node, "end_lineno", node.lineno),
+            "is_async": isinstance(node, ast.AsyncFunctionDef),
+            "parameters": extract_parameters(node) if hasattr(node, "args") else [],
+            "is_class": isinstance(node, ast.ClassDef),
+            "base_classes": (
+                extract_base_classes(node) if isinstance(node, ast.ClassDef) else []
+            ),
+            "structural_hash": calculate_structural_hash(node),
+        }
+
+        self.symbols[symbol_path_key] = symbol_data

body/introspection/semantic_clusterer.py

@@ -0,0 +1,128 @@
+# src/body/introspection/semantic_clusterer.py
+
+"""
+Performs semantic clustering on exported capability vectors to discover data-driven domains.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+from dotenv import load_dotenv
+
+# REFACTORED: Removed direct settings import
+from shared.exceptions import CoreError
+from shared.infrastructure.storage.file_handler import FileHandler
+from shared.logger import getLogger
+
+
+try:
+    from sklearn.cluster import KMeans
+except ImportError:
+    KMeans = None
+logger = getLogger(__name__)
+
+
+# ID: 04978d4c-740f-48cf-98f3-a85e28885ee6
+class SemanticClusteringError(CoreError):
+    """Raised when semantic clustering cannot complete."""
+
+
+# ID: 41c7d272-0d0d-4d84-9d99-984e9a698bd2
+def run_clustering(
+    input_path: Path | str,
+    output: Path | str,
+    n_clusters: int,
+    repo_root: Path | None = None,
+) -> None:
+    """
+    Loads exported vectors, runs K-Means clustering, and saves the proposed
+    capability-to-domain mappings to a JSON file.
+    """
+    if KMeans is None:
+        logger.error("scikit-learn is not installed. Aborting.")
+        raise SemanticClusteringError(
+            "scikit-learn is not installed for clustering.", exit_code=1
+        )
+    input_path = Path(input_path)
+    output_path = Path(output)
+    if repo_root is None:
+        raise ValueError("repo_root is required")
+    file_handler = FileHandler(str(repo_root))
+
+    logger.info("Starting semantic clustering process...")
+    file_handler.ensure_dir(str(output_path.parent.relative_to(repo_root)))
+    logger.info("   -> Loading vectors from %s...", input_path)
+    vectors = []
+    capability_keys = []
+    try:
+        with input_path.open("r", encoding="utf-8") as f:
+            for line in f:
+                record = json.loads(line)
+                if "vector" in record and "payload" in record:
+                    if "symbol" in record["payload"]:
+                        vectors.append(record["vector"])
+                        capability_keys.append(record["payload"]["symbol"])
+    except FileNotFoundError as exc:
+        logger.error("Input file not found: %s", input_path)
+        raise SemanticClusteringError("Input file not found.", exit_code=1) from exc
+
+    if not vectors:
+        logger.error("No valid vector data found in %s.", input_path)
+        raise SemanticClusteringError(
+            f"No valid vector data found in {input_path}.", exit_code=1
+        )
+    logger.info(
+        "   -> Loaded %s vectors for clustering into %s domains.",
+        len(vectors),
+        n_clusters,
+    )
+    X = np.array(vectors)
+    kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init="auto")
+    kmeans.fit(X)
+    labels = kmeans.labels_
+    proposed_domains = {
+        key: f"domain_{label}"
+        for key, label in zip(capability_keys, labels, strict=False)
+    }
+    rel_path = str(output_path.relative_to(repo_root))
+    file_handler.write_runtime_json(rel_path, proposed_domains)
+    logger.info(
+        "Successfully generated domain proposals for %s capabilities and saved to %s",
+        len(proposed_domains),
+        output_path,
+    )
+
+
+if __name__ == "__main__":
+    import argparse
+
+    load_dotenv()
+
+    parser = argparse.ArgumentParser(
+        description="Run semantic clustering on exported capability vectors."
+    )
+    parser.add_argument(
+        "input_path",
+        type=Path,
+        help="Path to the JSONL file containing exported vectors.",
+    )
+    parser.add_argument(
+        "output",
+        type=Path,
+        help="Path to write the proposed domain mapping JSON file.",
+    )
+    parser.add_argument(
+        "n_clusters",
+        type=int,
+        help="Number of clusters (domains) to produce.",
+    )
+
+    args = parser.parse_args()
+
+    try:
+        run_clustering(args.input_path, args.output, args.n_clusters)
+    except SemanticClusteringError as exc:
+        raise SystemExit(exc.exit_code) from exc

body/introspection/symbol_index_builder.py

@@ -0,0 +1,244 @@
+# src/body/introspection/symbol_index_builder.py
+"""
+Builds symbol_index.json from AST + patterns.
+
+CONSTITUTIONAL FIX:
+- Aligned with 'governance.artifact_mutation.traceable'.
+- Aligned with 'logic.logging.standard_only' (removed print statements).
+- Replaced direct Path writes with governed FileHandler mutations.
+- Fixed syntax error and ensured Python 3.12 compatibility.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+import sys
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from shared.infrastructure.storage.file_handler import FileHandler
+from shared.logger import getLogger
+
+
+logger = getLogger(__name__)
+
+try:
+    import yaml
+except Exception:
+    yaml = None
+
+
+@dataclass
+# ID: 3599bcdc-f7c4-4ee8-94f1-c30cf63c7104
+class Pattern:
+    name: str
+    description: str
+    match: dict[str, Any]
+    entry_point_type: str
+
+
+@dataclass
+# ID: b9f1867b-96ed-4fe0-b382-6ebea7d5500f
+class SymbolMeta:
+    key: str
+    filepath: str
+    name: str
+    type: str  # function | class | method
+    base_classes: list[str]
+    decorators: list[str]
+    is_public_function: bool
+    module_path: str
+
+
+def _load_patterns(patterns_path: Path) -> list[Pattern]:
+    if yaml is None or not patterns_path.exists():
+        return [
+            Pattern(
+                name="cli_command",
+                description="CLI entry points",
+                match={"type": "function", "module_path_contains": "src/cli/"},
+                entry_point_type="cli_command",
+            )
+        ]
+
+    data = yaml.safe_load(patterns_path.read_text(encoding="utf-8"))
+    items = data.get("patterns", []) if isinstance(data, dict) else []
+    return [
+        Pattern(
+            name=p.get("name", ""),
+            description=p.get("description", ""),
+            match=p.get("match", {}) or {},
+            entry_point_type=p.get("entry_point_type", ""),
+        )
+        for p in items
+    ]
+
+
+def _iter_py_files(root: Path) -> Iterable[Path]:
+    for p in root.rglob("*.py"):
+        s = str(p.as_posix())
+        if any(x in s for x in ["/.venv/", "/venv/", "/.git/", "reports/"]):
+            continue
+        yield p
+
+
+class _Visitor(ast.NodeVisitor):
+    def __init__(self, filepath: Path) -> None:
+        self.filepath = filepath
+        self.module_path = filepath.as_posix()
+        self.symbols: list[SymbolMeta] = []
+        self._class_stack: list[ast.ClassDef] = []
+
+    # ID: 53ffac44-279f-475f-8479-85ae61fbf17b
+    def visit_ClassDef(self, node: ast.ClassDef) -> Any:
+        bases = [self._name_of(b) for b in node.bases]
+        decorators = [self._name_of(d) for d in node.decorator_list]
+        self.symbols.append(
+            SymbolMeta(
+                key=f"{self.module_path}::{node.name}",
+                filepath=self.module_path,
+                name=node.name,
+                type="class",
+                base_classes=bases,
+                decorators=decorators,
+                is_public_function=not node.name.startswith("_"),
+                module_path=self.module_path,
+            )
+        )
+        self._class_stack.append(node)
+        self.generic_visit(node)
+        self._class_stack.pop()
+
+    # ID: 50c1acc1-14b2-4998-8741-aa255b1fa719
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
+        self._handle_func(node)
+
+    # ID: 8adec9fb-254d-4d09-896a-0155fcce78bf
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any:
+        self._handle_func(node)
+
+    def _handle_func(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None:
+        name = node.name
+        decorators = [self._name_of(d) for d in node.decorator_list]
+        bases: list[str] = []
+        if self._class_stack:
+            bases = [self._name_of(b) for b in self._class_stack[-1].bases]
+
+        self.symbols.append(
+            SymbolMeta(
+                key=f"{self.module_path}::{self._qn(name)}",
+                filepath=self.module_path,
+                name=name,
+                type="method" if self._class_stack else "function",
+                base_classes=bases,
+                decorators=decorators,
+                is_public_function=not name.startswith("_"),
+                module_path=self.module_path,
+            )
+        )
+
+    def _qn(self, name: str) -> str:
+        return f"{self._class_stack[-1].name}.{name}" if self._class_stack else name
+
+    def _name_of(self, node: ast.AST) -> str:
+        if isinstance(node, ast.Name):
+            return node.id
+        if isinstance(node, ast.Attribute):
+            return f"{self._name_of(node.value)}.{node.attr}"
+        try:
+            return ast.unparse(node)
+        except Exception:
+            return "unknown"
+
+
+def _match_pattern(sym: SymbolMeta, pat: Pattern) -> bool:
+    m = pat.match
+    if "type" in m:
+        if m["type"] == "function" and sym.type not in {"function", "method"}:
+            return False
+        if m["type"] == "class" and sym.type != "class":
+            return False
+    if "module_path_contains" in m and m["module_path_contains"] not in sym.module_path:
+        return False
+    if "name_regex" in m and not re.search(m["name_regex"], sym.name):
+        return False
+    return True
+
+
+def _classify(symbols: list[SymbolMeta], patterns: list[Pattern]) -> dict[str, dict]:
+    index = {}
+    for s in symbols:
+        for p in patterns:
+            if _match_pattern(s, p):
+                index[s.key] = {
+                    "entry_point_type": p.entry_point_type,
+                    "pattern_name": p.name,
+                    "entry_point_justification": p.description,
+                }
+                break
+    return index
+
+
+# ID: 2e550e18-7c09-4ccc-a967-e1d38fac6f8f
+def build_symbol_index(
+    project_root: str | Path = ".",
+    patterns_path: str | Path = ".intent/mind/knowledge/entry_point_patterns.yaml",
+) -> dict[str, dict]:
+    root = Path(project_root).resolve()
+    src = root / "src"
+    patterns_file = root / patterns_path
+
+    if not patterns_file.exists():
+        # Degrade gracefully if patterns are missing
+        patterns = []
+    else:
+        patterns = _load_patterns(patterns_file)
+
+    all_symbols: list[SymbolMeta] = []
+    for py in _iter_py_files(src):
+        try:
+            tree = ast.parse(py.read_text(encoding="utf-8"))
+            visitor = _Visitor(py)
+            visitor.visit(tree)
+            all_symbols.extend(visitor.symbols)
+        except Exception:
+            continue
+
+    return _classify(all_symbols, patterns)
+
+
+# ID: aeb56496-e95c-4537-aeb7-81ca8b3a9372
+def main(argv: list[str] | None = None) -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Build symbol index.")
+    parser.add_argument("--project-root", default=".")
+    parser.add_argument("--out", default="reports/symbol_index.json")
+    args = parser.parse_args(argv or sys.argv[1:])
+
+    try:
+        project_root = Path(args.project_root).resolve()
+        index = build_symbol_index(args.project_root)
+        fh = FileHandler(str(project_root))
+
+        # Resolve output path
+        out_path = Path(args.out)
+        repo_abs = project_root
+
+        if out_path.is_absolute():
+            rel_output = str(out_path.relative_to(repo_abs))
+        else:
+            rel_output = str(out_path).replace("\\", "/")
+
+        fh.write_runtime_json(rel_output, index)
+        return 0
+    except Exception as e:
+        logger.error("Failed to build symbol index: %s", e, exc_info=True)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

body/introspection/sync/engine.py

@@ -0,0 +1,92 @@
+# src/body/introspection/sync/engine.py
+
+"""Refactored logic for src/features/introspection/sync/engine.py."""
+
+from __future__ import annotations
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+
+# ID: 1d4f3f97-e8ce-4ebc-82b4-2d7f41ba55dd
+async def run_db_merge(session: AsyncSession, code_state: list[dict]) -> dict[str, int]:
+    """Executes the set-based merge logic exactly as in the original file."""
+    stats = {"scanned": len(code_state), "inserted": 0, "updated": 0, "deleted": 0}
+
+    await session.execute(
+        text(
+            """
+        CREATE TEMPORARY TABLE core_symbols_staging (LIKE core.symbols INCLUDING DEFAULTS) ON COMMIT DROP;
+    """
+        )
+    )
+
+    if code_state:
+        await session.execute(
+            text(
+                """
+            INSERT INTO core_symbols_staging (id, symbol_path, module, qualname, kind, ast_signature, fingerprint, state, is_public, calls, domain)
+            VALUES (:id, :symbol_path, :module, :qualname, :kind, :ast_signature, :fingerprint, :state, :is_public, :calls, :domain)
+        """
+            ),
+            code_state,
+        )
+
+    # 1. Calculate Deleted
+    stats["deleted"] = (
+        await session.execute(
+            text(
+                "SELECT COUNT(*) FROM core.symbols WHERE symbol_path NOT IN (SELECT symbol_path FROM core_symbols_staging)"
+            )
+        )
+    ).scalar_one()
+    # 2. Calculate Inserted
+    stats["inserted"] = (
+        await session.execute(
+            text(
+                "SELECT COUNT(*) FROM core_symbols_staging WHERE symbol_path NOT IN (SELECT symbol_path FROM core.symbols)"
+            )
+        )
+    ).scalar_one()
+    # 3. Calculate Updated
+    stats["updated"] = (
+        await session.execute(
+            text(
+                """
+        SELECT COUNT(*) FROM core.symbols s JOIN core_symbols_staging st ON s.symbol_path = st.symbol_path
+        WHERE s.fingerprint != st.fingerprint OR s.calls::text != st.calls::text OR s.domain != st.domain OR s.kind != st.kind
+    """
+            )
+        )
+    ).scalar_one()
+
+    # Apply Operations
+    await session.execute(
+        text(
+            "DELETE FROM core.symbols WHERE symbol_path NOT IN (SELECT symbol_path FROM core_symbols_staging)"
+        )
+    )
+
+    await session.execute(
+        text(
+            """
+        UPDATE core.symbols
+        SET fingerprint = st.fingerprint, calls = st.calls, domain = st.domain, kind = st.kind,
+            last_modified = NOW(), last_embedded = NULL, updated_at = NOW()
+        FROM core_symbols_staging st WHERE core.symbols.symbol_path = st.symbol_path
+        AND (core.symbols.fingerprint != st.fingerprint OR core.symbols.calls::text != st.calls::text OR core.symbols.domain != st.domain OR core.symbols.kind != st.kind);
+    """
+        )
+    )
+
+    await session.execute(
+        text(
+            """
+        INSERT INTO core.symbols (id, symbol_path, module, qualname, kind, ast_signature, fingerprint, state, is_public, calls, domain, created_at, updated_at, last_modified, first_seen, last_seen)
+        SELECT id, symbol_path, module, qualname, kind, ast_signature, fingerprint, state, is_public, calls, domain, NOW(), NOW(), NOW(), NOW(), NOW()
+        FROM core_symbols_staging ON CONFLICT (symbol_path) DO NOTHING;
+    """
+        )
+    )
+
+    return stats