core-runtime 0.1.1__py3-none-any.whl

shared/infrastructure/vector/adapters/constitutional_adapter.py

@@ -0,0 +1,309 @@
+# src/shared/infrastructure/vector/adapters/constitutional_adapter.py
+
+"""
+Constitutional Adapter - Constitution/Policies/Standards Vectorization
+
+Orchestrates transformation of constitutional documents into VectorizableItems
+for semantic search.
+
+CONSTITUTIONAL COMPLIANCE:
+- Uses IntentRepository as SSOT for all .intent/ access
+- NO direct filesystem crawling
+- Delegates discovery and loading to IntentRepository
+- Pure orchestration: IntentRepository → chunker → item_builder → VectorizableItems
+
+Architecture (Mind-Body-Will):
+    Mind (IntentRepository): Knows where files are, loads them
+    Body (ConstitutionalAdapter): Orchestrates transformation
+    Will (VectorIndexService): Stores vectors for semantic search
+
+Modular Design:
+- constitutional/chunker: Document chunking logic
+- constitutional/item_builder: VectorizableItem construction
+- constitutional/doc_key_resolver: Canonical key computation
+- This module: Orchestration only
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import ClassVar
+
+from shared.infrastructure.intent.intent_repository import (
+    IntentRepository,
+    PolicyRef,
+    get_intent_repository,
+)
+from shared.infrastructure.vector.adapters.constitutional.item_builder import (
+    data_to_items,
+)
+from shared.logger import getLogger
+from shared.models.vector_models import VectorizableItem
+
+
+logger = getLogger(__name__)
+
+
+# ID: bd536b63-ab28-435d-bdd6-bdd4d90b33f0
+class ConstitutionalAdapter:
+    """
+    Adapts constitutional files into vectorizable items.
+
+    CONSTITUTIONAL BOUNDARY:
+    - ALL .intent/ access goes through IntentRepository
+    - NO direct filesystem operations
+    - Pure orchestration: PolicyRef + document data → VectorizableItem
+    """
+
+    _EXTENSIONS: ClassVar[tuple[str, ...]] = (".json", ".yaml", ".yml")
+
+    def __init__(self, intent_repository: IntentRepository | None = None):
+        """
+        Initialize adapter.
+
+        Args:
+            intent_repository: Optional IntentRepository instance.
+                              If None, uses singleton from get_intent_repository().
+        """
+        self.intent_repo = intent_repository or get_intent_repository()
+        self.intent_repo.initialize()  # Ensure index is built
+
+        logger.debug(
+            "ConstitutionalAdapter initialized (intent_root=%s)", self.intent_repo.root
+        )
+
+    # -------------------------------------------------------------------------
+    # Public conversion APIs
+    # -------------------------------------------------------------------------
+
+    # ID: 7d7f430c-fd4b-4b99-9e35-76a28b93b492
+    def policies_to_items(self) -> list[VectorizableItem]:
+        """
+        Convert all executable governance policies into vector items.
+
+        Uses IntentRepository to discover policies from:
+        - .intent/policies/
+        - .intent/standards/
+        - .intent/rules/
+
+        Returns:
+            List of VectorizableItem objects ready for indexing
+        """
+        policy_refs = self.intent_repo.list_policies()
+        return self._process_policy_refs(
+            policy_refs, doc_type="policy", key_root="policies"
+        )
+
+    # ID: 5beb285c-cec2-4d2b-8107-4c948e62d818
+    def patterns_to_items(self) -> list[VectorizableItem]:
+        """
+        Convert architecture patterns into vector items.
+
+        Filters policies for those under */architecture/* paths.
+
+        Returns:
+            List of VectorizableItem objects for patterns
+        """
+        all_refs = self.intent_repo.list_policies()
+
+        # Filter for patterns under architecture subdirectories
+        pattern_refs = [
+            ref
+            for ref in all_refs
+            if "/architecture/" in str(ref.path).replace("\\", "/")
+        ]
+
+        if not pattern_refs:
+            logger.info("No architecture pattern files found")
+            return []
+
+        return self._process_policy_refs(
+            pattern_refs,
+            doc_type="pattern",
+            key_root="policies",  # Patterns are policies
+        )
+
+    # ID: a048dee6-165e-4f74-bd20-8dcacf87125f
+    def constitution_to_items(self) -> list[VectorizableItem]:
+        """
+        Convert constitution documents into vector items.
+
+        Processes files from .intent/constitution/ directory.
+
+        Returns:
+            List of VectorizableItem objects for constitution
+        """
+        return self._process_constitution_dir()
+
+    # ID: 6d5a1ee7-ebc0-44cd-bcaf-b30045d73547
+    def standards_to_items(self) -> list[VectorizableItem]:
+        """
+        Convert standards documents into vector items.
+
+        NOTE: Standards are already included in policies_to_items()
+        since IntentRepository searches ["policies", "standards", "rules"].
+        This method exists for backward compatibility and explicit standards querying.
+
+        Returns:
+            List of VectorizableItem objects for standards
+        """
+        all_refs = self.intent_repo.list_policies()
+
+        # Filter for standards only (path starts with standards/)
+        standards_refs = [
+            ref for ref in all_refs if str(ref.policy_id).startswith("standards/")
+        ]
+
+        if not standards_refs:
+            logger.info("No standards files found")
+            return []
+
+        return self._process_policy_refs(
+            standards_refs, doc_type="standard", key_root="standards"
+        )
+
+    # ID: bb43de2b-45e4-4889-aa23-c0bcd965d73d
+    def enforcement_policies_to_items(self) -> list[VectorizableItem]:
+        """Backward compatibility alias for policies_to_items()."""
+        return self.policies_to_items()
+
+    # -------------------------------------------------------------------------
+    # Processing - Uses IntentRepository data
+    # -------------------------------------------------------------------------
+
+    def _process_policy_refs(
+        self,
+        policy_refs: list[PolicyRef],
+        *,
+        doc_type: str,
+        key_root: str,
+    ) -> list[VectorizableItem]:
+        """
+        Process PolicyRef objects from IntentRepository into VectorizableItems.
+
+        Args:
+            policy_refs: List of PolicyRef from IntentRepository
+            doc_type: Document type (policy, pattern, standard)
+            key_root: Root for key generation (policies, standards)
+
+        Returns:
+            List of VectorizableItem objects
+        """
+        if not policy_refs:
+            logger.info("No %s files to process", doc_type)
+            return []
+
+        logger.info("Processing %s %s file(s)", len(policy_refs), doc_type)
+
+        items: list[VectorizableItem] = []
+        for ref in policy_refs:
+            try:
+                # Load document through IntentRepository (SSOT)
+                data = self.intent_repo.load_document(ref.path)
+
+                if not isinstance(data, dict):
+                    logger.warning(
+                        "Skipping non-dict document: %s (type=%s)",
+                        ref.path,
+                        type(data).__name__,
+                    )
+                    continue
+
+                # Transform to VectorizableItems (delegates to item_builder)
+                file_items = data_to_items(
+                    data,
+                    ref.path,
+                    doc_type,
+                    key_root=key_root,
+                    intent_root=self.intent_repo.root,
+                )
+                items.extend(file_items)
+
+            except Exception as exc:
+                logger.exception(
+                    "Failed to process %s (%s): %s", ref.path, doc_type, exc
+                )
+                continue
+
+        logger.info(
+            "Generated %s item(s) from %s file(s)", len(items), len(policy_refs)
+        )
+        return items
+
+    def _process_constitution_dir(self) -> list[VectorizableItem]:
+        """
+        Process constitution directory files.
+
+        Constitution files are not indexed by IntentRepository's policy index,
+        so we use direct directory resolution through IntentRepository.root.
+
+        Returns:
+            List of VectorizableItem objects
+        """
+        constitution_dir = self.intent_repo.root / "constitution"
+
+        if not constitution_dir.exists():
+            logger.warning("Constitution directory not found: %s", constitution_dir)
+            return []
+
+        files = self._collect_files(constitution_dir, recursive=True)
+        if not files:
+            logger.info("No constitution files found")
+            return []
+
+        logger.info("Processing %s constitution file(s)", len(files))
+
+        items: list[VectorizableItem] = []
+        for file_path in files:
+            try:
+                # Load through IntentRepository for consistency
+                data = self.intent_repo.load_document(file_path)
+
+                if not isinstance(data, dict):
+                    logger.warning(
+                        "Skipping non-dict document: %s (type=%s)",
+                        file_path,
+                        type(data).__name__,
+                    )
+                    continue
+
+                # Transform to VectorizableItems (delegates to item_builder)
+                file_items = data_to_items(
+                    data,
+                    file_path,
+                    doc_type="constitution",
+                    key_root="constitution",
+                    intent_root=self.intent_repo.root,
+                )
+                items.extend(file_items)
+
+            except Exception as exc:
+                logger.exception(
+                    "Failed to process constitution file %s: %s", file_path, exc
+                )
+                continue
+
+        logger.info("Generated %s item(s) from %s file(s)", len(items), len(files))
+        return items
+
+    def _collect_files(self, directory: Path, recursive: bool) -> list[Path]:
+        """
+        Collect JSON/YAML files from directory.
+
+        This is only used for constitution directory since those files
+        are not in the policy index.
+
+        Args:
+            directory: Directory to scan
+            recursive: Whether to scan recursively
+
+        Returns:
+            Sorted list of file paths
+        """
+        collected: set[Path] = set()
+        for ext in self._EXTENSIONS:
+            if recursive:
+                collected.update(p for p in directory.rglob(f"*{ext}") if p.is_file())
+            else:
+                collected.update(p for p in directory.glob(f"*{ext}") if p.is_file())
+        return sorted(collected)

shared/infrastructure/vector/adapters/specs_adapter.py

@@ -0,0 +1,171 @@
+# src/shared/infrastructure/vector/adapters/specs_adapter.py
+
+"""
+Specs Adapter - .specs/ Markdown Vectorization
+
+Converts human-authored specification documents under .specs/ into
+VectorizableItems for semantic search.
+
+CONSTITUTIONAL COMPLIANCE:
+- Uses SpecsRepository as SSOT for all .specs/ access
+- NO direct filesystem crawling or raw Path reads
+- Pure orchestration: SpecsRepository → chunker → VectorizableItems
+"""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+from shared.infrastructure.intent.operational_config import load_operational_config
+from shared.infrastructure.specs.specs_repository import (
+    SpecsRepository,
+    get_specs_repository,
+)
+from shared.logger import getLogger
+from shared.models.vector_models import VectorizableItem
+
+
+logger = getLogger(__name__)
+
+_CFG_CHK = load_operational_config().chunking
+
+_COLLECTION_NAME = "core_specs"
+_DOC_TYPE = "specs"
+_SOURCE = "specs"
+
+
+# ID: a7f3c91d-2b84-4e5a-9c1f-6d2e8a4b9c03
+class SpecsAdapter:
+    """
+    Adapts .specs/ markdown documents into vectorizable items.
+
+    All filesystem access is mediated by SpecsRepository; this class
+    performs pure orchestration and chunking.
+    """
+
+    # ID: b8e4d2a0-3c95-4f6b-ad28-7e3f9b5ca104
+    def __init__(self, specs_repository: SpecsRepository | None = None) -> None:
+        """
+        Args:
+            specs_repository: Optional SpecsRepository instance.
+                              Defaults to the singleton from get_specs_repository().
+        """
+        self.specs_repo = specs_repository or get_specs_repository()
+        logger.debug("SpecsAdapter initialized (specs_root=%s)", self.specs_repo.root)
+
+    # ID: c9f5e3b1-4da6-4a7c-be39-8f40ac6db215
+    def docs_to_items(self, subdir: str = "") -> list[VectorizableItem]:
+        """
+        Convert markdown documents under .specs/ into VectorizableItems.
+
+        Args:
+            subdir: Optional subdirectory (relative to .specs/ root) to scope
+                    the scan, e.g. "papers", "northstar", "requirements".
+                    Empty string scans the entire .specs/ tree.
+
+        Returns:
+            List of VectorizableItem objects ready for indexing into
+            the "core_specs" collection.
+        """
+        files = self.specs_repo.list_files(subdir=subdir, suffix=".md")
+        if not files:
+            logger.info(
+                "No .specs/ markdown files found (subdir=%r)", subdir or "<root>"
+            )
+            return []
+
+        logger.info("Processing %s spec document(s)", len(files))
+
+        items: list[VectorizableItem] = []
+        specs_root = self.specs_repo.root
+        for abs_path in files:
+            try:
+                rel_path = abs_path.relative_to(specs_root)
+            except ValueError:
+                logger.warning(
+                    "Skipping spec outside specs root: %s (root=%s)",
+                    abs_path,
+                    specs_root,
+                )
+                continue
+
+            doc_id = str(rel_path).replace("\\", "/")
+            try:
+                text = self.specs_repo.load_text(rel_path)
+            except Exception as exc:
+                logger.exception("Failed to load spec %s: %s", doc_id, exc)
+                continue
+
+            file_items = self._build_items_for_file(doc_id, rel_path, text)
+            items.extend(file_items)
+
+        logger.info("Generated %s item(s) from %s spec file(s)", len(items), len(files))
+        return items
+
+    # ID: d0a6f4c2-5eb7-4b8d-cf4a-9051bd7ec326
+    def _build_items_for_file(
+        self, doc_id: str, rel_path: Path, text: str
+    ) -> list[VectorizableItem]:
+        """Chunk a single markdown document and wrap each chunk as an item."""
+        body = text.strip()
+        if not body:
+            return []
+
+        chunks = self._chunk_markdown(body, rel_path.stem)
+
+        items: list[VectorizableItem] = []
+        for idx, (section, content) in enumerate(chunks):
+            clean = content.strip()
+            if not clean:
+                continue
+
+            item_id = f"{doc_id}:{idx}"
+            payload = {
+                "doc_id": doc_id,
+                "doc_type": _DOC_TYPE,
+                "source": _SOURCE,
+                "source_path": f".specs/{doc_id}",
+                "section": section,
+                "content_sha256": hashlib.sha256(clean.encode("utf-8")).hexdigest(),
+                "text": clean,
+            }
+            items.append(VectorizableItem(item_id=item_id, text=clean, payload=payload))
+
+        return items
+
+    # ID: e1b705d3-6fc8-4c9e-d05b-a162ce8fd437
+    def _chunk_markdown(
+        self, text: str, filename_section: str
+    ) -> list[tuple[str, str]]:
+        """
+        Split markdown into (section, content) pairs.
+
+        Small documents return a single chunk labelled with the filename stem.
+        Larger documents are split on "## " headings; preamble before the first
+        heading becomes its own chunk labelled with the filename stem.
+        """
+        if len(text) <= _CFG_CHK.max_chunk_chars:
+            return [(filename_section, text)]
+
+        sections: list[tuple[str, str]] = []
+        current_section = filename_section
+        current_lines: list[str] = []
+
+        for line in text.splitlines():
+            if line.startswith("## "):
+                if current_lines:
+                    sections.append((current_section, "\n".join(current_lines).strip()))
+                current_section = line[3:].strip() or filename_section
+                current_lines = [line]
+            else:
+                current_lines.append(line)
+
+        if current_lines:
+            sections.append((current_section, "\n".join(current_lines).strip()))
+
+        non_empty = [(s, c) for s, c in sections if c]
+        if not non_empty:
+            return [(filename_section, text)]
+
+        return non_empty

shared/infrastructure/vector/cognitive_adapter.py

@@ -0,0 +1,119 @@
+# src/shared/infrastructure/vector/cognitive_adapter.py
+"""
+CognitiveService Adapter for VectorIndexService
+
+Adapts CognitiveService to the Embeddable protocol so it can be used
+with VectorIndexService for smart deduplication.
+
+This allows constitutional vectorization to use the same embedding provider
+as code vectorization (database-configured LLM) instead of requiring
+separate local embedding settings.
+"""
+
+from __future__ import annotations
+
+from shared.logger import getLogger
+from shared.protocols.cognitive import CognitiveProtocol
+
+
+logger = getLogger(__name__)
+
+
+# ID: b4c348a3-9d0e-4079-a6bd-eb93be95686f
+class CognitiveEmbedderAdapter:
+    """
+    Adapts CognitiveService to the Embeddable protocol.
+
+    This allows VectorIndexService to use CognitiveService for embeddings,
+    enabling constitutional documents to use the same embedding provider
+    as code symbols (database-configured, not environment-based).
+
+    Usage:
+        cognitive_service = await registry.get_cognitive_service()
+        embedder = CognitiveEmbedderAdapter(cognitive_service)
+
+        service = VectorIndexService(
+            qdrant_service=qdrant,
+            collection_name="core_policies",
+            embedder=embedder  # ← Inject CognitiveService!
+        )
+
+        # Now uses smart deduplication + database-configured embeddings
+        await service.index_items(policy_items)
+    """
+
+    def __init__(self, cognitive_service: CognitiveProtocol):
+        """
+        Initialize adapter.
+
+        Args:
+            cognitive_service: Initialized CognitiveService instance
+        """
+        self._cognitive_service = cognitive_service
+        logger.debug("CognitiveEmbedderAdapter initialized")
+
+    # ID: d494615f-51d5-4796-a62a-ddf9e3e7bda3
+    async def get_embedding(self, text: str) -> list[float]:
+        """
+        Generate embedding using CognitiveService.
+
+        This delegates to cognitive_service.get_embedding_for_code() which
+        uses the database-configured LLM provider.
+
+        Args:
+            text: Text to embed
+
+        Returns:
+            Embedding vector as list of floats
+
+        Raises:
+            RuntimeError: If embedding generation fails
+        """
+        try:
+            embedding = await self._cognitive_service.get_embedding_for_code(text)
+
+            if not embedding:
+                raise RuntimeError("CognitiveService returned empty embedding")
+
+            return embedding
+
+        except Exception as e:
+            logger.error("Failed to generate embedding via CognitiveService: %s", e)
+            raise RuntimeError(f"Embedding generation failed: {e}") from e
+
+    # ID: 3a8f2e74-9c1d-4b85-a047-6e5f9c2a8d13
+    async def get_embeddings_batch(self, texts: list[str]) -> list[list[float]]:
+        """
+        Batch-generate embeddings for a list of texts (#461).
+
+        Delegates to cognitive_service.get_embeddings_for_code_batch().
+        Single round-trip to the embedding endpoint when the underlying
+        provider supports it (Ollama 0.4+); other providers fall back
+        to looped single-input calls (correct, not faster).
+
+        Args:
+            texts: Texts to embed, in order.
+
+        Returns:
+            Embedding vectors aligned to input order.
+
+        Raises:
+            RuntimeError: If embedding generation fails for the batch.
+        """
+        if not texts:
+            return []
+        try:
+            embeddings = await self._cognitive_service.get_embeddings_for_code_batch(
+                texts
+            )
+            if len(embeddings) != len(texts):
+                raise RuntimeError(
+                    f"CognitiveService returned {len(embeddings)} embeddings "
+                    f"for {len(texts)} inputs — batch response misalignment"
+                )
+            return embeddings
+        except Exception as e:
+            logger.error(
+                "Failed to generate batch embeddings via CognitiveService: %s", e
+            )
+            raise RuntimeError(f"Batch embedding generation failed: {e}") from e