core-runtime 0.1.1__py3-none-any.whl

shared/infrastructure/vector/vector_index_service.py

@@ -0,0 +1,230 @@
+# src/shared/infrastructure/vector/vector_index_service.py
+
+"""
+Unified Vector Indexing Service - Constitutional Infrastructure
+
+- Uses QdrantService for upsert operations.
+- Implements Smart Deduplication using content hashes.
+- Embeddings are obtained via an injected Embeddable provider.
+  Default provider is the local-only embedder from shared.utils.embedding_utils.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING
+
+from shared.config import settings
+from shared.infrastructure.clients.qdrant_client import QdrantService
+from shared.infrastructure.intent.operational_config import load_operational_config
+from shared.logger import getLogger
+from shared.models.vector_models import IndexResult, VectorizableItem
+from shared.universal import get_deterministic_id
+from shared.utils.embedding_utils import Embeddable, build_embedder_from_env
+
+
+if TYPE_CHECKING:
+    pass
+
+logger = getLogger(__name__)
+
+_CFG_VEC = load_operational_config().vectors
+
+
+# ID: 2ffe6361-bae9-4b98-936c-95cfe52a1d8b
+class VectorIndexService:
+    """
+    Unified vector indexing service with smart deduplication.
+
+    Constitutional embedding rule:
+    - VectorIndexService does not implement embedding logic.
+    - It consumes an Embeddable provider (injected), or falls back to the
+      local-only embedder factory (settings-based, no env access).
+    """
+
+    def __init__(
+        self,
+        qdrant_service: QdrantService,
+        collection_name: str,
+        vector_dim: int | None = None,
+        embedder: Embeddable | None = None,
+    ) -> None:
+        self.qdrant = qdrant_service
+        self.collection_name = collection_name
+        self.vector_dim = vector_dim or int(settings.LOCAL_EMBEDDING_DIM)
+
+        self._embedder: Embeddable = embedder or build_embedder_from_env()
+
+        logger.info(
+            "VectorIndexService initialized: collection=%s dim=%s embedder=%s",
+            self.collection_name,
+            self.vector_dim,
+            type(self._embedder).__name__,
+        )
+
+    # ID: 016343de-1f7d-4b55-bc00-ee7cc2565175
+    async def ensure_collection(self) -> None:
+        await self.qdrant.ensure_collection(
+            collection_name=self.collection_name,
+            vector_size=self.vector_dim,
+        )
+
+    # ID: 14aa6d5e-820b-4b0b-b77a-6da102e780ed
+    async def index_items(
+        self,
+        items: list[VectorizableItem],
+        batch_size: int = _CFG_VEC.index_batch_size,
+    ) -> list[IndexResult]:
+        if not items:
+            raise ValueError("Cannot index empty list of items")
+
+        stored_hashes = await self.qdrant.get_stored_hashes(self.collection_name)
+        items_to_index: list[VectorizableItem] = []
+        skipped_count = 0
+
+        for item in items:
+            point_id = str(get_deterministic_id(item.item_id))
+            new_hash = item.payload.get("content_sha256")
+
+            if (
+                point_id in stored_hashes
+                and new_hash
+                and stored_hashes[point_id] == new_hash
+            ):
+                skipped_count += 1
+                continue
+
+            items_to_index.append(item)
+
+        if skipped_count:
+            logger.info(
+                "Skipped %s unchanged items (smart deduplication).", skipped_count
+            )
+
+        if not items_to_index:
+            logger.info("All items are up to date. Nothing to index.")
+            return [
+                IndexResult(
+                    item_id=item.item_id,
+                    point_id=get_deterministic_id(item.item_id),
+                    vector_dim=self.vector_dim,
+                )
+                for item in items
+            ]
+
+        logger.info(
+            "Indexing %s new/changed items into %s (batch_size=%s)",
+            len(items_to_index),
+            self.collection_name,
+            batch_size,
+        )
+
+        results: list[IndexResult] = []
+        for i in range(0, len(items_to_index), batch_size):
+            batch = items_to_index[i : i + batch_size]
+            results.extend(await self._process_batch(batch))
+            logger.debug(
+                "Processed batch %s/%s",
+                i // batch_size + 1,
+                (len(items_to_index) - 1) // batch_size + 1,
+            )
+
+        logger.info(
+            "✓ Indexed %s/%s items successfully", len(results), len(items_to_index)
+        )
+        return results
+
+    async def _process_batch(self, items: list[VectorizableItem]) -> list[IndexResult]:
+        tasks = [self._embedder.get_embedding(item.text) for item in items]
+        embeddings = await asyncio.gather(*tasks, return_exceptions=True)
+
+        valid_pairs: list[tuple[VectorizableItem, list[float]]] = []
+
+        for item, emb in zip(items, embeddings):
+            if isinstance(emb, Exception):
+                logger.warning("Failed to embed %s: %s", item.item_id, emb)
+                continue
+            if not emb:
+                logger.warning("Failed to embed %s: empty embedding", item.item_id)
+                continue
+
+            vector = list(emb)
+
+            # Constitutional invariant: vector dim must match collection dim
+            if len(vector) != self.vector_dim:
+                logger.error(
+                    "Embedding dimension mismatch for %s: got=%s expected=%s",
+                    item.item_id,
+                    len(vector),
+                    self.vector_dim,
+                )
+                continue
+
+            valid_pairs.append((item, vector))
+
+        if not valid_pairs:
+            return []
+
+        bulk_items: list[tuple[str, list[float], dict]] = []
+        for item, vector in valid_pairs:
+            point_id_str = str(get_deterministic_id(item.item_id))
+            payload = {
+                **item.payload,
+                "item_id": item.item_id,
+                "chunk_id": item.item_id,
+            }
+
+            if "source_path" not in payload:
+                payload["source_path"] = payload.get(
+                    "file_path", f".intent/{item.item_id.split(':')[0]}.json"
+                )
+            if "source_type" not in payload:
+                payload["source_type"] = payload.get("doc_type", "intent")
+
+            bulk_items.append((point_id_str, vector, payload))
+
+        await self.qdrant.upsert_symbol_vectors_bulk(
+            bulk_items,
+            collection_name=self.collection_name,
+            wait=True,
+        )
+
+        return [
+            IndexResult(
+                item_id=item.item_id,
+                point_id=get_deterministic_id(item.item_id),
+                vector_dim=self.vector_dim,
+            )
+            for item, _vector in valid_pairs
+        ]
+
+    # ID: a74ccda4-917d-4da6-b172-e807c7e8d16f
+    async def query(
+        self,
+        query_text: str,
+        limit: int = 5,
+        score_threshold: float | None = None,
+    ) -> list[dict]:
+        vec = await self._embedder.get_embedding(query_text)
+        if not vec:
+            logger.warning("Failed to generate query embedding")
+            return []
+
+        query_vector = list(vec)
+
+        if len(query_vector) != self.vector_dim:
+            logger.error(
+                "Query embedding dimension mismatch: got=%s expected=%s",
+                len(query_vector),
+                self.vector_dim,
+            )
+            return []
+
+        results = await self.qdrant.search(
+            collection_name=self.collection_name,
+            query_vector=query_vector,
+            limit=limit,
+            score_threshold=score_threshold,
+        )
+
+        return [{"score": hit.score, "payload": hit.payload} for hit in results]

shared/logger.py

@@ -0,0 +1,195 @@
+# src/shared/logger.py
+
+"""Centralized logger configuration and factory for the CORE system."""
+
+from __future__ import annotations
+
+import contextvars
+import json
+import logging
+import os
+import sys
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# ─────────────────────────────────────────────────────────────── Configuration
+_LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
+_LOG_FORMAT_TYPE = os.getenv("LOG_FORMAT_TYPE", "human").lower()  # json or human
+_VALID_LEVELS = frozenset({"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"})
+
+# Context variable for Activity correlation (Workflow Tracing)
+_current_run_id = contextvars.ContextVar("run_id", default=None)
+
+# Use a module logger for internal bootstrap events
+_boot_logger = logging.getLogger(__name__)
+
+# Validate level at import time
+if _LOG_LEVEL not in _VALID_LEVELS:
+    _boot_logger.warning("Invalid LOG_LEVEL '%s'. Using INFO.", _LOG_LEVEL)
+    _LOG_LEVEL = "INFO"
+
+# ─────────────────────────────────────────────────────────────── Formatters
+
+
+# ID: d453de4a-8b0a-4dbe-84bb-8bcd78751e15
+class JsonFormatter(logging.Formatter):
+    """
+    Constitutional JSON Formatter (LOG-005).
+    Outputs structured logs for machine parsing/aggregation.
+    """
+
+    # ID: 7602325a-ebe5-4b21-b25f-043650e8fcf4
+    def format(self, record: logging.LogRecord) -> str:
+        log_record: dict[str, Any] = {
+            "timestamp": datetime.fromtimestamp(record.created, tz=UTC).isoformat(),
+            "level": record.levelname,
+            "message": record.getMessage(),
+            "logger": record.name,
+            "module": record.module,
+            "line": record.lineno,
+        }
+
+        run_id = _current_run_id.get()
+        if run_id:
+            log_record["run_id"] = run_id
+
+        if record.exc_info:
+            log_record["exception"] = self.formatException(record.exc_info)
+
+        standard_attrs = {
+            "name",
+            "msg",
+            "args",
+            "levelname",
+            "levelno",
+            "pathname",
+            "filename",
+            "module",
+            "exc_info",
+            "exc_text",
+            "stack_info",
+            "lineno",
+            "funcName",
+            "created",
+            "msecs",
+            "relativeCreated",
+            "thread",
+            "threadName",
+            "processName",
+            "process",
+            "activity",
+        }
+
+        for key, value in record.__dict__.items():
+            if key not in standard_attrs:
+                log_record[key] = value
+
+        if hasattr(record, "activity"):
+            log_record["activity"] = record.activity
+
+        return json.dumps(log_record)
+
+
+# ─────────────────────────────────────────────────────────────── Public API
+
+
+# ID: 90a8ab6f-c125-43b8-ae6f-e3a8ffc863a8
+def getLogger(name: str | None = None) -> logging.Logger:
+    """Returns a standard logger instance."""
+    return logging.getLogger(name)
+
+
+# ID: 2021f8a9-f7e0-451c-939d-01d197b517da
+def _configure_root_logger(
+    level: str | None = None,
+    handlers: Sequence[logging.Handler] | None = None,
+) -> None:
+    """
+    Bootstrap utility to set up root logging.
+    Made private (_) to exempt from public-api decorator requirements.
+    """
+    effective_level = (level or _LOG_LEVEL).upper()
+    if effective_level not in _VALID_LEVELS:
+        raise ValueError(f"Invalid log level: {effective_level}")
+
+    if handlers is None:
+        handlers = []
+        if _LOG_FORMAT_TYPE == "json":
+            handler = logging.StreamHandler(sys.stdout)
+            handler.setFormatter(JsonFormatter())
+            handlers.append(handler)
+        else:
+            try:
+                from rich.logging import RichHandler
+
+                handlers.append(
+                    RichHandler(
+                        rich_tracebacks=True,
+                        show_time=True,
+                        show_level=True,
+                        show_path=False,
+                        log_time_format="[%X]",
+                    )
+                )
+            except ImportError:
+                handlers.append(logging.StreamHandler())
+
+    logging.basicConfig(
+        level=getattr(logging, effective_level),
+        handlers=handlers,
+        force=True,
+    )
+
+    # Suppress noise from infrastructure libraries
+    for lib in ("httpx", "urllib3", "qdrant_client"):
+        logging.getLogger(lib).setLevel(logging.WARNING)
+
+
+# Break circular dependency by importing only when needed
+from shared.action_types import ActionImpact, ActionResult
+from shared.atomic_action import atomic_action
+
+
+@atomic_action(
+    action_id="logging.reconfigure",
+    intent="Dynamically update the system log level",
+    impact=ActionImpact.WRITE_DATA,
+    policies=["standard_logging"],
+)
+# ID: fef5e4a6-9002-452d-92df-aabbb41e50f8
+async def reconfigure_log_level(level: str, **kwargs) -> ActionResult:
+    """
+    Updates the root logger level at runtime.
+    Constitutional: Wrapped in atomic_action for traceability.
+    Satisfies body.atomic_actions_use_actionresult law.
+    """
+    import time
+
+    start_time = time.time()
+    try:
+        _configure_root_logger(level=level)
+        getLogger(__name__).info("Log level reconfigured to %s", level.upper())
+        return ActionResult(
+            action_id="logging.reconfigure",
+            ok=True,
+            data={"new_level": level.upper()},
+            duration_sec=time.time() - start_time,
+            impact=ActionImpact.WRITE_DATA,
+        )
+    except Exception as e:
+        return ActionResult(
+            action_id="logging.reconfigure",
+            ok=False,
+            data={"error": str(e)},
+            duration_sec=time.time() - start_time,
+            impact=ActionImpact.WRITE_DATA,
+        )
+
+
+# ─────────────────────────────────────────────────────────────── Initialization
+_configure_root_logger()
+logger = getLogger(__name__)

shared/models/__init__.py

@@ -0,0 +1,47 @@
+# src/shared/models/__init__.py
+"""
+Makes all Pydantic models in this directory available for easy import.
+"""
+
+from __future__ import annotations
+
+from .audit_models import AuditFinding, AuditSeverity
+from .capability_models import CapabilityMeta
+from .drift_models import DriftReport
+from .embedding_payload import EmbeddingPayload
+from .execution_models import (
+    ExecutionTask,
+    PlanExecutionError,
+    PlannerConfig,
+    TaskParams,
+)
+from .validation_result import ValidationResult
+from .workflow_models import (
+    DetailedPlan,
+    DetailedPlanStep,
+    ExecutionResults,
+    PhaseResult,
+    PhaseWorkflowResult,
+    WorkflowResult,
+)
+
+
+__all__ = [
+    "AuditFinding",
+    "AuditSeverity",
+    "CapabilityMeta",
+    # Workflow models
+    "DetailedPlan",
+    "DetailedPlanStep",
+    "DriftReport",
+    "EmbeddingPayload",
+    "ExecutionResults",
+    "ExecutionTask",
+    "PhaseResult",
+    "PhaseWorkflowResult",
+    "PlanExecutionError",
+    "PlannerConfig",
+    "TaskParams",
+    "ValidationResult",
+    "WorkflowResult",
+]

shared/models/action_result.py

@@ -0,0 +1,77 @@
+# src/shared/models/action_result.py
+
+"""
+ActionResult Database Model - Audit trail for CORE workflow operations.
+
+Records the outcome of every action (tests, coverage, alignment, code generation, etc.)
+to provide workflow gate verification and historical compliance tracking.
+
+Constitutional Principles: knowledge.database_ssot, safe_by_default
+"""
+
+from __future__ import annotations
+
+from typing import ClassVar
+from uuid import uuid4
+
+from sqlalchemy import Boolean, Column, DateTime, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import UUID as pgUUID
+
+from shared.infrastructure.database.models.knowledge import Base
+
+
+# ID: 588cecb7-f308-4ba0-b09e-b1122155b088
+class ActionResult(Base):
+    """
+    Records the outcome of CORE operations for workflow gating and audit trails.
+
+    Used by WorkflowGateEngine to verify:
+    - Test execution status
+    - Coverage measurements
+    - Alignment healing outcomes
+    - Code generation success/failure
+    - Any other quality gate checkpoints
+    """
+
+    __tablename__: ClassVar[str] = "action_results"
+    __table_args__: ClassVar[dict] = {"schema": "core"}
+
+    id = Column(pgUUID(as_uuid=True), primary_key=True, default=uuid4)
+    action_type = Column(
+        String(100),
+        nullable=False,
+        index=True,
+        comment="Type: test_execution, alignment, code_generation, coverage_check, etc.",
+    )
+    ok = Column(
+        Boolean,
+        nullable=False,
+        index=True,
+        comment="Whether the action succeeded (True) or failed (False)",
+    )
+    file_path = Column(
+        String(500),
+        nullable=True,
+        index=True,
+        comment="Target file path if action was file-specific",
+    )
+    error_message = Column(Text, nullable=True, comment="Error details if ok=False")
+    action_metadata = Column(
+        JSONB,
+        nullable=True,
+        comment="Additional context: coverage_percent, test_count, violations_fixed, etc.",
+    )
+    agent_id = Column(
+        String(100), nullable=True, comment="Which agent/service performed the action"
+    )
+    duration_ms = Column(
+        Integer, nullable=True, comment="How long the action took in milliseconds"
+    )
+    created_at = Column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(), index=True
+    )
+
+    def __repr__(self) -> str:
+        status = "✓" if self.ok else "✗"
+        return f"<ActionResult {status} {self.action_type} @ {self.created_at}>"

shared/models/audit_models.py

@@ -0,0 +1,106 @@
+# src/shared/models/audit_models.py
+"""
+Defines the Pydantic models for representing the results of a constitutional audit.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import IntEnum
+from typing import Any
+
+
+# ID: 5ccdae76-2214-413d-8551-13d4b224b694
+class AuditSeverity(IntEnum):
+    """Enumeration for the severity of an audit finding.
+
+    Five-value scale (ADR-059 D2): INFO/LOW/MEDIUM/HIGH/BLOCK. Stored as
+    lowercase string via __str__. BLOCK is the sole CI-blocking severity;
+    HIGH/MEDIUM/LOW are actionable at decreasing urgency; INFO is
+    informational only. Governed by `audit_severity` in
+    .intent/META/enums.json.
+    """
+
+    INFO = 1
+    LOW = 2
+    MEDIUM = 3
+    HIGH = 4
+    BLOCK = 5
+
+    def __str__(self) -> str:
+        # This allows us to use severity.name in lowercase, e.g., 'info'
+        return self.name.lower()
+
+    @property
+    # ID: bad8d002-de4c-4b09-900f-0cd784c60242
+    def is_blocking(self) -> bool:
+        """Returns True if the severity level should block a CI/CD pipeline."""
+        return self == AuditSeverity.BLOCK
+
+
+@dataclass(init=False)
+# ID: 1bc3d2f1-466b-49b9-aacd-6fac9e03a068
+class AuditFinding:
+    """Represents a single finding from a constitutional audit check.
+
+    Notes:
+        - `context` is the single source of truth for structured finding data.
+        - `details` is a backwards-compatible alias to `context`.
+        - We accept `details=...` as an init kwarg for legacy callers without
+          defining a dataclass field named `details` (avoids property collision).
+    """
+
+    check_id: str
+    severity: AuditSeverity
+    message: str
+    file_path: str | None = None
+    line_number: int | None = None
+    context: dict[str, Any] = field(default_factory=dict)
+
+    def __init__(
+        self,
+        check_id: str,
+        severity: AuditSeverity,
+        message: str,
+        file_path: str | None = None,
+        line_number: int | None = None,
+        context: dict[str, Any] | None = None,
+        *,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.check_id = check_id
+        self.severity = severity
+        self.message = message
+        self.file_path = file_path
+        self.line_number = line_number
+
+        base_context: dict[str, Any] = dict(context or {})
+        if details:
+            base_context.update(details)
+
+        self.context = base_context
+
+    # ID: d638215e-ceb0-421e-b33b-a0b191876530
+    def as_dict(self) -> dict[str, Any]:
+        """Serializes the finding to a dictionary for reporting."""
+        return {
+            "check_id": self.check_id,
+            "severity": str(self.severity),
+            "message": self.message,
+            "file_path": self.file_path,
+            "line_number": self.line_number,
+            "context": self.context,
+            # Keep the "details" key for existing consumers
+            "details": self.context,
+        }
+
+    @property
+    # ID: ed053380-56ba-4205-81d1-99e8550429f4
+    def details(self) -> dict[str, Any]:
+        """Backwards-compatible alias for structured finding context."""
+        return self.context
+
+    @details.setter
+    # ID: f2ff68c6-7a4c-4cca-a7dc-4071d8b49c13
+    def details(self, value: dict[str, Any] | None) -> None:
+        self.context = value or {}

shared/models/audit_rendering.py

@@ -0,0 +1,30 @@
+# src/shared/models/audit_rendering.py
+"""Data models and utilities for audit rendering."""
+
+from dataclasses import dataclass
+
+from shared.models import AuditFinding, AuditSeverity
+
+
+@dataclass(frozen=True)
+# ID: 94bf6a41-7736-4851-a138-c863f00c24a9
+class SeverityGroup:
+    """Immutable group of findings by severity."""
+
+    severity: AuditSeverity
+    findings: tuple[AuditFinding, ...]
+
+    @property
+    # ID: 68064f46-680f-4c86-b7e3-8fd054e56b97
+    def count(self) -> int:
+        return len(self.findings)
+
+
+# ID: 8d6c0049-4f2f-4102-829a-486e084b6bab
+def get_severity_style(severity: AuditSeverity) -> str:
+    """Get Rich console style string for a severity level."""
+    return {
+        AuditSeverity.BLOCK: "bold red",
+        AuditSeverity.HIGH: "bold yellow",
+        AuditSeverity.INFO: "cyan",
+    }.get(severity, "white")