core-runtime 0.1.1__py3-none-any.whl

shared/utils/embedding_utils.py

@@ -0,0 +1,182 @@
+# src/shared/utils/embedding_utils.py
+
+"""
+Provides utilities for handling text embeddings, including chunking and aggregation.
+
+CORE contract:
+- "Embeddings" are produced by the Vectorizer role and are ALWAYS local.
+- No provider switching (no OpenAI/DeepSeek here).
+- No os.getenv/os.environ.
+- No fallback chains. Missing required settings => error.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+from typing import Protocol
+
+import httpx
+import numpy as np
+
+from shared.config import settings
+from shared.infrastructure.intent.operational_config import load_operational_config
+from shared.logger import getLogger
+from shared.utils.common_knowledge import normalize_text
+
+
+logger = getLogger(__name__)
+
+_CFG_EMB = load_operational_config().embedding
+
+
+def _require_setting(name: str) -> str:
+    """
+    Strict settings read (CORE-style):
+    - only from shared.config.settings (+ model_extra if used as backing store)
+    - no fallback chains
+    - missing/empty => ValueError
+    """
+    value = None
+
+    if hasattr(settings, name):
+        value = getattr(settings, name)
+    else:
+        extra = getattr(settings, "model_extra", {}) or {}
+        value = extra.get(name)
+
+    if value is None:
+        raise ValueError(f"Missing required setting: {name}")
+    if isinstance(value, str) and not value.strip():
+        raise ValueError(f"Setting '{name}' is empty")
+
+    return str(value)
+
+
+# ID: 0c956ad0-a9d9-4cdf-bc8d-af9bccc4e30c
+class Embeddable(Protocol):
+    """Defines the interface for any service that can create embeddings."""
+
+    # ID: 3ace367e-4136-4dd0-95b9-ec75462ff78d
+    async def get_embedding(self, text: str) -> list[float]: ...
+
+
+class _Adapter:
+    """Internal adapter to make EmbeddingService conform to the Embeddable protocol."""
+
+    def __init__(self, service: Embeddable):
+        self._service = service
+
+    # ID: f6d67bd8-83e2-42d5-81d3-07c668642568
+    async def get_embedding(self, text: str) -> list[float]:
+        return await self._service.get_embedding(text)
+
+
+def _chunk_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
+    """Splits text into overlapping chunks."""
+    if not text:
+        return []
+    chunks: list[str] = []
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        chunks.append(text[start:end])
+        start += max(1, chunk_size - chunk_overlap)
+    return chunks
+
+
+# ID: 76aee7d7-fe49-4271-87b8-01fc9b074028
+def sha256_hex(text: str) -> str:
+    """Computes the SHA256 hex digest for a string."""
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+# ID: c3c32fe7-d434-43c6-b6a2-647afe213b4e
+class EmbeddingService:
+    """
+    Local-only embeddings client (Vectorizer role contract).
+
+    Expected settings (NO fallbacks):
+    - LOCAL_EMBEDDING_API_URL
+    - LOCAL_EMBEDDING_MODEL_NAME
+
+    Endpoint:
+    - POST {LOCAL_EMBEDDING_API_URL}/api/embed with {model, input} (Ollama 0.4+)
+    """
+
+    def __init__(self, timeout: float = _CFG_EMB.utils_request_timeout_sec) -> None:
+        self.base = _require_setting("LOCAL_EMBEDDING_API_URL").rstrip("/")
+        self.model = _require_setting("LOCAL_EMBEDDING_MODEL_NAME")
+        self.timeout = timeout
+
+        self.endpoint = "/api/embed"
+        self.headers: dict[str, str] = {"Content-Type": "application/json"}
+
+        logger.info(
+            "EmbeddingService initialized (local) base=%s model=%s",
+            self.base,
+            self.model,
+        )
+
+    # ID: b0db34ef-e89a-4910-b264-8e939cc14f9a
+    async def get_embedding(self, text: str) -> list[float]:
+        """Return a single embedding vector for the given text."""
+        url = f"{self.base}{self.endpoint}"
+        payload = {"model": self.model, "input": text, "options": {"num_ctx": 8192}}
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            resp = await client.post(url, json=payload, headers=self.headers)
+
+        if resp.status_code != 200:
+            logger.error(
+                "HTTP error from local embedding API: %s - %s",
+                resp.status_code,
+                resp.text,
+            )
+            raise RuntimeError(f"Embedding API HTTP {resp.status_code}")
+
+        data = resp.json()
+        embeddings = data.get("embeddings")
+        if not embeddings or not embeddings[0]:
+            logger.error("Local embedding service returned no vector.")
+            raise RuntimeError("No vector returned from embedding service")
+
+        return embeddings[0]
+
+
+# ID: 14fd20cf-3101-4970-84b0-942ea9fffda3
+def build_embedder_from_env() -> Embeddable:
+    """
+    Backwards-compatible factory name.
+
+    CORE contract: this is settings-based and local-only.
+    """
+    return _Adapter(EmbeddingService())
+
+
+# ID: 31b34c50-e03b-4839-b588-d2a0c76a9004
+async def chunk_and_embed(
+    embedder: Embeddable,
+    text: str,
+    chunk_size: int = _CFG_EMB.chunk_size,
+    chunk_overlap: int = _CFG_EMB.chunk_overlap,
+) -> np.ndarray:
+    """
+    Chunks text, gets embeddings for each chunk in parallel, and returns the
+    averaged embedding vector for the entire text.
+    """
+    text = normalize_text(text)
+    chunks = _chunk_text(text, chunk_size, chunk_overlap)
+    if not chunks:
+        raise ValueError("Cannot generate embedding for empty text.")
+
+    chunk_vectors = await asyncio.gather(*(embedder.get_embedding(c) for c in chunks))
+
+    vector_array = np.array(chunk_vectors, dtype=np.float32)
+    mean_vector = np.mean(vector_array, axis=0)
+
+    norm = np.linalg.norm(mean_vector)
+    if norm == 0:
+        return mean_vector
+
+    return mean_vector / norm

shared/utils/glob_match.py

@@ -0,0 +1,93 @@
+# src/shared/utils/glob_match.py
+
+"""
+Glob matching with gitignore semantics.
+
+Single entry point for path-pattern evaluation across src/. Replaces
+direct pathlib.Path.match() usage, which on Python 3.12 has two surprising
+semantics that produced silent under-enforcement at security-sensitive
+sites: ** is treated as single-segment *, and matching is suffix-only.
+
+Per ADR-012, all in-scope src/ call sites use these helpers; the
+audit_context.py compensation helpers (_include_matches, _is_excluded)
+are out of scope for this ADR and migrate separately under issue #117.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+from pathspec.patterns.gitignore.spec import GitIgnoreSpecPattern
+
+
+# ID: 57255383-a41d-4c3e-b003-dd5ba8ce622f
+def matches_glob(path: str | Path, pattern: str) -> bool:
+    """
+    Test whether a path matches a glob pattern under gitignore semantics.
+
+    Uses pathspec's GitIgnoreSpecPattern. Patterns follow gitignore.5
+    conventions: leading or middle '/' anchors at root; '*' matches any
+    run of characters except '/'; '**' matches any number of path
+    segments including zero; trailing '/' restricts to directories.
+
+    Args:
+        path: Path-like object or string. path inputs are treated as
+            repo-relative POSIX paths; absolute paths and ./ prefixes
+            are normalized to relative form. Converted to POSIX form for
+            consistent matching across platforms.
+        pattern: Glob pattern string. Empty patterns return False.
+
+    Returns:
+        True if the path matches the pattern, False otherwise.
+
+    Examples:
+        >>> matches_glob("src/main.py", "src/**/*.py")
+        True
+        >>> matches_glob("src/api/sub/main.py", "src/**/*.py")
+        True
+        >>> matches_glob("var/secrets/k.txt", "**/secrets/**")
+        True
+        >>> matches_glob("var/secrets/k.txt", "secrets/*")
+        False
+        >>> matches_glob("/src/main.py", "src/**/*.py")
+        True
+        >>> matches_glob("./src/main.py", "src/**/*.py")
+        True
+    """
+    if not pattern:
+        return False
+
+    if isinstance(path, Path):
+        path_str = path.as_posix()
+    else:
+        path_str = str(path).replace("\\", "/")
+
+    if path_str.startswith("./"):
+        path_str = path_str[2:]
+    path_str = path_str.lstrip("/")
+
+    compiled = GitIgnoreSpecPattern(pattern)
+    regex = compiled.regex
+    if regex is None:
+        return False
+    return bool(regex.match(path_str))
+
+
+# ID: b5d94dcf-5791-41a0-834f-8b2aa25b32f2
+def matches_any_glob(path: str | Path, patterns: Iterable[str]) -> bool:
+    """
+    Test whether a path matches any of multiple glob patterns.
+
+    Args:
+        path: Path-like object or string.
+        patterns: Iterable of pattern strings.
+
+    Returns:
+        True if the path matches at least one pattern.
+
+    Examples:
+        >>> matches_any_glob("var/secrets/k.txt", ["**/.env", "**/secrets/**"])
+        True
+    """
+    return any(matches_glob(path, pat) for pat in patterns)

shared/utils/header_tools.py

@@ -0,0 +1,177 @@
+# src/shared/utils/header_tools.py
+"""
+Provides a deterministic tool for parsing and reconstructing Python file headers
+according to CORE's constitutional style guide.
+"""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass, field
+
+
+@dataclass
+# ID: 4a498b02-ef0b-4ce2-bd66-d8289669cd8f
+class HeaderComponents:
+    """A data class to hold the parsed components of a Python file header."""
+
+    location: str | None = None
+    module_description: str | None = None
+    has_future_import: bool = False
+    other_imports: list[str] = field(default_factory=list)
+    body: list[str] = field(default_factory=list)
+
+
+class _HeaderTools:
+    """A stateless utility class for parsing and reconstructing file headers."""
+
+    @staticmethod
+    # ID: 8f8fa33d-1ab8-4ee8-8dc7-a71355167611
+    def parse(source_code: str) -> HeaderComponents:
+        """Parses the source code and extracts header components."""
+        components = HeaderComponents()
+        lines = source_code.splitlines()
+        if not lines:
+            return components
+
+        try:
+            tree = ast.parse(source_code)
+        except SyntaxError:
+            components.body = lines
+            return components
+
+        # Find the end of the header section (last docstring or import)
+        last_header_line = 0
+        header_nodes = []
+        for node in tree.body:
+            is_docstring = isinstance(node, ast.Expr) and isinstance(
+                node.value, ast.Constant
+            )
+            is_import = isinstance(node, (ast.Import, ast.ImportFrom))
+            if is_docstring or is_import:
+                # FIXED: Added null-safety for MyPy
+                node_end = getattr(node, "end_lineno", node.lineno)
+                last_header_line = node_end if node_end is not None else 0
+                header_nodes.append(node)
+            else:
+                # First non-header node marks the end of the header
+                break
+
+        # Body starts after the header, skipping blank lines
+        body_start_index = last_header_line
+        while body_start_index < len(lines) and not lines[body_start_index].strip():
+            body_start_index += 1
+
+        components.body = lines[body_start_index:]
+
+        # Process Header Content
+        if lines and lines[0].strip().startswith("#"):
+            components.location = lines[0]
+
+        # Extract docstring directly from source lines to preserve original quotes
+        docstring_node = (
+            tree.body[0] if tree.body and isinstance(tree.body[0], ast.Expr) else None
+        )
+        if (
+            docstring_node
+            and hasattr(docstring_node, "lineno")
+            and hasattr(docstring_node, "end_lineno")
+        ):
+            # FIXED: Added null-safety for MyPy arithmetic
+            s_lineno = docstring_node.lineno
+            e_lineno = docstring_node.end_lineno
+            if s_lineno is not None and e_lineno is not None:
+                start_line = s_lineno - 1
+                end_line = e_lineno - 1
+
+                # Extract lines including quotes
+                docstring_lines = lines[start_line : end_line + 1]
+
+                # Preserve original formatting by joining lines
+                if docstring_lines:
+                    # Detect if it's a multi-line docstring
+                    first_line = docstring_lines[0].strip()
+                    last_line = docstring_lines[-1].strip()
+
+                    # Check if it starts and ends with quotes
+                    if first_line.startswith(
+                        ('"""', "'''", '"', "'")
+                    ) and last_line.endswith(('"""', "'''", '"', "'")):
+                        # For single-line docstrings
+                        if len(docstring_lines) == 1:
+                            components.module_description = docstring_lines[0].strip()
+                        else:
+                            # For multi-line docstrings, preserve all lines
+                            # Find the indentation level
+                            base_indent = len(docstring_lines[0]) - len(
+                                docstring_lines[0].lstrip()
+                            )
+                            # Strip consistent indentation
+                            stripped_lines = []
+                            for line in docstring_lines:
+                                if line.startswith(" " * base_indent):
+                                    stripped_lines.append(line[base_indent:])
+                                else:
+                                    stripped_lines.append(line)
+                            components.module_description = "\n".join(stripped_lines)
+
+        for node in header_nodes:
+            if isinstance(node, (ast.Import, ast.ImportFrom)):
+                import_line = ast.unparse(node)
+                if "from __future__ import annotations" in import_line:
+                    components.has_future_import = True
+                else:
+                    components.other_imports.append(import_line)
+
+        return components
+
+    @staticmethod
+    # ID: e85d9dde-b46f-43f7-b83f-106a63103c48
+    def reconstruct(components: HeaderComponents) -> str:
+        """Reconstructs the source code from its parsed components."""
+        parts = []
+
+        if components.location:
+            parts.append(components.location)
+
+        if components.module_description:
+            if parts and parts[-1].strip():
+                parts.append("")
+            parts.append(components.module_description)
+
+        imports_present = components.has_future_import or components.other_imports
+        if imports_present:
+            if parts and parts[-1].strip():
+                parts.append("")
+
+            if components.has_future_import:
+                parts.append("from __future__ import annotations")
+
+            if components.other_imports:
+                # Add a blank line between future import and other imports
+                if components.has_future_import:
+                    parts.append("")
+                parts.extend(sorted(components.other_imports))
+
+        if components.body:
+            # If there was any header content, ensure two blank lines before the body
+            if parts:
+                while parts and not parts[-1].strip():
+                    parts.pop()
+                parts.append("")
+                parts.append("")
+
+            # Remove leading and trailing blank lines from body
+            body_lines = components.body[:]
+            while body_lines and not body_lines[0].strip():
+                body_lines.pop(0)
+            while body_lines and not body_lines[-1].strip():
+                body_lines.pop()
+
+            parts.extend(body_lines)
+
+        return "\n".join(parts) + "\n"
+
+
+# Public alias to satisfy callers and tests expecting `HeaderTools`.
+HeaderTools = _HeaderTools

shared/utils/parallel_processor.py

@@ -0,0 +1,60 @@
+# src/shared/utils/parallel_processor.py
+
+"""
+Provides a reusable, throttled parallel processor for running async tasks
+concurrently.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from typing import TypeVar
+
+from shared.config import settings
+from shared.logger import getLogger
+
+
+logger = getLogger(__name__)
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+# ID: 08955ac4-99b0-4bac-b3e4-3c9deb938e68
+class ThrottledParallelProcessor:
+    """
+    A dedicated executor for running a worker function over a list of items
+    in parallel, with concurrency limited by the constitution.
+    """
+
+    def __init__(self, description: str = "Processing items..."):
+        self.concurrency_limit = settings.CORE_MAX_CONCURRENT_REQUESTS
+        self.description = description
+        logger.info(
+            "ThrottledParallelProcessor initialized with concurrency limit: %s",
+            self.concurrency_limit,
+        )
+
+    async def _process_items_async(
+        self, items: list[T], worker_fn: Callable[[T], Awaitable[R]]
+    ) -> list[R]:
+        semaphore = asyncio.Semaphore(self.concurrency_limit)
+        results: list[R] = []
+
+        async def _worker(item: T) -> R:
+            async with semaphore:
+                return await worker_fn(item)
+
+        tasks = [asyncio.create_task(_worker(item)) for item in items]
+
+        logger.debug("%s", self.description)
+        for task in asyncio.as_completed(tasks):
+            results.append(await task)
+
+        return results
+
+    # ID: d64f09ac-d05d-4a32-ad5d-87bf95d0efcf
+    async def run_async(
+        self, items: list[T], worker_fn: Callable[[T], Awaitable[R]]
+    ) -> list[R]:
+        return await self._process_items_async(items, worker_fn)

shared/utils/parsing.py

@@ -0,0 +1,189 @@
+# src/shared/utils/parsing.py
+"""
+Shared utilities for parsing structured data from unstructured text,
+primarily from Large Language Model (LLM) outputs.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import re
+from typing import Any, cast
+
+from shared.infrastructure.intent.operational_config import load_operational_config
+
+
+_CFG = load_operational_config().parsing
+
+
+# ID: 03987fc0-13ec-460a-a399-a89c7289eac6
+def extract_json_from_response(text: str) -> dict[Any, Any] | list[Any] | None:
+    """
+    Extracts a JSON object or array from a raw text response, making it robust
+    against common LLM formatting issues like introductory text.
+    """
+    # 1. Try to extract JSON from markdown code blocks
+    json_data = _extract_from_markdown(text)
+    if json_data is not None:
+        return cast(dict[Any, Any] | list[Any], json_data)
+
+    # 2. Fallback: Find raw JSON by matching braces/brackets
+    return cast(dict[Any, Any] | list[Any] | None, _extract_raw_json(text))
+
+
+def _extract_from_markdown(text: str) -> dict[Any, Any] | list[Any] | None:
+    pattern = r"```(?:json)?\s*(\{[\s\S]*?\}|\[[\s\S]*?\])\s*```"
+    match = re.search(pattern, text, re.DOTALL)
+
+    if not match:
+        return None
+
+    try:
+        return cast(dict[Any, Any] | list[Any], json.loads(match.group(1)))
+    except json.JSONDecodeError:
+        return None
+
+
+def _extract_raw_json(text: str) -> dict[Any, Any] | list[Any] | None:
+    first_brace = text.find("{")
+    first_bracket = text.find("[")
+
+    if first_brace == -1 and first_bracket == -1:
+        return None
+
+    if first_brace != -1 and (first_bracket == -1 or first_brace < first_bracket):
+        end_char = "}"
+        start_index = first_brace
+    else:
+        end_char = "]"
+        start_index = first_bracket
+
+    last_index = text.rfind(end_char)
+    if last_index <= start_index:
+        return None
+
+    try:
+        json_str = text[start_index : last_index + 1]
+        return cast(dict[Any, Any] | list[Any], json.loads(json_str))
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+
+# ID: d4c82c76-0762-4358-b7b4-13d4819fce6c
+def parse_write_blocks(text: str) -> dict[str, str]:
+    """
+    Parses a string for one or more [[write:file_path]]...[[/write]] blocks.
+    """
+    pattern = r"\[\[write:(.+?)\]\]\s*\n(.*?)\n\s*\[\[/write\]\]"
+    matches = re.findall(pattern, text, re.DOTALL)
+    return {path.strip(): content.strip() for path, content in matches}
+
+
+def _normalize_python_snippet(code: str) -> str:
+    """
+    Normalize a Python snippet extracted from an LLM response.
+    """
+    if not code:
+        return code
+
+    lines = code.splitlines()
+    if not lines:
+        return code
+
+    fixed_lines: list[str] = []
+    for idx, line in enumerate(lines):
+        if idx == 0:
+            stripped = line.lstrip()
+            if stripped.startswith(r"\n"):
+                stripped = stripped[2:]
+            if stripped.startswith("\\") and not stripped.startswith("\\\\"):
+                stripped = stripped.lstrip("\\")
+            fixed_lines.append(stripped)
+        else:
+            fixed_lines.append(line)
+
+    normalized = "\n".join(fixed_lines).strip()
+    try:
+        ast.parse(normalized)
+        return normalized
+    except SyntaxError:
+        return code
+
+
+def _is_valid_python_block(code: str) -> bool:
+    """
+    Heuristic check to see if a block contains actual Python logic.
+    Filters out blocks that are just comments or lack keywords.
+    """
+    if not code or not code.strip():
+        return False
+
+    lines = [line.strip() for line in code.splitlines() if line.strip()]
+    if not lines:
+        return False
+
+    # Reject if every line is a comment
+    if all(line.startswith("#") for line in lines):
+        return False
+
+    # Must contain at least one structural keyword
+    keywords = {
+        "def ",
+        "class ",
+        "import ",
+        "from ",
+        "@",
+        "async def ",
+        "return ",
+        "assert ",
+    }
+    return any(any(k in line for k in keywords) for line in lines)
+
+
+# ID: 44c9f1bf-9a35-46d1-8059-f0d82b745a58
+def extract_python_code_from_response(text: str) -> str | None:
+    """
+    Extract Python code from an LLM response using a prioritized scoring strategy.
+    """
+    if not text:
+        return None
+
+    candidates = []
+
+    pattern = r"```(\w*)\s*\n(.*?)\n\s*```"
+    matches = re.findall(pattern, text, re.DOTALL)
+
+    for lang, content in matches:
+        cleaned = content.strip()
+        if lang and lang.lower() not in ("python", "py", ""):
+            continue
+
+        if len(cleaned) > _CFG.min_block_len and _is_valid_python_block(cleaned):
+            candidates.append(cleaned)
+
+    if not candidates:
+        stripped = text.strip()
+        if _is_valid_python_block(stripped):
+            candidates.append(stripped)
+
+    if not candidates:
+        return None
+
+    # ID: 46c042d9-977d-4567-9dff-4bb63bb042b0
+    def score_candidate(code: str) -> float:
+        score = 0.0
+        if "def test_" in code:
+            score += _CFG.score_test_fn
+        if "class Test" in code:
+            score += _CFG.score_test_class
+        if "import " in code or "from " in code:
+            score += _CFG.score_import
+        if "pytest" in code or "unittest" in code:
+            score += _CFG.score_pytest
+        score += min(len(code), 5000) / 10000.0
+        return score
+
+    candidates.sort(key=score_candidate, reverse=True)
+
+    return cast(str, _normalize_python_snippet(candidates[0]))