codrspot-processor-mcp 0.1.0__py3-none-any.whl

codepreproc_client/layer2_tooling/materialization/snippet/assembler.py

@@ -0,0 +1,503 @@
+"""Two-phase snippet assembler (Subfase 4.2 + 4.3).
+
+Phase 1 (low-cost model): intent resolution → snippet selection → variable binding
+                           → scope definition → template instantiation → AssemblyPayload.
+Phase 2 (high-capability model): controlled integration → files written to disk.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+from codepreproc_client.layer1_business.api_client.reasoning_client import LLMRouter
+from contracts.dtos import (
+    AssemblyExecutionPack,
+    AssemblyIntent,
+    AssemblyPayload,
+    AssemblyResult,
+    AssemblyScope,
+    FailureReason,
+    InstantiatedSnippet,
+    LLMPolicy,
+    LLMUsageStats,
+    ProjectConfig,
+    SnippetSelection,
+    VariableBinding,
+)
+from codepreproc_client.layer2_tooling.materialization.snippet.loader import SnippetLoader
+
+logger = logging.getLogger(__name__)
+
+_ASSEMBLY_SYSTEM_PROMPT = """\
+You are a DDD snippet assembly planner. You select snippets from a library and resolve variables.
+You must respond ONLY with valid JSON. No markdown, no explanation.
+
+DDD layer rules:
+- domain: no external dependencies
+- application: orchestrates domain, no infrastructure or interface imports
+- infrastructure: implements adapters, no interface imports
+- interface: handles IO, may import application and domain
+
+Forbidden actions:
+- generating code without a snippet
+- modifying out-of-scope files
+- introducing new architecture
+- duplicating logic
+- cross-layer violations
+"""
+
+_INTEGRATION_SYSTEM_PROMPT = """\
+You are a DDD integration specialist. You integrate pre-instantiated code snippets into a codebase.
+
+Rules:
+- Do not rewrite entire files unless required
+- Preserve existing logic
+- Merge changes locally
+- Respect imports and structure
+- Maintain layer boundaries
+- No missing imports
+- No unused variables
+- No broken references
+- No duplicate logic
+- Consistent naming
+- Valid DDD layering
+
+Output format: JSON with keys "files_modified" (list of paths), "files_created" (list of paths),
+"integration_notes" (list of strings explaining integration decisions).
+No explanations outside the JSON.
+"""
+
+_DDD_CONSTRAINTS = [
+    "domain has no external dependencies",
+    "application orchestrates domain only",
+    "infrastructure implements adapters, no interface imports",
+    "interface handles IO, may import application and domain",
+    "no circular dependencies between layers",
+    "no free code generation outside snippet templates",
+    "no modifications to out-of-scope files",
+]
+
+
+class SnippetAssembler:
+    """Runs the two-phase assembly pipeline."""
+
+    def __init__(
+        self,
+        loader: SnippetLoader,
+        router: LLMRouter,
+        project: ProjectConfig,
+        logger: logging.Logger | None = None,
+    ) -> None:
+        self._loader = loader
+        self._router = router
+        self._project = project
+        self._logger = logger or logging.getLogger(__name__)
+
+    async def assemble(
+        self,
+        user_prompt: str,
+        task_id: str,
+        policy: LLMPolicy,
+    ) -> AssemblyExecutionPack:
+        """Run both phases and return the execution pack."""
+        usage_log: list[LLMUsageStats] = []
+
+        # ------------------------------------------------------------------
+        # Phase 1: intent → selection → variables → scope → instantiation
+        # ------------------------------------------------------------------
+        intent, usage = await self._resolve_intent(user_prompt, policy)
+        usage_log.append(usage)
+        if intent is None:
+            return AssemblyExecutionPack(
+                success=False,
+                task_id=task_id,
+                project_id=self._project.project_id,
+                failure=FailureReason(
+                    code="ASSEMBLY_INTENT_FAILED",
+                    stage="intent_resolution",
+                    message="Low-cost model could not resolve snippet assembly intent",
+                    actionable_hint="Ensure the prompt specifies entity, action, framework and language",
+                ),
+                llm_usage=usage_log,
+            )
+
+        available = self._loader.list_snippets(framework=intent.framework, language=intent.language)
+        if not available:
+            return AssemblyExecutionPack(
+                success=False,
+                task_id=task_id,
+                project_id=self._project.project_id,
+                intent=intent,
+                failure=FailureReason(
+                    code="NO_SNIPPETS_AVAILABLE",
+                    stage="snippet_selection",
+                    message=f"No snippets for framework={intent.framework} language={intent.language}",
+                    actionable_hint="Check snippets in the cp_snippets PostgreSQL table and the snippets/ directory",
+                ),
+                llm_usage=usage_log,
+            )
+
+        selection, usage = await self._select_snippets(user_prompt, intent, available, policy)
+        usage_log.append(usage)
+
+        variables, usage = await self._resolve_variables(user_prompt, intent, policy)
+        usage_log.append(usage)
+
+        scope = self._build_scope(selection, variables, intent)
+
+        snippets = self._loader.get_snippets_by_ids(
+            selection.selected_snippet_ids, intent.framework, intent.language
+        )
+        if not snippets:
+            return AssemblyExecutionPack(
+                success=False,
+                task_id=task_id,
+                project_id=self._project.project_id,
+                intent=intent,
+                selection=selection,
+                failure=FailureReason(
+                    code="NO_SNIPPETS_SELECTED",
+                    stage="snippet_selection",
+                    message="No valid snippets could be loaded for the selected IDs",
+                    actionable_hint=f"Selected IDs: {selection.selected_snippet_ids}. "
+                                    "Verify ids exist in the registry for the resolved framework/language.",
+                ),
+                llm_usage=usage_log,
+            )
+        instantiated = [self._instantiate(s, variables) for s in snippets]
+
+        payload = AssemblyPayload(
+            snippets=instantiated,
+            variables=variables,
+            scope=scope,
+            target_files=[path for s in instantiated for path in s.produces],
+            ddd_constraints=_DDD_CONSTRAINTS,
+        )
+
+        # ------------------------------------------------------------------
+        # Phase 2: controlled integration
+        # ------------------------------------------------------------------
+        result, usage = await self._integrate(user_prompt, payload, policy)
+        usage_log.append(usage)
+
+        # Write files to disk
+        written = self._write_files(instantiated, payload.scope)
+
+        return AssemblyExecutionPack(
+            success=True,
+            task_id=task_id,
+            project_id=self._project.project_id,
+            intent=intent,
+            selection=selection,
+            variables=variables,
+            payload=payload,
+            result=result or AssemblyResult(
+                files_created=written,
+                integration_notes=["Files written from instantiated snippets"],
+            ),
+            llm_usage=usage_log,
+        )
+
+    # ------------------------------------------------------------------
+    # Phase 1 steps
+    # ------------------------------------------------------------------
+
+    async def _resolve_intent(
+        self, user_prompt: str, policy: LLMPolicy
+    ) -> tuple[AssemblyIntent | None, LLMUsageStats]:
+        available_frameworks = list({
+            s.framework for s in self._loader.list_snippets()
+        })
+        available_languages = list({
+            s.language for s in self._loader.list_snippets()
+        })
+        messages = [
+            {"role": "system", "content": _ASSEMBLY_SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": (
+                    f"User request: {user_prompt}\n\n"
+                    f"Available frameworks: {available_frameworks}\n"
+                    f"Available languages: {available_languages}\n\n"
+                    "Respond with JSON:\n"
+                    '{"intent": "...", "entity": "EntityName", "action": "create|update|delete|list", '
+                    '"framework": "nest|fastapi|flutter|laravel", "language": "typescript|python|dart|php", "pattern": "ddd"}'
+                ),
+            },
+        ]
+        result, usage = await self._router.chat(
+            task_type="snippet_assembly",
+            policy=policy,
+            messages=messages,
+            schema=None,
+            max_tokens=256,
+            temperature=0.0,
+            project_id=self._project.project_id,
+            task_id="intent",
+        )
+        try:
+            data = _parse_json(result)
+            return AssemblyIntent.model_validate(data), usage
+        except Exception as exc:
+            self._logger.warning("Intent resolution failed: %s", exc)
+            return None, usage
+
+    async def _select_snippets(
+        self,
+        user_prompt: str,
+        intent: AssemblyIntent,
+        available: list,
+        policy: LLMPolicy,
+    ) -> tuple[SnippetSelection, LLMUsageStats]:
+        snippet_list = "\n".join(
+            f"- id={s.id} layer={s.layer} tags={s.tags}" for s in available
+        )
+        messages = [
+            {"role": "system", "content": _ASSEMBLY_SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": (
+                    f"User request: {user_prompt}\n"
+                    f"Intent: {intent.model_dump_json()}\n\n"
+                    f"Available snippets for {intent.framework}/{intent.language}:\n{snippet_list}\n\n"
+                    "Select only the snippets needed. Use exact ids. Respond with JSON:\n"
+                    '{"selected_snippet_ids": ["id1", "id2"], "rationale": "..."}'
+                ),
+            },
+        ]
+        result, usage = await self._router.chat(
+            task_type="snippet_assembly",
+            policy=policy,
+            messages=messages,
+            schema=None,
+            max_tokens=256,
+            temperature=0.0,
+            project_id=self._project.project_id,
+            task_id="selection",
+        )
+        try:
+            data = _parse_json(result)
+            return SnippetSelection.model_validate(data), usage
+        except Exception:
+            # Fallback: select all available snippets
+            return SnippetSelection(
+                selected_snippet_ids=[s.id for s in available],
+                rationale="fallback: all available snippets selected",
+            ), usage
+
+    async def _resolve_variables(
+        self, user_prompt: str, intent: AssemblyIntent, policy: LLMPolicy
+    ) -> tuple[VariableBinding, LLMUsageStats]:
+        messages = [
+            {"role": "system", "content": _ASSEMBLY_SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": (
+                    f"Entity: {intent.entity}\n"
+                    f"Framework: {intent.framework}\n\n"
+                    "Derive variable values. Rules:\n"
+                    "- EntityName: PascalCase of the entity\n"
+                    "- entityName: camelCase (TypeScript/Dart) or snake_case (Python/PHP)\n"
+                    "- moduleName: kebab-case (NestJS), snake_case (FastAPI/Flutter/Laravel)\n\n"
+                    "Respond with JSON:\n"
+                    '{"EntityName": "...", "entityName": "...", "moduleName": "..."}'
+                ),
+            },
+        ]
+        result, usage = await self._router.chat(
+            task_type="snippet_assembly",
+            policy=policy,
+            messages=messages,
+            schema=None,
+            max_tokens=128,
+            temperature=0.0,
+            project_id=self._project.project_id,
+            task_id="variables",
+        )
+        try:
+            data = _parse_json(result)
+            return VariableBinding.model_validate(data), usage
+        except Exception:
+            # Fallback: derive from intent.entity
+            entity = intent.entity
+            entity_pascal = _to_pascal(entity)
+            if intent.language in {"python", "dart", "php"}:
+                entity_lower = _to_snake(entity)
+                module = _to_snake(entity)
+            else:
+                entity_lower = _to_camel(entity)
+                module = _to_kebab(entity)
+            return VariableBinding(
+                EntityName=entity_pascal,
+                entityName=entity_lower,
+                moduleName=module,
+            ), usage
+
+    # ------------------------------------------------------------------
+    # Phase 2: controlled integration
+    # ------------------------------------------------------------------
+
+    async def _integrate(
+        self,
+        user_prompt: str,
+        payload: AssemblyPayload,
+        policy: LLMPolicy,
+    ) -> tuple[AssemblyResult | None, LLMUsageStats]:
+        snippets_summary = "\n".join(
+            f"- {s.snippet_id} ({s.layer}) → {s.produces}"
+            for s in payload.snippets
+        )
+        messages = [
+            {"role": "system", "content": _INTEGRATION_SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": (
+                    f"Request: {user_prompt}\n\n"
+                    f"Variables: {payload.variables.model_dump_json()}\n\n"
+                    f"Snippets to integrate:\n{snippets_summary}\n\n"
+                    f"Scope — allowed files:\n" + "\n".join(f"  - {f}" for f in payload.scope.allowed_files) + "\n\n"
+                    f"DDD constraints:\n" + "\n".join(f"  - {c}" for c in payload.ddd_constraints) + "\n\n"
+                    "Respond with JSON:\n"
+                    '{"files_modified": [], "files_created": ["path1", ...], "integration_notes": ["..."]}'
+                ),
+            },
+        ]
+        result, usage = await self._router.chat(
+            task_type="snippet_integration",
+            policy=policy,
+            messages=messages,
+            schema=None,
+            max_tokens=512,
+            temperature=0.0,
+            project_id=self._project.project_id,
+            task_id="integration",
+        )
+        try:
+            data = _parse_json(result)
+            return AssemblyResult.model_validate(data), usage
+        except Exception as exc:
+            self._logger.warning(
+                "Integration phase failed — files will be written without LLM guidance: %s", exc
+            )
+            return None, usage
+
+    # ------------------------------------------------------------------
+    # File writing
+    # ------------------------------------------------------------------
+
+    def _build_scope(
+        self, selection: SnippetSelection, variables: VariableBinding, intent: AssemblyIntent
+    ) -> AssemblyScope:
+        snippets = self._loader.get_snippets_by_ids(
+            selection.selected_snippet_ids, intent.framework, intent.language
+        )
+        allowed: list[str] = []
+        for spec in snippets:
+            for path_template in spec.produces:
+                resolved = _apply_variables(path_template, variables)
+                allowed.append(resolved)
+        return AssemblyScope(
+            allowed_files=allowed,
+            forbidden=["global_config", "unrelated_modules"],
+        )
+
+    def _instantiate(self, spec: Any, variables: VariableBinding) -> InstantiatedSnippet:
+        code = _apply_variables(spec.template, variables)
+        produces = [_apply_variables(p, variables) for p in spec.produces]
+        return InstantiatedSnippet(
+            snippet_id=spec.id,
+            framework=spec.framework,
+            language=spec.language,
+            layer=spec.layer,
+            produces=produces,
+            instantiated_code=code,
+            integration=spec.integration,
+        )
+
+    def _write_files(self, snippets: list[InstantiatedSnippet], scope: AssemblyScope) -> list[str]:
+        written: list[str] = []
+        for snippet in snippets:
+            for i, relative_path in enumerate(snippet.produces):
+                if relative_path not in scope.allowed_files:
+                    self._logger.warning("Skipping out-of-scope file: %s", relative_path)
+                    continue
+                full_path = self._project.root / relative_path
+                full_path.parent.mkdir(parents=True, exist_ok=True)
+                full_path.write_text(snippet.instantiated_code, encoding="utf-8")
+                written.append(str(full_path))
+                self._logger.info("Written: %s", full_path)
+        return written
+
+
+# ------------------------------------------------------------------
+# Utility helpers
+# ------------------------------------------------------------------
+
+def _parse_json(result: Any) -> dict:
+    """Extract first valid JSON object from LLM result.
+
+    Tolerates: markdown code fences, surrounding explanation text,
+    non-string content (empty dict from FakeProvider, etc.).
+    """
+    if isinstance(result, dict):
+        content = result.get("content", "")
+        if isinstance(content, dict):
+            # Provider returned a pre-parsed dict (e.g. FakeProvider) — use directly
+            return content
+        if not isinstance(content, str):
+            content = ""
+    else:
+        content = str(result)
+
+    # Strip markdown code fences (``` or ```json)
+    content = re.sub(r"```(?:json)?\s*", "", content)
+    content = content.strip()
+
+    # 1. Try parsing the whole cleaned content directly
+    try:
+        return json.loads(content)
+    except json.JSONDecodeError:
+        pass
+
+    # 2. Find the first {...} block in case there is surrounding text
+    match = re.search(r"\{.*\}", content, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group())
+        except json.JSONDecodeError:
+            pass
+
+    raise ValueError(f"No valid JSON object found in LLM response: {content[:300]!r}")
+
+
+def _apply_variables(template: str, variables: VariableBinding) -> str:
+    result = template
+    result = result.replace("{{EntityName}}", variables.EntityName)
+    result = result.replace("{{entityName}}", variables.entityName)
+    result = result.replace("{{moduleName}}", variables.moduleName)
+    for key, value in variables.extra.items():
+        result = result.replace(f"{{{{{key}}}}}", value)
+    return result
+
+
+def _to_pascal(name: str) -> str:
+    return "".join(word.capitalize() for word in re.split(r"[_\-\s]+", name) if word)
+
+
+def _to_camel(name: str) -> str:
+    words = [w for w in re.split(r"[_\-\s]+", name) if w]
+    return words[0].lower() + "".join(w.capitalize() for w in words[1:]) if words else name
+
+
+def _to_snake(name: str) -> str:
+    s = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", name)
+    s = re.sub(r"([a-z\d])([A-Z])", r"\1_\2", s)
+    return re.sub(r"[\-\s]+", "_", s).lower()
+
+
+def _to_kebab(name: str) -> str:
+    return _to_snake(name).replace("_", "-")

codepreproc_client/layer2_tooling/materialization/snippet/loader.py

@@ -0,0 +1,187 @@
+"""Loads snippets from the registry and snippet files (or PostgreSQL)."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Iterator
+
+import yaml
+
+from contracts.dtos import SnippetSpec
+
+if TYPE_CHECKING:
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+class SnippetLoader:
+    """Reads snippet_registry.yaml and resolves snippet files relative to a base dir."""
+
+    def __init__(self, registry_path: Path, base_dir: Path | None = None) -> None:
+        self._registry_path = registry_path
+        # base_dir defaults to registry_path.parent.parent:
+        # registry/snippet_registry.yaml → registry → home_dir
+        self._base_dir = base_dir or registry_path.parent.parent
+        self._registry: dict | None = None
+        # Tuple key (id, framework, language) avoids "::"-separator collisions
+        self._cache: dict[tuple[str, str, str], SnippetSpec] = {}
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def list_snippets(
+        self,
+        *,
+        framework: str | None = None,
+        language: str | None = None,
+        layer: str | None = None,
+        tags: list[str] | None = None,
+    ) -> list[SnippetSpec]:
+        """Return all snippets matching the given filters."""
+        results = []
+        for spec in self._iter_registry_entries():
+            if framework and spec.framework != framework:
+                continue
+            if language and spec.language != language:
+                continue
+            if layer and spec.layer != layer:
+                continue
+            if tags and not all(t in spec.tags for t in tags):
+                continue
+            results.append(spec)
+        return results
+
+    def get_snippet(self, snippet_id: str, framework: str, language: str) -> SnippetSpec | None:
+        """Return a fully loaded snippet by (id, framework, language)."""
+        cache_key = (snippet_id, framework, language)
+        if cache_key in self._cache:
+            return self._cache[cache_key]
+
+        for entry in self._registry_data().get("snippets", []):
+            if (
+                entry.get("id") == snippet_id
+                and entry.get("framework") == framework
+                and entry.get("language") == language
+            ):
+                spec = self._load_snippet_file(entry)
+                if spec is not None:
+                    self._cache[cache_key] = spec
+                return spec
+        return None
+
+    def get_snippets_by_ids(self, ids: list[str], framework: str, language: str) -> list[SnippetSpec]:
+        """Return snippets in full transitive dependency order (DFS, deepest deps first)."""
+        seen: set[str] = set()
+        result: list[SnippetSpec] = []
+        for snippet_id in ids:
+            self._collect_deps(snippet_id, framework, language, seen, result)
+        return result
+
+    def _collect_deps(
+        self,
+        snippet_id: str,
+        framework: str,
+        language: str,
+        seen: set[str],
+        result: list[SnippetSpec],
+    ) -> None:
+        """DFS: insert all transitive dependencies before the snippet itself."""
+        if snippet_id in seen:
+            return
+        seen.add(snippet_id)
+        spec = self.get_snippet(snippet_id, framework, language)
+        if spec is None:
+            return
+        for req_id in spec.requires:
+            self._collect_deps(req_id, framework, language, seen, result)
+        result.append(spec)
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    def _registry_data(self) -> dict:
+        if self._registry is None:
+            if not self._registry_path.exists():
+                logger.warning("snippet_registry.yaml not found at %s", self._registry_path)
+                self._registry = {}
+            else:
+                with self._registry_path.open(encoding="utf-8") as fh:
+                    self._registry = yaml.safe_load(fh) or {}
+        return self._registry
+
+    def _iter_registry_entries(self) -> Iterator[SnippetSpec]:
+        for entry in self._registry_data().get("snippets", []):
+            cache_key = (entry.get("id", ""), entry.get("framework", ""), entry.get("language", ""))
+            if cache_key in self._cache:
+                yield self._cache[cache_key]
+                continue
+            spec = self._load_snippet_file(entry)
+            if spec is not None:
+                yield spec
+
+    def _load_snippet_file(self, entry: dict) -> SnippetSpec | None:
+        relative_path = entry.get("path", "")
+        full_path = self._base_dir / relative_path
+        if not full_path.exists():
+            logger.warning("Snippet file not found: %s", full_path)
+            return None
+        try:
+            with full_path.open(encoding="utf-8") as fh:
+                data = yaml.safe_load(fh) or {}
+            # Merge registry-level metadata with file-level data
+            merged = {**entry, **data}
+            merged["path"] = relative_path
+            return SnippetSpec.model_validate(merged)
+        except Exception as exc:
+            logger.warning("Failed to load snippet %s: %s", full_path, exc)
+            return None
+
+    # ------------------------------------------------------------------
+    # API-backed factory
+    # ------------------------------------------------------------------
+
+    @classmethod
+    async def from_api(cls, *, base_url: str, jwt: str | None, base_dir: Path | None = None) -> "SnippetLoader":
+        """Build a SnippetLoader pre-populated from GET /v1/snippets.
+
+        Falls back to an empty snippet library (rather than raising) if no
+        bootstrap JWT is available yet - e.g. before any project is
+        registered - or the server is unreachable.
+        """
+        sentinel = Path("/dev/null")
+        instance = cls(registry_path=sentinel, base_dir=base_dir)
+
+        rows = []
+        if jwt is not None:
+            try:
+                from codepreproc_client.layer1_business.api_client.registry_client import list_snippets
+
+                rows = await list_snippets(base_url=base_url, jwt=jwt)
+            except Exception:
+                logger.warning("SnippetLoader: failed to load snippets from the API, starting empty", exc_info=True)
+
+        for spec in rows:
+            cache_key = (spec.id, spec.framework, spec.language)
+            instance._cache[cache_key] = spec
+            # Also populate _registry so list_snippets() works via _iter_registry_entries
+        # Build an in-memory registry structure so filtering still works
+        instance._registry = {
+            "snippets": [
+                {
+                    "id": spec.id,
+                    "framework": spec.framework,
+                    "language": spec.language,
+                    "type": spec.type,
+                    "layer": spec.layer,
+                    "tags": spec.tags,
+                    "path": spec.path,
+                }
+                for spec in rows
+            ]
+        }
+        logger.info("SnippetLoader: %d snippets loaded from the API", len(rows))
+        return instance