component-mapper 0.1.0__py3-none-any.whl

component_mapper/mcp/registry_fetcher.py

@@ -0,0 +1,214 @@
+import asyncio
+import json
+import logging
+import time
+from pathlib import Path
+import aiohttp
+import aiofiles
+from component_mapper.config import RegistryConfig
+from component_mapper.models import RegistrySource
+
+logger = logging.getLogger(__name__)
+
+
+class RegistryFetcher:
+    def __init__(self, config: RegistryConfig):
+        self.config = config
+        self._memory_cache: dict[str, dict] = {}
+        self._cache_timestamps: dict[str, float] = {}
+        self._semaphore = asyncio.Semaphore(config.max_concurrent_fetches)
+        self._disk_cache_dir = Path(".cache/registry_http")
+        self._disk_cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def _cache_key(self, name: str, source: RegistrySource) -> str:
+        return f"{source.value}:{name}"
+
+    def _disk_cache_path(self, name: str, source: RegistrySource) -> Path:
+        return self._disk_cache_dir / f"{source.value}_{name}.json"
+
+    def _is_cache_fresh(self, cache_key: str) -> bool:
+        ts = self._cache_timestamps.get(cache_key, 0)
+        ttl_seconds = self.config.http_cache_ttl_hours * 3600
+        return (time.time() - ts) < ttl_seconds
+
+    def _base_url(self, source: RegistrySource) -> str:
+        if source == RegistrySource.SHADCN:
+            return self.config.shadcn_registry_base_url
+        return (
+            self.config.custom_registry_base_url or self.config.shadcn_registry_base_url
+        )
+
+    async def fetch_component(
+        self,
+        name: str,
+        source: RegistrySource = RegistrySource.SHADCN,
+    ) -> dict:
+        cache_key = self._cache_key(name, source)
+
+        # Memory cache
+        if cache_key in self._memory_cache and self._is_cache_fresh(cache_key):
+            return self._memory_cache[cache_key]
+
+        # Disk cache
+        disk_path = self._disk_cache_path(name, source)
+        if disk_path.exists():
+            stat_age = time.time() - disk_path.stat().st_mtime
+            if stat_age < self.config.http_cache_ttl_hours * 3600:
+                try:
+                    async with aiofiles.open(disk_path, "r") as f:
+                        data = json.loads(await f.read())
+                    self._memory_cache[cache_key] = data
+                    self._cache_timestamps[cache_key] = time.time()
+                    return data
+                except Exception:
+                    pass
+
+        # HTTP fetch
+        async with self._semaphore:
+            url = f"{self._base_url(source)}/{name}.json"
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(
+                        url,
+                        timeout=aiohttp.ClientTimeout(
+                            total=self.config.fetch_timeout_seconds
+                        ),
+                    ) as resp:
+                        if resp.status == 200:
+                            data = await resp.json(content_type=None)
+                        else:
+                            logger.warning(
+                                "Registry fetch %s returned %d", url, resp.status
+                            )
+                            data = {"name": name, "files": []}
+            except Exception as exc:
+                logger.warning("Registry fetch failed for %s: %s", name, exc)
+                data = {"name": name, "files": []}
+
+        self._memory_cache[cache_key] = data
+        self._cache_timestamps[cache_key] = time.time()
+
+        # Persist to disk
+        try:
+            async with aiofiles.open(disk_path, "w") as f:
+                await f.write(json.dumps(data))
+        except Exception:
+            pass
+
+        return data
+
+    async def fetch_many(
+        self,
+        names: list[str],
+        source: RegistrySource = RegistrySource.SHADCN,
+    ) -> dict[str, dict]:
+        """Concurrent fetch under semaphore. Returns name -> registry JSON."""
+        tasks = [self.fetch_component(n, source) for n in names]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        return {
+            name: (r if isinstance(r, dict) else {"name": name, "files": []})
+            for name, r in zip(names, results)
+        }
+
+    async def fetch_source_code(self, name: str) -> str:
+        """Extract TypeScript source from registry JSON files array."""
+        data = await self.fetch_component(name, RegistrySource.SHADCN)
+        files = data.get("files", [])
+        if files:
+            return files[0].get("content", "")
+        return ""
+
+    async def fetch_from_external(
+        self,
+        url_template: str,
+        component_name: str,
+        registry_name: str,
+    ) -> dict:
+        """Fetch a component from an external registry using its URL template.
+
+        url_template uses {name} as the placeholder e.g.
+        "https://bundui.io/r/{name}.json" → "https://bundui.io/r/pagination.json"
+        """
+        cache_key = f"external:{registry_name}:{component_name}"
+
+        if cache_key in self._memory_cache and self._is_cache_fresh(cache_key):
+            return self._memory_cache[cache_key]
+
+        disk_path = self._disk_cache_dir / f"ext_{registry_name}_{component_name}.json"
+        if disk_path.exists():
+            stat_age = time.time() - disk_path.stat().st_mtime
+            if stat_age < self.config.http_cache_ttl_hours * 3600:
+                try:
+                    async with aiofiles.open(disk_path, "r") as f:
+                        data = json.loads(await f.read())
+                    self._memory_cache[cache_key] = data
+                    self._cache_timestamps[cache_key] = time.time()
+                    return data
+                except Exception:
+                    pass
+
+        url = url_template.replace("{name}", component_name)
+        async with self._semaphore:
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(
+                        url,
+                        timeout=aiohttp.ClientTimeout(
+                            total=self.config.fetch_timeout_seconds
+                        ),
+                    ) as resp:
+                        if resp.status == 200:
+                            data = await resp.json(content_type=None)
+                            logger.debug(
+                                "Fetched %s/%s from external registry",
+                                registry_name, component_name,
+                            )
+                        else:
+                            logger.warning(
+                                "External registry %s returned %d for %s",
+                                registry_name, resp.status, component_name,
+                            )
+                            data = {"name": component_name, "files": []}
+            except Exception as exc:
+                logger.warning(
+                    "External registry fetch failed %s/%s: %s",
+                    registry_name, component_name, exc,
+                )
+                data = {"name": component_name, "files": []}
+
+        self._memory_cache[cache_key] = data
+        self._cache_timestamps[cache_key] = time.time()
+        try:
+            async with aiofiles.open(disk_path, "w") as f:
+                await f.write(json.dumps(data))
+        except Exception:
+            pass
+
+        return data
+
+    async def fetch_all_external(
+        self,
+        external_registries: list,
+    ) -> dict[str, dict]:
+        """Fetch all components from all external registries concurrently.
+
+        Returns dict keyed by "registry_name/component_name".
+        """
+        tasks = {}
+        for reg in external_registries:
+            if not reg.open_source:
+                continue
+            for comp in reg.components:
+                key = f"{reg.name}/{comp}"
+                tasks[key] = self.fetch_from_external(
+                    reg.url_template, comp, reg.name
+                )
+
+        if not tasks:
+            return {}
+
+        results = await asyncio.gather(*tasks.values(), return_exceptions=True)
+        return {
+            key: (r if isinstance(r, dict) else {"name": key, "files": []})
+            for key, r in zip(tasks.keys(), results)
+        }

component_mapper/models.py

@@ -0,0 +1,159 @@
+from enum import Enum
+from typing import Any
+from pydantic import BaseModel, Field
+from segment_classifier.models import ClassifiedSegment, ComponentType
+
+
+class MappingStage(str, Enum):
+    CACHE_HIT = "cache_hit"
+    STRUCTURAL_MATCH = "structural_match"
+    LLM_MAPPED = "llm_mapped"
+    LLM_NOVEL = "llm_novel"
+    UNRESOLVED = "unresolved"
+
+
+class InteractivityMode(str, Enum):
+    STATIC = "static"
+    INTERACTIVE = "interactive"
+    PARTIAL = "partial"
+
+
+class RegistrySource(str, Enum):
+    SHADCN = "shadcn"
+    CUSTOM = "custom"
+    NOVEL = "novel"
+
+
+class PropDefinition(BaseModel):
+    name: str
+    type: str
+    required: bool = False
+    default_value: str | None = None
+    description: str = ""
+
+
+class ComponentSignature(BaseModel):
+    component_name: str
+    registry_source: RegistrySource
+
+    dom_skeleton: str
+    root_element: str
+    required_children: list[str]
+    optional_children: list[str]
+    structural_class_tokens: list[str]
+    typical_nesting_depth: int
+    child_tag_counts: dict[str, int]
+    unique_tag_count: int
+
+    compatible_component_types: list[ComponentType]
+    interactivity: InteractivityMode = InteractivityMode.STATIC
+    description: str = ""
+
+    props: list[PropDefinition] = Field(default_factory=list)
+
+    astro_import: str
+    install_command: str
+    requires_client_directive: bool = False
+
+
+class CustomComponentDefinition(BaseModel):
+    name: str
+    dom_skeleton: str
+    structural_class_tokens: list[str]
+    compatible_component_types: list[ComponentType]
+    props: list[PropDefinition]
+    astro_import: str
+    install_command: str = ""
+    interactivity: InteractivityMode = InteractivityMode.STATIC
+    description: str = ""
+    source: str = "manual"
+    confidence: float = 1.0
+
+
+class RankedCandidate(BaseModel):
+    component_name: str
+    registry_source: RegistrySource
+    signature: ComponentSignature
+
+    structural_score: float = Field(ge=0.0, le=1.0)
+    type_score: float = Field(ge=0.0, le=1.0)
+    class_token_score: float = Field(ge=0.0, le=1.0)
+    composite_score: float = Field(ge=0.0, le=1.0)
+
+
+class PropMapping(BaseModel):
+    mappings: list[dict[str, Any]] = Field(default_factory=list)
+    has_ambiguous: bool = False
+    unmapped_props: list[str] = Field(default_factory=list)
+
+
+class AstroImport(BaseModel):
+    identifier: str
+    source: str
+    is_default: bool = False
+
+
+class AstroComponent(BaseModel):
+    component_name: str
+    file_path: str
+    frontmatter: str
+    template: str
+    imports: list[AstroImport]
+    full_file_content: str
+    install_commands: list[str]
+    client_directive: str | None = None
+    is_collection_item: bool = False
+
+
+class ContentCollectionSchema(BaseModel):
+    collection_name: str
+    zod_schema: str
+    example_entry: str
+
+
+class MappingCacheRecord(BaseModel):
+    fingerprint_hash: str
+    component_name: str
+    registry_source: RegistrySource
+    prop_mapping: PropMapping
+    mapping_stage: MappingStage
+    confidence: float
+    hit_count: int = 1
+
+
+class MappedComponent(BaseModel):
+    segment_id: str
+    page_url: str
+    component_type: ComponentType
+    classification_stage: str
+
+    component_name: str
+    registry_source: RegistrySource
+    mapping_stage: MappingStage
+    mapping_confidence: float
+
+    prop_mapping: PropMapping
+
+    astro_component: AstroComponent
+    content_collection_schema: ContentCollectionSchema | None = None
+
+    llm_model_used: str | None = None
+    llm_reasoning: str | None = None
+
+
+class PipelineRunResult(BaseModel):
+    total_segments: int
+    mapped: list[MappedComponent]
+    unresolved: list[ClassifiedSegment]
+
+    stage_breakdown: dict[MappingStage, int]
+
+    llm_calls_made: int
+    llm_model_usage: dict[str, int]
+    mcp_calls_made: int
+
+    cache_hit_rate: float
+    structural_match_rate: float
+
+    install_commands: list[str]
+    unique_components_used: list[str]

component_mapper/pipeline.py

@@ -0,0 +1,182 @@
+import asyncio
+import logging
+from component_mapper.config import MapperSettings
+from component_mapper.models import (
+    PipelineRunResult,
+    MappingStage,
+)
+from component_mapper.mcp.official_client import OfficialMCPClient
+from component_mapper.mcp.registry_fetcher import RegistryFetcher
+from component_mapper.registry.signature_index import SignatureIndex
+from component_mapper.registry.custom_registry import CustomRegistry
+from component_mapper.cache.mapping_cache import MappingCache
+from component_mapper.stages.cache_lookup import CacheLookupStage
+from component_mapper.stages.structural_match import StructuralMatchStage
+from component_mapper.stages.llm_mapper import LLMMapperStage
+from component_mapper.stages.astro_stage import AstroStage
+from segment_classifier.models import ClassifiedSegment
+
+logger = logging.getLogger(__name__)
+
+
+class MapperPipeline:
+    def __init__(self, settings: MapperSettings):
+        self.settings = settings
+        self.mcp_client = OfficialMCPClient(settings.mcp)
+        self.fetcher = RegistryFetcher(settings.registry)
+        self.custom_registry = CustomRegistry()
+        self.mapping_cache = MappingCache(
+            settings.mapping_cache.cache_path,
+            settings.mapping_cache.auto_persist_every,
+        )
+        self.signature_index = SignatureIndex(settings, self.fetcher, self.mcp_client)
+
+        self.cache_stage = CacheLookupStage(self.mapping_cache, self.signature_index)
+        self.structural_stage = StructuralMatchStage(
+            self.signature_index, self.mapping_cache
+        )
+        self.llm_stage = LLMMapperStage(
+            settings, self.signature_index, self.custom_registry, self.mapping_cache
+        )
+        self.astro_stage = AstroStage(settings, self.signature_index)
+
+    async def initialize(self) -> None:
+        """One-time setup. Call before run()."""
+        logger.info("Initializing MapperPipeline")
+
+        # Connect MCP (non-fatal if unavailable)
+        await self.mcp_client.connect()
+
+        # Load caches
+        await asyncio.gather(
+            self.mapping_cache.load(),
+            self.custom_registry.load(
+                self.settings.signature_index.custom_registry_path
+            ),
+        )
+
+        # Build signature index (uses cache if fresh, else rebuilds)
+        await self.signature_index.build()
+
+        # Merge custom components into index (priority: custom first)
+        self.signature_index.merge_custom(self.custom_registry.get_all())
+
+        logger.info(
+            "Pipeline initialized: %d components in index, %d mapping cache records",
+            len(self.signature_index.get_all_component_names()),
+            self.mapping_cache.size,
+        )
+
+    async def run(self, segments: list[ClassifiedSegment]) -> PipelineRunResult:
+        """Map all segments. Returns PipelineRunResult."""
+        if not segments:
+            return PipelineRunResult(
+                total_segments=0,
+                mapped=[],
+                unresolved=[],
+                stage_breakdown={s: 0 for s in MappingStage},
+                llm_calls_made=0,
+                llm_model_usage={},
+                mcp_calls_made=self.mcp_client.calls_made,
+                cache_hit_rate=0.0,
+                structural_match_rate=0.0,
+                install_commands=[],
+                unique_components_used=[],
+            )
+
+        total = len(segments)
+        logger.info("Starting pipeline run: %d segments", total)
+
+        # Stage 1: Cache lookup
+        hits, misses = await self.cache_stage.process(segments)
+
+        # Stage 2: Structural match
+        direct_matches, ambiguous, novel = await self.structural_stage.process(misses)
+
+        # Stage 3: LLM mapping
+        llm_mapped, unresolved = await self.llm_stage.process(ambiguous, novel)
+
+        # Stage 4: Astro generation
+        all_mapped = hits + direct_matches + llm_mapped
+        all_mapped = await self.astro_stage.process(all_mapped)
+
+        # Collect install manifest
+        install_set: set[str] = set()
+        component_names: set[str] = set()
+        for comp in all_mapped:
+            if comp.astro_component:
+                for cmd in comp.astro_component.install_commands:
+                    if cmd:
+                        install_set.add(cmd)
+            component_names.add(comp.component_name)
+
+        # Stage breakdown
+        breakdown: dict[MappingStage, int] = {s: 0 for s in MappingStage}
+        for comp in all_mapped:
+            breakdown[comp.mapping_stage] = breakdown.get(comp.mapping_stage, 0) + 1
+        breakdown[MappingStage.UNRESOLVED] = len(unresolved)
+
+        miss_count = len(misses)
+        structural_match_rate = len(direct_matches) / miss_count if miss_count else 0.0
+
+        logger.info(
+            "Pipeline complete: %d mapped, %d unresolved | "
+            "cache=%.0f%% structural=%.0f%% llm=%d calls",
+            len(all_mapped),
+            len(unresolved),
+            (len(hits) / total * 100) if total else 0,
+            structural_match_rate * 100,
+            self.llm_stage.calls_made,
+        )
+
+        return PipelineRunResult(
+            total_segments=total,
+            mapped=all_mapped,
+            unresolved=unresolved,
+            stage_breakdown=breakdown,
+            llm_calls_made=self.llm_stage.calls_made,
+            llm_model_usage=self.llm_stage.get_model_usage(),
+            mcp_calls_made=self.mcp_client.calls_made,
+            cache_hit_rate=len(hits) / total if total else 0.0,
+            structural_match_rate=structural_match_rate,
+            install_commands=sorted(install_set),
+            unique_components_used=sorted(component_names),
+        )
+
+    async def shutdown(self) -> None:
+        """Persist state and install components. Call after run()."""
+        logger.info("Shutting down pipeline")
+
+        # Persist caches
+        await asyncio.gather(
+            self.mapping_cache.persist(),
+            self.custom_registry.persist(
+                self.settings.signature_index.custom_registry_path
+            ),
+        )
+
+        # Install unique shadcn components via MCP
+        unique_shadcn = [
+            name
+            for name in self.signature_index.get_all_component_names()
+            if (
+                self.signature_index.get_signature(name) is not None
+                and self.signature_index.get_signature(name).registry_source.value
+                == "shadcn"
+            )
+        ]
+
+        if unique_shadcn and self.mcp_client._connected:
+            try:
+                results = await self.mcp_client.install_components(unique_shadcn)
+                success_count = sum(1 for v in results.values() if v)
+                logger.info(
+                    "Installed %d/%d shadcn components",
+                    success_count,
+                    len(unique_shadcn),
+                )
+            except Exception as exc:
+                logger.warning("Component install failed: %s", exc)
+
+        await self.mcp_client.disconnect()
+        logger.info("Pipeline shutdown complete")