nc1709 1.15.4__py3-none-any.whl

nc1709/models/detector.py

@@ -0,0 +1,377 @@
+"""
+Auto-detect model capabilities from Ollama.
+
+When you download a new model that's not in the registry,
+this module can detect its basic capabilities and create a spec.
+"""
+
+import asyncio
+import re
+from typing import Optional, List, Dict, Any
+
+try:
+    import aiohttp
+    HAS_AIOHTTP = True
+except ImportError:
+    HAS_AIOHTTP = False
+
+try:
+    import httpx
+    HAS_HTTPX = True
+except ImportError:
+    HAS_HTTPX = False
+
+from .registry import (
+    ModelSpec, ModelCapability, PromptFormat,
+    KNOWN_MODELS, register_model
+)
+
+
+class ModelDetector:
+    """
+    Auto-detects model capabilities from Ollama.
+
+    When you download a new model that's not in the registry,
+    this class can detect its basic capabilities.
+
+    Example:
+        detector = ModelDetector()
+        models = await detector.list_installed_models()
+        spec = await detector.detect_model_spec("new-model:latest")
+    """
+
+    def __init__(self, ollama_url: str = "http://localhost:11434"):
+        self.ollama_url = ollama_url.rstrip("/")
+
+    async def list_installed_models(self) -> List[str]:
+        """
+        Get list of models installed in Ollama.
+
+        Returns:
+            List of model names
+        """
+        try:
+            data = await self._get(f"{self.ollama_url}/api/tags")
+            if data:
+                return [model["name"] for model in data.get("models", [])]
+        except Exception:
+            pass
+        return []
+
+    def list_installed_models_sync(self) -> List[str]:
+        """Synchronous version of list_installed_models"""
+        try:
+            return asyncio.get_event_loop().run_until_complete(
+                self.list_installed_models()
+            )
+        except RuntimeError:
+            # No event loop, create one
+            return asyncio.run(self.list_installed_models())
+
+    async def get_model_info(self, model_name: str) -> Optional[Dict[str, Any]]:
+        """
+        Get model info from Ollama.
+
+        Args:
+            model_name: The model name
+
+        Returns:
+            Model info dict or None
+        """
+        try:
+            return await self._post(
+                f"{self.ollama_url}/api/show",
+                {"name": model_name}
+            )
+        except Exception:
+            return None
+
+    async def detect_model_spec(self, model_name: str) -> ModelSpec:
+        """
+        Detect/create a ModelSpec for a model.
+
+        First checks the registry, then tries to auto-detect.
+
+        Args:
+            model_name: The model name to detect
+
+        Returns:
+            ModelSpec (from registry or auto-detected)
+        """
+        # Check registry first
+        if model_name in KNOWN_MODELS:
+            return KNOWN_MODELS[model_name]
+
+        # Try to get info from Ollama
+        info = await self.get_model_info(model_name)
+
+        # Detect capabilities based on model name and info
+        capabilities = self._detect_capabilities(model_name, info)
+        prompt_format = self._detect_prompt_format(model_name, info)
+        context_window = self._detect_context_window(info)
+
+        # Create spec
+        spec = ModelSpec(
+            name=self._format_name(model_name),
+            ollama_name=model_name,
+            context_window=context_window,
+            prompt_format=prompt_format,
+            capabilities=capabilities,
+            suitability=self._estimate_suitability(model_name, capabilities),
+            notes="Auto-detected model"
+        )
+
+        # Register it for future use
+        register_model(spec)
+
+        return spec
+
+    def detect_model_spec_sync(self, model_name: str) -> ModelSpec:
+        """Synchronous version of detect_model_spec"""
+        try:
+            return asyncio.get_event_loop().run_until_complete(
+                self.detect_model_spec(model_name)
+            )
+        except RuntimeError:
+            return asyncio.run(self.detect_model_spec(model_name))
+
+    async def sync_with_ollama(self) -> List[ModelSpec]:
+        """
+        Sync registry with installed Ollama models.
+
+        Detects any new models and adds them to the registry.
+
+        Returns:
+            List of newly detected models
+        """
+        installed = await self.list_installed_models()
+        new_models = []
+
+        for model_name in installed:
+            if model_name not in KNOWN_MODELS:
+                spec = await self.detect_model_spec(model_name)
+                new_models.append(spec)
+
+        return new_models
+
+    def sync_with_ollama_sync(self) -> List[ModelSpec]:
+        """Synchronous version of sync_with_ollama"""
+        try:
+            return asyncio.get_event_loop().run_until_complete(
+                self.sync_with_ollama()
+            )
+        except RuntimeError:
+            return asyncio.run(self.sync_with_ollama())
+
+    def _detect_capabilities(
+        self,
+        model_name: str,
+        info: Optional[Dict]
+    ) -> List[ModelCapability]:
+        """Detect capabilities from model name and info"""
+        capabilities = []
+        name_lower = model_name.lower()
+
+        # Detect from name
+        if "coder" in name_lower or "code" in name_lower:
+            capabilities.extend([
+                ModelCapability.CODE_GENERATION,
+                ModelCapability.CODE_COMPLETION,
+            ])
+
+        if any(x in name_lower for x in ["r1", "reason", "think"]):
+            capabilities.append(ModelCapability.REASONING)
+
+        if "math" in name_lower:
+            capabilities.append(ModelCapability.MATH)
+
+        if any(x in name_lower for x in [":1b", ":3b", ":0.5b", "3b", "1b"]):
+            capabilities.append(ModelCapability.FAST_INFERENCE)
+
+        if "vision" in name_lower or "llava" in name_lower:
+            capabilities.append(ModelCapability.VISION)
+
+        if "embed" in name_lower:
+            capabilities.append(ModelCapability.EMBEDDING)
+
+        if "tool" in name_lower or "function" in name_lower:
+            capabilities.append(ModelCapability.TOOL_USE)
+
+        # Check context window from info
+        if info:
+            ctx = self._detect_context_window(info)
+            if ctx >= 100000:
+                capabilities.append(ModelCapability.LONG_CONTEXT)
+
+        # Default: at least general reasoning
+        if not capabilities:
+            capabilities.append(ModelCapability.REASONING)
+
+        return list(set(capabilities))  # Remove duplicates
+
+    def _detect_prompt_format(
+        self,
+        model_name: str,
+        info: Optional[Dict]
+    ) -> PromptFormat:
+        """Detect prompt format from model name and info"""
+        name_lower = model_name.lower()
+
+        # Check model family
+        if "qwen" in name_lower:
+            return PromptFormat.CHATML
+        if "deepseek" in name_lower:
+            return PromptFormat.DEEPSEEK
+        if "llama" in name_lower or "codellama" in name_lower:
+            return PromptFormat.LLAMA
+        if "mistral" in name_lower or "mixtral" in name_lower:
+            return PromptFormat.MISTRAL
+        if "command" in name_lower:
+            return PromptFormat.COMMAND_R
+        if "alpaca" in name_lower or "vicuna" in name_lower:
+            return PromptFormat.ALPACA
+
+        # Check template in info
+        if info and "template" in info:
+            template = info["template"].lower()
+            if "im_start" in template or "chatml" in template:
+                return PromptFormat.CHATML
+            if "[inst]" in template:
+                return PromptFormat.LLAMA
+            if "### instruction" in template:
+                return PromptFormat.ALPACA
+
+        # Default to ChatML (most common)
+        return PromptFormat.CHATML
+
+    def _detect_context_window(self, info: Optional[Dict]) -> int:
+        """Detect context window from info"""
+        if info:
+            # Check parameters
+            params = info.get("parameters", {})
+            if isinstance(params, dict) and "num_ctx" in params:
+                return params["num_ctx"]
+
+            # Check modelfile
+            modelfile = info.get("modelfile", "")
+            if isinstance(modelfile, str) and "num_ctx" in modelfile:
+                match = re.search(r"num_ctx\s+(\d+)", modelfile)
+                if match:
+                    return int(match.group(1))
+
+            # Check details
+            details = info.get("details", {})
+            if isinstance(details, dict):
+                # Some models report context in details
+                families = details.get("families", [])
+                if "llama" in families:
+                    return 128000  # Llama 3 default
+                if "qwen2" in families:
+                    return 32768
+
+        return 32768  # Default
+
+    def _estimate_suitability(
+        self,
+        model_name: str,
+        capabilities: List[ModelCapability]
+    ) -> Dict[str, float]:
+        """Estimate suitability scores based on name and capabilities"""
+        scores = {
+            "coding": 0.5,
+            "reasoning": 0.5,
+            "general": 0.6,
+            "fast": 0.5,
+            "instant": 0.3,
+        }
+
+        name_lower = model_name.lower()
+
+        # Adjust based on capabilities
+        if ModelCapability.CODE_GENERATION in capabilities:
+            scores["coding"] = 0.85
+
+        if ModelCapability.REASONING in capabilities:
+            scores["reasoning"] = 0.80
+
+        if ModelCapability.FAST_INFERENCE in capabilities:
+            scores["fast"] = 0.90
+            scores["instant"] = 0.80
+            scores["coding"] *= 0.8  # Fast models usually less capable
+
+        if ModelCapability.EMBEDDING in capabilities:
+            scores = {"embedding": 1.0}
+            return scores
+
+        # Adjust based on size hints in name
+        if any(x in name_lower for x in ["70b", "72b", "65b"]):
+            scores["coding"] = min(scores["coding"] * 1.15, 0.95)
+            scores["reasoning"] = min(scores["reasoning"] * 1.15, 0.95)
+            scores["fast"] *= 0.3
+            scores["instant"] *= 0.2
+        elif any(x in name_lower for x in ["32b", "34b", "33b"]):
+            scores["coding"] = min(scores["coding"] * 1.1, 0.92)
+            scores["reasoning"] = min(scores["reasoning"] * 1.1, 0.90)
+            scores["fast"] *= 0.5
+        elif any(x in name_lower for x in ["7b", "8b"]):
+            scores["fast"] = min(scores["fast"] * 1.2, 0.92)
+        elif any(x in name_lower for x in ["3b", "1b", "0.5b"]):
+            scores["fast"] = 0.95
+            scores["instant"] = 0.95
+            scores["coding"] *= 0.6
+
+        return scores
+
+    def _format_name(self, model_name: str) -> str:
+        """Format model name for display"""
+        # Remove :latest suffix
+        name = model_name.replace(":latest", "")
+        # Capitalize parts
+        parts = name.replace("-", " ").replace(":", " ").split()
+        return " ".join(p.capitalize() for p in parts)
+
+    async def _get(self, url: str) -> Optional[Dict]:
+        """HTTP GET request"""
+        if HAS_AIOHTTP:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as response:
+                    if response.status == 200:
+                        return await response.json()
+        elif HAS_HTTPX:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(url, timeout=10)
+                if response.status_code == 200:
+                    return response.json()
+        return None
+
+    async def _post(self, url: str, data: Dict) -> Optional[Dict]:
+        """HTTP POST request"""
+        if HAS_AIOHTTP:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(
+                    url, json=data, timeout=aiohttp.ClientTimeout(total=10)
+                ) as response:
+                    if response.status == 200:
+                        return await response.json()
+        elif HAS_HTTPX:
+            async with httpx.AsyncClient() as client:
+                response = await client.post(url, json=data, timeout=10)
+                if response.status_code == 200:
+                    return response.json()
+        return None
+
+
+# Convenience function
+def auto_detect_model(model_name: str, ollama_url: str = "http://localhost:11434") -> ModelSpec:
+    """
+    Auto-detect a model's capabilities.
+
+    Args:
+        model_name: The model name
+        ollama_url: Ollama API URL
+
+    Returns:
+        ModelSpec for the model
+    """
+    detector = ModelDetector(ollama_url)
+    return detector.detect_model_spec_sync(model_name)

nc1709/models/formats.py

@@ -0,0 +1,315 @@
+"""
+Prompt format templates for different model families.
+
+Each model family expects prompts in a specific format. This module
+handles the conversion automatically based on the model's prompt_format.
+"""
+
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from .registry import PromptFormat
+
+
+@dataclass
+class Message:
+    """A chat message"""
+    role: str  # "system", "user", "assistant"
+    content: str
+
+
+class PromptFormatter:
+    """
+    Formats prompts for different model families.
+
+    Each model family has its own expected format. This class
+    handles the conversion automatically.
+
+    Example:
+        formatter = PromptFormatter()
+        messages = [
+            Message("system", "You are helpful"),
+            Message("user", "Hello!"),
+        ]
+        prompt = formatter.format(messages, PromptFormat.CHATML)
+    """
+
+    @staticmethod
+    def format(
+        messages: List[Message],
+        prompt_format: PromptFormat,
+        add_generation_prompt: bool = True
+    ) -> str:
+        """
+        Format messages for a specific model format.
+
+        Args:
+            messages: List of messages
+            prompt_format: The format to use
+            add_generation_prompt: Whether to add the assistant prompt at end
+
+        Returns:
+            Formatted prompt string
+        """
+        formatters = {
+            PromptFormat.CHATML: PromptFormatter._format_chatml,
+            PromptFormat.LLAMA: PromptFormatter._format_llama,
+            PromptFormat.ALPACA: PromptFormatter._format_alpaca,
+            PromptFormat.RAW: PromptFormatter._format_raw,
+            PromptFormat.DEEPSEEK: PromptFormatter._format_deepseek,
+            PromptFormat.MISTRAL: PromptFormatter._format_mistral,
+            PromptFormat.COMMAND_R: PromptFormatter._format_command_r,
+        }
+
+        formatter = formatters.get(prompt_format, PromptFormatter._format_raw)
+        return formatter(messages, add_generation_prompt)
+
+    @staticmethod
+    def format_from_dicts(
+        messages: List[Dict[str, str]],
+        prompt_format: PromptFormat,
+        add_generation_prompt: bool = True
+    ) -> str:
+        """
+        Format messages from dict format.
+
+        Args:
+            messages: List of {"role": "...", "content": "..."} dicts
+            prompt_format: The format to use
+            add_generation_prompt: Whether to add the assistant prompt at end
+
+        Returns:
+            Formatted prompt string
+        """
+        msg_objects = [
+            Message(role=m["role"], content=m["content"])
+            for m in messages
+        ]
+        return PromptFormatter.format(msg_objects, prompt_format, add_generation_prompt)
+
+    @staticmethod
+    def _format_chatml(messages: List[Message], add_gen: bool) -> str:
+        """
+        ChatML format (Qwen, OpenAI-style)
+
+        <|im_start|>system
+        You are helpful<|im_end|>
+        <|im_start|>user
+        Hello!<|im_end|>
+        <|im_start|>assistant
+        """
+        parts = []
+        for msg in messages:
+            parts.append(f"<|im_start|>{msg.role}\n{msg.content}<|im_end|>")
+
+        if add_gen:
+            parts.append("<|im_start|>assistant\n")
+
+        return "\n".join(parts)
+
+    @staticmethod
+    def _format_llama(messages: List[Message], add_gen: bool) -> str:
+        """
+        Llama/Llama2/Llama3 format
+
+        [INST] <<SYS>>
+        System prompt
+        <</SYS>>
+
+        User message [/INST] Assistant response
+        """
+        parts = []
+        system_msg = None
+
+        for msg in messages:
+            if msg.role == "system":
+                system_msg = msg.content
+            elif msg.role == "user":
+                if system_msg:
+                    parts.append(
+                        f"[INST] <<SYS>>\n{system_msg}\n<</SYS>>\n\n{msg.content} [/INST]"
+                    )
+                    system_msg = None
+                else:
+                    parts.append(f"[INST] {msg.content} [/INST]")
+            elif msg.role == "assistant":
+                parts.append(msg.content)
+
+        return "\n".join(parts)
+
+    @staticmethod
+    def _format_alpaca(messages: List[Message], add_gen: bool) -> str:
+        """
+        Alpaca format
+
+        ### System:
+        System prompt
+
+        ### Instruction:
+        User message
+
+        ### Response:
+        """
+        parts = []
+
+        for msg in messages:
+            if msg.role == "system":
+                parts.append(f"### System:\n{msg.content}\n")
+            elif msg.role == "user":
+                parts.append(f"### Instruction:\n{msg.content}\n")
+            elif msg.role == "assistant":
+                parts.append(f"### Response:\n{msg.content}\n")
+
+        if add_gen:
+            parts.append("### Response:\n")
+
+        return "\n".join(parts)
+
+    @staticmethod
+    def _format_deepseek(messages: List[Message], add_gen: bool) -> str:
+        """
+        DeepSeek format
+
+        <|system|>
+        System prompt
+        <|user|>
+        User message
+        <|assistant|>
+        """
+        parts = []
+
+        for msg in messages:
+            if msg.role == "system":
+                parts.append(f"<|system|>\n{msg.content}")
+            elif msg.role == "user":
+                parts.append(f"<|user|>\n{msg.content}")
+            elif msg.role == "assistant":
+                parts.append(f"<|assistant|>\n{msg.content}")
+
+        if add_gen:
+            parts.append("<|assistant|>\n")
+
+        return "\n".join(parts)
+
+    @staticmethod
+    def _format_mistral(messages: List[Message], add_gen: bool) -> str:
+        """
+        Mistral format
+
+        [INST] User message [/INST] Assistant response
+        """
+        parts = []
+
+        for msg in messages:
+            if msg.role == "user":
+                parts.append(f"[INST] {msg.content} [/INST]")
+            elif msg.role == "assistant":
+                parts.append(msg.content)
+            elif msg.role == "system":
+                # Mistral often handles system as first user message
+                parts.append(f"[INST] {msg.content}\n")
+
+        return "".join(parts)
+
+    @staticmethod
+    def _format_command_r(messages: List[Message], add_gen: bool) -> str:
+        """
+        Cohere Command-R format
+
+        <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>...<|END_OF_TURN_TOKEN|>
+        <|START_OF_TURN_TOKEN|><|USER_TOKEN|>...<|END_OF_TURN_TOKEN|>
+        <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
+        """
+        parts = []
+
+        for msg in messages:
+            if msg.role == "system":
+                parts.append(
+                    f"<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{msg.content}<|END_OF_TURN_TOKEN|>"
+                )
+            elif msg.role == "user":
+                parts.append(
+                    f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{msg.content}<|END_OF_TURN_TOKEN|>"
+                )
+            elif msg.role == "assistant":
+                parts.append(
+                    f"<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{msg.content}<|END_OF_TURN_TOKEN|>"
+                )
+
+        if add_gen:
+            parts.append("<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>")
+
+        return "".join(parts)
+
+    @staticmethod
+    def _format_raw(messages: List[Message], add_gen: bool) -> str:
+        """
+        Raw format (no special tokens)
+
+        System: ...
+        User: ...
+        Assistant: ...
+        """
+        parts = []
+
+        for msg in messages:
+            if msg.role == "system":
+                parts.append(f"System: {msg.content}\n")
+            elif msg.role == "user":
+                parts.append(f"User: {msg.content}\n")
+            elif msg.role == "assistant":
+                parts.append(f"Assistant: {msg.content}\n")
+
+        if add_gen:
+            parts.append("Assistant: ")
+
+        return "".join(parts)
+
+
+def get_format_info(prompt_format: PromptFormat) -> Dict[str, str]:
+    """
+    Get information about a prompt format.
+
+    Args:
+        prompt_format: The format to describe
+
+    Returns:
+        Dict with format details
+    """
+    info = {
+        PromptFormat.CHATML: {
+            "name": "ChatML",
+            "description": "OpenAI-style format used by Qwen, etc.",
+            "example": "<|im_start|>user\\nHello<|im_end|>",
+        },
+        PromptFormat.LLAMA: {
+            "name": "Llama",
+            "description": "Meta Llama format with [INST] tags",
+            "example": "[INST] Hello [/INST]",
+        },
+        PromptFormat.ALPACA: {
+            "name": "Alpaca",
+            "description": "Stanford Alpaca instruction format",
+            "example": "### Instruction:\\nHello\\n### Response:",
+        },
+        PromptFormat.DEEPSEEK: {
+            "name": "DeepSeek",
+            "description": "DeepSeek model format",
+            "example": "<|user|>\\nHello\\n<|assistant|>",
+        },
+        PromptFormat.MISTRAL: {
+            "name": "Mistral",
+            "description": "Mistral AI format",
+            "example": "[INST] Hello [/INST]",
+        },
+        PromptFormat.COMMAND_R: {
+            "name": "Command-R",
+            "description": "Cohere Command-R format",
+            "example": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello",
+        },
+        PromptFormat.RAW: {
+            "name": "Raw",
+            "description": "Plain text with role prefixes",
+            "example": "User: Hello\\nAssistant:",
+        },
+    }
+    return info.get(prompt_format, {"name": "Unknown", "description": "", "example": ""})