velune-cli 0.9.0__py3-none-any.whl

velune/models/registry.py

@@ -0,0 +1,251 @@
+"""Model capability registry with empirical probe evaluation."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile, ModelDescriptor
+from velune.models.profile_cache import ModelProfileCache
+from velune.providers.discovery.scanner import ModelDiscoveryScanner
+
+logger = logging.getLogger("velune.models.registry")
+
+
+class ModelCapabilityRegistry:
+    """Unified database cataloging discovered models and capabilities with empirical routing."""
+
+    def __init__(self, scanner: ModelDiscoveryScanner | None = None) -> None:
+        self.scanner = scanner or ModelDiscoveryScanner()
+        self._models: dict[str, ModelDescriptor] = {}
+
+    async def refresh(self) -> None:
+        """Scan all providers and refresh the local catalog cache with empirical profiles."""
+        try:
+            discovered = await self.scanner.scan_all()
+            self._models.clear()
+
+            profile_cache = ModelProfileCache(Path(".velune") / "model_profiles.json")
+            from velune.models.probes import FastProbe
+
+            fast_probe = FastProbe()
+
+            probing_tasks = []
+            models_to_probe = []
+
+            for model in discovered:
+                cached = profile_cache.get(model.model_id, model.provider_id)
+                if cached:
+                    # Apply cached probe results to capability profile
+                    self._apply_probe_results(model, cached["probes"])
+                else:
+                    # Check if provider is available to probe
+                    provider = None
+                    try:
+                        from velune.kernel.registry import get_container
+
+                        provider_reg = get_container().get("runtime.provider_registry")
+                        provider = provider_reg.get(model.provider_id)
+                    except Exception:
+                        pass
+
+                    if provider:
+                        models_to_probe.append(model)
+                        probing_tasks.append(fast_probe.ping(provider, model.model_id))
+
+            # Execute fast probes concurrently
+            if probing_tasks:
+                results = await asyncio.gather(*probing_tasks, return_exceptions=True)
+                for model, responsive in zip(models_to_probe, results, strict=False):
+                    if isinstance(responsive, Exception):
+                        responsive = False
+
+                    if responsive:
+                        model.metadata["validated"] = True
+                        try:
+                            from velune.daemon.client import DaemonClient
+
+                            if DaemonClient.is_running():
+                                # Delegate background probing to the active persistent Velune daemon!
+                                # Using create_task to fire-and-forget the IPC dispatch call
+                                asyncio.create_task(
+                                    DaemonClient.send_command(
+                                        "probe_model",
+                                        model_id=model.model_id,
+                                        provider_id=model.provider_id,
+                                    )
+                                )
+                                logger.info(
+                                    "Delegated full probing of model %s to the active Velune daemon process.",
+                                    model.model_id,
+                                )
+                            else:
+                                from velune.kernel.registry import get_container
+
+                                task_reg = get_container().get("runtime.task_registry")
+                                task_reg.submit(
+                                    name=f"full_probe_{model.model_id}",
+                                    coro=self._probe_model_background(model, profile_cache),
+                                    timeout_seconds=120.0,
+                                )
+                        except Exception:
+                            pass
+                    else:
+                        model.metadata["validated"] = False
+                        logger.info("Model %s is not responding, skipping probe", model.model_id)
+
+            # Store models in mapping
+            for model in discovered:
+                key = f"{model.provider_id}/{model.model_id}"
+                self._models[key] = model
+                if model.model_id not in self._models:
+                    self._models[model.model_id] = model
+
+            logger.info(
+                "Indexed %d models (%d validated)",
+                len(discovered),
+                sum(1 for m in discovered if m.metadata.get("validated", True)),
+            )
+        except Exception as e:
+            logger.error("Failed to discover models during catalog refresh: %s", e)
+
+    def register(self, descriptor: ModelDescriptor) -> None:
+        """Explicitly register a custom model descriptor."""
+        key = f"{descriptor.provider_id}/{descriptor.model_id}"
+        self._models[key] = descriptor
+        if descriptor.model_id not in self._models:
+            self._models[descriptor.model_id] = descriptor
+
+    def get(self, model_id: str, provider_id: str | None = None) -> ModelDescriptor | None:
+        """Look up a model descriptor by ID and optional provider prefix."""
+        if provider_id:
+            key = f"{provider_id}/{model_id}"
+            return self._models.get(key)
+
+        # Try direct match
+        if model_id in self._models:
+            return self._models[model_id]
+
+        # Try searching values
+        for model in self._models.values():
+            if model.model_id == model_id:
+                return model
+        return None
+
+    def list_all(self) -> list[ModelDescriptor]:
+        """List all currently indexed model descriptors."""
+        # Return unique descriptors
+        seen = set()
+        unique = []
+        for model in self._models.values():
+            ref = (model.provider_id, model.model_id)
+            if ref not in seen:
+                seen.add(ref)
+                unique.append(model)
+        return unique
+
+    def get_by_provider(self, provider_id: str) -> list[ModelDescriptor]:
+        """List all models registered under a specific provider."""
+        return [model for model in self.list_all() if model.provider_id == provider_id]
+
+    def _apply_probe_results(self, model: ModelDescriptor, probes: dict) -> None:
+        """Map float probe scores (0.0-1.0) to CapabilityLevel and update model descriptor.
+
+        Score mapping (empirical calibration):
+        - score > 0.85 → EXPERT (100)
+        - score > 0.70 → ADVANCED (75)
+        - score > 0.50 → INTERMEDIATE (50)
+        - else → BASIC (25)
+        """
+        if not model.capabilities:
+            model.capabilities = ModelCapabilityProfile()
+
+        def score_to_level(score: float) -> CapabilityLevel:
+            if score > 0.85:
+                return CapabilityLevel.EXPERT
+            elif score > 0.70:
+                return CapabilityLevel.ADVANCED
+            elif score > 0.50:
+                return CapabilityLevel.INTERMEDIATE
+            elif score > 0.0:
+                return CapabilityLevel.BASIC
+            return CapabilityLevel.NONE
+
+        coding_data = probes.get("coding", {})
+        coding_score = (
+            coding_data.score if hasattr(coding_data, "score") else coding_data.get("score", 0.0)
+        )
+
+        reasoning_data = probes.get("reasoning", {})
+        reasoning_score = (
+            reasoning_data.score
+            if hasattr(reasoning_data, "score")
+            else reasoning_data.get("score", 0.0)
+        )
+
+        instruction_data = probes.get("instruction", {})
+        instruction_score = (
+            instruction_data.score
+            if hasattr(instruction_data, "score")
+            else instruction_data.get("score", 0.0)
+        )
+
+        model.capabilities.coding = score_to_level(coding_score)
+        model.capabilities.reasoning = score_to_level(reasoning_score)
+        model.capabilities.instruction_following = score_to_level(instruction_score)
+
+        # Infer other capabilities from primary scores
+        if model.capabilities.reasoning >= CapabilityLevel.INTERMEDIATE:
+            model.capabilities.planning = CapabilityLevel.INTERMEDIATE
+        if model.capabilities.instruction_following >= CapabilityLevel.INTERMEDIATE:
+            model.capabilities.tool_use = CapabilityLevel.INTERMEDIATE
+        if model.capabilities.coding >= CapabilityLevel.INTERMEDIATE:
+            model.capabilities.code_analysis = CapabilityLevel.INTERMEDIATE
+
+        logger.debug(
+            "Applied probe results to %s: coding=%s (%.2f), reasoning=%s (%.2f), instruction=%s (%.2f)",
+            model.model_id,
+            model.capabilities.coding.name,
+            coding_score,
+            model.capabilities.reasoning.name,
+            reasoning_score,
+            model.capabilities.instruction_following.name,
+            instruction_score,
+        )
+
+    async def _probe_model_background(
+        self, model: ModelDescriptor, cache: ModelProfileCache
+    ) -> None:
+        """Run probes in background, update model in registry when done."""
+        try:
+            from velune.kernel.registry import get_container
+
+            container = get_container()
+            if not container.has("runtime.provider_registry"):
+                logger.debug(
+                    "No provider registry registered yet, skipping background probe for %s.",
+                    model.model_id,
+                )
+                return
+
+            provider_registry = container.get("runtime.provider_registry")
+            provider = provider_registry.get(model.provider_id)
+            if not provider:
+                logger.debug("No active provider found for %s, skipping probe.", model.model_id)
+                return
+
+            from velune.models.probes import ModelProber
+
+            prober = ModelProber(provider, model.model_id)
+            results = await prober.run_all_probes()
+            cache.set(model.model_id, model.provider_id, results)
+            self._apply_probe_results(model, results)
+            logger.info(
+                "Successfully probed %s: coding=%.2f reasoning=%.2f",
+                model.model_id,
+                results["coding"].score,
+                results["reasoning"].score,
+            )
+        except Exception as e:
+            logger.debug("Background probe failed for %s: %s", model.model_id, e)

velune/models/scorer.py

@@ -0,0 +1,233 @@
+"""Multi-factor scorer for model routing and selection with family-specific and quantization-aware adjustments."""
+
+from __future__ import annotations
+
+import logging
+
+from velune.core.types.model import CapabilityLevel, ModelDescriptor
+from velune.models.profiler import ModelProfile
+
+logger = logging.getLogger("velune.models.scorer")
+
+
+class ModelScorer:
+    """Calculates multidimensional matching scores for model selection."""
+
+    def __init__(
+        self,
+        w_capability: float = 0.4,
+        w_context: float = 0.2,
+        w_speed: float = 0.2,
+        w_reliability: float = 0.1,
+        w_cost: float = 0.1,
+    ) -> None:
+        """Initialize routing weights."""
+        self.w_capability = w_capability
+        self.w_context = w_context
+        self.w_speed = w_speed
+        self.w_reliability = w_reliability
+        self.w_cost = w_cost
+
+    def _detect_model_family(self, model_id: str) -> str:
+        """Detect model family from ID for family-specific scoring adjustments."""
+        lower = model_id.lower()
+        families = {
+            "qwen": ["qwen"],
+            "deepseek": ["deepseek"],
+            "llama": ["llama", "meta-llama"],
+            "mistral": ["mistral", "mixtral"],
+            "phi": ["phi"],
+            "gemma": ["gemma"],
+            "codellama": ["codellama"],
+            "starcoder": ["starcoder"],
+        }
+        for family, patterns in families.items():
+            if any(p in lower for p in patterns):
+                return family
+        return "unknown"
+
+    def _get_family_capability_adjustments(self, family: str, task_category: str) -> float:
+        """
+        Return capability score adjustments for known model families.
+        Positive = boost, Negative = penalty.
+        Based on community benchmarks and known model strengths.
+        """
+        adjustments = {
+            ("qwen", "coding"): +0.1,
+            ("qwen", "reasoning"): +0.05,
+            ("deepseek", "coding"): +0.15,
+            ("deepseek", "reasoning"): +0.1,
+            ("codellama", "coding"): +0.2,
+            ("codellama", "reasoning"): -0.1,  # Not a reasoning model
+            ("phi", "coding"): +0.05,
+            ("phi", "reasoning"): +0.15,  # Phi is surprisingly capable at reasoning for its size
+            ("mistral", "summarization"): +0.1,
+            ("gemma", "instruction_following"): +0.05,
+            ("starcoder", "coding"): +0.2,
+            ("starcoder", "reasoning"): -0.15,
+        }
+        return adjustments.get((family, task_category), 0.0)
+
+    def _get_quantization_penalty(self, model: ModelDescriptor) -> float:
+        """
+        Quantization reduces quality. Apply penalty for heavily quantized models
+        on reasoning and complex tasks.
+        """
+        quant = (model.quantization or "").upper()
+        penalties = {
+            "Q2": -0.25,
+            "Q3": -0.15,
+            "Q4_0": -0.08,
+            "Q4_K_M": -0.05,
+            "Q5": -0.02,
+            "Q5_K_M": -0.02,
+            "Q8_0": 0.0,
+            "FP16": +0.05,  # Slight quality boost for full precision
+        }
+        return penalties.get(quant, 0.0)
+
+    def score(
+        self,
+        model: ModelDescriptor,
+        task_category: str,
+        required_tokens: int = 0,
+        latency_requirement: str = "medium",
+        profile: ModelProfile | None = None,
+        local_preferred: bool = False,
+    ) -> float:
+        """
+        Calculate aggregated suitability score (0.0 - 1.0) for a model based on task constraints.
+
+        agg_score = w_cap * cap_match + w_ctx * ctx_fit + w_speed * speed + w_rel * reliability - w_cost * cost
+        """
+        # 1. Base Capability Score (0.0 to 1.0)
+        cap_score = self._calculate_capability_score(model, task_category)
+
+        # Apply model family adjustments
+        family = self._detect_model_family(model.model_id)
+        family_adj = self._get_family_capability_adjustments(family, task_category)
+        cap_score = max(0.0, min(1.0, cap_score + family_adj))
+
+        # Apply quantization penalty for reasoning-heavy tasks
+        if task_category in ("reasoning", "planning"):
+            quant_penalty = self._get_quantization_penalty(model)
+            cap_score = max(0.0, min(1.0, cap_score + quant_penalty))
+
+        # 2. Context Fit Score (0.0 to 1.0)
+        ctx_score = self._calculate_context_score(model.context_length, required_tokens)
+
+        # 3. Speed / Performance Score (0.0 to 1.0)
+        speed_score = self._calculate_speed_score(model, latency_requirement, profile)
+
+        # 4. Reliability / Validity Score (0.0 to 1.0)
+        reliability_score = self._calculate_reliability_score(model, profile, local_preferred)
+
+        # 5. Cost Penalty (0.0 to 1.0)
+        cost_penalty = self._calculate_cost_penalty(model)
+
+        # Aggregate weighted components
+        total_score = (
+            self.w_capability * cap_score
+            + self.w_context * ctx_score
+            + self.w_speed * speed_score
+            + self.w_reliability * reliability_score
+            - self.w_cost * cost_penalty
+        )
+
+        return max(0.0, min(1.0, total_score))
+
+    def _calculate_capability_score(self, model: ModelDescriptor, task_category: str) -> float:
+        """Evaluate how well model capability level matches task category."""
+        capabilities = getattr(model, "capabilities", None)
+        if not capabilities:
+            return 0.25  # Basic fallback
+
+        level = CapabilityLevel.NONE
+        # Extract capability level from Pydantic model or dictionary representation
+        if isinstance(capabilities, dict):
+            level_val = capabilities.get(task_category, CapabilityLevel.NONE)
+            if isinstance(level_val, int):
+                level = CapabilityLevel(level_val)
+        else:
+            if hasattr(capabilities, task_category):
+                level = getattr(capabilities, task_category)
+
+        # Score mapping
+        level_score_map = {
+            CapabilityLevel.NONE: 0.0,
+            CapabilityLevel.BASIC: 0.2,
+            CapabilityLevel.INTERMEDIATE: 0.5,
+            CapabilityLevel.ADVANCED: 0.8,
+            CapabilityLevel.EXPERT: 1.0,
+        }
+
+        return level_score_map.get(level, 0.2)
+
+    def _calculate_context_score(self, context_length: int, required_tokens: int) -> float:
+        """Evaluate how well context window size fits required token limits."""
+        if required_tokens <= 0:
+            return 1.0
+
+        if context_length >= required_tokens:
+            # Having extra headroom is good, but value decays as ratio grows
+            ratio = context_length / required_tokens
+            return min(1.0, 0.8 + 0.2 / ratio)
+        else:
+            # Severe penalty for context overflow
+            return max(0.0, (context_length / required_tokens) * 0.5)
+
+    def _calculate_speed_score(
+        self, model: ModelDescriptor, latency_requirement: str, profile: ModelProfile | None
+    ) -> float:
+        """Calculate speed score using empirical metrics (TPS/TTFT) if available, falling back to static tiers."""
+        # Dynamic scoring if profile metrics exist
+        if profile and profile.tps > 0:
+            # Estimate speed based on empirical tokens per second. (assume 80 TPS is maximum optimal score)
+            empirical_tps_score = min(1.0, profile.tps / 80.0)
+
+            # Penalize long TTFT (assume > 1.5 seconds starts decaying score)
+            ttft_penalty = (
+                max(0.0, min(0.5, (profile.ttft_ms - 1500.0) / 3000.0))
+                if profile.ttft_ms > 0
+                else 0.0
+            )
+            return max(0.1, empirical_tps_score - ttft_penalty)
+
+        # Fallback to static speed tiers
+        speed_map = {"fast": 1.0, "medium": 0.6, "slow": 0.3}
+        model_speed = speed_map.get(model.speed_tier, 0.6)
+
+        req_map = {"fast": 1.0, "medium": 0.6, "slow": 0.3}
+        req_speed = req_map.get(latency_requirement, 0.6)
+
+        if model_speed >= req_speed:
+            return 1.0
+        return model_speed / req_speed
+
+    def _calculate_reliability_score(
+        self, model: ModelDescriptor, profile: ModelProfile | None, local_preferred: bool
+    ) -> float:
+        """Determine reliability and preference score based on locality and validation history."""
+        score = 0.9  # Baseline reliability
+
+        if model.is_local:
+            # Boost if local models are requested
+            score += 0.1 if local_preferred else 0.05
+        else:
+            # Slight penalty if we strictly prefer local running
+            score -= 0.2 if local_preferred else 0.0
+
+        # Empirical JSON formatting validity penalty
+        if profile and profile.json_validity < 1.0:
+            score -= (1.0 - profile.json_validity) * 0.5
+
+        return max(0.0, min(1.0, score))
+
+    def _calculate_cost_penalty(self, model: ModelDescriptor) -> float:
+        """Calculate score penalty based on token cost."""
+        cost = model.cost_per_1k_tokens
+        if cost is None or cost <= 0.0:
+            return 0.0  # Zero cost for local offline models
+
+        # Standardize cost penalty (assuming max expected cost is $0.15 per 1k tokens)
+        return min(1.0, cost / 0.15)

velune/models/specializations.py

@@ -0,0 +1,205 @@
+"""Model specialization mapper for the Reasoning Council with role-specific context optimizations."""
+
+from __future__ import annotations
+
+import logging
+from enum import StrEnum
+
+from velune.core.types.model import ModelDescriptor
+from velune.models.profiler import ModelProfiler
+from velune.models.registry import ModelCapabilityRegistry
+from velune.models.scorer import ModelScorer
+
+logger = logging.getLogger("velune.models.specializations")
+
+
+class CouncilRole(StrEnum):
+    """Roles in the Velune Reasoning Council."""
+
+    PLANNER = "planner"
+    CODER = "coder"
+    REVIEWER = "reviewer"
+    CHALLENGER = "challenger"
+    SYNTHESIZER = "synthesizer"
+
+
+ROLE_CONTEXT_REQUIREMENTS = {
+    CouncilRole.PLANNER: 16384,  # Needs full repo context
+    CouncilRole.CODER: 32768,  # Needs code + context + plan
+    CouncilRole.REVIEWER: 32768,  # Needs to see full code
+    CouncilRole.CHALLENGER: 16384,  # Needs code summary
+    CouncilRole.SYNTHESIZER: 65536,  # Needs all outputs
+}
+
+
+class ModelSpecializationMapper:
+    """Intelligent mapper that assigns discovered models to council roles based on scoring and role-specific context bounds."""
+
+    def __init__(
+        self,
+        registry: ModelCapabilityRegistry,
+        scorer: ModelScorer | None = None,
+        profiler: ModelProfiler | None = None,
+    ) -> None:
+        self.registry = registry
+        self.scorer = scorer or ModelScorer()
+        self.profiler = profiler or ModelProfiler()
+        self.overrides: dict[CouncilRole, str] = {}
+
+    def map_roles(
+        self,
+        task_category: str = "coding",
+        required_tokens: int | None = None,
+        local_preferred: bool = False,
+    ) -> dict[CouncilRole, ModelDescriptor]:
+        """
+        Assigns the best available model for each CouncilRole based on their functional profiles and optimal context token sizes.
+
+        - Planner: High planning and instruction-following scores (optimizes for 16k context window).
+        - Coder: High coding and tool-use scores (optimizes for 32k context window).
+        - Reviewer: High reasoning and instruction-following scores (optimizes for 32k context window).
+        - Challenger: High reasoning and adversarial analysis capabilities (optimizes for 16k context window).
+        - Synthesizer: High summarization and context window capability (optimizes for 64k context window).
+        """
+        models = self.registry.list_all()
+        if not models:
+            logger.warning(
+                "No models found in the capability registry. Council mappings will be empty."
+            )
+            return {}
+
+        assignments: dict[CouncilRole, ModelDescriptor] = {}
+
+        def get_tokens(role: CouncilRole) -> int:
+            if required_tokens is not None:
+                if role == CouncilRole.REVIEWER:
+                    return required_tokens + 2048
+                elif role == CouncilRole.SYNTHESIZER:
+                    return required_tokens + 4096
+                else:
+                    return required_tokens
+            return ROLE_CONTEXT_REQUIREMENTS[role]
+
+        # 1. Map Planner
+        planner_model = self._select_best_model(
+            models=models,
+            role_category="planning",
+            required_tokens=get_tokens(CouncilRole.PLANNER),
+            latency_requirement="medium",
+            local_preferred=local_preferred,
+        )
+        if planner_model:
+            assignments[CouncilRole.PLANNER] = planner_model
+
+        # 2. Map Coder
+        coder_model = self._select_best_model(
+            models=models,
+            role_category="coding",
+            required_tokens=get_tokens(CouncilRole.CODER),
+            latency_requirement="medium",
+            local_preferred=local_preferred,
+        )
+        if coder_model:
+            assignments[CouncilRole.CODER] = coder_model
+
+        # 3. Map Reviewer (prefers slower, highly capable reasoning models)
+        reviewer_model = self._select_best_model(
+            models=models,
+            role_category="reasoning",
+            required_tokens=get_tokens(CouncilRole.REVIEWER),
+            latency_requirement="slow",
+            local_preferred=local_preferred,
+        )
+        if reviewer_model:
+            assignments[CouncilRole.REVIEWER] = reviewer_model
+
+        # 4. Map Challenger (needs strong reasoning)
+        challenger_model = self._select_best_model(
+            models=models,
+            role_category="reasoning",
+            required_tokens=get_tokens(CouncilRole.CHALLENGER),
+            latency_requirement="medium",
+            local_preferred=local_preferred,
+        )
+        if challenger_model:
+            assignments[CouncilRole.CHALLENGER] = challenger_model
+
+        # 5. Map Synthesizer (prefers faster summarization models with large context)
+        synthesizer_model = self._select_best_model(
+            models=models,
+            role_category="summarization",
+            required_tokens=get_tokens(CouncilRole.SYNTHESIZER),
+            latency_requirement="fast",
+            local_preferred=local_preferred,
+        )
+        if synthesizer_model:
+            assignments[CouncilRole.SYNTHESIZER] = synthesizer_model
+
+        # Ensure we have fallbacks for all roles if any fail to map
+        if models:
+            default_model = models[0]
+            for role in CouncilRole:
+                if role not in assignments:
+                    logger.info(
+                        "Falling back role %s to default model %s",
+                        role.value,
+                        default_model.model_id,
+                    )
+                    assignments[role] = default_model
+
+        # Apply explicitly assigned overrides
+        for role, overridden_model_id in self.overrides.items():
+            descriptor = self.registry.get(overridden_model_id)
+            if descriptor:
+                assignments[role] = descriptor
+
+        return assignments
+
+    def _select_best_model(
+        self,
+        models: list[ModelDescriptor],
+        role_category: str,
+        required_tokens: int,
+        latency_requirement: str,
+        local_preferred: bool,
+    ) -> ModelDescriptor | None:
+        """Helper to score all models and select the highest scoring candidate."""
+        try:
+            from velune.kernel.registry import get_container
+
+            gpu_info = get_container().get("runtime.gpu_info")
+            available_vram_gb = gpu_info.get("vram_free_gb")
+        except Exception:
+            available_vram_gb = None
+
+        best_model: ModelDescriptor | None = None
+        best_score = -1.0
+
+        for model in models:
+            # VRAM check for local models
+            if model.is_local and available_vram_gb is not None:
+                required_vram = model.vram_required_gb
+                if required_vram and required_vram > available_vram_gb:
+                    logger.info(
+                        "Skipping %s: requires %.1fGB VRAM, only %.1fGB available",
+                        model.model_id,
+                        required_vram,
+                        available_vram_gb,
+                    )
+                    continue  # Skip models that won't fit in VRAM
+
+            profile = self.profiler.get_profile(model.provider_id, model.model_id)
+            score = self.scorer.score(
+                model=model,
+                task_category=role_category,
+                required_tokens=required_tokens,
+                latency_requirement=latency_requirement,
+                profile=profile,
+                local_preferred=local_preferred,
+            )
+
+            if score > best_score:
+                best_score = score
+                best_model = model
+
+        return best_model