DeepFabric 4.4.0__py3-none-any.whl → 4.5.1__py3-none-any.whl

deepfabric/evaluation/backends/transformers_backend.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import sys
 
 from typing import Any
 
@@ -36,9 +37,15 @@ class TransformersBackend(InferenceBackend):
         """
         super().__init__(config)
 
+        # Check if model is pre-loaded (not a string path)
+        is_preloaded = not isinstance(config.model, str)
+
         # Determine device
         if config.device:
             self.device = config.device
+        elif is_preloaded:
+            # Get device from pre-loaded model
+            self.device = str(next(config.model.parameters()).device)
         # Auto-detect best available device
         elif torch.cuda.is_available():
             self.device = "cuda"
@@ -48,7 +55,7 @@ class TransformersBackend(InferenceBackend):
             self.device = "cpu"
 
         # Determine dtype based on device
-        if self.device == "cuda":
+        if self.device == "cuda" or self.device.startswith("cuda:"):
             dtype = torch.float16
             device_map = "auto"
         elif self.device == "mps":
@@ -58,11 +65,36 @@ class TransformersBackend(InferenceBackend):
             dtype = torch.float32
             device_map = None
 
+        # Handle pre-loaded model case - skip all loading logic
+        if is_preloaded:
+            self.model = config.model
+            self.tokenizer = config.tokenizer
+            self.loaded_with_unsloth = False
+
+            # Detect architecture from pre-loaded model's config
+            self._architectures = []
+            if hasattr(self.model, "config"):
+                self._architectures = getattr(self.model.config, "architectures", []) or []
+
+            # Initialize tool call parser
+            self._tool_call_parser: ToolCallParser = get_parser(self._architectures)
+            logger.info(
+                "Using pre-loaded model with %s parser for architectures: %s",
+                type(self._tool_call_parser).__name__,
+                self._architectures or ["unknown"],
+            )
+
+            # Set padding token if not set
+            if self.tokenizer.pad_token is None:
+                self.tokenizer.pad_token = self.tokenizer.eos_token
+
+            return  # Skip remaining initialization
+
         # Detect model architecture for parser selection and tokenizer config
-        self._architectures: list[str] = []
+        self._architectures = []
         tokenizer_kwargs: dict[str, Any] = {}
         try:
-            model_config = AutoConfig.from_pretrained(config.model_path)  # nosec
+            model_config = AutoConfig.from_pretrained(config.model)  # nosec
             self._architectures = getattr(model_config, "architectures", []) or []
             if any(arch in MISTRAL_ARCHITECTURES for arch in self._architectures):
                 tokenizer_kwargs["fix_mistral_regex"] = True
@@ -71,7 +103,7 @@ class TransformersBackend(InferenceBackend):
             logger.warning("Could not detect model architecture: %s", e)
 
         # Initialize tool call parser based on detected architecture
-        self._tool_call_parser: ToolCallParser = get_parser(self._architectures)
+        self._tool_call_parser = get_parser(self._architectures)
         logger.info(
             "Using %s for model architectures: %s",
             type(self._tool_call_parser).__name__,
@@ -79,19 +111,44 @@ class TransformersBackend(InferenceBackend):
         )
 
         self.loaded_with_unsloth = False
-        # Load with Unsloth if requested
-        if config.use_unsloth:
+
+        # Detect if Unsloth has already patched the environment
+        # This happens when user imports unsloth in the same runtime
+        unsloth_patched = "unsloth" in sys.modules
+
+        # Use Unsloth if explicitly requested OR if Unsloth has patched the environment
+        # (to avoid "apply_qkv" errors from patched attention classes)
+        use_unsloth_loading = config.use_unsloth or unsloth_patched
+
+        if use_unsloth_loading:
             try:
                 from unsloth import FastLanguageModel  # type: ignore # noqa: PLC0415
 
-                # Load from adapter path if provided, otherwise from model_path
-                load_path = config.adapter_path if config.adapter_path else config.model_path
-                self.model, self.tokenizer = FastLanguageModel.from_pretrained(
-                    model_name=load_path,
-                    max_seq_length=config.max_seq_length,
-                    dtype=dtype,
-                    load_in_4bit=config.load_in_4bit,
-                )
+                if unsloth_patched and not config.use_unsloth:
+                    logger.info(
+                        "Unsloth detected in environment, using Unsloth loader for compatibility"
+                    )
+
+                if config.adapter_path:
+                    # Load base model first, then apply adapter
+                    self.model, self.tokenizer = FastLanguageModel.from_pretrained(
+                        model_name=config.model,
+                        max_seq_length=config.max_seq_length,
+                        dtype=dtype,
+                        load_in_4bit=config.load_in_4bit,
+                    )
+                    # Load LoRA adapter using PEFT
+                    from peft import PeftModel  # noqa: PLC0415
+
+                    self.model = PeftModel.from_pretrained(self.model, config.adapter_path)
+                else:
+                    # Load merged model or base model directly
+                    self.model, self.tokenizer = FastLanguageModel.from_pretrained(
+                        model_name=config.model,
+                        max_seq_length=config.max_seq_length,
+                        dtype=dtype,
+                        load_in_4bit=config.load_in_4bit,
+                    )
                 FastLanguageModel.for_inference(self.model)
                 self.loaded_with_unsloth = True
             except ImportError:
@@ -104,11 +161,11 @@ class TransformersBackend(InferenceBackend):
         # Standard transformers/PEFT loading
         if not self.loaded_with_unsloth:
             self.tokenizer = AutoTokenizer.from_pretrained(  # nosec
-                config.model_path, **tokenizer_kwargs
+                config.model, **tokenizer_kwargs
             )
 
             self.model = AutoModelForCausalLM.from_pretrained(  # nosec
-                config.model_path,
+                config.model,
                 device_map=device_map,
                 dtype=dtype,
             )

deepfabric/evaluation/evaluator.py

@@ -36,12 +36,12 @@ class EvaluatorConfig(BaseModel):
         default=None,
         description="Path to save evaluation results",
     )
-    model_path: str | None = Field(
+    model: str | None = Field(
         default=None,
-        description="Path to model to evaluate (overrides inference_config.model_path)",
+        description="Model to evaluate (overrides inference_config.model)",
     )
     inference_config: InferenceConfig = Field(
-        description="Inference backend configuration (includes model_path)",
+        description="Inference backend configuration (includes model)",
     )
     batch_size: int = Field(
         default=1,
@@ -119,7 +119,7 @@ class Evaluator:
             "evaluator_created",
             {
                 "backend": self.config.inference_config.backend,
-                "model_path": self.config.inference_config.model_path,
+                "model": self.config.inference_config.model,
                 "has_adapter": self.config.inference_config.adapter_path is not None,
                 "evaluators": (
                     list(self.config.evaluators)
@@ -434,6 +434,7 @@ class Evaluator:
                 ground_truth=ground_truth,
                 response=response,
                 evaluator_results=evaluator_results,
+                tools=tools,
             )
 
         except Exception as e:  # noqa: BLE001
@@ -442,8 +443,9 @@ class Evaluator:
             expected_tool = None
             expected_params: dict[str, Any] = {}
             expected_answer = None
+            available_tool_names: list[str] = []
 
-            # Try to extract ground truth if available
+            # Try to extract ground truth and tools if available
             try:
                 gt = self.extract_ground_truth(sample)
                 query = gt.query
@@ -453,9 +455,16 @@ class Evaluator:
             except (KeyError, AttributeError, ValidationError):
                 pass
 
+            try:
+                tools = self.prepare_tools(sample)
+                available_tool_names = [t.name for t in tools]
+            except (KeyError, AttributeError, ValidationError):
+                pass
+
             return SampleEvaluation(
                 sample_id=sample_id,
                 query=query,
+                available_tools=available_tool_names,
                 expected_tool=expected_tool,
                 predicted_tool=None,
                 expected_parameters=expected_params,
@@ -560,6 +569,7 @@ class Evaluator:
                 ground_truth=ground_truth,
                 predicted_tool_calls=all_predicted_tool_calls,
                 final_content=final_content,
+                tools=tools,
             )
 
         except Exception as e:  # noqa: BLE001
@@ -568,6 +578,7 @@ class Evaluator:
             expected_tool = None
             expected_params: dict[str, Any] = {}
             expected_answer = None
+            available_tool_names: list[str] = []
 
             try:
                 gt = self.extract_ground_truth(sample)
@@ -578,9 +589,16 @@ class Evaluator:
             except (KeyError, AttributeError, ValidationError):
                 pass
 
+            try:
+                tools = self.prepare_tools(sample)
+                available_tool_names = [t.name for t in tools]
+            except (KeyError, AttributeError, ValidationError):
+                pass
+
             return SampleEvaluation(
                 sample_id=sample_id,
                 query=query,
+                available_tools=available_tool_names,
                 expected_tool=expected_tool,
                 predicted_tool=None,
                 expected_parameters=expected_params,
@@ -600,6 +618,7 @@ class Evaluator:
         ground_truth: GroundTruth,
         predicted_tool_calls: list[dict],
         final_content: str,
+        tools: list[ToolDefinition] | None = None,
     ) -> SampleEvaluation:
         """Compute metrics for multi-turn evaluation.
 
@@ -610,6 +629,7 @@ class Evaluator:
             ground_truth: Expected values including all expected tools
             predicted_tool_calls: All tool calls made by model across turns
             final_content: Final model response content
+            tools: List of available tools for this sample
 
         Returns:
             SampleEvaluation with computed metrics
@@ -652,9 +672,13 @@ class Evaluator:
         # Execution valid if we got through the conversation
         execution_valid = len(predicted_tool_calls) > 0 or final_content != ""
 
+        # Extract tool names for available_tools field
+        available_tool_names = [t.name for t in tools] if tools else []
+
         return SampleEvaluation(
             sample_id=sample_id,
             query=ground_truth.query,
+            available_tools=available_tool_names,
             expected_tool=ground_truth.expected_tool,
             predicted_tool=first_predicted_tool,
             expected_parameters=ground_truth.expected_parameters,
@@ -714,6 +738,7 @@ class Evaluator:
         ground_truth: GroundTruth,
         response: ModelResponse,
         evaluator_results: list[EvaluatorResult],
+        tools: list[ToolDefinition] | None = None,
     ) -> SampleEvaluation:
         """Aggregate evaluator results into SampleEvaluation.
 
@@ -722,6 +747,7 @@ class Evaluator:
             ground_truth: Expected values
             response: Model response
             evaluator_results: Results from all evaluators
+            tools: List of available tools for this sample
 
         Returns:
             SampleEvaluation with aggregated metrics
@@ -746,10 +772,14 @@ class Evaluator:
                 params_correct = metrics.get("parameter_accuracy", 0.0) == 1.0
                 execution_valid = metrics.get("execution_valid", 0.0) == 1.0
 
+        # Extract tool names for available_tools field
+        available_tool_names = [t.name for t in tools] if tools else []
+
         # Return backwards-compatible SampleEvaluation
         return SampleEvaluation(
             sample_id=sample_id,
             query=ground_truth.query,
+            available_tools=available_tool_names,
             expected_tool=ground_truth.expected_tool,
             predicted_tool=predicted_tool,
             expected_parameters=ground_truth.expected_parameters,
@@ -780,13 +810,17 @@ class Evaluator:
         console.print("[bold blue]Running evaluation...[/bold blue]")
         evaluations = []
 
-        for idx, sample in tqdm(enumerate(samples), total=len(samples), desc="Evaluating"):
+        pbar = tqdm(enumerate(samples), total=len(samples), desc="Evaluating")
+        for idx, sample in pbar:
             eval_result = self.evaluate_sample(sample, idx)
             evaluations.append(eval_result)
 
             # Stream sample to reporters (for cloud real-time tracking)
             self.reporter.report_sample(eval_result)
 
+            # Force refresh for notebook compatibility
+            pbar.refresh()
+
         console.print("[bold green]Evaluation complete![/bold green]")
 
         # Compute metrics
@@ -804,7 +838,7 @@ class Evaluator:
             "evaluation_completed",
             {
                 "backend": self.config.inference_config.backend,
-                "model_path": self.config.inference_config.model_path,
+                "model": self.config.inference_config.model,
                 "has_adapter": self.config.inference_config.adapter_path is not None,
                 "samples_evaluated": metrics.samples_evaluated,
                 "samples_processed": metrics.samples_processed,

deepfabric/evaluation/evaluators/builtin/tool_calling.py

@@ -63,14 +63,19 @@ class ToolCallingEvaluator(BaseEvaluator):
         # Compute metrics
         tool_correct = predicted_tool == ground_truth.expected_tool
 
-        # Validate parameters against the PREDICTED tool (not expected)
-        # This measures parameter extraction capability independently of tool selection
-        params_correct = compare_parameters(
-            ground_truth.expected_parameters,
-            predicted_params,
-            tool_name=predicted_tool,  # Use predicted tool for schema validation
-            tool_definitions=context.tools,
-        )
+        # Parameter accuracy requires a tool to have been called
+        # If no tool was predicted but one was expected, params cannot be correct
+        if predicted_tool is None and ground_truth.expected_tool is not None:
+            params_correct = False
+        else:
+            # Validate parameters against the PREDICTED tool (not expected)
+            # This measures parameter extraction capability independently of tool selection
+            params_correct = compare_parameters(
+                ground_truth.expected_parameters,
+                predicted_params,
+                tool_name=predicted_tool,  # Use predicted tool for schema validation
+                tool_definitions=context.tools,
+            )
 
         # Execution valid requires BOTH correct tool AND correct params
         execution_valid = tool_correct and params_correct

deepfabric/evaluation/inference.py

@@ -1,9 +1,9 @@
 """Model inference interfaces and implementations for evaluation."""
 
 from abc import ABC, abstractmethod
-from typing import Literal
+from typing import Any, Literal
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator
 
 from ..schemas import ToolDefinition
 
@@ -11,17 +11,40 @@ from ..schemas import ToolDefinition
 class InferenceConfig(BaseModel):
     """Configuration for model inference."""
 
-    model_path: str = Field(
-        description="Path to model (local path or HuggingFace Hub ID)",
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    model: str | Any = Field(
+        description="Model identifier (local path, HuggingFace Hub ID, or model name for cloud providers). "
+        "Can also be a pre-loaded model object to avoid reloading.",
+    )
+    tokenizer: Any | None = Field(
+        default=None,
+        description="Pre-loaded tokenizer object. Required when model is a pre-loaded model object.",
     )
     adapter_path: str | None = Field(
         default=None,
         description="Path to PEFT/LoRA adapter (if using adapter-based fine-tuning)",
     )
-    backend: Literal["transformers", "ollama"] = Field(
+    backend: Literal["transformers", "ollama", "llm"] = Field(
         default="transformers",
         description="Inference backend to use",
     )
+    provider: Literal["openai", "anthropic", "gemini", "openrouter"] | None = Field(
+        default=None,
+        description="Cloud LLM provider (required when backend='llm')",
+    )
+    api_key: str | None = Field(
+        default=None,
+        description="API key for the provider (falls back to environment variable if not set)",
+    )
+    base_url: str | None = Field(
+        default=None,
+        description="Custom base URL for the API (e.g., for OpenRouter or proxies)",
+    )
+    rate_limit_config: dict | None = Field(
+        default=None,
+        description="Rate limiting configuration overrides",
+    )
     use_unsloth: bool = Field(
         default=False,
         description="Use Unsloth for loading adapter (for adapters trained with Unsloth)",
@@ -62,6 +85,51 @@ class InferenceConfig(BaseModel):
         description="Batch size for inference",
     )
 
+    @field_serializer("model")
+    def serialize_model(self, value: str | Any) -> str:
+        """Serialize model field - convert objects to descriptive string."""
+        if isinstance(value, str):
+            return value
+        # For in-memory model objects, return a descriptive string
+        model_class = type(value).__name__
+        model_name = getattr(getattr(value, "config", None), "name_or_path", "unknown")
+        return f"<in-memory:{model_class}:{model_name}>"
+
+    @field_serializer("tokenizer")
+    def serialize_tokenizer(self, value: Any | None) -> str | None:
+        """Serialize tokenizer field - convert objects to descriptive string."""
+        if value is None:
+            return None
+        if isinstance(value, str):
+            return value
+        # For in-memory tokenizer objects, return a descriptive string
+        tokenizer_class = type(value).__name__
+        tokenizer_name = getattr(value, "name_or_path", "unknown")
+        return f"<in-memory:{tokenizer_class}:{tokenizer_name}>"
+
+    @model_validator(mode="after")
+    def validate_config(self) -> "InferenceConfig":
+        """Validate configuration consistency."""
+        # Ensure provider is set when using LLM backend
+        if self.backend == "llm" and self.provider is None:
+            msg = "provider must be specified when backend='llm'"
+            raise ValueError(msg)
+
+        # Check if model is a pre-loaded object (not a string path)
+        is_preloaded_model = not isinstance(self.model, str)
+
+        # If model is pre-loaded, tokenizer must also be provided
+        if is_preloaded_model and self.tokenizer is None:
+            msg = "tokenizer must be provided when using a pre-loaded model object"
+            raise ValueError(msg)
+
+        # Pre-loaded models only work with transformers backend
+        if is_preloaded_model and self.backend != "transformers":
+            msg = "pre-loaded model objects are only supported with backend='transformers'"
+            raise ValueError(msg)
+
+        return self
+
 
 class ModelResponse(BaseModel):
     """Model inference response."""
@@ -150,6 +218,10 @@ def create_inference_backend(config: InferenceConfig) -> InferenceBackend:
         from .backends.ollama_backend import OllamaBackend  # noqa: PLC0415
 
         return OllamaBackend(config)
+    if config.backend == "llm":
+        from .backends.llm_eval_backend import LLMEvalBackend  # noqa: PLC0415
+
+        return LLMEvalBackend(config)
 
     msg = f"Unsupported backend: {config.backend}"
     raise ValueError(msg)

deepfabric/evaluation/metrics.py

@@ -107,6 +107,10 @@ class SampleEvaluation(BaseModel):
 
     sample_id: int = Field(description="Sample index")
     query: str = Field(description="Input query")
+    available_tools: list[str] = Field(
+        default_factory=list,
+        description="List of tool names available for this sample",
+    )
     expected_tool: str | None = Field(
         default=None,
         description="Expected tool name",

deepfabric/evaluation/reporters/cloud_reporter.py

@@ -103,7 +103,7 @@ class CloudReporter(BaseReporter):
             run_data = {
                 "project_id": self.project_id,
                 "name": f"Evaluation - {datetime.now(UTC).strftime('%Y-%m-%d %H:%M')}",
-                "model_name": result.config.inference_config.model_path,
+                "model_name": result.config.inference_config.model,
                 "model_provider": result.config.inference_config.backend,
                 "config": {
                     "evaluators": getattr(result.config, "evaluators", ["tool_calling"]),

deepfabric/generator.py

@@ -213,6 +213,10 @@ class DataSetGeneratorConfig(BaseModel):
         le=20,
         description="Minimum number of tool calls required before allowing early conversation conclusion",
     )
+    tool_inclusion_strategy: Literal["all", "used_only"] = Field(
+        default="used_only",
+        description="Which tools to include in each sample: 'all' includes full catalog, 'used_only' includes only tools actually called (recommended for training)",
+    )
 
 
 class DataSetGenerator:

deepfabric/hf_hub.py

@@ -210,5 +210,5 @@ class HFUploader:
         else:
             return {
                 "status": "success",
-                "message": f"Dataset pushed successfully to {hf_dataset_repo}.",
+                "message": f"Dataset pushed successfully to https://huggingface.co/datasets/{hf_dataset_repo}",
             }

deepfabric/llm/client.py

@@ -1,5 +1,7 @@
 import asyncio
+import logging
 import os
+import sys
 
 from functools import lru_cache
 from typing import Any
@@ -21,6 +23,8 @@ from .rate_limit_config import (
 )
 from .retry_handler import RetryHandler, retry_with_backoff, retry_with_backoff_async
 
+logger = logging.getLogger(__name__)
+
 # JSON Schema union type keys that need recursive processing
 _UNION_KEYS = ("anyOf", "oneOf", "allOf")
 
@@ -1061,8 +1065,18 @@ def _get_cached_openai_schema(schema: type[BaseModel]) -> type[BaseModel]:
     OpenAICompatModel.__name__ = f"{schema.__name__}OpenAICompat"
     OpenAICompatModel.__doc__ = schema.__doc__
 
-    # Rebuild model to resolve forward references (e.g., PendingToolCall in AgentStep)
-    OpenAICompatModel.model_rebuild()
+    # Rebuild model with the schema's original module namespace to resolve
+    # forward references (e.g., PendingToolCall in AgentStep)
+    schema_module = sys.modules.get(schema.__module__)
+    if schema_module:
+        OpenAICompatModel.model_rebuild(_types_namespace=vars(schema_module))
+    else:
+        logger.warning(
+            "Could not find module '%s' in sys.modules. "
+            "Forward reference resolution for dynamically created models may fail.",
+            schema.__module__,
+        )
+        OpenAICompatModel.model_rebuild()
 
     return OpenAICompatModel
 

deepfabric/schemas.py

@@ -304,8 +304,8 @@ class ToolDefinition(BaseModel):
         description = func.get("description", "")
         params_schema = func.get("parameters", {})
 
-        properties = params_schema.get("properties", {})
-        required_params = set(params_schema.get("required", []))
+        properties = params_schema.get("properties") or {}
+        required_params = set(params_schema.get("required") or [])
 
         parameters = []
         for param_name, param_props in properties.items():

deepfabric/training/__init__.py

@@ -1,20 +1,27 @@
-"""DeepFabric training metrics logging.
+"""DeepFabric training utilities.
 
-This module provides integration with HuggingFace Trainer and TRL trainers
-to log training metrics to the DeepFabric SaaS backend.
+This module provides:
+- Integration with HuggingFace Trainer and TRL trainers for metrics logging
+- Dataset preparation utilities for optimizing training data
 
 Features:
 - Non-blocking async metrics sending
 - Notebook-friendly API key prompts (like wandb)
 - Graceful handling of failures without impacting training
+- Tool filtering to reduce sequence lengths and memory usage
 
 Usage:
-    from deepfabric.training import DeepFabricCallback
+    from deepfabric.training import DeepFabricCallback, prepare_dataset_for_training
 
+    # Prepare dataset (reduces tool overhead)
+    dataset = load_dataset("your/dataset", split="train")
+    prepared = prepare_dataset_for_training(dataset, tool_strategy="used_only")
+
+    # Train with metrics logging
     trainer = Trainer(
         model=model,
         args=training_args,
-        train_dataset=train_dataset,
+        train_dataset=prepared,
     )
     trainer.add_callback(DeepFabricCallback(trainer))
     trainer.train()
@@ -27,9 +34,21 @@ Environment Variables:
 from __future__ import annotations
 
 from .callback import DeepFabricCallback
+from .dataset_utils import (
+    ToolInclusionStrategy,
+    clean_tool_schema,
+    filter_tools_for_sample,
+    get_used_tool_names,
+    prepare_dataset_for_training,
+)
 from .metrics_sender import MetricsSender
 
 __all__ = [
     "DeepFabricCallback",
     "MetricsSender",
+    "ToolInclusionStrategy",
+    "clean_tool_schema",
+    "filter_tools_for_sample",
+    "get_used_tool_names",
+    "prepare_dataset_for_training",
 ]