DeepFabric 4.4.0__py3-none-any.whl

deepfabric/evaluation/reporters/base.py

@@ -0,0 +1,56 @@
+"""Base classes for evaluation result reporting."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..evaluator import EvaluationResult
+    from ..metrics import SampleEvaluation
+
+
+class BaseReporter(ABC):
+    """Base class for evaluation result reporters.
+
+    Reporters handle the output of evaluation results. They can write to
+    local files, send to cloud services, or perform other actions with
+    the evaluation data.
+    """
+
+    def __init__(self, config: dict | None = None):
+        """Initialize reporter with optional configuration.
+
+        Args:
+            config: Optional reporter-specific configuration
+        """
+        self.config = config or {}
+
+    @abstractmethod
+    def report(self, result: EvaluationResult) -> None:
+        """Report complete evaluation results.
+
+        Called once at the end of evaluation with all results.
+
+        Args:
+            result: Complete evaluation result
+        """
+
+    def report_sample(self, sample_eval: SampleEvaluation) -> None:  # noqa: B027
+        """Report individual sample evaluation (optional, for streaming).
+
+        Called after each sample is evaluated, allowing real-time reporting.
+        Default implementation does nothing.
+
+        Args:
+            sample_eval: Individual sample evaluation result
+        """
+        pass  # Optional hook for subclasses
+
+    def get_name(self) -> str:
+        """Return reporter identifier.
+
+        Returns:
+            Reporter name (e.g., "file", "cloud")
+        """
+        return self.__class__.__name__.replace("Reporter", "").lower()

deepfabric/evaluation/reporters/cloud_reporter.py

@@ -0,0 +1,195 @@
+"""Cloud-based reporter for sending results to DeepFabric Cloud."""
+
+from __future__ import annotations
+
+import json
+import os
+
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import httpx
+
+from rich.console import Console
+
+from .base import BaseReporter
+
+if TYPE_CHECKING:
+    from ..evaluator import EvaluationResult
+    from ..metrics import SampleEvaluation
+
+console = Console()
+
+
+def get_auth_token() -> str | None:
+    """Get authentication token from CLI config."""
+    config_file = Path.home() / ".deepfabric" / "config.json"
+    if not config_file.exists():
+        return None
+
+    try:
+        with open(config_file) as f:
+            config = json.load(f)
+            # Return API key if present, otherwise access token
+            return config.get("api_key") or config.get("access_token")
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+class CloudReporter(BaseReporter):
+    """Posts evaluation results to DeepFabric cloud service."""
+
+    def __init__(self, config: dict | None = None):
+        """Initialize cloud reporter.
+
+        Args:
+            config: Optional configuration with:
+                - api_url: DeepFabric API URL (default: https://api.deepfabric.dev")
+                - project_id: Project ID to associate results with
+                - auth_token: Authentication token (if not provided, will read from config file)
+                - enabled: Whether to enable cloud reporting (default: True if authenticated)
+        """
+        super().__init__(config)
+
+        # Get API URL from config or environment
+        self.api_url = os.getenv("DEEPFABRIC_API_URL", "https://api.deepfabric.dev")
+        if config and "api_url" in config:
+            self.api_url = config["api_url"]
+
+        # Get auth token from config or CLI config file
+        self.auth_token = None
+        if config and "auth_token" in config:
+            self.auth_token = config["auth_token"]
+        else:
+            self.auth_token = get_auth_token()
+
+        # Get project ID from config
+        self.project_id = config.get("project_id") if config else None
+
+        # Enable cloud reporting if authenticated
+        self.enabled = (
+            config.get("enabled", bool(self.auth_token)) if config else bool(self.auth_token)
+        )
+
+        # Generate unique run ID for this evaluation
+        self.run_id = None  # Will be set when creating run
+        self.evaluation_run_id = None  # Backend run ID
+
+    def report(self, result: EvaluationResult) -> None:
+        """Upload complete evaluation results to cloud service.
+
+        Args:
+            result: Complete evaluation result
+        """
+        if not self.enabled:
+            return
+
+        if not self.auth_token:
+            console.print(
+                "[yellow]Cloud reporting skipped: Not authenticated. "
+                "Run 'deepfabric auth login' to enable cloud sync.[/yellow]"
+            )
+            return
+
+        if not self.project_id:
+            console.print("[yellow]Cloud reporting skipped: No project_id configured.[/yellow]")
+            return
+
+        try:
+            console.print("[cyan]Uploading evaluation results to cloud...[/cyan]")
+
+            # Create evaluation run
+            run_data = {
+                "project_id": self.project_id,
+                "name": f"Evaluation - {datetime.now(UTC).strftime('%Y-%m-%d %H:%M')}",
+                "model_name": result.config.inference_config.model_path,
+                "model_provider": result.config.inference_config.backend,
+                "config": {
+                    "evaluators": getattr(result.config, "evaluators", ["tool_calling"]),
+                    "inference": result.config.inference_config.model_dump(),
+                },
+                "status": "completed",
+            }
+
+            with httpx.Client(timeout=30.0) as client:
+                # Create run
+                response = client.post(
+                    f"{self.api_url}/api/v1/evaluations/runs",
+                    json=run_data,
+                    headers={
+                        "Authorization": f"Bearer {self.auth_token}",
+                        "Content-Type": "application/json",
+                    },
+                )
+                response.raise_for_status()
+                run_response = response.json()
+                self.evaluation_run_id = run_response["id"]
+
+                console.print(f"[green]v[/green] Created evaluation run: {self.evaluation_run_id}")
+
+                # Upload metrics
+                metrics_data = {
+                    "overall_score": result.metrics.overall_score,
+                    "tool_selection_accuracy": result.metrics.tool_selection_accuracy,
+                    "parameter_accuracy": result.metrics.parameter_accuracy,
+                    "execution_success_rate": result.metrics.execution_success_rate,
+                    "response_quality": result.metrics.response_quality,
+                    "samples_evaluated": result.metrics.samples_evaluated,
+                    "samples_processed": result.metrics.samples_processed,
+                    "processing_errors": result.metrics.processing_errors,
+                }
+
+                response = client.post(
+                    f"{self.api_url}/api/v1/evaluations/runs/{self.evaluation_run_id}/metrics",
+                    json=metrics_data,
+                    headers={
+                        "Authorization": f"Bearer {self.auth_token}",
+                        "Content-Type": "application/json",
+                    },
+                )
+                response.raise_for_status()
+
+                console.print("[green]v[/green] Uploaded metrics")
+
+                # Upload samples in batches
+                batch_size = 100
+                samples = []
+                for s in result.predictions:
+                    sample_dict = s.model_dump()
+                    # Convert sample_id to string (backend expects str, CLI uses int)
+                    sample_dict["sample_id"] = str(sample_dict["sample_id"])
+                    samples.append(sample_dict)
+
+                for i in range(0, len(samples), batch_size):
+                    batch = samples[i : i + batch_size]
+                    response = client.post(
+                        f"{self.api_url}/api/v1/evaluations/runs/{self.evaluation_run_id}/samples",
+                        json={"samples": batch},
+                        headers={
+                            "Authorization": f"Bearer {self.auth_token}",
+                            "Content-Type": "application/json",
+                        },
+                    )
+                    response.raise_for_status()
+
+                console.print(f"[green]v[/green] Uploaded {len(samples)} samples")
+                console.print("[green]Results uploaded successfully![/green]")
+                console.print(
+                    f"View at: {self.api_url.replace(':8080', ':3000')}/studio/evaluations/{self.evaluation_run_id}"
+                )
+
+        except httpx.HTTPError as e:
+            console.print(f"[red]Cloud upload failed: {e}[/red]")
+        except Exception as e:
+            console.print(f"[red]Cloud upload error: {e}[/red]")
+
+    def report_sample(self, sample_eval: SampleEvaluation) -> None:  # noqa: ARG002
+        """Stream individual sample to cloud for real-time progress tracking.
+
+        Args:
+            sample_eval: Individual sample evaluation result
+        """
+        # Real-time streaming not implemented yet
+        # Samples are uploaded in batch in report()
+        pass

deepfabric/evaluation/reporters/file_reporter.py

@@ -0,0 +1,61 @@
+"""File-based reporter for writing results to local JSON files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+
+from .base import BaseReporter
+
+if TYPE_CHECKING:
+    from ..evaluator import EvaluationResult
+    from ..metrics import SampleEvaluation
+
+console = Console()
+
+
+class FileReporter(BaseReporter):
+    """Writes evaluation results to local JSON file.
+
+    This is the default reporter that maintains backwards compatibility
+    with the original file-based output.
+    """
+
+    def __init__(self, config: dict | None = None):
+        """Initialize file reporter.
+
+        Args:
+            config: Optional configuration with 'path' key
+        """
+        super().__init__(config)
+        self.output_path = config.get("path") if config else None
+
+    def report(self, result: EvaluationResult) -> None:
+        """Write evaluation results to JSON file.
+
+        Args:
+            result: Complete evaluation result
+        """
+        # Use path from config, or fall back to result's config
+        output_path = self.output_path or result.config.output_path
+
+        if output_path is None:
+            console.print("[yellow]No output path specified, skipping file write[/yellow]")
+            return
+
+        path = Path(output_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        with path.open("w") as f:
+            f.write(result.model_dump_json(indent=2))
+
+        console.print(f"[green]Results saved to {path}[/green]")
+
+    def report_sample(self, sample_eval: SampleEvaluation) -> None:
+        """File reporter doesn't support streaming (waits for final results).
+
+        Args:
+            sample_eval: Individual sample evaluation (ignored)
+        """

deepfabric/evaluation/reporters/multi_reporter.py

@@ -0,0 +1,56 @@
+"""Multi-reporter for running multiple reporters simultaneously."""
+
+from __future__ import annotations
+
+from contextlib import suppress
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+
+from .base import BaseReporter
+
+if TYPE_CHECKING:
+    from ..evaluator import EvaluationResult
+    from ..metrics import SampleEvaluation
+
+console = Console()
+
+
+class MultiReporter(BaseReporter):
+    """Runs multiple reporters (e.g., file + cloud).
+
+    This reporter allows sending results to multiple destinations
+    simultaneously. Errors in one reporter don't affect others.
+    """
+
+    def __init__(self, reporters: list[BaseReporter]):
+        """Initialize multi-reporter.
+
+        Args:
+            reporters: List of reporter instances to run
+        """
+        super().__init__()
+        self.reporters = reporters
+
+    def report(self, result: EvaluationResult) -> None:
+        """Report to all reporters.
+
+        Args:
+            result: Complete evaluation result
+        """
+        for reporter in self.reporters:
+            try:
+                reporter.report(result)
+            except Exception as e:  # noqa: BLE001
+                console.print(f"[red]Reporter {reporter.get_name()} failed: {e}[/red]")
+
+    def report_sample(self, sample_eval: SampleEvaluation) -> None:
+        """Report sample to all reporters.
+
+        Args:
+            sample_eval: Individual sample evaluation result
+        """
+        for reporter in self.reporters:
+            # Silently fail on sample streaming errors
+            with suppress(Exception):
+                reporter.report_sample(sample_eval)

deepfabric/exceptions.py

@@ -0,0 +1,67 @@
+class DeepFabricError(Exception):
+    """Base exception class for DeepFabric."""
+
+    def __init__(self, message: str, context: dict | None = None):
+        super().__init__(message)
+        self.message = message
+        self.context = context or {}
+
+
+class ConfigurationError(DeepFabricError):
+    """Raised when there is an error in configuration."""
+
+    pass
+
+
+class ValidationError(DeepFabricError):
+    """Raised when data validation fails."""
+
+    pass
+
+
+class ModelError(DeepFabricError):
+    """Raised when there is an error with LLM model operations."""
+
+    pass
+
+
+class TreeError(DeepFabricError):
+    """Raised when there is an error in topic tree operations."""
+
+    pass
+
+
+class DataSetGeneratorError(DeepFabricError):
+    """Raised when there is an error in data engine operations."""
+
+    pass
+
+
+class DatasetError(DeepFabricError):
+    """Raised when there is an error in dataset operations."""
+
+    pass
+
+
+class HubUploadError(DeepFabricError):
+    """Raised when there is an error uploading to Hugging Face Hub."""
+
+    pass
+
+
+class JSONParsingError(ValidationError):
+    """Raised when JSON parsing fails."""
+
+    pass
+
+
+class APIError(ModelError):
+    """Raised when API calls fail."""
+
+    pass
+
+
+class RetryExhaustedError(ModelError):
+    """Raised when maximum retries are exceeded."""
+
+    pass

deepfabric/factory.py

@@ -0,0 +1,26 @@
+from .config import DeepFabricConfig
+from .graph import Graph
+from .topic_model import TopicModel
+from .tree import Tree
+
+
+def create_topic_generator(
+    config: DeepFabricConfig,
+    topics_overrides: dict | None = None,
+) -> TopicModel:
+    """Factory function to create a topic generator based on the configuration.
+
+    Args:
+        config: DeepFabricConfig object with topics configuration
+        topics_overrides: Override parameters for topic generation
+
+    Returns:
+        TopicModel (Tree or Graph) based on topics.mode
+    """
+    topics_params = config.get_topics_params(**(topics_overrides or {}))
+
+    if config.topics.mode == "graph":
+        return Graph(**topics_params)
+
+    # Default to tree mode
+    return Tree(**topics_params)