DeepFabric 4.5.1__py3-none-any.whl → 4.7.0__py3-none-any.whl

deepfabric/training/callback.py

@@ -51,6 +51,7 @@ class DeepFabricCallback:
         trainer: Any | None = None,
         api_key: str | None = None,
         endpoint: str | None = None,
+        pipeline_id: str | None = None,
         enabled: bool = True,
     ):
         """Initialize the DeepFabric callback.
@@ -60,11 +61,14 @@ class DeepFabricCallback:
             api_key: DeepFabric API key (falls back to DEEPFABRIC_API_KEY env var,
                      then prompts in interactive environments)
             endpoint: API endpoint URL (falls back to DEEPFABRIC_API_URL env var)
+            pipeline_id: Pipeline ID to associate training with (falls back to
+                         DEEPFABRIC_PIPELINE_ID env var or pipeline_id.txt file)
             enabled: Whether logging is enabled (default: True)
         """
         # Get API key from arg, env, or prompt
         self.api_key = api_key or get_api_key()
         self.endpoint = endpoint or os.getenv("DEEPFABRIC_API_URL", "https://api.deepfabric.ai")
+        self.pipeline_id = pipeline_id or self._get_pipeline_id()
         self.run_id = str(uuid.uuid4())
         self.enabled = enabled and self.api_key is not None
 
@@ -75,14 +79,26 @@ class DeepFabricCallback:
         self.sender = MetricsSender(
             endpoint=self.endpoint,
             api_key=self.api_key if self.enabled else None,
+            pipeline_id=self.pipeline_id,
         )
 
         self._run_started = False
         self._model_name: str | None = None
         self._training_args_logged = False
+        self._start_time: datetime | None = None
 
         if self.enabled:
-            logger.debug(f"DeepFabric callback initialized (run_id={self.run_id})")
+            if self.pipeline_id:
+                logger.debug(
+                    f"DeepFabric callback initialized (run_id={self.run_id}, "
+                    f"pipeline_id={self.pipeline_id})"
+                )
+            else:
+                logger.warning(
+                    "DeepFabric callback initialized but no pipeline_id set. "
+                    "Metrics will not be sent. Set DEEPFABRIC_PIPELINE_ID env var "
+                    "or create pipeline_id.txt file."
+                )
         else:
             logger.debug("DeepFabric callback disabled (no API key)")
 
@@ -101,6 +117,7 @@ class DeepFabricCallback:
             return
 
         self._run_started = True
+        self._start_time = datetime.now(timezone.utc)
 
         # Extract model name from various sources
         model = kwargs.get("model")
@@ -121,6 +138,7 @@ class DeepFabricCallback:
                     "num_train_epochs": state.num_train_epochs,
                     "is_world_process_zero": getattr(state, "is_world_process_zero", True),
                 },
+                "started_at": self._start_time.isoformat(),
             }
         )
 
@@ -204,6 +222,8 @@ class DeepFabricCallback:
         if not self.enabled or not self._run_started:
             return
 
+        completed_at = datetime.now(timezone.utc)
+
         self.sender.send_run_end(
             {
                 "run_id": self.run_id,
@@ -212,6 +232,7 @@ class DeepFabricCallback:
                 "total_flos": getattr(state, "total_flos", None),
                 "best_metric": getattr(state, "best_metric", None),
                 "best_model_checkpoint": getattr(state, "best_model_checkpoint", None),
+                "completed_at": completed_at.isoformat(),
             }
         )
 
@@ -246,6 +267,27 @@ class DeepFabricCallback:
             }
         )
 
+    def _get_pipeline_id(self) -> str | None:
+        """Get pipeline ID from environment or file.
+
+        Returns:
+            Pipeline ID or None
+        """
+        # Try environment variable first
+        pipeline_id = os.getenv("DEEPFABRIC_PIPELINE_ID", "")
+        if pipeline_id:
+            return pipeline_id
+
+        # Try pipeline_id.txt file
+        pipeline_file = "pipeline_id.txt"
+        if os.path.exists(pipeline_file):
+            with open(pipeline_file) as f:
+                pipeline_id = f.read().strip()
+                if pipeline_id:
+                    return pipeline_id
+
+        return None
+
     def _extract_model_name(self, args: TrainingArguments, model: Any | None) -> str | None:
         """Extract model name from various sources.
 

deepfabric/training/metrics_sender.py

@@ -35,6 +35,7 @@ class MetricsSender:
         self,
         endpoint: str,
         api_key: str | None,
+        pipeline_id: str | None = None,
         batch_size: int = 10,
         flush_interval: float = 5.0,
         max_queue_size: int = 1000,
@@ -45,6 +46,7 @@ class MetricsSender:
         Args:
             endpoint: Base URL for the DeepFabric API
             api_key: API key for authentication (None disables sending)
+            pipeline_id: Pipeline ID to associate training runs with (required)
             batch_size: Number of metrics to batch before sending
             flush_interval: Seconds between automatic flushes
             max_queue_size: Maximum queue size (overflow drops metrics)
@@ -52,12 +54,14 @@ class MetricsSender:
         """
         self.endpoint = endpoint.rstrip("/")
         self.api_key = api_key
+        self.pipeline_id = pipeline_id
         self.batch_size = batch_size
         self.flush_interval = flush_interval
         self.timeout = timeout
 
         self._queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=max_queue_size)
         self._stop_event = threading.Event()
+        self._flush_event = threading.Event()
         self._enabled = api_key is not None
 
         # Start background sender thread
@@ -177,19 +181,25 @@ class MetricsSender:
                 should_flush = (
                     len(batch) >= self.batch_size
                     or (time.monotonic() - last_flush) >= self.flush_interval
+                    or self._flush_event.is_set()
                 )
 
                 if should_flush:
                     self._flush_batch(batch)
                     batch = []
                     last_flush = time.monotonic()
+                    self._flush_event.clear()
 
             except queue.Empty:
-                # Timeout - flush if we have pending items
-                if batch and (time.monotonic() - last_flush) >= self.flush_interval:
+                # Timeout - flush if we have pending items or flush requested
+                if batch and (
+                    (time.monotonic() - last_flush) >= self.flush_interval
+                    or self._flush_event.is_set()
+                ):
                     self._flush_batch(batch)
                     batch = []
                     last_flush = time.monotonic()
+                    self._flush_event.clear()
 
         # On shutdown, drain the queue and flush everything
         while not self._queue.empty():
@@ -209,21 +219,34 @@ class MetricsSender:
         if not batch or not self._enabled:
             return
 
+        if not self.pipeline_id:
+            logger.debug("No pipeline_id set, skipping metrics send")
+            return
+
         # Separate events and metrics
-        events = [item for item in batch if item["type"] != "metrics"]
+        run_start_events = [item for item in batch if item["type"] == "run_start"]
+        run_end_events = [item for item in batch if item["type"] == "run_end"]
         metrics = [item["data"] for item in batch if item["type"] == "metrics"]
 
-        # Send events first (run_start, run_end)
-        for event in events:
-            self._send_to_api(
-                endpoint=f"{self.endpoint}/v1/training/runs",
-                payload={"event_type": event["type"], **event["data"]},
-            )
+        # Build query string with pipeline_id
+        query = f"?pipeline_id={self.pipeline_id}"
+
+        def send_run_events(events: list[dict[str, Any]]) -> None:
+            """Send run start/end events."""
+            for event in events:
+                self._send_to_api(
+                    endpoint=f"{self.endpoint}/api/v1/training/runs{query}",
+                    payload={"event_type": event["type"], **event["data"]},
+                )
+
+        # Send run events, ensuring start events are processed before end events
+        send_run_events(run_start_events)
+        send_run_events(run_end_events)
 
         # Send metrics batch
         if metrics:
             self._send_to_api(
-                endpoint=f"{self.endpoint}/v1/training/metrics",
+                endpoint=f"{self.endpoint}/api/v1/training/metrics{query}",
                 payload={"metrics": metrics},
             )
             self._metrics_sent += len(metrics)
@@ -252,22 +275,27 @@ class MetricsSender:
 
             if not response.ok:
                 self._send_errors += 1
-                logger.debug(f"API request failed: {response.status_code} {response.text[:100]}")
+                logger.warning(
+                    "API error: %s %s (endpoint: %s)",
+                    response.status_code,
+                    response.text[:200],
+                    endpoint,
+                )
                 return False
 
         except requests.exceptions.Timeout:
             self._send_errors += 1
-            logger.debug("API request timed out")
+            logger.warning("Request timed out: %s", endpoint)
             return False
 
-        except requests.exceptions.ConnectionError:
+        except requests.exceptions.ConnectionError as e:
             self._send_errors += 1
-            logger.debug("API connection error")
+            logger.warning("Connection error: %s (endpoint: %s)", e, endpoint)
             return False
 
         except requests.exceptions.RequestException as e:
             self._send_errors += 1
-            logger.debug(f"API request error: {e}")
+            logger.warning("Request error: %s (endpoint: %s)", e, endpoint)
             return False
 
         else:
@@ -282,8 +310,14 @@ class MetricsSender:
         if not self._enabled:
             return
 
+        # Signal the background thread to flush its current batch
+        self._flush_event.set()
+
         start = time.monotonic()
-        while not self._queue.empty() and (time.monotonic() - start) < timeout:
+        # Wait for queue to empty and flush event to be cleared (indicates batch was sent)
+        while (time.monotonic() - start) < timeout:
+            if self._queue.empty() and not self._flush_event.is_set():
+                break
             time.sleep(0.1)
 
     def shutdown(self) -> None:

deepfabric/tui.py

@@ -41,14 +41,22 @@ class TopicBuildingMixin:
 
     Subclasses must have these attributes:
     - tui: DeepFabricTUI instance
+    - live_display: Live | None
     - live_layout: Layout | None
     - events_log: deque
     """
 
     tui: "DeepFabricTUI"
+    live_display: "Live | None"
     live_layout: "Layout | None"
     events_log: "deque"
 
+    def stop_live(self) -> None:
+        """Stop the Live display if it's running."""
+        if self.live_display:
+            self.live_display.stop()
+            self.live_display = None
+
     def _refresh_left(self) -> None:
         """Update events panel in left column."""
         if self.live_layout is not None:
@@ -910,7 +918,7 @@ class DatasetGenerationTUI(StreamObserver):
             # Map conversation types to friendly names
             type_map = {
                 "basic": "Basic Q&A",
-                "chain_of_thought": "Chain of Thought",
+                "cot": "Chain of Thought",
                 "single_turn_agent": "Single-Turn Agent (Tool Calling)",
                 "multi_turn_agent": "Multi-Turn Agent (Tool Calling)",
             }

deepfabric/utils.py

@@ -1,6 +1,7 @@
 import ast
 import asyncio
 import json
+import os
 import re
 
 VALIDATION_ERROR_INDICATORS = [
@@ -147,4 +148,17 @@ def read_topic_tree_from_jsonl(file_path: str) -> list[dict]:
     with open(file_path) as file:
         for line in file:
             topic_tree.append(json.loads(line.strip()))
+
     return topic_tree
+
+
+def get_bool_env(key: str, default: bool = False) -> bool:
+    """Get a boolean environment variable.
+
+    Supports: '1', 'true', 'yes', 'on' (case-insensitive) as True.
+    Everything else is False unless default is True and key is missing.
+    """
+    val = os.getenv(key)
+    if val is None:
+        return default
+    return val.lower() in ("1", "true", "yes", "on")

deepfabric/validation.py

@@ -79,7 +79,7 @@ def validate_path_requirements(
             for steps, batch in optimal_combinations[:3]:  # Show top 3
                 total_samples = steps * batch
                 recommendations.append(
-                    f"    --num-steps {steps} --batch-size {batch}  (generates {total_samples} samples)"
+                    f"    --num-samples {steps} --batch-size {batch}  (generates {total_samples} samples)"
                 )
 
         recommendations.extend(

{deepfabric-4.5.1.dist-info → deepfabric-4.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DeepFabric
-Version: 4.5.1
+Version: 4.7.0
 Summary: Curate High Quality Datasets, Train, Evaluate and Ship
 Author-email: Luke Hinds <luke@alwaysfurther.ai>
 License-File: LICENSE
@@ -29,10 +29,12 @@ Requires-Dist: sentencepiece>=0.1.99
 Requires-Dist: spin-sdk>=3.4.1
 Requires-Dist: torch>=2.4.0
 Requires-Dist: transformers>=4.57.1
+Requires-Dist: trl>=0.26.2
 Provides-Extra: dev
 Requires-Dist: bandit>=1.7.10; extra == 'dev'
 Requires-Dist: mermaid-py>=0.2.0; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest-httpx>=0.30.0; extra == 'dev'
 Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
 Requires-Dist: pytest>=7.0.0; extra == 'dev'
 Requires-Dist: requests-mock>=1.11.0; extra == 'dev'
@@ -45,7 +47,7 @@ Description-Content-Type: text/markdown
 <div align="center">
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="./assets/logo-light.png" />
-    <img alt="DeepFabric logo" src="./assets/logo-light-hols.png" style="width:40%;max-width:40%;height:auto;display:block;margin:0 auto;" />
+    <img alt="DeepFabric logo" src="./assets/logo-light.png" style="width:40%;max-width:40%;height:auto;display:block;margin:0 auto;" />
   </picture>
   <h3>Training Model Behavior in Agentic Systems</h3>
 
@@ -123,7 +125,15 @@ This generates a topic graph and creates 27 unique nodes, then generates 27 trai
 
 ## Configuration
 
-DeepFabric also uses YAML configuration with three main sections and optional shared LLM defaults:
+DeepFabric also uses YAML configuration with three main sections and optional shared LLM defaults
+
+> [!NOTE]  
+> The following uses mocked tool execution, so will require a runing Spin service, which we provide in a docker image:
+```bash
+docker run -d -p 3000:3000 ghcr.io/always-further/deepfabric/tools-sdk:latest`
+```
+
+Save the following as `config.yaml`:
 
 ```yaml
 # Optional: Shared LLM defaults (inherited by topics and generation)
@@ -146,34 +156,74 @@ topics:
 # GENERATION: Create training samples from topics
 generation:
   system_prompt: |
-    You are an expert Python backend developer and technical educator.
+    You are an expert Python backend developer specializing in REST API design.
     Create practical, production-ready code examples with clear explanations.
     Include error handling, type hints, and follow PEP 8 conventions.
+    Use the following tools to read, write, and list files in the virtual filesystem:
+    - read_file
+    - write_file
+    - list_files
 
   # Additional instructions for sample generation
   instructions: |
-    Focus on real-world scenarios developers encounter daily.
+    Focus on real-world scenarios developers encounter daily when building REST APIs with Python.
     Include both happy path and edge case handling.
-    Provide context on when and why to use specific patterns.
+    Provide context on when and why to use specific patterns or libraries.
+    Ensure code is modular, testable, and maintainable.
 
   conversation:
-    type: chain_of_thought      # basic | chain_of_thought
-    reasoning_style: agent      # freetext | agent (for chain_of_thought)
+    type: cot      # basic | cot
+    reasoning_style: agent      # freetext | agent (for cot)
     agent_mode: single_turn     # single_turn | multi_turn (for agent)
   
   # Tool configuration (required for agent modes)
   tools:
     spin_endpoint: "http://localhost:3000"  # Spin service for tool execution
-    available:                  # Filter to specific tools (empty = all VFS tools)
-      - read_file
-      - write_file
-      - list_files
+    components:                 # Map component name to tool names
+      builtin:                  # Routes to /vfs/execute
+        - read_file
+        - write_file
+        - list_files
     max_per_query: 3            # Maximum tools per query
     max_agent_steps: 5          # Max ReAct reasoning iterations
 
-    max_retries: 3                # Retries for failed generations
-    sample_retries: 2             # Retries for validation failures
-    max_tokens: 2000              # Max tokens per generation
+  # Optional: Seed initial files into the spin before generation, used for tool calling
+    scenario_seed:
+      files:
+        "Dockerfile": |
+          FROM python:3.13
+          WORKDIR /usr/local/app
+
+          # Install the application dependencies
+          COPY requirements.txt ./
+          RUN pip install --no-cache-dir -r requirements.txt
+
+          # Copy in the source code
+          COPY src ./src
+          EXPOSE 8080
+
+          # Setup an app user so the container doesn't run as the root user
+          RUN useradd app
+          USER app
+
+          CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
+        "main.py": |
+          def greet(name):
+              return f"Hello, {name}!"
+
+          if __name__ == "__main__":
+              print(greet("World"))
+        "config.json": |
+          {
+            "version": "1.0.0",
+            "debug": true,
+            "max_retries": 3
+          }
+
+  # Generation control and retry settings
+  max_retries: 3                # Retries for failed generations
+  sample_retries: 2             # Retries for validation failures
+  max_tokens: 2000              # Max tokens per generation
 
   # Optional: Override shared LLM settings
   llm:
@@ -193,13 +243,13 @@ output:
   batch_size: 3                 # Parallel generation batch size
   save_as: "api-dataset.jsonl"
 
-# Optional: Upload to Hugging Face
-huggingface:
-  repository: "your-username/api-dataset-training-name"
-  tags: ["python", "programming"]
+ Optional: Upload to Hugging Face
+ huggingface:
+   repository: "your-username/api-dataset-training-name"
+   tags: ["python", "programming"]
 ```
 
-Run with:
+Run generation by sourcing the `config.yaml`:
 
 ```bash
 deepfabric generate config.yaml
@@ -209,6 +259,14 @@ deepfabric generate config.yaml
 
 DeepFabric returns standard HuggingFace datasets, making it easy to integrate with any training framework.
 
+### Colab Notebooks:
+
+A quick way of seeing DeepFabric in action is via our notebooks in the [notebooks/](./notebooks/) folder or on Google Colab:
+
+**Qwen4b Blender MCP**:
+
+[![Qwen4b Blender MCP](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1EG1V40v5xkJKLf6Ra6W4378vYqlZNVWqb)
+
 ### 1. Generate Dataset
 
 ```bash
@@ -327,7 +385,6 @@ config = EvaluatorConfig(
         model_path="Qwen/Qwen2.5-7B-Instruct",    # Base model
         adapter_path="./output/lora-adapter",     # LoRA adapter path
         backend="transformers",
-        use_unsloth=True,      # Use Unsloth for adapters trained with Unsloth
         load_in_4bit=True,     # 4-bit quantization
         max_seq_length=2048,
     ),
@@ -418,159 +475,6 @@ evaluator = Evaluator(config)
 results = evaluator.evaluate(dataset=eval_dataset)
 ```
 
-## Training Metrics
-
-DeepFabric provides a training callback that automatically logs metrics to the DeepFabric cloud during model training. This enables real-time monitoring and tracking of training runs.
-
-### Basic Usage with HuggingFace Trainer
-
-```python
-from transformers import Trainer, TrainingArguments
-from deepfabric import DeepFabricCallback
-
-# Set up training arguments
-training_args = TrainingArguments(
-    output_dir="./output",
-    num_train_epochs=3,
-    per_device_train_batch_size=4,
-    logging_steps=10,
-)
-
-# Create trainer
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-)
-
-# Add DeepFabric callback for metrics logging
-trainer.add_callback(DeepFabricCallback(trainer))
-
-# Train - metrics are automatically logged
-trainer.train()
-```
-
-### Usage with TRL SFTTrainer
-
-```python
-from trl import SFTTrainer, SFTConfig
-from deepfabric import DeepFabricCallback
-
-trainer = SFTTrainer(
-    model=model,
-    tokenizer=tokenizer,
-    train_dataset=train_dataset,
-    args=SFTConfig(
-        output_dir="./output",
-        num_train_epochs=3,
-        logging_steps=10,
-    ),
-)
-
-# Add callback - works with any Trainer-compatible class
-trainer.add_callback(DeepFabricCallback(trainer))
-trainer.train()
-```
-
-### Configuration Options
-
-```python
-from deepfabric import DeepFabricCallback
-
-callback = DeepFabricCallback(
-    trainer=trainer,                              # Optional: Trainer instance
-    api_key="your-api-key",                       # Or set DEEPFABRIC_API_KEY env var
-    endpoint="https://api.deepfabric.ai",         # Custom endpoint (optional)
-    enabled=True,                                 # Disable to skip logging
-)
-```
-
-### Environment Variables
-
-```bash
-# API key for authentication
-export DEEPFABRIC_API_KEY="your-api-key"
-
-# Custom API endpoint (optional)
-export DEEPFABRIC_API_URL="https://api.deepfabric.ai"
-```
-
-### Logged Metrics
-
-The callback automatically captures and logs:
-
-| Metric Type | Examples |
-|-------------|----------|
-| Training | `loss`, `learning_rate`, `epoch`, `global_step` |
-| Throughput | `train_runtime`, `train_samples_per_second` |
-| Evaluation | `eval_loss`, `eval_accuracy` (when evaluation is run) |
-| TRL-specific | `rewards/chosen`, `rewards/rejected`, `kl_divergence` |
-| Checkpoints | Checkpoint save events with step numbers |
-
-### Callback Events
-
-```python
-# The callback hooks into these Trainer events:
-# - on_train_begin: Logs run start with training configuration
-# - on_log: Logs training metrics (loss, lr, etc.)
-# - on_evaluate: Logs evaluation metrics
-# - on_save: Logs checkpoint events
-# - on_train_end: Logs run completion and flushes pending metrics
-```
-
-### Non-Blocking Design
-
-The callback uses a background thread to send metrics asynchronously, ensuring training is never blocked by network operations:
-
-```python
-from deepfabric.training import MetricsSender
-
-# Direct access to sender for advanced use cases
-sender = MetricsSender(
-    endpoint="https://api.deepfabric.ai",
-    api_key="your-key",
-    batch_size=10,        # Batch metrics before sending
-    flush_interval=5.0,   # Auto-flush every 5 seconds
-    max_queue_size=1000,  # Queue capacity
-)
-
-# Manually send metrics
-sender.send_metrics({"custom_metric": 0.95, "step": 100})
-
-# Flush pending metrics (blocking)
-sender.flush(timeout=30.0)
-
-# Check sender statistics
-print(sender.stats)
-# {'metrics_sent': 150, 'metrics_dropped': 0, 'send_errors': 0, 'queue_size': 0}
-```
-
-### Interactive API Key Prompt
-
-When running in an interactive environment (Jupyter notebook, terminal) without an API key configured, the callback will prompt for authentication:
-
-```python
-from deepfabric import DeepFabricCallback
-
-# If DEEPFABRIC_API_KEY is not set, prompts for login
-callback = DeepFabricCallback(trainer)
-# > DeepFabric API key not found. Log in to enable cloud metrics.
-# > Visit: https://app.deepfabric.ai/signup
-```
-
-### Disabling Metrics Logging
-
-```python
-# Disable via constructor
-callback = DeepFabricCallback(trainer, enabled=False)
-
-# Or set API key to None
-callback = DeepFabricCallback(trainer, api_key=None)
-
-# Or don't set DEEPFABRIC_API_KEY environment variable
-```
-
 ## Providers
 
 | Provider | Local/Cloud | Best For |
@@ -628,7 +532,7 @@ Enable tool tracing in your YAML config:
 ```yaml
 generation:
   conversation:
-    type: chain_of_thought
+    type: cot
     reasoning_style: agent
     agent_mode: single_turn