DeepFabric 4.5.1__tar.gz → 4.6.0__tar.gz

{deepfabric-4.5.1 → deepfabric-4.6.0}/.github/workflows/integration.yml

@@ -19,6 +19,12 @@ jobs:
     name: Run integration tests (Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest
 
+    services:
+      spin:
+        image: ghcr.io/always-further/deepfabric/tools-sdk:latest
+        ports:
+          - 3000:3000
+
     strategy:
       fail-fast: false
       matrix:
@@ -28,6 +34,23 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v5
 
+      - name: Wait for Spin service
+        run: |
+          echo "Waiting for Spin service to be ready..."
+          for i in {1..30}; do
+            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3000/vfs/execute -X POST \
+              -H "Content-Type: application/json" \
+              -d '{"session_id":"healthcheck","tool":"list_files","args":{"directory":"/"}}' 2>/dev/null || echo "000")
+            if [ "$response" = "200" ]; then
+              echo "Spin service is ready"
+              exit 0
+            fi
+            echo "Attempt $i: waiting... (HTTP $response)"
+            sleep 2
+          done
+          echo "Spin service failed to become ready"
+          exit 1
+
       - name: Set up Python
         uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6
         with:
@@ -46,3 +69,5 @@ jobs:
         run: make test-integration
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          SPIN_ENDPOINT: http://localhost:3000

{deepfabric-4.5.1 → deepfabric-4.6.0}/.gitignore

@@ -129,8 +129,12 @@ target/
 # Project files
 venv/
 *.jsonl
+*.json
 formats/*
 # coverage files
 .coverage.*
 tools-sdk/.spin/*
 notebooks/lora-output/*
+
+# docs
+site/

deepfabric-4.6.0/Makefile

@@ -0,0 +1,65 @@
+.PHONY: clean install format lint test-unit test-integration test-integration-verbose security build all
+.PHONY: test-integration-openai test-integration-gemini test-integration-llm
+.PHONY: test-integration-hubs test-integration-spin test-integration-quick
+.PHONY: test-integration-graph test-integration-generator
+
+# Base command for integration tests
+PYTEST_INTEGRATION = uv run pytest tests/integration --tb=short -v
+
+clean:
+	rm -rf build/
+	rm -rf dist/
+	rm -rf *.egg-info
+	rm -f .coverage
+	find . -type d -name '__pycache__' -exec rm -rf {} +
+	find . -type f -name '*.pyc' -delete
+
+install:
+	uv sync --all-extras
+
+format: ## Format code with ruff (parallel)
+	uv run ruff format deepfabric/ tests/
+
+lint:
+	uv run ruff check . --exclude notebooks/
+
+test-unit:
+	uv run pytest tests/unit/
+
+test-integration:
+	$(PYTEST_INTEGRATION) --maxfail=1
+
+test-integration-verbose:
+	uv run pytest tests/integration -v -rA --durations=10
+
+test-integration-openai:
+	$(PYTEST_INTEGRATION) -m openai
+
+test-integration-gemini:
+	$(PYTEST_INTEGRATION) -m gemini
+
+test-integration-llm:
+	$(PYTEST_INTEGRATION) -m "openai or gemini"
+
+test-integration-hubs:
+	$(PYTEST_INTEGRATION) -m huggingface
+
+test-integration-spin:
+	$(PYTEST_INTEGRATION) -m spin
+
+test-integration-quick:
+	$(PYTEST_INTEGRATION) -m "not huggingface"
+
+test-integration-graph:
+	$(PYTEST_INTEGRATION) tests/integration/test_graph_integration.py
+
+test-integration-generator:
+	$(PYTEST_INTEGRATION) tests/integration/test_generator_integration.py
+
+security:
+	uv run bandit -r deepfabric/
+
+build: clean test-unit
+	uv build
+
+all: clean install format lint test-unit test-integration security build

{deepfabric-4.5.1 → deepfabric-4.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DeepFabric
-Version: 4.5.1
+Version: 4.6.0
 Summary: Curate High Quality Datasets, Train, Evaluate and Ship
 Author-email: Luke Hinds <luke@alwaysfurther.ai>
 License-File: LICENSE
@@ -29,10 +29,12 @@ Requires-Dist: sentencepiece>=0.1.99
 Requires-Dist: spin-sdk>=3.4.1
 Requires-Dist: torch>=2.4.0
 Requires-Dist: transformers>=4.57.1
+Requires-Dist: trl>=0.26.2
 Provides-Extra: dev
 Requires-Dist: bandit>=1.7.10; extra == 'dev'
 Requires-Dist: mermaid-py>=0.2.0; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest-httpx>=0.30.0; extra == 'dev'
 Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
 Requires-Dist: pytest>=7.0.0; extra == 'dev'
 Requires-Dist: requests-mock>=1.11.0; extra == 'dev'
@@ -45,7 +47,7 @@ Description-Content-Type: text/markdown
 <div align="center">
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="./assets/logo-light.png" />
-    <img alt="DeepFabric logo" src="./assets/logo-light-hols.png" style="width:40%;max-width:40%;height:auto;display:block;margin:0 auto;" />
+    <img alt="DeepFabric logo" src="./assets/logo-light.png" style="width:40%;max-width:40%;height:auto;display:block;margin:0 auto;" />
   </picture>
   <h3>Training Model Behavior in Agentic Systems</h3>
 
@@ -123,7 +125,15 @@ This generates a topic graph and creates 27 unique nodes, then generates 27 trai
 
 ## Configuration
 
-DeepFabric also uses YAML configuration with three main sections and optional shared LLM defaults:
+DeepFabric also uses YAML configuration with three main sections and optional shared LLM defaults
+
+> [!NOTE]  
+> The following uses mocked tool execution, so will require a runing Spin service, which we provide in a docker image:
+```bash
+docker run -d -p 3000:3000 ghcr.io/always-further/deepfabric/tools-sdk:latest`
+```
+
+Save the following as `config.yaml`:
 
 ```yaml
 # Optional: Shared LLM defaults (inherited by topics and generation)
@@ -146,34 +156,74 @@ topics:
 # GENERATION: Create training samples from topics
 generation:
   system_prompt: |
-    You are an expert Python backend developer and technical educator.
+    You are an expert Python backend developer specializing in REST API design.
     Create practical, production-ready code examples with clear explanations.
     Include error handling, type hints, and follow PEP 8 conventions.
+    Use the following tools to read, write, and list files in the virtual filesystem:
+    - read_file
+    - write_file
+    - list_files
 
   # Additional instructions for sample generation
   instructions: |
-    Focus on real-world scenarios developers encounter daily.
+    Focus on real-world scenarios developers encounter daily when building REST APIs with Python.
     Include both happy path and edge case handling.
-    Provide context on when and why to use specific patterns.
+    Provide context on when and why to use specific patterns or libraries.
+    Ensure code is modular, testable, and maintainable.
 
   conversation:
-    type: chain_of_thought      # basic | chain_of_thought
-    reasoning_style: agent      # freetext | agent (for chain_of_thought)
+    type: cot      # basic | cot
+    reasoning_style: agent      # freetext | agent (for cot)
     agent_mode: single_turn     # single_turn | multi_turn (for agent)
   
   # Tool configuration (required for agent modes)
   tools:
     spin_endpoint: "http://localhost:3000"  # Spin service for tool execution
-    available:                  # Filter to specific tools (empty = all VFS tools)
-      - read_file
-      - write_file
-      - list_files
+    components:                 # Map component name to tool names
+      builtin:                  # Routes to /vfs/execute
+        - read_file
+        - write_file
+        - list_files
     max_per_query: 3            # Maximum tools per query
     max_agent_steps: 5          # Max ReAct reasoning iterations
 
-    max_retries: 3                # Retries for failed generations
-    sample_retries: 2             # Retries for validation failures
-    max_tokens: 2000              # Max tokens per generation
+  # Optional: Seed initial files into the spin before generation, used for tool calling
+    scenario_seed:
+      files:
+        "Dockerfile": |
+          FROM python:3.13
+          WORKDIR /usr/local/app
+
+          # Install the application dependencies
+          COPY requirements.txt ./
+          RUN pip install --no-cache-dir -r requirements.txt
+
+          # Copy in the source code
+          COPY src ./src
+          EXPOSE 8080
+
+          # Setup an app user so the container doesn't run as the root user
+          RUN useradd app
+          USER app
+
+          CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
+        "main.py": |
+          def greet(name):
+              return f"Hello, {name}!"
+
+          if __name__ == "__main__":
+              print(greet("World"))
+        "config.json": |
+          {
+            "version": "1.0.0",
+            "debug": true,
+            "max_retries": 3
+          }
+
+  # Generation control and retry settings
+  max_retries: 3                # Retries for failed generations
+  sample_retries: 2             # Retries for validation failures
+  max_tokens: 2000              # Max tokens per generation
 
   # Optional: Override shared LLM settings
   llm:
@@ -193,13 +243,13 @@ output:
   batch_size: 3                 # Parallel generation batch size
   save_as: "api-dataset.jsonl"
 
-# Optional: Upload to Hugging Face
-huggingface:
-  repository: "your-username/api-dataset-training-name"
-  tags: ["python", "programming"]
+ Optional: Upload to Hugging Face
+ huggingface:
+   repository: "your-username/api-dataset-training-name"
+   tags: ["python", "programming"]
 ```
 
-Run with:
+Run generation by sourcing the `config.yaml`:
 
 ```bash
 deepfabric generate config.yaml
@@ -209,6 +259,14 @@ deepfabric generate config.yaml
 
 DeepFabric returns standard HuggingFace datasets, making it easy to integrate with any training framework.
 
+### Colab Notebooks:
+
+A quick way of seeing DeepFabric in action is via our notebooks in the [notebooks/](./notebooks/) folder or on Google Colab:
+
+**Qwen4b Blender MCP**:
+
+[![Qwen4b Blender MCP](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1EG1V40v5xkJKLf6Ra6W4378vYqlZNVWqb)
+
 ### 1. Generate Dataset
 
 ```bash
@@ -327,7 +385,6 @@ config = EvaluatorConfig(
         model_path="Qwen/Qwen2.5-7B-Instruct",    # Base model
         adapter_path="./output/lora-adapter",     # LoRA adapter path
         backend="transformers",
-        use_unsloth=True,      # Use Unsloth for adapters trained with Unsloth
         load_in_4bit=True,     # 4-bit quantization
         max_seq_length=2048,
     ),
@@ -418,159 +475,6 @@ evaluator = Evaluator(config)
 results = evaluator.evaluate(dataset=eval_dataset)
 ```
 
-## Training Metrics
-
-DeepFabric provides a training callback that automatically logs metrics to the DeepFabric cloud during model training. This enables real-time monitoring and tracking of training runs.
-
-### Basic Usage with HuggingFace Trainer
-
-```python
-from transformers import Trainer, TrainingArguments
-from deepfabric import DeepFabricCallback
-
-# Set up training arguments
-training_args = TrainingArguments(
-    output_dir="./output",
-    num_train_epochs=3,
-    per_device_train_batch_size=4,
-    logging_steps=10,
-)
-
-# Create trainer
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-)
-
-# Add DeepFabric callback for metrics logging
-trainer.add_callback(DeepFabricCallback(trainer))
-
-# Train - metrics are automatically logged
-trainer.train()
-```
-
-### Usage with TRL SFTTrainer
-
-```python
-from trl import SFTTrainer, SFTConfig
-from deepfabric import DeepFabricCallback
-
-trainer = SFTTrainer(
-    model=model,
-    tokenizer=tokenizer,
-    train_dataset=train_dataset,
-    args=SFTConfig(
-        output_dir="./output",
-        num_train_epochs=3,
-        logging_steps=10,
-    ),
-)
-
-# Add callback - works with any Trainer-compatible class
-trainer.add_callback(DeepFabricCallback(trainer))
-trainer.train()
-```
-
-### Configuration Options
-
-```python
-from deepfabric import DeepFabricCallback
-
-callback = DeepFabricCallback(
-    trainer=trainer,                              # Optional: Trainer instance
-    api_key="your-api-key",                       # Or set DEEPFABRIC_API_KEY env var
-    endpoint="https://api.deepfabric.ai",         # Custom endpoint (optional)
-    enabled=True,                                 # Disable to skip logging
-)
-```
-
-### Environment Variables
-
-```bash
-# API key for authentication
-export DEEPFABRIC_API_KEY="your-api-key"
-
-# Custom API endpoint (optional)
-export DEEPFABRIC_API_URL="https://api.deepfabric.ai"
-```
-
-### Logged Metrics
-
-The callback automatically captures and logs:
-
-| Metric Type | Examples |
-|-------------|----------|
-| Training | `loss`, `learning_rate`, `epoch`, `global_step` |
-| Throughput | `train_runtime`, `train_samples_per_second` |
-| Evaluation | `eval_loss`, `eval_accuracy` (when evaluation is run) |
-| TRL-specific | `rewards/chosen`, `rewards/rejected`, `kl_divergence` |
-| Checkpoints | Checkpoint save events with step numbers |
-
-### Callback Events
-
-```python
-# The callback hooks into these Trainer events:
-# - on_train_begin: Logs run start with training configuration
-# - on_log: Logs training metrics (loss, lr, etc.)
-# - on_evaluate: Logs evaluation metrics
-# - on_save: Logs checkpoint events
-# - on_train_end: Logs run completion and flushes pending metrics
-```
-
-### Non-Blocking Design
-
-The callback uses a background thread to send metrics asynchronously, ensuring training is never blocked by network operations:
-
-```python
-from deepfabric.training import MetricsSender
-
-# Direct access to sender for advanced use cases
-sender = MetricsSender(
-    endpoint="https://api.deepfabric.ai",
-    api_key="your-key",
-    batch_size=10,        # Batch metrics before sending
-    flush_interval=5.0,   # Auto-flush every 5 seconds
-    max_queue_size=1000,  # Queue capacity
-)
-
-# Manually send metrics
-sender.send_metrics({"custom_metric": 0.95, "step": 100})
-
-# Flush pending metrics (blocking)
-sender.flush(timeout=30.0)
-
-# Check sender statistics
-print(sender.stats)
-# {'metrics_sent': 150, 'metrics_dropped': 0, 'send_errors': 0, 'queue_size': 0}
-```
-
-### Interactive API Key Prompt
-
-When running in an interactive environment (Jupyter notebook, terminal) without an API key configured, the callback will prompt for authentication:
-
-```python
-from deepfabric import DeepFabricCallback
-
-# If DEEPFABRIC_API_KEY is not set, prompts for login
-callback = DeepFabricCallback(trainer)
-# > DeepFabric API key not found. Log in to enable cloud metrics.
-# > Visit: https://app.deepfabric.ai/signup
-```
-
-### Disabling Metrics Logging
-
-```python
-# Disable via constructor
-callback = DeepFabricCallback(trainer, enabled=False)
-
-# Or set API key to None
-callback = DeepFabricCallback(trainer, api_key=None)
-
-# Or don't set DEEPFABRIC_API_KEY environment variable
-```
-
 ## Providers
 
 | Provider | Local/Cloud | Best For |
@@ -628,7 +532,7 @@ Enable tool tracing in your YAML config:
 ```yaml
 generation:
   conversation:
-    type: chain_of_thought
+    type: cot
     reasoning_style: agent
     agent_mode: single_turn