DeepFabric 4.4.1__tar.gz → 4.6.0__tar.gz

{deepfabric-4.4.1 → deepfabric-4.6.0}/.github/workflows/integration.yml

@@ -19,6 +19,12 @@ jobs:
     name: Run integration tests (Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest
 
+    services:
+      spin:
+        image: ghcr.io/always-further/deepfabric/tools-sdk:latest
+        ports:
+          - 3000:3000
+
     strategy:
       fail-fast: false
       matrix:
@@ -28,6 +34,23 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v5
 
+      - name: Wait for Spin service
+        run: |
+          echo "Waiting for Spin service to be ready..."
+          for i in {1..30}; do
+            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3000/vfs/execute -X POST \
+              -H "Content-Type: application/json" \
+              -d '{"session_id":"healthcheck","tool":"list_files","args":{"directory":"/"}}' 2>/dev/null || echo "000")
+            if [ "$response" = "200" ]; then
+              echo "Spin service is ready"
+              exit 0
+            fi
+            echo "Attempt $i: waiting... (HTTP $response)"
+            sleep 2
+          done
+          echo "Spin service failed to become ready"
+          exit 1
+
       - name: Set up Python
         uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6
         with:
@@ -46,3 +69,5 @@ jobs:
         run: make test-integration
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          SPIN_ENDPOINT: http://localhost:3000

{deepfabric-4.4.1 → deepfabric-4.6.0}/.gitignore

@@ -129,7 +129,12 @@ target/
 # Project files
 venv/
 *.jsonl
+*.json
 formats/*
 # coverage files
 .coverage.*
 tools-sdk/.spin/*
+notebooks/lora-output/*
+
+# docs
+site/

{deepfabric-4.4.1 → deepfabric-4.6.0}/CLAUDE.md

@@ -97,4 +97,5 @@ deepfabric start config.yaml --model gpt-4 --temperature 0.8 --hf-repo user/data
 - Bandit for security analysis
 - Python 3.11+ required
 - Google-style docstrings preferred
-- do not place imports anywhere but the top of the file
+- do not place imports anywhere but the top of the file
+- When updating `docs/` documentation, if new Markdown files are added or removed, consider updating `mkdocs.yml`.

deepfabric-4.6.0/Makefile

@@ -0,0 +1,65 @@
+.PHONY: clean install format lint test-unit test-integration test-integration-verbose security build all
+.PHONY: test-integration-openai test-integration-gemini test-integration-llm
+.PHONY: test-integration-hubs test-integration-spin test-integration-quick
+.PHONY: test-integration-graph test-integration-generator
+
+# Base command for integration tests
+PYTEST_INTEGRATION = uv run pytest tests/integration --tb=short -v
+
+clean:
+	rm -rf build/
+	rm -rf dist/
+	rm -rf *.egg-info
+	rm -f .coverage
+	find . -type d -name '__pycache__' -exec rm -rf {} +
+	find . -type f -name '*.pyc' -delete
+
+install:
+	uv sync --all-extras
+
+format: ## Format code with ruff (parallel)
+	uv run ruff format deepfabric/ tests/
+
+lint:
+	uv run ruff check . --exclude notebooks/
+
+test-unit:
+	uv run pytest tests/unit/
+
+test-integration:
+	$(PYTEST_INTEGRATION) --maxfail=1
+
+test-integration-verbose:
+	uv run pytest tests/integration -v -rA --durations=10
+
+test-integration-openai:
+	$(PYTEST_INTEGRATION) -m openai
+
+test-integration-gemini:
+	$(PYTEST_INTEGRATION) -m gemini
+
+test-integration-llm:
+	$(PYTEST_INTEGRATION) -m "openai or gemini"
+
+test-integration-hubs:
+	$(PYTEST_INTEGRATION) -m huggingface
+
+test-integration-spin:
+	$(PYTEST_INTEGRATION) -m spin
+
+test-integration-quick:
+	$(PYTEST_INTEGRATION) -m "not huggingface"
+
+test-integration-graph:
+	$(PYTEST_INTEGRATION) tests/integration/test_graph_integration.py
+
+test-integration-generator:
+	$(PYTEST_INTEGRATION) tests/integration/test_generator_integration.py
+
+security:
+	uv run bandit -r deepfabric/
+
+build: clean test-unit
+	uv build
+
+all: clean install format lint test-unit test-integration security build

{deepfabric-4.4.1 → deepfabric-4.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DeepFabric
-Version: 4.4.1
+Version: 4.6.0
 Summary: Curate High Quality Datasets, Train, Evaluate and Ship
 Author-email: Luke Hinds <luke@alwaysfurther.ai>
 License-File: LICENSE
@@ -29,10 +29,12 @@ Requires-Dist: sentencepiece>=0.1.99
 Requires-Dist: spin-sdk>=3.4.1
 Requires-Dist: torch>=2.4.0
 Requires-Dist: transformers>=4.57.1
+Requires-Dist: trl>=0.26.2
 Provides-Extra: dev
 Requires-Dist: bandit>=1.7.10; extra == 'dev'
 Requires-Dist: mermaid-py>=0.2.0; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest-httpx>=0.30.0; extra == 'dev'
 Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
 Requires-Dist: pytest>=7.0.0; extra == 'dev'
 Requires-Dist: requests-mock>=1.11.0; extra == 'dev'
@@ -45,7 +47,7 @@ Description-Content-Type: text/markdown
 <div align="center">
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="./assets/logo-light.png" />
-    <img alt="DeepFabric logo" src="./assets/logo-light-hols.png" style="width:40%;max-width:40%;height:auto;display:block;margin:0 auto;" />
+    <img alt="DeepFabric logo" src="./assets/logo-light.png" style="width:40%;max-width:40%;height:auto;display:block;margin:0 auto;" />
   </picture>
   <h3>Training Model Behavior in Agentic Systems</h3>
 
@@ -77,6 +79,9 @@ Description-Content-Type: text/markdown
     <a href="https://discord.gg/pPcjYzGvbS">
       <img src="https://img.shields.io/discord/1384081906773131274?color=7289da&label=Discord&logo=discord&logoColor=white" alt="Discord"/>
     </a>
+    <a href="https://www.reddit.com/r/deepfabric/">
+      <img src="https://img.shields.io/badge/Reddit-r%2Fdeepfabric-FF4500?logo=reddit&logoColor=white" alt="Reddit"/>
+    </a>
   </p>
 </div>
 
@@ -86,7 +91,7 @@ What sets DeepFabric apart from other dataset generation tools is its ability to
 
 <img src="/assets/df-demo.gif" width="100%" height="100%"/>
 
-Constrained decoding and response validation, along with real tool executions within isolated webassembly environments, ensure that generated samples strictly adhere to structured schema, variable constraints, and execution correctness, ensuring datasets have exact syntax and structure for use in model training pipelines. Tool definations can be either directly imported from MCP (Model Context Protocol) server schemas and automatically mocked, real life interfaces along with a standard set of common tools (`list_files()`, 'read_file()` etc)
+Constrained decoding and response validation, along with real tool executions within isolated webassembly environments, ensure that generated samples strictly adhere to structured schema, variable constraints, and execution correctness, ensuring datasets have exact syntax and structure for use in model training pipelines. Tool definations can be either directly imported from MCP (Model Context Protocol) server schemas and automatically mocked, real life interfaces along with a standard set of common tools (`list_files()`, `'read_file()` etc)
 
 Once your dataset is generated, it can be automatically uploaded to Hugging Face and directly imported into popular training frameworks like TRL, Unsloth, and Axolotl. 
 
@@ -120,7 +125,15 @@ This generates a topic graph and creates 27 unique nodes, then generates 27 trai
 
 ## Configuration
 
-DeepFabric also uses YAML configuration with three main sections and optional shared LLM defaults:
+DeepFabric also uses YAML configuration with three main sections and optional shared LLM defaults
+
+> [!NOTE]  
+> The following uses mocked tool execution, so will require a runing Spin service, which we provide in a docker image:
+```bash
+docker run -d -p 3000:3000 ghcr.io/always-further/deepfabric/tools-sdk:latest`
+```
+
+Save the following as `config.yaml`:
 
 ```yaml
 # Optional: Shared LLM defaults (inherited by topics and generation)
@@ -143,34 +156,74 @@ topics:
 # GENERATION: Create training samples from topics
 generation:
   system_prompt: |
-    You are an expert Python backend developer and technical educator.
+    You are an expert Python backend developer specializing in REST API design.
     Create practical, production-ready code examples with clear explanations.
     Include error handling, type hints, and follow PEP 8 conventions.
+    Use the following tools to read, write, and list files in the virtual filesystem:
+    - read_file
+    - write_file
+    - list_files
 
   # Additional instructions for sample generation
   instructions: |
-    Focus on real-world scenarios developers encounter daily.
+    Focus on real-world scenarios developers encounter daily when building REST APIs with Python.
     Include both happy path and edge case handling.
-    Provide context on when and why to use specific patterns.
+    Provide context on when and why to use specific patterns or libraries.
+    Ensure code is modular, testable, and maintainable.
 
   conversation:
-    type: chain_of_thought      # basic | chain_of_thought
-    reasoning_style: agent      # freetext | agent (for chain_of_thought)
+    type: cot      # basic | cot
+    reasoning_style: agent      # freetext | agent (for cot)
     agent_mode: single_turn     # single_turn | multi_turn (for agent)
   
   # Tool configuration (required for agent modes)
   tools:
     spin_endpoint: "http://localhost:3000"  # Spin service for tool execution
-    available:                  # Filter to specific tools (empty = all VFS tools)
-      - read_file
-      - write_file
-      - list_files
+    components:                 # Map component name to tool names
+      builtin:                  # Routes to /vfs/execute
+        - read_file
+        - write_file
+        - list_files
     max_per_query: 3            # Maximum tools per query
     max_agent_steps: 5          # Max ReAct reasoning iterations
 
-    max_retries: 3                # Retries for failed generations
-    sample_retries: 2             # Retries for validation failures
-    max_tokens: 2000              # Max tokens per generation
+  # Optional: Seed initial files into the spin before generation, used for tool calling
+    scenario_seed:
+      files:
+        "Dockerfile": |
+          FROM python:3.13
+          WORKDIR /usr/local/app
+
+          # Install the application dependencies
+          COPY requirements.txt ./
+          RUN pip install --no-cache-dir -r requirements.txt
+
+          # Copy in the source code
+          COPY src ./src
+          EXPOSE 8080
+
+          # Setup an app user so the container doesn't run as the root user
+          RUN useradd app
+          USER app
+
+          CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
+        "main.py": |
+          def greet(name):
+              return f"Hello, {name}!"
+
+          if __name__ == "__main__":
+              print(greet("World"))
+        "config.json": |
+          {
+            "version": "1.0.0",
+            "debug": true,
+            "max_retries": 3
+          }
+
+  # Generation control and retry settings
+  max_retries: 3                # Retries for failed generations
+  sample_retries: 2             # Retries for validation failures
+  max_tokens: 2000              # Max tokens per generation
 
   # Optional: Override shared LLM settings
   llm:
@@ -190,13 +243,13 @@ output:
   batch_size: 3                 # Parallel generation batch size
   save_as: "api-dataset.jsonl"
 
-# Optional: Upload to Hugging Face
-huggingface:
-  repository: "your-username/api-dataset-training-name"
-  tags: ["python", "programming"]
+ Optional: Upload to Hugging Face
+ huggingface:
+   repository: "your-username/api-dataset-training-name"
+   tags: ["python", "programming"]
 ```
 
-Run with:
+Run generation by sourcing the `config.yaml`:
 
 ```bash
 deepfabric generate config.yaml
@@ -206,6 +259,14 @@ deepfabric generate config.yaml
 
 DeepFabric returns standard HuggingFace datasets, making it easy to integrate with any training framework.
 
+### Colab Notebooks:
+
+A quick way of seeing DeepFabric in action is via our notebooks in the [notebooks/](./notebooks/) folder or on Google Colab:
+
+**Qwen4b Blender MCP**:
+
+[![Qwen4b Blender MCP](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1EG1V40v5xkJKLf6Ra6W4378vYqlZNVWqb)
+
 ### 1. Generate Dataset
 
 ```bash
@@ -215,7 +276,7 @@ deepfabric generate config.yaml --output-save-as dataset.jsonl
 Or upload to HuggingFace Hub:
 
 ```bash
-deepfabric upload dataset.jsonl --repo your-username/my-dataset
+deepfabric upload-hf dataset.jsonl --repo your-username/my-dataset
 ```
 
 ### 2. Load and Split for Training
@@ -324,7 +385,6 @@ config = EvaluatorConfig(
         model_path="Qwen/Qwen2.5-7B-Instruct",    # Base model
         adapter_path="./output/lora-adapter",     # LoRA adapter path
         backend="transformers",
-        use_unsloth=True,      # Use Unsloth for adapters trained with Unsloth
         load_in_4bit=True,     # 4-bit quantization
         max_seq_length=2048,
     ),
@@ -415,159 +475,6 @@ evaluator = Evaluator(config)
 results = evaluator.evaluate(dataset=eval_dataset)
 ```
 
-## Training Metrics
-
-DeepFabric provides a training callback that automatically logs metrics to the DeepFabric cloud during model training. This enables real-time monitoring and tracking of training runs.
-
-### Basic Usage with HuggingFace Trainer
-
-```python
-from transformers import Trainer, TrainingArguments
-from deepfabric import DeepFabricCallback
-
-# Set up training arguments
-training_args = TrainingArguments(
-    output_dir="./output",
-    num_train_epochs=3,
-    per_device_train_batch_size=4,
-    logging_steps=10,
-)
-
-# Create trainer
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-)
-
-# Add DeepFabric callback for metrics logging
-trainer.add_callback(DeepFabricCallback(trainer))
-
-# Train - metrics are automatically logged
-trainer.train()
-```
-
-### Usage with TRL SFTTrainer
-
-```python
-from trl import SFTTrainer, SFTConfig
-from deepfabric import DeepFabricCallback
-
-trainer = SFTTrainer(
-    model=model,
-    tokenizer=tokenizer,
-    train_dataset=train_dataset,
-    args=SFTConfig(
-        output_dir="./output",
-        num_train_epochs=3,
-        logging_steps=10,
-    ),
-)
-
-# Add callback - works with any Trainer-compatible class
-trainer.add_callback(DeepFabricCallback(trainer))
-trainer.train()
-```
-
-### Configuration Options
-
-```python
-from deepfabric import DeepFabricCallback
-
-callback = DeepFabricCallback(
-    trainer=trainer,                              # Optional: Trainer instance
-    api_key="your-api-key",                       # Or set DEEPFABRIC_API_KEY env var
-    endpoint="https://api.deepfabric.ai",         # Custom endpoint (optional)
-    enabled=True,                                 # Disable to skip logging
-)
-```
-
-### Environment Variables
-
-```bash
-# API key for authentication
-export DEEPFABRIC_API_KEY="your-api-key"
-
-# Custom API endpoint (optional)
-export DEEPFABRIC_API_URL="https://api.deepfabric.ai"
-```
-
-### Logged Metrics
-
-The callback automatically captures and logs:
-
-| Metric Type | Examples |
-|-------------|----------|
-| Training | `loss`, `learning_rate`, `epoch`, `global_step` |
-| Throughput | `train_runtime`, `train_samples_per_second` |
-| Evaluation | `eval_loss`, `eval_accuracy` (when evaluation is run) |
-| TRL-specific | `rewards/chosen`, `rewards/rejected`, `kl_divergence` |
-| Checkpoints | Checkpoint save events with step numbers |
-
-### Callback Events
-
-```python
-# The callback hooks into these Trainer events:
-# - on_train_begin: Logs run start with training configuration
-# - on_log: Logs training metrics (loss, lr, etc.)
-# - on_evaluate: Logs evaluation metrics
-# - on_save: Logs checkpoint events
-# - on_train_end: Logs run completion and flushes pending metrics
-```
-
-### Non-Blocking Design
-
-The callback uses a background thread to send metrics asynchronously, ensuring training is never blocked by network operations:
-
-```python
-from deepfabric.training import MetricsSender
-
-# Direct access to sender for advanced use cases
-sender = MetricsSender(
-    endpoint="https://api.deepfabric.ai",
-    api_key="your-key",
-    batch_size=10,        # Batch metrics before sending
-    flush_interval=5.0,   # Auto-flush every 5 seconds
-    max_queue_size=1000,  # Queue capacity
-)
-
-# Manually send metrics
-sender.send_metrics({"custom_metric": 0.95, "step": 100})
-
-# Flush pending metrics (blocking)
-sender.flush(timeout=30.0)
-
-# Check sender statistics
-print(sender.stats)
-# {'metrics_sent': 150, 'metrics_dropped': 0, 'send_errors': 0, 'queue_size': 0}
-```
-
-### Interactive API Key Prompt
-
-When running in an interactive environment (Jupyter notebook, terminal) without an API key configured, the callback will prompt for authentication:
-
-```python
-from deepfabric import DeepFabricCallback
-
-# If DEEPFABRIC_API_KEY is not set, prompts for login
-callback = DeepFabricCallback(trainer)
-# > DeepFabric API key not found. Log in to enable cloud metrics.
-# > Visit: https://app.deepfabric.ai/signup
-```
-
-### Disabling Metrics Logging
-
-```python
-# Disable via constructor
-callback = DeepFabricCallback(trainer, enabled=False)
-
-# Or set API key to None
-callback = DeepFabricCallback(trainer, api_key=None)
-
-# Or don't set DEEPFABRIC_API_KEY environment variable
-```
-
 ## Providers
 
 | Provider | Local/Cloud | Best For |
@@ -625,7 +532,7 @@ Enable tool tracing in your YAML config:
 ```yaml
 generation:
   conversation:
-    type: chain_of_thought
+    type: cot
     reasoning_style: agent
     agent_mode: single_turn