DeepFabric 4.4.0__tar.gz

deepfabric-4.4.0/.github/config.yml

@@ -0,0 +1,50 @@
+# Configuration for welcome - https://github.com/behaviorbot/welcome
+
+# Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome
+
+# Comment to be posted to on first time issues
+newIssueWelcomeComment: >
+  🎉 **Welcome to the DeepFabric project!** 🎉  
+  Thanks for opening your first issue here — we’re *so excited* to have you contributing! 😄  
+  Please make sure to follow the issue template so we can help you faster 🧵✨  
+
+  💬 Join our amazing community on Discord for live discussions, sneak peeks, and good vibes:  
+  👉 [**DeepFabric Discord**](https://discord.gg/pPcjYzGvbS)  
+
+  🧠 Let’s build something *deep* together! 🚀💡  
+
+  ![welcome gif](https://media.giphy.com/media/OkJat1YNdoD3W/giphy.gif)
+
+---
+
+# Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome
+
+# Comment to be posted to on PRs from first time contributors in your repository
+newPRWelcomeComment: >
+  👏 **Thank you for your first pull request!** 👏  
+  Welcome to **DeepFabric** — you’re officially part of the weave now! 🧶✨  
+
+  Please take a moment to check out our **contributing guidelines** before your PR is reviewed 💪  
+  Every thread you add helps strengthen the fabric of this project 🕸️💙  
+
+  💬 Hop into our Discord to say hi or share your progress:  
+  👉 [**DeepFabric Discord**](https://discord.gg/pPcjYzGvbS)  
+
+  ![thank you gif](https://media.giphy.com/media/l0MYt5jPR6QX5pnqM/giphy.gif)
+
+---
+
+# Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge
+
+# Comment to be posted to on pull requests merged by a first time user
+firstPRMergeComment: >
+  🥳 **Congratulations on merging your first PR!** 🥳  
+  The **DeepFabric** team is *so proud* of you! 🌟  
+
+  You’ve officially added your thread to the ever-growing fabric of this project 🧵💫  
+  Keep up the great work and let’s keep building amazing things together! 🚀  
+
+  💬 Come celebrate with us on Discord — you’ve earned it!  
+  👉 [**DeepFabric Discord**](https://discord.gg/pPcjYzGvbS)  
+
+  ![celebration gif](https://media.giphy.com/media/111ebonMs90YLu/giphy.gif)

deepfabric-4.4.0/.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  # - package-ecosystem: "docker"
+  #   directory: "/"
+  #   schedule:
+  #     interval: "daily"

deepfabric-4.4.0/.github/workflows/docs.yml

@@ -0,0 +1,60 @@
+name: Build and Deploy Documentation
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7
+
+      - name: Set up Python
+        run: uv python install
+
+      - name: Create virtual environment
+        run: uv venv
+
+      - name: Install dependencies
+        run: uv pip install mkdocs-material "mkdocstrings[python]>=0.30.0"
+
+      - name: Build documentation
+        run: |
+          source .venv/bin/activate
+          mkdocs build --strict
+
+      - name: Setup Pages
+        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4
+        with:
+          path: ./site
+
+  deploy:
+    if: github.ref == 'refs/heads/main'
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4

deepfabric-4.4.0/.github/workflows/integration.yml

@@ -0,0 +1,48 @@
+name: Integration Tests
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**/README.md'
+      - '**/docs/**'
+      - '**/.github/workflows/publish.yml'
+  pull_request:
+    paths-ignore:
+      - '**/README.md'
+      - '**/docs/**'
+      - '**/.github/workflows/publish.yml'
+
+jobs:
+  integration-tests:
+    name: Run integration tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ '3.10', '3.11', '3.12' ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v5
+
+      - name: Set up Python
+        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7
+        with:
+            enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Run integration tests
+        run: make test-integration
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

deepfabric-4.4.0/.github/workflows/publish.yml

@@ -0,0 +1,35 @@
+name: CI/CD
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/p/deepfabric
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v5
+
+      - name: Set up Python
+        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6
+        with:
+          python-version: '3.x'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7
+
+      - name: Build distributions
+        run: uv build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
+        with:
+          attestations: true

deepfabric-4.4.0/.github/workflows/test.yml

@@ -0,0 +1,45 @@
+name: Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11']
+
+    steps:
+    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v5
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7
+      with:
+        enable-cache: true
+
+    - name: Install dependencies
+      run: uv sync --all-extras
+
+    - name: Run code formatting
+      run: make format
+
+    - name: Run linting
+      run: make lint
+
+    - name: Run tests
+      run: make test-unit
+
+    - name: Run security checks
+      run: make security
+
+    - name: Run build
+      run: make build

deepfabric-4.4.0/.github/workflows/tools-sdk-docker.yml

@@ -0,0 +1,147 @@
+name: Tools-SDK Docker Build and Publish
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'tools-sdk/**'
+      - '.github/workflows/tools-sdk-docker.yml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'tools-sdk/**'
+      - '.github/workflows/tools-sdk-docker.yml'
+  release:
+    types: [published]
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}/tools-sdk
+
+jobs:
+  build:
+    name: Build ${{ matrix.platform }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+
+      - name: Log in to Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          context: ./tools-sdk
+          file: ./tools-sdk/Dockerfile
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
+
+      - name: Export digest
+        if: github.event_name != 'pull_request'
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: github.event_name != 'pull_request'
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: digests-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Merge and Push Manifest
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    needs: build
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix=
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
+
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0
+
+      - name: Sign the container image
+        env:
+          DIGEST: ${{ steps.meta.outputs.version }}
+        run: |
+          cosign sign --yes "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${DIGEST}"

deepfabric-4.4.0/.gitignore

@@ -0,0 +1,135 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+*.cover
+*.py,cover
+.hypothesis/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Pytype static type analyzer
+.pytype/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# Celery
+celerybeat-schedule
+*.sage.py
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook metadata
+.ipynb_checkpoints
+
+# VS Code settings
+.vscode/
+.vscode/settings.json
+
+# macOS specific
+.DS_Store
+
+# Temporary and backup files
+*.bak
+*.swp
+*.swo
+
+# IDE specific settings
+.idea/
+*.iml
+.serena/
+
+# Logs
+*.log
+*.out
+*.err
+
+# Project files
+venv/
+*.jsonl
+formats/*
+# coverage files
+.coverage.*
+tools-sdk/.spin/*

deepfabric-4.4.0/CLAUDE.md

@@ -0,0 +1,100 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+DeepFabric is a Python library for generating synthetic datasets using LLM providers. It consists of three main components that work together in a pipeline:
+
+1. **Tree**: Generates hierarchical topic structures from a root prompt
+2. **DataSetGenerator**: Creates training examples based on topics 
+3. **Dataset**: Manages and exports the final dataset
+
+## Development Commands
+
+### Package Management
+- `uv sync --all-extras` - Install all dependencies including dev tools
+- `make install` - Same as above via Makefile
+
+### Code Quality
+- `make format` - Format code with black and ruff auto-fix
+- `make lint` - Run ruff linting checks
+- `make security` - Run bandit security analysis on deepfabric/ directory
+
+### Testing
+- `make test` - Run pytest test suite
+- `uv run pytest` - Direct pytest execution
+- Test files are in `tests/` directory
+
+### Build and Release
+- `make build` - Clean, test, and build package
+- `make clean` - Remove build artifacts and cache files
+- `make all` - Complete workflow: install, format, lint, test, security, build
+
+## Core Architecture
+
+### Configuration System
+- YAML-based configuration with direct system prompt specification
+- CLI supports extensive parameter overrides
+
+### Data Flow
+1. Tree generates topic hierarchy from root prompt
+2. DataSetGenerator uses topics to create question/answer pairs
+3. Dataset validates, stores, and exports training examples
+4. Optional HuggingFace Hub upload with auto-generated dataset cards
+
+### Key Classes and Their Relationships
+- `DeepFabricConfig`: Loads YAML and provides argument objects
+- `TreeArguments`/`DataSetGeneratorArguments`: Pydantic-style dataclasses for parameters
+- `Dataset`: Handles JSONL import/export and validation
+- `HFUploader`: Manages Hugging Face Hub integration
+
+## Important Implementation Details
+
+### JSON Validation
+The engine includes robust JSON parsing with regex extraction and retry logic for handling LLM response inconsistencies.
+
+### Error Handling
+- Max retries configurable for failed LLM calls
+- Failed samples tracked separately from successful ones
+- Comprehensive error reporting in final summary
+
+### System Message Control
+The `sys_msg` parameter controls whether system messages are included in the final dataset format - this affects training data structure.
+
+## Configuration Patterns
+
+### YAML Structure
+```yaml
+dataset_system_prompt: "..."
+topic_tree:
+  args: {...}
+  save_as: "file.jsonl"
+data_engine:
+  args: {...}
+dataset:
+  creation: {...}
+  save_as: "file.jsonl"
+huggingface: {...}  # optional
+```
+
+### Provider Configuration
+Specify provider and model separately in config, or use combined format in code:
+- Config: `provider: "ollama"`, `model: "mistral:latest"`
+- Code: `model_name: "ollama/mistral:latest"`
+
+## CLI Usage Patterns
+
+The CLI supports both YAML configuration and parameter overrides:
+```bash
+deepfabric start config.yaml --model gpt-4 --temperature 0.8 --hf-repo user/dataset
+```
+
+## Code Style Notes
+- Uses uv for dependency management
+- Ruff for linting with extensive rule set
+- Black for formatting
+- Bandit for security analysis
+- Python 3.11+ required
+- Google-style docstrings preferred
+- do not place imports anywhere but the top of the file