puffinflow 2.dev0__tar.gz → 2.1.0__tar.gz

puffinflow-2.1.0/.dockerignore

@@ -0,0 +1,17 @@
+.git
+.github
+.mypy_cache
+.pytest_cache
+.ruff_cache
+__pycache__
+*.pyc
+*.pyo
+*.egg-info
+build/
+dist/
+htmlcov/
+docs/
+.venv
+venv/
+.env
+*.log

{puffinflow-2.dev0 → puffinflow-2.1.0}/.github/workflows/ci-cd.yml

@@ -126,6 +126,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -171,6 +174,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -203,6 +209,8 @@ jobs:
     name: Documentation
     runs-on: ubuntu-latest
     if: github.event_name != 'schedule' && (contains(github.event.head_commit.message, '[docs]') || github.event_name == 'pull_request' || github.ref == 'refs/heads/main')
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v4
         with:
@@ -214,6 +222,9 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -301,35 +312,112 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: docs/_build/html
 
-  # Build package
-  build:
-    name: Build Package
-    runs-on: ubuntu-latest
+  # Build wheels for each platform using maturin
+  build-wheels:
+    name: Build wheels (${{ matrix.os }}, ${{ matrix.target }})
+    runs-on: ${{ matrix.os }}
     needs: [security-scan, lint-and-format, test]
-    if: github.event_name != 'schedule' && !failure()
+    if: github.event_name != 'schedule' && !cancelled() && needs.lint-and-format.result == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Linux x86_64
+          - os: ubuntu-latest
+            target: x86_64
+            manylinux: auto
+          # Linux aarch64 (cross-compiled via QEMU)
+          - os: ubuntu-latest
+            target: aarch64
+            manylinux: auto
+          # macOS x86_64 (cross-compiled from ARM runner)
+          - os: macos-14
+            target: x86_64
+          # macOS Apple Silicon
+          - os: macos-14
+            target: aarch64
+          # Windows x86_64
+          - os: windows-latest
+            target: x64
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Needed for setuptools_scm
+          fetch-depth: 0
+
+      - name: Set up QEMU (for Linux cross-compilation)
+        if: matrix.target == 'aarch64' && runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
 
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Install build dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install build twine setuptools_scm
+      - name: Build wheels with maturin
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --interpreter 3.9 3.10 3.11 3.12 3.13
+          manylinux: ${{ matrix.manylinux || 'auto' }}
+
+      - name: Upload wheel artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}-${{ matrix.target }}
+          path: dist/*.whl
+          retention-days: 90
+
+  # Build source distribution
+  build-sdist:
+    name: Build sdist
+    runs-on: ubuntu-latest
+    needs: [security-scan, lint-and-format, test]
+    if: github.event_name != 'schedule' && !cancelled() && needs.lint-and-format.result == 'success'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
-      - name: Build package
-        run: python -m build
+      - name: Build sdist with maturin
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
 
-      - name: Check package
-        run: twine check dist/*
+      - name: Upload sdist artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+          retention-days: 90
 
-      - name: Upload build artifacts
+  # Merge all build artifacts into a single dist directory
+  merge-artifacts:
+    name: Merge Build Artifacts
+    runs-on: ubuntu-latest
+    needs: [build-wheels, build-sdist]
+    steps:
+      - name: Download all wheel artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+
+      - name: Download sdist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - name: List artifacts
+        run: ls -lh dist/
+
+      - name: Upload merged dist
         uses: actions/upload-artifact@v4
         with:
           name: dist
@@ -340,8 +428,10 @@ jobs:
   release:
     name: Create Release
     runs-on: ubuntu-latest
-    needs: [build]
+    needs: [merge-artifacts]
     if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && github.event.inputs.version)
+    permissions:
+      contents: write
     outputs:
       version: ${{ steps.get-version.outputs.version }}
       is-prerelease: ${{ steps.check-prerelease.outputs.is-prerelease }}
@@ -441,7 +531,7 @@ jobs:
   publish-test-pypi:
     name: Publish to Test PyPI
     runs-on: ubuntu-latest
-    needs: [build, release]
+    needs: [merge-artifacts, release]
     if: needs.release.outputs.is-prerelease == 'true'
     environment:
       name: test-pypi
@@ -464,7 +554,7 @@ jobs:
   publish-pypi:
     name: Publish to PyPI
     runs-on: ubuntu-latest
-    needs: [build, release]
+    needs: [merge-artifacts, release]
     if: needs.release.outputs.is-prerelease == 'false' && (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release')
     environment:
       name: pypi
@@ -487,7 +577,7 @@ jobs:
   notification:
     name: Workflow Summary
     runs-on: ubuntu-latest
-    needs: [security-scan, lint-and-format, test, docs, build, release, publish-test-pypi, publish-pypi]
+    needs: [security-scan, lint-and-format, test, docs, build-wheels, build-sdist, merge-artifacts, release, publish-test-pypi, publish-pypi]
     if: always()
     steps:
       - name: Workflow Summary
@@ -530,9 +620,9 @@ jobs:
           fi
 
           # Build Results
-          if [ "${{ needs.build.result }}" == "success" ]; then
-            echo "✅ **Package Build**: Successful" >> $GITHUB_STEP_SUMMARY
-          elif [ "${{ needs.build.result }}" == "skipped" ]; then
+          if [ "${{ needs.merge-artifacts.result }}" == "success" ]; then
+            echo "✅ **Package Build**: Successful (wheels for all platforms)" >> $GITHUB_STEP_SUMMARY
+          elif [ "${{ needs.merge-artifacts.result }}" == "skipped" ]; then
             echo "⏭️ **Package Build**: Skipped" >> $GITHUB_STEP_SUMMARY
           else
             echo "❌ **Package Build**: Failed" >> $GITHUB_STEP_SUMMARY

{puffinflow-2.dev0 → puffinflow-2.1.0}/.github/workflows/compliance.yml

@@ -32,6 +32,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install quality analysis tools
         run: |
           python -m pip install --upgrade pip
@@ -113,6 +116,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install documentation tools
         run: |
           python -m pip install --upgrade pip
@@ -166,6 +172,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

{puffinflow-2.dev0 → puffinflow-2.1.0}/.github/workflows/dependency-audit.yml

@@ -45,6 +45,9 @@ jobs:
           python -m pip install --upgrade pip
           pip install safety pip-audit semgrep cyclonedx-bom
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install project dependencies
         run: |
           pip install -e ".[dev,test]"
@@ -160,6 +163,9 @@ jobs:
           python -m pip install --upgrade pip
           pip install pip-licenses licensecheck
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install project dependencies
         run: pip install -e ".[dev]"
 

{puffinflow-2.dev0 → puffinflow-2.1.0}/.github/workflows/performance.yml

@@ -41,6 +41,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -86,6 +89,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -155,6 +161,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

{puffinflow-2.dev0 → puffinflow-2.1.0}/.gitignore

@@ -165,7 +165,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 # Abstra
 # Abstra is an AI-powered process automation framework.
@@ -207,3 +207,16 @@ trufflehog-results.json
 .bandit
 .safety
 .semgrep
+.vercel
+
+# Marketing (internal only)
+marketing/
+
+# Studio build artifacts and database
+studio.db
+studio/web/.next/
+studio/web/node_modules/
+
+# Extracted repositories (now separate repos)
+doc-site/
+puffinflow-visual-editor/

{puffinflow-2.dev0 → puffinflow-2.1.0}/.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.8
+    rev: v0.15.0
     hooks:
       - id: ruff
         args: [--fix]

puffinflow-2.1.0/BENCHMARKS.md

@@ -0,0 +1,193 @@
+# PuffinFlow Benchmarks
+
+Orchestration overhead measured across workflow frameworks using identical workloads. Only framework overhead differs.
+
+---
+
+## Methodology
+
+**Workload:** `sum(i*i for i in range(5000))` — a pure CPU computation used identically across all frameworks. This isolates framework orchestration overhead from application logic.
+
+**Setup:**
+- Each test runs 20 times (median reported)
+- 3 warmup iterations before measurement
+- Throughput measured over 3-second window
+- Memory measured with `tracemalloc` across 500 sequential workflow executions
+- Import time measured in isolated subprocess (Python startup time subtracted)
+- All frameworks use async execution where supported
+
+**Environment:**
+- Python 3.12+
+- PuffinFlow (Rust core via PyO3)
+- LangGraph (latest)
+- LlamaIndex Workflows (latest)
+
+**Reproduce:**
+```bash
+pip install puffinflow langgraph llama-index-core
+python benchmarks/benchmark.py
+```
+
+For JSON output:
+```bash
+python benchmarks/benchmark.py --json
+```
+
+---
+
+## Results
+
+### Lightweight Frameworks (async/graph-based)
+
+| Test | PuffinFlow | LangGraph | LlamaIndex Workflows |
+|------|-----------|-----------|---------------------|
+| Sequential 3-step | **0.7 ms** | 1.5 ms | 2.0 ms |
+| Sequential 5-step | **1.3 ms** | 2.5 ms | 3.0 ms |
+| Per-step overhead | **0.3 ms** | 0.4 ms | 0.5 ms |
+| Fan-out (3 branches + 1 aggregate) | **1.2 ms** | 3.4 ms | 1.6 ms |
+| Throughput (workflows/sec) | **1,150** | 680 | 430 |
+| Peak memory (500 workflows) | 2.50 MB | 4.93 MB | **0.67 MB** |
+| Import time (cold start) | **252 ms** | 1,000 ms | 1,600 ms |
+
+Per-step overhead = `(5-step − 3-step) / 2`. Import time measures `from pkg import ...` with real symbols (e.g. `from puffinflow import Agent, state`) in a cold subprocess, Python startup subtracted.
+
+---
+
+## Key Takeaways
+
+### Latency: 2x faster
+
+PuffinFlow's Rust core executes the state machine hot path — transition resolution, state dispatching, context management — in compiled Rust via PyO3. This delivers:
+
+- **2x lower latency** on sequential workflows (1.3ms vs 2.5ms for 5 steps)
+- **0.3ms per-step overhead** vs 0.4ms for LangGraph
+- Near-identical performance to raw asyncio for simple chains
+
+### Throughput: 1.7x higher
+
+Running workflows back-to-back over a 3-second window:
+
+- **PuffinFlow: 1,150 workflows/sec**
+- LangGraph: 680 workflows/sec
+- LlamaIndex: 430 workflows/sec
+
+### Import Time: 4x faster
+
+Cold-start import time matters for scripts, tests, notebooks, and CI/CD. The benchmark measures real-world imports with actual symbols (e.g. `from puffinflow import Agent, state`), not bare `import pkg`:
+
+- **PuffinFlow: 252ms** (lazy imports — only loads what you use)
+- LangGraph: 1,000ms (nearly 1 second)
+- LlamaIndex: 1,600ms (nearly 2 seconds)
+
+PuffinFlow uses a lazy import system (`__getattr__`-based) at every level of the package. Submodules and heavy dependencies like structlog are only imported when first accessed, keeping the import path lean.
+
+### Memory: Competitive
+
+PuffinFlow uses 2.50 MB for 500 sequential workflow executions. LlamaIndex is lower at 0.67 MB (lightweight event-driven model). LangGraph uses 4.93 MB. PuffinFlow's memory usage is well-controlled thanks to the Rust core's explicit memory management.
+
+---
+
+## Fan-Out Performance
+
+The fan-out benchmark measures a common pattern: one start node dispatching to 3 parallel branches, then aggregating results.
+
+| Framework | Fan-out latency |
+|-----------|----------------|
+| **PuffinFlow** | **1.2 ms** |
+| LlamaIndex | 1.6 ms |
+| LangGraph | 3.4 ms |
+
+PuffinFlow's `Send` API and reducer system handle parallel dispatch efficiently. The Rust core resolves transitions and merges without Python-level coordination overhead.
+
+---
+
+## What These Numbers Mean in Practice
+
+### For scripts and tests
+
+If you import your agent framework 50 times/day (script runs, test runs, notebook restarts):
+
+| Framework | Daily import overhead |
+|-----------|---------------------|
+| PuffinFlow | 12.6 seconds |
+| LangGraph | 50.0 seconds |
+| LlamaIndex | 80.0 seconds |
+
+### For CI/CD
+
+In CI pipelines, cold-start import time adds directly to build time. PuffinFlow's 252ms import is 4x faster than LangGraph and 6x faster than LlamaIndex.
+
+### For production throughput
+
+At 1,150 workflows/sec, PuffinFlow can handle high-frequency agent invocations without becoming a bottleneck. The framework overhead is negligible compared to actual LLM inference time (typically 100ms-10s per call).
+
+---
+
+## Benchmark Architecture
+
+### PuffinFlow benchmark agents
+
+```python
+from puffinflow import Agent, state
+
+class SeqAgent(Agent):
+    def __init__(self):
+        super().__init__("bench-seq")
+        for i in range(n_steps):
+            name = f"step{i}"
+            next_name = f"step{i + 1}" if i < n_steps - 1 else None
+
+            async def make_fn(ctx, _next=next_name):
+                workload()
+                return _next
+
+            make_fn.__name__ = name
+            decorated = state()(make_fn)
+            self.add_state(name, decorated)
+```
+
+### LangGraph benchmark graphs
+
+```python
+from langgraph.graph import StateGraph
+from typing_extensions import TypedDict
+
+class St(TypedDict):
+    value: int
+
+builder = StateGraph(St)
+for name in names:
+    builder.add_node(name, lambda s: {"value": workload()})
+builder.set_entry_point(names[0])
+for i in range(len(names) - 1):
+    builder.add_edge(names[i], names[i + 1])
+builder.set_finish_point(names[-1])
+graph = builder.compile()
+```
+
+Both frameworks execute the same `workload()` function in each step. Only orchestration overhead differs.
+
+---
+
+## Running Your Own Benchmarks
+
+```bash
+# Install all frameworks
+pip install puffinflow langgraph llama-index-core
+
+# Run comparison
+python benchmarks/benchmark.py
+
+# JSON output (for programmatic analysis)
+python benchmarks/benchmark.py --json
+```
+
+The benchmark source is at [`benchmarks/benchmark.py`](./benchmarks/benchmark.py). It's designed to be transparent and reproducible. Pull requests to improve methodology are welcome.
+
+---
+
+## Notes
+
+- **LlamaIndex memory advantage**: LlamaIndex Workflows use a lightweight event-driven model that allocates less per-workflow overhead. This is a genuine strength of their architecture.
+- **Rust core fallback**: If the Rust extension (`_rust_core`) is not available, PuffinFlow falls back to a pure-Python implementation. Performance numbers above are with the Rust core enabled.
+- **Real-world caveat**: In production AI agent workflows, LLM inference time (100ms-10s) dominates total latency. Framework overhead matters most for: (1) high-frequency orchestration, (2) complex multi-step pipelines, (3) CI/CD and testing, (4) import-heavy development workflows.

puffinflow-2.1.0/CLAUDE.md

@@ -0,0 +1,73 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+PuffinFlow is an async Python workflow orchestration framework for building LLM workflows and multi-agent systems. It uses a src-layout (`src/puffinflow/`) with Python 3.9+ and is built on asyncio, Pydantic v2, and structlog.
+
+## Commands
+
+```bash
+# Install
+pip install -e .                              # Basic
+pip install -e ".[dev,performance,observability,all]"  # Full dev setup
+
+# Test
+pytest tests/ -v                              # All tests
+pytest tests/unit/ -v                         # Unit tests only
+pytest tests/integration/ -v                  # Integration tests only
+pytest tests/e2e/ -v                          # End-to-end tests only
+pytest tests/unit/core/test_agent.py -v       # Single file
+pytest tests/unit/core/test_agent.py::TestClass::test_method  # Single test
+pytest tests/ --cov=src/puffinflow --cov-report=html  # With coverage (85% min)
+
+# Lint & Format
+ruff check src/ tests/ benchmarks/ examples/  # Lint
+mypy src/puffinflow/                          # Type check (strict mode)
+black src/ tests/ benchmarks/ examples/       # Format
+ruff check --fix src/ tests/ benchmarks/ examples/  # Auto-fix lint issues
+
+# Benchmarks
+python benchmarks/benchmark.py                # Multi-framework comparison
+python benchmarks/benchmark.py --json         # JSON output
+
+# Makefile shortcuts
+make dev-test    # format → lint → test (full cycle)
+make test        # pytest tests/ -v
+make lint        # ruff + mypy
+make format      # black + ruff --fix
+```
+
+## Architecture
+
+### Core Module Layout (`src/puffinflow/core/`)
+
+- **agent/** — Central abstraction. `Agent` is a state machine where each state is an async function decorated with `@state()`. States return the name of the next state to transition to. `Context` carries variables, outputs, metadata, shared state, and a TTL cache between states.
+- **coordination/** — Multi-agent patterns: `AgentTeam` (messaging + coordination), `AgentPool` (dynamic scaling + work queues), `AgentOrchestrator` (staged execution with strategies like PIPELINE, FAN_OUT, CONDITIONAL). Also includes primitives (locks, barriers, events), rate limiting, and deadlock detection.
+- **resources/** — `ResourcePool` manages CPU/memory/GPU/IO/network allocation across agents. States declare requirements via decorators (`@cpu_intensive`, `@memory_intensive`, etc.). Supports allocation strategies: FIRST_FIT, BEST_FIT, PRIORITY, FAIR_SHARE, ROUND_ROBIN, WEIGHTED.
+- **reliability/** — `CircuitBreaker` (three-state failure protection), `Bulkhead` (concurrency isolation), `ResourceLeakDetector`. States have built-in `RetryPolicy` with exponential backoff and dead-letter support.
+- **observability/** — `ObservableAgent` wraps Agent with automatic Prometheus metrics and OpenTelemetry tracing. Includes alerting (webhooks), buffered event processing, and distributed context propagation.
+- **config.py** — Pydantic Settings with environment variable support. `Settings` for resource limits/concurrency, `Features` for feature flags.
+
+### Key Patterns
+
+- **State decorators** define resource requirements inline: `@state(cpu=2.0, memory=512.0, priority="high", timeout=30.0)`
+- **Execution modes**: `SEQUENTIAL` (one state at a time) vs `PARALLEL` (all independent states run concurrently)
+- **Context methods**: `context.set_variable()`/`get_variable()` for state-local data, `context.set_state()`/`get_state()` for shared data, `context.set_typed()` for Pydantic-validated data
+- **State functions** return the next state name as a string, or `None` to end
+
+### Test Organization
+
+- `tests/unit/` — Fast tests (<1s each), organized mirroring `src/puffinflow/core/` subdirectories
+- `tests/integration/` — Component interaction tests (1-10s)
+- `tests/e2e/` — Full workflow tests (10-60s)
+- Async mode is `auto` — async test functions are detected automatically without needing `@pytest.mark.asyncio`
+- Markers: `unit`, `integration`, `e2e`, `slow`, `benchmark`, `observability`, `tracing`, `metrics`, `alerting`
+
+## Code Style
+
+- Black + Ruff formatting, 88-char line length
+- MyPy strict mode (disallow_untyped_defs, no_implicit_optional, strict_equality)
+- Target Python 3.9 — use `from __future__ import annotations` or `typing_extensions` for newer typing features
+- Version is auto-generated by setuptools_scm into `src/puffinflow/version.py` — do not edit manually