ai-pipeline-core 0.3.0__tar.gz → 0.3.3__tar.gz

{ai_pipeline_core-0.3.0 → ai_pipeline_core-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-pipeline-core
-Version: 0.3.0
+Version: 0.3.3
 Summary: Core utilities for AI-powered processing pipelines using prefect
 Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
 Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core
@@ -22,6 +22,7 @@ Requires-Dist: httpx>=0.28.1
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: lmnr>=0.7.18
 Requires-Dist: openai>=1.109.1
+Requires-Dist: pillow>=10.0.0
 Requires-Dist: prefect-gcp[cloud-storage]>=0.6.10
 Requires-Dist: prefect>=3.4.21
 Requires-Dist: pydantic-settings>=2.10.1
@@ -124,7 +125,7 @@ async def analyze_flow(
     for doc in documents:
         # Use AIMessages for LLM interaction
         response = await llm.generate(
-            model="gpt-5",
+            model="gpt-5.1",
             messages=AIMessages([doc])
         )
 
@@ -151,7 +152,7 @@ class Analysis(BaseModel):
 
 # Generate structured output
 response = await llm.generate_structured(
-    model="gpt-5",
+    model="gpt-5.1",
     response_format=Analysis,
     messages="Analyze this product review: ..."
 )
@@ -246,7 +247,7 @@ from ai_pipeline_core import llm, AIMessages, ModelOptions
 
 # Simple generation
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     messages="Explain quantum computing"
 )
 print(response.content)
@@ -256,21 +257,21 @@ static_context = AIMessages([large_document])
 
 # First call: caches context
 r1 = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,  # Cached for 120 seconds by default
     messages="Summarize"     # Dynamic query
 )
 
 # Second call: reuses cache
 r2 = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,  # Reused from cache!
     messages="Key points?"   # Different query
 )
 
 # Custom cache TTL
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,
     messages="Analyze",
     options=ModelOptions(cache_ttl="300s")  # Cache for 5 minutes
@@ -278,7 +279,7 @@ response = await llm.generate(
 
 # Disable caching for dynamic contexts
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=dynamic_context,
     messages="Process",
     options=ModelOptions(cache_ttl=None)  # No caching
@@ -335,6 +336,68 @@ async def main_flow(
     return DocumentList(results)
 ```
 
+### Local Trace Debugging
+
+Save all trace spans to the local filesystem for LLM-assisted debugging:
+
+```bash
+export TRACE_DEBUG_PATH=/path/to/debug/output
+```
+
+This creates a hierarchical directory structure that mirrors the execution flow with automatic deduplication:
+
+```
+20260128_152932_abc12345_my_flow/
+├── _trace.yaml           # Trace metadata
+├── _index.yaml           # Span ID → path mapping
+├── _summary.md           # Unified summary for human inspection and LLM debugging
+├── artifacts/            # Deduplicated content storage
+│   └── sha256/
+│       └── ab/cd/        # Sharded by hash prefix
+│           └── abcdef...1234.txt  # Large content (>10KB)
+└── 0001_my_flow/         # Root span (numbered for execution order)
+    ├── _span.yaml        # Span metadata (timing, status, I/O refs)
+    ├── input.yaml        # Structured inputs (inline or refs)
+    ├── output.yaml       # Structured outputs (inline or refs)
+    ├── 0002_task_1/      # Child spans nested inside parent
+    │   ├── _span.yaml
+    │   ├── input.yaml
+    │   ├── output.yaml
+    │   └── 0003_llm_call/
+    │       ├── _span.yaml
+    │       ├── input.yaml   # LLM messages with inline/external content
+    │       └── output.yaml
+    └── 0004_task_2/
+        └── ...
+```
+
+**Key Features:**
+- **Automatic Deduplication**: Identical content (e.g., system prompts) stored once in `artifacts/`
+- **Smart Externalization**: Large content (>10KB) externalized with 2KB inline previews
+- **AI-Friendly**: Files capped at 50KB for easy LLM processing
+- **Lossless**: Full content reconstruction via `content_ref` pointers
+
+Example `input.yaml` with externalization:
+```yaml
+format_version: 3
+type: llm_messages
+messages:
+  - role: system
+    parts:
+      - type: text
+        size_bytes: 28500
+        content_ref:  # Large content → artifact
+          hash: sha256:a1b2c3d4...
+          path: artifacts/sha256/a1/b2/a1b2c3d4...txt
+        excerpt: "You are a helpful assistant...\n[TRUNCATED]"
+  - role: user
+    parts:
+      - type: text
+        content: "Hello!"  # Small content stays inline
+```
+
+Run `tree` on the output directory to visualize the entire execution hierarchy. Feed `_summary.md` to an LLM for debugging assistance - it combines high-level overview with detailed navigation for comprehensive trace analysis.
+
 ## Configuration
 
 ### Environment Variables
@@ -348,6 +411,9 @@ OPENAI_API_KEY=your-api-key
 LMNR_PROJECT_API_KEY=your-lmnr-key
 LMNR_DEBUG=true  # Enable debug traces
 
+# Optional: Local Trace Debugging
+TRACE_DEBUG_PATH=/path/to/trace/output  # Save traces locally for LLM-assisted debugging
+
 # Optional: Orchestration
 PREFECT_API_URL=http://localhost:4200/api
 PREFECT_API_KEY=your-prefect-key

{ai_pipeline_core-0.3.0 → ai_pipeline_core-0.3.3}/README.md

@@ -79,7 +79,7 @@ async def analyze_flow(
     for doc in documents:
         # Use AIMessages for LLM interaction
         response = await llm.generate(
-            model="gpt-5",
+            model="gpt-5.1",
             messages=AIMessages([doc])
         )
 
@@ -106,7 +106,7 @@ class Analysis(BaseModel):
 
 # Generate structured output
 response = await llm.generate_structured(
-    model="gpt-5",
+    model="gpt-5.1",
     response_format=Analysis,
     messages="Analyze this product review: ..."
 )
@@ -201,7 +201,7 @@ from ai_pipeline_core import llm, AIMessages, ModelOptions
 
 # Simple generation
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     messages="Explain quantum computing"
 )
 print(response.content)
@@ -211,21 +211,21 @@ static_context = AIMessages([large_document])
 
 # First call: caches context
 r1 = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,  # Cached for 120 seconds by default
     messages="Summarize"     # Dynamic query
 )
 
 # Second call: reuses cache
 r2 = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,  # Reused from cache!
     messages="Key points?"   # Different query
 )
 
 # Custom cache TTL
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,
     messages="Analyze",
     options=ModelOptions(cache_ttl="300s")  # Cache for 5 minutes
@@ -233,7 +233,7 @@ response = await llm.generate(
 
 # Disable caching for dynamic contexts
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=dynamic_context,
     messages="Process",
     options=ModelOptions(cache_ttl=None)  # No caching
@@ -290,6 +290,68 @@ async def main_flow(
     return DocumentList(results)
 ```
 
+### Local Trace Debugging
+
+Save all trace spans to the local filesystem for LLM-assisted debugging:
+
+```bash
+export TRACE_DEBUG_PATH=/path/to/debug/output
+```
+
+This creates a hierarchical directory structure that mirrors the execution flow with automatic deduplication:
+
+```
+20260128_152932_abc12345_my_flow/
+├── _trace.yaml           # Trace metadata
+├── _index.yaml           # Span ID → path mapping
+├── _summary.md           # Unified summary for human inspection and LLM debugging
+├── artifacts/            # Deduplicated content storage
+│   └── sha256/
+│       └── ab/cd/        # Sharded by hash prefix
+│           └── abcdef...1234.txt  # Large content (>10KB)
+└── 0001_my_flow/         # Root span (numbered for execution order)
+    ├── _span.yaml        # Span metadata (timing, status, I/O refs)
+    ├── input.yaml        # Structured inputs (inline or refs)
+    ├── output.yaml       # Structured outputs (inline or refs)
+    ├── 0002_task_1/      # Child spans nested inside parent
+    │   ├── _span.yaml
+    │   ├── input.yaml
+    │   ├── output.yaml
+    │   └── 0003_llm_call/
+    │       ├── _span.yaml
+    │       ├── input.yaml   # LLM messages with inline/external content
+    │       └── output.yaml
+    └── 0004_task_2/
+        └── ...
+```
+
+**Key Features:**
+- **Automatic Deduplication**: Identical content (e.g., system prompts) stored once in `artifacts/`
+- **Smart Externalization**: Large content (>10KB) externalized with 2KB inline previews
+- **AI-Friendly**: Files capped at 50KB for easy LLM processing
+- **Lossless**: Full content reconstruction via `content_ref` pointers
+
+Example `input.yaml` with externalization:
+```yaml
+format_version: 3
+type: llm_messages
+messages:
+  - role: system
+    parts:
+      - type: text
+        size_bytes: 28500
+        content_ref:  # Large content → artifact
+          hash: sha256:a1b2c3d4...
+          path: artifacts/sha256/a1/b2/a1b2c3d4...txt
+        excerpt: "You are a helpful assistant...\n[TRUNCATED]"
+  - role: user
+    parts:
+      - type: text
+        content: "Hello!"  # Small content stays inline
+```
+
+Run `tree` on the output directory to visualize the entire execution hierarchy. Feed `_summary.md` to an LLM for debugging assistance - it combines high-level overview with detailed navigation for comprehensive trace analysis.
+
 ## Configuration
 
 ### Environment Variables
@@ -303,6 +365,9 @@ OPENAI_API_KEY=your-api-key
 LMNR_PROJECT_API_KEY=your-lmnr-key
 LMNR_DEBUG=true  # Enable debug traces
 
+# Optional: Local Trace Debugging
+TRACE_DEBUG_PATH=/path/to/trace/output  # Save traces locally for LLM-assisted debugging
+
 # Optional: Orchestration
 PREFECT_API_URL=http://localhost:4200/api
 PREFECT_API_KEY=your-prefect-key

{ai_pipeline_core-0.3.0 → ai_pipeline_core-0.3.3}/ai_pipeline_core/__init__.py

@@ -59,7 +59,7 @@ Quick Start:
     ... ) -> DocumentList:
     ...     # Messages accept AIMessages or str. Wrap documents: AIMessages([doc])
     ...     response = await llm.generate(
-    ...         "gpt-5",
+    ...         "gpt-5.1",
     ...         messages=AIMessages([documents[0]])
     ...     )
     ...     result = OutputDoc.create(
@@ -95,6 +95,15 @@ from .documents import (
     sanitize_url,
 )
 from .flow import FlowConfig, FlowOptions
+from .images import (
+    ImagePart,
+    ImagePreset,
+    ImageProcessingConfig,
+    ImageProcessingError,
+    ProcessedImage,
+    process_image,
+    process_image_to_documents,
+)
 from .llm import (
     AIMessages,
     AIMessageType,
@@ -121,7 +130,7 @@ from .settings import Settings
 from .tracing import TraceInfo, TraceLevel, set_trace_cost, trace
 from .utils.remote_deployment import remote_deployment
 
-__version__ = "0.3.0"
+__version__ = "0.3.3"
 
 __all__ = [
     # Config/Settings
@@ -175,6 +184,14 @@ __all__ = [
     # Prompt Builder
     "PromptBuilder",
     "EnvironmentVariable",
+    # Images
+    "process_image",
+    "process_image_to_documents",
+    "ImagePreset",
+    "ImageProcessingConfig",
+    "ProcessedImage",
+    "ImagePart",
+    "ImageProcessingError",
     # Utils
     "PromptManager",
 ]

ai_pipeline_core-0.3.3/ai_pipeline_core/debug/__init__.py

@@ -0,0 +1,26 @@
+"""Local trace debugging system for AI pipelines.
+
+This module provides filesystem-based trace debugging that saves all spans
+with their inputs/outputs for LLM-assisted debugging.
+
+Enable by setting TRACE_DEBUG_PATH environment variable.
+"""
+
+from .config import TraceDebugConfig
+from .content import ArtifactStore, ContentRef, ContentWriter, reconstruct_span_content
+from .processor import LocalDebugSpanProcessor
+from .summary import generate_summary
+from .writer import LocalTraceWriter, TraceState, WriteJob
+
+__all__ = [
+    "TraceDebugConfig",
+    "ContentRef",
+    "ContentWriter",
+    "ArtifactStore",
+    "reconstruct_span_content",
+    "LocalDebugSpanProcessor",
+    "LocalTraceWriter",
+    "TraceState",
+    "WriteJob",
+    "generate_summary",
+]

ai_pipeline_core-0.3.3/ai_pipeline_core/debug/config.py

@@ -0,0 +1,91 @@
+"""Configuration for local trace debugging."""
+
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TraceDebugConfig(BaseModel):
+    """Configuration for local trace debugging.
+
+    Controls how traces are written to the local filesystem for debugging.
+    Enable by setting TRACE_DEBUG_PATH environment variable.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    path: Path = Field(description="Directory for debug traces")
+    enabled: bool = Field(default=True, description="Whether debug tracing is enabled")
+
+    # Content size limits (Issue #2)
+    max_file_bytes: int = Field(
+        default=50_000,
+        description="Max bytes for input.yaml or output.yaml. Elements externalized to stay under.",
+    )
+    max_element_bytes: int = Field(
+        default=10_000,
+        description="Max bytes for single element. Above this, partial + artifact ref.",
+    )
+    element_excerpt_bytes: int = Field(
+        default=2_000,
+        description="Bytes of content to keep inline when element exceeds max_element_bytes.",
+    )
+    max_content_bytes: int = Field(
+        default=10_000_000,
+        description="Max bytes for any single artifact. Above this, truncate.",
+    )
+
+    # Image handling (Issue #7 - no changes per user)
+    extract_base64_images: bool = Field(
+        default=True,
+        description="Extract base64 images to artifact files",
+    )
+
+    # Span optimization (Issue #4)
+    merge_wrapper_spans: bool = Field(
+        default=True,
+        description="Merge Prefect wrapper spans with inner traced function spans",
+    )
+
+    # Events (Issue #12)
+    events_file_mode: str = Field(
+        default="errors_only",
+        description="When to write events.yaml: 'all', 'errors_only', 'none'",
+    )
+
+    # Indexes (Issue #1)
+    include_llm_index: bool = Field(
+        default=True,
+        description="Generate _llm_calls.yaml with LLM-specific details",
+    )
+    include_error_index: bool = Field(
+        default=True,
+        description="Generate _errors.yaml with failed span details",
+    )
+
+    # Cleanup
+    max_traces: int | None = Field(
+        default=None,
+        description="Max number of traces to keep. None for unlimited.",
+    )
+
+    # Security - default redaction patterns for common secrets
+    redact_patterns: tuple[str, ...] = Field(
+        default=(
+            r"sk-[a-zA-Z0-9]{20,}",  # OpenAI API keys
+            r"sk-proj-[a-zA-Z0-9\-_]{20,}",  # OpenAI project keys
+            r"AKIA[0-9A-Z]{16}",  # AWS access keys
+            r"ghp_[a-zA-Z0-9]{36}",  # GitHub personal tokens
+            r"gho_[a-zA-Z0-9]{36}",  # GitHub OAuth tokens
+            r"xoxb-[a-zA-Z0-9\-]+",  # Slack bot tokens
+            r"xoxp-[a-zA-Z0-9\-]+",  # Slack user tokens
+            r"(?i)password\s*[:=]\s*['\"]?[^\s'\"]+",  # Passwords
+            r"(?i)secret\s*[:=]\s*['\"]?[^\s'\"]+",  # Secrets
+            r"(?i)api[_\-]?key\s*[:=]\s*['\"]?[^\s'\"]+",  # API keys
+            r"(?i)bearer\s+[a-zA-Z0-9\-_\.]+",  # Bearer tokens
+        ),
+        description="Regex patterns for secrets to redact",
+    )
+
+    # Summary
+    generate_summary: bool = Field(default=True, description="Generate _summary.md")