ai-pipeline-core 0.2.9__py3-none-any.whl → 0.3.3__py3-none-any.whl

ai_pipeline_core/__init__.py

@@ -59,7 +59,7 @@ Quick Start:
     ... ) -> DocumentList:
     ...     # Messages accept AIMessages or str. Wrap documents: AIMessages([doc])
     ...     response = await llm.generate(
-    ...         "gpt-5",
+    ...         "gpt-5.1",
     ...         messages=AIMessages([documents[0]])
     ...     )
     ...     result = OutputDoc.create(
@@ -82,7 +82,8 @@ Optional Environment Variables:
     - LMNR_DEBUG: Set to "true" to enable debug-level traces
 """
 
-from . import llm
+from . import llm, progress
+from .deployment import DeploymentContext, DeploymentResult, PipelineDeployment
 from .documents import (
     Document,
     DocumentList,
@@ -94,6 +95,15 @@ from .documents import (
     sanitize_url,
 )
 from .flow import FlowConfig, FlowOptions
+from .images import (
+    ImagePart,
+    ImagePreset,
+    ImageProcessingConfig,
+    ImageProcessingError,
+    ProcessedImage,
+    process_image,
+    process_image_to_documents,
+)
 from .llm import (
     AIMessages,
     AIMessageType,
@@ -114,11 +124,13 @@ from .logging import (
 from .logging import get_pipeline_logger as get_logger
 from .pipeline import pipeline_flow, pipeline_task
 from .prefect import disable_run_logger, prefect_test_harness
+from .prompt_builder import EnvironmentVariable, PromptBuilder
 from .prompt_manager import PromptManager
 from .settings import Settings
 from .tracing import TraceInfo, TraceLevel, set_trace_cost, trace
+from .utils.remote_deployment import remote_deployment
 
-__version__ = "0.2.9"
+__version__ = "0.3.3"
 
 __all__ = [
     # Config/Settings
@@ -148,6 +160,12 @@ __all__ = [
     # Prefect decorators (clean, no tracing)
     "prefect_test_harness",
     "disable_run_logger",
+    # Deployment
+    "PipelineDeployment",
+    "DeploymentContext",
+    "DeploymentResult",
+    "remote_deployment",
+    "progress",
     # LLM
     "llm",  # for backward compatibility
     "generate",
@@ -163,8 +181,17 @@ __all__ = [
     "TraceLevel",
     "TraceInfo",
     "set_trace_cost",
+    # Prompt Builder
+    "PromptBuilder",
+    "EnvironmentVariable",
+    # Images
+    "process_image",
+    "process_image_to_documents",
+    "ImagePreset",
+    "ImageProcessingConfig",
+    "ProcessedImage",
+    "ImagePart",
+    "ImageProcessingError",
     # Utils
     "PromptManager",
-    "generate",
-    "generate_structured",
 ]

ai_pipeline_core/debug/__init__.py

@@ -0,0 +1,26 @@
+"""Local trace debugging system for AI pipelines.
+
+This module provides filesystem-based trace debugging that saves all spans
+with their inputs/outputs for LLM-assisted debugging.
+
+Enable by setting TRACE_DEBUG_PATH environment variable.
+"""
+
+from .config import TraceDebugConfig
+from .content import ArtifactStore, ContentRef, ContentWriter, reconstruct_span_content
+from .processor import LocalDebugSpanProcessor
+from .summary import generate_summary
+from .writer import LocalTraceWriter, TraceState, WriteJob
+
+__all__ = [
+    "TraceDebugConfig",
+    "ContentRef",
+    "ContentWriter",
+    "ArtifactStore",
+    "reconstruct_span_content",
+    "LocalDebugSpanProcessor",
+    "LocalTraceWriter",
+    "TraceState",
+    "WriteJob",
+    "generate_summary",
+]

ai_pipeline_core/debug/config.py

@@ -0,0 +1,91 @@
+"""Configuration for local trace debugging."""
+
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TraceDebugConfig(BaseModel):
+    """Configuration for local trace debugging.
+
+    Controls how traces are written to the local filesystem for debugging.
+    Enable by setting TRACE_DEBUG_PATH environment variable.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    path: Path = Field(description="Directory for debug traces")
+    enabled: bool = Field(default=True, description="Whether debug tracing is enabled")
+
+    # Content size limits (Issue #2)
+    max_file_bytes: int = Field(
+        default=50_000,
+        description="Max bytes for input.yaml or output.yaml. Elements externalized to stay under.",
+    )
+    max_element_bytes: int = Field(
+        default=10_000,
+        description="Max bytes for single element. Above this, partial + artifact ref.",
+    )
+    element_excerpt_bytes: int = Field(
+        default=2_000,
+        description="Bytes of content to keep inline when element exceeds max_element_bytes.",
+    )
+    max_content_bytes: int = Field(
+        default=10_000_000,
+        description="Max bytes for any single artifact. Above this, truncate.",
+    )
+
+    # Image handling (Issue #7 - no changes per user)
+    extract_base64_images: bool = Field(
+        default=True,
+        description="Extract base64 images to artifact files",
+    )
+
+    # Span optimization (Issue #4)
+    merge_wrapper_spans: bool = Field(
+        default=True,
+        description="Merge Prefect wrapper spans with inner traced function spans",
+    )
+
+    # Events (Issue #12)
+    events_file_mode: str = Field(
+        default="errors_only",
+        description="When to write events.yaml: 'all', 'errors_only', 'none'",
+    )
+
+    # Indexes (Issue #1)
+    include_llm_index: bool = Field(
+        default=True,
+        description="Generate _llm_calls.yaml with LLM-specific details",
+    )
+    include_error_index: bool = Field(
+        default=True,
+        description="Generate _errors.yaml with failed span details",
+    )
+
+    # Cleanup
+    max_traces: int | None = Field(
+        default=None,
+        description="Max number of traces to keep. None for unlimited.",
+    )
+
+    # Security - default redaction patterns for common secrets
+    redact_patterns: tuple[str, ...] = Field(
+        default=(
+            r"sk-[a-zA-Z0-9]{20,}",  # OpenAI API keys
+            r"sk-proj-[a-zA-Z0-9\-_]{20,}",  # OpenAI project keys
+            r"AKIA[0-9A-Z]{16}",  # AWS access keys
+            r"ghp_[a-zA-Z0-9]{36}",  # GitHub personal tokens
+            r"gho_[a-zA-Z0-9]{36}",  # GitHub OAuth tokens
+            r"xoxb-[a-zA-Z0-9\-]+",  # Slack bot tokens
+            r"xoxp-[a-zA-Z0-9\-]+",  # Slack user tokens
+            r"(?i)password\s*[:=]\s*['\"]?[^\s'\"]+",  # Passwords
+            r"(?i)secret\s*[:=]\s*['\"]?[^\s'\"]+",  # Secrets
+            r"(?i)api[_\-]?key\s*[:=]\s*['\"]?[^\s'\"]+",  # API keys
+            r"(?i)bearer\s+[a-zA-Z0-9\-_\.]+",  # Bearer tokens
+        ),
+        description="Regex patterns for secrets to redact",
+    )
+
+    # Summary
+    generate_summary: bool = Field(default=True, description="Generate _summary.md")