ai-pipeline-core 0.3.0__py3-none-any.whl → 0.3.4__py3-none-any.whl

ai_pipeline_core/__init__.py

@@ -59,7 +59,7 @@ Quick Start:
     ... ) -> DocumentList:
     ...     # Messages accept AIMessages or str. Wrap documents: AIMessages([doc])
     ...     response = await llm.generate(
-    ...         "gpt-5",
+    ...         "gpt-5.1",
     ...         messages=AIMessages([documents[0]])
     ...     )
     ...     result = OutputDoc.create(
@@ -82,6 +82,26 @@ Optional Environment Variables:
     - LMNR_DEBUG: Set to "true" to enable debug-level traces
 """
 
+import os
+import sys
+
+# Disable Prefect's built-in OpenTelemetry spans to prevent duplicates.
+# All tracing is handled by our @trace decorator and Laminar SDK.
+# Must be set before Prefect is imported by submodules below.
+os.environ.setdefault("PREFECT_CLOUD_ENABLE_ORCHESTRATION_TELEMETRY", "false")
+
+# If Prefect was already imported (user imported it before us), refresh its cached settings.
+if "prefect" in sys.modules:
+    try:
+        from prefect.settings import get_current_settings  # noqa: PLC0415
+
+        if get_current_settings().cloud.enable_orchestration_telemetry:
+            from prefect.context import refresh_global_settings_context  # noqa: PLC0415
+
+            refresh_global_settings_context()
+    except (ImportError, AttributeError):
+        pass
+
 from . import llm, progress
 from .deployment import DeploymentContext, DeploymentResult, PipelineDeployment
 from .documents import (
@@ -95,6 +115,15 @@ from .documents import (
     sanitize_url,
 )
 from .flow import FlowConfig, FlowOptions
+from .images import (
+    ImagePart,
+    ImagePreset,
+    ImageProcessingConfig,
+    ImageProcessingError,
+    ProcessedImage,
+    process_image,
+    process_image_to_documents,
+)
 from .llm import (
     AIMessages,
     AIMessageType,
@@ -121,7 +150,7 @@ from .settings import Settings
 from .tracing import TraceInfo, TraceLevel, set_trace_cost, trace
 from .utils.remote_deployment import remote_deployment
 
-__version__ = "0.3.0"
+__version__ = "0.3.4"
 
 __all__ = [
     # Config/Settings
@@ -175,6 +204,14 @@ __all__ = [
     # Prompt Builder
     "PromptBuilder",
     "EnvironmentVariable",
+    # Images
+    "process_image",
+    "process_image_to_documents",
+    "ImagePreset",
+    "ImageProcessingConfig",
+    "ProcessedImage",
+    "ImagePart",
+    "ImageProcessingError",
     # Utils
     "PromptManager",
 ]

ai_pipeline_core/debug/__init__.py

@@ -0,0 +1,26 @@
+"""Local trace debugging system for AI pipelines.
+
+This module provides filesystem-based trace debugging that saves all spans
+with their inputs/outputs for LLM-assisted debugging.
+
+Enable by setting TRACE_DEBUG_PATH environment variable.
+"""
+
+from .config import TraceDebugConfig
+from .content import ArtifactStore, ContentRef, ContentWriter, reconstruct_span_content
+from .processor import LocalDebugSpanProcessor
+from .summary import generate_summary
+from .writer import LocalTraceWriter, TraceState, WriteJob
+
+__all__ = [
+    "TraceDebugConfig",
+    "ContentRef",
+    "ContentWriter",
+    "ArtifactStore",
+    "reconstruct_span_content",
+    "LocalDebugSpanProcessor",
+    "LocalTraceWriter",
+    "TraceState",
+    "WriteJob",
+    "generate_summary",
+]

ai_pipeline_core/debug/config.py

@@ -0,0 +1,91 @@
+"""Configuration for local trace debugging."""
+
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TraceDebugConfig(BaseModel):
+    """Configuration for local trace debugging.
+
+    Controls how traces are written to the local filesystem for debugging.
+    Enable by setting TRACE_DEBUG_PATH environment variable.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    path: Path = Field(description="Directory for debug traces")
+    enabled: bool = Field(default=True, description="Whether debug tracing is enabled")
+
+    # Content size limits (Issue #2)
+    max_file_bytes: int = Field(
+        default=50_000,
+        description="Max bytes for input.yaml or output.yaml. Elements externalized to stay under.",
+    )
+    max_element_bytes: int = Field(
+        default=10_000,
+        description="Max bytes for single element. Above this, partial + artifact ref.",
+    )
+    element_excerpt_bytes: int = Field(
+        default=2_000,
+        description="Bytes of content to keep inline when element exceeds max_element_bytes.",
+    )
+    max_content_bytes: int = Field(
+        default=10_000_000,
+        description="Max bytes for any single artifact. Above this, truncate.",
+    )
+
+    # Image handling (Issue #7 - no changes per user)
+    extract_base64_images: bool = Field(
+        default=True,
+        description="Extract base64 images to artifact files",
+    )
+
+    # Span optimization (Issue #4)
+    merge_wrapper_spans: bool = Field(
+        default=True,
+        description="Merge Prefect wrapper spans with inner traced function spans",
+    )
+
+    # Events (Issue #12)
+    events_file_mode: str = Field(
+        default="errors_only",
+        description="When to write events.yaml: 'all', 'errors_only', 'none'",
+    )
+
+    # Indexes (Issue #1)
+    include_llm_index: bool = Field(
+        default=True,
+        description="Generate _llm_calls.yaml with LLM-specific details",
+    )
+    include_error_index: bool = Field(
+        default=True,
+        description="Generate _errors.yaml with failed span details",
+    )
+
+    # Cleanup
+    max_traces: int | None = Field(
+        default=None,
+        description="Max number of traces to keep. None for unlimited.",
+    )
+
+    # Security - default redaction patterns for common secrets
+    redact_patterns: tuple[str, ...] = Field(
+        default=(
+            r"sk-[a-zA-Z0-9]{20,}",  # OpenAI API keys
+            r"sk-proj-[a-zA-Z0-9\-_]{20,}",  # OpenAI project keys
+            r"AKIA[0-9A-Z]{16}",  # AWS access keys
+            r"ghp_[a-zA-Z0-9]{36}",  # GitHub personal tokens
+            r"gho_[a-zA-Z0-9]{36}",  # GitHub OAuth tokens
+            r"xoxb-[a-zA-Z0-9\-]+",  # Slack bot tokens
+            r"xoxp-[a-zA-Z0-9\-]+",  # Slack user tokens
+            r"(?i)password\s*[:=]\s*['\"]?[^\s'\"]+",  # Passwords
+            r"(?i)secret\s*[:=]\s*['\"]?[^\s'\"]+",  # Secrets
+            r"(?i)api[_\-]?key\s*[:=]\s*['\"]?[^\s'\"]+",  # API keys
+            r"(?i)bearer\s+[a-zA-Z0-9\-_\.]+",  # Bearer tokens
+        ),
+        description="Regex patterns for secrets to redact",
+    )
+
+    # Summary
+    generate_summary: bool = Field(default=True, description="Generate _summary.md")