ai-pipeline-core 0.4.6__tar.gz → 0.4.8__tar.gz

{ai_pipeline_core-0.4.6 → ai_pipeline_core-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-pipeline-core
-Version: 0.4.6
+Version: 0.4.8
 Summary: Core utilities for AI-powered processing pipelines using prefect
 Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
 Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core

{ai_pipeline_core-0.4.6 → ai_pipeline_core-0.4.8}/ai_pipeline_core/__init__.py

@@ -64,7 +64,7 @@ from .prompt_manager import PromptManager
 from .settings import Settings
 from .testing import disable_run_logger, prefect_test_harness
 
-__version__ = "0.4.6"
+__version__ = "0.4.8"
 
 __all__ = [
     "AIMessageType",

{ai_pipeline_core-0.4.6 → ai_pipeline_core-0.4.8}/ai_pipeline_core/deployment/base.py

@@ -661,7 +661,10 @@ class PipelineDeployment(Generic[TOptions, TResult]):
         except Exception as e:
             logger.warning(f"Failed to initialize observability: {e}")
             with contextlib.suppress(Exception):
-                Laminar.initialize(export_timeout_seconds=15)
+                # Use canonical initializer to ensure consistent Laminar setup
+                from ai_pipeline_core.observability import tracing
+
+                tracing._initialise_laminar()
 
         deployment = self
 
@@ -882,13 +885,34 @@ class PipelineDeployment(Generic[TOptions, TResult]):
             options: FlowOptions,
             context: DeploymentContext,
         ) -> DeploymentResult:
+            # Initialize observability for remote workers
+            try:
+                initialize_observability()
+            except Exception as e:
+                logger.warning(f"Failed to initialize observability: {e}")
+                with contextlib.suppress(Exception):
+                    # Use canonical initializer to ensure consistent Laminar setup
+                    from ai_pipeline_core.observability import tracing
+
+                    tracing._initialise_laminar()
+
+            # Set session ID from Prefect flow run for trace grouping
+            flow_run_id = str(runtime.flow_run.get_id()) if runtime.flow_run else str(uuid4())  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType]
+            os.environ["LMNR_SESSION_ID"] = flow_run_id
+
             store = create_document_store(
                 settings,
                 summary_generator=_build_summary_generator(),
             )
             set_document_store(store)
             try:
-                return await deployment.run(project_name, documents, cast(Any, options), context)
+                # Create parent span to group all traces under a single deployment trace
+                with Laminar.start_as_current_span(
+                    name=f"{deployment.name}-{project_name}",
+                    input={"project_name": project_name, "options": options.model_dump()},
+                    session_id=flow_run_id,
+                ):
+                    return await deployment.run(project_name, documents, cast(Any, options), context)
             finally:
                 store.shutdown()
                 set_document_store(None)

{ai_pipeline_core-0.4.6 → ai_pipeline_core-0.4.8}/ai_pipeline_core/llm/ai_messages.py

@@ -38,6 +38,34 @@ def _ensure_llm_compatible_image(content: bytes, mime_type: str) -> tuple[bytes,
     return buf.getvalue(), "image/png"
 
 
+def _looks_like_text(content: bytes) -> bool:
+    """Check if content is valid UTF-8 text (not binary).
+
+    Uses heuristics: must decode as UTF-8 and have no null bytes.
+    Null bytes are common in binary files but rare in text.
+    """
+    if not content:
+        return True
+    # Null bytes indicate binary content
+    if b"\x00" in content:
+        return False
+    try:
+        content.decode("utf-8")
+        return True
+    except UnicodeDecodeError:
+        return False
+
+
+def _has_pdf_signature(content: bytes) -> bool:
+    """Check if content starts with PDF magic bytes (%PDF-).
+
+    Real PDFs start with %PDF- (possibly after whitespace).
+    This prevents false positives when a real PDF happens to be
+    partly UTF-8 decodable (e.g., ASCII-heavy PDF metadata).
+    """
+    return content.lstrip().startswith(b"%PDF-")
+
+
 AIMessageType = str | Document | ModelResponse
 """Type for messages in AIMessages container.
 
@@ -350,7 +378,7 @@ class AIMessages(list[AIMessageType]):  # noqa: PLR0904
         return count
 
     @staticmethod
-    def document_to_prompt(document: Document) -> list[ChatCompletionContentPartParam]:  # noqa: PLR0912, PLR0914
+    def document_to_prompt(document: Document) -> list[ChatCompletionContentPartParam]:  # noqa: C901, PLR0912, PLR0914, PLR0915
         """Convert a document to prompt format for LLM consumption.
 
         Renders the document as XML with text/image/PDF content, followed by any
@@ -368,8 +396,15 @@ class AIMessages(list[AIMessageType]):  # noqa: PLR0904
         description = f"<description>{document.description}</description>\n" if document.description else ""
         header_text = f"<document>\n<id>{document.id}</id>\n<name>{document.name}</name>\n{description}"
 
+        # Check if "PDF" is actually text (misnamed file from URL ending in .pdf)
+        # Real PDFs start with %PDF- magic bytes; if missing and content is UTF-8, it's text
+        is_text = document.is_text
+        if not is_text and document.is_pdf and _looks_like_text(document.content) and not _has_pdf_signature(document.content):
+            is_text = True
+            logger.debug(f"Document '{document.name}' has PDF extension but contains text content - sending as text")
+
         # Handle text documents
-        if document.is_text:
+        if is_text:
             text_content = document.content.decode("utf-8")
             content_text = f"{header_text}<content>\n{text_content}\n</content>\n"
             prompt.append({"type": "text", "text": content_text})
@@ -407,8 +442,16 @@ class AIMessages(list[AIMessageType]):  # noqa: PLR0904
             desc_attr = f' description="{att.description}"' if att.description else ""
             att_open = f'<attachment name="{att.name}"{desc_attr}>\n'
 
-            if att.is_text:
-                prompt.append({"type": "text", "text": f"{att_open}{att.text}\n</attachment>\n"})
+            # Check if "PDF" attachment is actually text (same logic as document)
+            att_is_text = att.is_text
+            if not att_is_text and att.is_pdf and _looks_like_text(att.content) and not _has_pdf_signature(att.content):
+                att_is_text = True
+                logger.debug(f"Attachment '{att.name}' has PDF extension but contains text content - sending as text")
+
+            if att_is_text:
+                # Use content.decode() directly - att.text property raises ValueError if is_text is False
+                att_text = att.content.decode("utf-8")
+                prompt.append({"type": "text", "text": f"{att_open}{att_text}\n</attachment>\n"})
             elif att.is_image or att.is_pdf:
                 prompt.append({"type": "text", "text": att_open})
 

{ai_pipeline_core-0.4.6 → ai_pipeline_core-0.4.8}/ai_pipeline_core/observability/_initialization.py

@@ -8,7 +8,6 @@ import importlib
 from typing import Any, Protocol
 from uuid import UUID
 
-from lmnr import Laminar
 from opentelemetry import trace as otel_trace
 from pydantic import BaseModel, ConfigDict
 
@@ -180,10 +179,12 @@ def initialize_observability(config: ObservabilityConfig | None = None) -> None:
     if config is None:
         config = _build_config_from_settings()
 
-    # 1. Laminar
+    # 1. Laminar - use canonical initializer from tracing module
     if config.has_lmnr:
         try:
-            Laminar.initialize(project_api_key=config.lmnr_project_api_key, export_timeout_seconds=15)
+            from ai_pipeline_core.observability import tracing  # noqa: PLC0415
+
+            tracing._initialise_laminar()
             logger.info("Laminar initialized")
         except Exception as e:
             logger.warning(f"Laminar initialization failed: {e}")

{ai_pipeline_core-0.4.6 → ai_pipeline_core-0.4.8}/ai_pipeline_core/observability/tracing.py

@@ -10,6 +10,7 @@ import contextlib
 import inspect
 import json
 import os
+import threading
 from collections.abc import Callable
 from functools import wraps
 from typing import Any, Literal, ParamSpec, TypeVar, cast, overload
@@ -220,19 +221,42 @@ class TraceInfo(BaseModel):
 # ---------------------------------------------------------------------------
 
 
+_laminar_initialized = False
+_laminar_init_lock = threading.Lock()
+
+
 def _initialise_laminar() -> None:
-    """Initialize Laminar SDK with project configuration.
+    """Initialize Laminar SDK with project configuration (lazy, once per process).
 
     Sets up the Laminar observability client with the project API key
     from settings. Disables automatic OpenAI instrumentation to avoid
     conflicts with our custom tracing.
 
-    Called once per process. Multiple calls are safe (Laminar handles idempotency).
+    IMPORTANT: This is called lazily at first trace execution (not at decoration time)
+    to allow LMNR_SPAN_CONTEXT environment variable to be set before initialization.
+    Laminar reads LMNR_SPAN_CONTEXT during initialize() to establish parent context
+    for cross-process tracing.
+
+    Uses double-checked locking pattern for thread safety. The flag is set AFTER
+    successful initialization to prevent permanently disabled tracing on init failure.
     """
-    if settings.lmnr_project_api_key:
-        Laminar.initialize(
-            project_api_key=settings.lmnr_project_api_key, disabled_instruments=[Instruments.OPENAI] if Instruments.OPENAI else [], export_timeout_seconds=15
-        )
+    global _laminar_initialized  # noqa: PLW0603
+
+    # Fast path: already initialized (no lock needed)
+    if _laminar_initialized:
+        return
+
+    with _laminar_init_lock:
+        # Double-check inside lock
+        if _laminar_initialized:
+            return
+
+        if settings.lmnr_project_api_key:
+            disabled = [Instruments.OPENAI] if Instruments.OPENAI else []
+            Laminar.initialize(project_api_key=settings.lmnr_project_api_key, disabled_instruments=disabled, export_timeout_seconds=15)
+
+        # Set flag AFTER successful initialization
+        _laminar_initialized = True
 
 
 # Overload for calls like @trace(name="...", level="debug")
@@ -400,7 +424,9 @@ def trace(  # noqa: UP047
             return f
 
         # --- Pre-computation (done once when the function is decorated) ---
-        _initialise_laminar()
+        # NOTE: _initialise_laminar() is NOT called here (at decoration/import time)
+        # to allow LMNR_SPAN_CONTEXT to be set before Laminar.initialize() runs.
+        # It's called lazily in the wrapper functions at first execution.
         sig = inspect.signature(f)
         is_coroutine = inspect.iscoroutinefunction(f)
         observe_name = name or f.__name__
@@ -550,6 +576,9 @@ def trace(  # noqa: UP047
             Returns:
                 The result of the wrapped function.
             """
+            # Lazy initialization: called at first execution, not at decoration time.
+            # This allows LMNR_SPAN_CONTEXT to be set before Laminar.initialize().
+            _initialise_laminar()
             observe_params = _prepare_and_get_observe_params(kwargs)
             observed_func = bound_observe(**observe_params)(f)
             return observed_func(*args, **kwargs)
@@ -561,6 +590,9 @@ def trace(  # noqa: UP047
             Returns:
                 The result of the wrapped function.
             """
+            # Lazy initialization: called at first execution, not at decoration time.
+            # This allows LMNR_SPAN_CONTEXT to be set before Laminar.initialize().
+            _initialise_laminar()
             observe_params = _prepare_and_get_observe_params(kwargs)
             observed_func = bound_observe(**observe_params)(f)
             return await observed_func(*args, **kwargs)  # pyright: ignore[reportGeneralTypeIssues, reportUnknownVariableType]

{ai_pipeline_core-0.4.6 → ai_pipeline_core-0.4.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ai-pipeline-core"
-version = "0.4.6"
+version = "0.4.8"
 description = "Core utilities for AI-powered processing pipelines using prefect"
 readme = "README.md"
 license = {text = "MIT"}