ai-pipeline-core 0.4.6__py3-none-any.whl → 0.4.7__py3-none-any.whl

ai_pipeline_core/llm/ai_messages.py

@@ -38,6 +38,34 @@ def _ensure_llm_compatible_image(content: bytes, mime_type: str) -> tuple[bytes,
     return buf.getvalue(), "image/png"
 
 
+def _looks_like_text(content: bytes) -> bool:
+    """Check if content is valid UTF-8 text (not binary).
+
+    Uses heuristics: must decode as UTF-8 and have no null bytes.
+    Null bytes are common in binary files but rare in text.
+    """
+    if not content:
+        return True
+    # Null bytes indicate binary content
+    if b"\x00" in content:
+        return False
+    try:
+        content.decode("utf-8")
+        return True
+    except UnicodeDecodeError:
+        return False
+
+
+def _has_pdf_signature(content: bytes) -> bool:
+    """Check if content starts with PDF magic bytes (%PDF-).
+
+    Real PDFs start with %PDF- (possibly after whitespace).
+    This prevents false positives when a real PDF happens to be
+    partly UTF-8 decodable (e.g., ASCII-heavy PDF metadata).
+    """
+    return content.lstrip().startswith(b"%PDF-")
+
+
 AIMessageType = str | Document | ModelResponse
 """Type for messages in AIMessages container.
 
@@ -350,7 +378,7 @@ class AIMessages(list[AIMessageType]):  # noqa: PLR0904
         return count
 
     @staticmethod
-    def document_to_prompt(document: Document) -> list[ChatCompletionContentPartParam]:  # noqa: PLR0912, PLR0914
+    def document_to_prompt(document: Document) -> list[ChatCompletionContentPartParam]:  # noqa: C901, PLR0912, PLR0914, PLR0915
         """Convert a document to prompt format for LLM consumption.
 
         Renders the document as XML with text/image/PDF content, followed by any
@@ -368,8 +396,15 @@ class AIMessages(list[AIMessageType]):  # noqa: PLR0904
         description = f"<description>{document.description}</description>\n" if document.description else ""
         header_text = f"<document>\n<id>{document.id}</id>\n<name>{document.name}</name>\n{description}"
 
+        # Check if "PDF" is actually text (misnamed file from URL ending in .pdf)
+        # Real PDFs start with %PDF- magic bytes; if missing and content is UTF-8, it's text
+        is_text = document.is_text
+        if not is_text and document.is_pdf and _looks_like_text(document.content) and not _has_pdf_signature(document.content):
+            is_text = True
+            logger.debug(f"Document '{document.name}' has PDF extension but contains text content - sending as text")
+
         # Handle text documents
-        if document.is_text:
+        if is_text:
             text_content = document.content.decode("utf-8")
             content_text = f"{header_text}<content>\n{text_content}\n</content>\n"
             prompt.append({"type": "text", "text": content_text})
@@ -407,8 +442,16 @@ class AIMessages(list[AIMessageType]):  # noqa: PLR0904
             desc_attr = f' description="{att.description}"' if att.description else ""
             att_open = f'<attachment name="{att.name}"{desc_attr}>\n'
 
-            if att.is_text:
-                prompt.append({"type": "text", "text": f"{att_open}{att.text}\n</attachment>\n"})
+            # Check if "PDF" attachment is actually text (same logic as document)
+            att_is_text = att.is_text
+            if not att_is_text and att.is_pdf and _looks_like_text(att.content) and not _has_pdf_signature(att.content):
+                att_is_text = True
+                logger.debug(f"Attachment '{att.name}' has PDF extension but contains text content - sending as text")
+
+            if att_is_text:
+                # Use content.decode() directly - att.text property raises ValueError if is_text is False
+                att_text = att.content.decode("utf-8")
+                prompt.append({"type": "text", "text": f"{att_open}{att_text}\n</attachment>\n"})
             elif att.is_image or att.is_pdf:
                 prompt.append({"type": "text", "text": att_open})
 

{ai_pipeline_core-0.4.6.dist-info → ai_pipeline_core-0.4.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-pipeline-core
-Version: 0.4.6
+Version: 0.4.7
 Summary: Core utilities for AI-powered processing pipelines using prefect
 Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
 Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core

{ai_pipeline_core-0.4.6.dist-info → ai_pipeline_core-0.4.7.dist-info}/RECORD

@@ -37,7 +37,7 @@ ai_pipeline_core/documents/utils.py,sha256=9WOW3zvKYxQPnM8LjYFy3V9-yqc6hwgCaiog3
 ai_pipeline_core/images/__init__.py,sha256=Hc2QKR27Q2Q-h5nH-EbzfxdE3dHArBm-st5_xjOKFh0,8854
 ai_pipeline_core/images/_processing.py,sha256=MrCuPGsyyEl9UlXYIPhZs0wN8CPTMZmejV2Lo2wyCZk,4362
 ai_pipeline_core/llm/__init__.py,sha256=oyRvYD5DLDl7JIRTBUaiVz6jUC5dLLujkMNFpfRp2zc,795
-ai_pipeline_core/llm/ai_messages.py,sha256=Ycmntk5d6NUFqVVsnNR_IDwJUFuHYEH7CPvmmDfYaJI,17424
+ai_pipeline_core/llm/ai_messages.py,sha256=Ieldm2za0tVd-5ysxYTjietWq1gtJ8kWbP-AqWqNJNg,19308
 ai_pipeline_core/llm/client.py,sha256=N8eH9bY2rF28U5kGK0HQ3ibKvphcipSMLVVxtxtut8Y,30275
 ai_pipeline_core/llm/model_options.py,sha256=hg8xR0RJdJKp8QJNA4EbLnfFsnkE4HnxD85aYxc--hM,9164
 ai_pipeline_core/llm/model_response.py,sha256=Ml9wcssSssqibReJxCc9EQu488pz69Cmq_XNBs_xmak,12219
@@ -70,7 +70,7 @@ ai_pipeline_core/observability/_tracking/_writer.py,sha256=xZjwYyIxDzzzPxqkKjYAY
 ai_pipeline_core/pipeline/__init__.py,sha256=uMv1jwSyq8Ym8Hbn5097twBJLdwN1iMeqnVM4EWyrhA,282
 ai_pipeline_core/pipeline/decorators.py,sha256=CDJAeOjGLt5Ewc0Jc9zEuwLZwKyutOv89LSRS9dcXmI,37456
 ai_pipeline_core/pipeline/options.py,sha256=KF4FcT085-IwX8r649v0a9ua5xnApM0qG2wJHWbq39A,438
-ai_pipeline_core-0.4.6.dist-info/METADATA,sha256=hyy3vHyR5xZ5GRg4Nrp8BxKojp3pa-RDg1KWPPm5_O8,29947
-ai_pipeline_core-0.4.6.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-ai_pipeline_core-0.4.6.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
-ai_pipeline_core-0.4.6.dist-info/RECORD,,
+ai_pipeline_core-0.4.7.dist-info/METADATA,sha256=yFjXJ9fHXFtmrF2jIFx62k5spfR1PEipR_Uekbn3bmo,29947
+ai_pipeline_core-0.4.7.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+ai_pipeline_core-0.4.7.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
+ai_pipeline_core-0.4.7.dist-info/RECORD,,