ai-pipeline-core 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

ai_pipeline_core/documents/document.py

@@ -1,6 +1,7 @@
 import base64
 import hashlib
 import json
+import re
 from abc import ABC, abstractmethod
 from base64 import b32encode
 from enum import StrEnum
@@ -26,6 +27,7 @@ class Document(BaseModel, ABC):
 
     MAX_CONTENT_SIZE: ClassVar[int] = 10 * 1024 * 1024  # 10MB default
     DESCRIPTION_EXTENSION: ClassVar[str] = ".description.md"
+    MARKDOWN_LIST_SEPARATOR: ClassVar[str] = "\n\n---\n\n"
 
     def __init__(self, **data: Any) -> None:
         """Prevent direct instantiation of abstract Document class."""
@@ -199,15 +201,34 @@ class Document(BaseModel, ABC):
 
     def as_yaml(self) -> Any:
         """Parse document as YAML"""
-        if not self.is_text:
-            raise ValueError(f"Document is not text: {self.name}")
-        return YAML().load(self.content.decode("utf-8"))  # type: ignore
+        return YAML().load(self.as_text())
 
     def as_json(self) -> Any:
         """Parse document as JSON"""
-        if not self.is_text:
-            raise ValueError(f"Document is not text: {self.name}")
-        return json.loads(self.content.decode("utf-8"))
+        return json.loads(self.as_text())
+
+    def as_markdown_list(self) -> list[str]:
+        """Parse document as a markdown list"""
+        return self.as_text().split(self.MARKDOWN_LIST_SEPARATOR)
+
+    @classmethod
+    def create(cls, name: str, description: str | None, content: bytes | str) -> Self:
+        """Create a document from a name, description, and content"""
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+        return cls(name=name, description=description, content=content)
+
+    @classmethod
+    def create_as_markdown_list(cls, name: str, description: str | None, items: list[str]) -> Self:
+        """Create a document from a name, description, and list of strings"""
+        # remove other list separators (lines that are only the separator + whitespace)
+        separator = Document.MARKDOWN_LIST_SEPARATOR.strip()
+        pattern = re.compile(rf"^[ \t]*{re.escape(separator)}[ \t]*(?:\r?\n|$)", flags=re.MULTILINE)
+        # Normalize CRLF/CR to LF before cleaning to ensure consistent behavior
+        normalized_items = [re.sub(r"\r\n?", "\n", item) for item in items]
+        cleaned_items = [pattern.sub("", item) for item in normalized_items]
+        content = Document.MARKDOWN_LIST_SEPARATOR.join(cleaned_items)
+        return cls.create(name, description, content)
 
     def serialize_model(self) -> dict[str, Any]:
         """Serialize document to a dictionary with proper encoding."""

ai_pipeline_core/llm/client.py

@@ -162,7 +162,7 @@ async def generate(
 T = TypeVar("T", bound=BaseModel)
 
 
-@trace
+@trace(ignore_inputs=["context"])
 async def generate_structured(
     model: ModelName,
     response_format: type[T],

{ai_pipeline_core-0.1.2.dist-info → ai_pipeline_core-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-pipeline-core
-Version: 0.1.2
+Version: 0.1.3
 Summary: Core utilities for AI-powered processing pipelines using prefect
 Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
 Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core

{ai_pipeline_core-0.1.2.dist-info → ai_pipeline_core-0.1.3.dist-info}/RECORD

@@ -5,7 +5,7 @@ ai_pipeline_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ai_pipeline_core/settings.py,sha256=Zl2BPa6IHzh-B5V7cg5mtySr1dhWZQYYKxXz3BwrHlQ,615
 ai_pipeline_core/tracing.py,sha256=UcQ_z1F8KrBLq5ZJaXyoIZsiMdqjHzBhADV07pYXY-w,7663
 ai_pipeline_core/documents/__init__.py,sha256=rEnKj-sSlZ9WnFlZAmSGVi1P8vnsHmU9O9_YwtP40ms,242
-ai_pipeline_core/documents/document.py,sha256=Gj4WR57VW67hhRN1360oqHfVQg6Xxj4jx_XueK4cvl0,8941
+ai_pipeline_core/documents/document.py,sha256=LNKomZnkhgiJHuH2pIzZJUHwrl1rBMBypOiSKg_AcHY,10068
 ai_pipeline_core/documents/document_list.py,sha256=HOG_uZDazA9CJB7Lr_tNcDFzb5Ff9RUt0ELWQK_eYNM,4940
 ai_pipeline_core/documents/flow_document.py,sha256=qsV-2JYOMhkvAj7lW54ZNH_4QUclld9h06CoU59tWww,815
 ai_pipeline_core/documents/mime_type.py,sha256=tMWGH9PVmHe6a_IzdaJUqIHf4qnwQOwOCBhsgW2AyTE,2244
@@ -15,7 +15,7 @@ ai_pipeline_core/flow/__init__.py,sha256=_Sji2yY1ICkvVX6QiiGWKzqIXtg9UAiuvhjHSK_
 ai_pipeline_core/flow/config.py,sha256=crbe_OvNE6qulIKv1D8yKoe8xrEsIlvICyxjhqHHBxQ,2266
 ai_pipeline_core/llm/__init__.py,sha256=3XVK-bSJdOe0s6KmmO7PDbsXHfjlcZEG1MVBmaz3EeU,442
 ai_pipeline_core/llm/ai_messages.py,sha256=lzzp4t6xDU9ULhwbmORFTQbdl0BbsqxD5YLzlHstQwU,4333
-ai_pipeline_core/llm/client.py,sha256=FrSWmO2CiwinGLXo431YgtOY_ePAGuHgMRBjSwzyL0I,7663
+ai_pipeline_core/llm/client.py,sha256=vVzgJXmWcWZ1limRvq_PGkFAhrfSNp18UECSiJtGhYI,7690
 ai_pipeline_core/llm/model_options.py,sha256=TvAAlDFZN-TP9-J-RZBuU_dpSocskf6paaQMw1XY9UE,1321
 ai_pipeline_core/llm/model_response.py,sha256=fIWueaemgo0cMruvToMZyKsRPzKwL6IlvUJN7DLG710,5558
 ai_pipeline_core/llm/model_types.py,sha256=rIwY6voT8-xdfsKPDC0Gkdl2iTp9Q2LuvWGSRU9Mp3k,342
@@ -23,7 +23,7 @@ ai_pipeline_core/logging/__init__.py,sha256=DOO6ckgnMVXl29Sy7q6jhO-iW96h54pCHQDz
 ai_pipeline_core/logging/logging.yml,sha256=YTW48keO_K5bkkb-KXGM7ZuaYKiquLsjsURei8Ql0V4,1353
 ai_pipeline_core/logging/logging_config.py,sha256=6MBz9nnVNvqiLDoyy9-R3sWkn6927Re5hdz4hwTptpI,4903
 ai_pipeline_core/logging/logging_mixin.py,sha256=RDaR2ju2-vKTJRzXGa0DquGPT8_UxahWjvKJnaD0IV8,7810
-ai_pipeline_core-0.1.2.dist-info/METADATA,sha256=LO-DGxVRhZPcwRwb2_zibFzp8aRE59STOYp2BxDag8M,15869
-ai_pipeline_core-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ai_pipeline_core-0.1.2.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
-ai_pipeline_core-0.1.2.dist-info/RECORD,,
+ai_pipeline_core-0.1.3.dist-info/METADATA,sha256=0TBoEep5C9LSmlOKA3koR2vfKSiIba49SLYLnVLiaWc,15869
+ai_pipeline_core-0.1.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ai_pipeline_core-0.1.3.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
+ai_pipeline_core-0.1.3.dist-info/RECORD,,