ai-pipeline-core 0.1.13__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_pipeline_core/__init__.py +25 -14
- ai_pipeline_core/documents/__init__.py +2 -1
- ai_pipeline_core/documents/document.py +317 -49
- ai_pipeline_core/documents/document_list.py +136 -33
- ai_pipeline_core/documents/flow_document.py +8 -29
- ai_pipeline_core/documents/task_document.py +6 -27
- ai_pipeline_core/documents/temporary_document.py +6 -27
- ai_pipeline_core/documents/utils.py +64 -1
- ai_pipeline_core/flow/config.py +174 -5
- ai_pipeline_core/flow/options.py +2 -2
- ai_pipeline_core/llm/__init__.py +6 -1
- ai_pipeline_core/llm/ai_messages.py +14 -7
- ai_pipeline_core/llm/client.py +143 -55
- ai_pipeline_core/llm/model_options.py +20 -5
- ai_pipeline_core/llm/model_response.py +77 -29
- ai_pipeline_core/llm/model_types.py +38 -40
- ai_pipeline_core/logging/__init__.py +0 -2
- ai_pipeline_core/logging/logging_config.py +0 -6
- ai_pipeline_core/logging/logging_mixin.py +2 -10
- ai_pipeline_core/pipeline.py +68 -65
- ai_pipeline_core/prefect.py +12 -3
- ai_pipeline_core/prompt_manager.py +6 -7
- ai_pipeline_core/settings.py +13 -5
- ai_pipeline_core/simple_runner/__init__.py +1 -11
- ai_pipeline_core/simple_runner/cli.py +13 -12
- ai_pipeline_core/simple_runner/simple_runner.py +34 -172
- ai_pipeline_core/storage/__init__.py +8 -0
- ai_pipeline_core/storage/storage.py +628 -0
- ai_pipeline_core/tracing.py +110 -26
- {ai_pipeline_core-0.1.13.dist-info → ai_pipeline_core-0.2.0.dist-info}/METADATA +60 -23
- ai_pipeline_core-0.2.0.dist-info/RECORD +38 -0
- ai_pipeline_core-0.1.13.dist-info/RECORD +0 -36
- {ai_pipeline_core-0.1.13.dist-info → ai_pipeline_core-0.2.0.dist-info}/WHEEL +0 -0
- {ai_pipeline_core-0.1.13.dist-info → ai_pipeline_core-0.2.0.dist-info}/licenses/LICENSE +0 -0
ai_pipeline_core/tracing.py
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
"""Tracing utilities that integrate Laminar (``lmnr``) with our code-base.
|
|
2
2
|
|
|
3
|
-
@public
|
|
4
|
-
|
|
5
3
|
This module centralizes:
|
|
6
4
|
- ``TraceInfo`` - a small helper object for propagating contextual metadata.
|
|
7
5
|
- ``trace`` decorator - augments a callable with Laminar tracing, automatic
|
|
@@ -15,7 +13,7 @@ import os
|
|
|
15
13
|
from functools import wraps
|
|
16
14
|
from typing import Any, Callable, Literal, ParamSpec, TypeVar, cast, overload
|
|
17
15
|
|
|
18
|
-
from lmnr import Instruments, Laminar, observe
|
|
16
|
+
from lmnr import Attributes, Instruments, Laminar, observe
|
|
19
17
|
from pydantic import BaseModel
|
|
20
18
|
|
|
21
19
|
from ai_pipeline_core.settings import settings
|
|
@@ -29,8 +27,6 @@ R = TypeVar("R")
|
|
|
29
27
|
TraceLevel = Literal["always", "debug", "off"]
|
|
30
28
|
"""Control level for tracing activation.
|
|
31
29
|
|
|
32
|
-
@public
|
|
33
|
-
|
|
34
30
|
Values:
|
|
35
31
|
- "always": Always trace (default, production mode)
|
|
36
32
|
- "debug": Only trace when LMNR_DEBUG == "true"
|
|
@@ -51,16 +47,12 @@ class TraceInfo(BaseModel):
|
|
|
51
47
|
|
|
52
48
|
Attributes:
|
|
53
49
|
session_id: Unique identifier for the current session/conversation.
|
|
54
|
-
Falls back to LMNR_SESSION_ID environment variable.
|
|
55
50
|
user_id: Identifier for the user triggering the operation.
|
|
56
|
-
Falls back to LMNR_USER_ID environment variable.
|
|
57
51
|
metadata: Key-value pairs for additional trace context.
|
|
58
52
|
Useful for filtering and searching in LMNR dashboard.
|
|
59
53
|
tags: List of tags for categorizing traces (e.g., ["production", "v2"]).
|
|
60
54
|
|
|
61
55
|
Environment fallbacks:
|
|
62
|
-
- LMNR_SESSION_ID: Default session_id if not explicitly set
|
|
63
|
-
- LMNR_USER_ID: Default user_id if not explicitly set
|
|
64
56
|
- LMNR_DEBUG: Controls debug-level tracing when set to "true"
|
|
65
57
|
|
|
66
58
|
Note: These variables are read directly by the tracing layer and are
|
|
@@ -102,8 +94,8 @@ class TraceInfo(BaseModel):
|
|
|
102
94
|
|
|
103
95
|
Returns:
|
|
104
96
|
Dictionary with keys:
|
|
105
|
-
- session_id: From field or
|
|
106
|
-
- user_id: From field or
|
|
97
|
+
- session_id: From field or environment variable fallback
|
|
98
|
+
- user_id: From field or environment variable fallback
|
|
107
99
|
- metadata: Dictionary of custom metadata (if set)
|
|
108
100
|
- tags: List of tags (if set)
|
|
109
101
|
|
|
@@ -212,15 +204,13 @@ def trace(
|
|
|
212
204
|
) -> Callable[[Callable[P, R]], Callable[P, R]] | Callable[P, R]:
|
|
213
205
|
"""Add Laminar observability tracing to any function.
|
|
214
206
|
|
|
215
|
-
@public
|
|
216
|
-
|
|
217
207
|
The trace decorator integrates functions with Laminar (LMNR) for
|
|
218
208
|
distributed tracing, performance monitoring, and debugging. It
|
|
219
209
|
automatically handles both sync and async functions, propagates
|
|
220
210
|
trace context, and provides fine-grained control over what gets traced.
|
|
221
211
|
|
|
222
212
|
USAGE GUIDELINE - Defaults First:
|
|
223
|
-
|
|
213
|
+
By default, use WITHOUT any parameters unless instructed otherwise.
|
|
224
214
|
The defaults are optimized for most use cases.
|
|
225
215
|
|
|
226
216
|
Args:
|
|
@@ -306,8 +296,6 @@ def trace(
|
|
|
306
296
|
|
|
307
297
|
Environment variables:
|
|
308
298
|
- LMNR_DEBUG: Set to "true" to enable debug-level traces
|
|
309
|
-
- LMNR_SESSION_ID: Default session ID if not in TraceInfo
|
|
310
|
-
- LMNR_USER_ID: Default user ID if not in TraceInfo
|
|
311
299
|
- LMNR_PROJECT_API_KEY: Required for trace submission
|
|
312
300
|
|
|
313
301
|
Performance:
|
|
@@ -320,11 +308,6 @@ def trace(
|
|
|
320
308
|
- Works with both sync and async functions
|
|
321
309
|
- Preserves function signature and metadata
|
|
322
310
|
- Thread-safe and async-safe
|
|
323
|
-
|
|
324
|
-
See Also:
|
|
325
|
-
- TraceInfo: Container for trace metadata
|
|
326
|
-
- pipeline_task: Task decorator with built-in tracing
|
|
327
|
-
- pipeline_flow: Flow decorator with built-in tracing
|
|
328
311
|
"""
|
|
329
312
|
if level == "off":
|
|
330
313
|
if func:
|
|
@@ -370,7 +353,7 @@ def trace(
|
|
|
370
353
|
# Store the new parameters
|
|
371
354
|
_session_id = session_id
|
|
372
355
|
_user_id = user_id
|
|
373
|
-
_metadata = metadata
|
|
356
|
+
_metadata = metadata or {}
|
|
374
357
|
_tags = tags or []
|
|
375
358
|
_span_type = span_type
|
|
376
359
|
_ignore_input = ignore_input
|
|
@@ -404,10 +387,8 @@ def trace(
|
|
|
404
387
|
observe_params["session_id"] = _session_id
|
|
405
388
|
if _user_id:
|
|
406
389
|
observe_params["user_id"] = _user_id
|
|
407
|
-
|
|
408
|
-
# Merge decorator-level metadata and tags
|
|
409
390
|
if _metadata:
|
|
410
|
-
observe_params["metadata"] =
|
|
391
|
+
observe_params["metadata"] = _metadata
|
|
411
392
|
if _tags:
|
|
412
393
|
observe_params["tags"] = observe_params.get("tags", []) + _tags
|
|
413
394
|
if _span_type:
|
|
@@ -473,4 +454,107 @@ def trace(
|
|
|
473
454
|
return decorator # Called as @trace(...)
|
|
474
455
|
|
|
475
456
|
|
|
476
|
-
|
|
457
|
+
def set_trace_cost(cost: float | str) -> None:
|
|
458
|
+
"""Set cost attributes for the current trace span.
|
|
459
|
+
|
|
460
|
+
Sets cost metadata in the current LMNR trace span for tracking expenses
|
|
461
|
+
of custom operations. This function should be called within a traced
|
|
462
|
+
function to dynamically set or update the cost associated with the
|
|
463
|
+
current operation. Particularly useful for tracking costs of external
|
|
464
|
+
API calls, compute resources, or custom billing scenarios.
|
|
465
|
+
|
|
466
|
+
The cost is stored in three metadata fields for compatibility:
|
|
467
|
+
- gen_ai.usage.output_cost: Standard OpenAI cost field
|
|
468
|
+
- gen_ai.usage.cost: Alternative cost field
|
|
469
|
+
- cost: Simple cost field
|
|
470
|
+
|
|
471
|
+
Args:
|
|
472
|
+
cost: The cost value to set. Can be:
|
|
473
|
+
- float: Cost in dollars (e.g., 0.05 for 5 cents)
|
|
474
|
+
- str: USD format with dollar sign (e.g., "$0.05" or "$1.25")
|
|
475
|
+
Only positive values will be set; zero or negative values are ignored.
|
|
476
|
+
|
|
477
|
+
Example:
|
|
478
|
+
>>> # Track cost of external API call
|
|
479
|
+
>>> @trace
|
|
480
|
+
>>> async def call_translation_api(text: str) -> str:
|
|
481
|
+
... # External API charges per character
|
|
482
|
+
... char_count = len(text)
|
|
483
|
+
... cost_per_char = 0.00001 # $0.00001 per character
|
|
484
|
+
...
|
|
485
|
+
... result = await external_api.translate(text)
|
|
486
|
+
...
|
|
487
|
+
... # Set the cost for this operation
|
|
488
|
+
... set_trace_cost(char_count * cost_per_char)
|
|
489
|
+
... return result
|
|
490
|
+
>>>
|
|
491
|
+
>>> # Track compute resource costs
|
|
492
|
+
>>> @trace
|
|
493
|
+
>>> def process_video(video_path: str) -> dict:
|
|
494
|
+
... duration = get_video_duration(video_path)
|
|
495
|
+
... cost_per_minute = 0.10 # $0.10 per minute
|
|
496
|
+
...
|
|
497
|
+
... result = process_video_content(video_path)
|
|
498
|
+
...
|
|
499
|
+
... # Set cost using string format
|
|
500
|
+
... set_trace_cost(f"${duration * cost_per_minute:.2f}")
|
|
501
|
+
... return result
|
|
502
|
+
>>>
|
|
503
|
+
>>> # Combine with LLM costs in pipeline
|
|
504
|
+
>>> @pipeline_task
|
|
505
|
+
>>> async def enriched_generation(prompt: str) -> str:
|
|
506
|
+
... # LLM cost tracked automatically via ModelResponse
|
|
507
|
+
... response = await llm.generate("gpt-5", messages=prompt)
|
|
508
|
+
...
|
|
509
|
+
... # Add cost for post-processing
|
|
510
|
+
... processing_cost = 0.02 # Fixed cost for enrichment
|
|
511
|
+
... set_trace_cost(processing_cost)
|
|
512
|
+
...
|
|
513
|
+
... return enrich_response(response.content)
|
|
514
|
+
|
|
515
|
+
Raises:
|
|
516
|
+
ValueError: If string format is invalid (not a valid USD amount).
|
|
517
|
+
|
|
518
|
+
Note:
|
|
519
|
+
- This function only works within a traced context (function decorated
|
|
520
|
+
with @trace, @pipeline_task, or @pipeline_flow)
|
|
521
|
+
- LLM costs are tracked automatically via ModelResponse; use this for non-LLM costs
|
|
522
|
+
- Cost should be a positive number representing actual monetary cost in USD
|
|
523
|
+
- The cost is added to the current span's attributes/metadata
|
|
524
|
+
- Multiple calls overwrite the previous cost (not cumulative)
|
|
525
|
+
- If called outside a traced context (no active span), it has no effect
|
|
526
|
+
and does not raise an error
|
|
527
|
+
"""
|
|
528
|
+
# Parse string format if provided
|
|
529
|
+
if isinstance(cost, str):
|
|
530
|
+
# Remove dollar sign and any whitespace
|
|
531
|
+
cost_str = cost.strip()
|
|
532
|
+
if not cost_str.startswith("$"):
|
|
533
|
+
raise ValueError(f"Invalid USD format: {cost!r}. Must start with '$' (e.g., '$0.50')")
|
|
534
|
+
|
|
535
|
+
try:
|
|
536
|
+
# Remove $ and convert to float
|
|
537
|
+
cost_value = float(cost_str[1:])
|
|
538
|
+
except ValueError as e:
|
|
539
|
+
raise ValueError(
|
|
540
|
+
f"Invalid USD format: {cost!r}. Must be a valid number after '$'"
|
|
541
|
+
) from e
|
|
542
|
+
else:
|
|
543
|
+
cost_value = cost
|
|
544
|
+
|
|
545
|
+
if cost_value > 0:
|
|
546
|
+
# Build the attributes dictionary with cost metadata
|
|
547
|
+
attributes: dict[Attributes | str, float] = {
|
|
548
|
+
"gen_ai.usage.output_cost": cost_value,
|
|
549
|
+
"gen_ai.usage.cost": cost_value,
|
|
550
|
+
"cost": cost_value,
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
try:
|
|
554
|
+
Laminar.set_span_attributes(attributes)
|
|
555
|
+
except Exception:
|
|
556
|
+
# Silently ignore if not in a traced context
|
|
557
|
+
pass
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
__all__ = ["trace", "TraceLevel", "TraceInfo", "set_trace_cost"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ai-pipeline-core
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Core utilities for AI-powered processing pipelines using prefect
|
|
5
5
|
Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
|
|
6
6
|
Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core
|
|
@@ -22,6 +22,7 @@ Requires-Dist: httpx>=0.28.1
|
|
|
22
22
|
Requires-Dist: jinja2>=3.1.6
|
|
23
23
|
Requires-Dist: lmnr>=0.7.6
|
|
24
24
|
Requires-Dist: openai>=1.99.9
|
|
25
|
+
Requires-Dist: prefect-gcp[cloud-storage]>=0.6.10
|
|
25
26
|
Requires-Dist: prefect>=3.4.13
|
|
26
27
|
Requires-Dist: pydantic-settings>=2.10.1
|
|
27
28
|
Requires-Dist: pydantic>=2.11.7
|
|
@@ -57,11 +58,11 @@ AI Pipeline Core is a production-ready framework that combines document processi
|
|
|
57
58
|
|
|
58
59
|
### Key Features
|
|
59
60
|
|
|
60
|
-
- **Document Processing**: Type-safe handling of text, JSON, YAML, PDFs, and images with automatic MIME type detection
|
|
61
|
-
- **LLM Integration**: Unified interface to any model via LiteLLM proxy with
|
|
61
|
+
- **Document Processing**: Type-safe handling of text, JSON, YAML, PDFs, and images with automatic MIME type detection and provenance tracking
|
|
62
|
+
- **LLM Integration**: Unified interface to any model via LiteLLM proxy with configurable context caching
|
|
62
63
|
- **Structured Output**: Type-safe generation with Pydantic model validation
|
|
63
64
|
- **Workflow Orchestration**: Prefect-based flows and tasks with automatic retries
|
|
64
|
-
- **Observability**: Built-in distributed tracing via Laminar (LMNR) for debugging and monitoring
|
|
65
|
+
- **Observability**: Built-in distributed tracing via Laminar (LMNR) with cost tracking for debugging and monitoring
|
|
65
66
|
- **Local Development**: Simple runner for testing pipelines without infrastructure
|
|
66
67
|
|
|
67
68
|
## Installation
|
|
@@ -111,15 +112,13 @@ class AnalysisConfig(FlowConfig):
|
|
|
111
112
|
INPUT_DOCUMENT_TYPES = [InputDoc]
|
|
112
113
|
OUTPUT_DOCUMENT_TYPE = OutputDoc
|
|
113
114
|
|
|
114
|
-
# Create pipeline flow
|
|
115
|
-
@pipeline_flow
|
|
115
|
+
# Create pipeline flow with required config
|
|
116
|
+
@pipeline_flow(config=AnalysisConfig)
|
|
116
117
|
async def analyze_flow(
|
|
117
118
|
project_name: str,
|
|
118
119
|
documents: DocumentList,
|
|
119
120
|
flow_options: FlowOptions
|
|
120
121
|
) -> DocumentList:
|
|
121
|
-
config = AnalysisConfig()
|
|
122
|
-
|
|
123
122
|
# Process documents
|
|
124
123
|
outputs = []
|
|
125
124
|
for doc in documents:
|
|
@@ -136,7 +135,7 @@ async def analyze_flow(
|
|
|
136
135
|
outputs.append(output)
|
|
137
136
|
|
|
138
137
|
# RECOMMENDED: Always validate output
|
|
139
|
-
return
|
|
138
|
+
return AnalysisConfig.create_and_validate_output(outputs)
|
|
140
139
|
```
|
|
141
140
|
|
|
142
141
|
### Structured Output
|
|
@@ -178,6 +177,19 @@ doc = MyDocument.create(
|
|
|
178
177
|
# Parse back to original type
|
|
179
178
|
data = doc.parse(dict) # Returns {"key": "value"}
|
|
180
179
|
|
|
180
|
+
# Document provenance tracking (new in v0.1.14)
|
|
181
|
+
doc_with_sources = MyDocument.create(
|
|
182
|
+
name="derived.json",
|
|
183
|
+
content={"result": "processed"},
|
|
184
|
+
sources=[source_doc.sha256, "https://api.example.com/data"]
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
# Check provenance
|
|
188
|
+
for hash in doc_with_sources.get_source_documents():
|
|
189
|
+
print(f"Derived from document: {hash}")
|
|
190
|
+
for ref in doc_with_sources.get_source_references():
|
|
191
|
+
print(f"External source: {ref}")
|
|
192
|
+
|
|
181
193
|
# Temporary documents (never persisted)
|
|
182
194
|
temp = TemporaryDocument.create(
|
|
183
195
|
name="api_response.json",
|
|
@@ -211,6 +223,10 @@ if doc.is_text:
|
|
|
211
223
|
|
|
212
224
|
# Parse structured data
|
|
213
225
|
data = doc.as_json() # or as_yaml(), as_pydantic_model()
|
|
226
|
+
|
|
227
|
+
# Enhanced filtering (new in v0.1.14)
|
|
228
|
+
filtered = documents.filter_by([Doc1, Doc2, Doc3]) # Multiple types
|
|
229
|
+
named = documents.filter_by(["file1.txt", "file2.txt"]) # Multiple names
|
|
214
230
|
```
|
|
215
231
|
|
|
216
232
|
### LLM Integration
|
|
@@ -233,7 +249,7 @@ static_context = AIMessages([large_document])
|
|
|
233
249
|
# First call: caches context
|
|
234
250
|
r1 = await llm.generate(
|
|
235
251
|
model="gpt-5",
|
|
236
|
-
context=static_context, # Cached for 120 seconds
|
|
252
|
+
context=static_context, # Cached for 120 seconds by default
|
|
237
253
|
messages="Summarize" # Dynamic query
|
|
238
254
|
)
|
|
239
255
|
|
|
@@ -243,6 +259,22 @@ r2 = await llm.generate(
|
|
|
243
259
|
context=static_context, # Reused from cache!
|
|
244
260
|
messages="Key points?" # Different query
|
|
245
261
|
)
|
|
262
|
+
|
|
263
|
+
# Custom cache TTL (new in v0.1.14)
|
|
264
|
+
response = await llm.generate(
|
|
265
|
+
model="gpt-5",
|
|
266
|
+
context=static_context,
|
|
267
|
+
messages="Analyze",
|
|
268
|
+
options=ModelOptions(cache_ttl="300s") # Cache for 5 minutes
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
# Disable caching for dynamic contexts
|
|
272
|
+
response = await llm.generate(
|
|
273
|
+
model="gpt-5",
|
|
274
|
+
context=dynamic_context,
|
|
275
|
+
messages="Process",
|
|
276
|
+
options=ModelOptions(cache_ttl=None) # No caching
|
|
277
|
+
)
|
|
246
278
|
```
|
|
247
279
|
|
|
248
280
|
### Flow Configuration
|
|
@@ -256,15 +288,15 @@ class ProcessingConfig(FlowConfig):
|
|
|
256
288
|
INPUT_DOCUMENT_TYPES = [RawDataDocument]
|
|
257
289
|
OUTPUT_DOCUMENT_TYPE = ProcessedDocument # Must be different!
|
|
258
290
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
291
|
+
# Use in flows for validation
|
|
292
|
+
@pipeline_flow(config=ProcessingConfig)
|
|
293
|
+
async def process(
|
|
294
|
+
project_name: str,
|
|
295
|
+
documents: DocumentList,
|
|
296
|
+
flow_options: FlowOptions
|
|
297
|
+
) -> DocumentList:
|
|
298
|
+
# ... processing logic ...
|
|
299
|
+
return ProcessingConfig.create_and_validate_output(outputs)
|
|
268
300
|
```
|
|
269
301
|
|
|
270
302
|
### Pipeline Decorators
|
|
@@ -272,13 +304,15 @@ class ProcessingConfig(FlowConfig):
|
|
|
272
304
|
Enhanced decorators with built-in tracing and monitoring:
|
|
273
305
|
|
|
274
306
|
```python
|
|
275
|
-
from ai_pipeline_core import pipeline_flow, pipeline_task
|
|
307
|
+
from ai_pipeline_core import pipeline_flow, pipeline_task, set_trace_cost
|
|
276
308
|
|
|
277
309
|
@pipeline_task # Automatic retry, tracing, and monitoring
|
|
278
310
|
async def process_chunk(data: str) -> str:
|
|
279
|
-
|
|
311
|
+
result = await transform(data)
|
|
312
|
+
set_trace_cost(0.05) # Track costs (new in v0.1.14)
|
|
313
|
+
return result
|
|
280
314
|
|
|
281
|
-
@pipeline_flow # Full observability and orchestration
|
|
315
|
+
@pipeline_flow(config=MyFlowConfig) # Full observability and orchestration
|
|
282
316
|
async def main_flow(
|
|
283
317
|
project_name: str,
|
|
284
318
|
documents: DocumentList,
|
|
@@ -304,6 +338,9 @@ LMNR_DEBUG=true # Enable debug traces
|
|
|
304
338
|
# Optional: Orchestration
|
|
305
339
|
PREFECT_API_URL=http://localhost:4200/api
|
|
306
340
|
PREFECT_API_KEY=your-prefect-key
|
|
341
|
+
|
|
342
|
+
# Optional: Storage (for Google Cloud Storage)
|
|
343
|
+
GCS_SERVICE_ACCOUNT_FILE=/path/to/service-account.json # GCS auth file
|
|
307
344
|
```
|
|
308
345
|
|
|
309
346
|
### Settings Management
|
|
@@ -331,7 +368,7 @@ print(settings.app_name)
|
|
|
331
368
|
|
|
332
369
|
### Framework Rules (90% Use Cases)
|
|
333
370
|
|
|
334
|
-
1. **Decorators**: Use `@
|
|
371
|
+
1. **Decorators**: Use `@pipeline_task` WITHOUT parameters, `@pipeline_flow` WITH config
|
|
335
372
|
2. **Logging**: Use `get_pipeline_logger(__name__)` - NEVER `print()` or `logging` module
|
|
336
373
|
3. **LLM calls**: Use `AIMessages` or `str`. Wrap Documents in `AIMessages`
|
|
337
374
|
4. **Options**: Omit `ModelOptions` unless specifically needed (defaults are optimal)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
ai_pipeline_core/__init__.py,sha256=BBZn5MBlfCWAq1nFwNxsKnvBLfmPB43TnSEH7edde64,5720
|
|
2
|
+
ai_pipeline_core/exceptions.py,sha256=vx-XLTw2fJSPs-vwtXVYtqoQUcOc0JeI7UmHqRqQYWU,1569
|
|
3
|
+
ai_pipeline_core/pipeline.py,sha256=z3zTHAvDkXAsTJEzkpw1gXonNH8hioNAN2wUybGa1j0,28372
|
|
4
|
+
ai_pipeline_core/prefect.py,sha256=91ZgLJHsDsRUW77CpNmkKxYs3RCJuucPM3pjKmNBeDg,2199
|
|
5
|
+
ai_pipeline_core/prompt_manager.py,sha256=p7D0vv_nMmny0rmvxrVyYmXPRjmPJo9qI-pRZe4__Bk,11690
|
|
6
|
+
ai_pipeline_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
ai_pipeline_core/settings.py,sha256=-a9jVGg77xifj2SagCR9shXfzXUd-2MlrlquEu4htG8,5035
|
|
8
|
+
ai_pipeline_core/tracing.py,sha256=gy5E4OVr3KX-wZ4zWkk3RSER5Ulw8Q_qyp9YoEMCRj4,21963
|
|
9
|
+
ai_pipeline_core/documents/__init__.py,sha256=WHStvGZiSyybOcMTYxSV24U6MA3Am_0_Az5p-DuMFrk,738
|
|
10
|
+
ai_pipeline_core/documents/document.py,sha256=dCbKcgemW8nWHEUJdK5MqSe3XTIo-voi6td4hqWqvSw,62270
|
|
11
|
+
ai_pipeline_core/documents/document_list.py,sha256=iiW8p5p8PhoUgMJnCnPE5GF5xpf6lWJKIzDyYEf6BkM,13738
|
|
12
|
+
ai_pipeline_core/documents/flow_document.py,sha256=g9wlRJRJgy4RsrrZ_P5Qu6kj0FuUFfhfUsRFgtq4NIM,3918
|
|
13
|
+
ai_pipeline_core/documents/mime_type.py,sha256=DkW88K95el5nAmhC00XLS0G3WpDXgs5IRsBWbKiqG3Y,7995
|
|
14
|
+
ai_pipeline_core/documents/task_document.py,sha256=40tFavBLX3FhK9-CRsuOH-3gUZ0zvEkqv9XcMFr8ySk,4077
|
|
15
|
+
ai_pipeline_core/documents/temporary_document.py,sha256=Sam344Mm5AlZTm3_l01YdDWeF26F6pR2tytGRL1doQY,2711
|
|
16
|
+
ai_pipeline_core/documents/utils.py,sha256=ZyJNjFN7ihWno0K7dJZed7twYmmPLA0z40UzFw1A3A8,5465
|
|
17
|
+
ai_pipeline_core/flow/__init__.py,sha256=2BfWYMOPYW5teGzwo-qzpn_bom1lxxry0bPsjVgcsCk,188
|
|
18
|
+
ai_pipeline_core/flow/config.py,sha256=3PCDph2n8dj-txqAvd9Wflbi_6lmfXFR9rUhM-szGSQ,18887
|
|
19
|
+
ai_pipeline_core/flow/options.py,sha256=2rKR2GifhXcyw8avI_oiEDMLC2jm5Qzpw8z56pbxUMo,2285
|
|
20
|
+
ai_pipeline_core/llm/__init__.py,sha256=tSj3Mll8SebivP4J5khdXhM9fnbujnbRh0i5yQRoDJQ,857
|
|
21
|
+
ai_pipeline_core/llm/ai_messages.py,sha256=eSmMwTqGvtBeMoWuukzciQRDIIAfs-cnEXjlaADIYkw,9027
|
|
22
|
+
ai_pipeline_core/llm/client.py,sha256=pxedLnxb9dEu5I9XHTFgXEYWxMv7HOVHhESxIw1hANA,22946
|
|
23
|
+
ai_pipeline_core/llm/model_options.py,sha256=YT_lHazZPa0IbHOuLbWXerRODEDb62sKFM97olSxcAU,7693
|
|
24
|
+
ai_pipeline_core/llm/model_response.py,sha256=xKJPsqFHtOGfqpKlsGzyBHPbqjEjNfP-Ix3lGVdiTjQ,15289
|
|
25
|
+
ai_pipeline_core/llm/model_types.py,sha256=HrQCe_R86yWv5z_83yB-zoMFp6M5Ee9nSeimZmckqtA,2791
|
|
26
|
+
ai_pipeline_core/logging/__init__.py,sha256=Nz6-ghAoENsgNmLD2ma9TW9M0U2_QfxuQ5DDW6Vt6M0,651
|
|
27
|
+
ai_pipeline_core/logging/logging.yml,sha256=YTW48keO_K5bkkb-KXGM7ZuaYKiquLsjsURei8Ql0V4,1353
|
|
28
|
+
ai_pipeline_core/logging/logging_config.py,sha256=pV2x6GgMPXrzPH27sicCSXfw56beio4C2JKCJ3NsXrg,6207
|
|
29
|
+
ai_pipeline_core/logging/logging_mixin.py,sha256=OTye2pbUbG5oYZkI06TNkGCEa4y0ldePz5IAfdmNUPU,8090
|
|
30
|
+
ai_pipeline_core/simple_runner/__init__.py,sha256=9krT-CcDAZ0jB2MjWqFYhaK5qtUDMpB5qWzjRLa4Zhk,322
|
|
31
|
+
ai_pipeline_core/simple_runner/cli.py,sha256=yVyuxLY2RZvdNwmwT5LCe-km2nQJzWTPI0vSWn4_yms,9344
|
|
32
|
+
ai_pipeline_core/simple_runner/simple_runner.py,sha256=f6cIodYkul-Apu1d63T6kR5DZpiaCWpphUcEPp5XjFo,9102
|
|
33
|
+
ai_pipeline_core/storage/__init__.py,sha256=tcIkjJ3zPBLCyetwiJDewBvS2sbRJrDlBh3gEsQm08E,184
|
|
34
|
+
ai_pipeline_core/storage/storage.py,sha256=ClMr419Y-eU2RuOjZYd51dC0stWQk28Vb56PvQaoUwc,20007
|
|
35
|
+
ai_pipeline_core-0.2.0.dist-info/METADATA,sha256=3U4rWNVFQ_Agwpv6NH2e4k0falKwNpDHn-6GncwbcSs,14556
|
|
36
|
+
ai_pipeline_core-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
37
|
+
ai_pipeline_core-0.2.0.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
|
|
38
|
+
ai_pipeline_core-0.2.0.dist-info/RECORD,,
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
ai_pipeline_core/__init__.py,sha256=l2kbSaXDuBJMgFYHDCHKV5u5QxCB8eSTeGz0AL9lMXc,5396
|
|
2
|
-
ai_pipeline_core/exceptions.py,sha256=vx-XLTw2fJSPs-vwtXVYtqoQUcOc0JeI7UmHqRqQYWU,1569
|
|
3
|
-
ai_pipeline_core/pipeline.py,sha256=EETCEoKkFcVvsQqtVIV5S5DrVII4XPSuCnqF7zA2_jc,28137
|
|
4
|
-
ai_pipeline_core/prefect.py,sha256=CC8qeIpVqzNq8m6YWNIcRYeDEqkcAFiNjFwcuwwKO0k,2064
|
|
5
|
-
ai_pipeline_core/prompt_manager.py,sha256=XZwah5fp3GyZ0e0na_yOs6m4ngCcotysh-K_cU2U978,11572
|
|
6
|
-
ai_pipeline_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
ai_pipeline_core/settings.py,sha256=X8m13zUHWte953l3or45wG8o7ZQ4X-XBe6umk4PlkMQ,4598
|
|
8
|
-
ai_pipeline_core/tracing.py,sha256=ptaJLLWbI6Aq12RYifjChjL5CJByUFFkkF8Vo0u6lu4,18405
|
|
9
|
-
ai_pipeline_core/documents/__init__.py,sha256=FOYBUKipW_uuzFieW3MigvNLEpbCI1jeY9_0VxJsoS0,692
|
|
10
|
-
ai_pipeline_core/documents/document.py,sha256=tDUBootN_hK75k3XASyBxt-ZqtZuynTwgcL_L5ZWN1Q,51521
|
|
11
|
-
ai_pipeline_core/documents/document_list.py,sha256=m8ei2jLTHt47uCKHmn8BuMMbfwPIKH5g9oPO1jvIPmg,8282
|
|
12
|
-
ai_pipeline_core/documents/flow_document.py,sha256=bJxLjvUE6xfR5-JCy1oi7XtZOwLBOSuUBBJjRaQk6TI,4748
|
|
13
|
-
ai_pipeline_core/documents/mime_type.py,sha256=DkW88K95el5nAmhC00XLS0G3WpDXgs5IRsBWbKiqG3Y,7995
|
|
14
|
-
ai_pipeline_core/documents/task_document.py,sha256=HonlS7hO6zisu67JFXu6LjGLyc7BuylUHuP7SMcHCY4,4910
|
|
15
|
-
ai_pipeline_core/documents/temporary_document.py,sha256=cMuNLnRlWrMryzfZXOIrTrZ2blsxjmflnL8O3zicWIk,3240
|
|
16
|
-
ai_pipeline_core/documents/utils.py,sha256=VHq1gAWTMmJNS55HSZ2--sDErE4DVtFsxrv6y2KXmdk,3472
|
|
17
|
-
ai_pipeline_core/flow/__init__.py,sha256=2BfWYMOPYW5teGzwo-qzpn_bom1lxxry0bPsjVgcsCk,188
|
|
18
|
-
ai_pipeline_core/flow/config.py,sha256=4JVc30tztSW9sYufWLN3hx6qSeR1VX31H1aI9I2jIrA,12114
|
|
19
|
-
ai_pipeline_core/flow/options.py,sha256=UiddkxWrXAhsFtOOt06JhttTp-0gejc8kG0K8Falg1c,2297
|
|
20
|
-
ai_pipeline_core/llm/__init__.py,sha256=QWhMVs4OLTgIvOHfxb7AQhpfCXlgGpoGiJ8PSbVyzZs,661
|
|
21
|
-
ai_pipeline_core/llm/ai_messages.py,sha256=YS3tfqivj9kS6sYsDAgw9LHEjg9cKS1SAHviCzkvAos,8492
|
|
22
|
-
ai_pipeline_core/llm/client.py,sha256=8n1AzDLQMzGjLb-kZ3OTdNa4-648M7GhVJfPPx7sjwU,18194
|
|
23
|
-
ai_pipeline_core/llm/model_options.py,sha256=HX3yI9JPCtjR3LIS--Ku9R4YPTfUkqp1kSsMenQejxs,6923
|
|
24
|
-
ai_pipeline_core/llm/model_response.py,sha256=IkNXl7AsEPrHIAhsOncpZ52bkOfLp6e3EaXqdf0LtLI,12701
|
|
25
|
-
ai_pipeline_core/llm/model_types.py,sha256=Jv2h8oMhIDEBTulWbkmhVS4aiMI39Mzlhk_xOlkCQXo,2829
|
|
26
|
-
ai_pipeline_core/logging/__init__.py,sha256=4iXN4jNiOXLfCYGH3wZB0-Zf-SlU-gQ07f1AyP2H5-s,660
|
|
27
|
-
ai_pipeline_core/logging/logging.yml,sha256=YTW48keO_K5bkkb-KXGM7ZuaYKiquLsjsURei8Ql0V4,1353
|
|
28
|
-
ai_pipeline_core/logging/logging_config.py,sha256=QYI-vz9BqNA02RxoIWPdKhomZpZJkXeFINIuu08O3hY,6242
|
|
29
|
-
ai_pipeline_core/logging/logging_mixin.py,sha256=UFd_CfyJ6YP_XVA-CrpAszOr8g1FH8RwRIwiY23kRG0,8131
|
|
30
|
-
ai_pipeline_core/simple_runner/__init__.py,sha256=OXKFOu3rRcqXCWwBBxnZ7Vz8KRFF5g-G3eJq-vm3CUY,521
|
|
31
|
-
ai_pipeline_core/simple_runner/cli.py,sha256=sbIvv_d401o8h-b5JlcIJQhwzte1sttdmUi2a3As-wY,9357
|
|
32
|
-
ai_pipeline_core/simple_runner/simple_runner.py,sha256=Q7PRAgf_VUPS72WV9pER9WT0AQo9r4WbRclsRXXJiW4,14329
|
|
33
|
-
ai_pipeline_core-0.1.13.dist-info/METADATA,sha256=20BrzUHIzJ3qYUQy7lx5bl0mBcFtmhzLSkhjS2moAUU,13192
|
|
34
|
-
ai_pipeline_core-0.1.13.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
35
|
-
ai_pipeline_core-0.1.13.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
|
|
36
|
-
ai_pipeline_core-0.1.13.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|