ai-pipeline-core 0.1.10__py3-none-any.whl → 0.1.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_pipeline_core/__init__.py +84 -4
- ai_pipeline_core/documents/__init__.py +9 -0
- ai_pipeline_core/documents/document.py +1044 -152
- ai_pipeline_core/documents/document_list.py +147 -38
- ai_pipeline_core/documents/flow_document.py +112 -11
- ai_pipeline_core/documents/mime_type.py +173 -15
- ai_pipeline_core/documents/task_document.py +117 -12
- ai_pipeline_core/documents/temporary_document.py +84 -5
- ai_pipeline_core/documents/utils.py +41 -9
- ai_pipeline_core/exceptions.py +47 -11
- ai_pipeline_core/flow/__init__.py +2 -0
- ai_pipeline_core/flow/config.py +236 -27
- ai_pipeline_core/flow/options.py +50 -1
- ai_pipeline_core/llm/__init__.py +6 -0
- ai_pipeline_core/llm/ai_messages.py +125 -27
- ai_pipeline_core/llm/client.py +278 -26
- ai_pipeline_core/llm/model_options.py +130 -1
- ai_pipeline_core/llm/model_response.py +239 -35
- ai_pipeline_core/llm/model_types.py +67 -0
- ai_pipeline_core/logging/__init__.py +13 -0
- ai_pipeline_core/logging/logging_config.py +72 -20
- ai_pipeline_core/logging/logging_mixin.py +38 -32
- ai_pipeline_core/pipeline.py +363 -60
- ai_pipeline_core/prefect.py +48 -1
- ai_pipeline_core/prompt_manager.py +209 -24
- ai_pipeline_core/settings.py +108 -4
- ai_pipeline_core/simple_runner/__init__.py +5 -0
- ai_pipeline_core/simple_runner/cli.py +96 -11
- ai_pipeline_core/simple_runner/simple_runner.py +237 -4
- ai_pipeline_core/tracing.py +253 -30
- ai_pipeline_core-0.1.12.dist-info/METADATA +450 -0
- ai_pipeline_core-0.1.12.dist-info/RECORD +36 -0
- ai_pipeline_core-0.1.10.dist-info/METADATA +0 -538
- ai_pipeline_core-0.1.10.dist-info/RECORD +0 -36
- {ai_pipeline_core-0.1.10.dist-info → ai_pipeline_core-0.1.12.dist-info}/WHEEL +0 -0
- {ai_pipeline_core-0.1.10.dist-info → ai_pipeline_core-0.1.12.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
"""Logging mixin for consistent logging across components using Prefect logging
|
|
1
|
+
"""Logging mixin for consistent logging across components using Prefect logging.
|
|
2
|
+
|
|
3
|
+
@public
|
|
4
|
+
"""
|
|
2
5
|
|
|
3
6
|
import contextlib
|
|
4
7
|
import time
|
|
@@ -12,12 +15,17 @@ from prefect.logging import get_logger
|
|
|
12
15
|
|
|
13
16
|
|
|
14
17
|
class LoggerMixin:
|
|
15
|
-
"""
|
|
16
|
-
|
|
18
|
+
"""Mixin class that provides consistent logging functionality using Prefect's logging system.
|
|
19
|
+
|
|
20
|
+
@public
|
|
21
|
+
|
|
22
|
+
Note for users: In your code, always obtain loggers via get_pipeline_logger(__name__).
|
|
23
|
+
The mixin's internal behavior routes to the appropriate backend; you should not call
|
|
24
|
+
logging.getLogger directly.
|
|
17
25
|
|
|
18
26
|
Automatically uses appropriate logger based on context:
|
|
19
|
-
- get_run_logger() when in flow/task context
|
|
20
|
-
-
|
|
27
|
+
- prefect.get_run_logger() when in flow/task context
|
|
28
|
+
- Internal routing when outside flow/task context
|
|
21
29
|
"""
|
|
22
30
|
|
|
23
31
|
_logger_name: Optional[str] = None
|
|
@@ -30,7 +38,11 @@ class LoggerMixin:
|
|
|
30
38
|
return get_logger(self._logger_name or self.__class__.__module__)
|
|
31
39
|
|
|
32
40
|
def _get_run_logger(self):
|
|
33
|
-
"""Attempt to get Prefect run logger.
|
|
41
|
+
"""Attempt to get Prefect run logger.
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
The Prefect run logger if in a flow/task context, None otherwise.
|
|
45
|
+
"""
|
|
34
46
|
# Intentionally broad: Must handle any exception when checking context
|
|
35
47
|
with contextlib.suppress(Exception):
|
|
36
48
|
if FlowRunContext.get() or TaskRunContext.get():
|
|
@@ -38,28 +50,27 @@ class LoggerMixin:
|
|
|
38
50
|
return None
|
|
39
51
|
|
|
40
52
|
def log_debug(self, message: str, **kwargs: Any) -> None:
|
|
41
|
-
"""Log debug message with optional context"""
|
|
53
|
+
"""Log debug message with optional context."""
|
|
42
54
|
self.logger.debug(message, extra=kwargs)
|
|
43
55
|
|
|
44
56
|
def log_info(self, message: str, **kwargs: Any) -> None:
|
|
45
|
-
"""Log info message with optional context"""
|
|
57
|
+
"""Log info message with optional context."""
|
|
46
58
|
self.logger.info(message, extra=kwargs)
|
|
47
59
|
|
|
48
60
|
def log_warning(self, message: str, **kwargs: Any) -> None:
|
|
49
|
-
"""Log warning message with optional context"""
|
|
61
|
+
"""Log warning message with optional context."""
|
|
50
62
|
self.logger.warning(message, extra=kwargs)
|
|
51
63
|
|
|
52
64
|
def log_error(self, message: str, exc_info: bool = False, **kwargs: Any) -> None:
|
|
53
|
-
"""Log error message with optional exception info"""
|
|
65
|
+
"""Log error message with optional exception info."""
|
|
54
66
|
self.logger.error(message, exc_info=exc_info, extra=kwargs)
|
|
55
67
|
|
|
56
68
|
def log_critical(self, message: str, exc_info: bool = False, **kwargs: Any) -> None:
|
|
57
|
-
"""Log critical message with optional exception info"""
|
|
69
|
+
"""Log critical message with optional exception info."""
|
|
58
70
|
self.logger.critical(message, exc_info=exc_info, extra=kwargs)
|
|
59
71
|
|
|
60
72
|
def log_with_context(self, level: str, message: str, context: Dict[str, Any]) -> None:
|
|
61
|
-
"""
|
|
62
|
-
Log message with structured context
|
|
73
|
+
"""Log message with structured context.
|
|
63
74
|
|
|
64
75
|
Args:
|
|
65
76
|
level: Log level (debug, info, warning, error, critical)
|
|
@@ -83,13 +94,13 @@ class LoggerMixin:
|
|
|
83
94
|
|
|
84
95
|
|
|
85
96
|
class StructuredLoggerMixin(LoggerMixin):
|
|
86
|
-
"""
|
|
87
|
-
|
|
97
|
+
"""Extended mixin for structured logging with Prefect.
|
|
98
|
+
|
|
99
|
+
@public
|
|
88
100
|
"""
|
|
89
101
|
|
|
90
102
|
def log_event(self, event: str, **kwargs: Any) -> None:
|
|
91
|
-
"""
|
|
92
|
-
Log a structured event
|
|
103
|
+
"""Log a structured event.
|
|
93
104
|
|
|
94
105
|
Args:
|
|
95
106
|
event: Event name
|
|
@@ -104,8 +115,7 @@ class StructuredLoggerMixin(LoggerMixin):
|
|
|
104
115
|
self.logger.info(event, extra={"event": event, "structured": True, **kwargs})
|
|
105
116
|
|
|
106
117
|
def log_metric(self, metric_name: str, value: float, unit: str = "", **tags: Any) -> None:
|
|
107
|
-
"""
|
|
108
|
-
Log a metric value
|
|
118
|
+
"""Log a metric value.
|
|
109
119
|
|
|
110
120
|
Args:
|
|
111
121
|
metric_name: Name of the metric
|
|
@@ -129,8 +139,7 @@ class StructuredLoggerMixin(LoggerMixin):
|
|
|
129
139
|
)
|
|
130
140
|
|
|
131
141
|
def log_span(self, operation: str, duration_ms: float, **attributes: Any) -> None:
|
|
132
|
-
"""
|
|
133
|
-
Log a span (operation with duration)
|
|
142
|
+
"""Log a span (operation with duration).
|
|
134
143
|
|
|
135
144
|
Args:
|
|
136
145
|
operation: Operation name
|
|
@@ -153,8 +162,7 @@ class StructuredLoggerMixin(LoggerMixin):
|
|
|
153
162
|
|
|
154
163
|
@contextmanager
|
|
155
164
|
def log_operation(self, operation: str, **context: Any) -> Generator[None, None, None]:
|
|
156
|
-
"""
|
|
157
|
-
Context manager for logging operations with timing
|
|
165
|
+
"""Context manager for logging operations with timing.
|
|
158
166
|
|
|
159
167
|
Args:
|
|
160
168
|
operation: Operation name
|
|
@@ -188,36 +196,34 @@ class StructuredLoggerMixin(LoggerMixin):
|
|
|
188
196
|
|
|
189
197
|
|
|
190
198
|
class PrefectLoggerMixin(StructuredLoggerMixin):
|
|
191
|
-
"""
|
|
192
|
-
Enhanced mixin specifically for Prefect flows and tasks
|
|
193
|
-
"""
|
|
199
|
+
"""Enhanced mixin specifically for Prefect flows and tasks."""
|
|
194
200
|
|
|
195
201
|
def log_flow_start(self, flow_name: str, parameters: Dict[str, Any]) -> None:
|
|
196
|
-
"""Log flow start with parameters"""
|
|
202
|
+
"""Log flow start with parameters."""
|
|
197
203
|
self.log_event("flow_started", flow_name=flow_name, parameters=parameters)
|
|
198
204
|
|
|
199
205
|
def log_flow_end(self, flow_name: str, status: str, duration_ms: float) -> None:
|
|
200
|
-
"""Log flow completion"""
|
|
206
|
+
"""Log flow completion."""
|
|
201
207
|
self.log_event(
|
|
202
208
|
"flow_completed", flow_name=flow_name, status=status, duration_ms=duration_ms
|
|
203
209
|
)
|
|
204
210
|
|
|
205
211
|
def log_task_start(self, task_name: str, inputs: Dict[str, Any]) -> None:
|
|
206
|
-
"""Log task start with inputs"""
|
|
212
|
+
"""Log task start with inputs."""
|
|
207
213
|
self.log_event("task_started", task_name=task_name, inputs=inputs)
|
|
208
214
|
|
|
209
215
|
def log_task_end(self, task_name: str, status: str, duration_ms: float) -> None:
|
|
210
|
-
"""Log task completion"""
|
|
216
|
+
"""Log task completion."""
|
|
211
217
|
self.log_event(
|
|
212
218
|
"task_completed", task_name=task_name, status=status, duration_ms=duration_ms
|
|
213
219
|
)
|
|
214
220
|
|
|
215
221
|
def log_retry(self, operation: str, attempt: int, max_attempts: int, error: str) -> None:
|
|
216
|
-
"""Log retry attempt"""
|
|
222
|
+
"""Log retry attempt."""
|
|
217
223
|
self.log_warning(
|
|
218
224
|
f"Retrying {operation}", attempt=attempt, max_attempts=max_attempts, error=error
|
|
219
225
|
)
|
|
220
226
|
|
|
221
227
|
def log_checkpoint(self, checkpoint_name: str, **data: Any) -> None:
|
|
222
|
-
"""Log a checkpoint in processing"""
|
|
228
|
+
"""Log a checkpoint in processing."""
|
|
223
229
|
self.log_info(f"Checkpoint: {checkpoint_name}", checkpoint=checkpoint_name, **data)
|