moose-lib 0.6.148.dev3442438466__py3-none-any.whl → 0.6.283__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moose_lib/__init__.py +34 -3
- moose_lib/blocks.py +416 -52
- moose_lib/clients/redis_client.py +26 -14
- moose_lib/commons.py +37 -30
- moose_lib/config/config_file.py +5 -1
- moose_lib/config/runtime.py +73 -34
- moose_lib/data_models.py +331 -61
- moose_lib/dmv2/__init__.py +69 -73
- moose_lib/dmv2/_registry.py +2 -1
- moose_lib/dmv2/_source_capture.py +37 -0
- moose_lib/dmv2/consumption.py +55 -32
- moose_lib/dmv2/ingest_api.py +9 -2
- moose_lib/dmv2/ingest_pipeline.py +35 -16
- moose_lib/dmv2/life_cycle.py +3 -1
- moose_lib/dmv2/materialized_view.py +24 -14
- moose_lib/dmv2/moose_model.py +165 -0
- moose_lib/dmv2/olap_table.py +299 -151
- moose_lib/dmv2/registry.py +18 -3
- moose_lib/dmv2/sql_resource.py +16 -8
- moose_lib/dmv2/stream.py +75 -23
- moose_lib/dmv2/types.py +14 -8
- moose_lib/dmv2/view.py +13 -6
- moose_lib/dmv2/web_app.py +11 -6
- moose_lib/dmv2/web_app_helpers.py +5 -1
- moose_lib/dmv2/workflow.py +37 -9
- moose_lib/internal.py +340 -56
- moose_lib/main.py +87 -56
- moose_lib/query_builder.py +18 -5
- moose_lib/query_param.py +54 -20
- moose_lib/secrets.py +122 -0
- moose_lib/streaming/streaming_function_runner.py +233 -117
- moose_lib/utilities/sql.py +0 -1
- {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/METADATA +18 -1
- moose_lib-0.6.283.dist-info/RECORD +63 -0
- tests/__init__.py +1 -1
- tests/conftest.py +6 -5
- tests/test_backward_compatibility.py +85 -0
- tests/test_cluster_validation.py +85 -0
- tests/test_codec.py +75 -0
- tests/test_column_formatting.py +80 -0
- tests/test_fixedstring.py +43 -0
- tests/test_iceberg_config.py +105 -0
- tests/test_int_types.py +211 -0
- tests/test_kafka_config.py +141 -0
- tests/test_materialized.py +74 -0
- tests/test_metadata.py +37 -0
- tests/test_moose.py +21 -30
- tests/test_moose_model.py +153 -0
- tests/test_olap_table_moosemodel.py +89 -0
- tests/test_olap_table_versioning.py +52 -58
- tests/test_query_builder.py +97 -9
- tests/test_redis_client.py +10 -3
- tests/test_s3queue_config.py +211 -110
- tests/test_secrets.py +239 -0
- tests/test_simple_aggregate.py +42 -40
- tests/test_web_app.py +11 -5
- moose_lib-0.6.148.dev3442438466.dist-info/RECORD +0 -47
- {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/WHEEL +0 -0
- {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/top_level.txt +0 -0
moose_lib/dmv2/workflow.py
CHANGED
|
@@ -4,6 +4,7 @@ Workflow definitions for Moose Data Model v2 (dmv2).
|
|
|
4
4
|
This module provides classes for defining and configuring workflows composed of tasks,
|
|
5
5
|
including task dependencies, configurations, and execution functions.
|
|
6
6
|
"""
|
|
7
|
+
|
|
7
8
|
import dataclasses
|
|
8
9
|
from typing import Any, Optional, Dict, List, Callable, Union, Awaitable, Generic
|
|
9
10
|
from pydantic import BaseModel
|
|
@@ -11,6 +12,7 @@ from pydantic import BaseModel
|
|
|
11
12
|
from .types import TypedMooseResource, T_none, U_none
|
|
12
13
|
from ._registry import _workflows
|
|
13
14
|
|
|
15
|
+
|
|
14
16
|
@dataclasses.dataclass
|
|
15
17
|
class TaskContext(Generic[T_none]):
|
|
16
18
|
"""Context object passed to task handlers.
|
|
@@ -18,10 +20,15 @@ class TaskContext(Generic[T_none]):
|
|
|
18
20
|
- When a task declares an input model `T`, `input` is of type `T` (not Optional).
|
|
19
21
|
- For no-input tasks (`T` is `None`), `input` is exactly `None`.
|
|
20
22
|
"""
|
|
23
|
+
|
|
21
24
|
state: Dict[str, Any]
|
|
22
25
|
input: T_none
|
|
23
26
|
|
|
24
|
-
|
|
27
|
+
|
|
28
|
+
type TaskRunFunc[T_none, U_none] = Callable[
|
|
29
|
+
[TaskContext[T_none]], Union[U_none, Awaitable[U_none]]
|
|
30
|
+
]
|
|
31
|
+
|
|
25
32
|
|
|
26
33
|
@dataclasses.dataclass
|
|
27
34
|
class TaskConfig(Generic[T_none, U_none]):
|
|
@@ -36,12 +43,16 @@ class TaskConfig(Generic[T_none, U_none]):
|
|
|
36
43
|
timeout: Optional timeout string (e.g. "5m", "1h", "never").
|
|
37
44
|
retries: Optional number of retry attempts.
|
|
38
45
|
"""
|
|
46
|
+
|
|
39
47
|
run: TaskRunFunc[T_none, U_none]
|
|
40
48
|
on_complete: Optional[list["Task[U_none, Any]"]] = None
|
|
41
|
-
on_cancel: Optional[
|
|
49
|
+
on_cancel: Optional[
|
|
50
|
+
Callable[[TaskContext[T_none]], Union[None, Awaitable[None]]]
|
|
51
|
+
] = None
|
|
42
52
|
timeout: Optional[str] = None
|
|
43
53
|
retries: Optional[int] = None
|
|
44
54
|
|
|
55
|
+
|
|
45
56
|
class Task(TypedMooseResource, Generic[T_none, U_none]):
|
|
46
57
|
"""Represents a task that can be executed as part of a workflow.
|
|
47
58
|
|
|
@@ -61,6 +72,7 @@ class Task(TypedMooseResource, Generic[T_none, U_none]):
|
|
|
61
72
|
name (str): The name of the task.
|
|
62
73
|
model_type (type[T]): The Pydantic model associated with this task's input.
|
|
63
74
|
"""
|
|
75
|
+
|
|
64
76
|
config: TaskConfig[T_none, U_none]
|
|
65
77
|
|
|
66
78
|
def __init__(self, name: str, config: TaskConfig[T_none, U_none], **kwargs):
|
|
@@ -70,17 +82,27 @@ class Task(TypedMooseResource, Generic[T_none, U_none]):
|
|
|
70
82
|
|
|
71
83
|
@classmethod
|
|
72
84
|
def _get_type(cls, keyword_args: dict):
|
|
73
|
-
t = keyword_args.get(
|
|
85
|
+
t = keyword_args.get("t")
|
|
74
86
|
if t is None:
|
|
75
|
-
raise ValueError(
|
|
87
|
+
raise ValueError(
|
|
88
|
+
f"Use `{cls.__name__}[T, U](name='...')` to supply both input and output types"
|
|
89
|
+
)
|
|
76
90
|
if not isinstance(t, tuple) or len(t) != 2:
|
|
77
|
-
raise ValueError(
|
|
91
|
+
raise ValueError(
|
|
92
|
+
f"Use `{cls.__name__}[T, U](name='...')` to supply both input and output types"
|
|
93
|
+
)
|
|
78
94
|
|
|
79
95
|
input_type, output_type = t
|
|
80
|
-
if input_type is not None and (
|
|
96
|
+
if input_type is not None and (
|
|
97
|
+
not isinstance(input_type, type) or not issubclass(input_type, BaseModel)
|
|
98
|
+
):
|
|
81
99
|
raise ValueError(f"Input type {input_type} is not a Pydantic model or None")
|
|
82
|
-
if output_type is not None and (
|
|
83
|
-
|
|
100
|
+
if output_type is not None and (
|
|
101
|
+
not isinstance(output_type, type) or not issubclass(output_type, BaseModel)
|
|
102
|
+
):
|
|
103
|
+
raise ValueError(
|
|
104
|
+
f"Output type {output_type} is not a Pydantic model or None"
|
|
105
|
+
)
|
|
84
106
|
return t
|
|
85
107
|
|
|
86
108
|
def _set_type(self, name: str, t: tuple[type[T_none], type[U_none]]):
|
|
@@ -89,6 +111,7 @@ class Task(TypedMooseResource, Generic[T_none, U_none]):
|
|
|
89
111
|
self._u = output_type
|
|
90
112
|
self.name = name
|
|
91
113
|
|
|
114
|
+
|
|
92
115
|
@dataclasses.dataclass
|
|
93
116
|
class WorkflowConfig:
|
|
94
117
|
"""Configuration for a workflow.
|
|
@@ -99,11 +122,13 @@ class WorkflowConfig:
|
|
|
99
122
|
timeout: Optional timeout string for the entire workflow.
|
|
100
123
|
schedule: Optional cron-like schedule string for recurring execution.
|
|
101
124
|
"""
|
|
125
|
+
|
|
102
126
|
starting_task: Task[Any, Any]
|
|
103
127
|
retries: Optional[int] = None
|
|
104
128
|
timeout: Optional[str] = None
|
|
105
129
|
schedule: Optional[str] = None
|
|
106
130
|
|
|
131
|
+
|
|
107
132
|
class Workflow:
|
|
108
133
|
"""Represents a workflow composed of one or more tasks.
|
|
109
134
|
|
|
@@ -118,6 +143,7 @@ class Workflow:
|
|
|
118
143
|
name (str): The name of the workflow.
|
|
119
144
|
config (WorkflowConfig): The configuration for this workflow.
|
|
120
145
|
"""
|
|
146
|
+
|
|
121
147
|
def __init__(self, name: str, config: WorkflowConfig):
|
|
122
148
|
self.name = name
|
|
123
149
|
self.config = config
|
|
@@ -130,6 +156,7 @@ class Workflow:
|
|
|
130
156
|
Returns:
|
|
131
157
|
list[str]: List of task names in the workflow, including all child tasks
|
|
132
158
|
"""
|
|
159
|
+
|
|
133
160
|
def collect_task_names(task: Task) -> list[str]:
|
|
134
161
|
names = [task.name]
|
|
135
162
|
if task.config.on_complete:
|
|
@@ -148,6 +175,7 @@ class Workflow:
|
|
|
148
175
|
Returns:
|
|
149
176
|
Optional[Task]: The task if found, None otherwise
|
|
150
177
|
"""
|
|
178
|
+
|
|
151
179
|
def find_task(task: Task) -> Optional[Task]:
|
|
152
180
|
if task.name == task_name:
|
|
153
181
|
return task
|
|
@@ -158,4 +186,4 @@ class Workflow:
|
|
|
158
186
|
return found
|
|
159
187
|
return None
|
|
160
188
|
|
|
161
|
-
return find_task(self.config.starting_task)
|
|
189
|
+
return find_task(self.config.starting_task)
|