moose-lib 0.6.148.dev3442438466__py3-none-any.whl → 0.6.283__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. moose_lib/__init__.py +34 -3
  2. moose_lib/blocks.py +416 -52
  3. moose_lib/clients/redis_client.py +26 -14
  4. moose_lib/commons.py +37 -30
  5. moose_lib/config/config_file.py +5 -1
  6. moose_lib/config/runtime.py +73 -34
  7. moose_lib/data_models.py +331 -61
  8. moose_lib/dmv2/__init__.py +69 -73
  9. moose_lib/dmv2/_registry.py +2 -1
  10. moose_lib/dmv2/_source_capture.py +37 -0
  11. moose_lib/dmv2/consumption.py +55 -32
  12. moose_lib/dmv2/ingest_api.py +9 -2
  13. moose_lib/dmv2/ingest_pipeline.py +35 -16
  14. moose_lib/dmv2/life_cycle.py +3 -1
  15. moose_lib/dmv2/materialized_view.py +24 -14
  16. moose_lib/dmv2/moose_model.py +165 -0
  17. moose_lib/dmv2/olap_table.py +299 -151
  18. moose_lib/dmv2/registry.py +18 -3
  19. moose_lib/dmv2/sql_resource.py +16 -8
  20. moose_lib/dmv2/stream.py +75 -23
  21. moose_lib/dmv2/types.py +14 -8
  22. moose_lib/dmv2/view.py +13 -6
  23. moose_lib/dmv2/web_app.py +11 -6
  24. moose_lib/dmv2/web_app_helpers.py +5 -1
  25. moose_lib/dmv2/workflow.py +37 -9
  26. moose_lib/internal.py +340 -56
  27. moose_lib/main.py +87 -56
  28. moose_lib/query_builder.py +18 -5
  29. moose_lib/query_param.py +54 -20
  30. moose_lib/secrets.py +122 -0
  31. moose_lib/streaming/streaming_function_runner.py +233 -117
  32. moose_lib/utilities/sql.py +0 -1
  33. {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/METADATA +18 -1
  34. moose_lib-0.6.283.dist-info/RECORD +63 -0
  35. tests/__init__.py +1 -1
  36. tests/conftest.py +6 -5
  37. tests/test_backward_compatibility.py +85 -0
  38. tests/test_cluster_validation.py +85 -0
  39. tests/test_codec.py +75 -0
  40. tests/test_column_formatting.py +80 -0
  41. tests/test_fixedstring.py +43 -0
  42. tests/test_iceberg_config.py +105 -0
  43. tests/test_int_types.py +211 -0
  44. tests/test_kafka_config.py +141 -0
  45. tests/test_materialized.py +74 -0
  46. tests/test_metadata.py +37 -0
  47. tests/test_moose.py +21 -30
  48. tests/test_moose_model.py +153 -0
  49. tests/test_olap_table_moosemodel.py +89 -0
  50. tests/test_olap_table_versioning.py +52 -58
  51. tests/test_query_builder.py +97 -9
  52. tests/test_redis_client.py +10 -3
  53. tests/test_s3queue_config.py +211 -110
  54. tests/test_secrets.py +239 -0
  55. tests/test_simple_aggregate.py +42 -40
  56. tests/test_web_app.py +11 -5
  57. moose_lib-0.6.148.dev3442438466.dist-info/RECORD +0 -47
  58. {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/WHEEL +0 -0
  59. {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/top_level.txt +0 -0
@@ -4,6 +4,7 @@ Workflow definitions for Moose Data Model v2 (dmv2).
4
4
  This module provides classes for defining and configuring workflows composed of tasks,
5
5
  including task dependencies, configurations, and execution functions.
6
6
  """
7
+
7
8
  import dataclasses
8
9
  from typing import Any, Optional, Dict, List, Callable, Union, Awaitable, Generic
9
10
  from pydantic import BaseModel
@@ -11,6 +12,7 @@ from pydantic import BaseModel
11
12
  from .types import TypedMooseResource, T_none, U_none
12
13
  from ._registry import _workflows
13
14
 
15
+
14
16
  @dataclasses.dataclass
15
17
  class TaskContext(Generic[T_none]):
16
18
  """Context object passed to task handlers.
@@ -18,10 +20,15 @@ class TaskContext(Generic[T_none]):
18
20
  - When a task declares an input model `T`, `input` is of type `T` (not Optional).
19
21
  - For no-input tasks (`T` is `None`), `input` is exactly `None`.
20
22
  """
23
+
21
24
  state: Dict[str, Any]
22
25
  input: T_none
23
26
 
24
- type TaskRunFunc[T_none, U_none] = Callable[[TaskContext[T_none]], Union[U_none, Awaitable[U_none]]]
27
+
28
+ type TaskRunFunc[T_none, U_none] = Callable[
29
+ [TaskContext[T_none]], Union[U_none, Awaitable[U_none]]
30
+ ]
31
+
25
32
 
26
33
  @dataclasses.dataclass
27
34
  class TaskConfig(Generic[T_none, U_none]):
@@ -36,12 +43,16 @@ class TaskConfig(Generic[T_none, U_none]):
36
43
  timeout: Optional timeout string (e.g. "5m", "1h", "never").
37
44
  retries: Optional number of retry attempts.
38
45
  """
46
+
39
47
  run: TaskRunFunc[T_none, U_none]
40
48
  on_complete: Optional[list["Task[U_none, Any]"]] = None
41
- on_cancel: Optional[Callable[[TaskContext[T_none]], Union[None, Awaitable[None]]]] = None
49
+ on_cancel: Optional[
50
+ Callable[[TaskContext[T_none]], Union[None, Awaitable[None]]]
51
+ ] = None
42
52
  timeout: Optional[str] = None
43
53
  retries: Optional[int] = None
44
54
 
55
+
45
56
  class Task(TypedMooseResource, Generic[T_none, U_none]):
46
57
  """Represents a task that can be executed as part of a workflow.
47
58
 
@@ -61,6 +72,7 @@ class Task(TypedMooseResource, Generic[T_none, U_none]):
61
72
  name (str): The name of the task.
62
73
  model_type (type[T]): The Pydantic model associated with this task's input.
63
74
  """
75
+
64
76
  config: TaskConfig[T_none, U_none]
65
77
 
66
78
  def __init__(self, name: str, config: TaskConfig[T_none, U_none], **kwargs):
@@ -70,17 +82,27 @@ class Task(TypedMooseResource, Generic[T_none, U_none]):
70
82
 
71
83
  @classmethod
72
84
  def _get_type(cls, keyword_args: dict):
73
- t = keyword_args.get('t')
85
+ t = keyword_args.get("t")
74
86
  if t is None:
75
- raise ValueError(f"Use `{cls.__name__}[T, U](name='...')` to supply both input and output types")
87
+ raise ValueError(
88
+ f"Use `{cls.__name__}[T, U](name='...')` to supply both input and output types"
89
+ )
76
90
  if not isinstance(t, tuple) or len(t) != 2:
77
- raise ValueError(f"Use `{cls.__name__}[T, U](name='...')` to supply both input and output types")
91
+ raise ValueError(
92
+ f"Use `{cls.__name__}[T, U](name='...')` to supply both input and output types"
93
+ )
78
94
 
79
95
  input_type, output_type = t
80
- if input_type is not None and (not isinstance(input_type, type) or not issubclass(input_type, BaseModel)):
96
+ if input_type is not None and (
97
+ not isinstance(input_type, type) or not issubclass(input_type, BaseModel)
98
+ ):
81
99
  raise ValueError(f"Input type {input_type} is not a Pydantic model or None")
82
- if output_type is not None and (not isinstance(output_type, type) or not issubclass(output_type, BaseModel)):
83
- raise ValueError(f"Output type {output_type} is not a Pydantic model or None")
100
+ if output_type is not None and (
101
+ not isinstance(output_type, type) or not issubclass(output_type, BaseModel)
102
+ ):
103
+ raise ValueError(
104
+ f"Output type {output_type} is not a Pydantic model or None"
105
+ )
84
106
  return t
85
107
 
86
108
  def _set_type(self, name: str, t: tuple[type[T_none], type[U_none]]):
@@ -89,6 +111,7 @@ class Task(TypedMooseResource, Generic[T_none, U_none]):
89
111
  self._u = output_type
90
112
  self.name = name
91
113
 
114
+
92
115
  @dataclasses.dataclass
93
116
  class WorkflowConfig:
94
117
  """Configuration for a workflow.
@@ -99,11 +122,13 @@ class WorkflowConfig:
99
122
  timeout: Optional timeout string for the entire workflow.
100
123
  schedule: Optional cron-like schedule string for recurring execution.
101
124
  """
125
+
102
126
  starting_task: Task[Any, Any]
103
127
  retries: Optional[int] = None
104
128
  timeout: Optional[str] = None
105
129
  schedule: Optional[str] = None
106
130
 
131
+
107
132
  class Workflow:
108
133
  """Represents a workflow composed of one or more tasks.
109
134
 
@@ -118,6 +143,7 @@ class Workflow:
118
143
  name (str): The name of the workflow.
119
144
  config (WorkflowConfig): The configuration for this workflow.
120
145
  """
146
+
121
147
  def __init__(self, name: str, config: WorkflowConfig):
122
148
  self.name = name
123
149
  self.config = config
@@ -130,6 +156,7 @@ class Workflow:
130
156
  Returns:
131
157
  list[str]: List of task names in the workflow, including all child tasks
132
158
  """
159
+
133
160
  def collect_task_names(task: Task) -> list[str]:
134
161
  names = [task.name]
135
162
  if task.config.on_complete:
@@ -148,6 +175,7 @@ class Workflow:
148
175
  Returns:
149
176
  Optional[Task]: The task if found, None otherwise
150
177
  """
178
+
151
179
  def find_task(task: Task) -> Optional[Task]:
152
180
  if task.name == task_name:
153
181
  return task
@@ -158,4 +186,4 @@ class Workflow:
158
186
  return found
159
187
  return None
160
188
 
161
- return find_task(self.config.starting_task)
189
+ return find_task(self.config.starting_task)