moose-lib 0.6.148.dev3442438466__py3-none-any.whl → 0.6.283__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moose_lib/__init__.py +34 -3
- moose_lib/blocks.py +416 -52
- moose_lib/clients/redis_client.py +26 -14
- moose_lib/commons.py +37 -30
- moose_lib/config/config_file.py +5 -1
- moose_lib/config/runtime.py +73 -34
- moose_lib/data_models.py +331 -61
- moose_lib/dmv2/__init__.py +69 -73
- moose_lib/dmv2/_registry.py +2 -1
- moose_lib/dmv2/_source_capture.py +37 -0
- moose_lib/dmv2/consumption.py +55 -32
- moose_lib/dmv2/ingest_api.py +9 -2
- moose_lib/dmv2/ingest_pipeline.py +35 -16
- moose_lib/dmv2/life_cycle.py +3 -1
- moose_lib/dmv2/materialized_view.py +24 -14
- moose_lib/dmv2/moose_model.py +165 -0
- moose_lib/dmv2/olap_table.py +299 -151
- moose_lib/dmv2/registry.py +18 -3
- moose_lib/dmv2/sql_resource.py +16 -8
- moose_lib/dmv2/stream.py +75 -23
- moose_lib/dmv2/types.py +14 -8
- moose_lib/dmv2/view.py +13 -6
- moose_lib/dmv2/web_app.py +11 -6
- moose_lib/dmv2/web_app_helpers.py +5 -1
- moose_lib/dmv2/workflow.py +37 -9
- moose_lib/internal.py +340 -56
- moose_lib/main.py +87 -56
- moose_lib/query_builder.py +18 -5
- moose_lib/query_param.py +54 -20
- moose_lib/secrets.py +122 -0
- moose_lib/streaming/streaming_function_runner.py +233 -117
- moose_lib/utilities/sql.py +0 -1
- {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/METADATA +18 -1
- moose_lib-0.6.283.dist-info/RECORD +63 -0
- tests/__init__.py +1 -1
- tests/conftest.py +6 -5
- tests/test_backward_compatibility.py +85 -0
- tests/test_cluster_validation.py +85 -0
- tests/test_codec.py +75 -0
- tests/test_column_formatting.py +80 -0
- tests/test_fixedstring.py +43 -0
- tests/test_iceberg_config.py +105 -0
- tests/test_int_types.py +211 -0
- tests/test_kafka_config.py +141 -0
- tests/test_materialized.py +74 -0
- tests/test_metadata.py +37 -0
- tests/test_moose.py +21 -30
- tests/test_moose_model.py +153 -0
- tests/test_olap_table_moosemodel.py +89 -0
- tests/test_olap_table_versioning.py +52 -58
- tests/test_query_builder.py +97 -9
- tests/test_redis_client.py +10 -3
- tests/test_s3queue_config.py +211 -110
- tests/test_secrets.py +239 -0
- tests/test_simple_aggregate.py +42 -40
- tests/test_web_app.py +11 -5
- moose_lib-0.6.148.dev3442438466.dist-info/RECORD +0 -47
- {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/WHEEL +0 -0
- {moose_lib-0.6.148.dev3442438466.dist-info → moose_lib-0.6.283.dist-info}/top_level.txt +0 -0
|
@@ -4,6 +4,7 @@ Materialized View definitions for Moose Data Model v2 (dmv2).
|
|
|
4
4
|
This module provides classes for defining Materialized Views,
|
|
5
5
|
including their SQL statements, target tables, and dependencies.
|
|
6
6
|
"""
|
|
7
|
+
|
|
7
8
|
from typing import Any, Optional, Union, Generic, TypeVar
|
|
8
9
|
from pydantic import BaseModel, ConfigDict, model_validator
|
|
9
10
|
|
|
@@ -29,6 +30,7 @@ class MaterializedViewOptions(BaseModel):
|
|
|
29
30
|
engines like ReplacingMergeTree).
|
|
30
31
|
model_config: ConfigDict for Pydantic validation
|
|
31
32
|
"""
|
|
33
|
+
|
|
32
34
|
select_statement: str
|
|
33
35
|
select_tables: list[Union[OlapTable, SqlResource]]
|
|
34
36
|
# Backward-compatibility: allow specifying just the table_name and engine
|
|
@@ -62,14 +64,15 @@ class MaterializedView(SqlResource, BaseTypedResource, Generic[T]):
|
|
|
62
64
|
pulls_data_from (list[SqlObject]): Source tables/views.
|
|
63
65
|
pushes_data_to (list[SqlObject]): The target table.
|
|
64
66
|
"""
|
|
67
|
+
|
|
65
68
|
target_table: OlapTable[T]
|
|
66
69
|
config: MaterializedViewOptions
|
|
67
70
|
|
|
68
71
|
def __init__(
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
72
|
+
self,
|
|
73
|
+
options: MaterializedViewOptions,
|
|
74
|
+
target_table: Optional[OlapTable[T]] = None,
|
|
75
|
+
**kwargs,
|
|
73
76
|
):
|
|
74
77
|
self._set_type(options.materialized_view_name, self._get_type(kwargs))
|
|
75
78
|
|
|
@@ -77,27 +80,34 @@ class MaterializedView(SqlResource, BaseTypedResource, Generic[T]):
|
|
|
77
80
|
if target_table:
|
|
78
81
|
self.target_table = target_table
|
|
79
82
|
if self._t != target_table._t:
|
|
80
|
-
raise ValueError(
|
|
83
|
+
raise ValueError(
|
|
84
|
+
"Target table must have the same type as the materialized view"
|
|
85
|
+
)
|
|
81
86
|
else:
|
|
82
87
|
# Backward-compatibility path using table_name/engine/order_by_fields
|
|
83
88
|
if not options.table_name:
|
|
84
|
-
raise ValueError(
|
|
89
|
+
raise ValueError(
|
|
90
|
+
"Name of target table is not specified. Provide 'target_table' or 'table_name'."
|
|
91
|
+
)
|
|
85
92
|
target_table = OlapTable(
|
|
86
93
|
name=options.table_name,
|
|
87
94
|
config=OlapConfig(
|
|
88
|
-
order_by_fields=options.order_by_fields or [],
|
|
89
|
-
engine=options.engine
|
|
95
|
+
order_by_fields=options.order_by_fields or [], engine=options.engine
|
|
90
96
|
),
|
|
91
|
-
t=self._t
|
|
97
|
+
t=self._t,
|
|
92
98
|
)
|
|
93
|
-
|
|
99
|
+
|
|
94
100
|
if target_table.name == options.materialized_view_name:
|
|
95
|
-
raise ValueError(
|
|
101
|
+
raise ValueError(
|
|
102
|
+
"Target table name cannot be the same as the materialized view name"
|
|
103
|
+
)
|
|
96
104
|
|
|
97
105
|
setup = [
|
|
98
106
|
f"CREATE MATERIALIZED VIEW IF NOT EXISTS {quote_identifier(options.materialized_view_name)} TO {quote_identifier(target_table.name)} AS {options.select_statement}",
|
|
99
107
|
]
|
|
100
|
-
teardown = [
|
|
108
|
+
teardown = [
|
|
109
|
+
f"DROP VIEW IF EXISTS {quote_identifier(options.materialized_view_name)}"
|
|
110
|
+
]
|
|
101
111
|
|
|
102
112
|
super().__init__(
|
|
103
113
|
options.materialized_view_name,
|
|
@@ -105,8 +115,8 @@ class MaterializedView(SqlResource, BaseTypedResource, Generic[T]):
|
|
|
105
115
|
teardown,
|
|
106
116
|
pulls_data_from=options.select_tables,
|
|
107
117
|
pushes_data_to=[target_table],
|
|
108
|
-
metadata=options.metadata
|
|
118
|
+
metadata=options.metadata,
|
|
109
119
|
)
|
|
110
120
|
|
|
111
121
|
self.target_table = target_table
|
|
112
|
-
self.config = options
|
|
122
|
+
self.config = options
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MooseModel base class for data models with LSP-friendly column access.
|
|
3
|
+
|
|
4
|
+
This module provides MooseModel, a Pydantic BaseModel subclass that adds
|
|
5
|
+
Column descriptors for each field, enabling LSP autocomplete when accessing
|
|
6
|
+
columns for SQL query construction.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
from ..data_models import Column, _to_columns
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ColsNamespace:
|
|
15
|
+
"""
|
|
16
|
+
Namespace object that provides column access via attributes.
|
|
17
|
+
|
|
18
|
+
This is created at the class level for backward compatibility with
|
|
19
|
+
the existing table.cols.field_name pattern.
|
|
20
|
+
|
|
21
|
+
Example:
|
|
22
|
+
>>> class User(MooseModel):
|
|
23
|
+
... user_id: int
|
|
24
|
+
>>> User.cols.user_id # Returns Column object
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, columns: list[Column]):
|
|
28
|
+
"""
|
|
29
|
+
Initialize cols namespace with columns.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
columns: List of Column objects for the model
|
|
33
|
+
"""
|
|
34
|
+
self._columns = {c.name: c for c in columns}
|
|
35
|
+
|
|
36
|
+
# Set each column as an attribute for direct access
|
|
37
|
+
for col in columns:
|
|
38
|
+
setattr(self, col.name, col)
|
|
39
|
+
|
|
40
|
+
def __getitem__(self, item: str) -> Column:
|
|
41
|
+
"""
|
|
42
|
+
Allow bracket notation access to columns.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
item: Column name
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Column object
|
|
49
|
+
|
|
50
|
+
Raises:
|
|
51
|
+
KeyError: If column name not found
|
|
52
|
+
|
|
53
|
+
Example:
|
|
54
|
+
>>> User.cols['user_id'] # Returns Column object
|
|
55
|
+
"""
|
|
56
|
+
if item not in self._columns:
|
|
57
|
+
raise KeyError(f"{item} is not a valid column name")
|
|
58
|
+
return self._columns[item]
|
|
59
|
+
|
|
60
|
+
def __getattr__(self, item: str) -> Column:
|
|
61
|
+
"""
|
|
62
|
+
Fallback for attribute access (shouldn't be needed due to setattr).
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
item: Column name
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
Column object
|
|
69
|
+
|
|
70
|
+
Raises:
|
|
71
|
+
AttributeError: If column name not found
|
|
72
|
+
"""
|
|
73
|
+
if item.startswith("_"):
|
|
74
|
+
# Allow access to private attributes
|
|
75
|
+
raise AttributeError(
|
|
76
|
+
f"'{type(self).__name__}' object has no attribute '{item}'"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
if item in self._columns:
|
|
80
|
+
return self._columns[item]
|
|
81
|
+
raise AttributeError(f"{item} is not a valid column name")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class MooseModelMeta(type(BaseModel)):
|
|
85
|
+
"""
|
|
86
|
+
Metaclass for MooseModel that adds Column descriptors.
|
|
87
|
+
|
|
88
|
+
This metaclass runs after Pydantic's metaclass creates the model class.
|
|
89
|
+
It adds Column objects as class attributes for each model field, enabling:
|
|
90
|
+
1. Direct column access: Model.field_name returns Column object
|
|
91
|
+
2. LSP autocomplete: LSP sees field_name from annotations
|
|
92
|
+
3. Backward compatibility: Model.cols.field_name also works
|
|
93
|
+
|
|
94
|
+
The Column descriptors coexist with Pydantic's instance fields because
|
|
95
|
+
Pydantic separates class-level attributes from instance-level fields.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __new__(mcs, name, bases, namespace, **kwargs):
|
|
99
|
+
"""
|
|
100
|
+
Create new MooseModel class with Column descriptors.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
name: Name of the class being created
|
|
104
|
+
bases: Base classes
|
|
105
|
+
namespace: Class namespace dictionary
|
|
106
|
+
**kwargs: Additional keyword arguments
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
New class with Column descriptors added
|
|
110
|
+
"""
|
|
111
|
+
# Let Pydantic's metaclass create the class first
|
|
112
|
+
cls = super().__new__(mcs, name, bases, namespace, **kwargs)
|
|
113
|
+
|
|
114
|
+
# Skip for MooseModel base class itself
|
|
115
|
+
if name == "MooseModel":
|
|
116
|
+
return cls
|
|
117
|
+
|
|
118
|
+
# Add Column descriptors if this is a model with fields
|
|
119
|
+
if hasattr(cls, "model_fields") and cls.model_fields:
|
|
120
|
+
# Generate columns from model fields
|
|
121
|
+
columns = _to_columns(cls)
|
|
122
|
+
|
|
123
|
+
# Add Column object for each field as class attribute
|
|
124
|
+
# This enables: Model.field_name → Column
|
|
125
|
+
for col in columns:
|
|
126
|
+
setattr(cls, col.name, col)
|
|
127
|
+
|
|
128
|
+
# Add .cols namespace for backward compatibility
|
|
129
|
+
# This enables: Model.cols.field_name → Column
|
|
130
|
+
cls.cols = ColsNamespace(columns)
|
|
131
|
+
|
|
132
|
+
return cls
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class MooseModel(BaseModel, metaclass=MooseModelMeta):
|
|
136
|
+
"""
|
|
137
|
+
Base class for Moose data models with LSP-friendly column access.
|
|
138
|
+
|
|
139
|
+
MooseModel extends Pydantic's BaseModel by adding Column descriptors
|
|
140
|
+
for each field, enabling autocomplete when constructing SQL queries.
|
|
141
|
+
|
|
142
|
+
Usage Patterns:
|
|
143
|
+
|
|
144
|
+
1. Direct column access (NEW - with autocomplete):
|
|
145
|
+
>>> class User(MooseModel):
|
|
146
|
+
... user_id: int
|
|
147
|
+
... email: str
|
|
148
|
+
>>> query = f"SELECT {User.user_id:col}, {User.email:col} FROM users"
|
|
149
|
+
>>> # LSP provides autocomplete for User.user_id
|
|
150
|
+
|
|
151
|
+
2. Legacy .cols access (OLD - backward compatible):
|
|
152
|
+
>>> query = f"SELECT {User.cols.user_id} FROM users"
|
|
153
|
+
>>> # Works but requires type stubs for autocomplete
|
|
154
|
+
|
|
155
|
+
3. Pydantic instance behavior (unchanged):
|
|
156
|
+
>>> user = User(user_id=123, email="test@example.com")
|
|
157
|
+
>>> user.user_id # Returns 123 (int), not Column
|
|
158
|
+
|
|
159
|
+
The metaclass ensures:
|
|
160
|
+
- Class attributes (Model.field) return Column objects
|
|
161
|
+
- Instance attributes (instance.field) return actual values
|
|
162
|
+
- Full Pydantic compatibility maintained
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
pass
|