moose-lib 0.6.72__tar.gz → 0.6.74__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of moose-lib might be problematic. Click here for more details.
- {moose_lib-0.6.72 → moose_lib-0.6.74}/PKG-INFO +1 -1
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/materialized_view.py +1 -1
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/olap_table.py +11 -4
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/types.py +45 -2
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/internal.py +1 -1
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/main.py +35 -8
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib.egg-info/PKG-INFO +1 -1
- {moose_lib-0.6.72 → moose_lib-0.6.74}/README.md +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/__init__.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/blocks.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/clients/__init__.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/clients/redis_client.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/commons.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/config/__init__.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/config/config_file.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/config/runtime.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/data_models.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/__init__.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/_registry.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/consumption.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/ingest_api.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/ingest_pipeline.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/life_cycle.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/registry.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/sql_resource.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/stream.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/view.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2/workflow.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/dmv2_serializer.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/query_param.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/streaming/__init__.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/streaming/streaming_function_runner.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/utilities/__init__.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib/utilities/sql.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib.egg-info/SOURCES.txt +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib.egg-info/dependency_links.txt +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib.egg-info/requires.txt +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/moose_lib.egg-info/top_level.txt +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/setup.cfg +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/setup.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/tests/__init__.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/tests/conftest.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/tests/test_moose.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/tests/test_redis_client.py +0 -0
- {moose_lib-0.6.72 → moose_lib-0.6.74}/tests/test_s3queue_config.py +0 -0
|
@@ -7,7 +7,7 @@ including their SQL statements, target tables, and dependencies.
|
|
|
7
7
|
from typing import Any, Optional, Union, Generic, TypeVar
|
|
8
8
|
from pydantic import BaseModel, ConfigDict, model_validator
|
|
9
9
|
|
|
10
|
-
from
|
|
10
|
+
from ..blocks import ClickHouseEngines
|
|
11
11
|
from ..utilities.sql import quote_identifier
|
|
12
12
|
from .types import BaseTypedResource, T
|
|
13
13
|
from .olap_table import OlapTable, OlapConfig
|
|
@@ -12,12 +12,11 @@ from clickhouse_connect.driver.exceptions import ClickHouseError
|
|
|
12
12
|
from dataclasses import dataclass
|
|
13
13
|
from pydantic import BaseModel
|
|
14
14
|
from typing import List, Optional, Any, Literal, Union, Tuple, TypeVar, Generic, Iterator
|
|
15
|
-
from
|
|
16
|
-
from ..
|
|
17
|
-
from ..commons import Logger # Add import for Moose logging
|
|
15
|
+
from ..blocks import ClickHouseEngines, EngineConfig
|
|
16
|
+
from ..commons import Logger
|
|
18
17
|
from ..config.runtime import RuntimeClickHouseConfig
|
|
19
18
|
from ..utilities.sql import quote_identifier
|
|
20
|
-
from .types import TypedMooseResource, T
|
|
19
|
+
from .types import TypedMooseResource, T, Cols
|
|
21
20
|
from ._registry import _tables
|
|
22
21
|
from ..data_models import Column, is_array_nested_type, is_nested_type, _to_columns
|
|
23
22
|
from .life_cycle import LifeCycle
|
|
@@ -140,12 +139,16 @@ class OlapTable(TypedMooseResource, Generic[T]):
|
|
|
140
139
|
_memoized_client: Optional[Client] = None
|
|
141
140
|
_config_hash: Optional[str] = None
|
|
142
141
|
_cached_table_name: Optional[str] = None
|
|
142
|
+
_column_list: list[Column]
|
|
143
|
+
_cols: Cols
|
|
143
144
|
|
|
144
145
|
def __init__(self, name: str, config: OlapConfig = OlapConfig(), **kwargs):
|
|
145
146
|
super().__init__()
|
|
146
147
|
self._set_type(name, self._get_type(kwargs))
|
|
147
148
|
self.config = config
|
|
148
149
|
self.metadata = config.metadata
|
|
150
|
+
self._column_list = _to_columns(self._t)
|
|
151
|
+
self._cols = Cols(self._column_list)
|
|
149
152
|
_tables[name] = self
|
|
150
153
|
|
|
151
154
|
# Check if using legacy enum-based engine configuration
|
|
@@ -180,6 +183,10 @@ class OlapTable(TypedMooseResource, Generic[T]):
|
|
|
180
183
|
stacklevel=2
|
|
181
184
|
)
|
|
182
185
|
|
|
186
|
+
@property
|
|
187
|
+
def cols(self):
|
|
188
|
+
return self._cols
|
|
189
|
+
|
|
183
190
|
def _generate_table_name(self) -> str:
|
|
184
191
|
"""Generate the versioned table name following Moose's naming convention.
|
|
185
192
|
|
|
@@ -6,8 +6,11 @@ the dmv2 package, including generic type parameters, type aliases, and base
|
|
|
6
6
|
resource classes.
|
|
7
7
|
"""
|
|
8
8
|
from typing import Any, Generic, TypeVar, Union
|
|
9
|
+
|
|
10
|
+
import typing_extensions
|
|
9
11
|
from pydantic import BaseModel
|
|
10
12
|
from pydantic.fields import FieldInfo
|
|
13
|
+
from ..data_models import Column
|
|
11
14
|
|
|
12
15
|
T = TypeVar('T', bound=BaseModel)
|
|
13
16
|
U = TypeVar('U', bound=BaseModel)
|
|
@@ -15,6 +18,41 @@ T_none = TypeVar('T_none', bound=Union[BaseModel, None])
|
|
|
15
18
|
U_none = TypeVar('U_none', bound=Union[BaseModel, None])
|
|
16
19
|
type ZeroOrMany[T] = Union[T, list[T], None]
|
|
17
20
|
|
|
21
|
+
|
|
22
|
+
class Cols:
|
|
23
|
+
"""Provides runtime checked column name access for Moose resources.
|
|
24
|
+
|
|
25
|
+
Instead of using string literals for column names, you can use attribute access
|
|
26
|
+
on this object, which will verify the name against the Pydantic model's fields.
|
|
27
|
+
|
|
28
|
+
Example:
|
|
29
|
+
>>> class MyModel(BaseModel):
|
|
30
|
+
... user_id: int
|
|
31
|
+
... event_name: str
|
|
32
|
+
>>> table = OlapTable[MyModel]("my_models")
|
|
33
|
+
>>> print(table.cols.user_id) # Output: a column object
|
|
34
|
+
>>> print(table.cols.non_existent) # Raises AttributeError
|
|
35
|
+
"""
|
|
36
|
+
_columns: dict[str, Column]
|
|
37
|
+
|
|
38
|
+
def __init__(self, columns: list[Column]):
|
|
39
|
+
self._columns = {c.name: c for c in columns}
|
|
40
|
+
|
|
41
|
+
def __getattr__(self, item: str) -> Column:
|
|
42
|
+
if item in self._columns:
|
|
43
|
+
return self._columns[item]
|
|
44
|
+
raise AttributeError(f"{item} is not a valid column name")
|
|
45
|
+
|
|
46
|
+
def __getitem__(self, item: str) -> Column:
|
|
47
|
+
"""Allow bracket access to columns, equivalent to attribute access.
|
|
48
|
+
|
|
49
|
+
Example:
|
|
50
|
+
table.cols["user_id"] is the same as table.cols.user_id
|
|
51
|
+
"""
|
|
52
|
+
return self.__getattr__(item)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@typing_extensions.deprecated('use cols in OlapTable instead')
|
|
18
56
|
class Columns(Generic[T]):
|
|
19
57
|
"""Provides runtime checked column name access for Moose resources.
|
|
20
58
|
|
|
@@ -42,6 +80,7 @@ class Columns(Generic[T]):
|
|
|
42
80
|
return item # or some Column representation
|
|
43
81
|
raise AttributeError(f"{item} is not a valid column name")
|
|
44
82
|
|
|
83
|
+
|
|
45
84
|
class BaseTypedResource(Generic[T]):
|
|
46
85
|
"""Base class for Moose resources that are typed with a Pydantic model.
|
|
47
86
|
|
|
@@ -79,6 +118,7 @@ class BaseTypedResource(Generic[T]):
|
|
|
79
118
|
|
|
80
119
|
return curried_constructor
|
|
81
120
|
|
|
121
|
+
|
|
82
122
|
class TypedMooseResource(BaseTypedResource, Generic[T]):
|
|
83
123
|
"""Base class for Moose resources that have columns derived from a Pydantic model.
|
|
84
124
|
|
|
@@ -88,8 +128,11 @@ class TypedMooseResource(BaseTypedResource, Generic[T]):
|
|
|
88
128
|
Attributes:
|
|
89
129
|
columns (Columns[T]): An object providing attribute access to column names.
|
|
90
130
|
"""
|
|
91
|
-
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
@typing_extensions.deprecated('use cols in OlapTable instead', category=None)
|
|
134
|
+
def columns(self):
|
|
135
|
+
return Columns[T](self._t)
|
|
92
136
|
|
|
93
137
|
def _set_type(self, name: str, t: type[T]):
|
|
94
138
|
super()._set_type(name, t)
|
|
95
|
-
self.columns = Columns[T](self._t)
|
|
@@ -436,7 +436,7 @@ def to_infra_map() -> dict:
|
|
|
436
436
|
|
|
437
437
|
tables[name] = TableConfig(
|
|
438
438
|
name=name,
|
|
439
|
-
columns=
|
|
439
|
+
columns=table._column_list,
|
|
440
440
|
order_by=table.config.order_by_fields,
|
|
441
441
|
partition_by=table.config.partition_by,
|
|
442
442
|
engine_config=engine_config,
|
|
@@ -6,6 +6,7 @@ and utilities for defining data models and SQL queries.
|
|
|
6
6
|
"""
|
|
7
7
|
from clickhouse_connect.driver.client import Client as ClickhouseClient
|
|
8
8
|
from clickhouse_connect import get_client
|
|
9
|
+
from moose_lib.dmv2 import OlapTable
|
|
9
10
|
from pydantic import BaseModel
|
|
10
11
|
from dataclasses import dataclass, asdict
|
|
11
12
|
from enum import Enum
|
|
@@ -18,9 +19,11 @@ import asyncio
|
|
|
18
19
|
from string import Formatter
|
|
19
20
|
from temporalio.client import Client as TemporalClient, TLSConfig
|
|
20
21
|
from temporalio.common import RetryPolicy, WorkflowIDConflictPolicy, WorkflowIDReusePolicy
|
|
21
|
-
from datetime import timedelta
|
|
22
|
+
from datetime import timedelta, datetime
|
|
22
23
|
from time import perf_counter
|
|
23
24
|
from humanfriendly import format_timespan
|
|
25
|
+
|
|
26
|
+
from .data_models import Column
|
|
24
27
|
from .config.runtime import RuntimeClickHouseConfig
|
|
25
28
|
|
|
26
29
|
from moose_lib.commons import EnhancedJSONEncoder
|
|
@@ -183,7 +186,7 @@ class QueryClient:
|
|
|
183
186
|
|
|
184
187
|
def execute(self, input, variables, row_type: Type[BaseModel] = None):
|
|
185
188
|
params = {}
|
|
186
|
-
values = {}
|
|
189
|
+
values: dict[str, Any] = {}
|
|
187
190
|
preview_params = {}
|
|
188
191
|
|
|
189
192
|
for i, (_, variable_name, _, _) in enumerate(Formatter().parse(input)):
|
|
@@ -193,12 +196,29 @@ class QueryClient:
|
|
|
193
196
|
# handling passing the value of the query string dict directly to variables
|
|
194
197
|
value = value[0]
|
|
195
198
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
199
|
+
if isinstance(value, Column) or isinstance(value, OlapTable):
|
|
200
|
+
params[variable_name] = f'{{p{i}: Identifier}}'
|
|
201
|
+
values[f'p{i}'] = value.name
|
|
202
|
+
else:
|
|
203
|
+
if isinstance(value, bool):
|
|
204
|
+
params[variable_name] = f'{{p{i}: Bool}}'
|
|
205
|
+
values[f'p{i}'] = value
|
|
206
|
+
elif isinstance(value, datetime):
|
|
207
|
+
params[variable_name] = f'{{p{i}: DateTime}}'
|
|
208
|
+
values[f'p{i}'] = value
|
|
209
|
+
elif isinstance(value, int):
|
|
210
|
+
params[variable_name] = f'{{p{i}: Int64}}'
|
|
211
|
+
values[f'p{i}'] = value
|
|
212
|
+
elif isinstance(value, float):
|
|
213
|
+
params[variable_name] = f'{{p{i}: Float64}}'
|
|
214
|
+
values[f'p{i}'] = value
|
|
215
|
+
elif isinstance(value, str):
|
|
216
|
+
params[variable_name] = f'{{p{i}: String}}'
|
|
217
|
+
values[f'p{i}'] = value
|
|
218
|
+
else:
|
|
219
|
+
print(f"unhandled type in QueryClient {type(value)}", file=sys.stderr)
|
|
220
|
+
params[variable_name] = f'{{p{i}: String}}'
|
|
221
|
+
values[f'p{i}'] = str(value)
|
|
202
222
|
preview_params[variable_name] = self._format_value_for_preview(value)
|
|
203
223
|
|
|
204
224
|
clickhouse_query = input.format_map(params)
|
|
@@ -258,6 +278,13 @@ class QueryClient:
|
|
|
258
278
|
escaped = value.replace('\\', '\\\\').replace("'", "\\'")
|
|
259
279
|
return f"'{escaped}'"
|
|
260
280
|
|
|
281
|
+
# DateTime
|
|
282
|
+
if isinstance(value, datetime):
|
|
283
|
+
return f"'{value.strftime('%Y-%m-%d %H:%M:%S')}'"
|
|
284
|
+
|
|
285
|
+
if isinstance(value, Column) or isinstance(value, OlapTable):
|
|
286
|
+
return value.name
|
|
287
|
+
|
|
261
288
|
# Lists / tuples (format as [item1, item2, ...])
|
|
262
289
|
if isinstance(value, (list, tuple)):
|
|
263
290
|
formatted_items = ', '.join(self._format_value_for_preview(v) for v in value)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|