moose-lib 0.6.114__tar.gz → 0.6.116__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of moose-lib might be problematic. Click here for more details.
- {moose_lib-0.6.114 → moose_lib-0.6.116}/PKG-INFO +1 -1
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/__init__.py +4 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/blocks.py +98 -2
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/data_models.py +6 -6
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/olap_table.py +6 -1
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/internal.py +153 -31
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/PKG-INFO +1 -1
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/SOURCES.txt +1 -0
- moose_lib-0.6.116/tests/conftest.py +42 -0
- moose_lib-0.6.116/tests/test_olap_table_versioning.py +216 -0
- moose_lib-0.6.114/tests/conftest.py +0 -6
- {moose_lib-0.6.114 → moose_lib-0.6.116}/README.md +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/clients/__init__.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/clients/redis_client.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/commons.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/config/__init__.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/config/config_file.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/config/runtime.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/__init__.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/_registry.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/consumption.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/ingest_api.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/ingest_pipeline.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/life_cycle.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/materialized_view.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/registry.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/sql_resource.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/stream.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/types.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/view.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/workflow.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2_serializer.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/main.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/query_builder.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/query_param.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/streaming/__init__.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/streaming/streaming_function_runner.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/utilities/__init__.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/utilities/sql.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/dependency_links.txt +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/requires.txt +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/top_level.txt +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/setup.cfg +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/setup.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/__init__.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/test_moose.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/test_query_builder.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/test_redis_client.py +0 -0
- {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/test_s3queue_config.py +0 -0
|
@@ -19,6 +19,10 @@ from .blocks import (
|
|
|
19
19
|
ReplacingMergeTreeEngine,
|
|
20
20
|
AggregatingMergeTreeEngine,
|
|
21
21
|
SummingMergeTreeEngine,
|
|
22
|
+
ReplicatedMergeTreeEngine,
|
|
23
|
+
ReplicatedReplacingMergeTreeEngine,
|
|
24
|
+
ReplicatedAggregatingMergeTreeEngine,
|
|
25
|
+
ReplicatedSummingMergeTreeEngine,
|
|
22
26
|
S3QueueEngine,
|
|
23
27
|
EngineConfig,
|
|
24
28
|
# Legacy enum (already exported via .blocks import, but explicit for clarity)
|
|
@@ -14,6 +14,10 @@ class ClickHouseEngines(Enum):
|
|
|
14
14
|
VersionedCollapsingMergeTree = "VersionedCollapsingMergeTree"
|
|
15
15
|
GraphiteMergeTree = "GraphiteMergeTree"
|
|
16
16
|
S3Queue = "S3Queue"
|
|
17
|
+
ReplicatedMergeTree = "ReplicatedMergeTree"
|
|
18
|
+
ReplicatedReplacingMergeTree = "ReplicatedReplacingMergeTree"
|
|
19
|
+
ReplicatedAggregatingMergeTree = "ReplicatedAggregatingMergeTree"
|
|
20
|
+
ReplicatedSummingMergeTree = "ReplicatedSummingMergeTree"
|
|
17
21
|
|
|
18
22
|
# ==========================
|
|
19
23
|
# New Engine Configuration Classes
|
|
@@ -51,8 +55,100 @@ class AggregatingMergeTreeEngine(EngineConfig):
|
|
|
51
55
|
|
|
52
56
|
@dataclass
|
|
53
57
|
class SummingMergeTreeEngine(EngineConfig):
|
|
54
|
-
"""Configuration for SummingMergeTree engine
|
|
55
|
-
|
|
58
|
+
"""Configuration for SummingMergeTree engine
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
columns: Optional list of column names to sum
|
|
62
|
+
"""
|
|
63
|
+
columns: Optional[List[str]] = None
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class ReplicatedMergeTreeEngine(EngineConfig):
|
|
67
|
+
"""Configuration for ReplicatedMergeTree engine (replicated version of MergeTree)
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
keeper_path: Keeper path for replication (e.g., '/clickhouse/tables/{database}/{shard}/table_name')
|
|
71
|
+
Optional: omit for ClickHouse Cloud which manages replication automatically
|
|
72
|
+
replica_name: Replica name (e.g., '{replica}')
|
|
73
|
+
Optional: omit for ClickHouse Cloud which manages replication automatically
|
|
74
|
+
|
|
75
|
+
Note: Both keeper_path and replica_name must be provided together, or both omitted.
|
|
76
|
+
"""
|
|
77
|
+
keeper_path: Optional[str] = None
|
|
78
|
+
replica_name: Optional[str] = None
|
|
79
|
+
|
|
80
|
+
def __post_init__(self):
|
|
81
|
+
# Both must be provided or both must be None
|
|
82
|
+
if (self.keeper_path is None) != (self.replica_name is None):
|
|
83
|
+
raise ValueError("keeper_path and replica_name must both be provided or both be None")
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class ReplicatedReplacingMergeTreeEngine(EngineConfig):
|
|
87
|
+
"""Configuration for ReplicatedReplacingMergeTree engine (replicated version with deduplication)
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
keeper_path: Keeper path for replication (e.g., '/clickhouse/tables/{database}/{shard}/table_name')
|
|
91
|
+
Optional: omit for ClickHouse Cloud which manages replication automatically
|
|
92
|
+
replica_name: Replica name (e.g., '{replica}')
|
|
93
|
+
Optional: omit for ClickHouse Cloud which manages replication automatically
|
|
94
|
+
ver: Optional column name for version tracking
|
|
95
|
+
is_deleted: Optional column name for deletion marking (requires ver)
|
|
96
|
+
|
|
97
|
+
Note: Both keeper_path and replica_name must be provided together, or both omitted.
|
|
98
|
+
"""
|
|
99
|
+
keeper_path: Optional[str] = None
|
|
100
|
+
replica_name: Optional[str] = None
|
|
101
|
+
ver: Optional[str] = None
|
|
102
|
+
is_deleted: Optional[str] = None
|
|
103
|
+
|
|
104
|
+
def __post_init__(self):
|
|
105
|
+
# Both must be provided or both must be None
|
|
106
|
+
if (self.keeper_path is None) != (self.replica_name is None):
|
|
107
|
+
raise ValueError("keeper_path and replica_name must both be provided or both be None")
|
|
108
|
+
if self.is_deleted and not self.ver:
|
|
109
|
+
raise ValueError("is_deleted requires ver to be specified")
|
|
110
|
+
|
|
111
|
+
@dataclass
|
|
112
|
+
class ReplicatedAggregatingMergeTreeEngine(EngineConfig):
|
|
113
|
+
"""Configuration for ReplicatedAggregatingMergeTree engine (replicated version for aggregations)
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
keeper_path: Keeper path for replication (e.g., '/clickhouse/tables/{database}/{shard}/table_name')
|
|
117
|
+
Optional: omit for ClickHouse Cloud which manages replication automatically
|
|
118
|
+
replica_name: Replica name (e.g., '{replica}')
|
|
119
|
+
Optional: omit for ClickHouse Cloud which manages replication automatically
|
|
120
|
+
|
|
121
|
+
Note: Both keeper_path and replica_name must be provided together, or both omitted.
|
|
122
|
+
"""
|
|
123
|
+
keeper_path: Optional[str] = None
|
|
124
|
+
replica_name: Optional[str] = None
|
|
125
|
+
|
|
126
|
+
def __post_init__(self):
|
|
127
|
+
# Both must be provided or both must be None
|
|
128
|
+
if (self.keeper_path is None) != (self.replica_name is None):
|
|
129
|
+
raise ValueError("keeper_path and replica_name must both be provided or both be None")
|
|
130
|
+
|
|
131
|
+
@dataclass
|
|
132
|
+
class ReplicatedSummingMergeTreeEngine(EngineConfig):
|
|
133
|
+
"""Configuration for ReplicatedSummingMergeTree engine (replicated version for summation)
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
keeper_path: Keeper path for replication (e.g., '/clickhouse/tables/{database}/{shard}/table_name')
|
|
137
|
+
Optional: omit for ClickHouse Cloud which manages replication automatically
|
|
138
|
+
replica_name: Replica name (e.g., '{replica}')
|
|
139
|
+
Optional: omit for ClickHouse Cloud which manages replication automatically
|
|
140
|
+
columns: Optional list of column names to sum
|
|
141
|
+
|
|
142
|
+
Note: Both keeper_path and replica_name must be provided together, or both omitted.
|
|
143
|
+
"""
|
|
144
|
+
keeper_path: Optional[str] = None
|
|
145
|
+
replica_name: Optional[str] = None
|
|
146
|
+
columns: Optional[List[str]] = None
|
|
147
|
+
|
|
148
|
+
def __post_init__(self):
|
|
149
|
+
# Both must be provided or both must be None
|
|
150
|
+
if (self.keeper_path is None) != (self.replica_name is None):
|
|
151
|
+
raise ValueError("keeper_path and replica_name must both be provided or both be None")
|
|
56
152
|
|
|
57
153
|
@dataclass
|
|
58
154
|
class S3QueueEngine(EngineConfig):
|
|
@@ -16,17 +16,17 @@ type Key[T: (str, int)] = T
|
|
|
16
16
|
type JWT[T] = T
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
@dataclasses.dataclass # a BaseModel in the annotations will confuse pydantic
|
|
19
|
+
@dataclasses.dataclass(frozen=True) # a BaseModel in the annotations will confuse pydantic
|
|
20
20
|
class ClickhousePrecision:
|
|
21
21
|
precision: int
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
@dataclasses.dataclass
|
|
24
|
+
@dataclasses.dataclass(frozen=True)
|
|
25
25
|
class ClickhouseSize:
|
|
26
26
|
size: int
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
@dataclasses.dataclass
|
|
29
|
+
@dataclasses.dataclass(frozen=True)
|
|
30
30
|
class ClickhouseDefault:
|
|
31
31
|
expression: str
|
|
32
32
|
|
|
@@ -53,13 +53,13 @@ def aggregated[T](
|
|
|
53
53
|
agg_func: str,
|
|
54
54
|
param_types: list[type | GenericAlias | _BaseGenericAlias]
|
|
55
55
|
) -> Type[T]:
|
|
56
|
-
return Annotated[result_type, AggregateFunction(agg_func=agg_func, param_types=param_types)]
|
|
56
|
+
return Annotated[result_type, AggregateFunction(agg_func=agg_func, param_types=tuple(param_types))]
|
|
57
57
|
|
|
58
58
|
|
|
59
|
-
@dataclasses.dataclass
|
|
59
|
+
@dataclasses.dataclass(frozen=True)
|
|
60
60
|
class AggregateFunction:
|
|
61
61
|
agg_func: str
|
|
62
|
-
param_types:
|
|
62
|
+
param_types: tuple[type | GenericAlias | _BaseGenericAlias, ...]
|
|
63
63
|
|
|
64
64
|
def to_dict(self):
|
|
65
65
|
return {
|
|
@@ -149,7 +149,12 @@ class OlapTable(TypedMooseResource, Generic[T]):
|
|
|
149
149
|
self.metadata = config.metadata
|
|
150
150
|
self._column_list = _to_columns(self._t)
|
|
151
151
|
self._cols = Cols(self._column_list)
|
|
152
|
-
|
|
152
|
+
registry_key = f"{name}_{config.version}" if config.version else name
|
|
153
|
+
if registry_key in _tables:
|
|
154
|
+
raise ValueError(
|
|
155
|
+
f"OlapTable with name {name} and version {config.version or 'unversioned'} already exists"
|
|
156
|
+
)
|
|
157
|
+
_tables[registry_key] = self
|
|
153
158
|
|
|
154
159
|
# Check if using legacy enum-based engine configuration
|
|
155
160
|
if config.engine and isinstance(config.engine, ClickHouseEngines):
|
|
@@ -83,6 +83,38 @@ class AggregatingMergeTreeConfigDict(BaseEngineConfigDict):
|
|
|
83
83
|
class SummingMergeTreeConfigDict(BaseEngineConfigDict):
|
|
84
84
|
"""Configuration for SummingMergeTree engine."""
|
|
85
85
|
engine: Literal["SummingMergeTree"] = "SummingMergeTree"
|
|
86
|
+
columns: Optional[List[str]] = None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class ReplicatedMergeTreeConfigDict(BaseEngineConfigDict):
|
|
90
|
+
"""Configuration for ReplicatedMergeTree engine."""
|
|
91
|
+
engine: Literal["ReplicatedMergeTree"] = "ReplicatedMergeTree"
|
|
92
|
+
keeper_path: Optional[str] = None
|
|
93
|
+
replica_name: Optional[str] = None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class ReplicatedReplacingMergeTreeConfigDict(BaseEngineConfigDict):
|
|
97
|
+
"""Configuration for ReplicatedReplacingMergeTree engine."""
|
|
98
|
+
engine: Literal["ReplicatedReplacingMergeTree"] = "ReplicatedReplacingMergeTree"
|
|
99
|
+
keeper_path: Optional[str] = None
|
|
100
|
+
replica_name: Optional[str] = None
|
|
101
|
+
ver: Optional[str] = None
|
|
102
|
+
is_deleted: Optional[str] = None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class ReplicatedAggregatingMergeTreeConfigDict(BaseEngineConfigDict):
|
|
106
|
+
"""Configuration for ReplicatedAggregatingMergeTree engine."""
|
|
107
|
+
engine: Literal["ReplicatedAggregatingMergeTree"] = "ReplicatedAggregatingMergeTree"
|
|
108
|
+
keeper_path: Optional[str] = None
|
|
109
|
+
replica_name: Optional[str] = None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class ReplicatedSummingMergeTreeConfigDict(BaseEngineConfigDict):
|
|
113
|
+
"""Configuration for ReplicatedSummingMergeTree engine."""
|
|
114
|
+
engine: Literal["ReplicatedSummingMergeTree"] = "ReplicatedSummingMergeTree"
|
|
115
|
+
keeper_path: Optional[str] = None
|
|
116
|
+
replica_name: Optional[str] = None
|
|
117
|
+
columns: Optional[List[str]] = None
|
|
86
118
|
|
|
87
119
|
|
|
88
120
|
class S3QueueConfigDict(BaseEngineConfigDict):
|
|
@@ -102,6 +134,10 @@ EngineConfigDict = Union[
|
|
|
102
134
|
ReplacingMergeTreeConfigDict,
|
|
103
135
|
AggregatingMergeTreeConfigDict,
|
|
104
136
|
SummingMergeTreeConfigDict,
|
|
137
|
+
ReplicatedMergeTreeConfigDict,
|
|
138
|
+
ReplicatedReplacingMergeTreeConfigDict,
|
|
139
|
+
ReplicatedAggregatingMergeTreeConfigDict,
|
|
140
|
+
ReplicatedSummingMergeTreeConfigDict,
|
|
105
141
|
S3QueueConfigDict
|
|
106
142
|
]
|
|
107
143
|
|
|
@@ -318,6 +354,110 @@ def _map_sql_resource_ref(r: Any) -> InfrastructureSignatureJson:
|
|
|
318
354
|
raise TypeError(f"Object {r} lacks a 'kind' attribute for dependency mapping.")
|
|
319
355
|
|
|
320
356
|
|
|
357
|
+
def _convert_basic_engine_instance(engine: "EngineConfig") -> Optional[EngineConfigDict]:
|
|
358
|
+
"""Convert basic MergeTree engine instances to config dict.
|
|
359
|
+
|
|
360
|
+
Args:
|
|
361
|
+
engine: An EngineConfig instance
|
|
362
|
+
|
|
363
|
+
Returns:
|
|
364
|
+
EngineConfigDict if matched, None otherwise
|
|
365
|
+
"""
|
|
366
|
+
from moose_lib.blocks import (
|
|
367
|
+
MergeTreeEngine, ReplacingMergeTreeEngine,
|
|
368
|
+
AggregatingMergeTreeEngine, SummingMergeTreeEngine
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
if isinstance(engine, MergeTreeEngine):
|
|
372
|
+
return MergeTreeConfigDict()
|
|
373
|
+
elif isinstance(engine, ReplacingMergeTreeEngine):
|
|
374
|
+
return ReplacingMergeTreeConfigDict(
|
|
375
|
+
ver=engine.ver,
|
|
376
|
+
is_deleted=engine.is_deleted
|
|
377
|
+
)
|
|
378
|
+
elif isinstance(engine, AggregatingMergeTreeEngine):
|
|
379
|
+
return AggregatingMergeTreeConfigDict()
|
|
380
|
+
elif isinstance(engine, SummingMergeTreeEngine):
|
|
381
|
+
return SummingMergeTreeConfigDict(columns=engine.columns)
|
|
382
|
+
return None
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _convert_replicated_engine_instance(engine: "EngineConfig") -> Optional[EngineConfigDict]:
|
|
386
|
+
"""Convert replicated MergeTree engine instances to config dict.
|
|
387
|
+
|
|
388
|
+
Args:
|
|
389
|
+
engine: An EngineConfig instance
|
|
390
|
+
|
|
391
|
+
Returns:
|
|
392
|
+
EngineConfigDict if matched, None otherwise
|
|
393
|
+
"""
|
|
394
|
+
from moose_lib.blocks import (
|
|
395
|
+
ReplicatedMergeTreeEngine, ReplicatedReplacingMergeTreeEngine,
|
|
396
|
+
ReplicatedAggregatingMergeTreeEngine, ReplicatedSummingMergeTreeEngine
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
if isinstance(engine, ReplicatedMergeTreeEngine):
|
|
400
|
+
return ReplicatedMergeTreeConfigDict(
|
|
401
|
+
keeper_path=engine.keeper_path,
|
|
402
|
+
replica_name=engine.replica_name
|
|
403
|
+
)
|
|
404
|
+
elif isinstance(engine, ReplicatedReplacingMergeTreeEngine):
|
|
405
|
+
return ReplicatedReplacingMergeTreeConfigDict(
|
|
406
|
+
keeper_path=engine.keeper_path,
|
|
407
|
+
replica_name=engine.replica_name,
|
|
408
|
+
ver=engine.ver,
|
|
409
|
+
is_deleted=engine.is_deleted
|
|
410
|
+
)
|
|
411
|
+
elif isinstance(engine, ReplicatedAggregatingMergeTreeEngine):
|
|
412
|
+
return ReplicatedAggregatingMergeTreeConfigDict(
|
|
413
|
+
keeper_path=engine.keeper_path,
|
|
414
|
+
replica_name=engine.replica_name
|
|
415
|
+
)
|
|
416
|
+
elif isinstance(engine, ReplicatedSummingMergeTreeEngine):
|
|
417
|
+
return ReplicatedSummingMergeTreeConfigDict(
|
|
418
|
+
keeper_path=engine.keeper_path,
|
|
419
|
+
replica_name=engine.replica_name,
|
|
420
|
+
columns=engine.columns
|
|
421
|
+
)
|
|
422
|
+
return None
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineConfigDict:
|
|
426
|
+
"""Convert an EngineConfig instance to config dict format.
|
|
427
|
+
|
|
428
|
+
Args:
|
|
429
|
+
engine: An EngineConfig instance
|
|
430
|
+
|
|
431
|
+
Returns:
|
|
432
|
+
EngineConfigDict with engine-specific configuration
|
|
433
|
+
"""
|
|
434
|
+
from moose_lib.blocks import S3QueueEngine
|
|
435
|
+
|
|
436
|
+
# Try S3Queue first
|
|
437
|
+
if isinstance(engine, S3QueueEngine):
|
|
438
|
+
return S3QueueConfigDict(
|
|
439
|
+
s3_path=engine.s3_path,
|
|
440
|
+
format=engine.format,
|
|
441
|
+
aws_access_key_id=engine.aws_access_key_id,
|
|
442
|
+
aws_secret_access_key=engine.aws_secret_access_key,
|
|
443
|
+
compression=engine.compression,
|
|
444
|
+
headers=engine.headers
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
# Try basic engines
|
|
448
|
+
basic_config = _convert_basic_engine_instance(engine)
|
|
449
|
+
if basic_config:
|
|
450
|
+
return basic_config
|
|
451
|
+
|
|
452
|
+
# Try replicated engines
|
|
453
|
+
replicated_config = _convert_replicated_engine_instance(engine)
|
|
454
|
+
if replicated_config:
|
|
455
|
+
return replicated_config
|
|
456
|
+
|
|
457
|
+
# Fallback for any other EngineConfig subclass
|
|
458
|
+
return BaseEngineConfigDict(engine=engine.__class__.__name__.replace("Engine", ""))
|
|
459
|
+
|
|
460
|
+
|
|
321
461
|
def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig],
|
|
322
462
|
table: OlapTable) -> EngineConfigDict:
|
|
323
463
|
"""Convert engine enum or EngineConfig instance to new engine config format.
|
|
@@ -330,38 +470,12 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig
|
|
|
330
470
|
EngineConfigDict with engine-specific configuration
|
|
331
471
|
"""
|
|
332
472
|
from moose_lib import ClickHouseEngines
|
|
333
|
-
from moose_lib.blocks import
|
|
334
|
-
EngineConfig, S3QueueEngine, MergeTreeEngine,
|
|
335
|
-
ReplacingMergeTreeEngine, AggregatingMergeTreeEngine,
|
|
336
|
-
SummingMergeTreeEngine
|
|
337
|
-
)
|
|
473
|
+
from moose_lib.blocks import EngineConfig
|
|
338
474
|
from moose_lib.commons import Logger
|
|
339
475
|
|
|
340
476
|
# Check if engine is an EngineConfig instance (new API)
|
|
341
477
|
if isinstance(engine, EngineConfig):
|
|
342
|
-
|
|
343
|
-
return S3QueueConfigDict(
|
|
344
|
-
s3_path=engine.s3_path,
|
|
345
|
-
format=engine.format,
|
|
346
|
-
aws_access_key_id=engine.aws_access_key_id,
|
|
347
|
-
aws_secret_access_key=engine.aws_secret_access_key,
|
|
348
|
-
compression=engine.compression,
|
|
349
|
-
headers=engine.headers
|
|
350
|
-
)
|
|
351
|
-
elif isinstance(engine, ReplacingMergeTreeEngine):
|
|
352
|
-
return ReplacingMergeTreeConfigDict(
|
|
353
|
-
ver=engine.ver,
|
|
354
|
-
is_deleted=engine.is_deleted
|
|
355
|
-
)
|
|
356
|
-
elif isinstance(engine, AggregatingMergeTreeEngine):
|
|
357
|
-
return AggregatingMergeTreeConfigDict()
|
|
358
|
-
elif isinstance(engine, SummingMergeTreeEngine):
|
|
359
|
-
return SummingMergeTreeConfigDict()
|
|
360
|
-
elif isinstance(engine, MergeTreeEngine):
|
|
361
|
-
return MergeTreeConfigDict()
|
|
362
|
-
else:
|
|
363
|
-
# Fallback for any other EngineConfig subclass - use base class
|
|
364
|
-
return BaseEngineConfigDict(engine=engine.__class__.__name__.replace("Engine", ""))
|
|
478
|
+
return _convert_engine_instance_to_config_dict(engine)
|
|
365
479
|
|
|
366
480
|
# Handle legacy enum-based engine configuration
|
|
367
481
|
if isinstance(engine, ClickHouseEngines):
|
|
@@ -393,6 +507,10 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig
|
|
|
393
507
|
"ReplacingMergeTree": ReplacingMergeTreeConfigDict,
|
|
394
508
|
"AggregatingMergeTree": AggregatingMergeTreeConfigDict,
|
|
395
509
|
"SummingMergeTree": SummingMergeTreeConfigDict,
|
|
510
|
+
"ReplicatedMergeTree": ReplicatedMergeTreeConfigDict,
|
|
511
|
+
"ReplicatedReplacingMergeTree": ReplicatedReplacingMergeTreeConfigDict,
|
|
512
|
+
"ReplicatedAggregatingMergeTree": ReplicatedAggregatingMergeTreeConfigDict,
|
|
513
|
+
"ReplicatedSummingMergeTree": ReplicatedSummingMergeTreeConfigDict,
|
|
396
514
|
}
|
|
397
515
|
|
|
398
516
|
config_class = engine_map.get(engine_name)
|
|
@@ -421,7 +539,7 @@ def to_infra_map() -> dict:
|
|
|
421
539
|
sql_resources = {}
|
|
422
540
|
workflows = {}
|
|
423
541
|
|
|
424
|
-
for
|
|
542
|
+
for _registry_key, table in get_tables().items():
|
|
425
543
|
# Convert engine configuration to new format
|
|
426
544
|
engine_config = None
|
|
427
545
|
if table.config.engine:
|
|
@@ -436,8 +554,12 @@ def to_infra_map() -> dict:
|
|
|
436
554
|
if "mode" not in table_settings:
|
|
437
555
|
table_settings["mode"] = "unordered"
|
|
438
556
|
|
|
439
|
-
|
|
440
|
-
name
|
|
557
|
+
id_key = (
|
|
558
|
+
f"{table.name}_{table.config.version}" if table.config.version else table.name
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
tables[id_key] = TableConfig(
|
|
562
|
+
name=table.name,
|
|
441
563
|
columns=table._column_list,
|
|
442
564
|
order_by=table.config.order_by_fields,
|
|
443
565
|
partition_by=table.config.partition_by,
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
# Add the package root to Python path for imports
|
|
6
|
+
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
|
7
|
+
|
|
8
|
+
@pytest.fixture(autouse=True)
|
|
9
|
+
def clear_registries():
|
|
10
|
+
"""Clear all global registries before each test to prevent conflicts."""
|
|
11
|
+
from moose_lib.dmv2._registry import (
|
|
12
|
+
_tables,
|
|
13
|
+
_streams,
|
|
14
|
+
_ingest_apis,
|
|
15
|
+
_apis,
|
|
16
|
+
_api_name_aliases,
|
|
17
|
+
_api_path_map,
|
|
18
|
+
_sql_resources,
|
|
19
|
+
_workflows,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
# Clear all registries
|
|
23
|
+
_tables.clear()
|
|
24
|
+
_streams.clear()
|
|
25
|
+
_ingest_apis.clear()
|
|
26
|
+
_apis.clear()
|
|
27
|
+
_api_name_aliases.clear()
|
|
28
|
+
_api_path_map.clear()
|
|
29
|
+
_sql_resources.clear()
|
|
30
|
+
_workflows.clear()
|
|
31
|
+
|
|
32
|
+
yield
|
|
33
|
+
|
|
34
|
+
# Clean up after test (optional, but good practice)
|
|
35
|
+
_tables.clear()
|
|
36
|
+
_streams.clear()
|
|
37
|
+
_ingest_apis.clear()
|
|
38
|
+
_apis.clear()
|
|
39
|
+
_api_name_aliases.clear()
|
|
40
|
+
_api_path_map.clear()
|
|
41
|
+
_sql_resources.clear()
|
|
42
|
+
_workflows.clear()
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tests for OlapTable versioning functionality.
|
|
3
|
+
|
|
4
|
+
This test module verifies that multiple versions of OlapTables with the same name
|
|
5
|
+
can coexist and that the infrastructure map generation handles versioned keys correctly.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
from moose_lib import OlapTable, OlapConfig, ClickHouseEngines, MergeTreeEngine, ReplacingMergeTreeEngine
|
|
10
|
+
from moose_lib.dmv2.registry import get_tables
|
|
11
|
+
from moose_lib.internal import to_infra_map
|
|
12
|
+
from pydantic import BaseModel
|
|
13
|
+
from typing import Optional
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class UserEvent(BaseModel):
|
|
17
|
+
"""Sample model for testing OlapTable versioning."""
|
|
18
|
+
user_id: str
|
|
19
|
+
event_type: str
|
|
20
|
+
timestamp: float
|
|
21
|
+
metadata: Optional[str] = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class UserEventV2(BaseModel):
|
|
25
|
+
"""Updated model with additional fields for version testing."""
|
|
26
|
+
user_id: str
|
|
27
|
+
event_type: str
|
|
28
|
+
timestamp: float
|
|
29
|
+
metadata: Optional[str] = None
|
|
30
|
+
session_id: str
|
|
31
|
+
user_agent: Optional[str] = None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_multiple_olap_table_versions_can_coexist():
|
|
35
|
+
"""Test that multiple versions of the same table can be registered simultaneously."""
|
|
36
|
+
# Create version 1.0 of the table
|
|
37
|
+
table_v1 = OlapTable[UserEvent](
|
|
38
|
+
"UserEvents",
|
|
39
|
+
OlapConfig(
|
|
40
|
+
version="1.0",
|
|
41
|
+
engine=MergeTreeEngine(),
|
|
42
|
+
order_by_fields=["user_id", "timestamp"]
|
|
43
|
+
)
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# Create version 2.0 of the table with different configuration
|
|
47
|
+
table_v2 = OlapTable[UserEventV2](
|
|
48
|
+
"UserEvents",
|
|
49
|
+
OlapConfig(
|
|
50
|
+
version="2.0",
|
|
51
|
+
engine=ReplacingMergeTreeEngine(),
|
|
52
|
+
order_by_fields=["user_id", "timestamp", "session_id"]
|
|
53
|
+
)
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# Both tables should be registered successfully
|
|
57
|
+
tables = get_tables()
|
|
58
|
+
assert "UserEvents_1.0" in tables
|
|
59
|
+
assert "UserEvents_2.0" in tables
|
|
60
|
+
|
|
61
|
+
# Verify they are different instances
|
|
62
|
+
assert tables["UserEvents_1.0"] is table_v1
|
|
63
|
+
assert tables["UserEvents_2.0"] is table_v2
|
|
64
|
+
|
|
65
|
+
# Verify configurations are different
|
|
66
|
+
assert table_v1.config.version == "1.0"
|
|
67
|
+
assert table_v2.config.version == "2.0"
|
|
68
|
+
assert isinstance(table_v1.config.engine, MergeTreeEngine)
|
|
69
|
+
assert isinstance(table_v2.config.engine, ReplacingMergeTreeEngine)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_unversioned_and_versioned_tables_can_coexist():
|
|
73
|
+
"""Test that unversioned and versioned tables with the same name can coexist."""
|
|
74
|
+
# Create unversioned table
|
|
75
|
+
unversioned_table = OlapTable[UserEvent](
|
|
76
|
+
"EventData",
|
|
77
|
+
OlapConfig(engine=MergeTreeEngine())
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
# Create versioned table with same name
|
|
81
|
+
versioned_table = OlapTable[UserEvent](
|
|
82
|
+
"EventData",
|
|
83
|
+
OlapConfig(
|
|
84
|
+
version="1.5",
|
|
85
|
+
engine=MergeTreeEngine()
|
|
86
|
+
)
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# Both should be registered
|
|
90
|
+
tables = get_tables()
|
|
91
|
+
assert "EventData" in tables # Unversioned
|
|
92
|
+
assert "EventData_1.5" in tables # Versioned
|
|
93
|
+
|
|
94
|
+
assert tables["EventData"] is unversioned_table
|
|
95
|
+
assert tables["EventData_1.5"] is versioned_table
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def test_duplicate_version_registration_fails():
|
|
99
|
+
"""Test that registering the same table name and version twice fails."""
|
|
100
|
+
# Create first table
|
|
101
|
+
OlapTable[UserEvent](
|
|
102
|
+
"DuplicateTest",
|
|
103
|
+
OlapConfig(version="1.0", engine=MergeTreeEngine())
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
# Attempting to create another table with same name and version should fail
|
|
107
|
+
with pytest.raises(ValueError, match="OlapTable with name DuplicateTest and version 1.0 already exists"):
|
|
108
|
+
OlapTable[UserEvent](
|
|
109
|
+
"DuplicateTest",
|
|
110
|
+
OlapConfig(version="1.0", engine=MergeTreeEngine())
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_infrastructure_map_uses_versioned_keys():
|
|
115
|
+
"""Test that infrastructure map generation uses versioned keys for tables."""
|
|
116
|
+
# Create multiple versions of tables
|
|
117
|
+
table_v1 = OlapTable[UserEvent](
|
|
118
|
+
"InfraMapTest",
|
|
119
|
+
OlapConfig(
|
|
120
|
+
version="1.0",
|
|
121
|
+
engine=MergeTreeEngine(),
|
|
122
|
+
order_by_fields=["user_id"]
|
|
123
|
+
)
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
table_v2 = OlapTable[UserEvent](
|
|
127
|
+
"InfraMapTest",
|
|
128
|
+
OlapConfig(
|
|
129
|
+
version="2.0",
|
|
130
|
+
engine=ReplacingMergeTreeEngine(),
|
|
131
|
+
order_by_fields=["user_id", "timestamp"]
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
unversioned_table = OlapTable[UserEvent](
|
|
136
|
+
"UnversionedInfraTest",
|
|
137
|
+
OlapConfig(engine=MergeTreeEngine())
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# Generate infrastructure map
|
|
141
|
+
tables_registry = get_tables()
|
|
142
|
+
infra_map = to_infra_map()
|
|
143
|
+
|
|
144
|
+
# Verify versioned keys are used in infrastructure map
|
|
145
|
+
assert "InfraMapTest_1.0" in infra_map["tables"]
|
|
146
|
+
assert "InfraMapTest_2.0" in infra_map["tables"]
|
|
147
|
+
assert "UnversionedInfraTest" in infra_map["tables"]
|
|
148
|
+
|
|
149
|
+
# Verify table configurations in infra map
|
|
150
|
+
v1_config = infra_map["tables"]["InfraMapTest_1.0"]
|
|
151
|
+
v2_config = infra_map["tables"]["InfraMapTest_2.0"]
|
|
152
|
+
unversioned_config = infra_map["tables"]["UnversionedInfraTest"]
|
|
153
|
+
|
|
154
|
+
assert v1_config["name"] == "InfraMapTest"
|
|
155
|
+
assert v1_config["version"] == "1.0"
|
|
156
|
+
assert v1_config["engineConfig"]["engine"] == "MergeTree"
|
|
157
|
+
|
|
158
|
+
assert v2_config["name"] == "InfraMapTest"
|
|
159
|
+
assert v2_config["version"] == "2.0"
|
|
160
|
+
assert v2_config["engineConfig"]["engine"] == "ReplacingMergeTree"
|
|
161
|
+
|
|
162
|
+
assert unversioned_config["name"] == "UnversionedInfraTest"
|
|
163
|
+
assert unversioned_config.get("version") is None
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_version_with_dots_handled_correctly():
|
|
167
|
+
"""Test that versions with dots are handled correctly in keys."""
|
|
168
|
+
# Create table with semantic version
|
|
169
|
+
table = OlapTable[UserEvent](
|
|
170
|
+
"SemanticVersionTest",
|
|
171
|
+
OlapConfig(
|
|
172
|
+
version="1.2.3",
|
|
173
|
+
engine=MergeTreeEngine()
|
|
174
|
+
)
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
# Should be registered with version in key
|
|
178
|
+
tables = get_tables()
|
|
179
|
+
assert "SemanticVersionTest_1.2.3" in tables
|
|
180
|
+
assert tables["SemanticVersionTest_1.2.3"] is table
|
|
181
|
+
|
|
182
|
+
# Verify in infrastructure map
|
|
183
|
+
infra_map = to_infra_map()
|
|
184
|
+
assert "SemanticVersionTest_1.2.3" in infra_map["tables"]
|
|
185
|
+
|
|
186
|
+
table_config = infra_map["tables"]["SemanticVersionTest_1.2.3"]
|
|
187
|
+
assert table_config["version"] == "1.2.3"
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def test_backward_compatibility_with_legacy_engines():
|
|
191
|
+
"""Test that versioning works with legacy enum-based engine configuration."""
|
|
192
|
+
# Create table with legacy enum engine (should show deprecation warning)
|
|
193
|
+
table = OlapTable[UserEvent](
|
|
194
|
+
"LegacyEngineTest",
|
|
195
|
+
OlapConfig(
|
|
196
|
+
version="1.0",
|
|
197
|
+
engine=ClickHouseEngines.ReplacingMergeTree
|
|
198
|
+
)
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# Should still be registered correctly
|
|
202
|
+
tables = get_tables()
|
|
203
|
+
assert "LegacyEngineTest_1.0" in tables
|
|
204
|
+
assert tables["LegacyEngineTest_1.0"] is table
|
|
205
|
+
|
|
206
|
+
# Should work in infrastructure map
|
|
207
|
+
infra_map = to_infra_map()
|
|
208
|
+
assert "LegacyEngineTest_1.0" in infra_map["tables"]
|
|
209
|
+
|
|
210
|
+
table_config = infra_map["tables"]["LegacyEngineTest_1.0"]
|
|
211
|
+
assert table_config["version"] == "1.0"
|
|
212
|
+
assert table_config["engineConfig"]["engine"] == "ReplacingMergeTree"
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
if __name__ == "__main__":
|
|
216
|
+
pytest.main([__file__, "-v"])
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|