moose-lib 0.6.114__tar.gz → 0.6.116__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of moose-lib might be problematic. Click here for more details.

Files changed (49) hide show
  1. {moose_lib-0.6.114 → moose_lib-0.6.116}/PKG-INFO +1 -1
  2. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/__init__.py +4 -0
  3. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/blocks.py +98 -2
  4. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/data_models.py +6 -6
  5. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/olap_table.py +6 -1
  6. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/internal.py +153 -31
  7. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/PKG-INFO +1 -1
  8. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/SOURCES.txt +1 -0
  9. moose_lib-0.6.116/tests/conftest.py +42 -0
  10. moose_lib-0.6.116/tests/test_olap_table_versioning.py +216 -0
  11. moose_lib-0.6.114/tests/conftest.py +0 -6
  12. {moose_lib-0.6.114 → moose_lib-0.6.116}/README.md +0 -0
  13. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/clients/__init__.py +0 -0
  14. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/clients/redis_client.py +0 -0
  15. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/commons.py +0 -0
  16. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/config/__init__.py +0 -0
  17. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/config/config_file.py +0 -0
  18. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/config/runtime.py +0 -0
  19. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/__init__.py +0 -0
  20. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/_registry.py +0 -0
  21. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/consumption.py +0 -0
  22. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/ingest_api.py +0 -0
  23. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/ingest_pipeline.py +0 -0
  24. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/life_cycle.py +0 -0
  25. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/materialized_view.py +0 -0
  26. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/registry.py +0 -0
  27. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/sql_resource.py +0 -0
  28. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/stream.py +0 -0
  29. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/types.py +0 -0
  30. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/view.py +0 -0
  31. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2/workflow.py +0 -0
  32. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/dmv2_serializer.py +0 -0
  33. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/main.py +0 -0
  34. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/query_builder.py +0 -0
  35. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/query_param.py +0 -0
  36. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/streaming/__init__.py +0 -0
  37. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/streaming/streaming_function_runner.py +0 -0
  38. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/utilities/__init__.py +0 -0
  39. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib/utilities/sql.py +0 -0
  40. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/dependency_links.txt +0 -0
  41. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/requires.txt +0 -0
  42. {moose_lib-0.6.114 → moose_lib-0.6.116}/moose_lib.egg-info/top_level.txt +0 -0
  43. {moose_lib-0.6.114 → moose_lib-0.6.116}/setup.cfg +0 -0
  44. {moose_lib-0.6.114 → moose_lib-0.6.116}/setup.py +0 -0
  45. {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/__init__.py +0 -0
  46. {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/test_moose.py +0 -0
  47. {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/test_query_builder.py +0 -0
  48. {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/test_redis_client.py +0 -0
  49. {moose_lib-0.6.114 → moose_lib-0.6.116}/tests/test_s3queue_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: moose_lib
3
- Version: 0.6.114
3
+ Version: 0.6.116
4
4
  Home-page: https://www.fiveonefour.com/moose
5
5
  Author: Fiveonefour Labs Inc.
6
6
  Author-email: support@fiveonefour.com
@@ -19,6 +19,10 @@ from .blocks import (
19
19
  ReplacingMergeTreeEngine,
20
20
  AggregatingMergeTreeEngine,
21
21
  SummingMergeTreeEngine,
22
+ ReplicatedMergeTreeEngine,
23
+ ReplicatedReplacingMergeTreeEngine,
24
+ ReplicatedAggregatingMergeTreeEngine,
25
+ ReplicatedSummingMergeTreeEngine,
22
26
  S3QueueEngine,
23
27
  EngineConfig,
24
28
  # Legacy enum (already exported via .blocks import, but explicit for clarity)
@@ -14,6 +14,10 @@ class ClickHouseEngines(Enum):
14
14
  VersionedCollapsingMergeTree = "VersionedCollapsingMergeTree"
15
15
  GraphiteMergeTree = "GraphiteMergeTree"
16
16
  S3Queue = "S3Queue"
17
+ ReplicatedMergeTree = "ReplicatedMergeTree"
18
+ ReplicatedReplacingMergeTree = "ReplicatedReplacingMergeTree"
19
+ ReplicatedAggregatingMergeTree = "ReplicatedAggregatingMergeTree"
20
+ ReplicatedSummingMergeTree = "ReplicatedSummingMergeTree"
17
21
 
18
22
  # ==========================
19
23
  # New Engine Configuration Classes
@@ -51,8 +55,100 @@ class AggregatingMergeTreeEngine(EngineConfig):
51
55
 
52
56
  @dataclass
53
57
  class SummingMergeTreeEngine(EngineConfig):
54
- """Configuration for SummingMergeTree engine"""
55
- pass
58
+ """Configuration for SummingMergeTree engine
59
+
60
+ Args:
61
+ columns: Optional list of column names to sum
62
+ """
63
+ columns: Optional[List[str]] = None
64
+
65
+ @dataclass
66
+ class ReplicatedMergeTreeEngine(EngineConfig):
67
+ """Configuration for ReplicatedMergeTree engine (replicated version of MergeTree)
68
+
69
+ Args:
70
+ keeper_path: Keeper path for replication (e.g., '/clickhouse/tables/{database}/{shard}/table_name')
71
+ Optional: omit for ClickHouse Cloud which manages replication automatically
72
+ replica_name: Replica name (e.g., '{replica}')
73
+ Optional: omit for ClickHouse Cloud which manages replication automatically
74
+
75
+ Note: Both keeper_path and replica_name must be provided together, or both omitted.
76
+ """
77
+ keeper_path: Optional[str] = None
78
+ replica_name: Optional[str] = None
79
+
80
+ def __post_init__(self):
81
+ # Both must be provided or both must be None
82
+ if (self.keeper_path is None) != (self.replica_name is None):
83
+ raise ValueError("keeper_path and replica_name must both be provided or both be None")
84
+
85
+ @dataclass
86
+ class ReplicatedReplacingMergeTreeEngine(EngineConfig):
87
+ """Configuration for ReplicatedReplacingMergeTree engine (replicated version with deduplication)
88
+
89
+ Args:
90
+ keeper_path: Keeper path for replication (e.g., '/clickhouse/tables/{database}/{shard}/table_name')
91
+ Optional: omit for ClickHouse Cloud which manages replication automatically
92
+ replica_name: Replica name (e.g., '{replica}')
93
+ Optional: omit for ClickHouse Cloud which manages replication automatically
94
+ ver: Optional column name for version tracking
95
+ is_deleted: Optional column name for deletion marking (requires ver)
96
+
97
+ Note: Both keeper_path and replica_name must be provided together, or both omitted.
98
+ """
99
+ keeper_path: Optional[str] = None
100
+ replica_name: Optional[str] = None
101
+ ver: Optional[str] = None
102
+ is_deleted: Optional[str] = None
103
+
104
+ def __post_init__(self):
105
+ # Both must be provided or both must be None
106
+ if (self.keeper_path is None) != (self.replica_name is None):
107
+ raise ValueError("keeper_path and replica_name must both be provided or both be None")
108
+ if self.is_deleted and not self.ver:
109
+ raise ValueError("is_deleted requires ver to be specified")
110
+
111
+ @dataclass
112
+ class ReplicatedAggregatingMergeTreeEngine(EngineConfig):
113
+ """Configuration for ReplicatedAggregatingMergeTree engine (replicated version for aggregations)
114
+
115
+ Args:
116
+ keeper_path: Keeper path for replication (e.g., '/clickhouse/tables/{database}/{shard}/table_name')
117
+ Optional: omit for ClickHouse Cloud which manages replication automatically
118
+ replica_name: Replica name (e.g., '{replica}')
119
+ Optional: omit for ClickHouse Cloud which manages replication automatically
120
+
121
+ Note: Both keeper_path and replica_name must be provided together, or both omitted.
122
+ """
123
+ keeper_path: Optional[str] = None
124
+ replica_name: Optional[str] = None
125
+
126
+ def __post_init__(self):
127
+ # Both must be provided or both must be None
128
+ if (self.keeper_path is None) != (self.replica_name is None):
129
+ raise ValueError("keeper_path and replica_name must both be provided or both be None")
130
+
131
+ @dataclass
132
+ class ReplicatedSummingMergeTreeEngine(EngineConfig):
133
+ """Configuration for ReplicatedSummingMergeTree engine (replicated version for summation)
134
+
135
+ Args:
136
+ keeper_path: Keeper path for replication (e.g., '/clickhouse/tables/{database}/{shard}/table_name')
137
+ Optional: omit for ClickHouse Cloud which manages replication automatically
138
+ replica_name: Replica name (e.g., '{replica}')
139
+ Optional: omit for ClickHouse Cloud which manages replication automatically
140
+ columns: Optional list of column names to sum
141
+
142
+ Note: Both keeper_path and replica_name must be provided together, or both omitted.
143
+ """
144
+ keeper_path: Optional[str] = None
145
+ replica_name: Optional[str] = None
146
+ columns: Optional[List[str]] = None
147
+
148
+ def __post_init__(self):
149
+ # Both must be provided or both must be None
150
+ if (self.keeper_path is None) != (self.replica_name is None):
151
+ raise ValueError("keeper_path and replica_name must both be provided or both be None")
56
152
 
57
153
  @dataclass
58
154
  class S3QueueEngine(EngineConfig):
@@ -16,17 +16,17 @@ type Key[T: (str, int)] = T
16
16
  type JWT[T] = T
17
17
 
18
18
 
19
- @dataclasses.dataclass # a BaseModel in the annotations will confuse pydantic
19
+ @dataclasses.dataclass(frozen=True) # a BaseModel in the annotations will confuse pydantic
20
20
  class ClickhousePrecision:
21
21
  precision: int
22
22
 
23
23
 
24
- @dataclasses.dataclass
24
+ @dataclasses.dataclass(frozen=True)
25
25
  class ClickhouseSize:
26
26
  size: int
27
27
 
28
28
 
29
- @dataclasses.dataclass
29
+ @dataclasses.dataclass(frozen=True)
30
30
  class ClickhouseDefault:
31
31
  expression: str
32
32
 
@@ -53,13 +53,13 @@ def aggregated[T](
53
53
  agg_func: str,
54
54
  param_types: list[type | GenericAlias | _BaseGenericAlias]
55
55
  ) -> Type[T]:
56
- return Annotated[result_type, AggregateFunction(agg_func=agg_func, param_types=param_types)]
56
+ return Annotated[result_type, AggregateFunction(agg_func=agg_func, param_types=tuple(param_types))]
57
57
 
58
58
 
59
- @dataclasses.dataclass
59
+ @dataclasses.dataclass(frozen=True)
60
60
  class AggregateFunction:
61
61
  agg_func: str
62
- param_types: list[type | GenericAlias | _BaseGenericAlias]
62
+ param_types: tuple[type | GenericAlias | _BaseGenericAlias, ...]
63
63
 
64
64
  def to_dict(self):
65
65
  return {
@@ -149,7 +149,12 @@ class OlapTable(TypedMooseResource, Generic[T]):
149
149
  self.metadata = config.metadata
150
150
  self._column_list = _to_columns(self._t)
151
151
  self._cols = Cols(self._column_list)
152
- _tables[name] = self
152
+ registry_key = f"{name}_{config.version}" if config.version else name
153
+ if registry_key in _tables:
154
+ raise ValueError(
155
+ f"OlapTable with name {name} and version {config.version or 'unversioned'} already exists"
156
+ )
157
+ _tables[registry_key] = self
153
158
 
154
159
  # Check if using legacy enum-based engine configuration
155
160
  if config.engine and isinstance(config.engine, ClickHouseEngines):
@@ -83,6 +83,38 @@ class AggregatingMergeTreeConfigDict(BaseEngineConfigDict):
83
83
  class SummingMergeTreeConfigDict(BaseEngineConfigDict):
84
84
  """Configuration for SummingMergeTree engine."""
85
85
  engine: Literal["SummingMergeTree"] = "SummingMergeTree"
86
+ columns: Optional[List[str]] = None
87
+
88
+
89
+ class ReplicatedMergeTreeConfigDict(BaseEngineConfigDict):
90
+ """Configuration for ReplicatedMergeTree engine."""
91
+ engine: Literal["ReplicatedMergeTree"] = "ReplicatedMergeTree"
92
+ keeper_path: Optional[str] = None
93
+ replica_name: Optional[str] = None
94
+
95
+
96
+ class ReplicatedReplacingMergeTreeConfigDict(BaseEngineConfigDict):
97
+ """Configuration for ReplicatedReplacingMergeTree engine."""
98
+ engine: Literal["ReplicatedReplacingMergeTree"] = "ReplicatedReplacingMergeTree"
99
+ keeper_path: Optional[str] = None
100
+ replica_name: Optional[str] = None
101
+ ver: Optional[str] = None
102
+ is_deleted: Optional[str] = None
103
+
104
+
105
+ class ReplicatedAggregatingMergeTreeConfigDict(BaseEngineConfigDict):
106
+ """Configuration for ReplicatedAggregatingMergeTree engine."""
107
+ engine: Literal["ReplicatedAggregatingMergeTree"] = "ReplicatedAggregatingMergeTree"
108
+ keeper_path: Optional[str] = None
109
+ replica_name: Optional[str] = None
110
+
111
+
112
+ class ReplicatedSummingMergeTreeConfigDict(BaseEngineConfigDict):
113
+ """Configuration for ReplicatedSummingMergeTree engine."""
114
+ engine: Literal["ReplicatedSummingMergeTree"] = "ReplicatedSummingMergeTree"
115
+ keeper_path: Optional[str] = None
116
+ replica_name: Optional[str] = None
117
+ columns: Optional[List[str]] = None
86
118
 
87
119
 
88
120
  class S3QueueConfigDict(BaseEngineConfigDict):
@@ -102,6 +134,10 @@ EngineConfigDict = Union[
102
134
  ReplacingMergeTreeConfigDict,
103
135
  AggregatingMergeTreeConfigDict,
104
136
  SummingMergeTreeConfigDict,
137
+ ReplicatedMergeTreeConfigDict,
138
+ ReplicatedReplacingMergeTreeConfigDict,
139
+ ReplicatedAggregatingMergeTreeConfigDict,
140
+ ReplicatedSummingMergeTreeConfigDict,
105
141
  S3QueueConfigDict
106
142
  ]
107
143
 
@@ -318,6 +354,110 @@ def _map_sql_resource_ref(r: Any) -> InfrastructureSignatureJson:
318
354
  raise TypeError(f"Object {r} lacks a 'kind' attribute for dependency mapping.")
319
355
 
320
356
 
357
+ def _convert_basic_engine_instance(engine: "EngineConfig") -> Optional[EngineConfigDict]:
358
+ """Convert basic MergeTree engine instances to config dict.
359
+
360
+ Args:
361
+ engine: An EngineConfig instance
362
+
363
+ Returns:
364
+ EngineConfigDict if matched, None otherwise
365
+ """
366
+ from moose_lib.blocks import (
367
+ MergeTreeEngine, ReplacingMergeTreeEngine,
368
+ AggregatingMergeTreeEngine, SummingMergeTreeEngine
369
+ )
370
+
371
+ if isinstance(engine, MergeTreeEngine):
372
+ return MergeTreeConfigDict()
373
+ elif isinstance(engine, ReplacingMergeTreeEngine):
374
+ return ReplacingMergeTreeConfigDict(
375
+ ver=engine.ver,
376
+ is_deleted=engine.is_deleted
377
+ )
378
+ elif isinstance(engine, AggregatingMergeTreeEngine):
379
+ return AggregatingMergeTreeConfigDict()
380
+ elif isinstance(engine, SummingMergeTreeEngine):
381
+ return SummingMergeTreeConfigDict(columns=engine.columns)
382
+ return None
383
+
384
+
385
+ def _convert_replicated_engine_instance(engine: "EngineConfig") -> Optional[EngineConfigDict]:
386
+ """Convert replicated MergeTree engine instances to config dict.
387
+
388
+ Args:
389
+ engine: An EngineConfig instance
390
+
391
+ Returns:
392
+ EngineConfigDict if matched, None otherwise
393
+ """
394
+ from moose_lib.blocks import (
395
+ ReplicatedMergeTreeEngine, ReplicatedReplacingMergeTreeEngine,
396
+ ReplicatedAggregatingMergeTreeEngine, ReplicatedSummingMergeTreeEngine
397
+ )
398
+
399
+ if isinstance(engine, ReplicatedMergeTreeEngine):
400
+ return ReplicatedMergeTreeConfigDict(
401
+ keeper_path=engine.keeper_path,
402
+ replica_name=engine.replica_name
403
+ )
404
+ elif isinstance(engine, ReplicatedReplacingMergeTreeEngine):
405
+ return ReplicatedReplacingMergeTreeConfigDict(
406
+ keeper_path=engine.keeper_path,
407
+ replica_name=engine.replica_name,
408
+ ver=engine.ver,
409
+ is_deleted=engine.is_deleted
410
+ )
411
+ elif isinstance(engine, ReplicatedAggregatingMergeTreeEngine):
412
+ return ReplicatedAggregatingMergeTreeConfigDict(
413
+ keeper_path=engine.keeper_path,
414
+ replica_name=engine.replica_name
415
+ )
416
+ elif isinstance(engine, ReplicatedSummingMergeTreeEngine):
417
+ return ReplicatedSummingMergeTreeConfigDict(
418
+ keeper_path=engine.keeper_path,
419
+ replica_name=engine.replica_name,
420
+ columns=engine.columns
421
+ )
422
+ return None
423
+
424
+
425
+ def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineConfigDict:
426
+ """Convert an EngineConfig instance to config dict format.
427
+
428
+ Args:
429
+ engine: An EngineConfig instance
430
+
431
+ Returns:
432
+ EngineConfigDict with engine-specific configuration
433
+ """
434
+ from moose_lib.blocks import S3QueueEngine
435
+
436
+ # Try S3Queue first
437
+ if isinstance(engine, S3QueueEngine):
438
+ return S3QueueConfigDict(
439
+ s3_path=engine.s3_path,
440
+ format=engine.format,
441
+ aws_access_key_id=engine.aws_access_key_id,
442
+ aws_secret_access_key=engine.aws_secret_access_key,
443
+ compression=engine.compression,
444
+ headers=engine.headers
445
+ )
446
+
447
+ # Try basic engines
448
+ basic_config = _convert_basic_engine_instance(engine)
449
+ if basic_config:
450
+ return basic_config
451
+
452
+ # Try replicated engines
453
+ replicated_config = _convert_replicated_engine_instance(engine)
454
+ if replicated_config:
455
+ return replicated_config
456
+
457
+ # Fallback for any other EngineConfig subclass
458
+ return BaseEngineConfigDict(engine=engine.__class__.__name__.replace("Engine", ""))
459
+
460
+
321
461
  def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig],
322
462
  table: OlapTable) -> EngineConfigDict:
323
463
  """Convert engine enum or EngineConfig instance to new engine config format.
@@ -330,38 +470,12 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig
330
470
  EngineConfigDict with engine-specific configuration
331
471
  """
332
472
  from moose_lib import ClickHouseEngines
333
- from moose_lib.blocks import (
334
- EngineConfig, S3QueueEngine, MergeTreeEngine,
335
- ReplacingMergeTreeEngine, AggregatingMergeTreeEngine,
336
- SummingMergeTreeEngine
337
- )
473
+ from moose_lib.blocks import EngineConfig
338
474
  from moose_lib.commons import Logger
339
475
 
340
476
  # Check if engine is an EngineConfig instance (new API)
341
477
  if isinstance(engine, EngineConfig):
342
- if isinstance(engine, S3QueueEngine):
343
- return S3QueueConfigDict(
344
- s3_path=engine.s3_path,
345
- format=engine.format,
346
- aws_access_key_id=engine.aws_access_key_id,
347
- aws_secret_access_key=engine.aws_secret_access_key,
348
- compression=engine.compression,
349
- headers=engine.headers
350
- )
351
- elif isinstance(engine, ReplacingMergeTreeEngine):
352
- return ReplacingMergeTreeConfigDict(
353
- ver=engine.ver,
354
- is_deleted=engine.is_deleted
355
- )
356
- elif isinstance(engine, AggregatingMergeTreeEngine):
357
- return AggregatingMergeTreeConfigDict()
358
- elif isinstance(engine, SummingMergeTreeEngine):
359
- return SummingMergeTreeConfigDict()
360
- elif isinstance(engine, MergeTreeEngine):
361
- return MergeTreeConfigDict()
362
- else:
363
- # Fallback for any other EngineConfig subclass - use base class
364
- return BaseEngineConfigDict(engine=engine.__class__.__name__.replace("Engine", ""))
478
+ return _convert_engine_instance_to_config_dict(engine)
365
479
 
366
480
  # Handle legacy enum-based engine configuration
367
481
  if isinstance(engine, ClickHouseEngines):
@@ -393,6 +507,10 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig
393
507
  "ReplacingMergeTree": ReplacingMergeTreeConfigDict,
394
508
  "AggregatingMergeTree": AggregatingMergeTreeConfigDict,
395
509
  "SummingMergeTree": SummingMergeTreeConfigDict,
510
+ "ReplicatedMergeTree": ReplicatedMergeTreeConfigDict,
511
+ "ReplicatedReplacingMergeTree": ReplicatedReplacingMergeTreeConfigDict,
512
+ "ReplicatedAggregatingMergeTree": ReplicatedAggregatingMergeTreeConfigDict,
513
+ "ReplicatedSummingMergeTree": ReplicatedSummingMergeTreeConfigDict,
396
514
  }
397
515
 
398
516
  config_class = engine_map.get(engine_name)
@@ -421,7 +539,7 @@ def to_infra_map() -> dict:
421
539
  sql_resources = {}
422
540
  workflows = {}
423
541
 
424
- for name, table in get_tables().items():
542
+ for _registry_key, table in get_tables().items():
425
543
  # Convert engine configuration to new format
426
544
  engine_config = None
427
545
  if table.config.engine:
@@ -436,8 +554,12 @@ def to_infra_map() -> dict:
436
554
  if "mode" not in table_settings:
437
555
  table_settings["mode"] = "unordered"
438
556
 
439
- tables[name] = TableConfig(
440
- name=name,
557
+ id_key = (
558
+ f"{table.name}_{table.config.version}" if table.config.version else table.name
559
+ )
560
+
561
+ tables[id_key] = TableConfig(
562
+ name=table.name,
441
563
  columns=table._column_list,
442
564
  order_by=table.config.order_by_fields,
443
565
  partition_by=table.config.partition_by,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: moose_lib
3
- Version: 0.6.114
3
+ Version: 0.6.116
4
4
  Home-page: https://www.fiveonefour.com/moose
5
5
  Author: Fiveonefour Labs Inc.
6
6
  Author-email: support@fiveonefour.com
@@ -40,6 +40,7 @@ moose_lib/utilities/sql.py
40
40
  tests/__init__.py
41
41
  tests/conftest.py
42
42
  tests/test_moose.py
43
+ tests/test_olap_table_versioning.py
43
44
  tests/test_query_builder.py
44
45
  tests/test_redis_client.py
45
46
  tests/test_s3queue_config.py
@@ -0,0 +1,42 @@
1
+ import pytest
2
+ import os
3
+ import sys
4
+
5
+ # Add the package root to Python path for imports
6
+ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
7
+
8
+ @pytest.fixture(autouse=True)
9
+ def clear_registries():
10
+ """Clear all global registries before each test to prevent conflicts."""
11
+ from moose_lib.dmv2._registry import (
12
+ _tables,
13
+ _streams,
14
+ _ingest_apis,
15
+ _apis,
16
+ _api_name_aliases,
17
+ _api_path_map,
18
+ _sql_resources,
19
+ _workflows,
20
+ )
21
+
22
+ # Clear all registries
23
+ _tables.clear()
24
+ _streams.clear()
25
+ _ingest_apis.clear()
26
+ _apis.clear()
27
+ _api_name_aliases.clear()
28
+ _api_path_map.clear()
29
+ _sql_resources.clear()
30
+ _workflows.clear()
31
+
32
+ yield
33
+
34
+ # Clean up after test (optional, but good practice)
35
+ _tables.clear()
36
+ _streams.clear()
37
+ _ingest_apis.clear()
38
+ _apis.clear()
39
+ _api_name_aliases.clear()
40
+ _api_path_map.clear()
41
+ _sql_resources.clear()
42
+ _workflows.clear()
@@ -0,0 +1,216 @@
1
+ """
2
+ Tests for OlapTable versioning functionality.
3
+
4
+ This test module verifies that multiple versions of OlapTables with the same name
5
+ can coexist and that the infrastructure map generation handles versioned keys correctly.
6
+ """
7
+
8
+ import pytest
9
+ from moose_lib import OlapTable, OlapConfig, ClickHouseEngines, MergeTreeEngine, ReplacingMergeTreeEngine
10
+ from moose_lib.dmv2.registry import get_tables
11
+ from moose_lib.internal import to_infra_map
12
+ from pydantic import BaseModel
13
+ from typing import Optional
14
+
15
+
16
+ class UserEvent(BaseModel):
17
+ """Sample model for testing OlapTable versioning."""
18
+ user_id: str
19
+ event_type: str
20
+ timestamp: float
21
+ metadata: Optional[str] = None
22
+
23
+
24
+ class UserEventV2(BaseModel):
25
+ """Updated model with additional fields for version testing."""
26
+ user_id: str
27
+ event_type: str
28
+ timestamp: float
29
+ metadata: Optional[str] = None
30
+ session_id: str
31
+ user_agent: Optional[str] = None
32
+
33
+
34
+ def test_multiple_olap_table_versions_can_coexist():
35
+ """Test that multiple versions of the same table can be registered simultaneously."""
36
+ # Create version 1.0 of the table
37
+ table_v1 = OlapTable[UserEvent](
38
+ "UserEvents",
39
+ OlapConfig(
40
+ version="1.0",
41
+ engine=MergeTreeEngine(),
42
+ order_by_fields=["user_id", "timestamp"]
43
+ )
44
+ )
45
+
46
+ # Create version 2.0 of the table with different configuration
47
+ table_v2 = OlapTable[UserEventV2](
48
+ "UserEvents",
49
+ OlapConfig(
50
+ version="2.0",
51
+ engine=ReplacingMergeTreeEngine(),
52
+ order_by_fields=["user_id", "timestamp", "session_id"]
53
+ )
54
+ )
55
+
56
+ # Both tables should be registered successfully
57
+ tables = get_tables()
58
+ assert "UserEvents_1.0" in tables
59
+ assert "UserEvents_2.0" in tables
60
+
61
+ # Verify they are different instances
62
+ assert tables["UserEvents_1.0"] is table_v1
63
+ assert tables["UserEvents_2.0"] is table_v2
64
+
65
+ # Verify configurations are different
66
+ assert table_v1.config.version == "1.0"
67
+ assert table_v2.config.version == "2.0"
68
+ assert isinstance(table_v1.config.engine, MergeTreeEngine)
69
+ assert isinstance(table_v2.config.engine, ReplacingMergeTreeEngine)
70
+
71
+
72
+ def test_unversioned_and_versioned_tables_can_coexist():
73
+ """Test that unversioned and versioned tables with the same name can coexist."""
74
+ # Create unversioned table
75
+ unversioned_table = OlapTable[UserEvent](
76
+ "EventData",
77
+ OlapConfig(engine=MergeTreeEngine())
78
+ )
79
+
80
+ # Create versioned table with same name
81
+ versioned_table = OlapTable[UserEvent](
82
+ "EventData",
83
+ OlapConfig(
84
+ version="1.5",
85
+ engine=MergeTreeEngine()
86
+ )
87
+ )
88
+
89
+ # Both should be registered
90
+ tables = get_tables()
91
+ assert "EventData" in tables # Unversioned
92
+ assert "EventData_1.5" in tables # Versioned
93
+
94
+ assert tables["EventData"] is unversioned_table
95
+ assert tables["EventData_1.5"] is versioned_table
96
+
97
+
98
+ def test_duplicate_version_registration_fails():
99
+ """Test that registering the same table name and version twice fails."""
100
+ # Create first table
101
+ OlapTable[UserEvent](
102
+ "DuplicateTest",
103
+ OlapConfig(version="1.0", engine=MergeTreeEngine())
104
+ )
105
+
106
+ # Attempting to create another table with same name and version should fail
107
+ with pytest.raises(ValueError, match="OlapTable with name DuplicateTest and version 1.0 already exists"):
108
+ OlapTable[UserEvent](
109
+ "DuplicateTest",
110
+ OlapConfig(version="1.0", engine=MergeTreeEngine())
111
+ )
112
+
113
+
114
+ def test_infrastructure_map_uses_versioned_keys():
115
+ """Test that infrastructure map generation uses versioned keys for tables."""
116
+ # Create multiple versions of tables
117
+ table_v1 = OlapTable[UserEvent](
118
+ "InfraMapTest",
119
+ OlapConfig(
120
+ version="1.0",
121
+ engine=MergeTreeEngine(),
122
+ order_by_fields=["user_id"]
123
+ )
124
+ )
125
+
126
+ table_v2 = OlapTable[UserEvent](
127
+ "InfraMapTest",
128
+ OlapConfig(
129
+ version="2.0",
130
+ engine=ReplacingMergeTreeEngine(),
131
+ order_by_fields=["user_id", "timestamp"]
132
+ )
133
+ )
134
+
135
+ unversioned_table = OlapTable[UserEvent](
136
+ "UnversionedInfraTest",
137
+ OlapConfig(engine=MergeTreeEngine())
138
+ )
139
+
140
+ # Generate infrastructure map
141
+ tables_registry = get_tables()
142
+ infra_map = to_infra_map()
143
+
144
+ # Verify versioned keys are used in infrastructure map
145
+ assert "InfraMapTest_1.0" in infra_map["tables"]
146
+ assert "InfraMapTest_2.0" in infra_map["tables"]
147
+ assert "UnversionedInfraTest" in infra_map["tables"]
148
+
149
+ # Verify table configurations in infra map
150
+ v1_config = infra_map["tables"]["InfraMapTest_1.0"]
151
+ v2_config = infra_map["tables"]["InfraMapTest_2.0"]
152
+ unversioned_config = infra_map["tables"]["UnversionedInfraTest"]
153
+
154
+ assert v1_config["name"] == "InfraMapTest"
155
+ assert v1_config["version"] == "1.0"
156
+ assert v1_config["engineConfig"]["engine"] == "MergeTree"
157
+
158
+ assert v2_config["name"] == "InfraMapTest"
159
+ assert v2_config["version"] == "2.0"
160
+ assert v2_config["engineConfig"]["engine"] == "ReplacingMergeTree"
161
+
162
+ assert unversioned_config["name"] == "UnversionedInfraTest"
163
+ assert unversioned_config.get("version") is None
164
+
165
+
166
+ def test_version_with_dots_handled_correctly():
167
+ """Test that versions with dots are handled correctly in keys."""
168
+ # Create table with semantic version
169
+ table = OlapTable[UserEvent](
170
+ "SemanticVersionTest",
171
+ OlapConfig(
172
+ version="1.2.3",
173
+ engine=MergeTreeEngine()
174
+ )
175
+ )
176
+
177
+ # Should be registered with version in key
178
+ tables = get_tables()
179
+ assert "SemanticVersionTest_1.2.3" in tables
180
+ assert tables["SemanticVersionTest_1.2.3"] is table
181
+
182
+ # Verify in infrastructure map
183
+ infra_map = to_infra_map()
184
+ assert "SemanticVersionTest_1.2.3" in infra_map["tables"]
185
+
186
+ table_config = infra_map["tables"]["SemanticVersionTest_1.2.3"]
187
+ assert table_config["version"] == "1.2.3"
188
+
189
+
190
+ def test_backward_compatibility_with_legacy_engines():
191
+ """Test that versioning works with legacy enum-based engine configuration."""
192
+ # Create table with legacy enum engine (should show deprecation warning)
193
+ table = OlapTable[UserEvent](
194
+ "LegacyEngineTest",
195
+ OlapConfig(
196
+ version="1.0",
197
+ engine=ClickHouseEngines.ReplacingMergeTree
198
+ )
199
+ )
200
+
201
+ # Should still be registered correctly
202
+ tables = get_tables()
203
+ assert "LegacyEngineTest_1.0" in tables
204
+ assert tables["LegacyEngineTest_1.0"] is table
205
+
206
+ # Should work in infrastructure map
207
+ infra_map = to_infra_map()
208
+ assert "LegacyEngineTest_1.0" in infra_map["tables"]
209
+
210
+ table_config = infra_map["tables"]["LegacyEngineTest_1.0"]
211
+ assert table_config["version"] == "1.0"
212
+ assert table_config["engineConfig"]["engine"] == "ReplacingMergeTree"
213
+
214
+
215
+ if __name__ == "__main__":
216
+ pytest.main([__file__, "-v"])
@@ -1,6 +0,0 @@
1
- import pytest
2
- import os
3
- import sys
4
-
5
- # Add the package root to Python path for imports
6
- sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
File without changes
File without changes
File without changes