moose-lib 0.6.94__tar.gz → 0.6.96__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of moose-lib might be problematic. Click here for more details.
- {moose_lib-0.6.94 → moose_lib-0.6.96}/PKG-INFO +1 -1
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/ingest_pipeline.py +28 -5
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib.egg-info/PKG-INFO +1 -1
- {moose_lib-0.6.94 → moose_lib-0.6.96}/README.md +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/__init__.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/blocks.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/clients/__init__.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/clients/redis_client.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/commons.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/config/__init__.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/config/config_file.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/config/runtime.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/data_models.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/__init__.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/_registry.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/consumption.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/ingest_api.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/life_cycle.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/materialized_view.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/olap_table.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/registry.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/sql_resource.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/stream.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/types.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/view.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2/workflow.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/dmv2_serializer.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/internal.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/main.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/query_builder.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/query_param.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/streaming/__init__.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/streaming/streaming_function_runner.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/utilities/__init__.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib/utilities/sql.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib.egg-info/SOURCES.txt +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib.egg-info/dependency_links.txt +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib.egg-info/requires.txt +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/moose_lib.egg-info/top_level.txt +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/setup.cfg +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/setup.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/tests/__init__.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/tests/conftest.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/tests/test_moose.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/tests/test_query_builder.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/tests/test_redis_client.py +0 -0
- {moose_lib-0.6.94 → moose_lib-0.6.96}/tests/test_s3queue_config.py +0 -0
|
@@ -4,8 +4,9 @@ Ingestion Pipeline definitions for Moose Data Model v2 (dmv2).
|
|
|
4
4
|
This module provides classes for defining and configuring complete ingestion pipelines,
|
|
5
5
|
which combine tables, streams, and ingestion APIs into a single cohesive unit.
|
|
6
6
|
"""
|
|
7
|
+
import warnings
|
|
7
8
|
from typing import Any, Optional, Generic, TypeVar
|
|
8
|
-
from pydantic import BaseModel
|
|
9
|
+
from pydantic import BaseModel, model_validator
|
|
9
10
|
|
|
10
11
|
from .types import TypedMooseResource, T
|
|
11
12
|
from .olap_table import OlapTable, OlapConfig
|
|
@@ -23,7 +24,7 @@ class IngestPipelineConfig(BaseModel):
|
|
|
23
24
|
Attributes:
|
|
24
25
|
table: Configuration for the OLAP table component.
|
|
25
26
|
stream: Configuration for the stream component.
|
|
26
|
-
|
|
27
|
+
ingest_api: Configuration for the ingest API component.
|
|
27
28
|
dead_letter_queue: Configuration for the dead letter queue.
|
|
28
29
|
version: Optional version string applied to all created components.
|
|
29
30
|
path: Optional custom path for the ingestion API endpoint.
|
|
@@ -32,12 +33,34 @@ class IngestPipelineConfig(BaseModel):
|
|
|
32
33
|
"""
|
|
33
34
|
table: bool | OlapConfig = True
|
|
34
35
|
stream: bool | StreamConfig = True
|
|
35
|
-
|
|
36
|
+
ingest_api: bool | IngestConfig = True
|
|
36
37
|
dead_letter_queue: bool | StreamConfig = True
|
|
37
38
|
version: Optional[str] = None
|
|
38
39
|
path: Optional[str] = None
|
|
39
40
|
metadata: Optional[dict] = None
|
|
40
41
|
life_cycle: Optional[LifeCycle] = None
|
|
42
|
+
|
|
43
|
+
# Legacy support - will be removed in future version
|
|
44
|
+
ingest: Optional[bool | IngestConfig] = None
|
|
45
|
+
|
|
46
|
+
@model_validator(mode='before')
|
|
47
|
+
@classmethod
|
|
48
|
+
def handle_legacy_ingest_param(cls, data):
|
|
49
|
+
"""Handle backwards compatibility for the deprecated 'ingest' parameter."""
|
|
50
|
+
if isinstance(data, dict) and 'ingest' in data:
|
|
51
|
+
warnings.warn(
|
|
52
|
+
"The 'ingest' parameter is deprecated and will be removed in a future version. "
|
|
53
|
+
"Please use 'ingest_api' instead.",
|
|
54
|
+
DeprecationWarning,
|
|
55
|
+
stacklevel=3
|
|
56
|
+
)
|
|
57
|
+
# If ingest_api is not explicitly set, use the ingest value
|
|
58
|
+
if 'ingest_api' not in data:
|
|
59
|
+
data['ingest_api'] = data['ingest']
|
|
60
|
+
# Remove the legacy parameter
|
|
61
|
+
data = data.copy()
|
|
62
|
+
del data['ingest']
|
|
63
|
+
return data
|
|
41
64
|
|
|
42
65
|
class IngestPipeline(TypedMooseResource, Generic[T]):
|
|
43
66
|
"""Creates and configures a linked Table, Stream, and Ingest API pipeline.
|
|
@@ -150,11 +173,11 @@ class IngestPipeline(TypedMooseResource, Generic[T]):
|
|
|
150
173
|
stream_config.version = config.version
|
|
151
174
|
stream_config.metadata = stream_metadata
|
|
152
175
|
self.stream = Stream(name, stream_config, t=self._t)
|
|
153
|
-
if config.
|
|
176
|
+
if config.ingest_api:
|
|
154
177
|
if self.stream is None:
|
|
155
178
|
raise ValueError("Ingest API needs a stream to write to.")
|
|
156
179
|
ingest_config_dict = (
|
|
157
|
-
IngestConfig() if config.
|
|
180
|
+
IngestConfig() if config.ingest_api is True else config.ingest_api
|
|
158
181
|
).model_dump()
|
|
159
182
|
ingest_config_dict["destination"] = self.stream
|
|
160
183
|
if config.version:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|