moose-lib 0.6.148.dev3442438466__py3-none-any.whl → 0.6.283__py3-none-any.whl

moose_lib/dmv2/registry.py

@@ -4,6 +4,7 @@ Global registries for Moose Data Model v2 (dmv2) resources.
 This module provides functions to access the registered resources.
 The actual registry dictionaries are maintained in _registry.py to avoid circular dependencies.
 """
+
 from typing import Optional, Dict
 from .olap_table import OlapTable
 from .stream import Stream
@@ -24,34 +25,42 @@ from ._registry import (
     _web_apps,
 )
 
+
 def get_tables() -> Dict[str, OlapTable]:
     """Get all registered OLAP tables."""
     return _tables
 
+
 def get_table(name: str) -> Optional[OlapTable]:
     """Get a registered OLAP table by name."""
     return _tables.get(name)
 
+
 def get_streams() -> Dict[str, Stream]:
     """Get all registered streams."""
     return _streams
 
+
 def get_stream(name: str) -> Optional[Stream]:
     """Get a registered stream by name."""
     return _streams.get(name)
 
+
 def get_ingest_apis() -> Dict[str, IngestApi]:
     """Get all registered ingestion APIs."""
     return _ingest_apis
 
+
 def get_ingest_api(name: str) -> Optional[IngestApi]:
     """Get a registered ingestion API by name."""
     return _ingest_apis.get(name)
 
+
 def get_apis() -> Dict[str, Api]:
     """Get all registered APIs."""
     return _apis
 
+
 def get_api(name: str) -> Optional[Api]:
     """Get a registered API by name or path.
 
@@ -64,35 +73,41 @@ def get_api(name: str) -> Optional[Api]:
     api = _apis.get(name)
     if api:
         return api
-    
+
     # Try alias lookup
     api = _api_name_aliases.get(name)
     if api:
         return api
-    
+
     # Try path-based lookup
     return _api_path_map.get(name)
 
+
 def get_sql_resources() -> Dict[str, SqlResource]:
     """Get all registered SQL resources."""
     return _sql_resources
 
+
 def get_sql_resource(name: str) -> Optional[SqlResource]:
     """Get a registered SQL resource by name."""
     return _sql_resources.get(name)
 
+
 def get_workflows() -> Dict[str, Workflow]:
     """Get all registered workflows."""
     return _workflows
 
+
 def get_workflow(name: str) -> Optional[Workflow]:
     """Get a registered workflow by name."""
     return _workflows.get(name)
 
+
 def get_web_apps() -> Dict[str, WebApp]:
     """Get all registered WebApps."""
     return _web_apps
 
+
 def get_web_app(name: str) -> Optional[WebApp]:
     """Get a registered WebApp by name."""
     return _web_apps.get(name)
@@ -103,4 +118,4 @@ get_consumption_apis = get_apis
 """@deprecated: Use get_apis instead of get_consumption_apis"""
 
 get_consumption_api = get_api
-"""@deprecated: Use get_api instead of get_consumption_api"""
+"""@deprecated: Use get_api instead of get_consumption_api"""

moose_lib/dmv2/sql_resource.py

@@ -4,11 +4,14 @@ Base SQL resource definitions for Moose Data Model v2 (dmv2).
 This module provides the base class for SQL resources like Views and Materialized Views,
 handling common functionality like setup/teardown SQL commands and dependency tracking.
 """
+
 from typing import Any, Optional, Union, List
 from pydantic import BaseModel
 
 from .olap_table import OlapTable
 from ._registry import _sql_resources
+from ._source_capture import get_source_file_from_stack
+
 
 class SqlResource:
     """Base class for SQL resources like Views and Materialized Views.
@@ -23,22 +26,25 @@ class SqlResource:
         pulls_data_from (list[SqlObject]): List of tables/views this resource reads from.
         pushes_data_to (list[SqlObject]): List of tables/views this resource writes to.
         kind: The kind of the SQL resource (e.g., "SqlResource").
+        source_file: Optional path to the source file where this resource was defined.
     """
+
     setup: list[str]
     teardown: list[str]
     name: str
     kind: str = "SqlResource"
     pulls_data_from: list[Union[OlapTable, "SqlResource"]]
     pushes_data_to: list[Union[OlapTable, "SqlResource"]]
+    source_file: Optional[str]
 
     def __init__(
-            self,
-            name: str,
-            setup: list[str],
-            teardown: list[str],
-            pulls_data_from: Optional[list[Union[OlapTable, "SqlResource"]]] = None,
-            pushes_data_to: Optional[list[Union[OlapTable, "SqlResource"]]] = None,
-            metadata: dict = None
+        self,
+        name: str,
+        setup: list[str],
+        teardown: list[str],
+        pulls_data_from: Optional[list[Union[OlapTable, "SqlResource"]]] = None,
+        pushes_data_to: Optional[list[Union[OlapTable, "SqlResource"]]] = None,
+        metadata: dict = None,
     ):
         self.name = name
         self.setup = setup
@@ -46,4 +52,6 @@ class SqlResource:
         self.pulls_data_from = pulls_data_from or []
         self.pushes_data_to = pushes_data_to or []
         self.metadata = metadata
-        _sql_resources[name] = self
+        # Capture source file from call stack
+        self.source_file = get_source_file_from_stack()
+        _sql_resources[name] = self

moose_lib/dmv2/stream.py

@@ -4,6 +4,7 @@ Stream definitions for Moose Data Model v2 (dmv2).
 This module provides classes for defining and configuring data streams,
 including stream transformations, consumers, and dead letter queues.
 """
+
 import dataclasses
 import datetime
 import json
@@ -50,6 +51,7 @@ class StreamConfig(BaseModel):
         life_cycle: Determines how changes in code will propagate to the resources.
         default_dead_letter_queue: default dead letter queue used by transforms/consumers
     """
+
     parallelism: int = 1
     retention_period: int = 60 * 60 * 24 * 7  # 7 days
     destination: Optional[OlapTable] = None
@@ -69,6 +71,7 @@ class TransformConfig(BaseModel):
         version: Optional version string to identify a specific transformation.
                  Allows multiple transformations to the same destination if versions differ.
     """
+
     version: Optional[str] = None
     dead_letter_queue: "Optional[DeadLetterQueue]" = None
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -82,6 +85,7 @@ class ConsumerConfig(BaseModel):
         version: Optional version string to identify a specific consumer.
                  Allows multiple consumers if versions differ.
     """
+
     version: Optional[str] = None
     dead_letter_queue: "Optional[DeadLetterQueue]" = None
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -90,6 +94,7 @@ class ConsumerConfig(BaseModel):
 @dataclasses.dataclass
 class _RoutedMessage:
     """Internal class representing a message routed to a specific stream."""
+
     destination: "Stream[Any]"
     values: ZeroOrMany[Any]
 
@@ -97,6 +102,7 @@ class _RoutedMessage:
 @dataclasses.dataclass
 class ConsumerEntry(Generic[T]):
     """Internal class representing a consumer with its configuration."""
+
     consumer: Callable[[T], None]
     config: ConsumerConfig
 
@@ -104,6 +110,7 @@ class ConsumerEntry(Generic[T]):
 @dataclasses.dataclass
 class TransformEntry(Generic[T]):
     """Internal class representing a transformation with its configuration."""
+
     destination: "Stream[Any]"
     transformation: Callable[[T], ZeroOrMany[Any]]
     config: TransformConfig
@@ -128,6 +135,7 @@ class Stream(TypedMooseResource, Generic[T]):
         name (str): The name of the stream.
         model_type (type[T]): The Pydantic model associated with this stream.
     """
+
     config: StreamConfig
     transformations: dict[str, list[TransformEntry[T]]]
     consumers: list[ConsumerEntry[T]]
@@ -146,8 +154,12 @@ class Stream(TypedMooseResource, Generic[T]):
         self.default_dead_letter_queue = self.config.default_dead_letter_queue
         _streams[name] = self
 
-    def add_transform(self, destination: "Stream[U]", transformation: Callable[[T], ZeroOrMany[U]],
-                      config: TransformConfig = None):
+    def add_transform(
+        self,
+        destination: "Stream[U]",
+        transformation: Callable[[T], ZeroOrMany[U]],
+        config: TransformConfig = None,
+    ):
         """Adds a transformation step from this stream to a destination stream.
 
         The transformation function receives a record of type `T` and should return
@@ -160,23 +172,37 @@ class Stream(TypedMooseResource, Generic[T]):
         """
         config = config or TransformConfig()
         if (
-                self.default_dead_letter_queue is not None
-                and config.dead_letter_queue is None
+            self.default_dead_letter_queue is not None
+            and config.dead_letter_queue is None
         ):
             config = config.model_copy()
             config.dead_letter_queue = self.default_dead_letter_queue
         if destination.name in self.transformations:
             existing_transforms = self.transformations[destination.name]
             # Check if a transform with this version already exists
-            has_version = any(t.config.version == config.version for t in existing_transforms)
+            has_version = any(
+                t.config.version == config.version for t in existing_transforms
+            )
             if not has_version:
                 existing_transforms.append(
-                    TransformEntry(destination=destination, transformation=transformation, config=config))
+                    TransformEntry(
+                        destination=destination,
+                        transformation=transformation,
+                        config=config,
+                    )
+                )
         else:
             self.transformations[destination.name] = [
-                TransformEntry(destination=destination, transformation=transformation, config=config)]
+                TransformEntry(
+                    destination=destination,
+                    transformation=transformation,
+                    config=config,
+                )
+            ]
 
-    def add_consumer(self, consumer: Callable[[T], None], config: ConsumerConfig = None):
+    def add_consumer(
+        self, consumer: Callable[[T], None], config: ConsumerConfig = None
+    ):
         """Adds a consumer function to be executed for each record in the stream.
 
         Consumers are typically used for side effects like logging or triggering external actions.
@@ -187,8 +213,8 @@ class Stream(TypedMooseResource, Generic[T]):
         """
         config = config or ConsumerConfig()
         if (
-                self.default_dead_letter_queue is not None
-                and config.dead_letter_queue is None
+            self.default_dead_letter_queue is not None
+            and config.dead_letter_queue is None
         ):
             config = config.model_copy()
             config.dead_letter_queue = self.default_dead_letter_queue
@@ -243,12 +269,15 @@ class Stream(TypedMooseResource, Generic[T]):
 
     def _build_full_topic_name(self, namespace: Optional[str]) -> str:
         """Build full topic name with optional namespace and version suffix."""
-        version_suffix = f"_{self.config.version.replace('.', '_')}" if self.config.version else ""
+        version_suffix = (
+            f"_{self.config.version.replace('.', '_')}" if self.config.version else ""
+        )
         base = f"{self.name}{version_suffix}"
         return f"{namespace}.{base}" if namespace else base
 
     def _create_kafka_config_hash(self, cfg: RuntimeKafkaConfig) -> str:
         import hashlib
+
         config_string = ":".join(
             str(x)
             for x in (
@@ -273,11 +302,17 @@ class Stream(TypedMooseResource, Generic[T]):
         cfg: RuntimeKafkaConfig = config_registry.get_kafka_config()
         current_hash = self._create_kafka_config_hash(cfg)
 
-        if self._memoized_producer is not None and self._kafka_config_hash == current_hash:
+        if (
+            self._memoized_producer is not None
+            and self._kafka_config_hash == current_hash
+        ):
             return self._memoized_producer, cfg
 
         # Close previous producer if config changed
-        if self._memoized_producer is not None and self._kafka_config_hash != current_hash:
+        if (
+            self._memoized_producer is not None
+            and self._kafka_config_hash != current_hash
+        ):
             try:
                 self._memoized_producer.flush()
                 self._memoized_producer.close()
@@ -296,7 +331,7 @@ class Stream(TypedMooseResource, Generic[T]):
             sasl_password=cfg.sasl_password,
             sasl_mechanism=cfg.sasl_mechanism,
             security_protocol=cfg.security_protocol,
-            value_serializer=lambda v: v.model_dump_json().encode('utf-8'),
+            value_serializer=lambda v: v.model_dump_json().encode("utf-8"),
             acks="all",
         )
 
@@ -371,7 +406,9 @@ class Stream(TypedMooseResource, Generic[T]):
             elif isinstance(sr.reference, SubjectLatest):
                 schema = client.get_latest_version(sr.reference.name).schema
             else:
-                schema = client.get_version(sr.reference.subject, sr.reference.version).schema
+                schema = client.get_version(
+                    sr.reference.subject, sr.reference.version
+                ).schema
 
             serializer = JSONSerializer(schema, client)
 
@@ -395,9 +432,12 @@ class DeadLetterModel(BaseModel, Generic[T]):
         failed_at: Timestamp when the error occurred.
         source: Source of the error ("api", "transform", or "table").
     """
-    model_config = ConfigDict(alias_generator=AliasGenerator(
-        serialization_alias=to_camel,
-    ))
+
+    model_config = ConfigDict(
+        alias_generator=AliasGenerator(
+            serialization_alias=to_camel,
+        )
+    )
     original_record: Any
     error_message: str
     error_type: str
@@ -429,10 +469,16 @@ class DeadLetterQueue(Stream, Generic[T]):
         """
         self._model_type = self._get_type(kwargs)
         kwargs["t"] = DeadLetterModel[self._model_type]
-        super().__init__(name, config if config is not None else StreamConfig(), **kwargs)
+        super().__init__(
+            name, config if config is not None else StreamConfig(), **kwargs
+        )
 
-    def add_transform(self, destination: Stream[U], transformation: Callable[[DeadLetterModel[T]], ZeroOrMany[U]],
-                      config: TransformConfig = None):
+    def add_transform(
+        self,
+        destination: Stream[U],
+        transformation: Callable[[DeadLetterModel[T]], ZeroOrMany[U]],
+        config: TransformConfig = None,
+    ):
         def wrapped_transform(record: DeadLetterModel[T]):
             record._t = self._model_type
             return transformation(record)
@@ -440,7 +486,11 @@ class DeadLetterQueue(Stream, Generic[T]):
         config = config or TransformConfig()
         super().add_transform(destination, wrapped_transform, config)
 
-    def add_consumer(self, consumer: Callable[[DeadLetterModel[T]], None], config: ConsumerConfig = None):
+    def add_consumer(
+        self,
+        consumer: Callable[[DeadLetterModel[T]], None],
+        config: ConsumerConfig = None,
+    ):
         def wrapped_consumer(record: DeadLetterModel[T]):
             record._t = self._model_type
             return consumer(record)
@@ -448,7 +498,9 @@ class DeadLetterQueue(Stream, Generic[T]):
         config = config or ConsumerConfig()
         super().add_consumer(wrapped_consumer, config)
 
-    def set_multi_transform(self, transformation: Callable[[DeadLetterModel[T]], list[_RoutedMessage]]):
+    def set_multi_transform(
+        self, transformation: Callable[[DeadLetterModel[T]], list[_RoutedMessage]]
+    ):
         def wrapped_transform(record: DeadLetterModel[T]):
             record._t = self._model_type
             return transformation(record)

moose_lib/dmv2/types.py

@@ -5,6 +5,7 @@ This module provides the core type definitions and base classes used across
 the dmv2 package, including generic type parameters, type aliases, and base
 resource classes.
 """
+
 from typing import Any, Generic, TypeVar, Union
 
 import typing_extensions
@@ -12,10 +13,10 @@ from pydantic import BaseModel
 from pydantic.fields import FieldInfo
 from ..data_models import Column
 
-T = TypeVar('T', bound=BaseModel)
-U = TypeVar('U', bound=BaseModel)
-T_none = TypeVar('T_none', bound=Union[BaseModel, None])
-U_none = TypeVar('U_none', bound=Union[BaseModel, None])
+T = TypeVar("T", bound=BaseModel)
+U = TypeVar("U", bound=BaseModel)
+T_none = TypeVar("T_none", bound=Union[BaseModel, None])
+U_none = TypeVar("U_none", bound=Union[BaseModel, None])
 type ZeroOrMany[T] = Union[T, list[T], None]
 
 
@@ -33,6 +34,7 @@ class Cols:
         >>> print(table.cols.user_id)  # Output: a column object
         >>> print(table.cols.non_existent) # Raises AttributeError
     """
+
     _columns: dict[str, Column]
 
     def __init__(self, columns: list[Column]):
@@ -52,7 +54,7 @@ class Cols:
         return self.__getattr__(item)
 
 
-@typing_extensions.deprecated('use cols in OlapTable instead')
+@typing_extensions.deprecated("use cols in OlapTable instead")
 class Columns(Generic[T]):
     """Provides runtime checked column name access for Moose resources.
 
@@ -70,6 +72,7 @@ class Columns(Generic[T]):
     Args:
         model: The Pydantic model type whose fields represent the columns.
     """
+
     _fields: dict[str, FieldInfo]
 
     def __init__(self, model: type[T]):
@@ -90,14 +93,17 @@ class BaseTypedResource(Generic[T]):
     Attributes:
         name (str): The name of the Moose resource.
     """
+
     _t: type[T]
     name: str
 
     @classmethod
     def _get_type(cls, keyword_args: dict):
-        t = keyword_args.get('t')
+        t = keyword_args.get("t")
         if t is None:
-            raise ValueError(f"Use `{cls.__name__}[T](name='...')` to supply the Pydantic model type`")
+            raise ValueError(
+                f"Use `{cls.__name__}[T](name='...')` to supply the Pydantic model type`"
+            )
         if not isinstance(t, type) or not issubclass(t, BaseModel):
             raise ValueError(f"{t} is not a Pydantic model")
         return t
@@ -130,7 +136,7 @@ class TypedMooseResource(BaseTypedResource, Generic[T]):
     """
 
     @property
-    @typing_extensions.deprecated('use cols in OlapTable instead', category=None)
+    @typing_extensions.deprecated("use cols in OlapTable instead", category=None)
     def columns(self):
         return Columns[T](self._t)
 

moose_lib/dmv2/view.py

@@ -4,12 +4,14 @@ View definitions for Moose Data Model v2 (dmv2).
 This module provides classes for defining standard SQL Views,
 including their SQL statements and dependencies.
 """
+
 from typing import Union, List, Optional
 from pydantic import BaseModel
 
 from .sql_resource import SqlResource
 from .olap_table import OlapTable
 
+
 class View(SqlResource):
     """Represents a standard SQL database View.
 
@@ -27,10 +29,15 @@ class View(SqlResource):
         pulls_data_from (list[SqlObject]): Source tables/views.
     """
 
-    def __init__(self, name: str, select_statement: str, base_tables: list[Union[OlapTable, SqlResource]],
-                 metadata: dict = None):
-        setup = [
-            f"CREATE VIEW IF NOT EXISTS {name} AS {select_statement}".strip()
-        ]
+    def __init__(
+        self,
+        name: str,
+        select_statement: str,
+        base_tables: list[Union[OlapTable, SqlResource]],
+        metadata: dict = None,
+    ):
+        setup = [f"CREATE VIEW IF NOT EXISTS {name} AS {select_statement}".strip()]
         teardown = [f"DROP VIEW IF EXISTS {name}"]
-        super().__init__(name, setup, teardown, pulls_data_from=base_tables, metadata=metadata)
+        super().__init__(
+            name, setup, teardown, pulls_data_from=base_tables, metadata=metadata
+        )

moose_lib/dmv2/web_app.py

@@ -5,6 +5,7 @@ This module allows developers to register FastAPI applications as WebApp resourc
 that are managed by the Moose infrastructure, similar to other resources like
 OlapTables, Streams, and APIs.
 """
+
 from typing import Optional, Dict, Any
 from dataclasses import dataclass
 
@@ -29,6 +30,7 @@ class WebAppMetadata:
     Attributes:
         description: Optional description of the WebApp's purpose.
     """
+
     description: Optional[str] = None
 
 
@@ -45,6 +47,7 @@ class WebAppConfig:
         inject_moose_utils: Whether to inject MooseClient utilities into requests.
                            Defaults to True.
     """
+
     mount_path: str
     metadata: Optional[WebAppMetadata] = None
     inject_moose_utils: bool = True
@@ -111,7 +114,9 @@ class WebApp:
         _web_apps[name] = self
 
     @staticmethod
-    def _validate(name: str, config: WebAppConfig, existing_web_apps: Dict[str, 'WebApp']) -> None:
+    def _validate(
+        name: str, config: WebAppConfig, existing_web_apps: Dict[str, "WebApp"]
+    ) -> None:
         """Validate WebApp configuration.
 
         Args:
@@ -129,7 +134,7 @@ class WebApp:
         # Validate mountPath - it is required
         if not config.mount_path:
             raise ValueError(
-                f"mountPath is required. Please specify a mount path for your WebApp (e.g., \"/myapi\")."
+                f'mountPath is required. Please specify a mount path for your WebApp (e.g., "/myapi").'
             )
 
         mount_path = config.mount_path
@@ -137,7 +142,7 @@ class WebApp:
         # Check for root path - not allowed as it would overlap reserved paths
         if mount_path == "/":
             raise ValueError(
-                f"mountPath cannot be \"/\" as it would allow routes to overlap with reserved paths: "
+                f'mountPath cannot be "/" as it would allow routes to overlap with reserved paths: '
                 f"{', '.join(RESERVED_MOUNT_PATHS)}"
             )
 
@@ -154,7 +159,7 @@ class WebApp:
                 raise ValueError(
                     f"mountPath cannot begin with a reserved path: "
                     f"{', '.join(RESERVED_MOUNT_PATHS)}. "
-                    f"Got: \"{mount_path}\""
+                    f'Got: "{mount_path}"'
                 )
 
         # Check for duplicate mount path
@@ -162,8 +167,8 @@ class WebApp:
             existing_mount = existing_app.config.mount_path
             if existing_mount == mount_path:
                 raise ValueError(
-                    f"WebApp with mountPath \"{mount_path}\" already exists "
-                    f"(used by WebApp \"{existing_name}\")"
+                    f'WebApp with mountPath "{mount_path}" already exists '
+                    f'(used by WebApp "{existing_name}")'
                 )
 
     def __repr__(self) -> str:

moose_lib/dmv2/web_app_helpers.py

@@ -4,6 +4,7 @@ Helper utilities for WebApp integration with FastAPI.
 This module provides utilities to access Moose services (ClickHouse, Temporal)
 from within FastAPI request handlers.
 """
+
 from typing import Optional, Any, Dict
 from dataclasses import dataclass
 
@@ -17,6 +18,7 @@ class ApiUtil:
         sql: SQL template function for building safe queries.
         jwt: JWT payload if authentication is enabled, None otherwise.
     """
+
     client: Any  # MooseClient, typed as Any to avoid circular import
     sql: Any  # sql function from moose_lib.main
     jwt: Optional[Dict[str, Any]] = None
@@ -55,7 +57,7 @@ def get_moose_utils(request: Any) -> Optional[ApiUtil]:
         ```
     """
     # FastAPI uses request.state for storing custom data
-    if hasattr(request, 'state') and hasattr(request.state, 'moose'):
+    if hasattr(request, "state") and hasattr(request.state, "moose"):
         return request.state.moose
     return None
 
@@ -83,10 +85,12 @@ def get_moose_dependency():
             return result
         ```
     """
+
     def moose_dependency(request: Any) -> ApiUtil:
         moose = get_moose_utils(request)
         if moose is None:
             # This should rarely happen if inject_moose_utils=True
             raise RuntimeError("Moose utilities not available in request")
         return moose
+
     return moose_dependency