moose-lib 0.6.90__py3-none-any.whl → 0.6.283__py3-none-any.whl

moose_lib/internal.py

@@ -6,6 +6,7 @@ of various Moose resources (tables, streams/topics, APIs) and functions
 to convert the user-defined resources (from `dmv2.py`) into a serializable
 JSON format expected by the Moose infrastructure management system.
 """
+
 from importlib import import_module
 from typing import Literal, Optional, List, Any, Dict, Union, TYPE_CHECKING
 from pydantic import BaseModel, ConfigDict, AliasGenerator, Field
@@ -19,17 +20,20 @@ from moose_lib.dmv2 import (
     get_apis,
     get_sql_resources,
     get_workflows,
+    get_web_apps,
     OlapTable,
-    View,
-    MaterializedView,
-    SqlResource
+    OlapConfig,
+    SqlResource,
 )
+from moose_lib.dmv2.stream import KafkaSchemaConfig
 from pydantic.alias_generators import to_camel
 from pydantic.json_schema import JsonSchemaValue
 
-model_config = ConfigDict(alias_generator=AliasGenerator(
-    serialization_alias=to_camel,
-))
+model_config = ConfigDict(
+    alias_generator=AliasGenerator(
+        serialization_alias=to_camel,
+    )
+)
 
 
 class Target(BaseModel):
@@ -41,6 +45,7 @@ class Target(BaseModel):
         version: Optional version of the target stream configuration.
         metadata: Optional metadata for the target stream.
     """
+
     kind: Literal["stream"]
     name: str
     version: Optional[str] = None
@@ -53,22 +58,26 @@ class Consumer(BaseModel):
     Attributes:
         version: Optional version of the consumer configuration.
     """
+
     version: Optional[str] = None
 
 
 class BaseEngineConfigDict(BaseModel):
     """Base engine configuration for all ClickHouse table engines."""
+
     model_config = model_config
     engine: str
 
 
 class MergeTreeConfigDict(BaseEngineConfigDict):
     """Configuration for MergeTree engine."""
+
     engine: Literal["MergeTree"] = "MergeTree"
 
 
 class ReplacingMergeTreeConfigDict(BaseEngineConfigDict):
     """Configuration for ReplacingMergeTree engine."""
+
     engine: Literal["ReplacingMergeTree"] = "ReplacingMergeTree"
     ver: Optional[str] = None
     is_deleted: Optional[str] = None
@@ -76,16 +85,91 @@ class ReplacingMergeTreeConfigDict(BaseEngineConfigDict):
 
 class AggregatingMergeTreeConfigDict(BaseEngineConfigDict):
     """Configuration for AggregatingMergeTree engine."""
+
     engine: Literal["AggregatingMergeTree"] = "AggregatingMergeTree"
 
 
 class SummingMergeTreeConfigDict(BaseEngineConfigDict):
     """Configuration for SummingMergeTree engine."""
+
     engine: Literal["SummingMergeTree"] = "SummingMergeTree"
+    columns: Optional[List[str]] = None
+
+
+class CollapsingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for CollapsingMergeTree engine."""
+
+    engine: Literal["CollapsingMergeTree"] = "CollapsingMergeTree"
+    sign: str
+
+
+class VersionedCollapsingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for VersionedCollapsingMergeTree engine."""
+
+    engine: Literal["VersionedCollapsingMergeTree"] = "VersionedCollapsingMergeTree"
+    sign: str
+    ver: str
+
+
+class ReplicatedMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for ReplicatedMergeTree engine."""
+
+    engine: Literal["ReplicatedMergeTree"] = "ReplicatedMergeTree"
+    keeper_path: Optional[str] = None
+    replica_name: Optional[str] = None
+
+
+class ReplicatedReplacingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for ReplicatedReplacingMergeTree engine."""
+
+    engine: Literal["ReplicatedReplacingMergeTree"] = "ReplicatedReplacingMergeTree"
+    keeper_path: Optional[str] = None
+    replica_name: Optional[str] = None
+    ver: Optional[str] = None
+    is_deleted: Optional[str] = None
+
+
+class ReplicatedAggregatingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for ReplicatedAggregatingMergeTree engine."""
+
+    engine: Literal["ReplicatedAggregatingMergeTree"] = "ReplicatedAggregatingMergeTree"
+    keeper_path: Optional[str] = None
+    replica_name: Optional[str] = None
+
+
+class ReplicatedSummingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for ReplicatedSummingMergeTree engine."""
+
+    engine: Literal["ReplicatedSummingMergeTree"] = "ReplicatedSummingMergeTree"
+    keeper_path: Optional[str] = None
+    replica_name: Optional[str] = None
+    columns: Optional[List[str]] = None
+
+
+class ReplicatedCollapsingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for ReplicatedCollapsingMergeTree engine."""
+
+    engine: Literal["ReplicatedCollapsingMergeTree"] = "ReplicatedCollapsingMergeTree"
+    keeper_path: Optional[str] = None
+    replica_name: Optional[str] = None
+    sign: str
+
+
+class ReplicatedVersionedCollapsingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for ReplicatedVersionedCollapsingMergeTree engine."""
+
+    engine: Literal["ReplicatedVersionedCollapsingMergeTree"] = (
+        "ReplicatedVersionedCollapsingMergeTree"
+    )
+    keeper_path: Optional[str] = None
+    replica_name: Optional[str] = None
+    sign: str
+    ver: str
 
 
 class S3QueueConfigDict(BaseEngineConfigDict):
     """Configuration for S3Queue engine with all specific fields."""
+
     engine: Literal["S3Queue"] = "S3Queue"
     s3_path: str
     format: str
@@ -95,13 +179,95 @@ class S3QueueConfigDict(BaseEngineConfigDict):
     headers: Optional[Dict[str, str]] = None
 
 
+class S3ConfigDict(BaseEngineConfigDict):
+    """Configuration for S3 engine."""
+
+    engine: Literal["S3"] = "S3"
+    path: str
+    format: str
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    compression: Optional[str] = None
+    partition_strategy: Optional[str] = None
+    partition_columns_in_data_file: Optional[str] = None
+
+
+class BufferConfigDict(BaseEngineConfigDict):
+    """Configuration for Buffer engine."""
+
+    engine: Literal["Buffer"] = "Buffer"
+    target_database: str
+    target_table: str
+    num_layers: int
+    min_time: int
+    max_time: int
+    min_rows: int
+    max_rows: int
+    min_bytes: int
+    max_bytes: int
+    flush_time: Optional[int] = None
+    flush_rows: Optional[int] = None
+    flush_bytes: Optional[int] = None
+
+
+class DistributedConfigDict(BaseEngineConfigDict):
+    """Configuration for Distributed engine."""
+
+    engine: Literal["Distributed"] = "Distributed"
+    cluster: str
+    target_database: str
+    target_table: str
+    sharding_key: Optional[str] = None
+    policy_name: Optional[str] = None
+
+
+class IcebergS3ConfigDict(BaseEngineConfigDict):
+    """Configuration for IcebergS3 engine."""
+
+    engine: Literal["IcebergS3"] = "IcebergS3"
+    path: str
+    format: str
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    compression: Optional[str] = None
+
+
+class KafkaConfigDict(BaseEngineConfigDict):
+    """Configuration for Kafka engine.
+
+    Constructor: ENGINE = Kafka('broker', 'topic', 'group', 'format')
+    Settings (kafka_schema, kafka_num_consumers, security, etc.) go in table settings.
+
+    Reference: https://clickhouse.com/docs/engines/table-engines/integrations/kafka
+    """
+
+    engine: Literal["Kafka"] = "Kafka"
+    broker_list: str
+    topic_list: str
+    group_name: str
+    format: str
+
+
 # Discriminated union of all engine configurations
 EngineConfigDict = Union[
     MergeTreeConfigDict,
     ReplacingMergeTreeConfigDict,
     AggregatingMergeTreeConfigDict,
     SummingMergeTreeConfigDict,
-    S3QueueConfigDict
+    CollapsingMergeTreeConfigDict,
+    VersionedCollapsingMergeTreeConfigDict,
+    ReplicatedMergeTreeConfigDict,
+    ReplicatedReplacingMergeTreeConfigDict,
+    ReplicatedAggregatingMergeTreeConfigDict,
+    ReplicatedSummingMergeTreeConfigDict,
+    ReplicatedCollapsingMergeTreeConfigDict,
+    ReplicatedVersionedCollapsingMergeTreeConfigDict,
+    S3QueueConfigDict,
+    S3ConfigDict,
+    BufferConfigDict,
+    DistributedConfigDict,
+    IcebergS3ConfigDict,
+    KafkaConfigDict,
 ]
 
 
@@ -113,23 +279,33 @@ class TableConfig(BaseModel):
         columns: List of columns with their types and attributes.
         order_by: List of columns used for the ORDER BY clause.
         partition_by: The column name used for the PARTITION BY clause.
+        sample_by_expression: Optional SAMPLE BY expression for data sampling.
+        primary_key_expression: Optional PRIMARY KEY expression (overrides column-level primary_key flags when specified).
         engine_config: Engine configuration with type-safe, engine-specific parameters.
         version: Optional version string of the table configuration.
         metadata: Optional metadata for the table.
         life_cycle: Lifecycle management setting for the table.
         table_settings: Optional table-level settings that can be modified with ALTER TABLE MODIFY SETTING.
+        cluster: Optional cluster name for ON CLUSTER support in ClickHouse.
     """
+
     model_config = model_config
 
     name: str
     columns: List[Column]
-    order_by: List[str]
+    order_by: List[str] | str
     partition_by: Optional[str]
-    engine_config: Optional[EngineConfigDict] = Field(None, discriminator='engine')
+    sample_by_expression: Optional[str] = None
+    primary_key_expression: Optional[str] = None
+    engine_config: Optional[EngineConfigDict] = Field(None, discriminator="engine")
     version: Optional[str] = None
     metadata: Optional[dict] = None
     life_cycle: Optional[str] = None
-    table_settings: Optional[Dict[str, str]] = None
+    table_settings: Optional[dict[str, str]] = None
+    indexes: list[OlapConfig.TableIndex] = []
+    ttl: Optional[str] = None
+    database: Optional[str] = None
+    cluster: Optional[str] = None
 
 
 class TopicConfig(BaseModel):
@@ -149,6 +325,7 @@ class TopicConfig(BaseModel):
         metadata: Optional metadata for the topic.
         life_cycle: Lifecycle management setting for the topic.
     """
+
     model_config = model_config
 
     name: str
@@ -163,6 +340,7 @@ class TopicConfig(BaseModel):
     consumers: List[Consumer]
     metadata: Optional[dict] = None
     life_cycle: Optional[str] = None
+    schema_config: Optional[KafkaSchemaConfig] = None
 
 
 class IngestApiConfig(BaseModel):
@@ -176,7 +354,10 @@ class IngestApiConfig(BaseModel):
         version: Optional version string of the API configuration.
         path: Optional custom path for the ingestion endpoint.
         metadata: Optional metadata for the API.
+        allow_extra_fields: Whether this API allows extra fields beyond the defined columns.
+            When true, extra fields in payloads are passed through to streaming functions.
     """
+
     model_config = model_config
 
     name: str
@@ -187,6 +368,7 @@ class IngestApiConfig(BaseModel):
     path: Optional[str] = None
     metadata: Optional[dict] = None
     json_schema: dict[str, Any] = Field(serialization_alias="schema")
+    allow_extra_fields: bool = False
 
 
 class InternalApiConfig(BaseModel):
@@ -200,6 +382,7 @@ class InternalApiConfig(BaseModel):
         path: Optional custom path for the API endpoint.
         metadata: Optional metadata for the API.
     """
+
     model_config = model_config
 
     name: str
@@ -219,6 +402,7 @@ class WorkflowJson(BaseModel):
         timeout: Optional timeout string for the entire workflow.
         schedule: Optional cron-like schedule string for recurring execution.
     """
+
     model_config = model_config
 
     name: str
@@ -227,6 +411,34 @@ class WorkflowJson(BaseModel):
     schedule: Optional[str] = None
 
 
+class WebAppMetadataJson(BaseModel):
+    """Internal representation of WebApp metadata for serialization.
+
+    Attributes:
+        description: Optional description of the WebApp.
+    """
+
+    model_config = model_config
+
+    description: Optional[str] = None
+
+
+class WebAppJson(BaseModel):
+    """Internal representation of a WebApp configuration for serialization.
+
+    Attributes:
+        name: Name of the WebApp.
+        mount_path: The URL path where the WebApp is mounted.
+        metadata: Optional metadata for documentation purposes.
+    """
+
+    model_config = model_config
+
+    name: str
+    mount_path: str
+    metadata: Optional[WebAppMetadataJson] = None
+
+
 class InfrastructureSignatureJson(BaseModel):
     """Represents the unique signature of an infrastructure component (Table, Topic, etc.).
 
@@ -236,8 +448,16 @@ class InfrastructureSignatureJson(BaseModel):
         id: A unique identifier for the resource instance (often name + version).
         kind: The type of the infrastructure component.
     """
+
     id: str
-    kind: Literal["Table", "Topic", "ApiEndpoint", "TopicToTableSyncProcess", "View", "SqlResource"]
+    kind: Literal[
+        "Table",
+        "Topic",
+        "ApiEndpoint",
+        "TopicToTableSyncProcess",
+        "View",
+        "SqlResource",
+    ]
 
 
 class SqlResourceConfig(BaseModel):
@@ -249,8 +469,10 @@ class SqlResourceConfig(BaseModel):
         teardown: List of SQL commands required to drop the resource.
         pulls_data_from: List of infrastructure components this resource reads from.
         pushes_data_to: List of infrastructure components this resource writes to.
+        source_file: Optional path to the source file where this resource is defined.
         metadata: Optional metadata for the resource.
     """
+
     model_config = model_config
 
     name: str
@@ -258,6 +480,7 @@ class SqlResourceConfig(BaseModel):
     teardown: list[str]
     pulls_data_from: list[InfrastructureSignatureJson]
     pushes_data_to: list[InfrastructureSignatureJson]
+    source_file: Optional[str] = None
     metadata: Optional[dict] = None
 
 
@@ -273,7 +496,9 @@ class InfrastructureMap(BaseModel):
         apis: Dictionary mapping API names to their configurations.
         sql_resources: Dictionary mapping SQL resource names to their configurations.
         workflows: Dictionary mapping workflow names to their configurations.
+        web_apps: Dictionary mapping WebApp names to their configurations.
     """
+
     model_config = model_config
 
     tables: dict[str, TableConfig]
@@ -282,6 +507,7 @@ class InfrastructureMap(BaseModel):
     apis: dict[str, InternalApiConfig]
     sql_resources: dict[str, SqlResourceConfig]
     workflows: dict[str, WorkflowJson]
+    web_apps: dict[str, WebAppJson]
 
 
 def _map_sql_resource_ref(r: Any) -> InfrastructureSignatureJson:
@@ -299,11 +525,15 @@ def _map_sql_resource_ref(r: Any) -> InfrastructureSignatureJson:
     Raises:
         TypeError: If the input object is not a recognized SQL resource type.
     """
-    if hasattr(r, 'kind'):
+    if hasattr(r, "kind"):
         if r.kind == "OlapTable":
             # Explicitly cast for type hint checking if needed, though Python is dynamic
             table = r  # type: OlapTable
-            res_id = f"{table.name}_{table.config.version}" if table.config.version else table.name
+            res_id = (
+                f"{table.name}_{table.config.version}"
+                if table.config.version
+                else table.name
+            )
             return InfrastructureSignatureJson(id=res_id, kind="Table")
         elif r.kind == "SqlResource":
             # Explicitly cast for type hint checking if needed
@@ -316,50 +546,220 @@ def _map_sql_resource_ref(r: Any) -> InfrastructureSignatureJson:
         raise TypeError(f"Object {r} lacks a 'kind' attribute for dependency mapping.")
 
 
-def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig],
-                                   table: OlapTable) -> EngineConfigDict:
+def _convert_basic_engine_instance(
+    engine: "EngineConfig",
+) -> Optional[EngineConfigDict]:
+    """Convert basic MergeTree engine instances to config dict.
+
+    Args:
+        engine: An EngineConfig instance
+
+    Returns:
+        EngineConfigDict if matched, None otherwise
+    """
+    from moose_lib.blocks import (
+        MergeTreeEngine,
+        ReplacingMergeTreeEngine,
+        AggregatingMergeTreeEngine,
+        SummingMergeTreeEngine,
+        CollapsingMergeTreeEngine,
+        VersionedCollapsingMergeTreeEngine,
+    )
+
+    if isinstance(engine, MergeTreeEngine):
+        return MergeTreeConfigDict()
+    elif isinstance(engine, ReplacingMergeTreeEngine):
+        return ReplacingMergeTreeConfigDict(
+            ver=engine.ver, is_deleted=engine.is_deleted
+        )
+    elif isinstance(engine, AggregatingMergeTreeEngine):
+        return AggregatingMergeTreeConfigDict()
+    elif isinstance(engine, SummingMergeTreeEngine):
+        return SummingMergeTreeConfigDict(columns=engine.columns)
+    elif isinstance(engine, CollapsingMergeTreeEngine):
+        return CollapsingMergeTreeConfigDict(sign=engine.sign)
+    elif isinstance(engine, VersionedCollapsingMergeTreeEngine):
+        return VersionedCollapsingMergeTreeConfigDict(sign=engine.sign, ver=engine.ver)
+    return None
+
+
+def _convert_replicated_engine_instance(
+    engine: "EngineConfig",
+) -> Optional[EngineConfigDict]:
+    """Convert replicated MergeTree engine instances to config dict.
+
+    Args:
+        engine: An EngineConfig instance
+
+    Returns:
+        EngineConfigDict if matched, None otherwise
+    """
+    from moose_lib.blocks import (
+        ReplicatedMergeTreeEngine,
+        ReplicatedReplacingMergeTreeEngine,
+        ReplicatedAggregatingMergeTreeEngine,
+        ReplicatedSummingMergeTreeEngine,
+        ReplicatedCollapsingMergeTreeEngine,
+        ReplicatedVersionedCollapsingMergeTreeEngine,
+    )
+
+    if isinstance(engine, ReplicatedMergeTreeEngine):
+        return ReplicatedMergeTreeConfigDict(
+            keeper_path=engine.keeper_path, replica_name=engine.replica_name
+        )
+    elif isinstance(engine, ReplicatedReplacingMergeTreeEngine):
+        return ReplicatedReplacingMergeTreeConfigDict(
+            keeper_path=engine.keeper_path,
+            replica_name=engine.replica_name,
+            ver=engine.ver,
+            is_deleted=engine.is_deleted,
+        )
+    elif isinstance(engine, ReplicatedAggregatingMergeTreeEngine):
+        return ReplicatedAggregatingMergeTreeConfigDict(
+            keeper_path=engine.keeper_path, replica_name=engine.replica_name
+        )
+    elif isinstance(engine, ReplicatedSummingMergeTreeEngine):
+        return ReplicatedSummingMergeTreeConfigDict(
+            keeper_path=engine.keeper_path,
+            replica_name=engine.replica_name,
+            columns=engine.columns,
+        )
+    elif isinstance(engine, ReplicatedCollapsingMergeTreeEngine):
+        return ReplicatedCollapsingMergeTreeConfigDict(
+            keeper_path=engine.keeper_path,
+            replica_name=engine.replica_name,
+            sign=engine.sign,
+        )
+    elif isinstance(engine, ReplicatedVersionedCollapsingMergeTreeEngine):
+        return ReplicatedVersionedCollapsingMergeTreeConfigDict(
+            keeper_path=engine.keeper_path,
+            replica_name=engine.replica_name,
+            sign=engine.sign,
+            ver=engine.ver,
+        )
+    return None
+
+
+def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineConfigDict:
+    """Convert an EngineConfig instance to config dict format.
+
+    Args:
+        engine: An EngineConfig instance
+
+    Returns:
+        EngineConfigDict with engine-specific configuration
+    """
+    from moose_lib.blocks import (
+        S3QueueEngine,
+        S3Engine,
+        BufferEngine,
+        DistributedEngine,
+        IcebergS3Engine,
+        KafkaEngine,
+    )
+
+    # Try S3Queue first
+    if isinstance(engine, S3QueueEngine):
+        return S3QueueConfigDict(
+            s3_path=engine.s3_path,
+            format=engine.format,
+            aws_access_key_id=engine.aws_access_key_id,
+            aws_secret_access_key=engine.aws_secret_access_key,
+            compression=engine.compression,
+            headers=engine.headers,
+        )
+
+    # Try S3
+    if isinstance(engine, S3Engine):
+        return S3ConfigDict(
+            path=engine.path,
+            format=engine.format,
+            aws_access_key_id=engine.aws_access_key_id,
+            aws_secret_access_key=engine.aws_secret_access_key,
+            compression=engine.compression,
+            partition_strategy=engine.partition_strategy,
+            partition_columns_in_data_file=engine.partition_columns_in_data_file,
+        )
+
+    # Try Buffer
+    if isinstance(engine, BufferEngine):
+        return BufferConfigDict(
+            target_database=engine.target_database,
+            target_table=engine.target_table,
+            num_layers=engine.num_layers,
+            min_time=engine.min_time,
+            max_time=engine.max_time,
+            min_rows=engine.min_rows,
+            max_rows=engine.max_rows,
+            min_bytes=engine.min_bytes,
+            max_bytes=engine.max_bytes,
+            flush_time=engine.flush_time,
+            flush_rows=engine.flush_rows,
+            flush_bytes=engine.flush_bytes,
+        )
+
+    # Try Distributed
+    if isinstance(engine, DistributedEngine):
+        return DistributedConfigDict(
+            cluster=engine.cluster,
+            target_database=engine.target_database,
+            target_table=engine.target_table,
+            sharding_key=engine.sharding_key,
+            policy_name=engine.policy_name,
+        )
+
+    # Try IcebergS3
+    if isinstance(engine, IcebergS3Engine):
+        return IcebergS3ConfigDict(
+            path=engine.path,
+            format=engine.format,
+            aws_access_key_id=engine.aws_access_key_id,
+            aws_secret_access_key=engine.aws_secret_access_key,
+            compression=engine.compression,
+        )
+
+    # Try Kafka
+    if isinstance(engine, KafkaEngine):
+        return KafkaConfigDict(
+            broker_list=engine.broker_list,
+            topic_list=engine.topic_list,
+            group_name=engine.group_name,
+            format=engine.format,
+        )
+
+    # Try basic engines
+    basic_config = _convert_basic_engine_instance(engine)
+    if basic_config:
+        return basic_config
+
+    # Try replicated engines
+    replicated_config = _convert_replicated_engine_instance(engine)
+    if replicated_config:
+        return replicated_config
+
+    # Fallback for any other EngineConfig subclass
+    return BaseEngineConfigDict(engine=engine.__class__.__name__.replace("Engine", ""))
+
+
+def _convert_engine_to_config_dict(
+    engine: Union[ClickHouseEngines, EngineConfig], table: OlapTable
+) -> EngineConfigDict:
     """Convert engine enum or EngineConfig instance to new engine config format.
-    
+
     Args:
         engine: Either a ClickHouseEngines enum value or an EngineConfig instance
         table: The OlapTable instance with configuration
-        
+
     Returns:
         EngineConfigDict with engine-specific configuration
     """
     from moose_lib import ClickHouseEngines
-    from moose_lib.blocks import (
-        EngineConfig, S3QueueEngine, MergeTreeEngine,
-        ReplacingMergeTreeEngine, AggregatingMergeTreeEngine,
-        SummingMergeTreeEngine
-    )
+    from moose_lib.blocks import EngineConfig
     from moose_lib.commons import Logger
 
     # Check if engine is an EngineConfig instance (new API)
     if isinstance(engine, EngineConfig):
-        if isinstance(engine, S3QueueEngine):
-            return S3QueueConfigDict(
-                s3_path=engine.s3_path,
-                format=engine.format,
-                aws_access_key_id=engine.aws_access_key_id,
-                aws_secret_access_key=engine.aws_secret_access_key,
-                compression=engine.compression,
-                headers=engine.headers
-            )
-        elif isinstance(engine, ReplacingMergeTreeEngine):
-            return ReplacingMergeTreeConfigDict(
-                ver=engine.ver,
-                is_deleted=engine.is_deleted
-            )
-        elif isinstance(engine, AggregatingMergeTreeEngine):
-            return AggregatingMergeTreeConfigDict()
-        elif isinstance(engine, SummingMergeTreeEngine):
-            return SummingMergeTreeConfigDict()
-        elif isinstance(engine, MergeTreeEngine):
-            return MergeTreeConfigDict()
-        else:
-            # Fallback for any other EngineConfig subclass - use base class
-            return BaseEngineConfigDict(engine=engine.__class__.__name__.replace("Engine", ""))
+        return _convert_engine_instance_to_config_dict(engine)
 
     # Handle legacy enum-based engine configuration
     if isinstance(engine, ClickHouseEngines):
@@ -368,7 +768,7 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig
         engine_name = str(engine)
 
     # For S3Queue with legacy configuration, check for s3_queue_engine_config
-    if engine_name == "S3Queue" and hasattr(table.config, 's3_queue_engine_config'):
+    if engine_name == "S3Queue" and hasattr(table.config, "s3_queue_engine_config"):
         s3_config = table.config.s3_queue_engine_config
         if s3_config:
             logger = Logger(action="S3QueueConfig")
@@ -382,7 +782,7 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig
                 aws_access_key_id=s3_config.aws_access_key_id,
                 aws_secret_access_key=s3_config.aws_secret_access_key,
                 compression=s3_config.compression,
-                headers=s3_config.headers
+                headers=s3_config.headers,
             )
 
     # Map engine names to specific config classes
@@ -391,6 +791,10 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig
         "ReplacingMergeTree": ReplacingMergeTreeConfigDict,
         "AggregatingMergeTree": AggregatingMergeTreeConfigDict,
         "SummingMergeTree": SummingMergeTreeConfigDict,
+        "ReplicatedMergeTree": ReplicatedMergeTreeConfigDict,
+        "ReplicatedReplacingMergeTree": ReplicatedReplacingMergeTreeConfigDict,
+        "ReplicatedAggregatingMergeTree": ReplicatedAggregatingMergeTreeConfigDict,
+        "ReplicatedSummingMergeTree": ReplicatedSummingMergeTreeConfigDict,
     }
 
     config_class = engine_map.get(engine_name)
@@ -418,8 +822,9 @@ def to_infra_map() -> dict:
     apis = {}
     sql_resources = {}
     workflows = {}
+    web_apps = {}
 
-    for name, table in get_tables().items():
+    for _registry_key, table in get_tables().items():
         # Convert engine configuration to new format
         engine_config = None
         if table.config.engine:
@@ -434,17 +839,45 @@ def to_infra_map() -> dict:
             if "mode" not in table_settings:
                 table_settings["mode"] = "unordered"
 
-        tables[name] = TableConfig(
-            name=name,
+        id_key = (
+            f"{table.name}_{table.config.version}"
+            if table.config.version
+            else table.name
+        )
+
+        # Determine ORDER BY: list of fields or single expression
+        has_fields = bool(table.config.order_by_fields)
+        has_expr = table.config.order_by_expression is not None
+        if has_fields and has_expr:
+            raise ValueError(
+                f"Table {table.name}: Provide either order_by_fields or order_by_expression, not both."
+            )
+
+        order_by_value = (
+            table.config.order_by_expression
+            if has_expr
+            else table.config.order_by_fields
+        )
+
+        tables[id_key] = TableConfig(
+            name=table.name,
             columns=table._column_list,
-            order_by=table.config.order_by_fields,
+            order_by=order_by_value,
             partition_by=table.config.partition_by,
+            sample_by_expression=table.config.sample_by_expression,
+            primary_key_expression=table.config.primary_key_expression,
             engine_config=engine_config,
             version=table.config.version,
             metadata=getattr(table, "metadata", None),
-            life_cycle=table.config.life_cycle.value if table.config.life_cycle else None,
+            life_cycle=(
+                table.config.life_cycle.value if table.config.life_cycle else None
+            ),
             # Map 'settings' to 'table_settings' for internal use
             table_settings=table_settings if table_settings else None,
+            indexes=table.config.indexes,
+            ttl=table.config.ttl,
+            database=table.config.database,
+            cluster=table.config.cluster,
         )
 
     for name, stream in get_streams().items():
@@ -460,15 +893,20 @@ def to_infra_map() -> dict:
         ]
 
         consumers = [
-            Consumer(version=consumer.config.version)
-            for consumer in stream.consumers
+            Consumer(version=consumer.config.version) for consumer in stream.consumers
         ]
 
         topics[name] = TopicConfig(
             name=name,
             columns=_to_columns(stream._t),
-            target_table=stream.config.destination.name if stream.config.destination else None,
-            target_table_version=stream.config.destination.config.version if stream.config.destination else None,
+            target_table=(
+                stream.config.destination.name if stream.config.destination else None
+            ),
+            target_table_version=(
+                stream.config.destination.config.version
+                if stream.config.destination
+                else None
+            ),
             retention_period=stream.config.retention_period,
             partition_count=stream.config.parallelism,
             version=stream.config.version,
@@ -476,24 +914,33 @@ def to_infra_map() -> dict:
             has_multi_transform=stream._multipleTransformations is not None,
             consumers=consumers,
             metadata=getattr(stream, "metadata", None),
-            life_cycle=stream.config.life_cycle.value if stream.config.life_cycle else None,
+            life_cycle=(
+                stream.config.life_cycle.value if stream.config.life_cycle else None
+            ),
+            schema_config=stream.config.schema_config,
         )
 
     for name, api in get_ingest_apis().items():
+        # Check if the Pydantic model allows extra fields (extra='allow')
+        # This is the Python equivalent of TypeScript's index signatures
+        model_allows_extra = api._t.model_config.get("extra") == "allow"
+
         ingest_apis[name] = IngestApiConfig(
             name=name,
             columns=_to_columns(api._t),
             version=api.config.version,
             path=api.config.path,
-            write_to=Target(
-                kind="stream",
-                name=api.config.destination.name
-            ),
+            write_to=Target(kind="stream", name=api.config.destination.name),
             metadata=getattr(api, "metadata", None),
             json_schema=api._t.model_json_schema(
-                ref_template='#/components/schemas/{model}'
+                ref_template="#/components/schemas/{model}"
             ),
-            dead_letter_queue=api.config.dead_letter_queue.name if api.config.dead_letter_queue else None
+            dead_letter_queue=(
+                api.config.dead_letter_queue.name
+                if api.config.dead_letter_queue
+                else None
+            ),
+            allow_extra_fields=model_allows_extra,
         )
 
     for name, api in get_apis().items():
@@ -511,8 +958,13 @@ def to_infra_map() -> dict:
             name=resource.name,
             setup=resource.setup,
             teardown=resource.teardown,
-            pulls_data_from=[_map_sql_resource_ref(dep) for dep in resource.pulls_data_from],
-            pushes_data_to=[_map_sql_resource_ref(dep) for dep in resource.pushes_data_to],
+            pulls_data_from=[
+                _map_sql_resource_ref(dep) for dep in resource.pulls_data_from
+            ],
+            pushes_data_to=[
+                _map_sql_resource_ref(dep) for dep in resource.pushes_data_to
+            ],
+            source_file=getattr(resource, "source_file", None),
             metadata=getattr(resource, "metadata", None),
         )
 
@@ -524,16 +976,30 @@ def to_infra_map() -> dict:
             schedule=workflow.config.schedule,
         )
 
+    for name, web_app in get_web_apps().items():
+        mount_path = web_app.config.mount_path or "/"
+        metadata = None
+        if web_app.config.metadata:
+            metadata = WebAppMetadataJson(
+                description=web_app.config.metadata.description
+            )
+        web_apps[name] = WebAppJson(
+            name=web_app.name,
+            mount_path=mount_path,
+            metadata=metadata,
+        )
+
     infra_map = InfrastructureMap(
         tables=tables,
         topics=topics,
         ingest_apis=ingest_apis,
         apis=apis,
         sql_resources=sql_resources,
-        workflows=workflows
+        workflows=workflows,
+        web_apps=web_apps,
     )
 
-    return infra_map.model_dump(by_alias=True)
+    return infra_map.model_dump(by_alias=True, exclude_none=False)
 
 
 def load_models():
@@ -549,7 +1015,10 @@ def load_models():
        (`___MOOSE_STUFF___start` and `end___MOOSE_STUFF___`), which the
        calling system uses to extract the configuration.
     """
-    import_module("app.main")
+    import os
+
+    source_dir = os.environ.get("MOOSE_SOURCE_DIR", "app")
+    import_module(f"{source_dir}.main")
 
     # Generate the infrastructure map
     infra_map = to_infra_map()