moose-lib 0.4.212__py3-none-any.whl → 0.4.214__py3-none-any.whl

moose_lib/dmv2.py

@@ -10,7 +10,7 @@ of data infrastructure using Python and Pydantic models.
 """
 import dataclasses
 import datetime
-from typing import Any, Generic, Optional, TypeVar, Callable, Union, Literal
+from typing import Any, Generic, Optional, TypeVar, Callable, Union, Literal, Awaitable
 from pydantic import BaseModel, ConfigDict, AliasGenerator
 from pydantic.alias_generators import to_camel
 from pydantic.fields import FieldInfo
@@ -23,11 +23,24 @@ _streams: dict[str, "Stream"] = {}
 _ingest_apis: dict[str, "IngestApi"] = {}
 _egress_apis: dict[str, "ConsumptionApi"] = {}
 _sql_resources: dict[str, "SqlResource"] = {}
+_workflows: dict[str, "Workflow"] = {}
 
 T = TypeVar('T', bound=BaseModel)
 U = TypeVar('U', bound=BaseModel)
+T_none = TypeVar('T_none', bound=Union[BaseModel, None])
+U_none = TypeVar('U_none', bound=Union[BaseModel, None])
 type ZeroOrMany[T] = Union[T, list[T], None]
 
+type TaskRunFunc[T_none, U_none] = Union[
+    # Case 1: No input, no output
+    Callable[[], None],
+    # Case 2: No input, with output
+    Callable[[], Union[U_none, Awaitable[U_none]]],
+    # Case 3: With input, no output
+    Callable[[T_none], None],
+    # Case 4: With input, with output
+    Callable[[T_none], Union[U_none, Awaitable[U_none]]]
+]
 
 class Columns(Generic[T]):
     """Provides runtime checked column name access for Moose resources.
@@ -422,6 +435,7 @@ class IngestConfigWithDestination[T: BaseModel]:
         metadata: Optional metadata for the ingestion configuration.
     """
     destination: Stream[T]
+    dead_letter_queue: Optional[DeadLetterQueue[T]] = None
     version: Optional[str] = None
     metadata: Optional[dict] = None
 
@@ -443,6 +457,7 @@ class IngestPipelineConfig(BaseModel):
     table: bool | OlapConfig = True
     stream: bool | StreamConfig = True
     ingest: bool | IngestConfig = True
+    dead_letter_queue: bool | StreamConfig = True
     version: Optional[str] = None
     metadata: Optional[dict] = None
 
@@ -491,6 +506,7 @@ class IngestPipeline(TypedMooseResource, Generic[T]):
         table: The created `OlapTable` instance, if configured.
         stream: The created `Stream` instance, if configured.
         ingest_api: The created `IngestApi` instance, if configured.
+        dead_letter_queue: The created `DeadLetterQueue` instance, if configured.
         columns (Columns[T]): Helper for accessing data field names safely.
         name (str): The base name of the pipeline.
         model_type (type[T]): The Pydantic model associated with this pipeline.
@@ -498,6 +514,7 @@ class IngestPipeline(TypedMooseResource, Generic[T]):
     table: Optional[OlapTable[T]] = None
     stream: Optional[Stream[T]] = None
     ingest_api: Optional[IngestApi[T]] = None
+    dead_letter_queue: Optional[DeadLetterQueue[T]] = None
     metadata: Optional[dict] = None
 
     def get_table(self) -> OlapTable[T]:
@@ -561,6 +578,12 @@ class IngestPipeline(TypedMooseResource, Generic[T]):
                 stream_config.version = config.version
             stream_config.metadata = stream_metadata
             self.stream = Stream(name, stream_config, t=self._t)
+        if config.dead_letter_queue:
+            stream_config = StreamConfig() if config.dead_letter_queue is True else config.dead_letter_queue
+            if config.version:
+                stream_config.version = config.version
+            stream_config.metadata = stream_metadata
+            self.dead_letter_queue = DeadLetterQueue(f"{name}DeadLetterQueue", stream_config, t=self._t)
         if config.ingest:
             if self.stream is None:
                 raise ValueError("Ingest API needs a stream to write to.")
@@ -570,6 +593,8 @@ class IngestPipeline(TypedMooseResource, Generic[T]):
             ingest_config_dict["destination"] = self.stream
             if config.version:
                 ingest_config_dict["version"] = config.version
+            if self.dead_letter_queue:
+                ingest_config_dict["dead_letter_queue"] = self.dead_letter_queue
             ingest_config_dict["metadata"] = ingest_metadata
             ingest_config = IngestConfigWithDestination(**ingest_config_dict)
             self.ingest_api = IngestApi(name, ingest_config, t=self._t)
@@ -694,13 +719,13 @@ class SqlResource:
     pushes_data_to: list[Union[OlapTable, "SqlResource"]]
 
     def __init__(
-        self,
-        name: str,
-        setup: list[str],
-        teardown: list[str],
-        pulls_data_from: Optional[list[Union[OlapTable, "SqlResource"]]] = None,
-        pushes_data_to: Optional[list[Union[OlapTable, "SqlResource"]]] = None,
-        metadata: dict = None
+            self,
+            name: str,
+            setup: list[str],
+            teardown: list[str],
+            pulls_data_from: Optional[list[Union[OlapTable, "SqlResource"]]] = None,
+            pushes_data_to: Optional[list[Union[OlapTable, "SqlResource"]]] = None,
+            metadata: dict = None
     ):
         self.name = name
         self.setup = setup
@@ -721,7 +746,8 @@ class View(SqlResource):
                      that this view depends on.
     """
 
-    def __init__(self, name: str, select_statement: str, base_tables: list[Union[OlapTable, SqlResource]], metadata: dict = None):
+    def __init__(self, name: str, select_statement: str, base_tables: list[Union[OlapTable, SqlResource]],
+                 metadata: dict = None):
         setup = [
             f"CREATE VIEW IF NOT EXISTS {name} AS {select_statement}".strip()
         ]
@@ -810,3 +836,146 @@ class MaterializedView(SqlResource, BaseTypedResource, Generic[T]):
 
         self.target_table = target_table
         self.config = options
+
+
+@dataclasses.dataclass
+class TaskConfig(Generic[T_none, U_none]):
+    """Configuration for a Task.
+
+    Attributes:
+        run: The handler function that executes the task logic.
+        on_complete: Optional list of tasks to run after this task completes.
+        timeout: Optional timeout string (e.g. "5m", "1h").
+        retries: Optional number of retry attempts.
+    """
+    run: TaskRunFunc[T_none, U_none]
+    on_complete: Optional[list["Task[U_none, Any]"]] = None
+    timeout: Optional[str] = None
+    retries: Optional[int] = None
+
+
+class Task(TypedMooseResource, Generic[T_none, U_none]):
+    """Represents a task that can be executed as part of a workflow.
+
+    Tasks are the basic unit of work in a workflow, with typed input and output.
+    They can be chained together using the on_complete configuration.
+
+    Args:
+        name: The name of the task.
+        config: Configuration specifying the task's behavior.
+        t: The Pydantic model defining the task's input schema
+           (passed via `Task[InputModel, OutputModel](...)`).
+           OutputModel can be None for tasks that don't return a value.
+
+    Attributes:
+        config (TaskConfig[T, U]): The configuration for this task.
+        columns (Columns[T]): Helper for accessing input field names safely.
+        name (str): The name of the task.
+        model_type (type[T]): The Pydantic model associated with this task's input.
+    """
+    config: TaskConfig[T_none, U_none]
+
+    def __init__(self, name: str, config: TaskConfig[T_none, U_none], **kwargs):
+        super().__init__()
+        self._set_type(name, self._get_type(kwargs))
+        self.config = config
+
+    @classmethod
+    def _get_type(cls, keyword_args: dict):
+        t = keyword_args.get('t')
+        if t is None:
+            raise ValueError(f"Use `{cls.__name__}[T, U](name='...')` to supply both input and output types")
+        if not isinstance(t, tuple) or len(t) != 2:
+            raise ValueError(f"Use `{cls.__name__}[T, U](name='...')` to supply both input and output types")
+
+        input_type, output_type = t
+        if input_type is not None and (not isinstance(input_type, type) or not issubclass(input_type, BaseModel)):
+            raise ValueError(f"Input type {input_type} is not a Pydantic model or None")
+        if output_type is not None and (not isinstance(output_type, type) or not issubclass(output_type, BaseModel)):
+            raise ValueError(f"Output type {output_type} is not a Pydantic model or None")
+        return t
+
+    def _set_type(self, name: str, t: tuple[type[T_none], type[U_none]]):
+        input_type, output_type = t
+        self._t = input_type
+        self._u = output_type
+        self.name = name
+
+
+@dataclasses.dataclass
+class WorkflowConfig:
+    """Configuration for a workflow.
+
+    Attributes:
+        starting_task: The first task to execute in the workflow.
+        retries: Optional number of retry attempts for the entire workflow.
+        timeout: Optional timeout string for the entire workflow.
+        schedule: Optional cron-like schedule string for recurring execution.
+    """
+    starting_task: Task[Any, Any]
+    retries: Optional[int] = None
+    timeout: Optional[str] = None
+    schedule: Optional[str] = None
+
+
+class Workflow:
+    """Represents a workflow composed of one or more tasks.
+
+    Workflows define a sequence of tasks to be executed, with optional
+    scheduling, retries, and timeouts at the workflow level.
+
+    Args:
+        name: The name of the workflow.
+        config: Configuration specifying the workflow's behavior.
+
+    Attributes:
+        name (str): The name of the workflow.
+        config (WorkflowConfig): The configuration for this workflow.
+    """
+    def __init__(self, name: str, config: WorkflowConfig):
+        self.name = name
+        self.config = config
+        # Register the workflow in the internal registry
+        _workflows[name] = self
+
+    def get_task_names(self) -> list[str]:
+        """Get a list of all task names in this workflow.
+
+        Returns:
+            list[str]: List of task names in the workflow, including all child tasks
+        """
+        def collect_task_names(task: Task) -> list[str]:
+            names = [task.name]
+            if task.config.on_complete:
+                for child in task.config.on_complete:
+                    names.extend(collect_task_names(child))
+            return names
+
+        return collect_task_names(self.config.starting_task)
+
+    def get_task(self, task_name: str) -> Optional[Task]:
+        """Find a task in this workflow by name.
+
+        Args:
+            task_name: The name of the task to find
+
+        Returns:
+            Optional[Task]: The task if found, None otherwise
+        """
+        def find_task(task: Task) -> Optional[Task]:
+            if task.name == task_name:
+                return task
+            if task.config.on_complete:
+                for child in task.config.on_complete:
+                    found = find_task(child)
+                    if found:
+                        return found
+            return None
+
+        return find_task(self.config.starting_task)
+
+def _get_workflows() -> dict[str, Workflow]:
+    return _workflows
+
+def _get_workflow(name: str) -> Optional[Workflow]:
+    return _workflows.get(name)

moose_lib/internal.py

@@ -11,7 +11,7 @@ from typing import Literal, Optional, List, Any
 from pydantic import BaseModel, ConfigDict, AliasGenerator
 import json
 from .data_models import Column, _to_columns
-from moose_lib.dmv2 import _tables, _streams, _ingest_apis, _egress_apis, SqlResource, _sql_resources
+from moose_lib.dmv2 import _tables, _streams, _ingest_apis, _egress_apis, SqlResource, _sql_resources, _workflows
 from moose_lib.dmv2 import OlapTable, View, MaterializedView
 from pydantic.alias_generators import to_camel
 from pydantic.json_schema import JsonSchemaValue
@@ -110,6 +110,7 @@ class IngestApiConfig(BaseModel):
     name: str
     columns: List[Column]
     write_to: Target
+    dead_letter_queue: Optional[str] = None
     version: Optional[str] = None
     metadata: Optional[dict] = None
 
@@ -131,6 +132,22 @@ class EgressApiConfig(BaseModel):
     version: Optional[str] = None
     metadata: Optional[dict] = None
 
+class WorkflowJson(BaseModel):
+    """Internal representation of a workflow configuration for serialization.
+
+    Attributes:
+        name: Name of the workflow.
+        retries: Optional number of retry attempts for the entire workflow.
+        timeout: Optional timeout string for the entire workflow.
+        schedule: Optional cron-like schedule string for recurring execution.
+    """
+    model_config = model_config
+
+    name: str
+    retries: Optional[int] = None
+    timeout: Optional[str] = None
+    schedule: Optional[str] = None
+
 class InfrastructureSignatureJson(BaseModel):
     """Represents the unique signature of an infrastructure component (Table, Topic, etc.).
 
@@ -175,6 +192,7 @@ class InfrastructureMap(BaseModel):
         ingest_apis: Dictionary mapping ingest API names to their configurations.
         egress_apis: Dictionary mapping egress API names to their configurations.
         sql_resources: Dictionary mapping SQL resource names to their configurations.
+        workflows: Dictionary mapping workflow names to their configurations.
     """
     model_config = model_config
 
@@ -183,6 +201,7 @@ class InfrastructureMap(BaseModel):
     ingest_apis: dict[str, IngestApiConfig]
     egress_apis: dict[str, EgressApiConfig]
     sql_resources: dict[str, SqlResourceConfig]
+    workflows: dict[str, WorkflowJson]
 
 
 def _map_sql_resource_ref(r: Any) -> InfrastructureSignatureJson:
@@ -233,6 +252,7 @@ def to_infra_map() -> dict:
     ingest_apis = {}
     egress_apis = {}
     sql_resources = {}
+    workflows = {}
 
     for name, table in _tables.items():
         engine = table.config.engine
@@ -287,6 +307,7 @@ def to_infra_map() -> dict:
                 name=api.config.destination.name
             ),
             metadata=getattr(api, "metadata", None),
+            dead_letter_queue=api.config.dead_letter_queue.name
         )
 
     for name, api in _egress_apis.items():
@@ -308,12 +329,21 @@ def to_infra_map() -> dict:
             metadata=getattr(resource, "metadata", None),
         )
 
+    for name, workflow in _workflows.items():
+        workflows[name] = WorkflowJson(
+            name=workflow.name,
+            retries=workflow.config.retries,
+            timeout=workflow.config.timeout,
+            schedule=workflow.config.schedule,
+        )
+
     infra_map = InfrastructureMap(
         tables=tables,
         topics=topics,
         ingest_apis=ingest_apis,
         egress_apis=egress_apis,
-        sql_resources=sql_resources
+        sql_resources=sql_resources,
+        workflows=workflows
     )
 
     return infra_map.model_dump(by_alias=True)

{moose_lib-0.4.212.dist-info → moose_lib-0.4.214.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.4.212
+Version: 0.4.214
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com
@@ -11,6 +11,7 @@ Requires-Dist: pydantic==2.10.6
 Requires-Dist: temporalio==1.9.0
 Requires-Dist: kafka-python-ng==2.2.2
 Requires-Dist: redis==6.2.0
+Requires-Dist: humanfriendly==10.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{moose_lib-0.4.212.dist-info → moose_lib-0.4.214.dist-info}/RECORD

@@ -3,8 +3,8 @@ moose_lib/blocks.py,sha256=_wdvC2NC_Y3MMEnB71WTgWbeQ--zPNHk19xjToJW0C0,3185
 moose_lib/commons.py,sha256=BV5X78MuOWHiZV9bsWSN69JIvzTNWUi-gnuMiAtaO8A,2489
 moose_lib/data_models.py,sha256=R6do1eQqHK6AZ4GTP5tOPtSZaltjZurfx9_Asji7Dwc,8529
 moose_lib/dmv2-serializer.py,sha256=CL_Pvvg8tJOT8Qk6hywDNzY8MYGhMVdTOw8arZi3jng,49
-moose_lib/dmv2.py,sha256=jSqN9ST7xhYCqUP4kJLDxIHMCsRvbkiTJAd27kb6Nfk,31803
-moose_lib/internal.py,sha256=gREvC3XxBFN4i7JL5uMj0riCu_JUO2YyiMZvCokg1ME,13101
+moose_lib/dmv2.py,sha256=JFNRjTWUcuTw865Wy8E_UgjLeLZPWAU3m7Ec2rd3CKM,38404
+moose_lib/internal.py,sha256=8aEXQjB2DWz-dGUJ0ch8g5awHxELoZx1XYmlVXlhCWo,14193
 moose_lib/main.py,sha256=In-u7yA1FsLDeP_2bhIgBtHY_BkXaZqDwf7BxwyC21c,8471
 moose_lib/query_param.py,sha256=AB5BKu610Ji-h1iYGMBZKfnEFqt85rS94kzhDwhWJnc,6288
 moose_lib/tasks.py,sha256=6MXA0j7nhvQILAJVTQHCAsquwrSOi2zAevghAc_7kXs,1554
@@ -16,7 +16,7 @@ tests/__init__.py,sha256=0Gh4yzPkkC3TzBGKhenpMIxJcRhyrrCfxLSfpTZnPMQ,53
 tests/conftest.py,sha256=ZVJNbnr4DwbcqkTmePW6U01zAzE6QD0kNAEZjPG1f4s,169
 tests/test_moose.py,sha256=mBsx_OYWmL8ppDzL_7Bd7xR6qf_i3-pCIO3wm2iQNaA,2136
 tests/test_redis_client.py,sha256=d9_MLYsJ4ecVil_jPB2gW3Q5aWnavxmmjZg2uYI3LVo,3256
-moose_lib-0.4.212.dist-info/METADATA,sha256=v5sDuNe4eTw8OkY_RXFRlTE4Az-07umLE-Hyy-vJuZY,603
-moose_lib-0.4.212.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-moose_lib-0.4.212.dist-info/top_level.txt,sha256=XEns2-4aCmGp2XjJAeEH9TAUcGONLnSLy6ycT9FSJh8,16
-moose_lib-0.4.212.dist-info/RECORD,,
+moose_lib-0.4.214.dist-info/METADATA,sha256=rpuUzKwIew2DM6hjSWiSfvyxcJXSMXdOg64XyzhA_IQ,638
+moose_lib-0.4.214.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+moose_lib-0.4.214.dist-info/top_level.txt,sha256=XEns2-4aCmGp2XjJAeEH9TAUcGONLnSLy6ycT9FSJh8,16
+moose_lib-0.4.214.dist-info/RECORD,,