ddeutil-workflow 0.0.10__py3-none-any.whl → 0.0.11__py3-none-any.whl

ddeutil/workflow/stage.py

@@ -5,7 +5,7 @@
 # ------------------------------------------------------------------------------
 """Stage Model that use for getting stage data template from Job Model.
 The stage that handle the minimize task that run in some thread (same thread at
-its job owner) that mean it is the lowest executor of a pipeline workflow that
+its job owner) that mean it is the lowest executor of a workflow workflow that
 can tracking logs.
 
     The output of stage execution only return 0 status because I do not want to
@@ -551,12 +551,12 @@ class HookStage(BaseStage):
 
 
 class TriggerStage(BaseStage):
-    """Trigger Pipeline execution stage that execute another pipeline object.
+    """Trigger Workflow execution stage that execute another workflow object.
 
     Data Validate:
         >>> stage = {
-        ...     "name": "Trigger pipeline stage execution",
-        ...     "trigger": 'pipeline-name-for-loader',
+        ...     "name": "Trigger workflow stage execution",
+        ...     "trigger": 'workflow-name-for-loader',
         ...     "params": {
         ...         "run-date": "2024-08-01",
         ...         "source": "src",
@@ -564,31 +564,31 @@ class TriggerStage(BaseStage):
         ... }
     """
 
-    trigger: str = Field(description="A trigger pipeline name.")
+    trigger: str = Field(description="A trigger workflow name.")
     params: DictData = Field(
         default_factory=dict,
-        description="A parameter that want to pass to pipeline execution.",
+        description="A parameter that want to pass to workflow execution.",
     )
 
     @handler_result("Raise from TriggerStage")
     def execute(self, params: DictData) -> Result:
-        """Trigger pipeline execution.
+        """Trigger workflow execution.
 
         :param params: A parameter data that want to use in this execution.
         :rtype: Result
         """
-        from .pipeline import Pipeline
+        from . import Workflow
 
-        # NOTE: Loading pipeline object from trigger name.
+        # NOTE: Loading workflow object from trigger name.
         _trigger: str = param2template(self.trigger, params=params)
 
-        # NOTE: Set running pipeline ID from running stage ID to external
+        # NOTE: Set running workflow ID from running stage ID to external
         #   params on Loader object.
-        pipe: Pipeline = Pipeline.from_loader(
+        wf: Workflow = Workflow.from_loader(
             name=_trigger, externals={"run_id": self.run_id}
         )
         logger.info(f"({self.run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
-        return pipe.execute(params=param2template(self.params, params))
+        return wf.execute(params=param2template(self.params, params))
 
 
 # NOTE: Order of parsing stage data

ddeutil/workflow/utils.py

@@ -151,7 +151,7 @@ def config() -> ConfParams:
 
 class SimLoad:
     """Simple Load Object that will search config data by given some identity
-    value like name of pipeline or on.
+    value like name of workflow or on.
 
     :param name: A name of config data that will read by Yaml Loader object.
     :param params: A Params model object.
@@ -517,7 +517,7 @@ Param = Union[
 
 class Result(BaseModel):
     """Result Pydantic Model for passing parameter and receiving output from
-    the pipeline execution.
+    the workflow execution.
     """
 
     status: int = Field(default=2)
@@ -655,7 +655,7 @@ def get_args_const(
 
     if len(body) > 1:
         raise UtilException(
-            "Post-filter function should be only one calling per pipe"
+            "Post-filter function should be only one calling per wf"
         )
 
     caller: Union[Name, Call]
@@ -771,7 +771,7 @@ def str2template(
     ``RE_CALLER`` regular expression.
 
         The getter value that map a template should have typing support align
-    with the pipeline parameter types that is `str`, `int`, `datetime`, and
+    with the workflow parameter types that is `str`, `int`, `datetime`, and
     `list`.
 
     :param value: A string value that want to mapped with an params

{ddeutil_workflow-0.0.10.dist-info → ddeutil_workflow-0.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.10
+Version: 0.0.11
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -24,10 +24,9 @@ License-File: LICENSE
 Requires-Dist: ddeutil-io
 Requires-Dist: python-dotenv ==1.0.1
 Requires-Dist: typer <1.0.0,==0.12.5
+Requires-Dist: schedule <2.0.0,==1.2.2
 Provides-Extra: api
-Requires-Dist: fastapi[standard] <1.0.0,==0.112.2 ; extra == 'api'
-Provides-Extra: schedule
-Requires-Dist: schedule <2.0.0,==1.2.2 ; extra == 'schedule'
+Requires-Dist: fastapi <1.0.0,==0.112.2 ; extra == 'api'
 
 # Workflow
 
@@ -38,22 +37,22 @@ Requires-Dist: schedule <2.0.0,==1.2.2 ; extra == 'schedule'
 [![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 The **Lightweight workflow orchestration** with less dependencies the was created
-for easy to make a simple metadata driven for data pipeline orchestration.
+for easy to make a simple metadata driven for data workflow orchestration.
 It can to use for data operator by a `.yaml` template.
 
 > [!WARNING]
 > This package provide only orchestration workload. That mean you should not use
 > workflow stage to process any large data which use lot of compute usecase.
 
-In my opinion, I think it should not create duplicate pipeline codes if I can
-write with dynamic input parameters on the one template pipeline that just change
+In my opinion, I think it should not create duplicate workflow codes if I can
+write with dynamic input parameters on the one template workflow that just change
 the input parameters per use-case instead.
-This way I can handle a lot of logical pipelines in our orgs with only metadata
-configuration. It called **Metadata Driven Data Pipeline**.
+This way I can handle a lot of logical workflows in our orgs with only metadata
+configuration. It called **Metadata Driven Data Workflow**.
 
 Next, we should get some monitoring tools for manage logging that return from
-pipeline running. Because it not show us what is a use-case that running data
-pipeline.
+workflow running. Because it not show us what is a use-case that running data
+workflow.
 
 > [!NOTE]
 > _Disclaimer_: I inspire the dynamic statement from the GitHub Action `.yml` files
@@ -75,8 +74,7 @@ this package with application add-ons, you should add `app` in installation;
 | Usecase           | Install Optional                         | Support            |
 |-------------------|------------------------------------------|--------------------|
 | Python & CLI      | `pip install ddeutil-workflow`           | :heavy_check_mark: |
-| Scheduler Service | `pip install ddeutil-workflow[schedule]` | :x:                |
-| FastAPI Server    | `pip install ddeutil-workflow[api]`      | :x:                |
+| FastAPI Server    | `pip install ddeutil-workflow[api]`      | :heavy_check_mark: |
 
 
 > I added this feature to the main milestone.
@@ -97,67 +95,65 @@ use-case.
 
 > [!IMPORTANT]
 > I recommend you to use the `hook` stage for all actions that you want to do
-> with pipeline activity that you want to orchestrate. Because it able to dynamic
+> with workflow activity that you want to orchestrate. Because it able to dynamic
 > an input argument with the same hook function that make you use less time to
-> maintenance your data pipelines.
+> maintenance your data workflows.
 
 ```yaml
 run_py_local:
-  type: pipeline.Pipeline
-  on:
-    - cronjob: '*/5 * * * *'
-      timezone: "Asia/Bangkok"
-  params:
-    author-run: str
-    run-date: datetime
-  jobs:
-    getting-api-data:
-      stages:
-        - name: "Retrieve API Data"
-          id: retrieve-api
-          uses: tasks/get-api-with-oauth-to-s3@requests
-          with:
-            url: https://open-data/
-            auth: ${API_ACCESS_REFRESH_TOKEN}
-            aws_s3_path: my-data/open-data/
-
-            # This Authentication code should implement with your custom hook function.
-            # The template allow you to use environment variable.
-            aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
-            aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+   type: Workflow
+   on:
+      # If workflow deploy to schedule, it will running every 5 minutes
+      # with Asia/Bangkok timezone.
+      - cronjob: '*/5 * * * *'
+        timezone: "Asia/Bangkok"
+   params:
+      # Incoming execution parameters will validate with this type. It allow
+      # to set default value or templating.
+      author-run: str
+      run-date: datetime
+   jobs:
+      getting-api-data:
+         stages:
+            - name: "Retrieve API Data"
+              id: retrieve-api
+              uses: tasks/get-api-with-oauth-to-s3@requests
+              with:
+                 url: https://open-data/
+                 auth: ${API_ACCESS_REFRESH_TOKEN}
+                 aws_s3_path: my-data/open-data/
+
+                 # This Authentication code should implement with your custom hook function.
+                 # The template allow you to use environment variable.
+                 aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
+                 aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
 ```
 
 ## Configuration
 
-| Environment                         | Component | Default                      | Description                                                                |
-|-------------------------------------|-----------|------------------------------|----------------------------------------------------------------------------|
-| `WORKFLOW_ROOT_PATH`                | Core      | .                            | The root path of the workflow application                                  |
-| `WORKFLOW_CORE_REGISTRY`            | Core      | ddeutil.workflow,tests.utils | List of importable string for the hook stage                               |
-| `WORKFLOW_CORE_REGISTRY_FILTER`     | Core      | ddeutil.workflow.utils       | List of importable string for the filter template                          |
-| `WORKFLOW_CORE_PATH_CONF`           | Core      | conf                         | The config path that keep all template `.yaml` files                       |
-| `WORKFLOW_CORE_TIMEZONE`            | Core      | Asia/Bangkok                 | A Timezone string value that will pass to `ZoneInfo` object                |
-| `WORKFLOW_CORE_STAGE_DEFAULT_ID`    | Core      | true                         | A flag that enable default stage ID that use for catch an execution output |
-| `WORKFLOW_CORE_STAGE_RAISE_ERROR`   | Core      | true                         | A flag that all stage raise StageException from stage execution            |
-| `WORKFLOW_CORE_MAX_PIPELINE_POKING` | Core      | 4                            |                                                                            |
-| `WORKFLOW_CORE_MAX_JOB_PARALLEL`    | Core      | 2                            | The maximum job number that able to run parallel in pipeline executor      |
-| `WORKFLOW_LOG_DEBUG_MODE`           | Log       | true                         | A flag that enable logging with debug level mode                           |
-| `WORKFLOW_LOG_ENABLE_WRITE`         | Log       | true                         | A flag that enable logging object saving log to its destination            |
-
-
-**Application**:
-
-| Environment                         | Default                          | Description                                                             |
-|-------------------------------------|----------------------------------|-------------------------------------------------------------------------|
-| `WORKFLOW_APP_PROCESS_WORKER`       | 2                                | The maximum process worker number that run in scheduler app module      |
-| `WORKFLOW_APP_SCHEDULE_PER_PROCESS` | 100                              | A schedule per process that run parallel                                |
-| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format |
-
-**API server**:
-
-| Environment                          | Default | Description                                                                       |
-|--------------------------------------|---------|-----------------------------------------------------------------------------------|
-| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | true    | A flag that enable workflow route to manage execute manually and workflow logging |
-| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | true    | A flag that enable run scheduler                                                  |
+| Environment                         | Component | Default                          | Description                                                                |
+|-------------------------------------|-----------|----------------------------------|----------------------------------------------------------------------------|
+| `WORKFLOW_ROOT_PATH`                | Core      | .                                | The root path of the workflow application                                  |
+| `WORKFLOW_CORE_REGISTRY`            | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage                               |
+| `WORKFLOW_CORE_REGISTRY_FILTER`     | Core      | ddeutil.workflow.utils           | List of importable string for the filter template                          |
+| `WORKFLOW_CORE_PATH_CONF`           | Core      | conf                             | The config path that keep all template `.yaml` files                       |
+| `WORKFLOW_CORE_TIMEZONE`            | Core      | Asia/Bangkok                     | A Timezone string value that will pass to `ZoneInfo` object                |
+| `WORKFLOW_CORE_STAGE_DEFAULT_ID`    | Core      | true                             | A flag that enable default stage ID that use for catch an execution output |
+| `WORKFLOW_CORE_STAGE_RAISE_ERROR`   | Core      | true                             | A flag that all stage raise StageException from stage execution            |
+| `WORKFLOW_CORE_MAX_NUM_POKING`      | Core      | 4                                |                                                                            |
+| `WORKFLOW_CORE_MAX_JOB_PARALLEL`    | Core      | 2                                | The maximum job number that able to run parallel in workflow executor      |
+| `WORKFLOW_LOG_DEBUG_MODE`           | Log       | true                             | A flag that enable logging with debug level mode                           |
+| `WORKFLOW_LOG_ENABLE_WRITE`         | Log       | true                             | A flag that enable logging object saving log to its destination            |
+| `WORKFLOW_APP_PROCESS_WORKER`       | Schedule  | 2                                | The maximum process worker number that run in scheduler app module         |
+| `WORKFLOW_APP_SCHEDULE_PER_PROCESS` | Schedule  | 100                              | A schedule per process that run parallel                                   |
+| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`  | Schedule  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format    |
+
+**API Application**:
+
+| Environment                          | Component | Default | Description                                                                       |
+|--------------------------------------|-----------|---------|-----------------------------------------------------------------------------------|
+| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | API       | true    | A flag that enable workflow route to manage execute manually and workflow logging |
+| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | API       | true    | A flag that enable run scheduler                                                  |
 
 ## Deployment
 
@@ -165,18 +161,18 @@ This package able to run as a application service for receive manual trigger
 from the master node via RestAPI or use to be Scheduler background service
 like crontab job but via Python API.
 
-### Schedule Service
+### Schedule App
 
 ```shell
-(venv) $ python src.ddeutil.workflow.app
+(venv) $ ddeutil-workflow schedule
 ```
 
 ### API Server
 
 ```shell
-(venv) $ uvicorn src.ddeutil.workflow.api:app --host 0.0.0.0 --port 80 --reload
+(venv) $ uvicorn src.ddeutil.workflow.api:app --host 127.0.0.1 --port 80
 ```
 
 > [!NOTE]
 > If this package already deploy, it able to use
-> `uvicorn ddeutil.workflow.api:app --host 0.0.0.0 --port 80 --workers 4`
+> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`

ddeutil_workflow-0.0.11.dist-info/RECORD

@@ -0,0 +1,21 @@
+ddeutil/workflow/__about__.py,sha256=i9hlE_wZTG59lDKZ8plCEpWU3EdeUe141ACbjcsAewc,28
+ddeutil/workflow/__init__.py,sha256=aEQiEWwTPGhfwpzzdb99xXaHchi5ABWUHl2iLIyT18E,664
+ddeutil/workflow/__types.py,sha256=SYMoxbENQX8uPsiCZkjtpHAqqHOh8rUrarAFicAJd0E,1773
+ddeutil/workflow/api.py,sha256=xVP8eGu1nnR8HM0ULTwxs9TV9tsxCOjZ68cAffw2f3o,4802
+ddeutil/workflow/cli.py,sha256=Ikcq526WeIl-737-v55T0PwAZ2pNiZFxlN0Y-DjhDbQ,3374
+ddeutil/workflow/cron.py,sha256=uhp3E5pl_tX_H88bsDujcwdhZmOE53csyV-ouPpPdK8,25321
+ddeutil/workflow/exceptions.py,sha256=Uf1-Tn8rAzj0aiVHSqo4fBqO80W0za7UFZgKv24E-tg,706
+ddeutil/workflow/job.py,sha256=eESvmIbIyYtiKgVLfILtb0lTz-shMhy4Bi7kHrtZSk0,19663
+ddeutil/workflow/log.py,sha256=bZyyqf3oNBB8oRf8RI0YvII7wHHoj4wC-nmW_pQjQ1c,6036
+ddeutil/workflow/on.py,sha256=Sxwnu0vPbIrMR_WWvH3_rOvD0tbiJntcB5378WoV19M,7163
+ddeutil/workflow/repeat.py,sha256=e3dekPTlMlxCCizfBYsZ8dD8Juy4rtfqDZJU3Iky2oA,5011
+ddeutil/workflow/route.py,sha256=ABEk-WlVo9XGFc7zCPbckX33URCNH7woQFU1keX_8PQ,6970
+ddeutil/workflow/scheduler.py,sha256=ISiVoKM0puh3XaXZ9NTi-J-vREGxdEa-lk1jR6a4OXk,41639
+ddeutil/workflow/stage.py,sha256=nYsKKT5ZKelEsFnDScBwaitXOv_aUEFkubDPQVK5isM,20644
+ddeutil/workflow/utils.py,sha256=TbqgPkDDYBpqCZ7HV2TU3AH1_Mv-zfrJdwVL-l2SPUo,28559
+ddeutil_workflow-0.0.11.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.11.dist-info/METADATA,sha256=QOsSVDJmVdgrXzmtSYmE5WVI7u69EGlKfsALWjE0by8,9395
+ddeutil_workflow-0.0.11.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+ddeutil_workflow-0.0.11.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
+ddeutil_workflow-0.0.11.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.11.dist-info/RECORD,,

{ddeutil_workflow-0.0.10.dist-info → ddeutil_workflow-0.0.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (74.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any