ddeutil-workflow 0.0.27__py3-none-any.whl → 0.0.29__py3-none-any.whl

ddeutil/workflow/workflow.py

@@ -138,7 +138,7 @@ class WorkflowQueue:
         """Construct WorkflowQueue object from an input queue value that passing
         with list of datetime or list of WorkflowRelease.
 
-        :raise TypeError: If the type of an input queue does not valid.
+        :raise TypeError: If the type of input queue does not valid.
 
         :rtype: Self
         """
@@ -226,9 +226,9 @@ class WorkflowQueue:
 class Workflow(BaseModel):
     """Workflow Pydantic model.
 
-        This is the main future of this project because it use to be workflow
+        This is the main future of this project because it uses to be workflow
     data for running everywhere that you want or using it to scheduler task in
-    background. It use lightweight coding line from Pydantic Model and enhance
+    background. It uses lightweight coding line from Pydantic Model and enhance
     execute method on it.
     """
 
@@ -317,7 +317,7 @@ class Workflow(BaseModel):
     @model_validator(mode="before")
     def __prepare_model_before__(cls, values: DictData) -> DictData:
         """Prepare the params key in the data model before validating."""
-        # NOTE: Prepare params type if it passing with only type value.
+        # NOTE: Prepare params type if it is passing with only type value.
         if params := values.pop("params", {}):
             values["params"] = {
                 p: (
@@ -341,7 +341,7 @@ class Workflow(BaseModel):
     @field_validator("on", mode="after")
     def __on_no_dup_and_reach_limit__(cls, value: list[On]) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
-        contain the every minute value more than one value, it will remove to
+        contains the every minute value more than one value, it will remove to
         only one value.
 
         :raise ValueError: If it has some duplicate value.
@@ -359,8 +359,8 @@ class Workflow(BaseModel):
         # WARNING:
         # if '* * * * *' in set_ons and len(set_ons) > 1:
         #     raise ValueError(
-        #         "If it has every minute cronjob on value, it should has only "
-        #         "one value in the on field."
+        #         "If it has every minute cronjob on value, it should have "
+        #         "only one value in the on field."
         #     )
 
         if len(set_ons) > config.max_on_per_workflow:
@@ -372,7 +372,7 @@ class Workflow(BaseModel):
 
     @model_validator(mode="after")
     def __validate_jobs_need__(self) -> Self:
-        """Validate each need job in any jobs should exists.
+        """Validate each need job in any jobs should exist.
 
         :raise WorkflowException: If it has not exists need value in this
             workflow job.
@@ -591,10 +591,10 @@ class Workflow(BaseModel):
         """Generate queue of datetime from the cron runner that initialize from
         the on field. with offset value.
 
-        :param offset: A offset in second unit for time travel.
+        :param offset: An offset in second unit for time travel.
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
-        :param log: A log class that want to making log object.
+        :param log: A log class that want to make log object.
         :param force_run: A flag that allow to release workflow if the log with
             that release was pointed.
 
@@ -696,7 +696,7 @@ class Workflow(BaseModel):
             start_date: datetime = current_date
             offset: float = 0
 
-        # NOTE: End date is use to stop generate queue with an input periods
+        # NOTE: End date is using to stop generate queue with an input periods
         #   value.
         end_date: datetime = start_date + timedelta(minutes=periods)
 
@@ -812,7 +812,7 @@ class Workflow(BaseModel):
         :param params: A params that was parameterized from workflow execution.
         :param run_id: A workflow running ID for this job execution.
         :param raise_error: A flag that raise error instead catching to result
-            if it get exception from job execution.
+            if it gets exception from job execution.
 
         :rtype: Result
         :return: Return the result object that receive the job execution result
@@ -868,8 +868,8 @@ class Workflow(BaseModel):
         """Execute workflow with passing a dynamic parameters to all jobs that
         included in this workflow model with ``jobs`` field.
 
-            The result of execution process for each jobs and stages on this
-        workflow will keeping in dict which able to catch out with all jobs and
+            The result of execution process for each job and stages on this
+        workflow will keep in dict which able to catch out with all jobs and
         stages by dot annotation.
 
             For example, when I want to use the output from previous stage, I
@@ -884,7 +884,9 @@ class Workflow(BaseModel):
         :param run_id: A workflow running ID for this job execution.
         :type run_id: str | None (default: None)
         :param timeout: A workflow execution time out in second unit that use
-            for limit time of execution and waiting job dependency.
+            for limit time of execution and waiting job dependency. This value
+            does not force stop the task that still running more than this limit
+            time.
         :type timeout: int (default: 0)
 
         :rtype: Result
@@ -907,8 +909,8 @@ class Workflow(BaseModel):
             )
             return rs.catch(status=0, context=params)
 
-        # NOTE: Create a job queue that keep the job that want to running after
-        #   it dependency condition.
+        # NOTE: Create a job queue that keep the job that want to run after
+        #   its dependency condition.
         jq: Queue = Queue()
         for job_id in self.jobs:
             jq.put(job_id)
@@ -967,7 +969,7 @@ class Workflow(BaseModel):
 
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
-            timeout.
+            time out.
         :param job_queue: A job queue object.
         :param timeout: A second value unit that bounding running time.
         :param thread_timeout: A timeout to waiting all futures complete.
@@ -1064,7 +1066,7 @@ class Workflow(BaseModel):
 
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
-            timeout.
+            time out.
         :param timeout: A second value unit that bounding running time.
 
         :rtype: DictData
@@ -1090,7 +1092,7 @@ class Workflow(BaseModel):
                 continue
 
             # NOTE: Start workflow job execution with deep copy context data
-            #   before release. This job execution process will running until
+            #   before release. This job execution process will run until
             #   done before checking all execution timeout or not.
             #
             #   {
@@ -1182,7 +1184,7 @@ class WorkflowTask:
 
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
-        :param log: A log class that want to making log object.
+        :param log: A log class that want to make log object.
         :param force_run: A flag that allow to release workflow if the log with
             that release was pointed.
 
@@ -1220,6 +1222,7 @@ class WorkflowTask:
         return queue
 
     def __repr__(self) -> str:
+        """Override ___repr__ method."""
         return (
             f"{self.__class__.__name__}(alias={self.alias!r}, "
             f"workflow={self.workflow.name!r}, runner={self.runner!r}, "

ddeutil_workflow-0.0.29.dist-info/METADATA

@@ -0,0 +1,292 @@
+Metadata-Version: 2.2
+Name: ddeutil-workflow
+Version: 0.0.29
+Summary: Lightweight workflow orchestration
+Author-email: ddeutils <korawich.anu@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/ddeutils/ddeutil-workflow/
+Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
+Keywords: orchestration,workflow
+Classifier: Topic :: Utilities
+Classifier: Natural Language :: English
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.9.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: ddeutil>=0.4.6
+Requires-Dist: ddeutil-io[toml,yaml]>=0.2.3
+Requires-Dist: pydantic==2.10.6
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: schedule<2.0.0,==1.2.2
+Provides-Extra: api
+Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
+
+# Workflow Orchestration
+
+[![test](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
+[![codecov](https://codecov.io/gh/ddeutils/ddeutil-workflow/graph/badge.svg?token=3NDPN2I0H9)](https://codecov.io/gh/ddeutils/ddeutil-workflow)
+[![pypi version](https://img.shields.io/pypi/v/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
+[![python support version](https://img.shields.io/pypi/pyversions/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
+[![size](https://img.shields.io/github/languages/code-size/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow)
+[![gh license](https://img.shields.io/github/license/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
+[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
+The **Lightweight Workflow Orchestration** with fewer dependencies the was created
+for easy to make a simple metadata driven data workflow. It can use for data operator
+by a `.yaml` template.
+
+> [!WARNING]
+> This package provide only orchestration workload. That mean you should not
+> use the workflow stage to process any large volume data which use a lot of compute
+> resource :cold_sweat:.
+
+In my opinion, I think it should not create duplicate workflow codes if I can
+write with dynamic input parameters on the one template workflow that just change
+the input parameters per use-case instead.
+This way I can handle a lot of logical workflows in our orgs with only metadata
+configuration. It called **Metadata Driven Data Workflow**.
+
+---
+
+**:pushpin: <u>Rules of This Workflow engine</u>**:
+
+1. The Minimum frequency unit of scheduling is **1 minute** :warning:
+2. Can not re-run only failed stage and its pending downstream :rotating_light:
+3. All parallel tasks inside workflow engine use Multi-Threading
+   (Python 3.13 unlock GIL :unlock:)
+
+---
+
+**:memo: <u>Workflow Diagrams</u>**:
+
+This diagram show where is this application run on the production infrastructure.
+You will see that this application do only running code with stress-less which mean
+you should to set the data layer separate this core program before run this application.
+
+```mermaid
+flowchart LR
+    subgraph Interface
+    A((User))
+        subgraph Docker Container
+        G@{ shape: rounded, label: "Observe<br>Application" }
+        end
+    end
+
+    A --->|action| B(Workflow<br>Application)
+    B ---> |response| A
+    B -..-> |response| G
+    G -..-> |request| B
+
+    subgraph Docker Container
+    B
+    end
+
+    subgraph Data Context
+    D@{ shape: processes, label: "Logs" }
+    E@{ shape: lin-cyl, label: "Metadata" }
+    end
+
+    subgraph Git Context
+    F@{ shape: tag-rect, label: "YAML<br>files" }
+    end
+
+    B --->|disable| F
+    F --->|read| B
+
+    B --->|write| E
+    E --->|read| B
+    B --->|write| D
+
+    D -.->|read| G
+    E -.->|read| G
+```
+
+> [!NOTE]
+> _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
+> with `.yml` files and all configs file from several data orchestration framework
+> tools from my experience on Data Engineer. :grimacing:
+>
+> Other workflow tools that I interest on them and pick some interested feature
+> implement to this package:
+>
+> - [Google **Workflows**](https://cloud.google.com/workflows)
+> - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
+
+## :round_pushpin: Installation
+
+This project need `ddeutil` and `ddeutil-io` extension namespace packages.
+If you want to install this package with application add-ons, you should add
+`app` in installation;
+
+| Use-case       | Install Optional         | Support            |
+|----------------|--------------------------|--------------------|
+| Python         | `ddeutil-workflow`       | :heavy_check_mark: |
+| FastAPI Server | `ddeutil-workflow[api]`  | :heavy_check_mark: |
+
+## :beers: Usage
+
+This is examples that use workflow file for running common Data Engineering
+use-case.
+
+> [!IMPORTANT]
+> I recommend you to use the `hook` stage for all actions that you want to do
+> with workflow activity that you want to orchestrate. Because it is able to
+> dynamic an input argument with the same hook function that make you use less
+> time to maintenance your data workflows.
+
+```yaml
+run-py-local:
+
+   # Validate model that use to parsing exists for template file
+   type: Workflow
+   on:
+      # If workflow deploy to schedule, it will run every 5 minutes
+      # with Asia/Bangkok timezone.
+      - cronjob: '*/5 * * * *'
+        timezone: "Asia/Bangkok"
+   params:
+      # Incoming execution parameters will validate with this type. It allows
+      # to set default value or templating.
+      source-extract: str
+      run-date: datetime
+   jobs:
+      getting-api-data:
+         stages:
+            - name: "Retrieve API Data"
+              id: retrieve-api
+              uses: tasks/get-api-with-oauth-to-s3@requests
+              with:
+                 # Arguments of source data that want to retrieve.
+                 method: post
+                 url: https://finances/open-data/currency-pairs/
+                 body:
+                    resource: ${{ params.source-extract }}
+
+                    # You can use filtering like Jinja template but this
+                    # package does not use it.
+                    filter: ${{ params.run-date | fmt(fmt='%Y%m%d') }}
+                 auth:
+                    type: bearer
+                    keys: ${API_ACCESS_REFRESH_TOKEN}
+
+                 # Arguments of target data that want to land.
+                 writing_mode: flatten
+                 aws_s3_path: my-data/open-data/${{ params.source-extract }}
+
+                 # This Authentication code should implement with your custom hook
+                 # function. The template allow you to use environment variable.
+                 aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
+                 aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+```
+
+The above workflow template is main executor pipeline that you want to do. If you
+want to schedule this workflow, you want to dynamic its parameters change base on
+execution time such as `run-date` should change base on that workflow running date.
+
+So, this package provide the `Schedule` template for this action.
+
+```yaml
+schedule-run-local-wf:
+
+   # Validate model that use to parsing exists for template file
+   type: Schedule
+   workflows:
+
+      # Map existing workflow that want to deploy with scheduler application.
+      # It allows you to pass release parameter that dynamic change depend on the
+      # current context of this scheduler application releasing that time.
+      - name: run-py-local
+        params:
+          source-extract: "USD-THB"
+          asat-dt: "${{ release.logical_date }}"
+```
+
+## :cookie: Configuration
+
+The main configuration that use to dynamic changing with your objective of this
+application. If any configuration values do not set yet, it will use default value
+and do not raise any error to you.
+
+> [!IMPORTANT]
+> The config value that you will set on the environment should combine with
+> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
+
+| Name                         | Component | Default                           | Description                                                                                                        |
+|:-----------------------------|:---------:|:----------------------------------|:-------------------------------------------------------------------------------------------------------------------|
+| **ROOT_PATH**                |   Core    | `.`                               | The root path of the workflow application.                                                                         |
+| **REGISTRY**                 |   Core    | `.`                               | List of importable string for the hook stage.                                                                      |
+| **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`      | List of importable string for the filter template.                                                                 |
+| **CONF_PATH**                |   Core    | `conf`                            | The config path that keep all template `.yaml` files.                                                              |
+| **TIMEZONE**                 |   Core    | `Asia/Bangkok`                    | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
+| **STAGE_DEFAULT_ID**         |   Core    | `true`                            | A flag that enable default stage ID that use for catch an execution output.                                        |
+| **STAGE_RAISE_ERROR**        |   Core    | `false`                           | A flag that all stage raise StageException from stage execution.                                                   |
+| **JOB_DEFAULT_ID**           |   Core    | `false`                           | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
+| **JOB_RAISE_ERROR**          |   Core    | `true`                            | A flag that all job raise JobException from job strategy execution.                                                |
+| **MAX_NUM_POKING**           |   Core    | `4`                               | .                                                                                                                  |
+| **MAX_JOB_PARALLEL**         |   Core    | `2`                               | The maximum job number that able to run parallel in workflow executor.                                             |
+| **MAX_JOB_EXEC_TIMEOUT**     |   Core    | `600`                             |                                                                                                                    |
+| **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                               |                                                                                                                    |
+| **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                              |                                                                                                                    |
+| **GENERATE_ID_SIMPLE_MODE**  |   Core    | `true`                            | A flog that enable generating ID with `md5` algorithm.                                                             |
+| **PATH**                     |    Log    | `./logs`                          | The log path of the workflow saving log.                                                                           |
+| **DEBUG_MODE**               |    Log    | `true`                            | A flag that enable logging with debug level mode.                                                                  |
+| **ENABLE_WRITE**             |    Log    | `true`                            | A flag that enable logging object saving log to its destination.                                                   |
+| **MAX_PROCESS**              |    App    | `2`                               | The maximum process worker number that run in scheduler app module.                                                |
+| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                             | A schedule per process that run parallel.                                                                          |
+| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format.                                           |
+
+**API Application**:
+
+| Environment                |  Component  | Default | Description                                                                        |
+|:---------------------------|:-----------:|---------|------------------------------------------------------------------------------------|
+| **ENABLE_ROUTE_WORKFLOW**  |     API     | `true`  | A flag that enable workflow route to manage execute manually and workflow logging. |
+| **ENABLE_ROUTE_SCHEDULE**  |     API     | `true`  | A flag that enable run scheduler.                                                  |
+
+## :rocket: Deployment
+
+This package able to run as an application service for receive manual trigger
+from the master node via RestAPI or use to be Scheduler background service
+like crontab job but via Python API.
+
+### API Server
+
+```shell
+(venv) $ uvicorn src.ddeutil.workflow.api:app \
+  --host 127.0.0.1 \
+  --port 80 \
+  --no-access-log
+```
+
+> [!NOTE]
+> If this package already deploy, it is able to use multiprocess;
+> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
+
+### Docker Container
+
+Create Docker image;
+
+```shell
+$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
+```
+
+Run the above Docker image;
+
+```shell
+$ docker run -i ddeutil-workflow:latest
+```
+
+## :speech_balloon: Contribute
+
+I do not think this project will go around the world because it has specific propose,
+and you can create by your coding without this project dependency for long term
+solution. So, on this time, you can open [the GitHub issue on this project :raised_hands:](https://github.com/ddeutils/ddeutil-workflow/issues)
+for fix bug or request new feature if you want it.

ddeutil_workflow-0.0.29.dist-info/RECORD

@@ -0,0 +1,25 @@
+ddeutil/workflow/__about__.py,sha256=msVKiLUg4jRVo_KJlghj1cc0zwX_olhWZZkqcWYz16E,28
+ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
+ddeutil/workflow/__init__.py,sha256=dghn2lFl3Own4Pyq7SFHu-FMymOgLontJ6aCfxea9h4,1606
+ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
+ddeutil/workflow/conf.py,sha256=7lj_Im9jsa95fWUo19Q4-ZAcHa8Pu1HW-vaLgvrjNUM,17559
+ddeutil/workflow/cron.py,sha256=OLgniUxmrn65gzckk-uTmE2Pk1enJJyjYUKVeBbDQz0,7522
+ddeutil/workflow/exceptions.py,sha256=XUnpJSuxOyataClP0w_gpYjzn-NIwZK2BHro-J7Yw24,895
+ddeutil/workflow/hook.py,sha256=vgiJVbgm4aVl-tt_HVhHn-65UXCojzGLapdOPkoX9QA,5406
+ddeutil/workflow/job.py,sha256=XcewyALsLYYq94ycF6mkj3Ydr6if683z7t1oBqEVInE,24290
+ddeutil/workflow/params.py,sha256=svCjmFgEhim8yFJVjZhFmKP8JqTDHQ5EPhwJHVuDGno,5289
+ddeutil/workflow/result.py,sha256=k4pcj5KjbEcEPymsEUXeGY4gyLMfPkMTO6YDrAtfk7Q,3408
+ddeutil/workflow/scheduler.py,sha256=OlrnBZvVttoymeY1g-on9icEMU729OWISJReeX3jAKI,20452
+ddeutil/workflow/stage.py,sha256=wn8CARTvFJY4ZK1SwjzH8sKoMRz_eIeSGUMgnDWNi6g,24031
+ddeutil/workflow/templates.py,sha256=bVU_8gnMQmdhhw3W28ZqwmpEaOx10Nx_aauqiLS0lqg,10807
+ddeutil/workflow/utils.py,sha256=8LTqpvRPfrEYxsxhwszk6GKkyjrswxnwF3r_9vE8szw,6059
+ddeutil/workflow/workflow.py,sha256=vuy0Q3ceslBth04qbslXrp5NAQQ7XfpOochwgORzQ4Q,42349
+ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
+ddeutil/workflow/api/api.py,sha256=hmH45GtpyZ-kbiqQNmnHgjwEiCiDDXLKTGvcNa5nFos,4041
+ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
+ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
+ddeutil_workflow-0.0.29.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.29.dist-info/METADATA,sha256=UXsjCGddPiksHRAByDfUcsYAsGIqAbL1qJ87uQKWCVQ,14801
+ddeutil_workflow-0.0.29.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ddeutil_workflow-0.0.29.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.29.dist-info/RECORD,,

{ddeutil_workflow-0.0.27.dist-info → ddeutil_workflow-0.0.29.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.7.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any