ddeutil-workflow 0.0.80__tar.gz → 0.0.81__tar.gz

{ddeutil_workflow-0.0.80/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.80
+Version: 0.0.81
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

ddeutil_workflow-0.0.81/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.81"

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81}/src/ddeutil/workflow/cli.py

@@ -54,29 +54,30 @@ def init() -> None:
         dedent(
             """
         # Example workflow template.
-        wf-example:
-          type: Workflow
-          desc: |
-            An example workflow template that provide the demo of workflow.
-          params:
-            name:
-                type: str
-                default: "World"
-          jobs:
-            first-job:
-              stages:
-
-                - name: "Hello Stage"
-                  echo: "Start say hi to the console"
-
-                - name: "Call tasks"
-                  uses: tasks/say-hello-func@example
-                  with:
-                    name: ${{ params.name }}
-            second-job:
-
-                - name: "Hello Env"
-                  echo: "Start say hi with ${ WORKFLOW_DEMO_HELLO }"
+        name: wf-example:
+        type: Workflow
+        desc: |
+          An example workflow template that provide the demo of workflow.
+        params:
+          name:
+              type: str
+              default: "World"
+        jobs:
+          first-job:
+            stages:
+
+              - name: "Hello Stage"
+                echo: "Start say hi to the console"
+
+              - name: "Call tasks"
+                uses: tasks/say-hello-func@example
+                with:
+                  name: ${{ params.name }}
+
+          second-job:
+
+              - name: "Hello Env"
+                echo: "Start say hi with ${ WORKFLOW_DEMO_HELLO }"
         """
         ).lstrip("\n")
     )
@@ -89,12 +90,20 @@ def init() -> None:
         dummy_tasks_path.write_text(
             dedent(
                 """
+            from typing import Any, Optional
+
             from ddeutil.workflow import Result, tag
 
             @tag(name="example", alias="say-hello-func")
-            def hello_world_task(name: str, rs: Result) -> dict[str, str]:
+            def hello_world_task(name: str, rs: Result, extras: Optional[dict[str, Any]] = None) -> dict[str, str]:
                 \"\"\"Logging hello task function\"\"\"
-                rs.trace.info(f"Hello, {name}")
+                _extras = extras or {}
+                # NOTE: I will use custom newline logging if you pass `||`.
+                rs.trace.info(
+                    f"Hello, {name}||"
+                    f"> running ID: {rs.run_id}"
+                    f"> extras: {_extras}"
+                )
                 return {"name": name}
             """
             ).lstrip("\n")
@@ -106,18 +115,19 @@ def init() -> None:
     dotenv_file = Path(".env")
     mode: str = "a" if dotenv_file.exists() else "w"
     with dotenv_file.open(mode=mode) as f:
-        f.write("\n# Workflow env vars\n")
+        f.write("\n# Workflow Environment Variables\n")
         f.write(
             "WORKFLOW_DEMO_HELLO=foo\n"
             "WORKFLOW_CORE_DEBUG_MODE=true\n"
             "WORKFLOW_LOG_TIMEZONE=Asia/Bangkok\n"
-            "WORKFLOW_LOG_TRACE_ENABLE_WRITE=false\n"
+            'WORKFLOW_LOG_TRACE_HANDLERS=\'[{"type": "console"}]\'\n'
+            'WORKFLOW_LOG_AUDIT_CONF=\'{"type": "file", "path": "./audits"}\''
             "WORKFLOW_LOG_AUDIT_ENABLE_WRITE=true\n"
         )
 
     typer.echo("Starter command:")
     typer.echo(
-        "> `source .env && workflow-cli workflows execute --name=wf-example`"
+        ">>> `source .env && workflow-cli workflows execute --name=wf-example`"
     )
 
 
@@ -163,7 +173,7 @@ def api(
     debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
     workers: Annotated[int, typer.Option(help="A worker number")] = None,
     reload: Annotated[bool, typer.Option(help="A reload flag")] = False,
-):
+) -> None:
     """
     Provision API application from the FastAPI.
     """

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81}/src/ddeutil/workflow/conf.py

@@ -22,19 +22,6 @@ Functions:
     pass_env: Process environment variable substitution
     api_config: Get API-specific configuration settings
 
-Example:
-    ```python
-    from ddeutil.workflow.conf import Config, YamlParser
-
-    # Load workflow configuration
-    parser = YamlParser("my-workflow")
-    workflow_config = parser.data
-
-    # Access dynamic configuration
-    from ddeutil.workflow.conf import dynamic
-    log_level = dynamic("log_level", default="INFO")
-    ```
-
 Note:
     Configuration files support environment variable substitution using
     ${VAR_NAME} syntax and provide extensive validation capabilities.
@@ -155,7 +142,7 @@ class Config:  # pragma: no cov
         )
 
     @property
-    def audit_conf(self) -> str:
+    def audit_conf(self) -> dict[str, Any]:
         return json.loads(
             env("LOG_AUDIT_URL", '{"type": "file", "path": "./audits"}')
         )
@@ -288,9 +275,12 @@ class YamlParser:
                     continue
 
                 if data := cls.filter_yaml(file, name=name):
+
+                    # NOTE: Start adding file metadata.
                     file_stat: os.stat_result = file.lstat()
                     data["created_at"] = file_stat.st_ctime
                     data["updated_at"] = file_stat.st_mtime
+
                     if not obj_type:
                         all_data.append((file_stat.st_mtime, data))
                     elif (t := data.get("type")) and t == obj_type:
@@ -324,9 +314,8 @@ class YamlParser:
             extras: (DictData) An extra parameter that use to override core
                 config values.
             ignore_filename: (str) An ignore filename. Default is
-            ``.confignore`` filename.
-            tags: (list[str])
-                A list of tag that want to filter.
+                ``.confignore`` filename.
+            tags (list[str]): A list of tag that want to filter.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
@@ -365,6 +354,8 @@ class YamlParser:
                         continue
 
                     if (t := data.get("type")) and t == obj_type:
+
+                        # NOTE: Start adding file metadata.
                         file_stat: os.stat_result = file.lstat()
                         data["created_at"] = file_stat.st_ctime
                         data["updated_at"] = file_stat.st_mtime
@@ -372,6 +363,7 @@ class YamlParser:
                             file.lstat().st_mtime,
                             data,
                         )
+
                         if key in all_data:
                             all_data[key].append(marking)
                         else:
@@ -405,15 +397,23 @@ class YamlParser:
     def filter_yaml(cls, file: Path, name: Optional[str] = None) -> DictData:
         """Read a YAML file context from an input file path and specific name.
 
-        :param file: (Path) A file path that want to extract YAML context.
-        :param name: (str) A key name that search on a YAML context.
+        Args:
+            file (Path): A file path that want to extract YAML context.
+            name (str): A key name that search on a YAML context.
 
-        :rtype: DictData
+        Returns:
+            DictData: A data that read from this file if it is YAML format.
         """
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
             if values is not None:
-                return values.get(name, {}) if name else values
+                if name:
+                    if "name" in values and values.get("name") == name:
+                        return values
+                    return (
+                        values[name] | {"name": name} if name in values else {}
+                    )
+                return values
         return {}
 
     @cached_property

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81}/src/ddeutil/workflow/errors.py

@@ -135,11 +135,12 @@ class BaseError(Exception):
 
         Example:
             >>> error = BaseError("Something failed", refs="stage-1")
-            >>> # Simple format
+
+            Simple format
             >>> error.to_dict()
             >>> # Returns: {"name": "BaseError", "message": "Something failed"}
 
-            >>> # With reference mapping
+            With reference mapping
             >>> error.to_dict(with_refs=True)
             >>> # Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
             ```

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81}/src/ddeutil/workflow/reusables.py

@@ -25,22 +25,20 @@ Functions:
     create_model_from_caller: Generate Pydantic models from function signatures
 
 Example:
-    ```python
-    from ddeutil.workflow.reusables import tag
-
-    @tag("data-processing", alias="process-csv")
-    def process_csv_file(input_path: str, output_path: str) -> dict:
-        # Custom processing logic
-        return {"status": "completed", "rows_processed": 1000}
-
-    # Use in workflow YAML:
-    # stages:
-    #   - name: "Process data"
-    #     uses: "data-processing/process-csv@latest"
-    #     args:
-    #       input_path: "/data/input.csv"
-    #       output_path: "/data/output.csv"
-    ```
+
+    >>> from ddeutil.workflow.reusables import tag
+    >>>
+    >>> @tag("data-processing", alias="process-csv")
+    >>> def process_csv_file(input_path: str, output_path: str) -> dict:
+    >>>     return {"status": "completed", "rows_processed": 1000}
+
+    >>> # Use in workflow YAML:
+    >>> # stages:
+    >>> #   - name: "Process data"
+    >>> #     uses: "data-processing/process-csv@latest"
+    >>> #     args:
+    >>> #       input_path: "/data/input.csv"
+    >>> #       output_path: "/data/output.csv"
 
 Note:
     The registry system supports versioning and aliasing for better function
@@ -64,6 +62,7 @@ from typing import (
     Protocol,
     TypeVar,
     Union,
+    cast,
     get_type_hints,
 )
 
@@ -201,7 +200,7 @@ def get_args_const(
             f"Post-filter: {expr} does not valid because it raise syntax error."
         ) from None
 
-    body: list[Expr] = mod.body
+    body: list[Expr] = cast(list[Expr], mod.body)
     if len(body) > 1:
         raise UtilError(
             "Post-filter function should be only one calling per workflow."

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81}/src/ddeutil/workflow/utils.py

@@ -28,17 +28,6 @@ Functions:
     cut_id: Cut running ID to specified length
     dump_all: Serialize nested BaseModel objects to dictionaries
     obj_name: Get object name or class name
-
-Example:
-    ```python
-    from ddeutil.workflow.utils import gen_id, get_dt_now
-
-    # Generate unique ID
-    run_id = gen_id("workflow")
-
-    # Get current datetime
-    now = get_dt_now()
-    ```
 """
 from __future__ import annotations
 

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81}/src/ddeutil/workflow/workflow.py

@@ -198,17 +198,15 @@ class Workflow(BaseModel):
             FileNotFoundError: If workflow configuration file not found
 
         Example:
-            ```python
-            # Load from default config path
-            workflow = Workflow.from_conf('data-pipeline')
-
-            # Load with custom path and extras
-            workflow = Workflow.from_conf(
-                'data-pipeline',
-                path=Path('./custom-configs'),
-                extras={'environment': 'production'}
-            )
-            ```
+            >>> # Load from default config path
+            >>> workflow = Workflow.from_conf('data-pipeline')
+
+            >>> # Load with custom path and extras
+            >>> workflow = Workflow.from_conf(
+            ...     'data-pipeline',
+            ...     path=Path('./custom-configs'),
+            ...     extras={'env': 'prod'}
+            ... )
         """
         load: YamlParser = YamlParser(name, path=path, extras=extras, obj=cls)
         data: DictData = copy.deepcopy(load.data)

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81/src/ddeutil_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.80
+Version: 0.0.81
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81}/tests/test_audits.py

@@ -47,7 +47,10 @@ def test_base_audit():
             },
         }
     )
-    print(log.model_dump())
+    assert log.model_dump() == {
+        "type": "test",
+        "extras": {"foo": "bar", "datetime": datetime(2024, 1, 1, 1, 15)},
+    }
 
 
 @mock.patch.object(Config, "enable_write_audit", False)

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.81}/tests/test_conf.py

@@ -122,12 +122,14 @@ def test_load_file(target_path: Path):
 
     load = YamlParser("test_load_file", extras={"conf_paths": [target_path]})
     assert exclude_created_and_updated(load.data) == {
+        "name": "test_load_file",
         "type": "Workflow",
         "desc": "Test multi config path",
         "env": "${WORKFLOW_LOG_TIMEZONE}",
     }
     assert pass_env(load.data["env"]) == "Asia/Bangkok"
     assert exclude_created_and_updated(pass_env(load.data)) == {
+        "name": "test_load_file",
         "type": "Workflow",
         "desc": "Test multi config path",
         "env": "Asia/Bangkok",
@@ -137,6 +139,7 @@ def test_load_file(target_path: Path):
         "test_load_file", extras={"conf_paths": [target_path]}, obj="Workflow"
     )
     assert exclude_created_and_updated(load.data) == {
+        "name": "test_load_file",
         "type": "Workflow",
         "desc": "Test multi config path",
         "env": "${WORKFLOW_LOG_TIMEZONE}",
@@ -153,6 +156,56 @@ def test_load_file(target_path: Path):
         _ = load.type
 
 
+@pytest.fixture(scope="function")
+def mock_workflow_with_name_key(test_path):
+    target_p = test_path / "test_read_file_with_name_key"
+    target_p.mkdir(exist_ok=True)
+
+    with (target_p / "wf_1.yaml").open(mode="w") as f:
+        yaml.dump(
+            {
+                "name": "wf_1",
+                "type": "Workflow",
+                "value": 1,
+                "tags": [
+                    1,
+                ],
+            },
+            f,
+        )
+
+    with (target_p / "wf_2.yaml").open(mode="w") as f:
+        yaml.dump(
+            {
+                "name": "wf_2",
+                "type": "Workflow",
+                "value": 1,
+                "tags": [
+                    1,
+                ],
+            },
+            f,
+        )
+
+    yield target_p
+
+    shutil.rmtree(target_p)
+
+
+def test_load_file_with_name_key(mock_workflow_with_name_key):
+    assert exclude_created_and_updated(
+        YamlParser(
+            "wf_1", extras={"conf_paths": [mock_workflow_with_name_key]}
+        ).data
+    ) == {"name": "wf_1", "tags": [1], "type": "Workflow", "value": 1}
+
+    with pytest.raises(ValueError):
+        YamlParser(
+            "wf_not_exists",
+            extras={"conf_paths": [mock_workflow_with_name_key]},
+        )
+
+
 def test_load_file_filter(mock_conf: Path):
     assert (
         "wf_1",
@@ -197,10 +250,10 @@ def test_load_file_filter(mock_conf: Path):
 
     assert exclude_created_and_updated(
         YamlParser.find("wf_1", path=mock_conf)
-    ) == {"tags": [1], "type": "Workflow", "value": 1}
+    ) == {"name": "wf_1", "tags": [1], "type": "Workflow", "value": 1}
     assert exclude_created_and_updated(
         YamlParser.find("wf_2", path=mock_conf)
-    ) == {"tags": [2], "type": "Workflow", "value": 2}
+    ) == {"name": "wf_2", "tags": [2], "type": "Workflow", "value": 2}
     assert (
         exclude_created_and_updated(
             YamlParser.find("wf_3", path=mock_conf, obj="Workflow")
@@ -209,7 +262,7 @@ def test_load_file_filter(mock_conf: Path):
     )
     assert exclude_created_and_updated(
         YamlParser.find("wf_4", path=mock_conf, obj="Custom")
-    ) == {"type": "Custom", "value": 4}
+    ) == {"name": "wf_4", "type": "Custom", "value": 4}
 
     with pytest.raises(TypeError):
         list(YamlParser.finds("Custom", paths={"path": mock_conf}, tags=[1]))
@@ -285,6 +338,7 @@ def test_load_file_finds(target_path: Path):
 
     load = YamlParser.find("test_load_file", path=target_path, obj="Workflow")
     assert exclude_created_and_updated(load) == {
+        "name": "test_load_file",
         "type": "Workflow",
         "data": "foo",
     }
@@ -292,6 +346,7 @@ def test_load_file_finds(target_path: Path):
     # NOTE: Load with the same name, but it set different type.
     load = YamlParser.find("test_load_file", path=target_path, obj="Config")
     assert exclude_created_and_updated(load) == {
+        "name": "test_load_file",
         "type": "Config",
         "data": "bar",
     }
@@ -351,26 +406,3 @@ def test_dynamic():
 
     conf = dynamic("trace_handlers", extras={})
     assert conf == [{"type": "console"}]
-
-
-def test_parse_url():
-    from urllib.parse import ParseResult, urlparse
-
-    url: ParseResult = urlparse("./logs")
-    assert url == ParseResult(
-        scheme="", netloc="", path="./logs", params="", query="", fragment=""
-    )
-    assert url.scheme == ""
-    assert url.path == "./logs"
-
-    url: ParseResult = urlparse("file:///./logs")
-    assert url.scheme == "file"
-    assert url.path == "/./logs"
-
-    url: ParseResult = urlparse("sqlite:///home/warehouse/sqlite.db")
-    assert url.scheme == "sqlite"
-    assert url.path == "/home/warehouse/sqlite.db"
-
-    url: ParseResult = urlparse("file:./data.db")
-    assert url.scheme == "file"
-    assert url.path == "./data.db"

ddeutil_workflow-0.0.80/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.80"