ddeutil-workflow 0.0.3__tar.gz → 0.0.4__tar.gz

{ddeutil_workflow-0.0.3/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.3
+Version: 0.0.4
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.3 → ddeutil_workflow-0.0.4}/pyproject.toml

@@ -80,8 +80,8 @@ addopts = [
 ]
 filterwarnings = ["error"]
 log_cli = true
-log_cli_level = "INFO"
-log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
+log_cli_level = "DEBUG"
+log_cli_format = "%(asctime)s [%(levelname)7s] %(message)s (%(filename)s:%(lineno)s)"
 log_cli_date_format = "%Y-%m-%d %H:%M:%S"
 
 [tool.black]

ddeutil_workflow-0.0.4/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.4"

{ddeutil_workflow-0.0.3 → ddeutil_workflow-0.0.4}/src/ddeutil/workflow/pipeline.py

@@ -5,13 +5,17 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+import contextlib
 import inspect
 import itertools
 import logging
 import subprocess
+import sys
 import time
+import uuid
 from abc import ABC, abstractmethod
 from inspect import Parameter
+from pathlib import Path
 from queue import Queue
 from subprocess import CompletedProcess
 from typing import Any, Callable, Optional, Union
@@ -25,7 +29,7 @@ from .__regex import RegexConf
 from .__types import DictData, DictStr
 from .exceptions import TaskException
 from .loader import Loader, map_params
-from .utils import Params, make_registry
+from .utils import Params, make_exec, make_registry
 
 
 class BaseStage(BaseModel, ABC):
@@ -78,9 +82,27 @@ class ShellStage(BaseStage):
     env: DictStr = Field(default_factory=dict)
 
     @staticmethod
+    @contextlib.contextmanager
     def __prepare_shell(shell: str):
-        """Prepare shell statement string that include newline"""
-        return shell.replace("\n", ";")
+        """Return context of prepared shell statement that want to execute. This
+        step will write the `.sh` file before giving this file name to context.
+        After that, it will auto delete this file automatic.
+
+        :param shell: A shell statement that want to prepare.
+        """
+        f_name: str = f"{uuid.uuid4()}.sh"
+        f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
+        with open(f"./{f_name}", mode="w", newline="\n") as f:
+            f.write(f"#!/bin/{f_shebang}\n")
+
+            # NOTE: make sure that shell script file does not have `\r` char.
+            f.write(shell.replace("\r\n", "\n"))
+
+        make_exec(f"./{f_name}")
+
+        yield [f_shebang, f_name]
+
+        Path(f_name).unlink()
 
     def set_outputs(self, rs: CompletedProcess, params: DictData) -> DictData:
         """Set outputs to params"""
@@ -95,8 +117,7 @@ class ShellStage(BaseStage):
             # NOTE: The output will fileter unnecessary keys from ``_locals``.
             "outputs": {
                 "return_code": rs.returncode,
-                "stdout": rs.stdout,
-                "stderr": rs.stderr,
+                "stdout": rs.stdout.rstrip("\n"),
             },
         }
         return params
@@ -105,19 +126,21 @@ class ShellStage(BaseStage):
         """Execute the Shell & Powershell statement with the Python build-in
         ``subprocess`` package.
         """
-        rs: CompletedProcess = subprocess.run(
-            self.__prepare_shell(self.shell),
-            capture_output=True,
-            text=True,
-            shell=True,
-        )
+        with self.__prepare_shell(self.shell) as sh:
+            with open(sh[-1]) as f:
+                logging.debug(f.read())
+            logging.info(f"Shell-Execute: {sh}")
+            rs: CompletedProcess = subprocess.run(
+                sh,
+                shell=False,
+                capture_output=True,
+                text=True,
+            )
         if rs.returncode > 0:
-            print(f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}")
-            # FIXME: raise err for this execution.
-            # raise TaskException(
-            #     f"{rs.stderr}\nRunning Statement:\n---\n"
-            #     f"{self.shell}"
-            # )
+            logging.error(f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}")
+            raise TaskException(
+                f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}"
+            )
         self.set_outputs(rs, params)
         return params
 
@@ -472,6 +495,7 @@ class Pipeline(BaseModel):
             job_id: str = jq.get()
             logging.info(f"[PIPELINE]: Start execute the job: {job_id!r}")
             job: Job = self.jobs[job_id]
+
             # TODO: Condition on ``needs`` of this job was set. It should create
             #   multithreading process on this step.
             #   But, I don't know how to handle changes params between each job
@@ -480,6 +504,7 @@ class Pipeline(BaseModel):
             #   >>> import multiprocessing
             #   >>> with multiprocessing.Pool(processes=3) as pool:
             #   ...     results = pool.starmap(merge_names, ('', '', ...))
+            #
             if any(params["jobs"].get(need) for need in job.needs):
                 jq.put(job_id)
             job.execute(params=params)

{ddeutil_workflow-0.0.3 → ddeutil_workflow-0.0.4}/src/ddeutil/workflow/utils.py

@@ -6,10 +6,12 @@
 from __future__ import annotations
 
 import inspect
+import stat
 from abc import ABC, abstractmethod
 from datetime import date, datetime
 from functools import wraps
 from importlib import import_module
+from pathlib import Path
 from typing import Any, Callable, Literal, Optional, Protocol, Union
 
 from ddeutil.core import lazy
@@ -178,3 +180,8 @@ Params = Union[
     DatetimeParams,
     StrParams,
 ]
+
+
+def make_exec(path: str | Path):
+    f: Path = Path(path) if isinstance(path, str) else path
+    f.chmod(f.stat().st_mode | stat.S_IEXEC)

{ddeutil_workflow-0.0.3 → ddeutil_workflow-0.0.4/src/ddeutil_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.3
+Version: 0.0.4
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.3 → ddeutil_workflow-0.0.4}/tests/test_pipeline_run.py

@@ -63,7 +63,6 @@ def test_pipe_job_py():
     } == rs
 
 
-@pytest.mark.skipif(True, reason="Because subprocess call on different OS")
 def test_pipe_stage_shell():
     pipeline = pipe.Pipeline.from_loader(name="run_python", externals={})
     echo_env: pipe.Job = pipeline.job("shell-run").stage("echo-env")
@@ -73,35 +72,13 @@ def test_pipe_stage_shell():
             "echo-env": {
                 "outputs": {
                     "return_code": 0,
-                    "stdout": '"Hello World";;;;echo "Hello $Env:USER_SH";\n',
-                    "stderr": "",
+                    "stdout": "Hello World\nVariable Foo",
                 },
             },
         },
     } == rs
 
 
-def test_subprocess_shell():
-    import subprocess
-    from textwrap import dedent
-
-    rs = subprocess.run(
-        [
-            "powershell",
-            dedent(
-                """
-                echo "Hello World";
-                echo "Next line";
-            """
-            ).strip(),
-        ],
-        capture_output=True,
-        text=True,
-        shell=True,
-    )
-    print(rs)
-
-
 def test_pipe_params_py():
     pipeline = pipe.Pipeline.from_loader(
         name="run_python_with_params",

ddeutil_workflow-0.0.3/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.3"