ddeutil-workflow 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.3"
+__version__: str = "0.0.4"

ddeutil/workflow/pipeline.py

@@ -5,13 +5,17 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+import contextlib
 import inspect
 import itertools
 import logging
 import subprocess
+import sys
 import time
+import uuid
 from abc import ABC, abstractmethod
 from inspect import Parameter
+from pathlib import Path
 from queue import Queue
 from subprocess import CompletedProcess
 from typing import Any, Callable, Optional, Union
@@ -25,7 +29,7 @@ from .__regex import RegexConf
 from .__types import DictData, DictStr
 from .exceptions import TaskException
 from .loader import Loader, map_params
-from .utils import Params, make_registry
+from .utils import Params, make_exec, make_registry
 
 
 class BaseStage(BaseModel, ABC):
@@ -78,9 +82,27 @@ class ShellStage(BaseStage):
     env: DictStr = Field(default_factory=dict)
 
     @staticmethod
+    @contextlib.contextmanager
     def __prepare_shell(shell: str):
-        """Prepare shell statement string that include newline"""
-        return shell.replace("\n", ";")
+        """Return context of prepared shell statement that want to execute. This
+        step will write the `.sh` file before giving this file name to context.
+        After that, it will auto delete this file automatic.
+
+        :param shell: A shell statement that want to prepare.
+        """
+        f_name: str = f"{uuid.uuid4()}.sh"
+        f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
+        with open(f"./{f_name}", mode="w", newline="\n") as f:
+            f.write(f"#!/bin/{f_shebang}\n")
+
+            # NOTE: make sure that shell script file does not have `\r` char.
+            f.write(shell.replace("\r\n", "\n"))
+
+        make_exec(f"./{f_name}")
+
+        yield [f_shebang, f_name]
+
+        Path(f_name).unlink()
 
     def set_outputs(self, rs: CompletedProcess, params: DictData) -> DictData:
         """Set outputs to params"""
@@ -95,8 +117,7 @@ class ShellStage(BaseStage):
             # NOTE: The output will fileter unnecessary keys from ``_locals``.
             "outputs": {
                 "return_code": rs.returncode,
-                "stdout": rs.stdout,
-                "stderr": rs.stderr,
+                "stdout": rs.stdout.rstrip("\n"),
             },
         }
         return params
@@ -105,19 +126,21 @@ class ShellStage(BaseStage):
         """Execute the Shell & Powershell statement with the Python build-in
         ``subprocess`` package.
         """
-        rs: CompletedProcess = subprocess.run(
-            self.__prepare_shell(self.shell),
-            capture_output=True,
-            text=True,
-            shell=True,
-        )
+        with self.__prepare_shell(self.shell) as sh:
+            with open(sh[-1]) as f:
+                logging.debug(f.read())
+            logging.info(f"Shell-Execute: {sh}")
+            rs: CompletedProcess = subprocess.run(
+                sh,
+                shell=False,
+                capture_output=True,
+                text=True,
+            )
         if rs.returncode > 0:
-            print(f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}")
-            # FIXME: raise err for this execution.
-            # raise TaskException(
-            #     f"{rs.stderr}\nRunning Statement:\n---\n"
-            #     f"{self.shell}"
-            # )
+            logging.error(f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}")
+            raise TaskException(
+                f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}"
+            )
         self.set_outputs(rs, params)
         return params
 
@@ -472,6 +495,7 @@ class Pipeline(BaseModel):
             job_id: str = jq.get()
             logging.info(f"[PIPELINE]: Start execute the job: {job_id!r}")
             job: Job = self.jobs[job_id]
+
             # TODO: Condition on ``needs`` of this job was set. It should create
             #   multithreading process on this step.
             #   But, I don't know how to handle changes params between each job
@@ -480,6 +504,7 @@ class Pipeline(BaseModel):
             #   >>> import multiprocessing
             #   >>> with multiprocessing.Pool(processes=3) as pool:
             #   ...     results = pool.starmap(merge_names, ('', '', ...))
+            #
             if any(params["jobs"].get(need) for need in job.needs):
                 jq.put(job_id)
             job.execute(params=params)

ddeutil/workflow/utils.py

@@ -6,10 +6,12 @@
 from __future__ import annotations
 
 import inspect
+import stat
 from abc import ABC, abstractmethod
 from datetime import date, datetime
 from functools import wraps
 from importlib import import_module
+from pathlib import Path
 from typing import Any, Callable, Literal, Optional, Protocol, Union
 
 from ddeutil.core import lazy
@@ -178,3 +180,8 @@ Params = Union[
     DatetimeParams,
     StrParams,
 ]
+
+
+def make_exec(path: str | Path):
+    f: Path = Path(path) if isinstance(path, str) else path
+    f.chmod(f.stat().st_mode | stat.S_IEXEC)

{ddeutil_workflow-0.0.3.dist-info → ddeutil_workflow-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.3
+Version: 0.0.4
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.3.dist-info → ddeutil_workflow-0.0.4.dist-info}/RECORD

@@ -1,13 +1,13 @@
-ddeutil/workflow/__about__.py,sha256=smA9c0CTLewINRoxj2VBHoiYDESoFGtXYFDvRT31dgs,27
+ddeutil/workflow/__about__.py,sha256=mI2ex1MyAxyhuxFGZTcowuzhaiztnKZbPh3obkmKpsY,27
 ddeutil/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ddeutil/workflow/__regex.py,sha256=bOngaQ0zJgy3vfNwF2MlI8XhLu_Ei1Vz8y50iLj8ao4,1061
 ddeutil/workflow/__types.py,sha256=AkpQq6QlrclpurCZZVY9RMxoyS9z2WGzhaz_ikeTaCU,453
 ddeutil/workflow/conn.py,sha256=POtNcyqFNGxZnkg5J_H1OIvQVnnqG-ajmBBzjoHl9sg,7238
 ddeutil/workflow/exceptions.py,sha256=XAq82VHSMLNb4UjGatp7hYfjxFtMiKFtBqJyAhwTl-s,434
 ddeutil/workflow/loader.py,sha256=TXS4k2dqNycBYSTYcJ80WIsPMKNZbHNeBbcufX6lrJc,5483
-ddeutil/workflow/pipeline.py,sha256=fG6ta-SNx4OWS6n8w7YpYDadfnbqayj8A1uY03TvLUA,16942
+ddeutil/workflow/pipeline.py,sha256=Uo26aC-IEdhSao0bxV_dy03fmv72rX9xENxiyzIEfC0,17789
 ddeutil/workflow/schedule.py,sha256=RMbTC7L32D3fJ5gYxJDCn-vPr2RYEBMSD0G2kj1Qows,2712
-ddeutil/workflow/utils.py,sha256=z7evB9kOsgTr30uVuL994bmOMDNZB5xDY2KjO7gL1dc,5379
+ddeutil/workflow/utils.py,sha256=YWZA5npxzsx89O6bQ1-nxq3k_lbx4SDJURXkXwkmsmc,5556
 ddeutil/workflow/tasks/__init__.py,sha256=TIcw9JinrdepWgyazSMLk_QflUFms99ILI4GvLHUGD0,338
 ddeutil/workflow/tasks/_pandas.py,sha256=rqz5_VMSqkEdirk7i3EElZoqnRYFyyK_Z8_Zt8FyeTg,1693
 ddeutil/workflow/tasks/_polars.py,sha256=SYEBx-0I9tbY046QGSMokVugK8Fqjhiw4dzpL6y6Hww,2917
@@ -22,8 +22,8 @@ ddeutil/workflow/vendors/pd.py,sha256=J6Nkb4RqUnz3NMfo3cHX-Udw3HPjqjUimojS86rR4o
 ddeutil/workflow/vendors/pg.py,sha256=TGwkV6nsarGLbiRTT_wB4uAy3xCR89EPPCMWqlWhFe8,422
 ddeutil/workflow/vendors/pl.py,sha256=B-l9zcZ9vATAKVMLv5tjKiWo5Qt8ZIv_aQzuVFinKbY,5087
 ddeutil/workflow/vendors/sftp.py,sha256=lQn4mnHhgvE9g1pbpoQF7HvZOxab8Z2XaDtSIJvumGM,7090
-ddeutil_workflow-0.0.3.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.3.dist-info/METADATA,sha256=KcsTd-FjufMK-4fhiIq27yeQUuA7NeB8TCkbXADQ1Dc,7992
-ddeutil_workflow-0.0.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-ddeutil_workflow-0.0.3.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.3.dist-info/RECORD,,
+ddeutil_workflow-0.0.4.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.4.dist-info/METADATA,sha256=-Sx33xY1RAQQ5OOSiOIIK6Qw8fcrAq-jiINbaXRFvjo,7992
+ddeutil_workflow-0.0.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+ddeutil_workflow-0.0.4.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.4.dist-info/RECORD,,