ddeutil-workflow 0.0.72__tar.gz → 0.0.73__tar.gz

{ddeutil_workflow-0.0.72/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.72
+Version: 0.0.73
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.73}/pyproject.toml

@@ -116,7 +116,7 @@ filterwarnings = [
 ]
 log_cli = true
 log_cli_level = "DEBUG"
-log_cli_format = "%(asctime)s [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)"
+log_cli_format = "%(asctime)s [%(levelname)-7s] (%(cut_id)s) %(message)-120s (%(filename)s:%(lineno)s)"
 log_cli_date_format = "%Y%m%d %H:%M:%S"
 
 [tool.black]

ddeutil_workflow-0.0.73/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.73"

{ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.73}/src/ddeutil/workflow/conf.py

@@ -109,7 +109,7 @@ class Config:  # pragma: no cov
             "LOG_FORMAT",
             (
                 "%(asctime)s.%(msecs)03d (%(process)-5d, "
-                "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
+                "%(thread)-5d) [%(levelname)-7s] (%(cut_id)s) %(message)-120s "
                 "(%(filename)s:%(lineno)s) (%(name)-10s)"
             ),
         )
@@ -119,8 +119,8 @@ class Config:  # pragma: no cov
         return env(
             "LOG_FORMAT_FILE",
             (
-                "{datetime} ({process:5d}, {thread:5d}) {message:120s} "
-                "({filename}:{lineno})"
+                "{datetime} ({process:5d}, {thread:5d}) ({cut_id}) "
+                "{message:120s} ({filename}:{lineno})"
             ),
         )
 
@@ -222,6 +222,7 @@ class YamlParser:
         paths: Optional[list[Path]] = None,
         obj: Optional[Union[object, str]] = None,
         extras: Optional[DictData] = None,
+        ignore_filename: Optional[str] = None,
     ) -> DictData:
         """Find data with specific key and return the latest modify date data if
         this key exists multiple files.
@@ -229,9 +230,12 @@ class YamlParser:
         :param name: (str) A name of data that want to find.
         :param path: (Path) A config path object.
         :param paths: (list[Path]) A list of config path object.
-        :param obj:
+        :param obj: (object | str) An object that want to validate matching
+            before return.
         :param extras: (DictData)  An extra parameter that use to override core
             config values.
+        :param ignore_filename: (str) An ignore filename. Default is
+            ``.confignore`` filename.
 
         :rtype: DictData
         """
@@ -252,13 +256,13 @@ class YamlParser:
         for path in paths:
             for file in glob_files(path):
 
-                if cls.is_ignore(file, path):
+                if cls.is_ignore(file, path, ignore_filename=ignore_filename):
                     continue
 
                 if data := cls.filter_yaml(file, name=name):
                     if not obj_type:
                         all_data.append((file.lstat().st_mtime, data))
-                    elif data.get("type", "") == obj_type:
+                    elif (t := data.get("type")) and t == obj_type:
                         all_data.append((file.lstat().st_mtime, data))
                     else:
                         continue
@@ -274,6 +278,7 @@ class YamlParser:
         paths: Optional[list[Path]] = None,
         excluded: Optional[list[str]] = None,
         extras: Optional[DictData] = None,
+        ignore_filename: Optional[str] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
@@ -287,6 +292,8 @@ class YamlParser:
             data.
         :param extras: (DictData) An extra parameter that use to override core
             config values.
+        :param ignore_filename: (str) An ignore filename. Default is
+            ``.confignore`` filename.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
@@ -308,7 +315,7 @@ class YamlParser:
         for path in paths:
             for file in glob_files(path):
 
-                if cls.is_ignore(file, path):
+                if cls.is_ignore(file, path, ignore_filename=ignore_filename):
                     continue
 
                 for key, data in cls.filter_yaml(file).items():
@@ -316,7 +323,7 @@ class YamlParser:
                     if key in excluded:
                         continue
 
-                    if data.get("type", "") == obj_type:
+                    if (t := data.get("type")) and t == obj_type:
                         marking: tuple[float, DictData] = (
                             file.lstat().st_mtime,
                             data,

{ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.73}/src/ddeutil/workflow/stages.py

@@ -1311,6 +1311,17 @@ class CallStage(BaseRetryStage):
         alias="with",
     )
 
+    @field_validator("args", mode="before")
+    def __validate_args_key(cls, value: Any) -> Any:
+        if isinstance(value, dict):
+            if any(k in value for k in ("result", "extras")):
+                raise ValueError(
+                    "The argument on workflow template for the caller stage "
+                    "should not pass `result` and `extras`. They are special "
+                    "arguments."
+                )
+        return value
+
     def execute(
         self,
         params: DictData,
@@ -1348,9 +1359,10 @@ class CallStage(BaseRetryStage):
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = {"result": result} | param2template(
-            self.args, params, extras=self.extras
-        )
+        args: DictData = {
+            "result": result,
+            "extras": self.extras,
+        } | param2template(self.args, params, extras=self.extras)
         sig = inspect.signature(call_func)
         necessary_params: list[str] = []
         has_keyword: bool = False
@@ -1369,14 +1381,21 @@ class CallStage(BaseRetryStage):
             (k.removeprefix("_") not in args and k not in args)
             for k in necessary_params
         ):
+            necessary_params.remove("result")
+            necessary_params.remove("extras")
+            args.pop("result")
+            args.pop("extras")
             raise ValueError(
                 f"Necessary params, ({', '.join(necessary_params)}, ), "
-                f"does not set to args, {list(args.keys())}."
+                f"does not set to args. It already set {list(args.keys())}."
             )
 
         if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
+        if "extras" not in sig.parameters and not has_keyword:
+            args.pop("extras")
+
         if event and event.is_set():
             raise StageCancelError(
                 "Execution was canceled from the event before start parallel."
@@ -1441,9 +1460,10 @@ class CallStage(BaseRetryStage):
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = {"result": result} | param2template(
-            self.args, params, extras=self.extras
-        )
+        args: DictData = {
+            "result": result,
+            "extras": self.extras,
+        } | param2template(self.args, params, extras=self.extras)
         sig = inspect.signature(call_func)
         necessary_params: list[str] = []
         has_keyword: bool = False
@@ -1462,14 +1482,20 @@ class CallStage(BaseRetryStage):
             (k.removeprefix("_") not in args and k not in args)
             for k in necessary_params
         ):
+            necessary_params.remove("result")
+            necessary_params.remove("extras")
+            args.pop("result")
+            args.pop("extras")
             raise ValueError(
                 f"Necessary params, ({', '.join(necessary_params)}, ), "
-                f"does not set to args, {list(args.keys())}."
+                f"does not set to args. It already set {list(args.keys())}."
             )
-
         if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
+        if "extras" not in sig.parameters and not has_keyword:
+            args.pop("extras")
+
         args: DictData = self.validate_model_args(call_func, args, result)
         if inspect.iscoroutinefunction(call_func):
             rs: DictOrModel = await call_func(

{ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.73}/src/ddeutil/workflow/traces.py

@@ -76,7 +76,7 @@ PREFIX_LOGS_REGEX: re.Pattern[str] = re.compile(
 )  # pragma: no cov
 
 
-class PrefixMsg(BaseModel):
+class Message(BaseModel):
     """Prefix Message model for receive grouping dict from searching prefix data
     from logging message.
     """
@@ -92,9 +92,9 @@ class PrefixMsg(BaseModel):
             msg (str): A message that want to extract.
 
         Returns:
-            PrefixMsg: the validated model from a string message.
+            Message: the validated model from a string message.
         """
-        return PrefixMsg.model_validate(
+        return Message.model_validate(
             obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
         )
 
@@ -126,6 +126,9 @@ class TraceMeta(BaseModel):  # pragma: no cov
     process: int = Field(description="A process ID.")
     thread: int = Field(description="A thread ID.")
     message: str = Field(description="A message log.")
+    cut_id: Optional[str] = Field(
+        default=None, description="A cutting of running ID."
+    )
     filename: str = Field(description="A filename of this log.")
     lineno: int = Field(description="A line number of this log.")
 
@@ -157,6 +160,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
         mode: Literal["stdout", "stderr"],
         message: str,
         level: str,
+        cutting_id: str,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
@@ -166,6 +170,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
         :param mode: (Literal["stdout", "stderr"]) A metadata mode.
         :param message: (str) A message.
         :param level: (str) A log level.
+        :param cutting_id: (str)
         :param extras: (DictData) An extra parameter that want to override core
             config values.
 
@@ -185,6 +190,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
             process=os.getpid(),
             thread=get_ident(),
             message=message,
+            cut_id=cutting_id,
             filename=frame_info.filename.split(os.path.sep)[-1],
             lineno=frame_info.lineno,
         )
@@ -529,10 +535,7 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
 
         :rtype: str
         """
-        return prepare_newline(
-            f"({self.cut_id}) "
-            f"{PrefixMsg.from_str(message).prepare(self.extras)}"
-        )
+        return prepare_newline(Message.from_str(message).prepare(self.extras))
 
     def _logging(
         self, message: str, mode: str, *, is_err: bool = False
@@ -655,7 +658,11 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
 
         mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
         trace_meta: TraceMeta = TraceMeta.make(
-            mode=mode, level=level, message=message, extras=self.extras
+            mode=mode,
+            level=level,
+            message=message,
+            cutting_id=self.cut_id,
+            extras=self.extras,
         )
 
         with (self.pointer / f"{mode}.txt").open(
@@ -684,7 +691,11 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
 
         mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
         trace_meta: TraceMeta = TraceMeta.make(
-            mode=mode, level=level, message=message, extras=self.extras
+            mode=mode,
+            level=level,
+            message=message,
+            cutting_id=self.cut_id,
+            extras=self.extras,
         )
 
         async with aiofiles.open(

{ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.73/src/ddeutil_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.72
+Version: 0.0.73
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.73}/tests/test_traces.py

@@ -4,7 +4,7 @@ import pytest
 from ddeutil.workflow import Result
 from ddeutil.workflow.traces import (
     FileTrace,
-    PrefixMsg,
+    Message,
     TraceMeta,
 )
 
@@ -25,7 +25,7 @@ def test_trace_regex_message():
         "[STAGE]: Execute Empty-Stage: 'End trigger Priority Group': "
         "( End trigger Priority Group: 2 )"
     )
-    prefix: PrefixMsg = PrefixMsg.from_str(msg)
+    prefix: Message = Message.from_str(msg)
     assert prefix.name == "STAGE"
     assert prefix.message == (
         "Execute Empty-Stage: 'End trigger Priority Group': "
@@ -36,7 +36,7 @@ def test_trace_regex_message():
         "[]: Execute Empty-Stage: 'End trigger Priority Group': "
         "( End trigger Priority Group: 2 )"
     )
-    prefix: PrefixMsg = PrefixMsg.from_str(msg)
+    prefix: Message = Message.from_str(msg)
     assert prefix.name is None
     assert prefix.message == (
         "[]: Execute Empty-Stage: 'End trigger Priority Group': "
@@ -44,7 +44,7 @@ def test_trace_regex_message():
     )
 
     msg: str = ""
-    prefix: PrefixMsg = PrefixMsg.from_str(msg)
+    prefix: Message = Message.from_str(msg)
     assert prefix.name is None
     assert prefix.message == ""
 
@@ -52,7 +52,7 @@ def test_trace_regex_message():
         "[WORKFLOW]: Execute Empty-Stage:\n'End trigger Priority Group':\n"
         "( End trigger Priority Group: 2 )"
     )
-    prefix: PrefixMsg = PrefixMsg.from_str(msg)
+    prefix: Message = Message.from_str(msg)
     assert prefix.name == "WORKFLOW"
     assert prefix.message == (
         "Execute Empty-Stage:\n'End trigger Priority Group':\n"
@@ -69,13 +69,16 @@ def test_trace_regex_message():
 
 
 def test_trace_meta():
-    meta = TraceMeta.make(mode="stderr", message="Foo", level="info")
+    meta = TraceMeta.make(
+        mode="stderr", message="Foo", level="info", cutting_id=""
+    )
     assert meta.message == "Foo"
 
     meta = TraceMeta.make(
         mode="stderr",
         message="Foo",
         level="info",
+        cutting_id="",
         extras={"logs_trace_frame_layer": 1},
     )
     assert meta.filename == "test_traces.py"
@@ -84,6 +87,7 @@ def test_trace_meta():
         mode="stderr",
         message="Foo",
         level="info",
+        cutting_id="",
         extras={"logs_trace_frame_layer": 2},
     )
     assert meta.filename == "python.py"
@@ -94,6 +98,7 @@ def test_trace_meta():
             mode="stderr",
             message="Foo",
             level="info",
+            cutting_id="",
             extras={"logs_trace_frame_layer": 100},
         )
 

ddeutil_workflow-0.0.72/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.72"