ddeutil-workflow 0.0.72__py3-none-any.whl → 0.0.73__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.72"
+__version__: str = "0.0.73"

ddeutil/workflow/conf.py

@@ -109,7 +109,7 @@ class Config:  # pragma: no cov
             "LOG_FORMAT",
             (
                 "%(asctime)s.%(msecs)03d (%(process)-5d, "
-                "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
+                "%(thread)-5d) [%(levelname)-7s] (%(cut_id)s) %(message)-120s "
                 "(%(filename)s:%(lineno)s) (%(name)-10s)"
             ),
         )
@@ -119,8 +119,8 @@ class Config:  # pragma: no cov
         return env(
             "LOG_FORMAT_FILE",
             (
-                "{datetime} ({process:5d}, {thread:5d}) {message:120s} "
-                "({filename}:{lineno})"
+                "{datetime} ({process:5d}, {thread:5d}) ({cut_id}) "
+                "{message:120s} ({filename}:{lineno})"
             ),
         )
 
@@ -222,6 +222,7 @@ class YamlParser:
         paths: Optional[list[Path]] = None,
         obj: Optional[Union[object, str]] = None,
         extras: Optional[DictData] = None,
+        ignore_filename: Optional[str] = None,
     ) -> DictData:
         """Find data with specific key and return the latest modify date data if
         this key exists multiple files.
@@ -229,9 +230,12 @@ class YamlParser:
         :param name: (str) A name of data that want to find.
         :param path: (Path) A config path object.
         :param paths: (list[Path]) A list of config path object.
-        :param obj:
+        :param obj: (object | str) An object that want to validate matching
+            before return.
         :param extras: (DictData)  An extra parameter that use to override core
             config values.
+        :param ignore_filename: (str) An ignore filename. Default is
+            ``.confignore`` filename.
 
         :rtype: DictData
         """
@@ -252,13 +256,13 @@ class YamlParser:
         for path in paths:
             for file in glob_files(path):
 
-                if cls.is_ignore(file, path):
+                if cls.is_ignore(file, path, ignore_filename=ignore_filename):
                     continue
 
                 if data := cls.filter_yaml(file, name=name):
                     if not obj_type:
                         all_data.append((file.lstat().st_mtime, data))
-                    elif data.get("type", "") == obj_type:
+                    elif (t := data.get("type")) and t == obj_type:
                         all_data.append((file.lstat().st_mtime, data))
                     else:
                         continue
@@ -274,6 +278,7 @@ class YamlParser:
         paths: Optional[list[Path]] = None,
         excluded: Optional[list[str]] = None,
         extras: Optional[DictData] = None,
+        ignore_filename: Optional[str] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
@@ -287,6 +292,8 @@ class YamlParser:
             data.
         :param extras: (DictData) An extra parameter that use to override core
             config values.
+        :param ignore_filename: (str) An ignore filename. Default is
+            ``.confignore`` filename.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
@@ -308,7 +315,7 @@ class YamlParser:
         for path in paths:
             for file in glob_files(path):
 
-                if cls.is_ignore(file, path):
+                if cls.is_ignore(file, path, ignore_filename=ignore_filename):
                     continue
 
                 for key, data in cls.filter_yaml(file).items():
@@ -316,7 +323,7 @@ class YamlParser:
                     if key in excluded:
                         continue
 
-                    if data.get("type", "") == obj_type:
+                    if (t := data.get("type")) and t == obj_type:
                         marking: tuple[float, DictData] = (
                             file.lstat().st_mtime,
                             data,

ddeutil/workflow/stages.py

@@ -1311,6 +1311,17 @@ class CallStage(BaseRetryStage):
         alias="with",
     )
 
+    @field_validator("args", mode="before")
+    def __validate_args_key(cls, value: Any) -> Any:
+        if isinstance(value, dict):
+            if any(k in value for k in ("result", "extras")):
+                raise ValueError(
+                    "The argument on workflow template for the caller stage "
+                    "should not pass `result` and `extras`. They are special "
+                    "arguments."
+                )
+        return value
+
     def execute(
         self,
         params: DictData,
@@ -1348,9 +1359,10 @@ class CallStage(BaseRetryStage):
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = {"result": result} | param2template(
-            self.args, params, extras=self.extras
-        )
+        args: DictData = {
+            "result": result,
+            "extras": self.extras,
+        } | param2template(self.args, params, extras=self.extras)
         sig = inspect.signature(call_func)
         necessary_params: list[str] = []
         has_keyword: bool = False
@@ -1369,14 +1381,21 @@ class CallStage(BaseRetryStage):
             (k.removeprefix("_") not in args and k not in args)
             for k in necessary_params
         ):
+            necessary_params.remove("result")
+            necessary_params.remove("extras")
+            args.pop("result")
+            args.pop("extras")
             raise ValueError(
                 f"Necessary params, ({', '.join(necessary_params)}, ), "
-                f"does not set to args, {list(args.keys())}."
+                f"does not set to args. It already set {list(args.keys())}."
             )
 
         if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
+        if "extras" not in sig.parameters and not has_keyword:
+            args.pop("extras")
+
         if event and event.is_set():
             raise StageCancelError(
                 "Execution was canceled from the event before start parallel."
@@ -1441,9 +1460,10 @@ class CallStage(BaseRetryStage):
 
         # VALIDATE: check input task caller parameters that exists before
         #   calling.
-        args: DictData = {"result": result} | param2template(
-            self.args, params, extras=self.extras
-        )
+        args: DictData = {
+            "result": result,
+            "extras": self.extras,
+        } | param2template(self.args, params, extras=self.extras)
         sig = inspect.signature(call_func)
         necessary_params: list[str] = []
         has_keyword: bool = False
@@ -1462,14 +1482,20 @@ class CallStage(BaseRetryStage):
             (k.removeprefix("_") not in args and k not in args)
             for k in necessary_params
         ):
+            necessary_params.remove("result")
+            necessary_params.remove("extras")
+            args.pop("result")
+            args.pop("extras")
             raise ValueError(
                 f"Necessary params, ({', '.join(necessary_params)}, ), "
-                f"does not set to args, {list(args.keys())}."
+                f"does not set to args. It already set {list(args.keys())}."
             )
-
         if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
+        if "extras" not in sig.parameters and not has_keyword:
+            args.pop("extras")
+
         args: DictData = self.validate_model_args(call_func, args, result)
         if inspect.iscoroutinefunction(call_func):
             rs: DictOrModel = await call_func(

ddeutil/workflow/traces.py

@@ -76,7 +76,7 @@ PREFIX_LOGS_REGEX: re.Pattern[str] = re.compile(
 )  # pragma: no cov
 
 
-class PrefixMsg(BaseModel):
+class Message(BaseModel):
     """Prefix Message model for receive grouping dict from searching prefix data
     from logging message.
     """
@@ -92,9 +92,9 @@ class PrefixMsg(BaseModel):
             msg (str): A message that want to extract.
 
         Returns:
-            PrefixMsg: the validated model from a string message.
+            Message: the validated model from a string message.
         """
-        return PrefixMsg.model_validate(
+        return Message.model_validate(
             obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
         )
 
@@ -126,6 +126,9 @@ class TraceMeta(BaseModel):  # pragma: no cov
     process: int = Field(description="A process ID.")
     thread: int = Field(description="A thread ID.")
     message: str = Field(description="A message log.")
+    cut_id: Optional[str] = Field(
+        default=None, description="A cutting of running ID."
+    )
     filename: str = Field(description="A filename of this log.")
     lineno: int = Field(description="A line number of this log.")
 
@@ -157,6 +160,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
         mode: Literal["stdout", "stderr"],
         message: str,
         level: str,
+        cutting_id: str,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
@@ -166,6 +170,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
         :param mode: (Literal["stdout", "stderr"]) A metadata mode.
         :param message: (str) A message.
         :param level: (str) A log level.
+        :param cutting_id: (str)
         :param extras: (DictData) An extra parameter that want to override core
             config values.
 
@@ -185,6 +190,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
             process=os.getpid(),
             thread=get_ident(),
             message=message,
+            cut_id=cutting_id,
             filename=frame_info.filename.split(os.path.sep)[-1],
             lineno=frame_info.lineno,
         )
@@ -529,10 +535,7 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
 
         :rtype: str
         """
-        return prepare_newline(
-            f"({self.cut_id}) "
-            f"{PrefixMsg.from_str(message).prepare(self.extras)}"
-        )
+        return prepare_newline(Message.from_str(message).prepare(self.extras))
 
     def _logging(
         self, message: str, mode: str, *, is_err: bool = False
@@ -655,7 +658,11 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
 
         mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
         trace_meta: TraceMeta = TraceMeta.make(
-            mode=mode, level=level, message=message, extras=self.extras
+            mode=mode,
+            level=level,
+            message=message,
+            cutting_id=self.cut_id,
+            extras=self.extras,
         )
 
         with (self.pointer / f"{mode}.txt").open(
@@ -684,7 +691,11 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
 
         mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
         trace_meta: TraceMeta = TraceMeta.make(
-            mode=mode, level=level, message=message, extras=self.extras
+            mode=mode,
+            level=level,
+            message=message,
+            cutting_id=self.cut_id,
+            extras=self.extras,
         )
 
         async with aiofiles.open(

{ddeutil_workflow-0.0.72.dist-info → ddeutil_workflow-0.0.73.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.72
+Version: 0.0.73
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.72.dist-info → ddeutil_workflow-0.0.73.dist-info}/RECORD

@@ -1,19 +1,19 @@
-ddeutil/workflow/__about__.py,sha256=VthLlJATFFZHEcP5syc1TRdzGF0zHfM2k5hzVK_WmRk,28
+ddeutil/workflow/__about__.py,sha256=Ht-dVJZ9jffiZCn55sCsjQbRLgRrXbeX4g38xoyJcmo,28
 ddeutil/workflow/__cron.py,sha256=BOKQcreiex0SAigrK1gnLxpvOeF3aca_rQwyz9Kfve4,28751
 ddeutil/workflow/__init__.py,sha256=HUy9XkBe7ttpUupJS4JDuj3aGp2QmJZfz8m2kHAIwdw,927
 ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
 ddeutil/workflow/__types.py,sha256=uNfoRbVmNK5O37UUMVnqcmoghD9oMS1q9fXC0APnjSI,4584
 ddeutil/workflow/audits.py,sha256=1pg4a5wdZCAKOqMr1Z_ofzRAFsDarN1BIJenWwn9xkg,11435
 ddeutil/workflow/cli.py,sha256=7cdn4f9KYOBK4CFs7sGmM2c0iYvzXiPtHnklSuD7ObU,5676
-ddeutil/workflow/conf.py,sha256=ZkXNiiIljqNVoLo46Gbp37-NNMfDCz5g9JiyNeJNoXE,13973
+ddeutil/workflow/conf.py,sha256=dxPzaLL9Wsg_3k6diQn4jqxT9jkqdsjDEGl81qLhVpY,14472
 ddeutil/workflow/errors.py,sha256=O5rq80Sqj0QMeIsWXpRUhiFLTq0o8bwm5BQ4kuq6xmI,3013
 ddeutil/workflow/event.py,sha256=0fGcous2vAF5DZ-aengk_vovgHoWNRoIiidSxk9ytkw,11119
 ddeutil/workflow/job.py,sha256=kviOQeSUsx0Z7CL0foblTulg2m_l6a3M3SMRxg9RWeg,39151
 ddeutil/workflow/params.py,sha256=Pco3DyjptC5Jkx53dhLL9xlIQdJvNAZs4FLzMUfXpbQ,12402
 ddeutil/workflow/result.py,sha256=ctxNSaY9tZPHEAUgvDkjWWu2APeTmlZCf1Hb0XVbbFo,8173
 ddeutil/workflow/reusables.py,sha256=LSn0XTkzGHf4ulOmWub29F0JZHt0NEyzrFd4ZFx_g_k,21622
-ddeutil/workflow/stages.py,sha256=kzMEMRTEuG52EOw51zyVO6LE-oiiqTIRUCk_OMcWZTM,106506
-ddeutil/workflow/traces.py,sha256=DrKzxgp9FJUJd_oWYHiMtR1z8R7HeweAL4_Y8q0_pz8,25048
+ddeutil/workflow/stages.py,sha256=xJqVxJvah3nsJSgOHL3BFm5M_UzM8RSEeIIlx96qUAI,107564
+ddeutil/workflow/traces.py,sha256=2XTX0ZQnbtKAp__PU4KCuBrx_8fTDpiIiBnKqeFCrLY,25318
 ddeutil/workflow/utils.py,sha256=-yWVShFGUJwUL5DU4xi-f2UGLXEuuv8sTf-GmVGjNt0,9674
 ddeutil/workflow/workflow.py,sha256=sy-HEKLmGPE1tDNr_cpm3CQsMEDP6fkXDHC9UsVSVDE,36214
 ddeutil/workflow/api/__init__.py,sha256=W3fe6_NLHSUzr4Tsu79w3pmvrYjpLeP3zBk4mtpPyqg,2843
@@ -22,9 +22,9 @@ ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8M
 ddeutil/workflow/api/routes/job.py,sha256=x809G5gCbJS257txj9eLLTbCbFK8ercXWzPDLuv5gEM,2953
 ddeutil/workflow/api/routes/logs.py,sha256=HiXw93PeIiaK_xJjM8lbD2ED1Il-W1iM51085nc7qmg,5286
 ddeutil/workflow/api/routes/workflows.py,sha256=D76cdLb2_9Dkfe2_8xt06CvPhAyJMqxyYkUgAs8Qlnw,4402
-ddeutil_workflow-0.0.72.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.72.dist-info/METADATA,sha256=rXtrkQ-LS9dx4DIIWOAz44e0bXSGrm_zujUNvb-uujs,15221
-ddeutil_workflow-0.0.72.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddeutil_workflow-0.0.72.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
-ddeutil_workflow-0.0.72.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.72.dist-info/RECORD,,
+ddeutil_workflow-0.0.73.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.73.dist-info/METADATA,sha256=0R6aQshHh_MWHRGKQwlrCfUToucyG3cLRuEDv_QDTKY,15221
+ddeutil_workflow-0.0.73.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddeutil_workflow-0.0.73.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.73.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.73.dist-info/RECORD,,