ddeutil-workflow 0.0.9__py3-none-any.whl → 0.0.10__py3-none-any.whl

ddeutil/workflow/stage.py

@@ -19,7 +19,6 @@ from __future__ import annotations
 
 import contextlib
 import inspect
-import logging
 import os
 import subprocess
 import sys
@@ -46,6 +45,7 @@ from typing_extensions import Self
 
 from .__types import DictData, DictStr, Re, TupleStr
 from .exceptions import StageException
+from .log import get_logger
 from .utils import (
     Registry,
     Result,
@@ -58,6 +58,9 @@ from .utils import (
 )
 
 P = ParamSpec("P")
+logger = get_logger("ddeutil.workflow")
+
+
 __all__: TupleStr = (
     "Stage",
     "EmptyStage",
@@ -86,7 +89,7 @@ def handler_result(message: str | None = None) -> Callable[P, Result]:
                 return func(self, *args, **kwargs).set_run_id(self.run_id)
             except Exception as err:
                 # NOTE: Start catching error from the stage execution.
-                logging.error(
+                logger.error(
                     f"({self.run_id}) [STAGE]: {err.__class__.__name__}: {err}"
                 )
                 if str2bool(
@@ -141,6 +144,7 @@ class BaseStage(BaseModel, ABC):
         default=None,
         description="A running stage ID.",
         repr=False,
+        exclude=True,
     )
 
     @model_validator(mode="after")
@@ -191,7 +195,7 @@ class BaseStage(BaseModel, ABC):
             self.id
             or str2bool(os.getenv("WORKFLOW_CORE_STAGE_DEFAULT_ID", "false"))
         ):
-            logging.debug(
+            logger.debug(
                 f"({self.run_id}) [STAGE]: Output does not set because this "
                 f"stage does not set ID or default stage ID config flag not be "
                 f"True."
@@ -208,7 +212,7 @@ class BaseStage(BaseModel, ABC):
             _id: str = gen_id(param2template(self.name, params=to))
 
         # NOTE: Set the output to that stage generated ID.
-        logging.debug(
+        logger.debug(
             f"({self.run_id}) [STAGE]: Set output complete with stage ID: {_id}"
         )
         to["stages"][_id] = {"outputs": output}
@@ -231,7 +235,7 @@ class BaseStage(BaseModel, ABC):
                 raise TypeError("Return type of condition does not be boolean")
             return not rs
         except Exception as err:
-            logging.error(f"({self.run_id}) [STAGE]: {err}")
+            logger.error(f"({self.run_id}) [STAGE]: {err}")
             raise StageException(f"{err.__class__.__name__}: {err}") from err
 
 
@@ -258,7 +262,7 @@ class EmptyStage(BaseStage):
         :param params: A context data that want to add output result. But this
             stage does not pass any output.
         """
-        logging.info(
+        logger.info(
             f"({self.run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
@@ -314,7 +318,7 @@ class BashStage(BaseStage):
         # NOTE: Make this .sh file able to executable.
         make_exec(f"./{f_name}")
 
-        logging.debug(
+        logger.debug(
             f"({self.run_id}) [STAGE]: Start create `.sh` file and running a "
             f"bash statement."
         )
@@ -336,7 +340,7 @@ class BashStage(BaseStage):
         with self.__prepare_bash(
             bash=bash, env=param2template(self.env, params)
         ) as sh:
-            logging.info(f"({self.run_id}) [STAGE]: Shell-Execute: {sh}")
+            logger.info(f"({self.run_id}) [STAGE]: Shell-Execute: {sh}")
             rs: CompletedProcess = subprocess.run(
                 sh,
                 shell=False,
@@ -424,7 +428,7 @@ class PyStage(BaseStage):
         _locals: DictData = {}
 
         # NOTE: Start exec the run statement.
-        logging.info(f"({self.run_id}) [STAGE]: Py-Execute: {self.name}")
+        logger.info(f"({self.run_id}) [STAGE]: Py-Execute: {self.name}")
         exec(run, _globals, _locals)
 
         return Result(
@@ -531,7 +535,7 @@ class HookStage(BaseStage):
             if k.removeprefix("_") in args:
                 args[k] = args.pop(k.removeprefix("_"))
 
-        logging.info(
+        logger.info(
             f"({self.run_id}) [STAGE]: Hook-Execute: {t_func.name}@{t_func.tag}"
         )
         rs: DictData = t_func(**param2template(args, params))
@@ -583,7 +587,7 @@ class TriggerStage(BaseStage):
         pipe: Pipeline = Pipeline.from_loader(
             name=_trigger, externals={"run_id": self.run_id}
         )
-        logging.info(f"({self.run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
+        logger.info(f"({self.run_id}) [STAGE]: Trigger-Execute: {_trigger!r}")
         return pipe.execute(params=param2template(self.params, params))
 
 

ddeutil/workflow/utils.py

@@ -14,14 +14,14 @@ from abc import ABC, abstractmethod
 from ast import Call, Constant, Expr, Module, Name, parse
 from collections.abc import Iterator
 from datetime import date, datetime
-from functools import wraps
+from functools import cached_property, wraps
 from hashlib import md5
 from importlib import import_module
 from inspect import isfunction
 from itertools import chain, islice, product
 from pathlib import Path
 from random import randrange
-from typing import Any, Callable, Literal, Optional, Protocol, Union
+from typing import Any, Callable, Literal, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 try:
@@ -30,16 +30,20 @@ except ImportError:
     from typing_extensions import ParamSpec
 
 from ddeutil.core import getdot, hasdot, hash_str, import_string, lazy, str2bool
-from ddeutil.io import PathData, search_env_replace
+from ddeutil.io import PathData, PathSearch, YamlFlResolve, search_env_replace
 from ddeutil.io.models.lineage import dt_now
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, Matrix, Re
 from .exceptions import ParamValueException, UtilException
 
+logger = logging.getLogger("ddeutil.workflow")
 P = ParamSpec("P")
+AnyModel = TypeVar("AnyModel", bound=BaseModel)
+AnyModelType = type[AnyModel]
 
 
 def get_diff_sec(dt: datetime, tz: ZoneInfo | None = None) -> int:
@@ -51,11 +55,13 @@ def get_diff_sec(dt: datetime, tz: ZoneInfo | None = None) -> int:
     )
 
 
-def delay() -> None:
+def delay(second: float = 0) -> None:
     """Delay time that use time.sleep with random second value between
     0.00 - 0.99 seconds.
+
+    :param second: A second number that want to adds-on random value.
     """
-    time.sleep(randrange(0, 99, step=10) / 100)
+    time.sleep(second + randrange(0, 99, step=10) / 100)
 
 
 class Engine(BaseModel):
@@ -143,6 +149,112 @@ def config() -> ConfParams:
     )
 
 
+class SimLoad:
+    """Simple Load Object that will search config data by given some identity
+    value like name of pipeline or on.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param params: A Params model object.
+    :param externals: An external parameters
+
+    Noted:
+    ---
+        The config data should have ``type`` key for modeling validation that
+    make this loader know what is config should to do pass to.
+
+        ... <identity-key>:
+        ...     type: <importable-object>
+        ...     <key-data>: <value-data>
+        ...     ...
+
+    """
+
+    def __init__(
+        self,
+        name: str,
+        params: ConfParams,
+        externals: DictData | None = None,
+    ) -> None:
+        self.data: DictData = {}
+        for file in PathSearch(params.engine.paths.conf).files:
+            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
+                data := YamlFlResolve(file).read().get(name, {})
+            ):
+                self.data = data
+
+        # VALIDATE: check the data that reading should not empty.
+        if not self.data:
+            raise ValueError(f"Config {name!r} does not found on conf path")
+
+        self.conf_params: ConfParams = params
+        self.externals: DictData = externals or {}
+        self.data.update(self.externals)
+
+    @classmethod
+    def finds(
+        cls,
+        obj: object,
+        params: ConfParams,
+        *,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+    ) -> Iterator[tuple[str, DictData]]:
+        """Find all data that match with object type in config path. This class
+        method can use include and exclude list of identity name for filter and
+        adds-on.
+        """
+        exclude: list[str] = exclude or []
+        for file in PathSearch(params.engine.paths.conf).files:
+            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
+                values := YamlFlResolve(file).read()
+            ):
+                for key, data in values.items():
+                    if key in exclude:
+                        continue
+                    if issubclass(get_type(data["type"], params), obj) and (
+                        include is None or all(i in data for i in include)
+                    ):
+                        yield key, data
+
+    @cached_property
+    def type(self) -> AnyModelType:
+        """Return object of string type which implement on any registry. The
+        object type.
+
+        :rtype: AnyModelType
+        """
+        if not (_typ := self.data.get("type")):
+            raise ValueError(
+                f"the 'type' value: {_typ} does not exists in config data."
+            )
+        return get_type(_typ, self.conf_params)
+
+
+class Loader(SimLoad):
+    """Loader Object that get the config `yaml` file from current path.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param externals: An external parameters
+    """
+
+    @classmethod
+    def finds(
+        cls,
+        obj: object,
+        *,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+        **kwargs,
+    ) -> DictData:
+        """Override the find class method from the Simple Loader object."""
+        return super().finds(
+            obj=obj, params=config(), include=include, exclude=exclude
+        )
+
+    def __init__(self, name: str, externals: DictData) -> None:
+        super().__init__(name, config(), externals)
+
+
 def gen_id(
     value: Any,
     *,
@@ -176,6 +288,26 @@ def gen_id(
     ).hexdigest()
 
 
+def get_type(t: str, params: ConfParams) -> AnyModelType:
+    """Return import type from string importable value in the type key.
+
+    :param t: A importable type string.
+    :param params: A config parameters that use registry to search this
+        type.
+    :rtype: AnyModelType
+    """
+    try:
+        # NOTE: Auto adding module prefix if it does not set
+        return import_string(f"ddeutil.workflow.{t}")
+    except ModuleNotFoundError:
+        for registry in params.engine.registry:
+            try:
+                return import_string(f"{registry}.{t}")
+            except ModuleNotFoundError:
+                continue
+        return import_string(f"{t}")
+
+
 class TagFunc(Protocol):
     """Tag Function Protocol"""
 
@@ -260,6 +392,10 @@ class BaseParam(BaseModel, ABC):
             "Receive value and validate typing before return valid value."
         )
 
+    @field_serializer("type")
+    def __serializer_type(self, value: str) -> str:
+        return value
+
 
 class DefaultParam(BaseParam):
     """Default Parameter that will check default if it required"""
@@ -583,7 +719,7 @@ def map_post_filter(
             else:
                 value: Any = f_func(value, *args, **kwargs)
         except Exception as err:
-            logging.warning(str(err))
+            logger.warning(str(err))
             raise UtilException(
                 f"The post-filter function: {func_name} does not fit with "
                 f"{value} (type: {type(value).__name__})."

{ddeutil_workflow-0.0.9.dist-info → ddeutil_workflow-0.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.9
+Version: 0.0.10
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -21,13 +21,11 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.9.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: fmtutil
 Requires-Dist: ddeutil-io
 Requires-Dist: python-dotenv ==1.0.1
-Requires-Dist: typer ==0.12.4
+Requires-Dist: typer <1.0.0,==0.12.5
 Provides-Extra: api
-Requires-Dist: fastapi[standard] ==0.112.1 ; extra == 'api'
-Requires-Dist: croniter ==3.0.3 ; extra == 'api'
+Requires-Dist: fastapi[standard] <1.0.0,==0.112.2 ; extra == 'api'
 Provides-Extra: schedule
 Requires-Dist: schedule <2.0.0,==1.2.2 ; extra == 'schedule'
 
@@ -39,17 +37,6 @@ Requires-Dist: schedule <2.0.0,==1.2.2 ; extra == 'schedule'
 [![gh license](https://img.shields.io/github/license/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
 [![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
-**Table of Contents**:
-
-- [Installation](#installation)
-- [Getting Started](#getting-started)
-  - [On](#on)
-  - [Pipeline](#pipeline)
-- [Usage](#usage)
-- [Configuration](#configuration)
-- [Future](#future)
-- [Deployment](#deployment)
-
 The **Lightweight workflow orchestration** with less dependencies the was created
 for easy to make a simple metadata driven for data pipeline orchestration.
 It can to use for data operator by a `.yaml` template.
@@ -103,82 +90,6 @@ this package with application add-ons, you should add `app` in installation;
 > | ddeutil-workflow:python3.11 | `3.11`         | :x:     |
 > | ddeutil-workflow:python3.12 | `3.12`         | :x:     |
 
-## Getting Started
-
-The main feature of this project is the `Pipeline` object that can call any
-registries function. The pipeline can handle everything that you want to do, it
-will passing parameters and catching the output for re-use it to next step.
-
-### On
-
-The **On** is schedule object that receive crontab value and able to generate
-datetime value with next or previous with any start point of an input datetime.
-
-```yaml
-# This file should keep under this path: `./root-path/conf-path/*`
-on_every_5_min:
-  type: on.On
-  cron: "*/5 * * * *"
-```
-
-```python
-from ddeutil.workflow.on import On
-
-# NOTE: Start load the on data from `.yaml` template file with this key.
-schedule = On.from_loader(name='on_every_5_min', externals={})
-
-assert '*/5 * * * *' == str(schedule.cronjob)
-
-cron_iter = schedule.generate('2022-01-01 00:00:00')
-
-assert "2022-01-01 00:05:00" f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
-assert "2022-01-01 00:10:00" f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
-assert "2022-01-01 00:15:00" f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
-```
-
-### Pipeline
-
-The **Pipeline** object that is the core feature of this project.
-
-```yaml
-# This file should keep under this path: `./root-path/conf-path/*`
-pipeline-name:
-  type: ddeutil.workflow.pipeline.Pipeline
-  on: 'on_every_5_min'
-  params:
-    author-run:
-      type: str
-    run-date:
-      type: datetime
-  jobs:
-    first-job:
-      stages:
-        - name: "Empty stage do logging to console only!!"
-```
-
-```python
-from ddeutil.workflow.pipeline import Pipeline
-
-pipe = Pipeline.from_loader(name='pipeline-name', externals={})
-pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
-```
-
-> [!NOTE]
-> The above parameter can use short declarative statement. You can pass a parameter
-> type to the key of a parameter name but it does not handler default value if you
-> run this pipeline workflow with schedule.
->
-> ```yaml
-> ...
-> params:
->   author-run: str
->   run-date: datetime
-> ...
-> ```
->
-> And for the type, you can remove `ddeutil.workflow` prefix because we can find
-> it by looping search from `WORKFLOW_CORE_REGISTRY` value.
-
 ## Usage
 
 This is examples that use workflow file for running common Data Engineering
@@ -209,7 +120,9 @@ run_py_local:
             url: https://open-data/
             auth: ${API_ACCESS_REFRESH_TOKEN}
             aws_s3_path: my-data/open-data/
-            # This Authentication code should implement with your custom hook function
+
+            # This Authentication code should implement with your custom hook function.
+            # The template allow you to use environment variable.
             aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
             aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
 ```
@@ -227,28 +140,24 @@ run_py_local:
 | `WORKFLOW_CORE_STAGE_RAISE_ERROR`   | Core      | true                         | A flag that all stage raise StageException from stage execution            |
 | `WORKFLOW_CORE_MAX_PIPELINE_POKING` | Core      | 4                            |                                                                            |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`    | Core      | 2                            | The maximum job number that able to run parallel in pipeline executor      |
+| `WORKFLOW_LOG_DEBUG_MODE`           | Log       | true                         | A flag that enable logging with debug level mode                           |
 | `WORKFLOW_LOG_ENABLE_WRITE`         | Log       | true                         | A flag that enable logging object saving log to its destination            |
 
 
 **Application**:
 
-| Environment                         | Default | Description |
-|-------------------------------------|---------|-------------|
-| `WORKFLOW_APP_PROCESS_WORKER`       | 2       |             |
-| `WORKFLOW_APP_PIPELINE_PER_PROCESS` | 100     |             |
+| Environment                         | Default                          | Description                                                             |
+|-------------------------------------|----------------------------------|-------------------------------------------------------------------------|
+| `WORKFLOW_APP_PROCESS_WORKER`       | 2                                | The maximum process worker number that run in scheduler app module      |
+| `WORKFLOW_APP_SCHEDULE_PER_PROCESS` | 100                              | A schedule per process that run parallel                                |
+| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format |
 
 **API server**:
 
-| Environment           | Default                                                | Description                                                        |
-|-----------------------|--------------------------------------------------------|--------------------------------------------------------------------|
-| `WORKFLOW_API_DB_URL` | postgresql+asyncpg://user:pass@localhost:5432/schedule | A Database URL that will pass to SQLAlchemy create_engine function |
-
-## Future
-
-The current milestone that will develop and necessary features that should to
-implement on this project.
-
-- ...
+| Environment                          | Default | Description                                                                       |
+|--------------------------------------|---------|-----------------------------------------------------------------------------------|
+| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | true    | A flag that enable workflow route to manage execute manually and workflow logging |
+| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | true    | A flag that enable run scheduler                                                  |
 
 ## Deployment
 
@@ -270,4 +179,4 @@ like crontab job but via Python API.
 
 > [!NOTE]
 > If this package already deploy, it able to use
-> `uvicorn ddeutil.workflow.api:app --host 0.0.0.0 --port 80`
+> `uvicorn ddeutil.workflow.api:app --host 0.0.0.0 --port 80 --workers 4`

ddeutil_workflow-0.0.10.dist-info/RECORD

@@ -0,0 +1,21 @@
+ddeutil/workflow/__about__.py,sha256=KJfEGSDA5LiPvdupul6ulxozLD2x2GhgUvq8t60EXsI,28
+ddeutil/workflow/__init__.py,sha256=oGvg_BpKKb_FG76DlMvXTKD7BsYhqF9wB1r4x5Q_lQI,647
+ddeutil/workflow/__types.py,sha256=SYMoxbENQX8uPsiCZkjtpHAqqHOh8rUrarAFicAJd0E,1773
+ddeutil/workflow/api.py,sha256=WHgmjvnnkM4djwHt4bsAqsQjjcjAITRSrNrYYO6bgn8,2582
+ddeutil/workflow/cli.py,sha256=snJCM-LAqvWwhkSB-3KRWwcgbHAkHn4cZ_DtmfOL5gs,3360
+ddeutil/workflow/cron.py,sha256=uhp3E5pl_tX_H88bsDujcwdhZmOE53csyV-ouPpPdK8,25321
+ddeutil/workflow/exceptions.py,sha256=UHojJQmnG9OVuRhXBAzDW6KZn-uKxvxV034QhUBUzUI,686
+ddeutil/workflow/log.py,sha256=a5L5KWEGS5oiY_y6jugeAoRyAcnAhlt1HfeTU77YeI4,6036
+ddeutil/workflow/on.py,sha256=Sxwnu0vPbIrMR_WWvH3_rOvD0tbiJntcB5378WoV19M,7163
+ddeutil/workflow/pipeline.py,sha256=lPw9R3gOnBcU2eogClG8b4e4rTvpn5EbACLNZDuuR38,40825
+ddeutil/workflow/repeat.py,sha256=0O8voTRB8lNMWsk1AbOYcio_b2_CW98yrfiEzNBb6gA,4954
+ddeutil/workflow/route.py,sha256=pcn_oDzc2nl6txFhu_TWAnntLggEOFV9A3EVdnazcHI,2597
+ddeutil/workflow/scheduler.py,sha256=Vu9FZbiHDnshQ2O1SnkVX686eSfaZzip-1oQohfuH_Y,20140
+ddeutil/workflow/stage.py,sha256=XZEPImipk83kNX9UHrwu7wWUBigXZpEkWqagOG0oS70,20656
+ddeutil/workflow/utils.py,sha256=ehIcT_fIQL8N0wU16VJDKAFN9q4h1FyMxyT5uTeMIA0,28561
+ddeutil_workflow-0.0.10.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.10.dist-info/METADATA,sha256=7jdDYS2WtaZFpwUCo4ur8NAFWyi-omELh8YLB3EL9ok,9433
+ddeutil_workflow-0.0.10.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
+ddeutil_workflow-0.0.10.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
+ddeutil_workflow-0.0.10.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.10.dist-info/RECORD,,

ddeutil_workflow-0.0.10.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+ddeutil-workflow = ddeutil.workflow.cli:cli

ddeutil/workflow/loader.py

@@ -1,132 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-from collections.abc import Iterator
-from functools import cached_property
-from typing import TypeVar
-
-from ddeutil.core import import_string
-from ddeutil.io import PathSearch, YamlFlResolve
-from pydantic import BaseModel
-
-from .__types import DictData
-from .utils import ConfParams, config
-
-AnyModel = TypeVar("AnyModel", bound=BaseModel)
-AnyModelType = type[AnyModel]
-
-
-class SimLoad:
-    """Simple Load Object that will search config data by name.
-
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param params: A Params model object.
-    :param externals: An external parameters
-
-    Noted:
-        The config data should have ``type`` key for engine can know what is
-    config should to do next.
-    """
-
-    def __init__(
-        self,
-        name: str,
-        params: ConfParams,
-        externals: DictData | None = None,
-    ) -> None:
-        self.data: DictData = {}
-        for file in PathSearch(params.engine.paths.conf).files:
-            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
-                data := YamlFlResolve(file).read().get(name, {})
-            ):
-                self.data = data
-        if not self.data:
-            raise ValueError(f"Config {name!r} does not found on conf path")
-
-        # TODO: Validate the version of template data that mean if version of
-        #   Template were change it should raise to upgrade package version.
-        #   ---
-        #   <pipeline-name>:
-        #       version: 1
-        #       type: pipeline.Pipeline
-        #
-        self.conf_params: ConfParams = params
-        self.externals: DictData = externals or {}
-        self.data.update(self.externals)
-
-    @classmethod
-    def find(
-        cls,
-        obj: object,
-        params: ConfParams,
-        *,
-        include: list[str] | None = None,
-        exclude: list[str] | None = None,
-    ) -> Iterator[tuple[str, DictData]]:
-        """Find all object"""
-        exclude: list[str] = exclude or []
-        for file in PathSearch(params.engine.paths.conf).files:
-            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
-                values := YamlFlResolve(file).read()
-            ):
-                for key, data in values.items():
-                    if key in exclude:
-                        continue
-                    if (
-                        (t := data.get("type"))
-                        and issubclass(cls.get_type(t, params), obj)
-                        and all(i in data for i in (include or data.keys()))
-                    ):
-                        yield key, data
-
-    @classmethod
-    def get_type(cls, t: str, params: ConfParams) -> AnyModelType:
-        try:
-            # NOTE: Auto adding module prefix if it does not set
-            return import_string(f"ddeutil.workflow.{t}")
-        except ModuleNotFoundError:
-            for registry in params.engine.registry:
-                try:
-                    return import_string(f"{registry}.{t}")
-                except ModuleNotFoundError:
-                    continue
-            return import_string(f"{t}")
-
-    @cached_property
-    def type(self) -> AnyModelType:
-        """Return object of string type which implement on any registry. The
-        object type
-        """
-        if not (_typ := self.data.get("type")):
-            raise ValueError(
-                f"the 'type' value: {_typ} does not exists in config data."
-            )
-        return self.get_type(_typ, self.conf_params)
-
-
-class Loader(SimLoad):
-    """Loader Object that get the config `yaml` file from current path.
-
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param externals: An external parameters
-    """
-
-    @classmethod
-    def find(
-        cls,
-        obj,
-        *,
-        include: list[str] | None = None,
-        exclude: list[str] | None = None,
-        **kwargs,
-    ) -> DictData:
-        return super().find(
-            obj=obj, params=config(), include=include, exclude=exclude
-        )
-
-    def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(name, config(), externals)