ddeutil-workflow 0.0.40__py3-none-any.whl → 0.0.41__py3-none-any.whl

ddeutil/workflow/audit.py

@@ -1,257 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-"""Audit Log module."""
-from __future__ import annotations
-
-import json
-import os
-from abc import ABC, abstractmethod
-from collections.abc import Iterator
-from datetime import datetime
-from pathlib import Path
-from typing import ClassVar, Optional, Union
-
-from pydantic import BaseModel, Field
-from pydantic.functional_validators import model_validator
-from typing_extensions import Self
-
-from .__types import DictData, TupleStr
-from .conf import config
-from .logs import TraceLog, get_dt_tznow, get_trace
-
-__all__: TupleStr = (
-    "get_audit",
-    "FileAudit",
-    "SQLiteAudit",
-    "Audit",
-)
-
-
-class BaseAudit(BaseModel, ABC):
-    """Base Audit Pydantic Model with abstraction class property that implement
-    only model fields. This model should to use with inherit to logging
-    subclass like file, sqlite, etc.
-    """
-
-    name: str = Field(description="A workflow name.")
-    release: datetime = Field(description="A release datetime.")
-    type: str = Field(description="A running type before logging.")
-    context: DictData = Field(
-        default_factory=dict,
-        description="A context that receive from a workflow execution result.",
-    )
-    parent_run_id: Optional[str] = Field(
-        default=None, description="A parent running ID."
-    )
-    run_id: str = Field(description="A running ID")
-    update: datetime = Field(default_factory=get_dt_tznow)
-    execution_time: float = Field(default=0, description="An execution time.")
-
-    @model_validator(mode="after")
-    def __model_action(self) -> Self:
-        """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
-
-        :rtype: Self
-        """
-        if config.enable_write_audit:
-            self.do_before()
-        return self
-
-    def do_before(self) -> None:  # pragma: no cov
-        """To something before end up of initial log model."""
-
-    @abstractmethod
-    def save(self, excluded: list[str] | None) -> None:  # pragma: no cov
-        """Save this model logging to target logging store."""
-        raise NotImplementedError("Audit should implement ``save`` method.")
-
-
-class FileAudit(BaseAudit):
-    """File Audit Pydantic Model that use to saving log data from result of
-    workflow execution. It inherits from BaseAudit model that implement the
-    ``self.save`` method for file.
-    """
-
-    filename_fmt: ClassVar[str] = (
-        "workflow={name}/release={release:%Y%m%d%H%M%S}"
-    )
-
-    def do_before(self) -> None:
-        """Create directory of release before saving log file."""
-        self.pointer().mkdir(parents=True, exist_ok=True)
-
-    @classmethod
-    def find_audits(cls, name: str) -> Iterator[Self]:
-        """Generate the audit data that found from logs path with specific a
-        workflow name.
-
-        :param name: A workflow name that want to search release logging data.
-
-        :rtype: Iterator[Self]
-        """
-        pointer: Path = config.audit_path / f"workflow={name}"
-        if not pointer.exists():
-            raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
-
-        for file in pointer.glob("./release=*/*.log"):
-            with file.open(mode="r", encoding="utf-8") as f:
-                yield cls.model_validate(obj=json.load(f))
-
-    @classmethod
-    def find_audit_with_release(
-        cls,
-        name: str,
-        release: datetime | None = None,
-    ) -> Self:
-        """Return the audit data that found from logs path with specific
-        workflow name and release values. If a release does not pass to an input
-        argument, it will return the latest release from the current log path.
-
-        :param name: A workflow name that want to search log.
-        :param release: A release datetime that want to search log.
-
-        :raise FileNotFoundError:
-        :raise NotImplementedError: If an input release does not pass to this
-            method. Because this method does not implement latest log.
-
-        :rtype: Self
-        """
-        if release is None:
-            raise NotImplementedError("Find latest log does not implement yet.")
-
-        pointer: Path = (
-            config.audit_path
-            / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
-        )
-        if not pointer.exists():
-            raise FileNotFoundError(
-                f"Pointer: ./logs/workflow={name}/"
-                f"release={release:%Y%m%d%H%M%S} does not found."
-            )
-
-        with max(pointer.glob("./*.log"), key=os.path.getctime).open(
-            mode="r", encoding="utf-8"
-        ) as f:
-            return cls.model_validate(obj=json.load(f))
-
-    @classmethod
-    def is_pointed(cls, name: str, release: datetime) -> bool:
-        """Check the release log already pointed or created at the destination
-        log path.
-
-        :param name: A workflow name.
-        :param release: A release datetime.
-
-        :rtype: bool
-        :return: Return False if the release log was not pointed or created.
-        """
-        # NOTE: Return False if enable writing log flag does not set.
-        if not config.enable_write_audit:
-            return False
-
-        # NOTE: create pointer path that use the same logic of pointer method.
-        pointer: Path = config.audit_path / cls.filename_fmt.format(
-            name=name, release=release
-        )
-
-        return pointer.exists()
-
-    def pointer(self) -> Path:
-        """Return release directory path that was generated from model data.
-
-        :rtype: Path
-        """
-        return config.audit_path / self.filename_fmt.format(
-            name=self.name, release=self.release
-        )
-
-    def save(self, excluded: list[str] | None) -> Self:
-        """Save logging data that receive a context data from a workflow
-        execution result.
-
-        :param excluded: An excluded list of key name that want to pass in the
-            model_dump method.
-
-        :rtype: Self
-        """
-        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
-
-        # NOTE: Check environ variable was set for real writing.
-        if not config.enable_write_audit:
-            trace.debug("[LOG]: Skip writing log cause config was set")
-            return self
-
-        log_file: Path = (
-            self.pointer() / f"{self.parent_run_id or self.run_id}.log"
-        )
-        log_file.write_text(
-            json.dumps(
-                self.model_dump(exclude=excluded),
-                default=str,
-                indent=2,
-            ),
-            encoding="utf-8",
-        )
-        return self
-
-
-class SQLiteAudit(BaseAudit):  # pragma: no cov
-    """SQLite Audit Pydantic Model."""
-
-    table_name: ClassVar[str] = "audits"
-    schemas: ClassVar[
-        str
-    ] = """
-        workflow        str,
-        release         int,
-        type            str,
-        context         json,
-        parent_run_id   int,
-        run_id          int,
-        update          datetime
-        primary key ( run_id )
-        """
-
-    def save(self, excluded: list[str] | None) -> SQLiteAudit:
-        """Save logging data that receive a context data from a workflow
-        execution result.
-        """
-        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
-
-        # NOTE: Check environ variable was set for real writing.
-        if not config.enable_write_audit:
-            trace.debug("[LOG]: Skip writing log cause config was set")
-            return self
-
-        raise NotImplementedError("SQLiteAudit does not implement yet.")
-
-
-class RemoteFileAudit(FileAudit):  # pragma: no cov
-    """Remote File Audit Pydantic Model."""
-
-    def save(self, excluded: list[str] | None) -> RemoteFileAudit: ...
-
-
-class RedisAudit(BaseAudit):  # pragma: no cov
-    """Redis Audit Pydantic Model."""
-
-    def save(self, excluded: list[str] | None) -> RedisAudit: ...
-
-
-Audit = Union[
-    FileAudit,
-    SQLiteAudit,
-]
-
-
-def get_audit() -> type[Audit]:  # pragma: no cov
-    """Get an audit class that dynamic base on the config audit path value.
-
-    :rtype: type[Audit]
-    """
-    if config.audit_path.is_file():
-        return SQLiteAudit
-    return FileAudit

ddeutil/workflow/caller.py

@@ -1,179 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-import inspect
-import logging
-from dataclasses import dataclass
-from functools import wraps
-from importlib import import_module
-from typing import Any, Callable, Protocol, TypeVar
-
-try:
-    from typing import ParamSpec
-except ImportError:
-    from typing_extensions import ParamSpec
-
-from ddeutil.core import lazy
-
-from .__types import Re
-from .conf import config
-
-T = TypeVar("T")
-P = ParamSpec("P")
-
-logger = logging.getLogger("ddeutil.workflow")
-logging.getLogger("asyncio").setLevel(logging.INFO)
-
-
-class TagFunc(Protocol):
-    """Tag Function Protocol"""
-
-    name: str
-    tag: str
-
-    def __call__(self, *args, **kwargs): ...  # pragma: no cov
-
-
-ReturnTagFunc = Callable[P, TagFunc]
-DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
-
-
-def tag(
-    name: str, alias: str | None = None
-) -> DecoratorTagFunc:  # pragma: no cov
-    """Tag decorator function that set function attributes, ``tag`` and ``name``
-    for making registries variable.
-
-    :param: name: (str) A tag name for make different use-case of a function.
-    :param: alias: (str) A alias function name that keeping in registries.
-        If this value does not supply, it will use original function name
-        from `__name__` argument.
-
-    :rtype: Callable[P, TagFunc]
-    """
-
-    def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
-        func.tag = name
-        func.name = alias or func.__name__.replace("_", "-")
-
-        @wraps(func)
-        def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            return func(*args, **kwargs)
-
-        @wraps(func)
-        async def awrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            return await func(*args, **kwargs)
-
-        return awrapped if inspect.iscoroutinefunction(func) else wrapped
-
-    return func_internal
-
-
-Registry = dict[str, Callable[[], TagFunc]]
-
-
-def make_registry(submodule: str) -> dict[str, Registry]:
-    """Return registries of all functions that able to called with task.
-
-    :param submodule: (str) A module prefix that want to import registry.
-
-    :rtype: dict[str, Registry]
-    """
-    rs: dict[str, Registry] = {}
-    regis_calls: list[str] = config.regis_call
-    regis_calls.extend(["ddeutil.vendors"])
-    for module in regis_calls:
-        # NOTE: try to sequential import task functions
-        try:
-            importer = import_module(f"{module}.{submodule}")
-        except ModuleNotFoundError:
-            continue
-
-        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
-            # NOTE: check function attribute that already set tag by
-            #   ``utils.tag`` decorator.
-            if not (
-                hasattr(func, "tag") and hasattr(func, "name")
-            ):  # pragma: no cov
-                continue
-
-            # NOTE: Define type of the func value.
-            func: TagFunc
-
-            # NOTE: Create new register name if it not exists
-            if func.name not in rs:
-                rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
-                continue
-
-            if func.tag in rs[func.name]:
-                raise ValueError(
-                    f"The tag {func.tag!r} already exists on "
-                    f"{module}.{submodule}, you should change this tag name or "
-                    f"change it func name."
-                )
-            rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
-
-    return rs
-
-
-@dataclass(frozen=True)
-class CallSearchData:
-    """Call Search dataclass that use for receive regular expression grouping
-    dict from searching call string value.
-    """
-
-    path: str
-    func: str
-    tag: str
-
-
-def extract_call(call: str) -> Callable[[], TagFunc]:
-    """Extract Call function from string value to call partial function that
-    does run it at runtime.
-
-    :param call: (str) A call value that able to match with Task regex.
-
-        The format of call value should contain 3 regular expression groups
-    which match with the below config format:
-
-        >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
-
-    Examples:
-        >>> extract_call("tasks/el-postgres-to-delta@polars")
-        ...
-        >>> extract_call("tasks/return-type-not-valid@raise")
-        ...
-
-    :raise NotImplementedError: When the searching call's function result does
-        not exist in the registry.
-    :raise NotImplementedError: When the searching call's tag result does not
-        exist in the registry with its function key.
-
-    :rtype: Callable[[], TagFunc]
-    """
-    if not (found := Re.RE_TASK_FMT.search(call)):
-        raise ValueError(
-            f"Call {call!r} does not match with the call regex format."
-        )
-
-    # NOTE: Pass the searching call string to `path`, `func`, and `tag`.
-    call: CallSearchData = CallSearchData(**found.groupdict())
-
-    # NOTE: Registry object should implement on this package only.
-    rgt: dict[str, Registry] = make_registry(f"{call.path}")
-    if call.func not in rgt:
-        raise NotImplementedError(
-            f"`REGISTER-MODULES.{call.path}.registries` does not "
-            f"implement registry: {call.func!r}."
-        )
-
-    if call.tag not in rgt[call.func]:
-        raise NotImplementedError(
-            f"tag: {call.tag!r} does not found on registry func: "
-            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
-        )
-    return rgt[call.func][call.tag]

ddeutil_workflow-0.0.40.dist-info/RECORD

@@ -1,33 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=4_L4aAteV6ecIgxXQuBmMHF6LoFXtYgCVakxe2GlHjk,28
-ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
-ddeutil/workflow/__init__.py,sha256=hIM2Ha-7F3YF3aLsu4IY3N-UvD_EE1ai6DO6WL2mILg,1908
-ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
-ddeutil/workflow/audit.py,sha256=BpzLI6ZKXi6xQO8C2sAHxSi8RAfF31dMPdjn3DbYlw8,8364
-ddeutil/workflow/caller.py,sha256=M7_nan1DbRUAHEIQoQc1qIX4AHgI5fKFNx0KDbzhDMk,5736
-ddeutil/workflow/conf.py,sha256=USYzITPwBS5Q_pl3DTTdFK8OxeL9GQu4GUXqi8nKpJo,11819
-ddeutil/workflow/context.py,sha256=vsk4JQL7t3KsnKPfshw3O7YrPFo2h4rnnNd3B-G9Kj4,1700
-ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
-ddeutil/workflow/exceptions.py,sha256=fO37f9p7lOjIJgVOpKE_1X44yJTwBepyukZV9a7NNm4,1241
-ddeutil/workflow/job.py,sha256=LCQf_3gyTmnj6aAQ0ksz4lJxU10-F5MfVFHczkWt_VE,28669
-ddeutil/workflow/logs.py,sha256=mAKQjNri-oourd3dBLLG3Fqoo4QG27U9IO2h6IqCOU0,10196
-ddeutil/workflow/params.py,sha256=qw9XJyjh2ocf9pf6h_XiYHLOvQN4R5TMqPElmItKnRM,8019
-ddeutil/workflow/result.py,sha256=cDkItrhpzZfMS1Oj8IZX8O-KBD4KZYDi43XJZvvC3Gc,4318
-ddeutil/workflow/scheduler.py,sha256=daNdcTp8dQN0gXuK3cg7K7idspOcqKNmfOLTWirNEUM,27652
-ddeutil/workflow/stages.py,sha256=ONBR7GjgLEbA21wNioBrhOucwqj6M1v2ht5JhipoWSg,36994
-ddeutil/workflow/templates.py,sha256=yA2xgrSXcxfBxNT2hc6v06HkVY_0RKsc1UwdJRip9EE,11554
-ddeutil/workflow/utils.py,sha256=Fz5y-LK_JfikDfvMKcFaxad_VvCnr7UC2C9KFCbzPNA,7105
-ddeutil/workflow/workflow.py,sha256=8ForojeLboLL6dHmV5-NjtU9o4mIL7NtY2F2NFxdqZY,49400
-ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
-ddeutil/workflow/api/api.py,sha256=gGQtqkzyJNaJIfka_w2M1lrCS3Ep46re2Dznsk9RxYQ,5191
-ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
-ddeutil/workflow/api/repeat.py,sha256=cycd1-91j-4v6uY1SkrZHd9l95e-YgVC4UCSNNFuGJ8,5277
-ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
-ddeutil/workflow/api/routes/job.py,sha256=YVta083i8vU8-o4WdKFwDpfdC9vN1dZ6goZSmNlQXHA,1954
-ddeutil/workflow/api/routes/logs.py,sha256=uEQ6k5PwRg-k0eSiSFArXfyeDq5xzepxILrRnwgAe1I,5373
-ddeutil/workflow/api/routes/schedules.py,sha256=uWYDOwlV8w56hKQmfkQFwdZ6t2gZSJeCdBIzMmJenAQ,4824
-ddeutil/workflow/api/routes/workflows.py,sha256=KVywA7vD9b4QrfmWBdSFF5chj34yJe1zNCzl6iBMeGI,4538
-ddeutil_workflow-0.0.40.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.40.dist-info/METADATA,sha256=AdIiZTWsPhfcVba7x2OrWw5CN3y0OQB-dRdBb9RQh2o,19501
-ddeutil_workflow-0.0.40.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddeutil_workflow-0.0.40.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.40.dist-info/RECORD,,