ddeutil-workflow 0.0.81__py3-none-any.whl → 0.0.83__py3-none-any.whl

ddeutil/workflow/audits.py

@@ -49,33 +49,72 @@ import zlib
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import datetime, timedelta
+from enum import Enum
 from pathlib import Path
 from typing import Annotated, Any, ClassVar, Literal, Optional, Union
 from urllib.parse import ParseResult, urlparse
 
-from pydantic import BaseModel, Field, TypeAdapter
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData
 from .conf import dynamic
-from .traces import TraceManager, get_trace, set_logging
+from .traces import Trace, get_trace, set_logging
 
 logger = logging.getLogger("ddeutil.workflow")
 
 
+class ReleaseType(str, Enum):
+    """Release type enumeration for workflow execution modes.
+
+    This enum defines the different types of workflow releases that can be
+    triggered, each with specific behavior and use cases.
+
+    Attributes:
+        NORMAL: Standard workflow release execution
+        RERUN: Re-execution of previously failed workflow
+        DRYRUN: Dry-execution workflow
+        FORCE: Forced execution bypassing normal conditions
+    """
+
+    NORMAL = "normal"
+    RERUN = "rerun"
+    FORCE = "force"
+    DRYRUN = "dryrun"
+
+
+NORMAL = ReleaseType.NORMAL
+RERUN = ReleaseType.RERUN
+DRYRUN = ReleaseType.DRYRUN
+FORCE = ReleaseType.FORCE
+
+
 class AuditData(BaseModel):
+    """Audit Data model that use to be the core data for any Audit model manage
+    logging at the target pointer system or service like file-system, sqlite
+    database, etc.
+    """
+
+    model_config = ConfigDict(use_enum_values=True)
+
     name: str = Field(description="A workflow name.")
     release: datetime = Field(description="A release datetime.")
-    type: str = Field(description="A running type before logging.")
+    type: ReleaseType = Field(
+        default=NORMAL,
+        description=(
+            "An execution type that should be value in ('normal', 'rerun', "
+            "'force', 'dryrun')."
+        ),
+    )
     context: DictData = Field(
         default_factory=dict,
         description="A context that receive from a workflow execution result.",
     )
+    run_id: str = Field(description="A running ID")
     parent_run_id: Optional[str] = Field(
         default=None, description="A parent running ID."
     )
-    run_id: str = Field(description="A running ID")
     runs_metadata: DictData = Field(
         default_factory=dict,
         description="A runs metadata that will use to tracking this audit log.",
@@ -89,18 +128,17 @@ class BaseAudit(BaseModel, ABC):
     for logging subclasses like file, sqlite, etc.
     """
 
-    type: str
+    type: Literal["base"] = "base"
+    logging_name: str = "ddeutil.workflow"
     extras: DictData = Field(
         default_factory=dict,
         description="An extras parameter that want to override core config",
     )
 
     @field_validator("extras", mode="before")
-    def validate_extras(cls, v: Any) -> DictData:
+    def __prepare_extras(cls, v: Any) -> Any:
         """Validate extras field to ensure it's a dictionary."""
-        if v is None:
-            return {}
-        return v
+        return {} if v is None else v
 
     @model_validator(mode="after")
     def __model_action(self) -> Self:
@@ -116,13 +154,13 @@ class BaseAudit(BaseModel, ABC):
             self.do_before()
 
         # NOTE: Start setting log config in this line with cache.
-        set_logging("ddeutil.workflow")
+        set_logging(self.logging_name)
         return self
 
     @abstractmethod
     def is_pointed(
         self,
-        data: AuditData,
+        data: Any,
         *,
         extras: Optional[DictData] = None,
     ) -> bool:
@@ -216,7 +254,7 @@ class BaseAudit(BaseModel, ABC):
         raise NotImplementedError("Audit should implement `save` method.")
 
 
-class FileAudit(BaseAudit):
+class LocalFileAudit(BaseAudit):
     """File Audit Pydantic Model for saving log data from workflow execution.
 
     This class inherits from BaseAudit and implements file-based storage
@@ -224,19 +262,25 @@ class FileAudit(BaseAudit):
     in a structured directory hierarchy.
 
     Attributes:
-        filename_fmt: Class variable defining the filename format for audit files.
+        file_fmt: Class variable defining the filename format for audit log.
+        file_release_fmt: Class variable defining the filename format for audit
+            release log.
     """
 
-    filename_fmt: ClassVar[str] = (
-        "workflow={name}/release={release:%Y%m%d%H%M%S}"
-    )
+    file_fmt: ClassVar[str] = "workflow={name}"
+    file_release_fmt: ClassVar[str] = "release={release:%Y%m%d%H%M%S}"
 
     type: Literal["file"] = "file"
-    path: str = Field(
-        default="./audits",
+    path: Path = Field(
+        default=Path("./audits"),
         description="A file path that use to manage audit logs.",
     )
 
+    @field_validator("path", mode="before", json_schema_input_type=str)
+    def __prepare_path(cls, data: Any) -> Any:
+        """Prepare path that passing with string to Path instance."""
+        return Path(data) if isinstance(data, str) else data
+
     def do_before(self) -> None:
         """Create directory of release before saving log file.
 
@@ -246,7 +290,10 @@ class FileAudit(BaseAudit):
         Path(self.path).mkdir(parents=True, exist_ok=True)
 
     def find_audits(
-        self, name: str, *, extras: Optional[DictData] = None
+        self,
+        name: str,
+        *,
+        extras: Optional[DictData] = None,
     ) -> Iterator[AuditData]:
         """Generate audit data found from logs path for a specific workflow name.
 
@@ -260,7 +307,7 @@ class FileAudit(BaseAudit):
         Raises:
             FileNotFoundError: If the workflow directory does not exist.
         """
-        pointer: Path = Path(self.path) / f"workflow={name}"
+        pointer: Path = self.path / self.file_fmt.format(name=name)
         if not pointer.exists():
             raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
 
@@ -293,7 +340,7 @@ class FileAudit(BaseAudit):
             ValueError: If no releases found when release is None.
         """
         if release is None:
-            pointer: Path = Path(self.path) / f"workflow={name}"
+            pointer: Path = self.path / self.file_fmt.format(name=name)
             if not pointer.exists():
                 raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
 
@@ -328,21 +375,21 @@ class FileAudit(BaseAudit):
             return AuditData.model_validate(obj=json.load(f))
 
     def is_pointed(
-        self, data: AuditData, *, extras: Optional[DictData] = None
+        self,
+        data: Any,
+        *,
+        extras: Optional[DictData] = None,
     ) -> bool:
         """Check if the release log already exists at the destination log path.
 
         Args:
-            data: The workflow name.
+            data (str):
             extras: Optional extra parameters to override core config.
 
         Returns:
             bool: True if the release log exists, False otherwise.
         """
-        # NOTE: Return False if enable writing log flag does not set.
-        if not dynamic("enable_write_audit", extras=extras):
-            return False
-        return self.pointer(data).exists()
+        return self.pointer(AuditData.model_validate(data)).exists()
 
     def pointer(self, data: AuditData) -> Path:
         """Return release directory path generated from model data.
@@ -350,8 +397,10 @@ class FileAudit(BaseAudit):
         Returns:
             Path: The directory path for the current workflow and release.
         """
-        return Path(self.path) / self.filename_fmt.format(
-            name=data.name, release=data.release
+        return (
+            self.path
+            / self.file_fmt.format(**data.model_dump(by_alias=True))
+            / self.file_release_fmt.format(**data.model_dump(by_alias=True))
         )
 
     def save(self, data: Any, excluded: Optional[list[str]] = None) -> Self:
@@ -365,7 +414,7 @@ class FileAudit(BaseAudit):
             Self: The audit instance after saving.
         """
         audit = AuditData.model_validate(data)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             audit.run_id,
             parent_run_id=audit.parent_run_id,
             extras=self.extras,
@@ -427,7 +476,7 @@ class FileAudit(BaseAudit):
         return cleaned_count
 
 
-class SQLiteAudit(BaseAudit):  # pragma: no cov
+class LocalSQLiteAudit(BaseAudit):  # pragma: no cov
     """SQLite Audit model for database-based audit storage.
 
     This class inherits from BaseAudit and implements SQLite database storage
@@ -435,11 +484,11 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
 
     Attributes:
         table_name: Class variable defining the database table name.
-        schemas: Class variable defining the database schema.
+        ddl: Class variable defining the database schema.
     """
 
     table_name: ClassVar[str] = "audits"
-    schemas: ClassVar[
+    ddl: ClassVar[
         str
     ] = """
         CREATE TABLE IF NOT EXISTS audits (
@@ -457,22 +506,21 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         """
 
     type: Literal["sqlite"] = "sqlite"
-    path: str
+    path: Path = Field(
+        default=Path("./audits.db"),
+        description="A SQLite filepath.",
+    )
 
-    def _ensure_table_exists(self) -> None:
+    def do_before(self) -> None:
         """Ensure the audit table exists in the database."""
-        audit_url = dynamic("audit_url", extras=self.extras)
-        if audit_url is None or not audit_url.path:
+        if self.path.is_dir():
             raise ValueError(
-                "SQLite audit_url must specify a database file path"
+                "SQLite path must specify a database file path not dir."
             )
 
-        audit_url_parse: ParseResult = urlparse(audit_url)
-        db_path = Path(audit_url_parse.path)
-        db_path.parent.mkdir(parents=True, exist_ok=True)
-
-        with sqlite3.connect(db_path) as conn:
-            conn.execute(self.schemas)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with sqlite3.connect(self.path) as conn:
+            conn.execute(self.ddl)
             conn.commit()
 
     def is_pointed(
@@ -655,7 +703,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
             ValueError: If SQLite database is not properly configured.
         """
         audit = AuditData.model_validate(data)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             audit.run_id,
             parent_run_id=audit.parent_run_id,
             extras=self.extras,
@@ -739,27 +787,31 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
             return cursor.rowcount
 
 
+class PostgresAudit(BaseAudit, ABC): ...  # pragma: no cov
+
+
 Audit = Annotated[
     Union[
-        FileAudit,
-        SQLiteAudit,
+        LocalFileAudit,
+        LocalSQLiteAudit,
     ],
     Field(discriminator="type"),
 ]
 
 
 def get_audit(
-    *,
+    audit_conf: Optional[DictData] = None,
     extras: Optional[DictData] = None,
 ) -> Audit:  # pragma: no cov
     """Get an audit model dynamically based on the config audit path value.
 
     Args:
+        audit_conf (DictData):
         extras: Optional extra parameters to override the core config.
 
     Returns:
         Audit: The appropriate audit model class based on configuration.
     """
-    audit_conf = dynamic("audit_conf", extras=extras)
+    audit_conf = dynamic("audit_conf", f=audit_conf, extras=extras)
     model = TypeAdapter(Audit).validate_python(audit_conf | {"extras": extras})
     return model

ddeutil/workflow/conf.py

@@ -176,17 +176,6 @@ class YamlParser:
     """Base Load object that use to search config data by given some identity
     value like name of `Workflow` or `Crontab` templates.
 
-    :param name: (str) A name of key of config data that read with YAML
-        Environment object.
-    :param path: (Path) A config path object.
-    :param externals: (DictData) An external config data that want to add to
-        loaded config data.
-    :param extras: (DictDdata) An extra parameters that use to override core
-        config values.
-
-    :raise ValueError: If the data does not find on the config path with the
-        name parameter.
-
     Noted:
         The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
@@ -209,6 +198,23 @@ class YamlParser:
         extras: Optional[DictData] = None,
         obj: Optional[Union[object, str]] = None,
     ) -> None:
+        """Main constructure function.
+
+        Args:
+            name (str): A name of key of config data that read with YAML
+                Environment object.
+            path (Path): A config path object.
+            externals (DictData): An external config data that want to add to
+                loaded config data.
+            extras (DictDdata): An extra parameters that use to override core
+                config values.
+            obj (object | str): An object that want to validate from the `type`
+                key before keeping the config data.
+
+        Raises:
+            ValueError: If the data does not find on the config path with the
+                name parameter.
+        """
         self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
         self.extras: DictData = extras or {}
@@ -242,17 +248,19 @@ class YamlParser:
         """Find data with specific key and return the latest modify date data if
         this key exists multiple files.
 
-        :param name: (str) A name of data that want to find.
-        :param path: (Path) A config path object.
-        :param paths: (list[Path]) A list of config path object.
-        :param obj: (object | str) An object that want to validate matching
-            before return.
-        :param extras: (DictData)  An extra parameter that use to override core
-            config values.
-        :param ignore_filename: (str) An ignore filename. Default is
-            ``.confignore`` filename.
+        Args:
+            name (str): A name of data that want to find.
+            path (Path): A config path object.
+            paths (list[Path]): A list of config path object.
+            obj (object | str): An object that want to validate matching
+                before return.
+            extras (DictData):  An extra parameter that use to override core
+                config values.
+            ignore_filename (str): An ignore filename. Default is
+                ``.confignore`` filename.
 
-        :rtype: DictData
+        Returns:
+            DictData: A config data that was found on the searching paths.
         """
         path: Path = dynamic("conf_path", f=path, extras=extras)
         if not paths:
@@ -317,7 +325,9 @@ class YamlParser:
                 ``.confignore`` filename.
             tags (list[str]): A list of tag that want to filter.
 
-        :rtype: Iterator[tuple[str, DictData]]
+        Returns:
+            Iterator[tuple[str, DictData]]: An iterator of config data that was
+                found on the searching paths.
         """
         excluded: list[str] = excluded or []
         tags: list[str] = tags or []
@@ -353,8 +363,11 @@ class YamlParser:
                     ):
                         continue
 
-                    if (t := data.get("type")) and t == obj_type:
-
+                    if (
+                        # isinstance(data, dict) and
+                        (t := data.get("type"))
+                        and t == obj_type
+                    ):
                         # NOTE: Start adding file metadata.
                         file_stat: os.stat_result = file.lstat()
                         data["created_at"] = file_stat.st_ctime
@@ -397,6 +410,13 @@ class YamlParser:
     def filter_yaml(cls, file: Path, name: Optional[str] = None) -> DictData:
         """Read a YAML file context from an input file path and specific name.
 
+        Notes:
+            The data that will return from reading context will map with config
+            name if an input searching name does not pass to this function.
+
+                input: {"name": "foo", "type": "Some"}
+                output: {"foo": {"name": "foo", "type": "Some"}}
+
         Args:
             file (Path): A file path that want to extract YAML context.
             name (str): A key name that search on a YAML context.
@@ -413,7 +433,7 @@ class YamlParser:
                     return (
                         values[name] | {"name": name} if name in values else {}
                     )
-                return values
+                return {values["name"]: values} if "name" in values else values
         return {}
 
     @cached_property

ddeutil/workflow/errors.py

@@ -166,6 +166,15 @@ class StageCancelError(StageError): ...
 class StageSkipError(StageError): ...
 
 
+class StageNestedError(StageError): ...
+
+
+class StageNestedCancelError(StageNestedError): ...
+
+
+class StageNestedSkipError(StageNestedError): ...
+
+
 class JobError(BaseError): ...
 
 
@@ -175,6 +184,9 @@ class JobCancelError(JobError): ...
 class JobSkipError(JobError): ...
 
 
+class EventError(BaseError): ...
+
+
 class WorkflowError(BaseError): ...
 
 

ddeutil/workflow/event.py

@@ -16,13 +16,9 @@ Attributes:
     Interval: Type alias for scheduling intervals ('daily', 'weekly', 'monthly')
 
 Classes:
+    CrontabValue:
     Crontab: Main cron-based event scheduler.
     CrontabYear: Enhanced cron scheduler with year constraints.
-    ReleaseEvent: Release-based event triggers.
-    FileEvent: File system monitoring triggers.
-    WebhookEvent: API/webhook-based triggers.
-    DatabaseEvent: Database change monitoring triggers.
-    SensorEvent: Sensor-based event monitoring.
 
 Example:
     >>> from ddeutil.workflow.event import Crontab
@@ -50,6 +46,8 @@ from pydantic_extra_types.timezone_name import TimeZoneName
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData
+from .errors import EventError
+from .utils import UTC, replace_sec
 
 Interval = Literal["daily", "weekly", "monthly"]
 
@@ -393,19 +391,16 @@ class Event(BaseModel):
     )
 
     @field_validator("schedule", mode="after")
-    def __on_no_dup_and_reach_limit__(
-        cls,
-        value: list[Crontab],
-    ) -> list[Crontab]:
+    def __prepare_schedule__(cls, value: list[Crontab]) -> list[Crontab]:
         """Validate the on fields should not contain duplicate values and if it
         contains the every minute value more than one value, it will remove to
         only one value.
 
         Args:
-            value: A list of on object.
+            value (list[Crontab]): A list of on object.
 
         Returns:
-            list[CronJobYear | Crontab]: The validated list of Crontab objects.
+            list[Crontab]: The validated list of Crontab objects.
 
         Raises:
             ValueError: If it has some duplicate value.
@@ -434,3 +429,31 @@ class Event(BaseModel):
                 "The number of the on should not more than 10 crontabs."
             )
         return value
+
+    def validate_dt(self, dt: datetime) -> datetime:
+        """Validate the release datetime that should was replaced second and
+        millisecond to 0 and replaced timezone to None before checking it match
+        with the set `on` field.
+
+        Args:
+            dt (datetime): A datetime object that want to validate.
+
+        Returns:
+            datetime: The validated release datetime.
+        """
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=UTC)
+
+        release: datetime = replace_sec(dt.astimezone(UTC))
+
+        # NOTE: Return itself if schedule event does not set.
+        if not self.schedule:
+            return release
+
+        for on in self.schedule:
+            if release == on.cronjob.schedule(release, tz=UTC).next:
+                return release
+        raise EventError(
+            f"This datetime, {datetime}, does not support for this event "
+            f"schedule."
+        )