ddeutil-workflow 0.0.54__py3-none-any.whl → 0.0.55__py3-none-any.whl

ddeutil/workflow/workflow.py

@@ -166,6 +166,7 @@ class ReleaseQueue:
     extras: DictData = Field(
         default_factory=dict,
         description="An extra parameters that want to override config values.",
+        repr=False,
     )
 
     @classmethod
@@ -213,15 +214,6 @@ class ReleaseQueue:
         """
         return len(self.queue) > 0
 
-    @property
-    def first_queue(self) -> Release:
-        """Check an input Release object is the first value of the
-        waiting queue.
-
-        :rtype: Release
-        """
-        return self.queue[0]
-
     def check_queue(self, value: Release | datetime) -> bool:
         """Check a Release value already exists in list of tracking
         queues.
@@ -240,16 +232,6 @@ class ReleaseQueue:
             or (value in self.complete)
         )
 
-    def remove_running(self, value: Release) -> Self:
-        """Remove Release in the running queue if it exists.
-
-        :rtype: Self
-        """
-        if value in self.running:
-            self.running.remove(value)
-
-        return self
-
     def mark_complete(self, value: Release) -> Self:
         """Push Release to the complete queue.
 
@@ -702,10 +684,10 @@ class Workflow(BaseModel):
         if isinstance(release, datetime):
             release: Release = Release.from_dt(release, extras=self.extras)
 
-        result.trace.debug(
+        result.trace.info(
             f"[RELEASE]: Start {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
         )
-        self.execute(
+        rs: Result = self.execute(
             params=param2template(
                 params,
                 params={
@@ -724,7 +706,7 @@ class Workflow(BaseModel):
             parent_run_id=result.parent_run_id,
             timeout=timeout,
         )
-        result.trace.debug(
+        result.trace.info(
             f"[RELEASE]: End {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
         )
 
@@ -745,11 +727,12 @@ class Workflow(BaseModel):
         )
 
         if queue:
-            queue.remove_running(release)
+            if release in queue.running:
+                queue.running.remove(release)
             queue.mark_complete(release)
 
         return result.catch(
-            status=SUCCESS,
+            status=rs.status,
             context={
                 "params": params,
                 "release": {
@@ -1054,16 +1037,19 @@ class Workflow(BaseModel):
             For example with non-strategy job, when I want to use the output
         from previous stage, I can access it with syntax:
 
-            ... ${job-id}.stages.${stage-id}.outputs.${key}
-            ... ${job-id}.stages.${stage-id}.errors.${key}
+        ... ${job-id}.stages.${stage-id}.outputs.${key}
+        ... ${job-id}.stages.${stage-id}.errors.${key}
 
             But example for strategy job:
 
-            ... ${job-id}.strategies.${strategy-id}.stages.${stage-id}.outputs.${key}
-            ... ${job-id}.strategies.${strategy-id}.stages.${stage-id}.errors.${key}
+        ... ${job-id}.strategies.${strategy-id}.stages.${stage-id}.outputs.${key}
+        ... ${job-id}.strategies.${strategy-id}.stages.${stage-id}.errors.${key}
 
-        :param params: An input parameters that use on workflow execution that
-            will parameterize before using it.
+            This method already handle all exception class that can raise from
+        the job execution. It will warp that error and keep it in the key `errors`
+        at the result context.
+
+        :param params: A parameter data that will parameterize before execution.
         :param run_id: (str | None) A workflow running ID.
         :param parent_run_id: (str | None) A parent workflow running ID.
         :param result: (Result) A Result instance for return context and status.
@@ -1116,7 +1102,6 @@ class Workflow(BaseModel):
             ):
                 job_id: str = job_queue.get()
                 job: Job = self.job(name=job_id)
-
                 if (check := job.check_needs(context["jobs"])) == WAIT:
                     job_queue.task_done()
                     job_queue.put(job_id)
@@ -1227,8 +1212,8 @@ class WorkflowTask:
     :param alias: (str) An alias name of Workflow model.
     :param workflow: (Workflow) A Workflow model instance.
     :param runner: (CronRunner)
-    :param values:
-    :param extras:
+    :param values: A value data that want to parameterize.
+    :param extras: An extra parameter that use to override core config values.
     """
 
     alias: str

{ddeutil_workflow-0.0.54.dist-info → ddeutil_workflow-0.0.55.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.54
+Version: 0.0.55
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -121,12 +121,10 @@ flowchart LR
 
 > [!WARNING]
 > _**Disclaimer**_: I inspire the dynamic YAML statement from the [**GitHub Action**](https://github.com/features/actions),
-> and all configs pattern from several data orchestration framework tools from
-> my data engineering experience. :grimacing:
-
-> [!NOTE]
-> Other workflow orchestration tools that I interest and pick them to be inspiration
-> some for this package:
+> and my experience of data framework configs pattern. :grimacing:
+>
+> Other workflow orchestration services that I interest and pick them to be
+> this project inspiration:
 >
 > - [Google **Workflows**](https://cloud.google.com/workflows)
 > - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)

{ddeutil_workflow-0.0.54.dist-info → ddeutil_workflow-0.0.55.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ddeutil/workflow/__about__.py,sha256=KTbXC0Wvt30Z6UCFcSzVAD697wqCc-kgyHEuDu_xEtY,28
+ddeutil/workflow/__about__.py,sha256=kfTS6Gqr5xkMNDVKm-NTS4vmeIl2Zj4oMBQGmq19LHA,28
 ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
 ddeutil/workflow/__init__.py,sha256=noE8LNRcgq32m9OnIFcQqh0P7PXWdp-SGmvBCYIXgf4,1338
 ddeutil/workflow/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -6,26 +6,25 @@ ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4
 ddeutil/workflow/conf.py,sha256=80rgmJKFU7BlH5xTLnghGzGhE8C6LFAQykd9mjHSjo8,12528
 ddeutil/workflow/cron.py,sha256=WS2MInn0Sp5DKlZDZH5VFZ5AA0Q3_AnBnYEU4lZSv4I,9779
 ddeutil/workflow/exceptions.py,sha256=r4Jrf9qtVPALU4wh4bnb_OYqC-StqSQJEmFC-_QK934,1408
-ddeutil/workflow/job.py,sha256=uoj6grIq1f0iV5RFtQaysyel6z9keYr-urkkhFzyhcI,33831
+ddeutil/workflow/job.py,sha256=aVRWLMLv5vYFbckT6AKYrMu29FzXYESOEzDHhFIpUyo,34159
 ddeutil/workflow/logs.py,sha256=rsoBrUGQrooou18fg2yvPsB8NOaXnUA5ThQpBr_WVMg,26598
 ddeutil/workflow/params.py,sha256=FKY4Oo1Ze4QZKRfAk7rqKsi44YaJQAbqAtXM6vlO2hI,11392
-ddeutil/workflow/result.py,sha256=27nPQq9CETLCVczv4vvFEF9w2TllHZ_ROfyDoLFxRWM,5647
+ddeutil/workflow/result.py,sha256=rI0S8-HanFDk1l6_BsYRRamzSfzKUy7bkKJUae1w_aQ,5708
 ddeutil/workflow/reusables.py,sha256=iXcS7Gg-71qVX4ln0ILTDx03cTtUnj_rNoXHTVdVrxc,17636
-ddeutil/workflow/scheduler.py,sha256=4G5AogkmnsTKe7jKYSfU35qjubR82WQ8CLtEe9kqPTE,28304
-ddeutil/workflow/stages.py,sha256=131pGqE5RhhDo9aLy6KTXGrvVLFT3UIwr9fS3tVqoZQ,80466
+ddeutil/workflow/scheduler.py,sha256=hk3-9R63DZH9J0PWbtBzaD8rqHyiOf03vTvAJDgsxTA,28279
+ddeutil/workflow/stages.py,sha256=E5XoMVijjcvm_YK8AbiA8xGAQUphCPTtGazW-oLAdeI,82543
 ddeutil/workflow/utils.py,sha256=NZPvPPP_5g4cigFcD7tHjIKLtKMeYAcb3oUhNyhTpJ0,7947
-ddeutil/workflow/workflow.py,sha256=-7M3HdxOpuPpXeRF8oWhDh_S8anX3ivSoKlOsKnsh6c,46942
-ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
-ddeutil/workflow/api/api.py,sha256=xLrQ8yD7iOn-MkzaSxG-BADbdkqLikDna630oW3YEmc,5243
+ddeutil/workflow/workflow.py,sha256=F3I_c0LHMoJFZIFt1a92sxFMlQVkE-cH--TLyX2rwuo,46799
+ddeutil/workflow/api/__init__.py,sha256=kY30dL8HPY8tY_GBmm7y_3OdoXzB1-EA2a96PLU0AQw,5278
 ddeutil/workflow/api/logs.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
 ddeutil/workflow/api/utils.py,sha256=uTtUFVLpiYYahXvCVx8sueRQ03K2Xw1id_gW3IMmX1U,5295
 ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
-ddeutil/workflow/api/routes/job.py,sha256=GbDFmToksKsvPHn9KPXhoOCznzEPRTe79jUn7ew0snE,1953
+ddeutil/workflow/api/routes/job.py,sha256=8X5VLDJH6PumyNIY6JGRNBsf2gWN0eG9DzxRPSh6n4I,2190
 ddeutil/workflow/api/routes/logs.py,sha256=U6vOni3wd-ZTOwd3yVdSOpgyRmNdcgfngU5KlLM3Cww,5383
-ddeutil/workflow/api/routes/schedules.py,sha256=EgUjyRGhsm6UNaMj5luh6TcY6l571sCHcla-BL1iOfY,4829
-ddeutil/workflow/api/routes/workflows.py,sha256=JcDOrn1deK8ztFRcMTNATQejG6KMA7JxZLVc4QeBsP4,4527
-ddeutil_workflow-0.0.54.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.54.dist-info/METADATA,sha256=uVgjeBlD6MEq6XVs4pPNEfhiTuf44TrUzpESmy_8ADw,19081
-ddeutil_workflow-0.0.54.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddeutil_workflow-0.0.54.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.54.dist-info/RECORD,,
+ddeutil/workflow/api/routes/schedules.py,sha256=14RnaJKEGMSJtncI1H_QQVZNBe_jDS40PPRO6qFc3i0,4805
+ddeutil/workflow/api/routes/workflows.py,sha256=GJu5PiXEylswrXylEImpncySjeU9chrvrtjhiMCw2RQ,4529
+ddeutil_workflow-0.0.55.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.55.dist-info/METADATA,sha256=sTjAMsv4yIgw-CKKHyr-sg_yrgnnDT_0UMF3E2TP-Js,19008
+ddeutil_workflow-0.0.55.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddeutil_workflow-0.0.55.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.55.dist-info/RECORD,,

ddeutil/workflow/api/api.py

@@ -1,170 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-import contextlib
-from collections.abc import AsyncIterator
-from datetime import datetime, timedelta
-from typing import TypedDict
-
-from dotenv import load_dotenv
-from fastapi import FastAPI, Request
-from fastapi import status as st
-from fastapi.encoders import jsonable_encoder
-from fastapi.exceptions import RequestValidationError
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.middleware.gzip import GZipMiddleware
-from fastapi.responses import UJSONResponse
-
-from ..__about__ import __version__
-from ..conf import api_config, config
-from ..logs import get_logger
-from ..scheduler import ReleaseThread, ReleaseThreads
-from ..workflow import ReleaseQueue, WorkflowTask
-from .routes import job, log
-from .utils import repeat_at
-
-load_dotenv()
-logger = get_logger("uvicorn.error")
-
-
-class State(TypedDict):
-    """TypeDict for State of FastAPI application."""
-
-    scheduler: list[str]
-    workflow_threads: ReleaseThreads
-    workflow_tasks: list[WorkflowTask]
-    workflow_queue: dict[str, ReleaseQueue]
-
-
-@contextlib.asynccontextmanager
-async def lifespan(a: FastAPI) -> AsyncIterator[State]:
-    """Lifespan function for the FastAPI application."""
-    a.state.scheduler = []
-    a.state.workflow_threads = {}
-    a.state.workflow_tasks = []
-    a.state.workflow_queue = {}
-
-    yield {
-        # NOTE: Scheduler value should be contained a key of workflow and
-        #   list of datetime of queue and running.
-        #
-        #   ... {
-        #   ...     '<workflow-name>': (
-        #   ...         [<running-datetime>, ...], [<queue-datetime>, ...]
-        #   ...     )
-        #   ... }
-        #
-        "scheduler": a.state.scheduler,
-        "workflow_queue": a.state.workflow_queue,
-        "workflow_threads": a.state.workflow_threads,
-        "workflow_tasks": a.state.workflow_tasks,
-    }
-
-
-app = FastAPI(
-    titile="Workflow",
-    description=(
-        "This is a workflow FastAPI application that use to manage manual "
-        "execute, logging, and schedule workflow via RestAPI."
-    ),
-    version=__version__,
-    lifespan=lifespan,
-    default_response_class=UJSONResponse,
-)
-app.add_middleware(GZipMiddleware, minimum_size=1000)
-origins: list[str] = [
-    "http://localhost",
-    "http://localhost:88",
-    "http://localhost:80",
-]
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=origins,
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-@app.get("/")
-async def health():
-    """Index view that not return any template without json status."""
-    return {"message": "Workflow already start up with healthy status."}
-
-
-# NOTE Add the jobs and logs routes by default.
-app.include_router(job, prefix=api_config.prefix_path)
-app.include_router(log, prefix=api_config.prefix_path)
-
-
-# NOTE: Enable the workflows route.
-if api_config.enable_route_workflow:
-    from .routes import workflow
-
-    app.include_router(workflow, prefix=api_config.prefix_path)
-
-
-# NOTE: Enable the schedules route.
-if api_config.enable_route_schedule:
-    from ..logs import get_audit
-    from ..scheduler import schedule_task
-    from .routes import schedule
-
-    app.include_router(schedule, prefix=api_config.prefix_path)
-
-    @schedule.on_event("startup")
-    @repeat_at(cron="* * * * *", delay=2)
-    def scheduler_listener():
-        """Schedule broker every minute at 02 second."""
-        logger.debug(
-            f"[SCHEDULER]: Start listening schedule from queue "
-            f"{app.state.scheduler}"
-        )
-        if app.state.workflow_tasks:
-            schedule_task(
-                app.state.workflow_tasks,
-                stop=datetime.now(config.tz) + timedelta(minutes=1),
-                queue=app.state.workflow_queue,
-                threads=app.state.workflow_threads,
-                audit=get_audit(),
-            )
-
-    @schedule.on_event("startup")
-    @repeat_at(cron="*/5 * * * *", delay=10)
-    def monitoring():
-        """Monitoring workflow thread that running in the background."""
-        logger.debug("[MONITOR]: Start monitoring threading.")
-        snapshot_threads: list[str] = list(app.state.workflow_threads.keys())
-        for t_name in snapshot_threads:
-
-            thread_release: ReleaseThread = app.state.workflow_threads[t_name]
-
-            # NOTE: remove the thread that running success.
-            if not thread_release["thread"].is_alive():
-                app.state.workflow_threads.pop(t_name)
-
-
-@app.exception_handler(RequestValidationError)
-async def validation_exception_handler(
-    request: Request, exc: RequestValidationError
-):
-    _ = request
-    return UJSONResponse(
-        status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
-        content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}),
-    )
-
-
-if __name__ == "__main__":
-    import uvicorn
-
-    uvicorn.run(
-        app,
-        host="0.0.0.0",
-        port=80,
-        log_level="DEBUG",
-    )