ddeutil-workflow 0.0.6__py3-none-any.whl → 0.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ddeutil/workflow/utils.py CHANGED
@@ -6,9 +6,11 @@
6
6
  from __future__ import annotations
7
7
 
8
8
  import inspect
9
+ import logging
9
10
  import os
10
11
  import stat
11
12
  from abc import ABC, abstractmethod
13
+ from ast import Call, Constant, Expr, Module, Name, parse
12
14
  from collections.abc import Iterator
13
15
  from dataclasses import dataclass, field
14
16
  from datetime import date, datetime
@@ -20,14 +22,24 @@ from pathlib import Path
20
22
  from typing import Any, Callable, Literal, Optional, Protocol, Union
21
23
  from zoneinfo import ZoneInfo
22
24
 
23
- from ddeutil.core import getdot, hasdot, lazy
24
- from ddeutil.io import PathData
25
+ from ddeutil.core import getdot, hasdot, import_string, lazy
26
+ from ddeutil.io import PathData, search_env_replace
25
27
  from ddeutil.io.models.lineage import dt_now
26
- from pydantic import BaseModel, Field
28
+ from pydantic import BaseModel, ConfigDict, Field
27
29
  from pydantic.functional_validators import model_validator
28
30
  from typing_extensions import Self
29
31
 
30
32
  from .__types import DictData, Matrix, Re
33
+ from .exceptions import ParamValueException, UtilException
34
+
35
+
36
+ def get_diff_sec(dt: datetime, tz: ZoneInfo | None = None) -> int:
37
+ """Return second value that come from diff of an input datetime and the
38
+ current datetime with specific timezone.
39
+ """
40
+ return round(
41
+ (dt - datetime.now(tz=(tz or ZoneInfo("UTC")))).total_seconds()
42
+ )
31
43
 
32
44
 
33
45
  class Engine(BaseModel):
@@ -35,9 +47,10 @@ class Engine(BaseModel):
35
47
 
36
48
  paths: PathData = Field(default_factory=PathData)
37
49
  registry: list[str] = Field(
38
- default_factory=lambda: [
39
- "ddeutil.workflow",
40
- ],
50
+ default_factory=lambda: ["ddeutil.workflow"],
51
+ )
52
+ registry_filter: list[str] = Field(
53
+ default=lambda: ["ddeutil.workflow.utils"]
41
54
  )
42
55
 
43
56
  @model_validator(mode="before")
@@ -47,9 +60,21 @@ class Engine(BaseModel):
47
60
  """
48
61
  if (_regis := values.get("registry")) and isinstance(_regis, str):
49
62
  values["registry"] = [_regis]
63
+ if (_regis_filter := values.get("registry_filter")) and isinstance(
64
+ _regis, str
65
+ ):
66
+ values["registry_filter"] = [_regis_filter]
50
67
  return values
51
68
 
52
69
 
70
+ class CoreConf(BaseModel):
71
+ """Core Config Model"""
72
+
73
+ model_config = ConfigDict(arbitrary_types_allowed=True)
74
+
75
+ tz: ZoneInfo = Field(default_factory=lambda: ZoneInfo("UTC"))
76
+
77
+
53
78
  class ConfParams(BaseModel):
54
79
  """Params Model"""
55
80
 
@@ -57,16 +82,24 @@ class ConfParams(BaseModel):
57
82
  default_factory=Engine,
58
83
  description="A engine mapping values.",
59
84
  )
85
+ core: CoreConf = Field(
86
+ default_factory=CoreConf,
87
+ description="A core config value",
88
+ )
60
89
 
61
90
 
62
91
  def config() -> ConfParams:
63
92
  """Load Config data from ``workflows-conf.yaml`` file."""
64
93
  root_path: str = os.getenv("WORKFLOW_ROOT_PATH", ".")
65
94
 
66
- regis: list[str] = []
95
+ regis: list[str] = ["ddeutil.workflow"]
67
96
  if regis_env := os.getenv("WORKFLOW_CORE_REGISTRY"):
68
97
  regis = [r.strip() for r in regis_env.split(",")]
69
98
 
99
+ regis_filter: list[str] = ["ddeutil.workflow.utils"]
100
+ if regis_filter_env := os.getenv("WORKFLOW_CORE_REGISTRY_FILTER"):
101
+ regis_filter = [r.strip() for r in regis_filter_env.split(",")]
102
+
70
103
  conf_path: str = (
71
104
  f"{root_path}/{conf_env}"
72
105
  if (conf_env := os.getenv("WORKFLOW_CORE_PATH_CONF"))
@@ -76,6 +109,7 @@ def config() -> ConfParams:
76
109
  obj={
77
110
  "engine": {
78
111
  "registry": regis,
112
+ "registry_filter": regis_filter,
79
113
  "paths": {
80
114
  "root": root_path,
81
115
  "conf": conf_path,
@@ -115,24 +149,24 @@ class TagFunc(Protocol):
115
149
  def __call__(self, *args, **kwargs): ...
116
150
 
117
151
 
118
- def tag(value: str, name: str | None = None):
152
+ def tag(name: str, alias: str | None = None):
119
153
  """Tag decorator function that set function attributes, ``tag`` and ``name``
120
154
  for making registries variable.
121
155
 
122
- :param: value: A tag value for make different use-case of a function.
123
- :param: name: A name that keeping in registries.
156
+ :param: name: A tag value for make different use-case of a function.
157
+ :param: alias: A alias function name that keeping in registries. If this
158
+ value does not supply, it will use original function name from __name__.
124
159
  """
125
160
 
126
- def func_internal(func: callable) -> TagFunc:
127
- func.tag = value
128
- func.name = name or func.__name__.replace("_", "-")
161
+ def func_internal(func: Callable[[...], Any]) -> TagFunc:
162
+ func.tag = name
163
+ func.name = alias or func.__name__.replace("_", "-")
129
164
 
130
165
  @wraps(func)
131
166
  def wrapped(*args, **kwargs):
167
+ # NOTE: Able to do anything before calling hook function.
132
168
  return func(*args, **kwargs)
133
169
 
134
- # TODO: pass result from a wrapped to Result model
135
- # >>> return Result.model_validate(obj=wrapped)
136
170
  return wrapped
137
171
 
138
172
  return func_internal
@@ -145,6 +179,7 @@ def make_registry(submodule: str) -> dict[str, Registry]:
145
179
  """Return registries of all functions that able to called with task.
146
180
 
147
181
  :param submodule: A module prefix that want to import registry.
182
+ :rtype: dict[str, Registry]
148
183
  """
149
184
  rs: dict[str, Registry] = {}
150
185
  for module in config().engine.registry:
@@ -185,7 +220,7 @@ class BaseParam(BaseModel, ABC):
185
220
 
186
221
  @abstractmethod
187
222
  def receive(self, value: Optional[Any] = None) -> Any:
188
- raise ValueError(
223
+ raise NotImplementedError(
189
224
  "Receive value and validate typing before return valid value."
190
225
  )
191
226
 
@@ -197,14 +232,14 @@ class DefaultParam(BaseParam):
197
232
 
198
233
  @abstractmethod
199
234
  def receive(self, value: Optional[Any] = None) -> Any:
200
- raise ValueError(
235
+ raise NotImplementedError(
201
236
  "Receive value and validate typing before return valid value."
202
237
  )
203
238
 
204
239
  @model_validator(mode="after")
205
240
  def check_default(self) -> Self:
206
241
  if not self.required and self.default is None:
207
- raise ValueError(
242
+ raise ParamValueException(
208
243
  "Default should set when this parameter does not required."
209
244
  )
210
245
  return self
@@ -218,6 +253,7 @@ class DatetimeParam(DefaultParam):
218
253
  default: datetime = Field(default_factory=dt_now)
219
254
 
220
255
  def receive(self, value: str | datetime | date | None = None) -> datetime:
256
+ """Receive value that match with datetime."""
221
257
  if value is None:
222
258
  return self.default
223
259
 
@@ -226,7 +262,7 @@ class DatetimeParam(DefaultParam):
226
262
  elif isinstance(value, date):
227
263
  return datetime(value.year, value.month, value.day)
228
264
  elif not isinstance(value, str):
229
- raise ValueError(
265
+ raise ParamValueException(
230
266
  f"Value that want to convert to datetime does not support for "
231
267
  f"type: {type(value)}"
232
268
  )
@@ -239,6 +275,7 @@ class StrParam(DefaultParam):
239
275
  type: Literal["str"] = "str"
240
276
 
241
277
  def receive(self, value: Optional[str] = None) -> str | None:
278
+ """Receive value that match with str."""
242
279
  if value is None:
243
280
  return self.default
244
281
  return str(value)
@@ -250,13 +287,14 @@ class IntParam(DefaultParam):
250
287
  type: Literal["int"] = "int"
251
288
 
252
289
  def receive(self, value: Optional[int] = None) -> int | None:
290
+ """Receive value that match with int."""
253
291
  if value is None:
254
292
  return self.default
255
293
  if not isinstance(value, int):
256
294
  try:
257
295
  return int(str(value))
258
296
  except TypeError as err:
259
- raise ValueError(
297
+ raise ParamValueException(
260
298
  f"Value that want to convert to integer does not support "
261
299
  f"for type: {type(value)}"
262
300
  ) from err
@@ -264,6 +302,8 @@ class IntParam(DefaultParam):
264
302
 
265
303
 
266
304
  class ChoiceParam(BaseParam):
305
+ """Choice parameter."""
306
+
267
307
  type: Literal["choice"] = "choice"
268
308
  options: list[str]
269
309
 
@@ -274,13 +314,16 @@ class ChoiceParam(BaseParam):
274
314
  if value is None:
275
315
  return self.options[0]
276
316
  if any(value not in self.options):
277
- raise ValueError(f"{value} does not match any value in options")
317
+ raise ParamValueException(
318
+ f"{value!r} does not match any value in choice options."
319
+ )
278
320
  return value
279
321
 
280
322
 
281
323
  Param = Union[
282
324
  ChoiceParam,
283
325
  DatetimeParam,
326
+ IntParam,
284
327
  StrParam,
285
328
  ]
286
329
 
@@ -291,6 +334,11 @@ class Result:
291
334
  the pipeline execution.
292
335
  """
293
336
 
337
+ # TODO: Add running ID to this result dataclass.
338
+ # ---
339
+ # parent_run_id: str
340
+ # run_id: str
341
+ #
294
342
  status: int = field(default=2)
295
343
  context: DictData = field(default_factory=dict)
296
344
 
@@ -301,11 +349,216 @@ def make_exec(path: str | Path):
301
349
  f.chmod(f.stat().st_mode | stat.S_IEXEC)
302
350
 
303
351
 
304
- def param2template(
352
+ FILTERS: dict[str, callable] = {
353
+ "abs": abs,
354
+ "str": str,
355
+ "int": int,
356
+ "upper": lambda x: x.upper(),
357
+ "lower": lambda x: x.lower(),
358
+ "rstr": [str, repr],
359
+ }
360
+
361
+
362
+ class FilterFunc(Protocol):
363
+ """Tag Function Protocol"""
364
+
365
+ name: str
366
+
367
+ def __call__(self, *args, **kwargs): ...
368
+
369
+
370
+ def custom_filter(name: str):
371
+ """Custom filter decorator function that set function attributes, ``filter``
372
+ for making filter registries variable.
373
+
374
+ :param: name: A filter name for make different use-case of a function.
375
+ """
376
+
377
+ def func_internal(func: Callable[[...], Any]) -> TagFunc:
378
+ func.filter = name
379
+
380
+ @wraps(func)
381
+ def wrapped(*args, **kwargs):
382
+ # NOTE: Able to do anything before calling custom filter function.
383
+ return func(*args, **kwargs)
384
+
385
+ return wrapped
386
+
387
+ return func_internal
388
+
389
+
390
+ FilterRegistry = Union[FilterFunc, Callable[[...], Any]]
391
+
392
+
393
+ def make_filter_registry() -> dict[str, FilterRegistry]:
394
+ """Return registries of all functions that able to called with task.
395
+
396
+ :rtype: dict[str, Registry]
397
+ """
398
+ rs: dict[str, Registry] = {}
399
+ for module in config().engine.registry_filter:
400
+ # NOTE: try to sequential import task functions
401
+ try:
402
+ importer = import_module(module)
403
+ except ModuleNotFoundError:
404
+ continue
405
+
406
+ for fstr, func in inspect.getmembers(importer, inspect.isfunction):
407
+ # NOTE: check function attribute that already set tag by
408
+ # ``utils.tag`` decorator.
409
+ if not hasattr(func, "filter"):
410
+ continue
411
+
412
+ rs[func.filter] = import_string(f"{module}.{fstr}")
413
+
414
+ rs.update(FILTERS)
415
+ return rs
416
+
417
+
418
+ def get_args_const(
419
+ expr: str,
420
+ ) -> tuple[str, list[Constant], dict[str, Constant]]:
421
+ """Get arguments and keyword-arguments from function calling string."""
422
+ try:
423
+ mod: Module = parse(expr)
424
+ except SyntaxError:
425
+ raise UtilException(
426
+ f"Post-filter: {expr} does not valid because it raise syntax error."
427
+ ) from None
428
+ body: list[Expr] = mod.body
429
+
430
+ if len(body) > 1:
431
+ raise UtilException(
432
+ "Post-filter function should be only one calling per pipe"
433
+ )
434
+
435
+ caller: Union[Name, Call]
436
+ if isinstance((caller := body[0].value), Name):
437
+ return caller.id, [], {}
438
+ elif not isinstance(caller, Call):
439
+ raise UtilException(
440
+ f"Get arguments does not support for caller type: {type(caller)}"
441
+ )
442
+
443
+ name: Name = caller.func
444
+ args: list[Constant] = caller.args
445
+ keywords: dict[str, Constant] = {k.arg: k.value for k in caller.keywords}
446
+
447
+ if any(not isinstance(i, Constant) for i in args):
448
+ raise UtilException("Argument should be constant.")
449
+
450
+ return name.id, args, keywords
451
+
452
+
453
+ @custom_filter("fmt")
454
+ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
455
+ return value.strftime(fmt)
456
+
457
+
458
+ def map_post_filter(
305
459
  value: Any,
306
- params: dict[str, Any],
460
+ post_filter: list[str],
461
+ filters: dict[str, FilterRegistry],
462
+ ) -> Any:
463
+ """Mapping post-filter to value with sequence list of filter function name
464
+ that will get from the filter registry.
465
+
466
+ :param value: A string value that want to mapped with filter function.
467
+ :param post_filter: A list of post-filter function name.
468
+ :param filters: A filter registry.
469
+ """
470
+ for _filter in post_filter:
471
+ func_name, _args, _kwargs = get_args_const(_filter)
472
+ args = [arg.value for arg in _args]
473
+ kwargs = {k: v.value for k, v in _kwargs.items()}
474
+
475
+ if func_name not in filters:
476
+ raise UtilException(
477
+ f"The post-filter: {func_name} does not support yet."
478
+ )
479
+
480
+ try:
481
+ if isinstance((f_func := filters[func_name]), list):
482
+ if args or kwargs:
483
+ raise UtilException(
484
+ "Chain filter function does not support for passing "
485
+ "arguments."
486
+ )
487
+ for func in f_func:
488
+ value: Any = func(value)
489
+ else:
490
+ value: Any = f_func(value, *args, **kwargs)
491
+ except Exception as err:
492
+ logging.warning(str(err))
493
+ raise UtilException(
494
+ f"The post-filter function: {func_name} does not fit with "
495
+ f"{value} (type: {type(value).__name__})."
496
+ ) from None
497
+ return value
498
+
499
+
500
+ def str2template(
501
+ value: str,
502
+ params: DictData,
307
503
  *,
308
- repr_flag: bool = False,
504
+ filters: dict[str, FilterRegistry] | None = None,
505
+ ) -> Any:
506
+ """(Sub-function) Pass param to template string that can search by
507
+ ``RE_CALLER`` regular expression.
508
+
509
+ The getter value that map a template should have typing support align
510
+ with the pipeline parameter types that is `str`, `int`, `datetime`, and
511
+ `list`.
512
+
513
+ :param value: A string value that want to mapped with an params
514
+ :param params: A parameter value that getting with matched regular
515
+ expression.
516
+ :param filters:
517
+ """
518
+ filters: dict[str, FilterRegistry] = filters or make_filter_registry()
519
+
520
+ # NOTE: remove space before and after this string value.
521
+ value: str = value.strip()
522
+ for found in Re.RE_CALLER.finditer(value):
523
+ # NOTE:
524
+ # Get caller and filter values that setting inside;
525
+ #
526
+ # ... ``${{ <caller-value> [ | <filter-value>] ... }}``
527
+ #
528
+ caller: str = found.group("caller")
529
+ pfilter: list[str] = [
530
+ i.strip()
531
+ for i in (
532
+ found.group("post_filters").strip().removeprefix("|").split("|")
533
+ )
534
+ if i != ""
535
+ ]
536
+ if not hasdot(caller, params):
537
+ raise UtilException(f"The params does not set caller: {caller!r}.")
538
+
539
+ # NOTE: from validate step, it guarantee that caller exists in params.
540
+ getter: Any = getdot(caller, params)
541
+
542
+ # NOTE:
543
+ # If type of getter caller is not string type and it does not use to
544
+ # concat other string value, it will return origin value from the
545
+ # ``getdot`` function.
546
+ if value.replace(found.group(0), "", 1) == "":
547
+ return map_post_filter(getter, pfilter, filters=filters)
548
+
549
+ # NOTE: map post-filter function.
550
+ getter: Any = map_post_filter(getter, pfilter, filters=filters)
551
+ if not isinstance(getter, str):
552
+ getter: str = str(getter)
553
+
554
+ value: str = value.replace(found.group(0), getter, 1)
555
+
556
+ return search_env_replace(value)
557
+
558
+
559
+ def param2template(
560
+ value: Any,
561
+ params: DictData,
309
562
  ) -> Any:
310
563
  """Pass param to template string that can search by ``RE_CALLER`` regular
311
564
  expression.
@@ -313,47 +566,18 @@ def param2template(
313
566
  :param value: A value that want to mapped with an params
314
567
  :param params: A parameter value that getting with matched regular
315
568
  expression.
316
- :param repr_flag: A repr flag for using repr instead of str if it set be
317
- true.
318
569
 
319
570
  :rtype: Any
320
571
  :returns: An any getter value from the params input.
321
572
  """
573
+ filters: dict[str, FilterRegistry] = make_filter_registry()
322
574
  if isinstance(value, dict):
323
575
  return {k: param2template(value[k], params) for k in value}
324
576
  elif isinstance(value, (list, tuple, set)):
325
577
  return type(value)([param2template(i, params) for i in value])
326
578
  elif not isinstance(value, str):
327
579
  return value
328
-
329
- if not Re.RE_CALLER.search(value):
330
- return value
331
-
332
- for found in Re.RE_CALLER.finditer(value):
333
-
334
- # NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
335
- caller: str = found.group("caller")
336
- if not hasdot(caller, params):
337
- raise ValueError(f"params does not set caller: {caller!r}")
338
-
339
- getter: Any = getdot(caller, params)
340
-
341
- # NOTE: check type of vars
342
- if isinstance(getter, (str, int)):
343
- value: str = value.replace(
344
- found.group(0), (repr(getter) if repr_flag else str(getter)), 1
345
- )
346
- continue
347
-
348
- # NOTE:
349
- # If type of getter caller does not formatting, it will return origin
350
- # value from the ``getdot`` function.
351
- if value.replace(found.group(0), "", 1) != "":
352
- raise ValueError(
353
- "Callable variable should not pass other outside ${{ ... }}"
354
- )
355
- return getter
356
- return value
580
+ return str2template(value, params, filters=filters)
357
581
 
358
582
 
359
583
  def dash2underscore(
@@ -368,7 +592,7 @@ def dash2underscore(
368
592
  return values
369
593
 
370
594
 
371
- def cross_product(matrix: Matrix) -> Iterator:
595
+ def cross_product(matrix: Matrix) -> Iterator[DictData]:
372
596
  """Iterator of products value from matrix."""
373
597
  yield from (
374
598
  {_k: _v for e in mapped for _k, _v in e.items()}
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ddeutil-workflow
3
- Version: 0.0.6
3
+ Version: 0.0.7
4
4
  Summary: Data Developer & Engineer Workflow Utility Objects
5
5
  Author-email: ddeutils <korawich.anu@gmail.com>
6
6
  License: MIT
@@ -24,9 +24,12 @@ License-File: LICENSE
24
24
  Requires-Dist: fmtutil
25
25
  Requires-Dist: ddeutil-io
26
26
  Requires-Dist: python-dotenv ==1.0.1
27
+ Provides-Extra: api
28
+ Requires-Dist: fastapi[standard] ==0.112.0 ; extra == 'api'
29
+ Requires-Dist: apscheduler[sqlalchemy] <4.0.0,==3.10.4 ; extra == 'api'
30
+ Requires-Dist: croniter ==3.0.3 ; extra == 'api'
27
31
  Provides-Extra: app
28
- Requires-Dist: fastapi ==0.112.0 ; extra == 'app'
29
- Requires-Dist: apscheduler[sqlalchemy] ==3.10.4 ; extra == 'app'
32
+ Requires-Dist: schedule <2.0.0,==1.2.2 ; extra == 'app'
30
33
 
31
34
  # Workflow
32
35
 
@@ -39,7 +42,6 @@ Requires-Dist: apscheduler[sqlalchemy] ==3.10.4 ; extra == 'app'
39
42
 
40
43
  - [Installation](#installation)
41
44
  - [Getting Started](#getting-started)
42
- - [Core Features](#core-features)
43
45
  - [On](#on)
44
46
  - [Pipeline](#pipeline)
45
47
  - [Usage](#usage)
@@ -50,12 +52,14 @@ Requires-Dist: apscheduler[sqlalchemy] ==3.10.4 ; extra == 'app'
50
52
  - [Deployment](#deployment)
51
53
 
52
54
  This **Workflow** objects was created for easy to make a simple metadata
53
- driven pipeline that able to **ETL, T, EL, or ELT** by `.yaml` file.
55
+ driven for data pipeline orchestration that able to use for **ETL, T, EL, or
56
+ ELT** by a `.yaml` file template.
54
57
 
55
- I think we should not create the multiple pipeline per use-case if we able to
56
- write some dynamic pipeline that just change the input parameters per use-case
57
- instead. This way we can handle a lot of pipelines in our orgs with metadata only.
58
- It called **Metadata Driven**.
58
+ In my opinion, I think it should not create duplicate pipeline codes if I can
59
+ write with dynamic input parameters on the one template pipeline that just change
60
+ the input parameters per use-case instead.
61
+ This way I can handle a lot of logical pipelines in our orgs with only metadata
62
+ configuration. It called **Metadata Driven Data Pipeline**.
59
63
 
60
64
  Next, we should get some monitoring tools for manage logging that return from
61
65
  pipeline running. Because it not show us what is a use-case that running data
@@ -79,6 +83,10 @@ this package with application add-ons, you should add `app` in installation;
79
83
  pip install ddeutil-workflow[app]
80
84
  ```
81
85
 
86
+ ```shell
87
+ pip install ddeutil-workflow[api]
88
+ ```
89
+
82
90
  ## Getting Started
83
91
 
84
92
  The first step, you should start create the connections and datasets for In and
@@ -240,6 +248,18 @@ pipe_el_pg_to_lake:
240
248
  endpoint: "/${{ params.name }}"
241
249
  ```
242
250
 
251
+ Implement hook:
252
+
253
+ ```python
254
+ from ddeutil.workflow.utils import tag
255
+
256
+ @tag('polars', alias='postgres-to-delta')
257
+ def postgres_to_delta(source, sink):
258
+ return {
259
+ "source": source, "sink": sink
260
+ }
261
+ ```
262
+
243
263
  ### Hook (Transform)
244
264
 
245
265
  ```yaml
@@ -265,12 +285,30 @@ pipeline_hook_mssql_proc:
265
285
  target: ${{ params.target_name }}
266
286
  ```
267
287
 
288
+ Implement hook:
289
+
290
+ ```python
291
+ from ddeutil.workflow.utils import tag
292
+
293
+ @tag('odbc', alias='mssql-proc')
294
+ def odbc_mssql_procedure(_exec: str, params: dict):
295
+ return {
296
+ "exec": _exec, "params": params
297
+ }
298
+ ```
299
+
268
300
  ## Configuration
269
301
 
270
302
  ```bash
271
303
  export WORKFLOW_ROOT_PATH=.
272
304
  export WORKFLOW_CORE_REGISTRY=ddeutil.workflow,tests.utils
305
+ export WORKFLOW_CORE_REGISTRY_FILTER=ddeutil.workflow.utils
273
306
  export WORKFLOW_CORE_PATH_CONF=conf
307
+ export WORKFLOW_CORE_TIMEZONE=Asia/Bangkok
308
+ export WORKFLOW_CORE_DEFAULT_STAGE_ID=true
309
+
310
+ export WORKFLOW_CORE_MAX_PIPELINE_POKING=4
311
+ export WORKFLOW_CORE_MAX_JOB_PARALLEL=2
274
312
  ```
275
313
 
276
314
  Application config:
@@ -283,12 +321,21 @@ export WORKFLOW_APP_INTERVAL=10
283
321
  ## Deployment
284
322
 
285
323
  This package able to run as a application service for receive manual trigger
286
- from the master node via RestAPI.
324
+ from the master node via RestAPI or use to be Scheduler background service
325
+ like crontab job but via Python API.
326
+
327
+ ### Schedule Service
287
328
 
288
- > [!WARNING]
289
- > This feature do not start yet because I still research and find the best tool
290
- > to use it provision an app service, like `starlette`, `fastapi`, `apscheduler`.
329
+ ```shell
330
+ (venv) $ python src.ddeutil.workflow.app
331
+ ```
332
+
333
+ ### API Server
291
334
 
292
335
  ```shell
293
- (venv) $ workflow start -p 7070
336
+ (venv) $ uvicorn src.ddeutil.workflow.api:app --host 0.0.0.0 --port 80 --reload
294
337
  ```
338
+
339
+ > [!NOTE]
340
+ > If this package already deploy, it able to use
341
+ > `uvicorn ddeutil.workflow.api:app --host 0.0.0.0 --port 80`
@@ -0,0 +1,20 @@
1
+ ddeutil/workflow/__about__.py,sha256=b23XabBwtuoPOLmS_Hj_gSA4LZ0fRfAkACM6c3szVoc,27
2
+ ddeutil/workflow/__init__.py,sha256=4PEL3RdHmUowK0Dz-tK7fO0wvFX4u9CLd0Up7b3lrAQ,760
3
+ ddeutil/workflow/__types.py,sha256=SYMoxbENQX8uPsiCZkjtpHAqqHOh8rUrarAFicAJd0E,1773
4
+ ddeutil/workflow/api.py,sha256=d2Mmv9jTtN3FITIy-2mivyAKdBOGZxtkNWRMPbCLlFI,3341
5
+ ddeutil/workflow/app.py,sha256=GbdwvUkE8lO2Ze4pZ0-J-7p9mcZAaORfjkHwW_oZIP0,1076
6
+ ddeutil/workflow/exceptions.py,sha256=BH7COn_3uz3z7oJBZOQGiuo8osBFgeXL8HYymnjCOPQ,671
7
+ ddeutil/workflow/loader.py,sha256=_ZD-XP5P7VbUeqItrUVPaKIZu6dMUZ2aywbCbReW1hQ,2778
8
+ ddeutil/workflow/log.py,sha256=_GJEdJr7bqpcQDxZjrqHd-hkiW3NKFaVoR6voE6Ty0o,952
9
+ ddeutil/workflow/on.py,sha256=YoEqDbzJUwqOA3JRltbvlYr0rNTtxdmb7cWMxl8U19k,6717
10
+ ddeutil/workflow/pipeline.py,sha256=dKF09TFS_v5TCD-5o8tp1UhB5sGuWIQu4zl_UFtlIC0,25951
11
+ ddeutil/workflow/repeat.py,sha256=sNoRfbOR4cYm_edrSvlVy9N8Dk_osLIq9FC5GMZz32M,4621
12
+ ddeutil/workflow/route.py,sha256=Ck_O1xJwI-vKkMJr37El0-1PGKlwKF8__DDNWVQrf0A,2079
13
+ ddeutil/workflow/scheduler.py,sha256=FqmkvWCqwJ4eRf8aDn5Ce4FcNWqmcvu2aTTfL34lfgs,22184
14
+ ddeutil/workflow/stage.py,sha256=z05bKk2QFQDXjidSnQYCVOdceSpSO13sHXE0B1UH6XA,14978
15
+ ddeutil/workflow/utils.py,sha256=pDM2jaYVP-USH0pLd_XmHOguxVPGVzZ76hOh1AZdINU,18495
16
+ ddeutil_workflow-0.0.7.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
17
+ ddeutil_workflow-0.0.7.dist-info/METADATA,sha256=ba2nH57cpHB2P4ldQCRT8ZWDj3r1OPx9a1dgcB0a2Ws,9702
18
+ ddeutil_workflow-0.0.7.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
19
+ ddeutil_workflow-0.0.7.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
20
+ ddeutil_workflow-0.0.7.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (72.1.0)
2
+ Generator: setuptools (72.2.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5