zrb 0.0.119__py3-none-any.whl → 0.0.121__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zrb/__init__.py +3 -0
- zrb/action/runner.py +44 -33
- zrb/task/base_task/base_task.py +11 -0
- zrb/task/parallel.py +36 -0
- {zrb-0.0.119.dist-info → zrb-0.0.121.dist-info}/METADATA +33 -9
- {zrb-0.0.119.dist-info → zrb-0.0.121.dist-info}/RECORD +9 -8
- {zrb-0.0.119.dist-info → zrb-0.0.121.dist-info}/LICENSE +0 -0
- {zrb-0.0.119.dist-info → zrb-0.0.121.dist-info}/WHEEL +0 -0
- {zrb-0.0.119.dist-info → zrb-0.0.121.dist-info}/entry_points.txt +0 -0
zrb/__init__.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
from zrb.runner import runner
|
2
2
|
from zrb.task.decorator import python_task
|
3
3
|
from zrb.task.any_task import AnyTask
|
4
|
+
from zrb.task.parallel import AnyParallel, Parallel
|
4
5
|
from zrb.task.task import Task
|
5
6
|
from zrb.task.cmd_task import CmdTask
|
6
7
|
from zrb.task.docker_compose_task import DockerComposeTask, ServiceConfig
|
@@ -33,6 +34,8 @@ from zrb.helper.default_env import inject_default_env
|
|
33
34
|
|
34
35
|
assert runner
|
35
36
|
assert AnyTask
|
37
|
+
assert AnyParallel
|
38
|
+
assert Parallel
|
36
39
|
assert python_task
|
37
40
|
assert Task
|
38
41
|
assert CmdTask
|
zrb/action/runner.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
from zrb.helper.typing import Any, Callable,
|
1
|
+
from zrb.helper.typing import Any, Callable, List, Mapping, Union
|
2
2
|
from zrb.helper.typecheck import typechecked
|
3
3
|
from zrb.helper.log import logger
|
4
4
|
from zrb.helper.accessories.color import colored
|
@@ -19,71 +19,82 @@ class Runner():
|
|
19
19
|
|
20
20
|
def __init__(self, env_prefix: str = ''):
|
21
21
|
logger.info(colored('Create runner', attrs=['dark']))
|
22
|
-
self.
|
23
|
-
self.
|
24
|
-
self.
|
25
|
-
self.
|
26
|
-
self.
|
22
|
+
self.__env_prefix = env_prefix
|
23
|
+
self.__tasks: List[AnyTask] = []
|
24
|
+
self.__registered_groups: Mapping[str, click.Group] = {}
|
25
|
+
self.__top_levels: List[CliSubcommand] = []
|
26
|
+
self.__subcommands: Mapping[str, List[click.Group]] = {}
|
27
|
+
self.__registered_task_cmd_name: List[str] = []
|
27
28
|
logger.info(colored('Runner created', attrs=['dark']))
|
28
29
|
|
29
|
-
def register(self,
|
30
|
-
task
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
30
|
+
def register(self, *tasks: AnyTask):
|
31
|
+
for task in tasks:
|
32
|
+
task._set_has_cli_interface()
|
33
|
+
cmd_name = task._get_full_cmd_name()
|
34
|
+
if cmd_name in self.__registered_task_cmd_name:
|
35
|
+
raise RuntimeError(
|
36
|
+
f'Task "{cmd_name}" has already been registered'
|
37
|
+
)
|
38
|
+
logger.debug(
|
39
|
+
colored(f'Register task: "{cmd_name}"', attrs=['dark'])
|
40
|
+
)
|
41
|
+
self.__tasks.append(task)
|
42
|
+
self.__registered_task_cmd_name.append(cmd_name)
|
43
|
+
logger.debug(
|
44
|
+
colored(f'Task registered: "{cmd_name}"', attrs=['dark'])
|
45
|
+
)
|
35
46
|
|
36
47
|
def serve(self, cli: click.Group) -> click.Group:
|
37
|
-
for task in self.
|
38
|
-
subcommand = self.
|
39
|
-
if subcommand not in self.
|
40
|
-
self.
|
48
|
+
for task in self.__tasks:
|
49
|
+
subcommand = self.__create_cli_subcommand(task)
|
50
|
+
if subcommand not in self.__top_levels:
|
51
|
+
self.__top_levels.append(subcommand)
|
41
52
|
cli.add_command(subcommand)
|
42
53
|
return cli
|
43
54
|
|
44
|
-
def
|
55
|
+
def __create_cli_subcommand(
|
45
56
|
self, task: AnyTask
|
46
57
|
) -> Union[click.Group, click.Command]:
|
47
|
-
subcommand: CliSubcommand = self.
|
58
|
+
subcommand: CliSubcommand = self.__create_cli_command(task)
|
48
59
|
task_group = task._group
|
49
60
|
while task_group is not None:
|
50
|
-
group = self.
|
61
|
+
group = self.__register_sub_command(task_group, subcommand)
|
51
62
|
if task_group._parent is None:
|
52
63
|
return group
|
53
64
|
subcommand = group
|
54
65
|
task_group = task_group._parent
|
55
66
|
return subcommand
|
56
67
|
|
57
|
-
def
|
68
|
+
def __register_sub_command(
|
58
69
|
self, task_group: TaskGroup, subcommand: CliSubcommand
|
59
70
|
) -> click.Group:
|
60
71
|
task_group_id = task_group.get_id()
|
61
|
-
group = self.
|
62
|
-
if task_group_id not in self.
|
63
|
-
self.
|
64
|
-
if subcommand not in self.
|
72
|
+
group = self.__get_cli_group(task_group)
|
73
|
+
if task_group_id not in self.__subcommands:
|
74
|
+
self.__subcommands[task_group_id] = []
|
75
|
+
if subcommand not in self.__subcommands[task_group_id]:
|
65
76
|
group.add_command(subcommand)
|
66
|
-
self.
|
77
|
+
self.__subcommands[task_group_id].append(subcommand)
|
67
78
|
return group
|
68
79
|
|
69
|
-
def
|
80
|
+
def __get_cli_group(self, task_group: TaskGroup) -> click.Group:
|
70
81
|
task_group_id = task_group.get_id()
|
71
|
-
if task_group_id in self.
|
72
|
-
return self.
|
82
|
+
if task_group_id in self.__registered_groups:
|
83
|
+
return self.__registered_groups[task_group_id]
|
73
84
|
group_cmd_name = task_group.get_cmd_name()
|
74
85
|
group_description = task_group._description
|
75
86
|
group = click.Group(name=group_cmd_name, help=group_description)
|
76
|
-
self.
|
87
|
+
self.__registered_groups[task_group_id] = group
|
77
88
|
return group
|
78
89
|
|
79
|
-
def
|
90
|
+
def __create_cli_command(self, task: AnyTask) -> click.Command:
|
80
91
|
task_inputs = task._get_combined_inputs()
|
81
92
|
task_cmd_name = task.get_cmd_name()
|
82
93
|
task_description = task.get_description()
|
83
94
|
task_function = task.to_function(
|
84
|
-
env_prefix=self.
|
95
|
+
env_prefix=self.__env_prefix, raise_error=True
|
85
96
|
)
|
86
|
-
callback = self.
|
97
|
+
callback = self.__wrap_task_function(task_function)
|
87
98
|
command = click.Command(
|
88
99
|
callback=callback, name=task_cmd_name, help=task_description
|
89
100
|
)
|
@@ -101,7 +112,7 @@ class Runner():
|
|
101
112
|
command.params.append(click.Option(param_decl, **options))
|
102
113
|
return command
|
103
114
|
|
104
|
-
def
|
115
|
+
def __wrap_task_function(
|
105
116
|
self, function: Callable[..., Any]
|
106
117
|
) -> Callable[..., Any]:
|
107
118
|
def wrapped_function(*args: Any, **kwargs: Any) -> Any:
|
zrb/task/base_task/base_task.py
CHANGED
@@ -12,6 +12,7 @@ from zrb.task.base_task.component.trackers import (
|
|
12
12
|
)
|
13
13
|
from zrb.task.base_task.component.renderer import Renderer
|
14
14
|
from zrb.task.base_task.component.base_task_model import BaseTaskModel
|
15
|
+
from zrb.task.parallel import AnyParallel
|
15
16
|
from zrb.advertisement import advertisements
|
16
17
|
from zrb.task_group.group import Group
|
17
18
|
from zrb.task_env.env import Env
|
@@ -102,6 +103,16 @@ class BaseTask(
|
|
102
103
|
self.__is_execution_triggered: bool = False
|
103
104
|
self.__is_execution_started: bool = False
|
104
105
|
|
106
|
+
def __rshift__(self, operand: Union[AnyParallel, AnyTask]):
|
107
|
+
if isinstance(operand, AnyTask):
|
108
|
+
operand.add_upstream(self)
|
109
|
+
return operand
|
110
|
+
if isinstance(operand, AnyParallel):
|
111
|
+
other_tasks: List[AnyTask] = operand.get_tasks()
|
112
|
+
for other_task in other_tasks:
|
113
|
+
other_task.add_upstream(self)
|
114
|
+
return operand
|
115
|
+
|
105
116
|
def copy(self) -> AnyTask:
|
106
117
|
return copy.deepcopy(self)
|
107
118
|
|
zrb/task/parallel.py
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
from zrb.helper.typing import TypeVar, List, Union
|
2
|
+
from abc import ABC, abstractmethod
|
3
|
+
from zrb.helper.typecheck import typechecked
|
4
|
+
from zrb.task.any_task import AnyTask
|
5
|
+
|
6
|
+
|
7
|
+
TParallel = TypeVar('TParallel', bound='Parallel')
|
8
|
+
|
9
|
+
|
10
|
+
class AnyParallel(ABC):
|
11
|
+
@abstractmethod
|
12
|
+
def get_tasks(self) -> List[AnyTask]:
|
13
|
+
pass
|
14
|
+
|
15
|
+
|
16
|
+
@typechecked
|
17
|
+
class Parallel(AnyParallel):
|
18
|
+
def __init__(self, *tasks: AnyTask):
|
19
|
+
self.__tasks = list(tasks)
|
20
|
+
|
21
|
+
def get_tasks(self) -> List[AnyTask]:
|
22
|
+
return self.__tasks
|
23
|
+
|
24
|
+
def __rshift__(
|
25
|
+
self, operand: Union[AnyTask, AnyParallel]
|
26
|
+
) -> Union[AnyTask, AnyParallel]:
|
27
|
+
if isinstance(operand, AnyTask):
|
28
|
+
for task in self.__tasks:
|
29
|
+
operand.add_upstream(task)
|
30
|
+
return operand
|
31
|
+
if isinstance(operand, AnyParallel):
|
32
|
+
other_tasks: List[AnyTask] = operand.get_tasks()
|
33
|
+
for task in self.__tasks:
|
34
|
+
for other_task in other_tasks:
|
35
|
+
other_task.add_upstream(task)
|
36
|
+
return operand
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: zrb
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.121
|
4
4
|
Summary: A Framework to Enhance Your Workflow
|
5
5
|
Author-email: Go Frendi Gunawan <gofrendiasgard@gmail.com>
|
6
6
|
Requires-Python: >=3.10.0
|
@@ -114,12 +114,11 @@ download_dataset = CmdTask(
|
|
114
114
|
)
|
115
115
|
|
116
116
|
# 📊 Define a task named `show-stat` to show the statistics properties of the dataset.
|
117
|
-
#
|
118
|
-
#
|
119
|
-
#
|
117
|
+
# @python_task` decorator turns a function into a Zrb Task (i.e., `show_stat` is now a Zrb Task).
|
118
|
+
# If this task failed, we don't want to retry
|
119
|
+
# We also want to register the task so that it is accessible from the CLI
|
120
120
|
@python_task(
|
121
121
|
name='show-stat',
|
122
|
-
upstreams=[download_dataset, install_pandas],
|
123
122
|
retry=0
|
124
123
|
)
|
125
124
|
def show_stat(*args, **kwargs):
|
@@ -127,8 +126,12 @@ def show_stat(*args, **kwargs):
|
|
127
126
|
df = pd.read_csv('dataset.csv')
|
128
127
|
return df.describe()
|
129
128
|
|
130
|
-
#
|
131
|
-
|
129
|
+
# Define dependencies: `show_stat` depends on both, `download_dataset` and `install_pandas`
|
130
|
+
download_dataset >> show_stat
|
131
|
+
install_pandas >> show_stat
|
132
|
+
|
133
|
+
# Register the tasks so that they are accessbie from the CLI
|
134
|
+
runner.register(install_pandas, download_dataset, show_stat)
|
132
135
|
```
|
133
136
|
|
134
137
|
> __📝 NOTE:__ It is possible (although less readable) to define `show_stat` as `CmdTask`:
|
@@ -138,7 +141,6 @@ runner.register(show_stat)
|
|
138
141
|
> ```python
|
139
142
|
> show_stat = CmdTask(
|
140
143
|
> name='show-stat',
|
141
|
-
> upstreams=[download_dataset, install_pandas],
|
142
144
|
> cmd='python -c "import pandas as pd; df=pd.read_csv(\'dataset.csv\'); print(df.describe())"',
|
143
145
|
> retry=0
|
144
146
|
> )
|
@@ -152,8 +154,23 @@ Once you write the definitions, Zrb will automatically load your `zrb_init.py` s
|
|
152
154
|
zrb show-stat
|
153
155
|
```
|
154
156
|
|
157
|
+
The command will give you the statistics property of the dataset:
|
158
|
+
|
159
|
+
```
|
160
|
+
sepal_length sepal_width petal_length petal_width
|
161
|
+
count 150.000000 150.000000 150.000000 150.000000
|
162
|
+
mean 5.843333 3.054000 3.758667 1.198667
|
163
|
+
std 0.828066 0.433594 1.764420 0.763161
|
164
|
+
min 4.300000 2.000000 1.000000 0.100000
|
165
|
+
25% 5.100000 2.800000 1.600000 0.300000
|
166
|
+
50% 5.800000 3.000000 4.350000 1.300000
|
167
|
+
75% 6.400000 3.300000 5.100000 1.800000
|
168
|
+
max 7.900000 4.400000 6.900000 2.500000
|
169
|
+
|
170
|
+
```
|
171
|
+
|
155
172
|
<details>
|
156
|
-
<summary>
|
173
|
+
<summary>See the full output</summary>
|
157
174
|
|
158
175
|
```
|
159
176
|
Url [https://raw.githubusercontent.com/state-alchemists/datasets/main/iris.csv]:
|
@@ -196,6 +213,13 @@ To run again: zrb project show-stats --url "https://raw.githubusercontent.com/st
|
|
196
213
|
```
|
197
214
|
</details>
|
198
215
|
|
216
|
+
Since you have registered `install_pandas` and `download_dataset` (i.e., `runner.register(install_pandas, download_dataset)`), then you can also execute those tasks as well:
|
217
|
+
|
218
|
+
```bash
|
219
|
+
zrb install-pandas
|
220
|
+
zrb download-dataset
|
221
|
+
```
|
222
|
+
|
199
223
|
> __📝 NOTE:__ When executing a task, you can also provide the parameter directly, for example you want to provide the `url`
|
200
224
|
>
|
201
225
|
> ```bash
|
@@ -1,9 +1,9 @@
|
|
1
|
-
zrb/__init__.py,sha256=
|
1
|
+
zrb/__init__.py,sha256=1SSyh2yX2Zk3BnraVUTNiXDVsQOstiSC8cPR2ZukEtQ,2127
|
2
2
|
zrb/__main__.py,sha256=CdfuYSxqlJhnsJPOBOL2_uzEaTZHC3MtpyTuz8QUfUI,314
|
3
3
|
zrb/advertisement.py,sha256=e-1tFPlmEuz8IqaIJ_9-2p9x5cuGsxssJGu5F0wHthI,505
|
4
4
|
zrb/runner.py,sha256=MPCNPMCyiYNZeubSxd1eM2Zr6PCIKF-9pkG385AXElw,118
|
5
5
|
zrb/action/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
|
-
zrb/action/runner.py,sha256=
|
6
|
+
zrb/action/runner.py,sha256=YKNrqJatgaYkif3zaZPBjb8en1kq66pFbkBG76IyVss,4904
|
7
7
|
zrb/builtin/__init__.py,sha256=NwkM8HYaWHxr8sGE18gNF2qgf00FlhP2TWq1r1uDQAY,727
|
8
8
|
zrb/builtin/base64.py,sha256=nf_qWD2M0aX06zGef61U2nPdGHX_pfYN9E4gZmpBr5g,1346
|
9
9
|
zrb/builtin/env.py,sha256=hEjQios0i-3DantczxQnZnolzwZAGIQOE0Ygka330EU,1146
|
@@ -1156,6 +1156,7 @@ zrb/task/docker_compose_task.py,sha256=clbq1OOKC_zAz7IL3iBpNnCGR96JCF056r4o0v0E7
|
|
1156
1156
|
zrb/task/flow_task.py,sha256=l0xCb6mlaaNK5NVZxhKWE1qKXUI2akzC6CZfciGh7P8,3939
|
1157
1157
|
zrb/task/http_checker.py,sha256=PoneoC9S7bTw41IP7hB9ewaWO-lF3ZWT75MXEPyThYA,5244
|
1158
1158
|
zrb/task/notifier.py,sha256=UVhx1fLORhqyXprzqQqdoVYIFGrirwnTk-Ak18ZyucQ,5332
|
1159
|
+
zrb/task/parallel.py,sha256=OnCdh4aQwL-N2ICNf3gshhqx5-Hq_QPZD3ycMHqrFRg,1042
|
1159
1160
|
zrb/task/path_checker.py,sha256=BSBf8ZXEcTwVI3UskSgdXXWrCLWggnwk6sHHbEI0z5g,3382
|
1160
1161
|
zrb/task/path_watcher.py,sha256=9r36cMc7EaLehQVw0149TN-qOZitucR9mRW3S-w7SQw,4695
|
1161
1162
|
zrb/task/port_checker.py,sha256=IvOUhefkpYcKZJ9zGr0RNUZju5yLvXoNJlLlODpLZZ8,4111
|
@@ -1166,7 +1167,7 @@ zrb/task/rsync_task.py,sha256=3vxENH-unD1VLgZI09Ts_lAQRMi2i1UiJpsaI1dltzI,3966
|
|
1166
1167
|
zrb/task/task.py,sha256=l1jzJil7AqnsFn9e8WyO-rgCIG1JSvB6DaRBQZQo00I,246
|
1167
1168
|
zrb/task/time_watcher.py,sha256=cbmmKtbeMrkRwpRZi_-jyJ_hC4KnsOPHUIpzvns2hPQ,3728
|
1168
1169
|
zrb/task/base_task/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
1169
|
-
zrb/task/base_task/base_task.py,sha256=
|
1170
|
+
zrb/task/base_task/base_task.py,sha256=w593NE0-sfuV7WKjw4QxcdtT1lngnCIeXPg60ueJJcs,16540
|
1170
1171
|
zrb/task/base_task/component/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
1171
1172
|
zrb/task/base_task/component/base_task_model.py,sha256=mi7v0ScvVlqkXSM6N0GUEWomVd3wwzL492LU60u2jZQ,9845
|
1172
1173
|
zrb/task/base_task/component/common_task_model.py,sha256=9qwrnVARWafGm8PNpZtPmxlgVSD_3ZNt3hZxiOJO2m4,10478
|
@@ -1189,8 +1190,8 @@ zrb/task_input/int_input.py,sha256=VYduGcAE0m7eDItRQboTFDCW5whMur_uH1Y9qtBIwV4,1
|
|
1189
1190
|
zrb/task_input/password_input.py,sha256=i6U-smP6zFo8Ts9x14YXVgx7PKkRWr4U_EzPFp9nucw,1687
|
1190
1191
|
zrb/task_input/str_input.py,sha256=w9TTUMlMQtlOXh2ahHPbRnFyTiV94lXizucCc8SuWoo,1668
|
1191
1192
|
zrb/task_input/task_input.py,sha256=HotqM1iYSzwE4PIj8grnEUsvJqjx1dS6Ek7i6ZJLq2Y,83
|
1192
|
-
zrb-0.0.
|
1193
|
-
zrb-0.0.
|
1194
|
-
zrb-0.0.
|
1195
|
-
zrb-0.0.
|
1196
|
-
zrb-0.0.
|
1193
|
+
zrb-0.0.121.dist-info/entry_points.txt,sha256=xTgXc1kBKYhJHEujdaSPHUcJT3-hbyP1mLgwkv-5sSk,40
|
1194
|
+
zrb-0.0.121.dist-info/LICENSE,sha256=WfnGCl8G60EYOPAEkuc8C9m9pdXWDe08NsKj3TBbxsM,728
|
1195
|
+
zrb-0.0.121.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
|
1196
|
+
zrb-0.0.121.dist-info/METADATA,sha256=A07FBVLViYyRae2p0Efcs_hmBmnFojeeZeFk4hv-_8M,15079
|
1197
|
+
zrb-0.0.121.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|