runnable 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. runnable/__init__.py +1 -1
  2. runnable/catalog.py +2 -0
  3. runnable/cli.py +264 -307
  4. runnable/context.py +10 -3
  5. runnable/datastore.py +145 -22
  6. runnable/defaults.py +13 -54
  7. runnable/entrypoints.py +197 -185
  8. runnable/exceptions.py +22 -0
  9. runnable/executor.py +114 -86
  10. runnable/graph.py +0 -1
  11. runnable/nodes.py +36 -6
  12. runnable/sdk.py +132 -36
  13. runnable/tasks.py +6 -15
  14. runnable/utils.py +22 -30
  15. {runnable-0.14.0.dist-info → runnable-0.16.0.dist-info}/METADATA +6 -3
  16. runnable-0.16.0.dist-info/RECORD +23 -0
  17. {runnable-0.14.0.dist-info → runnable-0.16.0.dist-info}/entry_points.txt +12 -7
  18. runnable/integration.py +0 -197
  19. runnable-0.14.0.dist-info/RECORD +0 -24
  20. {runnable-0.14.0.dist-info → runnable-0.16.0.dist-info}/WHEEL +0 -0
  21. {runnable-0.14.0.dist-info → runnable-0.16.0.dist-info}/licenses/LICENSE +0 -0
runnable/__init__.py CHANGED
@@ -1,6 +1,5 @@
1
1
  # ruff: noqa
2
2
 
3
- # TODO: Might need to add Rich to pyinstaller part
4
3
  import logging
5
4
  import os
6
5
  from logging.config import dictConfig
@@ -20,6 +19,7 @@ task_console = Console(record=True)
20
19
  from runnable.sdk import ( # noqa
21
20
  Catalog,
22
21
  Fail,
22
+ Job,
23
23
  Map,
24
24
  NotebookTask,
25
25
  Parallel,
runnable/catalog.py CHANGED
@@ -10,6 +10,8 @@ from runnable.datastore import DataCatalog
10
10
 
11
11
  logger = logging.getLogger(defaults.LOGGER_NAME)
12
12
 
13
+ # TODO: Should ** be allowed as glob pattern as it can potentially copy everything to catalog
14
+
13
15
 
14
16
  def is_catalog_out_of_sync(
15
17
  catalog, synced_catalogs=Optional[List[DataCatalog]]
runnable/cli.py CHANGED
@@ -1,137 +1,154 @@
1
- # A dummy to trigger the PR
2
1
  import logging
2
+ from enum import Enum
3
+ from typing import Annotated
3
4
 
4
- import click
5
- from click_plugins import with_plugins
6
- from pkg_resources import iter_entry_points
5
+ import typer
7
6
 
8
7
  from runnable import defaults, entrypoints
9
8
 
10
9
  logger = logging.getLogger(defaults.LOGGER_NAME)
11
10
 
12
11
 
13
- @with_plugins(iter_entry_points("runnable.cli_plugins"))
14
- @click.group()
15
- @click.version_option()
16
- def cli():
17
- """
18
- Welcome to runnable. Please provide the command that you want to use.
19
- All commands have options that you can see by runnable <command> --help
20
- """
21
- pass
12
+ app = typer.Typer(
13
+ help=(
14
+ "Welcome to runnable. Please provide the command that you want to use."
15
+ "All commands have options that you can see by runnable <command> --help"
16
+ ),
17
+ )
22
18
 
23
19
 
24
- @cli.command("execute", short_help="Execute/translate a pipeline")
25
- @click.argument("filename")
26
- @click.option(
27
- "-c",
28
- "--config-file",
29
- default=None,
30
- help="config file, in yaml, to be used for the run",
31
- show_default=True,
32
- )
33
- @click.option(
34
- "-p",
35
- "--parameters-file",
36
- default=None,
37
- help="Parameters, in yaml, accessible by the application",
38
- show_default=True,
39
- )
40
- @click.option(
41
- "--log-level",
42
- default=defaults.LOG_LEVEL,
43
- help="The log level",
44
- show_default=True,
45
- type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
46
- )
47
- @click.option("--tag", default="", help="A tag attached to the run")
48
- @click.option(
49
- "--run-id", help="An optional run_id, one would be generated if not provided"
50
- )
20
+ class LogLevel(str, Enum):
21
+ INFO = "INFO"
22
+ DEBUG = "DEBUG"
23
+ WARNING = "WARNING"
24
+ ERROR = "ERROR"
25
+ FATAL = "FATAL"
26
+
27
+
28
+ class ExecutionMode(str, Enum):
29
+ YAML = "yaml"
30
+ PYTHON = "python"
31
+
32
+
33
+ class FanMode(str, Enum):
34
+ IN = "in"
35
+ OUT = "out"
36
+
37
+
38
+ @app.command()
51
39
  def execute(
52
- filename, config_file, parameters_file, log_level, tag, run_id
53
- ): # pragma: no cover
40
+ yaml_file: Annotated[str, typer.Argument(help="The pipeline definition file")],
41
+ config_file: Annotated[
42
+ str,
43
+ typer.Option(
44
+ "--config", "-c", help="The configuration file specifying the services"
45
+ ),
46
+ ] = "",
47
+ parameters_file: Annotated[
48
+ str,
49
+ typer.Option(
50
+ "--parameters",
51
+ "-p",
52
+ help="Parameters, in yaml, accessible by the application",
53
+ ),
54
+ ] = "",
55
+ log_level: Annotated[
56
+ LogLevel,
57
+ typer.Option(
58
+ "--log-level",
59
+ help="The log level",
60
+ show_default=True,
61
+ case_sensitive=False,
62
+ ),
63
+ ] = LogLevel.WARNING,
64
+ tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
65
+ run_id: Annotated[
66
+ str,
67
+ typer.Option(
68
+ help="An optional run_id, one would be generated if its not provided"
69
+ ),
70
+ ] = "",
71
+ ):
54
72
  """
55
- Execute a pipeline
73
+ Execute a pipeline defined by yaml file.
56
74
 
57
- Usage: runnable execute [OPTIONS]
75
+ The executor is defined by executor block of the configuration file.
58
76
 
59
- Options:
60
- -f, --file TEXT The pipeline definition file [default: pipeline.yaml]
61
- -c, --config-file TEXT config file, in yaml, to be used for the run [default: None]
62
- -p, --parameters-file TEXT Parameters, in yaml, accessible by the application [default: None]
63
- --log-level One of [INFO|DEBUG|WARNING|ERROR|FATAL]
64
- The log level
65
- [default: INFO]
66
- --tag TEXT A tag attached to the run
67
- [default: ]
68
- --run-id TEXT An optional run_id, one would be generated if not
69
- provided
77
+ The behavior of this command depends on the executor type:
78
+
79
+ -- For local executors (local, local-container), the pipeline is executed in the current environment.
80
+
81
+ -- For remote executors (argo, airflow), the pipeline translated to the specification.
70
82
  """
71
- logger.setLevel(log_level)
83
+ logger.setLevel(log_level.value)
72
84
 
73
- entrypoints.execute(
85
+ entrypoints.execute_pipeline_yaml_spec(
74
86
  configuration_file=config_file,
75
- pipeline_file=filename,
87
+ pipeline_file=yaml_file,
76
88
  tag=tag,
77
89
  run_id=run_id,
78
90
  parameters_file=parameters_file,
79
91
  )
80
92
 
81
93
 
82
- @cli.command(
83
- "execute_single_node",
84
- short_help="Internal entry point to execute a single node",
85
- hidden=True,
86
- )
87
- @click.argument("run_id")
88
- @click.argument("step_name")
89
- @click.option(
90
- "--map-variable",
91
- default="",
92
- help="The map variable dictionary in str",
93
- show_default=True,
94
- )
95
- @click.option(
96
- "-f", "--file", default="", help="The pipeline definition file", show_default=True
97
- )
98
- @click.option(
99
- "-c",
100
- "--config-file",
101
- default=None,
102
- help="config file, in yaml, to be used for the run",
103
- show_default=True,
104
- )
105
- @click.option(
106
- "-p",
107
- "--parameters-file",
108
- default=None,
109
- help="Parameters, in yaml, accessible by the application",
110
- show_default=True,
111
- )
112
- @click.option(
113
- "--log-level",
114
- default=defaults.LOG_LEVEL,
115
- help="The log level",
116
- show_default=True,
117
- type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
118
- )
119
- @click.option("--tag", default="", help="A tag attached to the run")
94
+ @app.command(hidden=True)
120
95
  def execute_single_node(
121
- run_id, step_name, map_variable, file, config_file, parameters_file, log_level, tag
96
+ run_id: Annotated[
97
+ str,
98
+ typer.Argument(
99
+ help="An optional run_id, one would be generated if its not provided"
100
+ ),
101
+ ],
102
+ yaml_or_python_file: Annotated[
103
+ str, typer.Argument(help="The pipeline definition file")
104
+ ],
105
+ step_name: Annotated[str, typer.Argument(help="The step name to execute")],
106
+ config_file: Annotated[
107
+ str,
108
+ typer.Option(
109
+ "--config", "-c", help="The configuration file specifying the services"
110
+ ),
111
+ ] = "",
112
+ parameters_file: Annotated[
113
+ str,
114
+ typer.Option(
115
+ "--parameters-file",
116
+ "-p",
117
+ help="Parameters, in yaml, accessible by the application",
118
+ ),
119
+ ] = "",
120
+ log_level: Annotated[
121
+ LogLevel,
122
+ typer.Option(
123
+ "--log-level",
124
+ help="The log level",
125
+ show_default=True,
126
+ case_sensitive=False,
127
+ ),
128
+ ] = LogLevel.INFO,
129
+ tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
130
+ mode: Annotated[
131
+ ExecutionMode,
132
+ typer.Option(
133
+ "--mode",
134
+ "-m",
135
+ help="spec in yaml or python sdk",
136
+ ),
137
+ ] = ExecutionMode.YAML,
138
+ map_variable: Annotated[
139
+ str,
140
+ typer.Option(
141
+ "--map-variable",
142
+ help="The map variable dictionary in str",
143
+ show_default=True,
144
+ ),
145
+ ] = "",
122
146
  ):
123
- """
124
- Internal entrypoint for runnable to execute a single node.
125
-
126
- Other than local executor, every other executor uses this entry point to execute a step in the context of runnable.
127
- Only chained executions should use this method. Unchained executions should use execute_
128
- """
129
- logger.setLevel(log_level)
130
-
131
- # Execute the node as part of the graph execution.
147
+ logger.setLevel(log_level.value)
132
148
  entrypoints.execute_single_node(
133
149
  configuration_file=config_file,
134
- pipeline_file=file,
150
+ pipeline_file=yaml_or_python_file,
151
+ mode=mode,
135
152
  step_name=step_name,
136
153
  map_variable=map_variable,
137
154
  run_id=run_id,
@@ -140,228 +157,168 @@ def execute_single_node(
140
157
  )
141
158
 
142
159
 
143
- @cli.command("execute_notebook", short_help="Entry point to execute a notebook")
144
- @click.argument("filename")
145
- @click.option("--entrypoint", default=defaults.ENTRYPOINT.USER.value, hidden=True)
146
- @click.option(
147
- "-c",
148
- "--config-file",
149
- default=None,
150
- help="config file, in yaml, to be used for the run",
151
- show_default=True,
152
- )
153
- @click.option(
154
- "-p",
155
- "--parameters-file",
156
- default=None,
157
- help="Parameters, in yaml, accessible by the application",
158
- show_default=True,
159
- )
160
- @click.option(
161
- "--log-level",
162
- default=defaults.LOG_LEVEL,
163
- help="The log level",
164
- show_default=True,
165
- type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
166
- )
167
- @click.option("--data-folder", "-d", default="data/", help="The catalog data folder")
168
- @click.option(
169
- "--put-in-catalog",
170
- "-put",
171
- default=None,
172
- multiple=True,
173
- help="The data to put from the catalog",
174
- )
175
- @click.option(
176
- "--notebook-output-path", default="", help="The output path for the notebook"
177
- )
178
- @click.option("--tag", help="A tag attached to the run")
179
- @click.option(
180
- "--run-id", help="An optional run_id, one would be generated if not provided"
181
- )
182
- def execute_notebook(
183
- filename,
184
- entrypoint,
185
- config_file,
186
- parameters_file,
187
- log_level,
188
- data_folder,
189
- put_in_catalog,
190
- notebook_output_path,
191
- tag,
192
- run_id,
160
+ @app.command(hidden=True)
161
+ def fan(
162
+ run_id: Annotated[str, typer.Argument(help="The run id")],
163
+ step_name: Annotated[str, typer.Argument(help="The step name")],
164
+ python_or_yaml_file: Annotated[
165
+ str, typer.Argument(help="The pipeline definition file")
166
+ ],
167
+ mode: Annotated[FanMode, typer.Option(help="fan in or fan out")],
168
+ map_variable: Annotated[
169
+ str,
170
+ typer.Option(
171
+ "--map-variable",
172
+ help="The map variable dictionary in str",
173
+ show_default=True,
174
+ ),
175
+ ],
176
+ config_file: Annotated[
177
+ str,
178
+ typer.Option(
179
+ "--config-file", "-c", help="The configuration file specifying the services"
180
+ ),
181
+ ] = "",
182
+ parameters_file: Annotated[
183
+ str,
184
+ typer.Option(
185
+ "--parameters-file",
186
+ "-p",
187
+ help="Parameters, in yaml, accessible by the application",
188
+ ),
189
+ ] = "",
190
+ log_level: Annotated[
191
+ LogLevel,
192
+ typer.Option(
193
+ "--log-level",
194
+ help="The log level",
195
+ show_default=True,
196
+ case_sensitive=False,
197
+ ),
198
+ ] = LogLevel.INFO,
199
+ tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
193
200
  ):
194
- """
195
- External entry point to execute a Jupyter notebook in isolation.
196
-
197
- The notebook would be executed in the environment defined by the config file or default if none.
198
- The execution plan is unchained.
199
- """
200
- logger.setLevel(log_level)
201
- catalog_config = {
202
- "compute_data_folder": data_folder,
203
- "put": list(put_in_catalog) if put_in_catalog else None,
204
- }
205
- if not filename.endswith(".ipynb"):
206
- raise Exception("A notebook should always have ipynb as the extension")
201
+ logger.setLevel(log_level.value)
207
202
 
208
- entrypoints.execute_notebook(
209
- entrypoint=entrypoint,
210
- notebook_file=filename,
211
- catalog_config=catalog_config,
203
+ # Fan in or out
204
+ entrypoints.fan(
212
205
  configuration_file=config_file,
213
- parameters_file=parameters_file,
214
- notebook_output_path=notebook_output_path,
215
- tag=tag,
206
+ pipeline_file=python_or_yaml_file,
207
+ step_name=step_name,
208
+ mode=mode,
209
+ map_variable=map_variable,
216
210
  run_id=run_id,
211
+ tag=tag,
212
+ parameters_file=parameters_file,
217
213
  )
218
214
 
219
215
 
220
- @cli.command("execute_function", short_help="Entry point to execute a python function")
221
- @click.argument("command")
222
- @click.option("--entrypoint", default=defaults.ENTRYPOINT.USER.value, hidden=True)
223
- @click.option(
224
- "-c",
225
- "--config-file",
226
- default=None,
227
- help="config file, in yaml, to be used for the run",
228
- show_default=True,
229
- )
230
- @click.option(
231
- "-p",
232
- "--parameters-file",
233
- default=None,
234
- help="Parameters, in yaml, accessible by the application",
235
- show_default=True,
236
- )
237
- @click.option(
238
- "--log-level",
239
- default=defaults.LOG_LEVEL,
240
- help="The log level",
241
- show_default=True,
242
- type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
243
- )
244
- @click.option("--data-folder", "-d", default="data/", help="The catalog data folder")
245
- @click.option(
246
- "--put-in-catalog",
247
- "-put",
248
- default=None,
249
- multiple=True,
250
- help="The data to put from the catalog",
251
- )
252
- @click.option("--tag", help="A tag attached to the run")
253
- @click.option(
254
- "--run-id", help="An optional run_id, one would be generated if not provided"
255
- )
256
- def execute_function(
257
- command,
258
- entrypoint,
259
- config_file,
260
- parameters_file,
261
- log_level,
262
- data_folder,
263
- put_in_catalog,
264
- tag,
265
- run_id,
216
+ @app.command()
217
+ def submit_job(
218
+ job_definition_file: Annotated[
219
+ str,
220
+ typer.Argument(
221
+ help=("The yaml file containing the job definition"),
222
+ ),
223
+ ],
224
+ config_file: Annotated[
225
+ str,
226
+ typer.Option(
227
+ "--config", "-c", help="The configuration file specifying the services"
228
+ ),
229
+ ] = "",
230
+ parameters_file: Annotated[
231
+ str,
232
+ typer.Option(
233
+ "--parameters",
234
+ "-p",
235
+ help="Parameters, in yaml, accessible by the application",
236
+ ),
237
+ ] = "",
238
+ log_level: Annotated[
239
+ LogLevel,
240
+ typer.Option(
241
+ "--log-level",
242
+ help="The log level",
243
+ show_default=True,
244
+ case_sensitive=False,
245
+ ),
246
+ ] = LogLevel.WARNING,
247
+ tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
248
+ run_id: Annotated[
249
+ str,
250
+ typer.Option(
251
+ help="An optional run_id, one would be generated if its not provided"
252
+ ),
253
+ ] = "",
266
254
  ):
267
- """
268
- External entry point to execute a python function in isolation.
255
+ logger.setLevel(log_level.value)
269
256
 
270
- The function would be executed in the environment defined by the config file or default if none.
271
- The execution plan is unchained.
272
- """
273
- logger.setLevel(log_level)
274
- catalog_config = {
275
- "compute_data_folder": data_folder,
276
- "put": list(put_in_catalog) if put_in_catalog else None,
277
- }
278
- entrypoints.execute_function(
279
- entrypoint=entrypoint,
280
- command=command,
281
- catalog_config=catalog_config,
257
+ entrypoints.execute_job_yaml_spec(
282
258
  configuration_file=config_file,
283
- parameters_file=parameters_file,
259
+ job_definition_file=job_definition_file,
284
260
  tag=tag,
285
261
  run_id=run_id,
262
+ parameters_file=parameters_file,
286
263
  )
287
264
 
288
265
 
289
- @cli.command(
290
- "fan",
291
- short_help="Internal entry point to fan in or out a composite node",
292
- hidden=True,
293
- )
294
- @click.argument("run_id")
295
- @click.argument("step_name")
296
- @click.option(
297
- "-m",
298
- "--mode",
299
- help="fan in or fan out",
300
- required=True,
301
- type=click.Choice(["in", "out"]),
302
- )
303
- @click.option(
304
- "--map-variable",
305
- default="",
306
- help="The map variable dictionary in str",
307
- show_default=True,
308
- )
309
- @click.option(
310
- "-f", "--file", default="", help="The pipeline definition file", show_default=True
311
- )
312
- @click.option(
313
- "-c",
314
- "--config-file",
315
- default=None,
316
- help="config file, in yaml, to be used for the run",
317
- show_default=True,
318
- )
319
- @click.option(
320
- "-p",
321
- "--parameters-file",
322
- default=None,
323
- help="Parameters, in yaml, accessible by the application",
324
- show_default=True,
325
- )
326
- @click.option(
327
- "--log-level",
328
- default=defaults.LOG_LEVEL,
329
- help="The log level",
330
- show_default=True,
331
- type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
332
- )
333
- @click.option("--tag", default="", help="A tag attached to the run")
334
- def fan(
335
- run_id,
336
- step_name,
337
- mode,
338
- map_variable,
339
- file,
340
- config_file,
341
- parameters_file,
342
- log_level,
343
- tag,
266
+ @app.command(hidden=True)
267
+ def execute_job(
268
+ job_definition_file: Annotated[
269
+ str,
270
+ typer.Argument(
271
+ help=("The yaml file containing the job definition"),
272
+ ),
273
+ ],
274
+ run_id: Annotated[
275
+ str,
276
+ typer.Argument(help="An run_id, one would be generated if its not provided"),
277
+ ],
278
+ config_file: Annotated[
279
+ str,
280
+ typer.Option(
281
+ "--config", "-c", help="The configuration file specifying the services"
282
+ ),
283
+ ] = "",
284
+ parameters_file: Annotated[
285
+ str,
286
+ typer.Option(
287
+ "--parameters",
288
+ "-p",
289
+ help="Parameters, in yaml, accessible by the application",
290
+ ),
291
+ ] = "",
292
+ mode: Annotated[
293
+ ExecutionMode,
294
+ typer.Option(
295
+ "--mode",
296
+ "-m",
297
+ help="spec in yaml or python sdk",
298
+ ),
299
+ ] = ExecutionMode.YAML,
300
+ log_level: Annotated[
301
+ LogLevel,
302
+ typer.Option(
303
+ "--log-level",
304
+ help="The log level",
305
+ show_default=True,
306
+ case_sensitive=False,
307
+ ),
308
+ ] = LogLevel.WARNING,
309
+ tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
344
310
  ):
345
- """
346
- Internal entrypoint for runnable to fan in or out a composite node.
347
-
348
- Only 3rd party orchestrators should use this entry point.
349
- """
350
- logger.setLevel(log_level)
311
+ logger.setLevel(log_level.value)
351
312
 
352
- # Fan in or out
353
- entrypoints.fan(
313
+ entrypoints.execute_job_non_local(
354
314
  configuration_file=config_file,
355
- pipeline_file=file,
356
- step_name=step_name,
315
+ job_definition_file=job_definition_file,
357
316
  mode=mode,
358
- map_variable=map_variable,
359
- run_id=run_id,
360
317
  tag=tag,
318
+ run_id=run_id,
361
319
  parameters_file=parameters_file,
362
320
  )
363
321
 
364
322
 
365
- # Needed for the binary creation
366
323
  if __name__ == "__main__":
367
- cli()
324
+ app()