metaflow 2.12.28__py2.py3-none-any.whl → 2.12.29__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. metaflow/__init__.py +2 -3
  2. metaflow/cli.py +23 -13
  3. metaflow/client/core.py +2 -2
  4. metaflow/clone_util.py +1 -1
  5. metaflow/cmd/develop/stub_generator.py +623 -233
  6. metaflow/datastore/task_datastore.py +1 -1
  7. metaflow/extension_support/plugins.py +1 -0
  8. metaflow/flowspec.py +2 -2
  9. metaflow/includefile.py +8 -14
  10. metaflow/metaflow_config.py +4 -0
  11. metaflow/metaflow_current.py +1 -1
  12. metaflow/parameters.py +3 -0
  13. metaflow/plugins/__init__.py +12 -3
  14. metaflow/plugins/airflow/airflow_cli.py +5 -0
  15. metaflow/plugins/airflow/airflow_decorator.py +1 -1
  16. metaflow/plugins/argo/argo_workflows_decorator.py +1 -1
  17. metaflow/plugins/argo/argo_workflows_deployer.py +77 -263
  18. metaflow/plugins/argo/argo_workflows_deployer_objects.py +381 -0
  19. metaflow/plugins/aws/batch/batch_cli.py +1 -1
  20. metaflow/plugins/aws/batch/batch_decorator.py +2 -2
  21. metaflow/plugins/aws/step_functions/step_functions_cli.py +7 -0
  22. metaflow/plugins/aws/step_functions/step_functions_decorator.py +1 -1
  23. metaflow/plugins/aws/step_functions/step_functions_deployer.py +65 -224
  24. metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py +236 -0
  25. metaflow/plugins/azure/includefile_support.py +2 -0
  26. metaflow/plugins/cards/card_cli.py +3 -2
  27. metaflow/plugins/cards/card_modules/components.py +9 -9
  28. metaflow/plugins/cards/card_server.py +39 -14
  29. metaflow/plugins/datatools/local.py +2 -0
  30. metaflow/plugins/datatools/s3/s3.py +2 -0
  31. metaflow/plugins/env_escape/__init__.py +3 -3
  32. metaflow/plugins/gcp/includefile_support.py +3 -0
  33. metaflow/plugins/kubernetes/kubernetes_cli.py +1 -1
  34. metaflow/plugins/kubernetes/kubernetes_decorator.py +5 -4
  35. metaflow/plugins/{metadata → metadata_providers}/local.py +2 -2
  36. metaflow/plugins/{metadata → metadata_providers}/service.py +2 -2
  37. metaflow/plugins/parallel_decorator.py +1 -1
  38. metaflow/plugins/pypi/conda_decorator.py +1 -1
  39. metaflow/plugins/test_unbounded_foreach_decorator.py +1 -1
  40. metaflow/runner/click_api.py +4 -0
  41. metaflow/runner/deployer.py +139 -269
  42. metaflow/runner/deployer_impl.py +167 -0
  43. metaflow/runner/metaflow_runner.py +10 -9
  44. metaflow/runner/nbdeploy.py +12 -13
  45. metaflow/runner/nbrun.py +3 -3
  46. metaflow/runner/utils.py +55 -8
  47. metaflow/runtime.py +1 -1
  48. metaflow/task.py +1 -1
  49. metaflow/version.py +1 -1
  50. {metaflow-2.12.28.dist-info → metaflow-2.12.29.dist-info}/METADATA +2 -2
  51. {metaflow-2.12.28.dist-info → metaflow-2.12.29.dist-info}/RECORD +60 -57
  52. /metaflow/{metadata → metadata_provider}/__init__.py +0 -0
  53. /metaflow/{metadata → metadata_provider}/heartbeat.py +0 -0
  54. /metaflow/{metadata → metadata_provider}/metadata.py +0 -0
  55. /metaflow/{metadata → metadata_provider}/util.py +0 -0
  56. /metaflow/plugins/{metadata → metadata_providers}/__init__.py +0 -0
  57. {metaflow-2.12.28.dist-info → metaflow-2.12.29.dist-info}/LICENSE +0 -0
  58. {metaflow-2.12.28.dist-info → metaflow-2.12.29.dist-info}/WHEEL +0 -0
  59. {metaflow-2.12.28.dist-info → metaflow-2.12.29.dist-info}/entry_points.txt +0 -0
  60. {metaflow-2.12.28.dist-info → metaflow-2.12.29.dist-info}/top_level.txt +0 -0
metaflow/__init__.py CHANGED
@@ -101,9 +101,7 @@ from .metaflow_current import current
101
101
  # Flow spec
102
102
  from .flowspec import FlowSpec
103
103
 
104
- from .parameters import Parameter, JSONTypeClass
105
-
106
- JSONType = JSONTypeClass()
104
+ from .parameters import Parameter, JSONTypeClass, JSONType
107
105
 
108
106
  # data layer
109
107
  # For historical reasons, we make metaflow.plugins.datatools accessible as
@@ -149,6 +147,7 @@ if sys.version_info >= (3, 7):
149
147
  from .runner.metaflow_runner import Runner
150
148
  from .runner.nbrun import NBRunner
151
149
  from .runner.deployer import Deployer
150
+ from .runner.deployer import DeployedFlow
152
151
  from .runner.nbdeploy import NBDeployer
153
152
 
154
153
  __ext_tl_modules__ = []
metaflow/cli.py CHANGED
@@ -282,21 +282,31 @@ def dump(obj, input_path, private=None, max_value_size=None, include=None, file=
282
282
  else:
283
283
  ds_list = list(datastore_set) # get all tasks
284
284
 
285
+ tasks_processed = False
285
286
  for ds in ds_list:
286
- echo(
287
- "Dumping output of run_id=*{run_id}* "
288
- "step=*{step}* task_id=*{task_id}*".format(
289
- run_id=ds.run_id, step=ds.step_name, task_id=ds.task_id
290
- ),
291
- fg="magenta",
292
- )
293
-
294
- if file is None:
295
- echo_always(
296
- ds.format(**kwargs), highlight="green", highlight_bold=False, err=False
287
+ if ds is not None:
288
+ tasks_processed = True
289
+ echo(
290
+ "Dumping output of run_id=*{run_id}* "
291
+ "step=*{step}* task_id=*{task_id}*".format(
292
+ run_id=ds.run_id, step=ds.step_name, task_id=ds.task_id
293
+ ),
294
+ fg="magenta",
297
295
  )
298
- else:
299
- output[ds.pathspec] = ds.to_dict(**kwargs)
296
+
297
+ if file is None:
298
+ echo_always(
299
+ ds.format(**kwargs),
300
+ highlight="green",
301
+ highlight_bold=False,
302
+ err=False,
303
+ )
304
+ else:
305
+ output[ds.pathspec] = ds.to_dict(**kwargs)
306
+
307
+ if not tasks_processed:
308
+ echo(f"No task(s) found for pathspec {input_path}", fg="red")
309
+ return
300
310
 
301
311
  if file is not None:
302
312
  with open(file, "wb") as f:
metaflow/client/core.py CHANGED
@@ -39,7 +39,7 @@ from ..info_file import INFO_FILE
39
39
  from .filecache import FileCache
40
40
 
41
41
  if TYPE_CHECKING:
42
- from metaflow.metadata import MetadataProvider
42
+ from metaflow.metadata_provider import MetadataProvider
43
43
 
44
44
  try:
45
45
  # python2
@@ -143,7 +143,7 @@ def default_metadata() -> str:
143
143
  if default:
144
144
  current_metadata = default[0]
145
145
  else:
146
- from metaflow.plugins.metadata import LocalMetadataProvider
146
+ from metaflow.plugins.metadata_providers import LocalMetadataProvider
147
147
 
148
148
  current_metadata = LocalMetadataProvider
149
149
  return get_metadata()
metaflow/clone_util.py CHANGED
@@ -1,5 +1,5 @@
1
1
  import time
2
- from .metadata import MetaDatum
2
+ from .metadata_provider import MetaDatum
3
3
 
4
4
 
5
5
  def clone_task_helper(