ob-metaflow 2.12.27.1__py2.py3-none-any.whl → 2.12.30.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow might be problematic. Click here for more details.

Files changed (63) hide show
  1. metaflow/__init__.py +2 -3
  2. metaflow/cli.py +27 -0
  3. metaflow/client/core.py +2 -2
  4. metaflow/clone_util.py +1 -1
  5. metaflow/cmd/develop/stub_generator.py +623 -233
  6. metaflow/datastore/task_datastore.py +1 -1
  7. metaflow/extension_support/plugins.py +1 -0
  8. metaflow/flowspec.py +2 -2
  9. metaflow/includefile.py +8 -14
  10. metaflow/metaflow_config.py +4 -7
  11. metaflow/metaflow_current.py +1 -1
  12. metaflow/parameters.py +3 -0
  13. metaflow/plugins/__init__.py +12 -8
  14. metaflow/plugins/airflow/airflow_cli.py +5 -0
  15. metaflow/plugins/airflow/airflow_decorator.py +1 -1
  16. metaflow/plugins/argo/argo_workflows_decorator.py +1 -1
  17. metaflow/plugins/argo/argo_workflows_deployer.py +77 -363
  18. metaflow/plugins/argo/argo_workflows_deployer_objects.py +381 -0
  19. metaflow/plugins/aws/batch/batch_cli.py +1 -1
  20. metaflow/plugins/aws/batch/batch_decorator.py +2 -2
  21. metaflow/plugins/aws/step_functions/step_functions_cli.py +7 -0
  22. metaflow/plugins/aws/step_functions/step_functions_decorator.py +1 -1
  23. metaflow/plugins/aws/step_functions/step_functions_deployer.py +65 -224
  24. metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py +236 -0
  25. metaflow/plugins/azure/includefile_support.py +2 -0
  26. metaflow/plugins/cards/card_cli.py +3 -2
  27. metaflow/plugins/cards/card_modules/components.py +9 -9
  28. metaflow/plugins/cards/card_server.py +39 -14
  29. metaflow/plugins/datatools/local.py +2 -0
  30. metaflow/plugins/datatools/s3/s3.py +2 -0
  31. metaflow/plugins/env_escape/__init__.py +3 -3
  32. metaflow/plugins/gcp/includefile_support.py +3 -0
  33. metaflow/plugins/kubernetes/kubernetes_cli.py +1 -1
  34. metaflow/plugins/kubernetes/kubernetes_decorator.py +5 -4
  35. metaflow/plugins/kubernetes/kubernetes_jobsets.py +43 -28
  36. metaflow/plugins/{metadata → metadata_providers}/local.py +2 -2
  37. metaflow/plugins/{metadata → metadata_providers}/service.py +2 -2
  38. metaflow/plugins/parallel_decorator.py +1 -1
  39. metaflow/plugins/pypi/conda_decorator.py +1 -1
  40. metaflow/plugins/test_unbounded_foreach_decorator.py +1 -1
  41. metaflow/runner/click_api.py +4 -0
  42. metaflow/runner/deployer.py +134 -303
  43. metaflow/runner/deployer_impl.py +167 -0
  44. metaflow/runner/metaflow_runner.py +10 -9
  45. metaflow/runner/nbdeploy.py +12 -13
  46. metaflow/runner/nbrun.py +3 -3
  47. metaflow/runner/utils.py +55 -8
  48. metaflow/runtime.py +1 -1
  49. metaflow/system/system_logger.py +1 -19
  50. metaflow/system/system_monitor.py +0 -24
  51. metaflow/task.py +5 -8
  52. metaflow/version.py +1 -1
  53. {ob_metaflow-2.12.27.1.dist-info → ob_metaflow-2.12.30.2.dist-info}/METADATA +2 -2
  54. {ob_metaflow-2.12.27.1.dist-info → ob_metaflow-2.12.30.2.dist-info}/RECORD +63 -60
  55. {ob_metaflow-2.12.27.1.dist-info → ob_metaflow-2.12.30.2.dist-info}/WHEEL +1 -1
  56. /metaflow/{metadata → metadata_provider}/__init__.py +0 -0
  57. /metaflow/{metadata → metadata_provider}/heartbeat.py +0 -0
  58. /metaflow/{metadata → metadata_provider}/metadata.py +0 -0
  59. /metaflow/{metadata → metadata_provider}/util.py +0 -0
  60. /metaflow/plugins/{metadata → metadata_providers}/__init__.py +0 -0
  61. {ob_metaflow-2.12.27.1.dist-info → ob_metaflow-2.12.30.2.dist-info}/LICENSE +0 -0
  62. {ob_metaflow-2.12.27.1.dist-info → ob_metaflow-2.12.30.2.dist-info}/entry_points.txt +0 -0
  63. {ob_metaflow-2.12.27.1.dist-info → ob_metaflow-2.12.30.2.dist-info}/top_level.txt +0 -0
metaflow/__init__.py CHANGED
@@ -101,9 +101,7 @@ from .metaflow_current import current
101
101
  # Flow spec
102
102
  from .flowspec import FlowSpec
103
103
 
104
- from .parameters import Parameter, JSONTypeClass
105
-
106
- JSONType = JSONTypeClass()
104
+ from .parameters import Parameter, JSONTypeClass, JSONType
107
105
 
108
106
  # data layer
109
107
  # For historical reasons, we make metaflow.plugins.datatools accessible as
@@ -149,6 +147,7 @@ if sys.version_info >= (3, 7):
149
147
  from .runner.metaflow_runner import Runner
150
148
  from .runner.nbrun import NBRunner
151
149
  from .runner.deployer import Deployer
150
+ from .runner.deployer import DeployedFlow
152
151
  from .runner.nbdeploy import NBDeployer
153
152
 
154
153
  __ext_tl_modules__ = []
metaflow/cli.py CHANGED
@@ -716,6 +716,20 @@ def resume(
716
716
  if runtime.should_skip_clone_only_execution():
717
717
  return
718
718
 
719
+ current._update_env(
720
+ {
721
+ "run_id": runtime.run_id,
722
+ }
723
+ )
724
+ _system_logger.log_event(
725
+ level="info",
726
+ module="metaflow.resume",
727
+ name="start",
728
+ payload={
729
+ "msg": "Resuming run",
730
+ },
731
+ )
732
+
719
733
  with runtime.run_heartbeat():
720
734
  if clone_only:
721
735
  runtime.clone_original_run()
@@ -775,6 +789,19 @@ def run(
775
789
  write_file(run_id_file, runtime.run_id)
776
790
 
777
791
  obj.flow._set_constants(obj.graph, kwargs)
792
+ current._update_env(
793
+ {
794
+ "run_id": runtime.run_id,
795
+ }
796
+ )
797
+ _system_logger.log_event(
798
+ level="info",
799
+ module="metaflow.run",
800
+ name="start",
801
+ payload={
802
+ "msg": "Starting run",
803
+ },
804
+ )
778
805
  runtime.print_workflow_info()
779
806
  runtime.persist_constants()
780
807
 
metaflow/client/core.py CHANGED
@@ -39,7 +39,7 @@ from ..info_file import INFO_FILE
39
39
  from .filecache import FileCache
40
40
 
41
41
  if TYPE_CHECKING:
42
- from metaflow.metadata import MetadataProvider
42
+ from metaflow.metadata_provider import MetadataProvider
43
43
 
44
44
  try:
45
45
  # python2
@@ -143,7 +143,7 @@ def default_metadata() -> str:
143
143
  if default:
144
144
  current_metadata = default[0]
145
145
  else:
146
- from metaflow.plugins.metadata import LocalMetadataProvider
146
+ from metaflow.plugins.metadata_providers import LocalMetadataProvider
147
147
 
148
148
  current_metadata = LocalMetadataProvider
149
149
  return get_metadata()
metaflow/clone_util.py CHANGED
@@ -1,5 +1,5 @@
1
1
  import time
2
- from .metadata import MetaDatum
2
+ from .metadata_provider import MetaDatum
3
3
 
4
4
 
5
5
  def clone_task_helper(