ob-metaflow 2.11.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (289) hide show
  1. metaflow/R.py +10 -7
  2. metaflow/__init__.py +40 -25
  3. metaflow/_vendor/imghdr/__init__.py +186 -0
  4. metaflow/_vendor/importlib_metadata/__init__.py +1063 -0
  5. metaflow/_vendor/importlib_metadata/_adapters.py +68 -0
  6. metaflow/_vendor/importlib_metadata/_collections.py +30 -0
  7. metaflow/_vendor/importlib_metadata/_compat.py +71 -0
  8. metaflow/_vendor/importlib_metadata/_functools.py +104 -0
  9. metaflow/_vendor/importlib_metadata/_itertools.py +73 -0
  10. metaflow/_vendor/importlib_metadata/_meta.py +48 -0
  11. metaflow/_vendor/importlib_metadata/_text.py +99 -0
  12. metaflow/_vendor/importlib_metadata/py.typed +0 -0
  13. metaflow/_vendor/typeguard/__init__.py +48 -0
  14. metaflow/_vendor/typeguard/_checkers.py +1070 -0
  15. metaflow/_vendor/typeguard/_config.py +108 -0
  16. metaflow/_vendor/typeguard/_decorators.py +233 -0
  17. metaflow/_vendor/typeguard/_exceptions.py +42 -0
  18. metaflow/_vendor/typeguard/_functions.py +308 -0
  19. metaflow/_vendor/typeguard/_importhook.py +213 -0
  20. metaflow/_vendor/typeguard/_memo.py +48 -0
  21. metaflow/_vendor/typeguard/_pytest_plugin.py +127 -0
  22. metaflow/_vendor/typeguard/_suppression.py +86 -0
  23. metaflow/_vendor/typeguard/_transformer.py +1229 -0
  24. metaflow/_vendor/typeguard/_union_transformer.py +55 -0
  25. metaflow/_vendor/typeguard/_utils.py +173 -0
  26. metaflow/_vendor/typeguard/py.typed +0 -0
  27. metaflow/_vendor/typing_extensions.py +3641 -0
  28. metaflow/_vendor/v3_7/importlib_metadata/__init__.py +1063 -0
  29. metaflow/_vendor/v3_7/importlib_metadata/_adapters.py +68 -0
  30. metaflow/_vendor/v3_7/importlib_metadata/_collections.py +30 -0
  31. metaflow/_vendor/v3_7/importlib_metadata/_compat.py +71 -0
  32. metaflow/_vendor/v3_7/importlib_metadata/_functools.py +104 -0
  33. metaflow/_vendor/v3_7/importlib_metadata/_itertools.py +73 -0
  34. metaflow/_vendor/v3_7/importlib_metadata/_meta.py +48 -0
  35. metaflow/_vendor/v3_7/importlib_metadata/_text.py +99 -0
  36. metaflow/_vendor/v3_7/importlib_metadata/py.typed +0 -0
  37. metaflow/_vendor/v3_7/typeguard/__init__.py +48 -0
  38. metaflow/_vendor/v3_7/typeguard/_checkers.py +906 -0
  39. metaflow/_vendor/v3_7/typeguard/_config.py +108 -0
  40. metaflow/_vendor/v3_7/typeguard/_decorators.py +237 -0
  41. metaflow/_vendor/v3_7/typeguard/_exceptions.py +42 -0
  42. metaflow/_vendor/v3_7/typeguard/_functions.py +310 -0
  43. metaflow/_vendor/v3_7/typeguard/_importhook.py +213 -0
  44. metaflow/_vendor/v3_7/typeguard/_memo.py +48 -0
  45. metaflow/_vendor/v3_7/typeguard/_pytest_plugin.py +100 -0
  46. metaflow/_vendor/v3_7/typeguard/_suppression.py +88 -0
  47. metaflow/_vendor/v3_7/typeguard/_transformer.py +1207 -0
  48. metaflow/_vendor/v3_7/typeguard/_union_transformer.py +54 -0
  49. metaflow/_vendor/v3_7/typeguard/_utils.py +169 -0
  50. metaflow/_vendor/v3_7/typeguard/py.typed +0 -0
  51. metaflow/_vendor/v3_7/typing_extensions.py +3072 -0
  52. metaflow/_vendor/yaml/__init__.py +427 -0
  53. metaflow/_vendor/yaml/composer.py +139 -0
  54. metaflow/_vendor/yaml/constructor.py +748 -0
  55. metaflow/_vendor/yaml/cyaml.py +101 -0
  56. metaflow/_vendor/yaml/dumper.py +62 -0
  57. metaflow/_vendor/yaml/emitter.py +1137 -0
  58. metaflow/_vendor/yaml/error.py +75 -0
  59. metaflow/_vendor/yaml/events.py +86 -0
  60. metaflow/_vendor/yaml/loader.py +63 -0
  61. metaflow/_vendor/yaml/nodes.py +49 -0
  62. metaflow/_vendor/yaml/parser.py +589 -0
  63. metaflow/_vendor/yaml/reader.py +185 -0
  64. metaflow/_vendor/yaml/representer.py +389 -0
  65. metaflow/_vendor/yaml/resolver.py +227 -0
  66. metaflow/_vendor/yaml/scanner.py +1435 -0
  67. metaflow/_vendor/yaml/serializer.py +111 -0
  68. metaflow/_vendor/yaml/tokens.py +104 -0
  69. metaflow/cards.py +5 -0
  70. metaflow/cli.py +331 -785
  71. metaflow/cli_args.py +17 -0
  72. metaflow/cli_components/__init__.py +0 -0
  73. metaflow/cli_components/dump_cmd.py +96 -0
  74. metaflow/cli_components/init_cmd.py +52 -0
  75. metaflow/cli_components/run_cmds.py +546 -0
  76. metaflow/cli_components/step_cmd.py +334 -0
  77. metaflow/cli_components/utils.py +140 -0
  78. metaflow/client/__init__.py +1 -0
  79. metaflow/client/core.py +467 -73
  80. metaflow/client/filecache.py +75 -35
  81. metaflow/clone_util.py +7 -1
  82. metaflow/cmd/code/__init__.py +231 -0
  83. metaflow/cmd/develop/stub_generator.py +756 -288
  84. metaflow/cmd/develop/stubs.py +12 -28
  85. metaflow/cmd/main_cli.py +6 -4
  86. metaflow/cmd/make_wrapper.py +78 -0
  87. metaflow/datastore/__init__.py +1 -0
  88. metaflow/datastore/content_addressed_store.py +41 -10
  89. metaflow/datastore/datastore_set.py +11 -2
  90. metaflow/datastore/flow_datastore.py +156 -10
  91. metaflow/datastore/spin_datastore.py +91 -0
  92. metaflow/datastore/task_datastore.py +154 -39
  93. metaflow/debug.py +5 -0
  94. metaflow/decorators.py +404 -78
  95. metaflow/exception.py +8 -2
  96. metaflow/extension_support/__init__.py +527 -376
  97. metaflow/extension_support/_empty_file.py +2 -2
  98. metaflow/extension_support/plugins.py +49 -31
  99. metaflow/flowspec.py +482 -33
  100. metaflow/graph.py +210 -42
  101. metaflow/includefile.py +84 -40
  102. metaflow/lint.py +141 -22
  103. metaflow/meta_files.py +13 -0
  104. metaflow/{metadata → metadata_provider}/heartbeat.py +24 -8
  105. metaflow/{metadata → metadata_provider}/metadata.py +86 -1
  106. metaflow/metaflow_config.py +175 -28
  107. metaflow/metaflow_config_funcs.py +51 -3
  108. metaflow/metaflow_current.py +4 -10
  109. metaflow/metaflow_environment.py +139 -53
  110. metaflow/metaflow_git.py +115 -0
  111. metaflow/metaflow_profile.py +18 -0
  112. metaflow/metaflow_version.py +150 -66
  113. metaflow/mflog/__init__.py +4 -3
  114. metaflow/mflog/save_logs.py +2 -2
  115. metaflow/multicore_utils.py +31 -14
  116. metaflow/package/__init__.py +673 -0
  117. metaflow/packaging_sys/__init__.py +880 -0
  118. metaflow/packaging_sys/backend.py +128 -0
  119. metaflow/packaging_sys/distribution_support.py +153 -0
  120. metaflow/packaging_sys/tar_backend.py +99 -0
  121. metaflow/packaging_sys/utils.py +54 -0
  122. metaflow/packaging_sys/v1.py +527 -0
  123. metaflow/parameters.py +149 -28
  124. metaflow/plugins/__init__.py +74 -5
  125. metaflow/plugins/airflow/airflow.py +40 -25
  126. metaflow/plugins/airflow/airflow_cli.py +22 -5
  127. metaflow/plugins/airflow/airflow_decorator.py +1 -1
  128. metaflow/plugins/airflow/airflow_utils.py +5 -3
  129. metaflow/plugins/airflow/sensors/base_sensor.py +4 -4
  130. metaflow/plugins/airflow/sensors/external_task_sensor.py +2 -2
  131. metaflow/plugins/airflow/sensors/s3_sensor.py +2 -2
  132. metaflow/plugins/argo/argo_client.py +78 -33
  133. metaflow/plugins/argo/argo_events.py +6 -6
  134. metaflow/plugins/argo/argo_workflows.py +2410 -527
  135. metaflow/plugins/argo/argo_workflows_cli.py +571 -121
  136. metaflow/plugins/argo/argo_workflows_decorator.py +43 -12
  137. metaflow/plugins/argo/argo_workflows_deployer.py +106 -0
  138. metaflow/plugins/argo/argo_workflows_deployer_objects.py +453 -0
  139. metaflow/plugins/argo/capture_error.py +73 -0
  140. metaflow/plugins/argo/conditional_input_paths.py +35 -0
  141. metaflow/plugins/argo/exit_hooks.py +209 -0
  142. metaflow/plugins/argo/jobset_input_paths.py +15 -0
  143. metaflow/plugins/argo/param_val.py +19 -0
  144. metaflow/plugins/aws/aws_client.py +10 -3
  145. metaflow/plugins/aws/aws_utils.py +55 -2
  146. metaflow/plugins/aws/batch/batch.py +72 -5
  147. metaflow/plugins/aws/batch/batch_cli.py +33 -10
  148. metaflow/plugins/aws/batch/batch_client.py +4 -3
  149. metaflow/plugins/aws/batch/batch_decorator.py +102 -35
  150. metaflow/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.py +13 -10
  151. metaflow/plugins/aws/step_functions/dynamo_db_client.py +0 -3
  152. metaflow/plugins/aws/step_functions/production_token.py +1 -1
  153. metaflow/plugins/aws/step_functions/step_functions.py +65 -8
  154. metaflow/plugins/aws/step_functions/step_functions_cli.py +101 -7
  155. metaflow/plugins/aws/step_functions/step_functions_decorator.py +1 -2
  156. metaflow/plugins/aws/step_functions/step_functions_deployer.py +97 -0
  157. metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py +264 -0
  158. metaflow/plugins/azure/azure_exceptions.py +1 -1
  159. metaflow/plugins/azure/azure_secret_manager_secrets_provider.py +240 -0
  160. metaflow/plugins/azure/azure_tail.py +1 -1
  161. metaflow/plugins/azure/includefile_support.py +2 -0
  162. metaflow/plugins/cards/card_cli.py +66 -30
  163. metaflow/plugins/cards/card_creator.py +25 -1
  164. metaflow/plugins/cards/card_datastore.py +21 -49
  165. metaflow/plugins/cards/card_decorator.py +132 -8
  166. metaflow/plugins/cards/card_modules/basic.py +112 -17
  167. metaflow/plugins/cards/card_modules/bundle.css +1 -1
  168. metaflow/plugins/cards/card_modules/card.py +16 -1
  169. metaflow/plugins/cards/card_modules/chevron/renderer.py +1 -1
  170. metaflow/plugins/cards/card_modules/components.py +665 -28
  171. metaflow/plugins/cards/card_modules/convert_to_native_type.py +36 -7
  172. metaflow/plugins/cards/card_modules/json_viewer.py +232 -0
  173. metaflow/plugins/cards/card_modules/main.css +1 -0
  174. metaflow/plugins/cards/card_modules/main.js +68 -49
  175. metaflow/plugins/cards/card_modules/renderer_tools.py +1 -0
  176. metaflow/plugins/cards/card_modules/test_cards.py +26 -12
  177. metaflow/plugins/cards/card_server.py +39 -14
  178. metaflow/plugins/cards/component_serializer.py +2 -9
  179. metaflow/plugins/cards/metadata.py +22 -0
  180. metaflow/plugins/catch_decorator.py +9 -0
  181. metaflow/plugins/datastores/azure_storage.py +10 -1
  182. metaflow/plugins/datastores/gs_storage.py +6 -2
  183. metaflow/plugins/datastores/local_storage.py +12 -6
  184. metaflow/plugins/datastores/spin_storage.py +12 -0
  185. metaflow/plugins/datatools/local.py +2 -0
  186. metaflow/plugins/datatools/s3/s3.py +126 -75
  187. metaflow/plugins/datatools/s3/s3op.py +254 -121
  188. metaflow/plugins/env_escape/__init__.py +3 -3
  189. metaflow/plugins/env_escape/client_modules.py +102 -72
  190. metaflow/plugins/env_escape/server.py +7 -0
  191. metaflow/plugins/env_escape/stub.py +24 -5
  192. metaflow/plugins/events_decorator.py +343 -185
  193. metaflow/plugins/exit_hook/__init__.py +0 -0
  194. metaflow/plugins/exit_hook/exit_hook_decorator.py +46 -0
  195. metaflow/plugins/exit_hook/exit_hook_script.py +52 -0
  196. metaflow/plugins/gcp/__init__.py +1 -1
  197. metaflow/plugins/gcp/gcp_secret_manager_secrets_provider.py +11 -6
  198. metaflow/plugins/gcp/gs_tail.py +10 -6
  199. metaflow/plugins/gcp/includefile_support.py +3 -0
  200. metaflow/plugins/kubernetes/kube_utils.py +108 -0
  201. metaflow/plugins/kubernetes/kubernetes.py +411 -130
  202. metaflow/plugins/kubernetes/kubernetes_cli.py +168 -36
  203. metaflow/plugins/kubernetes/kubernetes_client.py +104 -2
  204. metaflow/plugins/kubernetes/kubernetes_decorator.py +246 -88
  205. metaflow/plugins/kubernetes/kubernetes_job.py +253 -581
  206. metaflow/plugins/kubernetes/kubernetes_jobsets.py +1071 -0
  207. metaflow/plugins/kubernetes/spot_metadata_cli.py +69 -0
  208. metaflow/plugins/kubernetes/spot_monitor_sidecar.py +109 -0
  209. metaflow/plugins/logs_cli.py +359 -0
  210. metaflow/plugins/{metadata → metadata_providers}/local.py +144 -84
  211. metaflow/plugins/{metadata → metadata_providers}/service.py +103 -26
  212. metaflow/plugins/metadata_providers/spin.py +16 -0
  213. metaflow/plugins/package_cli.py +36 -24
  214. metaflow/plugins/parallel_decorator.py +128 -11
  215. metaflow/plugins/parsers.py +16 -0
  216. metaflow/plugins/project_decorator.py +51 -5
  217. metaflow/plugins/pypi/bootstrap.py +357 -105
  218. metaflow/plugins/pypi/conda_decorator.py +82 -81
  219. metaflow/plugins/pypi/conda_environment.py +187 -52
  220. metaflow/plugins/pypi/micromamba.py +157 -47
  221. metaflow/plugins/pypi/parsers.py +268 -0
  222. metaflow/plugins/pypi/pip.py +88 -13
  223. metaflow/plugins/pypi/pypi_decorator.py +37 -1
  224. metaflow/plugins/pypi/utils.py +48 -2
  225. metaflow/plugins/resources_decorator.py +2 -2
  226. metaflow/plugins/secrets/__init__.py +3 -0
  227. metaflow/plugins/secrets/secrets_decorator.py +26 -181
  228. metaflow/plugins/secrets/secrets_func.py +49 -0
  229. metaflow/plugins/secrets/secrets_spec.py +101 -0
  230. metaflow/plugins/secrets/utils.py +74 -0
  231. metaflow/plugins/tag_cli.py +4 -7
  232. metaflow/plugins/test_unbounded_foreach_decorator.py +41 -6
  233. metaflow/plugins/timeout_decorator.py +3 -3
  234. metaflow/plugins/uv/__init__.py +0 -0
  235. metaflow/plugins/uv/bootstrap.py +128 -0
  236. metaflow/plugins/uv/uv_environment.py +72 -0
  237. metaflow/procpoll.py +1 -1
  238. metaflow/pylint_wrapper.py +5 -1
  239. metaflow/runner/__init__.py +0 -0
  240. metaflow/runner/click_api.py +717 -0
  241. metaflow/runner/deployer.py +470 -0
  242. metaflow/runner/deployer_impl.py +201 -0
  243. metaflow/runner/metaflow_runner.py +714 -0
  244. metaflow/runner/nbdeploy.py +132 -0
  245. metaflow/runner/nbrun.py +225 -0
  246. metaflow/runner/subprocess_manager.py +650 -0
  247. metaflow/runner/utils.py +335 -0
  248. metaflow/runtime.py +1078 -260
  249. metaflow/sidecar/sidecar_worker.py +1 -1
  250. metaflow/system/__init__.py +5 -0
  251. metaflow/system/system_logger.py +85 -0
  252. metaflow/system/system_monitor.py +108 -0
  253. metaflow/system/system_utils.py +19 -0
  254. metaflow/task.py +521 -225
  255. metaflow/tracing/__init__.py +7 -7
  256. metaflow/tracing/span_exporter.py +31 -38
  257. metaflow/tracing/tracing_modules.py +38 -43
  258. metaflow/tuple_util.py +27 -0
  259. metaflow/user_configs/__init__.py +0 -0
  260. metaflow/user_configs/config_options.py +563 -0
  261. metaflow/user_configs/config_parameters.py +598 -0
  262. metaflow/user_decorators/__init__.py +0 -0
  263. metaflow/user_decorators/common.py +144 -0
  264. metaflow/user_decorators/mutable_flow.py +512 -0
  265. metaflow/user_decorators/mutable_step.py +424 -0
  266. metaflow/user_decorators/user_flow_decorator.py +264 -0
  267. metaflow/user_decorators/user_step_decorator.py +749 -0
  268. metaflow/util.py +243 -27
  269. metaflow/vendor.py +23 -7
  270. metaflow/version.py +1 -1
  271. ob_metaflow-2.19.7.1rc0.data/data/share/metaflow/devtools/Makefile +355 -0
  272. ob_metaflow-2.19.7.1rc0.data/data/share/metaflow/devtools/Tiltfile +726 -0
  273. ob_metaflow-2.19.7.1rc0.data/data/share/metaflow/devtools/pick_services.sh +105 -0
  274. ob_metaflow-2.19.7.1rc0.dist-info/METADATA +87 -0
  275. ob_metaflow-2.19.7.1rc0.dist-info/RECORD +445 -0
  276. {ob_metaflow-2.11.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/WHEEL +1 -1
  277. {ob_metaflow-2.11.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/entry_points.txt +1 -0
  278. metaflow/_vendor/v3_5/__init__.py +0 -1
  279. metaflow/_vendor/v3_5/importlib_metadata/__init__.py +0 -644
  280. metaflow/_vendor/v3_5/importlib_metadata/_compat.py +0 -152
  281. metaflow/package.py +0 -188
  282. ob_metaflow-2.11.13.1.dist-info/METADATA +0 -85
  283. ob_metaflow-2.11.13.1.dist-info/RECORD +0 -308
  284. /metaflow/_vendor/{v3_5/zipp.py → zipp.py} +0 -0
  285. /metaflow/{metadata → metadata_provider}/__init__.py +0 -0
  286. /metaflow/{metadata → metadata_provider}/util.py +0 -0
  287. /metaflow/plugins/{metadata → metadata_providers}/__init__.py +0 -0
  288. {ob_metaflow-2.11.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info/licenses}/LICENSE +0 -0
  289. {ob_metaflow-2.11.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/top_level.txt +0 -0
metaflow/plugins/cards/card_modules/renderer_tools.py CHANGED
@@ -40,6 +40,7 @@ def render_safely(func):
40
40
  This is a decorator that can be added to any `MetaflowCardComponent.render`
41
41
  The goal is to render subcomponents safely and ensure that they are JSON serializable.
42
42
  """
43
+
43
44
  # expects a renderer func
44
45
  def ret_func(self, *args, **kwargs):
45
46
  return _render_component_safely(self, func, True, *args, **kwargs)
metaflow/plugins/cards/card_modules/test_cards.py CHANGED
@@ -34,41 +34,41 @@ class TestPathSpecCard(MetaflowCard):
34
34
  class TestEditableCard(MetaflowCard):
35
35
  type = "test_editable_card"
36
36
 
37
- seperator = "$&#!!@*"
37
+ separator = "$&#!!@*"
38
38
 
39
39
  ALLOW_USER_COMPONENTS = True
40
40
 
41
- def __init__(self, options={}, components=[], graph=None):
41
+ def __init__(self, components=[], **kwargs):
42
42
  self._components = components
43
43
 
44
44
  def render(self, task):
45
- return self.seperator.join([str(comp) for comp in self._components])
45
+ return self.separator.join([str(comp) for comp in self._components])
46
46
 
47
47
 
48
48
  class TestEditableCard2(MetaflowCard):
49
49
  type = "test_editable_card_2"
50
50
 
51
- seperator = "$&#!!@*"
51
+ separator = "$&#!!@*"
52
52
 
53
53
  ALLOW_USER_COMPONENTS = True
54
54
 
55
- def __init__(self, options={}, components=[], graph=None):
55
+ def __init__(self, components=[], **kwargs):
56
56
  self._components = components
57
57
 
58
58
  def render(self, task):
59
- return self.seperator.join([str(comp) for comp in self._components])
59
+ return self.separator.join([str(comp) for comp in self._components])
60
60
 
61
61
 
62
62
  class TestNonEditableCard(MetaflowCard):
63
63
  type = "test_non_editable_card"
64
64
 
65
- seperator = "$&#!!@*"
65
+ separator = "$&#!!@*"
66
66
 
67
- def __init__(self, options={}, components=[], graph=None):
67
+ def __init__(self, components=[], **kwargs):
68
68
  self._components = components
69
69
 
70
70
  def render(self, task):
71
- return self.seperator.join([str(comp) for comp in self._components])
71
+ return self.separator.join([str(comp) for comp in self._components])
72
72
 
73
73
 
74
74
  class TestMockCard(MetaflowCard):
@@ -138,7 +138,6 @@ class TestJSONComponent(MetaflowCardComponent):
138
138
 
139
139
 
140
140
  class TestRefreshCard(MetaflowCard):
141
-
142
141
  """
143
142
  This card takes no components and helps test the `current.card.refresh(data)` interface.
144
143
  """
@@ -178,7 +177,6 @@ def _component_values_to_hash(components):
178
177
 
179
178
 
180
179
  class TestRefreshComponentCard(MetaflowCard):
181
-
182
180
  """
183
181
  This card takes components and helps test the `current.card.components["A"].update()`
184
182
  interface
@@ -195,7 +193,7 @@ class TestRefreshComponentCard(MetaflowCard):
195
193
 
196
194
  type = "test_component_refresh_card"
197
195
 
198
- def __init__(self, options={}, components=[], graph=None):
196
+ def __init__(self, components=[], **kwargs):
199
197
  self._components = components
200
198
 
201
199
  def render(self, task) -> str:
@@ -215,3 +213,19 @@ class TestRefreshComponentCard(MetaflowCard):
215
213
  if task.finished:
216
214
  return "final"
217
215
  return "runtime-%s" % _component_values_to_hash(data["components"])
216
+
217
+
218
+ class TestImageCard(MetaflowCard):
219
+ """Card that renders a tiny PNG using ``TaskToDict.parse_image``."""
220
+
221
+ type = "test_image_card"
222
+
223
+ def render(self, task):
224
+ from .convert_to_native_type import TaskToDict
225
+ import base64
226
+
227
+ png_bytes = base64.b64decode(
228
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABRDE8UwAAAABJRU5ErkJggg=="
229
+ )
230
+ img_src = TaskToDict().parse_image(png_bytes)
231
+ return f"<html><img src='{img_src}' /></html>"
metaflow/plugins/cards/card_server.py CHANGED
@@ -20,14 +20,9 @@ except ImportError:
20
20
  from .card_client import CardContainer
21
21
  from .exception import CardNotPresentException
22
22
  from .card_resolver import resolve_paths_from_task
23
- from metaflow.metaflow_config import DATASTORE_LOCAL_DIR
24
23
  from metaflow import namespace
25
- from metaflow.exception import (
26
- CommandException,
27
- MetaflowNotFound,
28
- MetaflowNamespaceMismatch,
29
- )
30
-
24
+ from metaflow.exception import MetaflowNotFound
25
+ from metaflow.plugins.datastores.local_storage import LocalStorage
31
26
 
32
27
  VIEWER_PATH = os.path.join(
33
28
  os.path.dirname(os.path.abspath(__file__)), "card_viewer", "viewer.html"
@@ -50,18 +45,48 @@ class RunWatcher(Thread):
50
45
  def __init__(self, flow_name, connection: Connection):
51
46
  super().__init__()
52
47
 
53
- self._watch_file = os.path.join(
54
- os.getcwd(), DATASTORE_LOCAL_DIR, flow_name, "latest_run"
55
- )
56
- self._current_run_id = self.get_run_id()
57
48
  self.daemon = True
58
49
  self._connection = connection
50
+ self._flow_name = flow_name
51
+
52
+ self._watch_file = self._initialize_watch_file()
53
+ if self._watch_file is None:
54
+ _ClickLogger(
55
+ "Warning: Could not initialize watch file location.", fg="yellow"
56
+ )
57
+
58
+ self._current_run_id = self.get_run_id()
59
+
60
+ def _initialize_watch_file(self):
61
+ local_root = LocalStorage.datastore_root
62
+ if local_root is None:
63
+ local_root = LocalStorage.get_datastore_root_from_config(
64
+ lambda _: None, create_on_absent=False
65
+ )
66
+
67
+ return (
68
+ os.path.join(local_root, self._flow_name, "latest_run")
69
+ if local_root
70
+ else None
71
+ )
59
72
 
60
73
  def get_run_id(self):
61
- if not os.path.exists(self._watch_file):
74
+ # Try to reinitialize watch file if needed
75
+ if not self._watch_file:
76
+ self._watch_file = self._initialize_watch_file()
77
+
78
+ # Early return if watch file is still None or doesn't exist
79
+ if not (self._watch_file and os.path.exists(self._watch_file)):
80
+ return None
81
+
82
+ try:
83
+ with open(self._watch_file, "r") as f:
84
+ return f.read().strip()
85
+ except (IOError, OSError) as e:
86
+ _ClickLogger(
87
+ "Warning: Could not read run ID from watch file: %s" % e, fg="yellow"
88
+ )
62
89
  return None
63
- with open(self._watch_file, "r") as f:
64
- return f.read().strip()
65
90
 
66
91
  def watch(self):
67
92
  while True:
metaflow/plugins/cards/component_serializer.py CHANGED
@@ -1,8 +1,8 @@
1
1
  from .card_modules import MetaflowCardComponent
2
+ from .card_modules.card import create_component_id
2
3
  from .card_modules.basic import ErrorComponent, SectionComponent
3
4
  from .card_modules.components import (
4
5
  UserComponent,
5
- create_component_id,
6
6
  StubComponent,
7
7
  )
8
8
  from .exception import ComponentOverwriteNotSupportedException
@@ -57,15 +57,8 @@ class ComponentStore:
57
57
  The `_component_map` attribute is supposed to be a dictionary so that we can access the components by their ids.
58
58
  But we also want to maintain order in which components are inserted since all of these components are going to be visible on a UI.
59
59
  Since python3.6 dictionaries are ordered by default so we can use the default python `dict`.
60
- For python3.5 and below we need to use an OrderedDict since `dict`'s are not ordered by default.
61
60
  """
62
- python_version = int(platform.python_version_tuple()[0]) * 10 + int(
63
- platform.python_version_tuple()[1]
64
- )
65
- if python_version < 36:
66
- self._component_map = OrderedDict()
67
- else:
68
- self._component_map = {}
61
+ self._component_map = {}
69
62
 
70
63
  def __init__(self, logger, card_type=None, components=None, user_set_id=None):
71
64
  self._logger = logger
metaflow/plugins/cards/metadata.py ADDED
@@ -0,0 +1,22 @@
1
+ import json
2
+ from metaflow.metadata_provider import MetaDatum
3
+
4
+
5
+ def _save_metadata(
6
+ metadata_provider,
7
+ run_id,
8
+ step_name,
9
+ task_id,
10
+ attempt_id,
11
+ card_uuid,
12
+ save_metadata,
13
+ ):
14
+ entries = [
15
+ MetaDatum(
16
+ field=card_uuid,
17
+ value=json.dumps(save_metadata),
18
+ type="card-info",
19
+ tags=["attempt_id:{0}".format(attempt_id)],
20
+ )
21
+ ]
22
+ metadata_provider.register_metadata(run_id, step_name, task_id, entries)
metaflow/plugins/catch_decorator.py CHANGED
@@ -52,6 +52,15 @@ class CatchDecorator(StepDecorator):
52
52
  "split steps." % step
53
53
  )
54
54
 
55
+ # Do not support catch on switch steps for now.
56
+ # When applying @catch to a switch step, we can not guarantee that the flow attribute used for the switching condition gets properly recorded.
57
+ if graph[step].type == "split-switch":
58
+ raise MetaflowException(
59
+ "@catch is defined for the step *%s* "
60
+ "but @catch is not supported in conditional "
61
+ "switch steps." % step
62
+ )
63
+
55
64
  def _print_exception(self, step, flow):
56
65
  self.logger(head="@catch caught an exception from %s" % flow, timestamp=False)
57
66
  for line in traceback.format_exc().splitlines():
metaflow/plugins/datastores/azure_storage.py CHANGED
@@ -145,12 +145,19 @@ class _AzureRootClient(object):
145
145
  # It is good enough 99.9% of the time.
146
146
  # Depending on ResourceExistsError is more costly, though
147
147
  # we are still going to handle it right.
148
+
149
+ # The default timeout in the Azure blobstore python SDK
150
+ # doesn't work well on slower network connections and largish
151
+ # files. Hence increasing the connection_timeout below.
152
+ # For more details, see this:
153
+ # https://github.com/Azure/azure-sdk-for-python/issues/23232
148
154
  if overwrite or not blob.exists():
149
155
  blob.upload_blob(
150
156
  byte_stream,
151
157
  overwrite=overwrite,
152
158
  metadata=metadata_to_upload,
153
159
  max_concurrency=AZURE_STORAGE_UPLOAD_MAX_CONCURRENCY,
160
+ connection_timeout=14400,
154
161
  )
155
162
  except ResourceExistsError:
156
163
  if overwrite:
@@ -346,7 +353,9 @@ class AzureStorage(DataStoreStorage):
346
353
  byte_stream, metadata = byte_stream
347
354
  tmp_filename = os.path.join(tmpdir, str(uuid.uuid4()))
348
355
  with open(tmp_filename, "wb") as f:
349
- f.write(byte_stream.read())
356
+ # make sure to close the file handle after reading.
357
+ with byte_stream as bytes:
358
+ f.write(bytes.read())
350
359
  # Fully finish writing the file, before submitting work. Careful with indentation.
351
360
 
352
361
  futures.append(
metaflow/plugins/datastores/gs_storage.py CHANGED
@@ -119,7 +119,9 @@ class _GSRootClient(object):
119
119
  blob.metadata = {"metaflow-user-attributes": json.dumps(metadata)}
120
120
  from google.cloud.storage.retry import DEFAULT_RETRY
121
121
 
122
- blob.upload_from_filename(tmpfile, retry=DEFAULT_RETRY)
122
+ blob.upload_from_filename(
123
+ tmpfile, retry=DEFAULT_RETRY, timeout=(14400, 60)
124
+ ) # generous timeout for massive uploads. Use the same values as for Azure (connection_timeout, read_timeout)
123
125
  except Exception as e:
124
126
  process_gs_exception(e)
125
127
 
@@ -225,7 +227,9 @@ class GSStorage(DataStoreStorage):
225
227
  byte_stream, metadata = byte_stream
226
228
  tmp_filename = os.path.join(tmpdir, str(uuid.uuid4()))
227
229
  with open(tmp_filename, "wb") as f:
228
- f.write(byte_stream.read())
230
+ # make sure to close the file handle after reading.
231
+ with byte_stream as bytes:
232
+ f.write(bytes.read())
229
233
  # Fully finish writing the file, before submitting work. Careful with indentation.
230
234
 
231
235
  futures.append(
metaflow/plugins/datastores/local_storage.py CHANGED
@@ -1,24 +1,29 @@
1
1
  import json
2
2
  import os
3
3
 
4
- from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, DATASTORE_SYSROOT_LOCAL
4
+ from metaflow.metaflow_config import (
5
+ DATASTORE_LOCAL_DIR,
6
+ DATASTORE_SYSROOT_LOCAL,
7
+ )
5
8
  from metaflow.datastore.datastore_storage import CloseAfterUse, DataStoreStorage
6
9
 
7
10
 
8
11
  class LocalStorage(DataStoreStorage):
9
12
  TYPE = "local"
10
13
  METADATA_DIR = "_meta"
14
+ DATASTORE_DIR = DATASTORE_LOCAL_DIR # ".metaflow"
15
+ SYSROOT_VAR = DATASTORE_SYSROOT_LOCAL
11
16
 
12
17
  @classmethod
13
18
  def get_datastore_root_from_config(cls, echo, create_on_absent=True):
14
- result = DATASTORE_SYSROOT_LOCAL
19
+ result = cls.SYSROOT_VAR
15
20
  if result is None:
16
21
  try:
17
22
  # Python2
18
23
  current_path = os.getcwdu()
19
24
  except: # noqa E722
20
25
  current_path = os.getcwd()
21
- check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR)
26
+ check_dir = os.path.join(current_path, cls.DATASTORE_DIR)
22
27
  check_dir = os.path.realpath(check_dir)
23
28
  orig_path = check_dir
24
29
  top_level_reached = False
@@ -28,12 +33,13 @@ class LocalStorage(DataStoreStorage):
28
33
  top_level_reached = True
29
34
  break # We are no longer making upward progress
30
35
  current_path = new_path
31
- check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR)
36
+ check_dir = os.path.join(current_path, cls.DATASTORE_DIR)
32
37
  if top_level_reached:
33
38
  if create_on_absent:
34
39
  # Could not find any directory to use so create a new one
35
40
  echo(
36
- "Creating local datastore in current directory (%s)" % orig_path
41
+ "Creating %s datastore in current directory (%s)"
42
+ % (cls.TYPE, orig_path)
37
43
  )
38
44
  os.mkdir(orig_path)
39
45
  result = orig_path
@@ -42,7 +48,7 @@ class LocalStorage(DataStoreStorage):
42
48
  else:
43
49
  result = check_dir
44
50
  else:
45
- result = os.path.join(result, DATASTORE_LOCAL_DIR)
51
+ result = os.path.join(result, cls.DATASTORE_DIR)
46
52
  return result
47
53
 
48
54
  @staticmethod
metaflow/plugins/datastores/spin_storage.py ADDED
@@ -0,0 +1,12 @@
1
+ from metaflow.metaflow_config import (
2
+ DATASTORE_SPIN_LOCAL_DIR,
3
+ DATASTORE_SYSROOT_SPIN,
4
+ )
5
+ from metaflow.plugins.datastores.local_storage import LocalStorage
6
+
7
+
8
+ class SpinStorage(LocalStorage):
9
+ TYPE = "spin"
10
+ METADATA_DIR = "_meta"
11
+ DATASTORE_DIR = DATASTORE_SPIN_LOCAL_DIR # ".metaflow_spin"
12
+ SYSROOT_VAR = DATASTORE_SYSROOT_SPIN
metaflow/plugins/datatools/local.py CHANGED
@@ -81,6 +81,8 @@ class Local(object):
81
81
  In the future, we may want to allow it to be used in a way similar to the S3() client.
82
82
  """
83
83
 
84
+ TYPE = "local"
85
+
84
86
  @staticmethod
85
87
  def _makedirs(path):
86
88
  try: