ob-metaflow 2.16.4.6__py2.py3-none-any.whl → 2.16.5.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow might be problematic. Click here for more details.

Files changed (30) hide show
  1. metaflow/_vendor/imghdr/__init__.py +0 -5
  2. metaflow/client/core.py +1 -6
  3. metaflow/extension_support/__init__.py +3 -4
  4. metaflow/metaflow_environment.py +6 -14
  5. metaflow/package/__init__.py +9 -18
  6. metaflow/packaging_sys/__init__.py +43 -53
  7. metaflow/packaging_sys/backend.py +6 -21
  8. metaflow/packaging_sys/tar_backend.py +3 -16
  9. metaflow/packaging_sys/v1.py +21 -21
  10. metaflow/plugins/argo/argo_client.py +31 -14
  11. metaflow/plugins/argo/argo_workflows.py +8 -75
  12. metaflow/plugins/argo/argo_workflows_cli.py +1 -2
  13. metaflow/plugins/argo/argo_workflows_deployer_objects.py +32 -0
  14. metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py +14 -0
  15. metaflow/plugins/kubernetes/kubernetes_decorator.py +1 -1
  16. metaflow/plugins/pypi/conda_decorator.py +2 -4
  17. metaflow/runner/click_api.py +7 -14
  18. metaflow/runner/deployer.py +80 -6
  19. metaflow/runner/subprocess_manager.py +12 -20
  20. metaflow/user_decorators/mutable_flow.py +1 -3
  21. metaflow/version.py +1 -1
  22. {ob_metaflow-2.16.4.6.dist-info → ob_metaflow-2.16.5.1.dist-info}/METADATA +2 -2
  23. {ob_metaflow-2.16.4.6.dist-info → ob_metaflow-2.16.5.1.dist-info}/RECORD +30 -30
  24. {ob_metaflow-2.16.4.6.data → ob_metaflow-2.16.5.1.data}/data/share/metaflow/devtools/Makefile +0 -0
  25. {ob_metaflow-2.16.4.6.data → ob_metaflow-2.16.5.1.data}/data/share/metaflow/devtools/Tiltfile +0 -0
  26. {ob_metaflow-2.16.4.6.data → ob_metaflow-2.16.5.1.data}/data/share/metaflow/devtools/pick_services.sh +0 -0
  27. {ob_metaflow-2.16.4.6.dist-info → ob_metaflow-2.16.5.1.dist-info}/WHEEL +0 -0
  28. {ob_metaflow-2.16.4.6.dist-info → ob_metaflow-2.16.5.1.dist-info}/entry_points.txt +0 -0
  29. {ob_metaflow-2.16.4.6.dist-info → ob_metaflow-2.16.5.1.dist-info}/licenses/LICENSE +0 -0
  30. {ob_metaflow-2.16.4.6.dist-info → ob_metaflow-2.16.5.1.dist-info}/top_level.txt +0 -0
@@ -1,14 +1,9 @@
1
1
  """Recognize image file formats based on their first few bytes."""
2
2
 
3
3
  from os import PathLike
4
- import warnings
5
4
 
6
5
  __all__ = ["what"]
7
6
 
8
-
9
- warnings._deprecated(__name__, remove=(3, 13))
10
-
11
-
12
7
  #-------------------------#
13
8
  # Recognize image headers #
14
9
  #-------------------------#
metaflow/client/core.py CHANGED
@@ -831,12 +831,10 @@ class MetaflowCode(object):
831
831
  )
832
832
  self._code_obj = BytesIO(blobdata)
833
833
  self._info = MetaflowPackage.cls_get_info(self._code_metadata, self._code_obj)
834
- self._code_obj.seek(0)
835
834
  if self._info:
836
835
  self._flowspec = MetaflowPackage.cls_get_content(
837
836
  self._code_metadata, self._code_obj, self._info["script"]
838
837
  )
839
- self._code_obj.seek(0)
840
838
  else:
841
839
  raise MetaflowInternalError("Code package metadata is invalid.")
842
840
 
@@ -887,9 +885,7 @@ class MetaflowCode(object):
887
885
  TarFile for everything in this code package
888
886
  """
889
887
  if self._backend.type == "tgz":
890
- to_return = self._backend.cls_open(self._code_obj)
891
- self._code_obj.seek(0)
892
- return to_return
888
+ return self._backend.cls_open(self._code_obj)
893
889
  raise RuntimeError("Archive is not a tarball")
894
890
 
895
891
  def extract(self) -> TemporaryDirectory:
@@ -925,7 +921,6 @@ class MetaflowCode(object):
925
921
  MetaflowPackage.cls_extract_into(
926
922
  self._code_metadata, self._code_obj, tmp.name, ContentType.USER_CONTENT
927
923
  )
928
- self._code_obj.seek(0)
929
924
  return tmp
930
925
 
931
926
  @property
@@ -205,10 +205,9 @@ def package_mfext_all():
205
205
  # the packaged metaflow_extensions directory "self-contained" so that
206
206
  # python doesn't go and search other parts of the system for more
207
207
  # metaflow_extensions.
208
- if _all_packages:
209
- yield os.path.join(
210
- os.path.dirname(os.path.abspath(__file__)), "_empty_file.py"
211
- ), os.path.join(EXT_PKG, "__init__.py")
208
+ yield os.path.join(
209
+ os.path.dirname(os.path.abspath(__file__)), "_empty_file.py"
210
+ ), os.path.join(EXT_PKG, "__init__.py")
212
211
 
213
212
  for p in _all_packages:
214
213
  for path_tuple in package_mfext_package(p):
@@ -203,19 +203,6 @@ class MetaflowEnvironment(object):
203
203
  "mfcontent_version": 1,
204
204
  }
205
205
  )
206
-
207
- extra_exports = []
208
- for k, v in MetaflowPackage.get_post_extract_env_vars(
209
- code_package_metadata, dest_dir="$(pwd)"
210
- ).items():
211
- if k.endswith(":"):
212
- # If the value ends with a colon, we override the existing value
213
- extra_exports.append("export %s=%s" % (k[:-1], v))
214
- else:
215
- extra_exports.append(
216
- "export %s=%s:$(printenv %s)" % (k, v.replace('"', '\\"'), k)
217
- )
218
-
219
206
  cmds = (
220
207
  [
221
208
  BASH_MFLOG,
@@ -239,7 +226,12 @@ class MetaflowEnvironment(object):
239
226
  + MetaflowPackage.get_extract_commands(
240
227
  code_package_metadata, "job.tar", dest_dir="."
241
228
  )
242
- + extra_exports
229
+ + [
230
+ "export %s=%s:$(printenv %s)" % (k, v.replace('"', '\\"'), k)
231
+ for k, v in MetaflowPackage.get_post_extract_env_vars(
232
+ code_package_metadata, dest_dir="."
233
+ ).items()
234
+ ]
243
235
  + [
244
236
  "mflog 'Task is starting.'",
245
237
  "flush_mflogs",
@@ -17,6 +17,7 @@ from ..packaging_sys.utils import suffix_filter, walk
17
17
  from ..metaflow_config import DEFAULT_PACKAGE_SUFFIXES
18
18
  from ..exception import MetaflowException
19
19
  from ..user_configs.config_parameters import dump_config_values
20
+ from ..util import get_metaflow_root
20
21
  from .. import R
21
22
 
22
23
  DEFAULT_SUFFIXES_LIST = DEFAULT_PACKAGE_SUFFIXES.split(",")
@@ -75,22 +76,12 @@ class MetaflowPackage(object):
75
76
  from ..user_decorators.user_flow_decorator import FlowMutatorMeta
76
77
  from ..user_decorators.user_step_decorator import UserStepDecoratorMeta
77
78
 
78
- # Be very defensive here to filter modules in case there are
79
- # some badly behaved modules that have weird values for
80
- # METAFLOW_PACKAGE_POLICY for example.
81
- try:
82
- if (
83
- m.__name__ in FlowMutatorMeta._import_modules
84
- or m.__name__ in UserStepDecoratorMeta._import_modules
85
- or (
86
- hasattr(m, "METAFLOW_PACKAGE_POLICY")
87
- and m.METAFLOW_PACKAGE_POLICY == "include"
88
- )
89
- ):
90
- return True
91
- return False
92
- except:
93
- return False
79
+ if (
80
+ m.__name__ in FlowMutatorMeta._import_modules
81
+ or m.__name__ in UserStepDecoratorMeta._import_modules
82
+ or hasattr(m, "METAFLOW_PACKAGE")
83
+ ):
84
+ return True
94
85
 
95
86
  if mfcontent is None:
96
87
  self._mfcontent = MetaflowCodeContentV1(criteria=_module_selector)
@@ -359,10 +350,10 @@ class MetaflowPackage(object):
359
350
  """
360
351
  backend = cls.get_backend(pkg_metadata)
361
352
  with backend.cls_open(archive) as opened_archive:
362
- include_members = MetaflowCodeContent.get_archive_content_members(
353
+ include_names = MetaflowCodeContent.get_archive_content_names(
363
354
  opened_archive, content_types, backend
364
355
  )
365
- backend.cls_extract_members(opened_archive, include_members, dest_dir)
356
+ backend.extract_members(include_names, dest_dir)
366
357
 
367
358
  def user_tuples(self, timeout: Optional[float] = None):
368
359
  # Wait for at least the blob to be formed
@@ -118,7 +118,9 @@ class MetaflowCodeContent:
118
118
  return handling_cls.get_filename_impl(mfcontent_info, filename, content_type)
119
119
 
120
120
  @classmethod
121
- def get_env_vars_for_packaged_metaflow(cls, dest_dir: str) -> Dict[str, str]:
121
+ def get_env_vars_for_packaged_metaflow(
122
+ cls, dest_dir: str
123
+ ) -> Optional[Dict[str, str]]:
122
124
  """
123
125
  Get the environment variables that are needed to run Metaflow when it is
124
126
  packaged. This is typically used to set the PYTHONPATH to include the
@@ -126,19 +128,17 @@ class MetaflowCodeContent:
126
128
 
127
129
  Returns
128
130
  -------
129
- Dict[str, str]
131
+ Optional[Dict[str, str]]
130
132
  The environment variables that are needed to run Metaflow when it is
131
- packaged it present.
133
+ packaged -- None if there are no such variables (not packaged for example)
132
134
  """
133
- mfcontent_info = cls._extract_mfcontent_info(dest_dir)
135
+ mfcontent_info = cls._extract_mfcontent_info()
134
136
  if mfcontent_info is None:
135
137
  # No MFCONTENT_MARKER file found -- this is not a packaged Metaflow code
136
138
  # package so no environment variables to set.
137
- return {}
139
+ return None
138
140
  handling_cls = cls._get_mfcontent_class(mfcontent_info)
139
- v = handling_cls.get_post_extract_env_vars_impl(dest_dir)
140
- v["METAFLOW_EXTRACTED_ROOT:"] = dest_dir
141
- return v
141
+ return handling_cls.get_post_extract_env_vars_impl(dest_dir)
142
142
 
143
143
  @classmethod
144
144
  def get_archive_info(
@@ -216,15 +216,15 @@ class MetaflowCodeContent:
216
216
  )
217
217
 
218
218
  @classmethod
219
- def get_archive_content_members(
219
+ def get_archive_content_names(
220
220
  cls,
221
221
  archive: Any,
222
222
  content_types: Optional[int] = None,
223
223
  packaging_backend: Type[PackagingBackend] = TarPackagingBackend,
224
- ) -> List[Any]:
224
+ ) -> List[str]:
225
225
  mfcontent_info = cls._extract_archive_mfcontent_info(archive, packaging_backend)
226
226
  handling_cls = cls._get_mfcontent_class(mfcontent_info)
227
- return handling_cls.get_archive_content_members_impl(
227
+ return handling_cls.get_archive_content_names_impl(
228
228
  mfcontent_info, archive, content_types, packaging_backend
229
229
  )
230
230
 
@@ -276,9 +276,7 @@ class MetaflowCodeContent:
276
276
  "Invalid package -- unknown version %s in info: %s"
277
277
  % (version_id, cls._mappings)
278
278
  )
279
- v = cls._mappings[version_id].get_post_extract_env_vars_impl(dest_dir)
280
- v["METAFLOW_EXTRACTED_ROOT:"] = dest_dir
281
- return v
279
+ return cls._mappings[version_id].get_post_extract_env_vars_impl(dest_dir)
282
280
 
283
281
  # Implement the _impl methods in the base subclass (in this file). These need to
284
282
  # happen with as few imports as possible to prevent circular dependencies.
@@ -339,14 +337,14 @@ class MetaflowCodeContent:
339
337
  raise NotImplementedError("get_archive_filename_impl not implemented")
340
338
 
341
339
  @classmethod
342
- def get_archive_content_members_impl(
340
+ def get_archive_content_names_impl(
343
341
  cls,
344
342
  mfcontent_info: Optional[Dict[str, Any]],
345
343
  archive: Any,
346
344
  content_types: Optional[int] = None,
347
345
  packaging_backend: Type[PackagingBackend] = TarPackagingBackend,
348
- ) -> List[Any]:
349
- raise NotImplementedError("get_archive_content_members_impl not implemented")
346
+ ) -> List[str]:
347
+ raise NotImplementedError("get_archive_content_names_impl not implemented")
350
348
 
351
349
  @classmethod
352
350
  def get_post_extract_env_vars_impl(cls, dest_dir: str) -> Dict[str, str]:
@@ -525,22 +523,19 @@ class MetaflowCodeContent:
525
523
  return mfcontent_info
526
524
 
527
525
  @classmethod
528
- def _extract_mfcontent_info(
529
- cls, target_dir: Optional[str] = None
530
- ) -> Optional[Dict[str, Any]]:
531
- target_dir = target_dir or "_local"
532
- if target_dir in cls._cached_mfcontent_info:
533
- return cls._cached_mfcontent_info[target_dir]
526
+ def _extract_mfcontent_info(cls) -> Optional[Dict[str, Any]]:
527
+ if "_local" in cls._cached_mfcontent_info:
528
+ return cls._cached_mfcontent_info["_local"]
534
529
 
535
530
  mfcontent_info = None # type: Optional[Dict[str, Any]]
536
- if target_dir == "_local":
537
- root = os.environ.get("METAFLOW_EXTRACTED_ROOT", get_metaflow_root())
538
- else:
539
- root = target_dir
540
- if os.path.exists(os.path.join(root, MFCONTENT_MARKER)):
541
- with open(os.path.join(root, MFCONTENT_MARKER), "r", encoding="utf-8") as f:
531
+ if os.path.exists(os.path.join(get_metaflow_root(), MFCONTENT_MARKER)):
532
+ with open(
533
+ os.path.join(get_metaflow_root(), MFCONTENT_MARKER),
534
+ "r",
535
+ encoding="utf-8",
536
+ ) as f:
542
537
  mfcontent_info = json.load(f)
543
- cls._cached_mfcontent_info[target_dir] = mfcontent_info
538
+ cls._cached_mfcontent_info["_local"] = mfcontent_info
544
539
  return mfcontent_info
545
540
 
546
541
  def get_package_version(self) -> int:
@@ -632,13 +627,13 @@ class MetaflowCodeContentV0(MetaflowCodeContent, version_id=0):
632
627
  return None
633
628
 
634
629
  @classmethod
635
- def get_archive_content_members_impl(
630
+ def get_archive_content_names_impl(
636
631
  cls,
637
632
  mfcontent_info: Optional[Dict[str, Any]],
638
633
  archive: Any,
639
634
  content_types: Optional[int] = None,
640
635
  packaging_backend: Type[PackagingBackend] = TarPackagingBackend,
641
- ) -> List[Any]:
636
+ ) -> List[str]:
642
637
  """
643
638
  For V0, we use a static list of known files to classify the content
644
639
  """
@@ -654,20 +649,16 @@ class MetaflowCodeContentV0(MetaflowCodeContent, version_id=0):
654
649
  "condav2-1.cnd": ContentType.OTHER_CONTENT.value,
655
650
  }
656
651
  to_return = []
657
- for member in packaging_backend.cls_list_members(archive):
658
- filename = packaging_backend.cls_member_name(member)
659
- added = False
652
+ for filename in packaging_backend.cls_list_members(archive):
660
653
  for prefix, classification in known_prefixes.items():
661
654
  if (
662
655
  prefix[-1] == "/" and filename.startswith(prefix)
663
656
  ) or prefix == filename:
664
657
  if content_types & classification:
665
- to_return.append(member)
666
- added = True
667
- break
668
- if not added and content_types & ContentType.USER_CONTENT.value:
669
- # Everything else is user content
670
- to_return.append(member)
658
+ to_return.append(filename)
659
+ elif content_types & ContentType.USER_CONTENT.value:
660
+ # Everything else is user content
661
+ to_return.append(filename)
671
662
  return to_return
672
663
 
673
664
  @classmethod
@@ -714,7 +705,7 @@ class MetaflowCodeContentV1Base(MetaflowCodeContent, version_id=1):
714
705
  cls, mfcontent_info: Optional[Dict[str, Any]], filename: str, in_archive: bool
715
706
  ) -> str:
716
707
  if in_archive:
717
- return os.path.join(cls._other_dir, filename)
708
+ return filename
718
709
  return os.path.join(get_metaflow_root(), "..", cls._other_dir, filename)
719
710
 
720
711
  @classmethod
@@ -722,7 +713,7 @@ class MetaflowCodeContentV1Base(MetaflowCodeContent, version_id=1):
722
713
  cls, mfcontent_info: Optional[Dict[str, Any]], filename: str, in_archive: bool
723
714
  ) -> str:
724
715
  if in_archive:
725
- return os.path.join(cls._code_dir, filename)
716
+ return filename
726
717
  return os.path.join(get_metaflow_root(), filename)
727
718
 
728
719
  @classmethod
@@ -841,38 +832,37 @@ class MetaflowCodeContentV1Base(MetaflowCodeContent, version_id=1):
841
832
  return None
842
833
 
843
834
  @classmethod
844
- def get_archive_content_members_impl(
835
+ def get_archive_content_names_impl(
845
836
  cls,
846
837
  mfcontent_info: Optional[Dict[str, Any]],
847
838
  archive: Any,
848
839
  content_types: Optional[int] = None,
849
840
  packaging_backend: Type[PackagingBackend] = TarPackagingBackend,
850
- ) -> List[Any]:
841
+ ) -> List[str]:
851
842
  to_return = []
852
843
  module_content = set(mfcontent_info.get("module_files", []))
853
- for member in packaging_backend.cls_list_members(archive):
854
- filename = packaging_backend.cls_member_name(member)
844
+ for filename in packaging_backend.cls_list_members(archive):
855
845
  if filename.startswith(cls._other_dir) and (
856
846
  content_types & ContentType.OTHER_CONTENT.value
857
847
  ):
858
- to_return.append(member)
848
+ to_return.append(filename)
859
849
  elif filename.startswith(cls._code_dir):
860
850
  # Special case for marker which is a other content even if in code.
861
- if filename == MFCONTENT_MARKER:
851
+ if filename == f"{cls._code_dir}/{MFCONTENT_MARKER}":
862
852
  if content_types & ContentType.OTHER_CONTENT.value:
863
- to_return.append(member)
853
+ to_return.append(filename)
864
854
  else:
865
855
  continue
866
856
  # Here it is either module or code
867
857
  if os.path.join(cls._code_dir, filename) in module_content:
868
858
  if content_types & ContentType.MODULE_CONTENT.value:
869
- to_return.append(member)
859
+ to_return.append(filename)
870
860
  elif content_types & ContentType.CODE_CONTENT.value:
871
- to_return.append(member)
861
+ to_return.append(filename)
872
862
  else:
873
863
  if content_types & ContentType.USER_CONTENT.value:
874
864
  # Everything else is user content
875
- to_return.append(member)
865
+ to_return.append(filename)
876
866
  return to_return
877
867
 
878
868
  @classmethod
@@ -57,15 +57,6 @@ class PackagingBackend(ABC):
57
57
  """Open the archive from the given content."""
58
58
  pass
59
59
 
60
- @classmethod
61
- @abstractmethod
62
- def cls_member_name(cls, member: Union[Any, str]) -> str:
63
- """
64
- Returns the name of the member as a string.
65
- This is used to ensure consistent naming across different archive formats.
66
- """
67
- pass
68
-
69
60
  @classmethod
70
61
  @abstractmethod
71
62
  def cls_has_member(cls, archive: Any, name: str) -> bool:
@@ -81,20 +72,14 @@ class PackagingBackend(ABC):
81
72
  def cls_extract_members(
82
73
  cls,
83
74
  archive: Any,
84
- members: Optional[List[Any]] = None,
75
+ members: Optional[List[str]] = None,
85
76
  dest_dir: str = ".",
86
77
  ) -> None:
87
78
  pass
88
79
 
89
80
  @classmethod
90
81
  @abstractmethod
91
- def cls_list_names(cls, archive: Any) -> Optional[List[str]]:
92
- pass
93
-
94
- @classmethod
95
- @abstractmethod
96
- def cls_list_members(cls, archive: Any) -> Optional[List[Any]]:
97
- """List all members in the archive."""
82
+ def cls_list_members(cls, archive: Any) -> Optional[List[str]]:
98
83
  pass
99
84
 
100
85
  def has_member(self, name: str) -> bool:
@@ -108,17 +93,17 @@ class PackagingBackend(ABC):
108
93
  raise ValueError("Cannot get member from an uncreated archive")
109
94
 
110
95
  def extract_members(
111
- self, members: Optional[List[Any]] = None, dest_dir: str = "."
96
+ self, members: Optional[List[str]] = None, dest_dir: str = "."
112
97
  ) -> None:
113
98
  if self._archive:
114
99
  self.cls_extract_members(self._archive, members, dest_dir)
115
100
  else:
116
101
  raise ValueError("Cannot extract from an uncreated archive")
117
102
 
118
- def list_names(self) -> Optional[List[str]]:
103
+ def list_members(self) -> Optional[List[str]]:
119
104
  if self._archive:
120
- return self.cls_list_names(self._archive)
121
- raise ValueError("Cannot list names from an uncreated archive")
105
+ return self.cls_list_members(self._archive)
106
+ raise ValueError("Cannot list members from an uncreated archive")
122
107
 
123
108
  def __enter__(self):
124
109
  self.create()
@@ -1,7 +1,7 @@
1
1
  import tarfile
2
2
 
3
3
  from io import BytesIO
4
- from typing import Any, IO, List, Optional, Union
4
+ from typing import IO, List, Optional, Union
5
5
 
6
6
  from .backend import PackagingBackend
7
7
 
@@ -56,13 +56,6 @@ class TarPackagingBackend(PackagingBackend):
56
56
  def cls_open(cls, content: IO[bytes]) -> tarfile.TarFile:
57
57
  return tarfile.open(fileobj=content, mode="r:gz")
58
58
 
59
- @classmethod
60
- def cls_member_name(cls, member: Union[tarfile.TarInfo, str]) -> str:
61
- """
62
- Returns the name of the member as a string.
63
- """
64
- return member.name if isinstance(member, tarfile.TarInfo) else member
65
-
66
59
  @classmethod
67
60
  def cls_has_member(cls, archive: tarfile.TarFile, name: str) -> bool:
68
61
  try:
@@ -83,17 +76,11 @@ class TarPackagingBackend(PackagingBackend):
83
76
  def cls_extract_members(
84
77
  cls,
85
78
  archive: tarfile.TarFile,
86
- members: Optional[List[Any]] = None,
79
+ members: Optional[List[str]] = None,
87
80
  dest_dir: str = ".",
88
81
  ) -> None:
89
82
  archive.extractall(path=dest_dir, members=members)
90
83
 
91
84
  @classmethod
92
- def cls_list_members(
93
- cls, archive: tarfile.TarFile
94
- ) -> Optional[List[tarfile.TarInfo]]:
95
- return archive.getmembers() or None
96
-
97
- @classmethod
98
- def cls_list_names(cls, archive: tarfile.TarFile) -> Optional[List[str]]:
85
+ def cls_list_members(cls, archive: tarfile.TarFile) -> Optional[List[str]]:
99
86
  return archive.getnames() or None
@@ -61,25 +61,23 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
61
61
  else:
62
62
  new_modules = []
63
63
 
64
- self._modules = {} # type: Dict[str, _ModuleInfo]
65
- # We do this explicitly module by module to harden it against misbehaving
66
- # modules like the one in:
67
- # https://github.com/Netflix/metaflow/issues/2512
68
- # We will silently ignore modules that are not well built.
69
- for name, mod in new_modules:
70
- try:
71
- minfo = _ModuleInfo(
72
- name,
73
- set(
74
- Path(p).resolve().as_posix()
75
- for p in getattr(mod, "__path__", [mod.__file__])
76
- ),
77
- mod,
78
- True, # This is a Metaflow module (see filter below)
79
- )
80
- except:
81
- continue
82
- self._modules[name] = minfo
64
+ self._modules = {
65
+ name: _ModuleInfo(
66
+ name,
67
+ set(
68
+ Path(p).resolve().as_posix()
69
+ for p in getattr(mod, "__path__", [mod.__file__])
70
+ ),
71
+ mod,
72
+ True, # This is a Metaflow module (see filter below)
73
+ )
74
+ for (name, mod) in new_modules
75
+ }
76
+
77
+ # Filter the modules
78
+ self._modules = {
79
+ name: info for name, info in self._modules.items() if criteria(info.module)
80
+ }
83
81
 
84
82
  # Contain metadata information regarding the distributions packaged.
85
83
  # This allows Metaflow to "fake" distribution information when packaged
@@ -357,14 +355,16 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
357
355
  )
358
356
  yield json.dumps(self.create_mfcontent_info()).encode(
359
357
  "utf-8"
360
- ), MFCONTENT_MARKER
358
+ ), os.path.join(self._code_dir, MFCONTENT_MARKER)
361
359
  else:
362
360
  for k in self._other_content.keys():
363
361
  yield "<generated %s content>" % (os.path.basename(k)), k
364
362
  yield "<generated %s content>" % (
365
363
  os.path.basename(self._dist_info_file)
366
364
  ), os.path.join(self._other_dir, self._dist_info_file)
367
- yield "<generated %s content>" % MFCONTENT_MARKER, MFCONTENT_MARKER
365
+ yield "<generated %s content>" % MFCONTENT_MARKER, os.path.join(
366
+ self._code_dir, MFCONTENT_MARKER
367
+ )
368
368
 
369
369
  def _metaflow_distribution_files(self) -> Generator[Tuple[str, str], None, None]:
370
370
  debug.package_exec("Including Metaflow from '%s'" % self._metaflow_root)
@@ -58,21 +58,38 @@ class ArgoClient(object):
58
58
  json.loads(e.body)["message"] if e.body is not None else e.reason
59
59
  )
60
60
 
61
- def get_workflow_templates(self):
61
+ def get_workflow_templates(self, page_size=100):
62
62
  client = self._client.get()
63
- try:
64
- return client.CustomObjectsApi().list_namespaced_custom_object(
65
- group=self._group,
66
- version=self._version,
67
- namespace=self._namespace,
68
- plural="workflowtemplates",
69
- )["items"]
70
- except client.rest.ApiException as e:
71
- if e.status == 404:
72
- return None
73
- raise ArgoClientException(
74
- json.loads(e.body)["message"] if e.body is not None else e.reason
75
- )
63
+ continue_token = None
64
+
65
+ while True:
66
+ try:
67
+ params = {"limit": page_size}
68
+ if continue_token:
69
+ params["_continue"] = continue_token
70
+
71
+ response = client.CustomObjectsApi().list_namespaced_custom_object(
72
+ group=self._group,
73
+ version=self._version,
74
+ namespace=self._namespace,
75
+ plural="workflowtemplates",
76
+ **params,
77
+ )
78
+
79
+ for item in response.get("items", []):
80
+ yield item
81
+
82
+ metadata = response.get("metadata", {})
83
+ continue_token = metadata.get("continue")
84
+
85
+ if not continue_token:
86
+ break
87
+ except client.rest.ApiException as e:
88
+ if e.status == 404:
89
+ return None
90
+ raise ArgoClientException(
91
+ json.loads(e.body)["message"] if e.body is not None else e.reason
92
+ )
76
93
 
77
94
  def register_workflow_template(self, name, workflow_template):
78
95
  # Unfortunately, Kubernetes client does not handle optimistic
@@ -216,23 +216,14 @@ class ArgoWorkflows(object):
216
216
  return name.replace(".", "-")
217
217
 
218
218
  @staticmethod
219
- def list_templates(flow_name, all=False):
219
+ def list_templates(flow_name, all=False, page_size=100):
220
220
  client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
221
221
 
222
- templates = client.get_workflow_templates()
223
- if templates is None:
224
- return []
225
-
226
- template_names = [
227
- template["metadata"]["name"]
228
- for template in templates
229
- if all
230
- or flow_name
231
- == template["metadata"]
232
- .get("annotations", {})
233
- .get("metaflow/flow_name", None)
234
- ]
235
- return template_names
222
+ for template in client.get_workflow_templates(page_size=page_size):
223
+ if all or flow_name == template["metadata"].get("annotations", {}).get(
224
+ "metaflow/flow_name", None
225
+ ):
226
+ yield template["metadata"]["name"]
236
227
 
237
228
  @staticmethod
238
229
  def delete(name):
@@ -3286,8 +3277,8 @@ class ArgoWorkflows(object):
3286
3277
  Trigger().template(
3287
3278
  TriggerTemplate(self.name)
3288
3279
  # Trigger a deployed workflow template
3289
- .k8s_trigger(
3290
- StandardK8STrigger()
3280
+ .argo_workflow_trigger(
3281
+ ArgoWorkflowTrigger()
3291
3282
  .source(
3292
3283
  {
3293
3284
  "resource": {
@@ -4265,10 +4256,6 @@ class TriggerTemplate(object):
4265
4256
  self.payload = tree()
4266
4257
  self.payload["name"] = name
4267
4258
 
4268
- def k8s_trigger(self, k8s_trigger):
4269
- self.payload["k8s"] = k8s_trigger.to_json()
4270
- return self
4271
-
4272
4259
  def argo_workflow_trigger(self, argo_workflow_trigger):
4273
4260
  self.payload["argoWorkflow"] = argo_workflow_trigger.to_json()
4274
4261
  return self
@@ -4343,57 +4330,3 @@ class TriggerParameter(object):
4343
4330
 
4344
4331
  def __str__(self):
4345
4332
  return json.dumps(self.payload, indent=4)
4346
-
4347
-
4348
- class StandardK8STrigger(object):
4349
- # https://pkg.go.dev/github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1#StandardK8STrigger
4350
-
4351
- def __init__(self):
4352
- tree = lambda: defaultdict(tree)
4353
- self.payload = tree()
4354
- self.payload["operation"] = "create"
4355
-
4356
- def operation(self, operation):
4357
- self.payload["operation"] = operation
4358
- return self
4359
-
4360
- def group(self, group):
4361
- self.payload["group"] = group
4362
- return self
4363
-
4364
- def version(self, version):
4365
- self.payload["version"] = version
4366
- return self
4367
-
4368
- def resource(self, resource):
4369
- self.payload["resource"] = resource
4370
- return self
4371
-
4372
- def namespace(self, namespace):
4373
- self.payload["namespace"] = namespace
4374
- return self
4375
-
4376
- def source(self, source):
4377
- self.payload["source"] = source
4378
- return self
4379
-
4380
- def parameters(self, trigger_parameters):
4381
- if "parameters" not in self.payload:
4382
- self.payload["parameters"] = []
4383
- for trigger_parameter in trigger_parameters:
4384
- self.payload["parameters"].append(trigger_parameter.to_json())
4385
- return self
4386
-
4387
- def live_object(self, live_object=True):
4388
- self.payload["liveObject"] = live_object
4389
- return self
4390
-
4391
- def patch_strategy(self, patch_strategy):
4392
- self.payload["patchStrategy"] = patch_strategy
4393
- return self
4394
-
4395
- def to_json(self):
4396
- return self.payload
4397
-
4398
- def __str__(self):
4399
- return json.dumps(self.payload, indent=4)
@@ -1011,8 +1011,7 @@ def terminate(obj, run_id, authorize=None):
1011
1011
  )
1012
1012
  @click.pass_obj
1013
1013
  def list_workflow_templates(obj, all=None):
1014
- templates = ArgoWorkflows.list_templates(obj.flow.name, all)
1015
- for template_name in templates:
1014
+ for template_name in ArgoWorkflows.list_templates(obj.flow.name, all):
1016
1015
  obj.echo_always(template_name)
1017
1016
 
1018
1017
 
@@ -203,6 +203,38 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
203
203
 
204
204
  TYPE: ClassVar[Optional[str]] = "argo-workflows"
205
205
 
206
+ @classmethod
207
+ def list_deployed_flows(cls, flow_name: Optional[str] = None):
208
+ """
209
+ List all deployed Argo Workflow templates.
210
+
211
+ Parameters
212
+ ----------
213
+ flow_name : str, optional, default None
214
+ If specified, only list deployed flows for this specific flow name.
215
+ If None, list all deployed flows.
216
+
217
+ Yields
218
+ ------
219
+ ArgoWorkflowsDeployedFlow
220
+ `ArgoWorkflowsDeployedFlow` objects representing deployed
221
+ workflow templates on Argo Workflows.
222
+ """
223
+ from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
224
+
225
+ # When flow_name is None, use all=True to get all templates
226
+ # When flow_name is specified, use all=False to filter by flow_name
227
+ all_templates = flow_name is None
228
+ for template_name in ArgoWorkflows.list_templates(
229
+ flow_name=flow_name, all=all_templates
230
+ ):
231
+ try:
232
+ deployed_flow = cls.from_deployment(template_name)
233
+ yield deployed_flow
234
+ except Exception:
235
+ # Skip templates that can't be converted to DeployedFlow objects
236
+ continue
237
+
206
238
  @classmethod
207
239
  def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
208
240
  """
@@ -56,6 +56,20 @@ class StepFunctionsDeployedFlow(DeployedFlow):
56
56
 
57
57
  TYPE: ClassVar[Optional[str]] = "step-functions"
58
58
 
59
+ @classmethod
60
+ def list_deployed_flows(cls, flow_name: Optional[str] = None):
61
+ """
62
+ This method is not currently implemented for Step Functions.
63
+
64
+ Raises
65
+ ------
66
+ NotImplementedError
67
+ This method is not implemented for Step Functions.
68
+ """
69
+ raise NotImplementedError(
70
+ "list_deployed_flows is not implemented for StepFunctions"
71
+ )
72
+
59
73
  @classmethod
60
74
  def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
61
75
  """
@@ -98,7 +98,7 @@ class KubernetesDecorator(StepDecorator):
98
98
  the scheduled node should not have GPUs.
99
99
  gpu_vendor : str, default KUBERNETES_GPU_VENDOR
100
100
  The vendor of the GPUs to be used for this step.
101
- tolerations : List[str], default []
101
+ tolerations : List[Dict[str,str]], default []
102
102
  The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
103
103
  Kubernetes tolerations to use when launching pod in Kubernetes.
104
104
  labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
@@ -243,11 +243,9 @@ class CondaStepDecorator(StepDecorator):
243
243
  # Ensure local installation of Metaflow is visible to user code
244
244
  python_path = self.__class__._metaflow_home.name
245
245
  addl_env_vars = {}
246
- if self.__class__._addl_env_vars:
246
+ if self.__class__._addl_env_vars is not None:
247
247
  for key, value in self.__class__._addl_env_vars.items():
248
- if key.endswith(":"):
249
- addl_env_vars[key[:-1]] = value
250
- elif key == "PYTHONPATH":
248
+ if key == "PYTHONPATH":
251
249
  addl_env_vars[key] = os.pathsep.join([value, python_path])
252
250
  else:
253
251
  addl_env_vars[key] = value
@@ -43,7 +43,6 @@ from metaflow._vendor.click.types import (
43
43
  )
44
44
  from metaflow.decorators import add_decorator_options
45
45
  from metaflow.exception import MetaflowException
46
- from metaflow.flowspec import _FlowState
47
46
  from metaflow.includefile import FilePathClass
48
47
  from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
49
48
  from metaflow.parameters import JSONTypeClass, flow_context
@@ -172,6 +171,7 @@ def _lazy_load_command(
172
171
  _self,
173
172
  name: str,
174
173
  ):
174
+
175
175
  # Context is not used in get_command so we can pass None. Since we pin click,
176
176
  # this won't change from under us.
177
177
 
@@ -516,11 +516,6 @@ class MetaflowAPI(object):
516
516
  # Note that if CLICK_API_PROCESS_CONFIG is False, we still do this because
517
517
  # it will init all parameters (config_options will be None)
518
518
  # We ignore any errors if we don't check the configs in the click API.
519
-
520
- # Init all values in the flow mutators and then process them
521
- for decorator in self._flow_cls._flow_state.get(_FlowState.FLOW_MUTATORS, []):
522
- decorator.external_init()
523
-
524
519
  new_cls = self._flow_cls._process_config_decorators(
525
520
  config_options, process_configs=CLICK_API_PROCESS_CONFIG
526
521
  )
@@ -546,16 +541,14 @@ def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
546
541
 
547
542
  for each_param in cmd_obj.params:
548
543
  if isinstance(each_param, click.Argument):
549
- (
550
- arg_params_sigs[each_param.name],
551
- annotations[each_param.name],
552
- ) = get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
544
+ arg_params_sigs[each_param.name], annotations[each_param.name] = (
545
+ get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
546
+ )
553
547
  arg_parameters[each_param.name] = each_param
554
548
  elif isinstance(each_param, click.Option):
555
- (
556
- opt_params_sigs[each_param.name],
557
- annotations[each_param.name],
558
- ) = get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
549
+ opt_params_sigs[each_param.name], annotations[each_param.name] = (
550
+ get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
551
+ )
559
552
  opt_parameters[each_param.name] = each_param
560
553
 
561
554
  defaults[each_param.name] = each_param.default
@@ -229,7 +229,7 @@ class DeployedFlowMeta(type):
229
229
  }
230
230
  )
231
231
 
232
- def _default_injected_method():
232
+ def _from_deployment_injected_method():
233
233
  def f(
234
234
  cls,
235
235
  identifier: str,
@@ -271,7 +271,7 @@ class DeployedFlowMeta(type):
271
271
  f.__name__ = "from_deployment"
272
272
  return f
273
273
 
274
- def _per_type_injected_method(method_name, impl):
274
+ def _per_type_from_deployment_injected_method(method_name, impl):
275
275
  def f(
276
276
  cls,
277
277
  identifier: str,
@@ -286,14 +286,88 @@ class DeployedFlowMeta(type):
286
286
  f.__name__ = method_name
287
287
  return f
288
288
 
289
- setattr(cls, "from_deployment", classmethod(_default_injected_method()))
289
+ def _list_deployed_flows_injected_method():
290
+ def f(
291
+ cls,
292
+ flow_name: Optional[str] = None,
293
+ impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
294
+ ):
295
+ """
296
+ List all deployed flows for the specified implementation.
297
+
298
+ Parameters
299
+ ----------
300
+ flow_name : str, optional, default None
301
+ If specified, only list deployed flows for this specific flow name.
302
+ If None, list all deployed flows.
303
+ impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
304
+ The default implementation to use if not specified
305
+
306
+ Yields
307
+ ------
308
+ DeployedFlow
309
+ `DeployedFlow` objects representing deployed flows.
310
+ """
311
+ if impl in allowed_providers:
312
+ return (
313
+ allowed_providers[impl]
314
+ .deployed_flow_type()
315
+ .list_deployed_flows(flow_name)
316
+ )
317
+ else:
318
+ raise ValueError(
319
+ f"No deployer '{impl}' exists; valid deployers are: "
320
+ f"{list(allowed_providers.keys())}"
321
+ )
322
+
323
+ f.__name__ = "list_deployed_flows"
324
+ return f
325
+
326
+ def _per_type_list_deployed_flows_injected_method(method_name, impl):
327
+ def f(
328
+ cls,
329
+ flow_name: Optional[str] = None,
330
+ ):
331
+ return (
332
+ allowed_providers[impl]
333
+ .deployed_flow_type()
334
+ .list_deployed_flows(flow_name)
335
+ )
336
+
337
+ f.__name__ = method_name
338
+ return f
339
+
340
+ setattr(
341
+ cls, "from_deployment", classmethod(_from_deployment_injected_method())
342
+ )
343
+ setattr(
344
+ cls,
345
+ "list_deployed_flows",
346
+ classmethod(_list_deployed_flows_injected_method()),
347
+ )
290
348
 
291
349
  for impl in allowed_providers:
292
- method_name = f"from_{impl}"
350
+ from_deployment_method_name = f"from_{impl}"
351
+ list_deployed_flows_method_name = f"list_{impl}"
352
+
293
353
  setattr(
294
354
  cls,
295
- method_name,
296
- classmethod(_per_type_injected_method(method_name, impl)),
355
+ from_deployment_method_name,
356
+ classmethod(
357
+ _per_type_from_deployment_injected_method(
358
+ from_deployment_method_name, impl
359
+ )
360
+ ),
361
+ )
362
+
363
+ setattr(
364
+ cls,
365
+ list_deployed_flows_method_name,
366
+ classmethod(
367
+ _per_type_list_deployed_flows_injected_method(
368
+ list_deployed_flows_method_name, impl
369
+ )
370
+ ),
297
371
  )
298
372
 
299
373
  return cls
@@ -152,20 +152,12 @@ class SubprocessManager(object):
152
152
  int
153
153
  The process ID of the subprocess.
154
154
  """
155
- env = env or {}
156
- installed_root = os.environ.get("METAFLOW_EXTRACTED_ROOT", get_metaflow_root())
157
-
158
- for k, v in MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
159
- installed_root
160
- ).items():
161
- if k.endswith(":"):
162
- # Override
163
- env[k[:-1]] = v
164
- elif k in env:
165
- env[k] = "%s:%s" % (v, env[k])
166
- else:
167
- env[k] = v
168
-
155
+ updated_env = MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
156
+ get_metaflow_root()
157
+ )
158
+ if updated_env:
159
+ env = env or {}
160
+ env.update(updated_env)
169
161
  command_obj = CommandManager(command, env, cwd)
170
162
  pid = command_obj.run(show_output=show_output)
171
163
  self.commands[pid] = command_obj
@@ -196,12 +188,12 @@ class SubprocessManager(object):
196
188
  int
197
189
  The process ID of the subprocess.
198
190
  """
199
- env = env or {}
200
- if "PYTHONPATH" in env:
201
- env["PYTHONPATH"] = "%s:%s" % (get_metaflow_root(), env["PYTHONPATH"])
202
- else:
203
- env["PYTHONPATH"] = get_metaflow_root()
204
-
191
+ updated_env = MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
192
+ get_metaflow_root()
193
+ )
194
+ if updated_env:
195
+ env = env or {}
196
+ env.update(updated_env)
205
197
  command_obj = CommandManager(command, env, cwd)
206
198
  pid = await command_obj.async_run()
207
199
  self.commands[pid] = command_obj
@@ -347,10 +347,8 @@ class MutableFlow:
347
347
  "Mutable flow adding flow decorator '%s'" % deco_type
348
348
  )
349
349
 
350
- # self._flow_cls._flow_decorators is a dictionary of form :
351
- # <deco_name> : [deco_instance, deco_instance, ...]
352
350
  existing_deco = [
353
- d for d in self._flow_cls._flow_decorators if d == flow_deco.name
351
+ d for d in self._flow_cls._flow_decorators if d.name == flow_deco.name
354
352
  ]
355
353
 
356
354
  if flow_deco.allow_multiple or not existing_deco:
metaflow/version.py CHANGED
@@ -1 +1 @@
1
- metaflow_version = "2.16.4.6"
1
+ metaflow_version = "2.16.5.1"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ob-metaflow
3
- Version: 2.16.4.6
3
+ Version: 2.16.5.1
4
4
  Summary: Metaflow: More AI and ML, Less Engineering
5
5
  Author: Netflix, Outerbounds & the Metaflow Community
6
6
  Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
12
12
  Requires-Dist: pylint
13
13
  Requires-Dist: kubernetes
14
14
  Provides-Extra: stubs
15
- Requires-Dist: metaflow-stubs==2.16.4.6; extra == "stubs"
15
+ Requires-Dist: metaflow-stubs==2.16.5.1; extra == "stubs"
16
16
  Dynamic: author
17
17
  Dynamic: author-email
18
18
  Dynamic: description
@@ -19,7 +19,7 @@ metaflow/meta_files.py,sha256=vlgJHI8GJUKzXoxdrVoH8yyCF5bhFgwYemUgnyd1wgM,342
19
19
  metaflow/metaflow_config.py,sha256=xEAsJ-Fa-fkDq1hzI72Hx9BC00-S1yQoqgSYG9K46AY,24267
20
20
  metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
21
21
  metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
22
- metaflow/metaflow_environment.py,sha256=20PIhA5R_rJneNj8f8UaWRmznGRPcEd6hP7goj_rc1s,11477
22
+ metaflow/metaflow_environment.py,sha256=YVq2j8cK6JjbmBkPXoTBNiAw29H1TVpDsnpVN-ZTBR4,11210
23
23
  metaflow/metaflow_git.py,sha256=Pb_VtvQzcjpuuM7UfC2u5kz85EbPMUfspl2UrPWBQMM,3647
24
24
  metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
25
25
  metaflow/metaflow_version.py,sha256=KJJAxhOMY28DaavMpvJUzvw-G6MI-29Fi2A6AEcQpok,7495
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
36
36
  metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
37
37
  metaflow/util.py,sha256=g2SOU_CRzJLgDM_UGF9QDMANMAIHAsDRXE6S76_YzsY,14594
38
38
  metaflow/vendor.py,sha256=EDZokNMrx1PU07jNMiWFMFtC7TL03pMXZ1kKn13k-2g,5139
39
- metaflow/version.py,sha256=0bLVnyslb9FTRAj4LmHwCMFnk1pe0Cy54ArUn2Uo5Tk,30
39
+ metaflow/version.py,sha256=e4483KufdR8M9TId-cCHXHvz60Of-pT7QBtPrqTJNGk,30
40
40
  metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
41
41
  metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
42
42
  metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -57,7 +57,7 @@ metaflow/_vendor/click/termui.py,sha256=G7QBEKIepRIGLvNdGwBTYiEtSImRxvTO_AglVpyH
57
57
  metaflow/_vendor/click/testing.py,sha256=EUEsDUqNXFgCLhZ0ZFOROpaVDA5I_rijwnNPE6qICgA,12854
58
58
  metaflow/_vendor/click/types.py,sha256=wuubik4VqgqAw5dvbYFkDt-zSAx97y9TQXuXcVaRyQA,25045
59
59
  metaflow/_vendor/click/utils.py,sha256=4VEcJ7iEHwjnFuzEuRtkT99o5VG3zqSD7Q2CVzv13WU,15940
60
- metaflow/_vendor/imghdr/__init__.py,sha256=wbtS6ggW20xfNCBlWrSkdvs4KXCRQekfGla5xv4obVY,4398
60
+ metaflow/_vendor/imghdr/__init__.py,sha256=s4fE82lm6KaADg6_Xwl6xLeyW9NZDaIEbVoVFsg_kKI,4332
61
61
  metaflow/_vendor/importlib_metadata/__init__.py,sha256=dH_kBRM61VLiIV0zuU5macqk3XYY-ljq_9gt8ZcJDMI,30525
62
62
  metaflow/_vendor/importlib_metadata/_adapters.py,sha256=B6fCi5-8mLVDFUZj3krI5nAo-mKp1dH_qIavyIyFrJs,1862
63
63
  metaflow/_vendor/importlib_metadata/_collections.py,sha256=CJ0OTCHIjWA0ZIVS4voORAsn2R4R2cQBEtPsZEJpASY,743
@@ -140,7 +140,7 @@ metaflow/cli_components/run_cmds.py,sha256=xULZQ2UrxLNsWjQIZd38EbOGNBw8UJT7w_T19
140
140
  metaflow/cli_components/step_cmd.py,sha256=zGJgTv7wxrv34nWDi__CHaC2eS6kItR95EdVGJX803w,4766
141
141
  metaflow/cli_components/utils.py,sha256=gpoDociadjnJD7MuiJup_MDR02ZJjjleejr0jPBu29c,6057
142
142
  metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
143
- metaflow/client/core.py,sha256=tj2PuqQt1RXg8GuyLQ_WRuxYEvTaDyi_lW18YGwWvAQ,83714
143
+ metaflow/client/core.py,sha256=h_sPTG36U2UPiZRRwL-oQ5EAeJ6ijfXaibhinb22Hj0,83548
144
144
  metaflow/client/filecache.py,sha256=Wy0yhhCqC1JZgebqi7z52GCwXYnkAqMZHTtxThvwBgM,15229
145
145
  metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
146
146
  metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
@@ -160,7 +160,7 @@ metaflow/datastore/exceptions.py,sha256=r7Ab5FvHIzyFh6kwiptA1lO5nLqWg0xRBoeYGefv
160
160
  metaflow/datastore/flow_datastore.py,sha256=rDMEHdYwub1PwLp2uaK-8CHdd8hiwxqeELXzsUfuqZs,10250
161
161
  metaflow/datastore/inputs.py,sha256=i43dXr2xvgtsgKMO9allgCR18bk80GeayeQFyUTH36w,449
162
162
  metaflow/datastore/task_datastore.py,sha256=bEti1X5rvKBQykfvsoAnmHXel_itZbI5MeLrEpWPHPQ,35059
163
- metaflow/extension_support/__init__.py,sha256=xLkhh0IzQ70IfF9j6MopMY02SMpEVI_eguksIOEXbDs,52522
163
+ metaflow/extension_support/__init__.py,sha256=Y0DoVZ1Eh4W7aO9DMIXrI8xUoJFP40Oux2gVWv-xoYY,52488
164
164
  metaflow/extension_support/_empty_file.py,sha256=vz61sSExf5DZH3JCqdfwkp7l_NrJR8HV175kG82yUas,133
165
165
  metaflow/extension_support/cmd.py,sha256=hk8iBUUINqvKCDxInKgWpum8ThiRZtHSJP7qBASHzl8,5711
166
166
  metaflow/extension_support/integrations.py,sha256=AWAh-AZ-vo9IxuAVEjGw3s8p_NMm2DKHYx10oC51gPU,5506
@@ -174,13 +174,13 @@ metaflow/mflog/mflog.py,sha256=VebXxqitOtNAs7VJixnNfziO_i_urG7bsJ5JiB5IXgY,4370
174
174
  metaflow/mflog/save_logs.py,sha256=4p1OwozsHJBslOzAf0wUq2XPMNpEOZWM68MgWzh_jJY,2330
175
175
  metaflow/mflog/save_logs_periodically.py,sha256=2Uvk9hi-zlCqXxOQoXmmjH1SCugfw6eG6w70WgfI-ho,1256
176
176
  metaflow/mflog/tee.py,sha256=wTER15qeHuiRpCkOqo-bd-r3Gj-EVlf3IvWRCA4beW4,887
177
- metaflow/package/__init__.py,sha256=1ckiqYx4mJ0Rtp5vr5ta3fIx1LuCSkTdoHy0Gzx_Lwo,26384
178
- metaflow/packaging_sys/__init__.py,sha256=HzpznaW1T9xppCIsxI7MzcTIu6QasyDZ3pJ5fRjzuao,31994
179
- metaflow/packaging_sys/backend.py,sha256=uiJNMTTh3RV9_Zqkd5MBd2TnukfxpY2OEGBOmDroeTM,3534
177
+ metaflow/package/__init__.py,sha256=O2fCTo7Ktm4TnjA6blkepUWs8sf3fTY1q5y8I5nk8Ak,25962
178
+ metaflow/packaging_sys/__init__.py,sha256=VIDa-J_UNyD-Axf5Iuom6SUssQfqAcTq4ezHU6_uqIM,31529
179
+ metaflow/packaging_sys/backend.py,sha256=b3OX42Io8xP5l56zhXZxi-VLqXBr-B2q8AM7cWZvmMw,3104
180
180
  metaflow/packaging_sys/distribution_support.py,sha256=VvikZBCH8N1TBZZ2Twk8jH1brmiinKWCD3y_aFqBsIw,4937
181
- metaflow/packaging_sys/tar_backend.py,sha256=nFWuXiwYjWQkFdV2KaZ6gazNVvtY84Eqsh9txhU3pNY,3010
181
+ metaflow/packaging_sys/tar_backend.py,sha256=EYZD5iGEzPoO4L6IQhmrZC1QlaOPV471SBKyOYBS2XU,2593
182
182
  metaflow/packaging_sys/utils.py,sha256=x8SVglJvY5mIAilS7MqZi2PpMr6IEyi6RCg3l8hN3G0,2972
183
- metaflow/packaging_sys/v1.py,sha256=kbNK0-pDAv3QJPZ789TE0UirGXcHbXkVQiyNT815H7A,20631
183
+ metaflow/packaging_sys/v1.py,sha256=_YlVPR7oSYt7B8dHOS0Fb6ZPrxaSZMSQyRhdN8CL8ZY,20483
184
184
  metaflow/plugins/__init__.py,sha256=yFxjJOlnfap7tQMNgSgaso2tl_zr1BcWL7KoUKk4c9Y,8617
185
185
  metaflow/plugins/catch_decorator.py,sha256=UOM2taN_OL2RPpuJhwEOA9ZALm0-hHD0XS2Hn2GUev0,4061
186
186
  metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
@@ -211,13 +211,13 @@ metaflow/plugins/airflow/sensors/base_sensor.py,sha256=s-OQBfPWZ_T3wn96Ua59CCEj1
211
211
  metaflow/plugins/airflow/sensors/external_task_sensor.py,sha256=zhYlrZnXT20KW8-fVk0fCNtTyNiKJB5PMVASacu30r0,6034
212
212
  metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqNA90nARrjjjEEk_x4,3275
213
213
  metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
214
- metaflow/plugins/argo/argo_client.py,sha256=A1kI9rjVjCadDsBscZ2Wk8xRBI6GNgWV6SU7TyrdfrI,16530
214
+ metaflow/plugins/argo/argo_client.py,sha256=jLz0FjCTBvFLZt-8lZcMQhDcInhgEcGdPrU2Gvh67zA,17080
215
215
  metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
216
- metaflow/plugins/argo/argo_workflows.py,sha256=fM2mI1drSxmUpEV-P8f7A8CXtYF3gGyrUBU7vbc-va0,191426
217
- metaflow/plugins/argo/argo_workflows_cli.py,sha256=pfqYH0nqkgmkDcz3EPIz5yR_PSdd3CfLb92wzplS8fY,38810
216
+ metaflow/plugins/argo/argo_workflows.py,sha256=0tZ7tuOBsMxfbjYfgZXeCdMfIrnxkEm3jX7ruH2qArQ,189693
217
+ metaflow/plugins/argo/argo_workflows_cli.py,sha256=Le_GgvLVE1MhxQeOv-k5xp4L51tzQ6ZIq3_P-YEphIk,38784
218
218
  metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
219
219
  metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
220
- metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=7OiapcIM_r-aBkuIobhofgLC5NRJHC-p9bvBmxvhqoM,12500
220
+ metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=1qiY69jB2yq0ryIH2SKzFHgUXRYvr0VTXd9TDQWG-s0,13693
221
221
  metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
222
222
  metaflow/plugins/argo/exit_hooks.py,sha256=nh8IEkzAtQnbKVnh3N9CVnVKZB39Bjm3e0LFrACsLz8,6109
223
223
  metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
@@ -243,7 +243,7 @@ metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=tLIfDwgdcfBjkjm
243
243
  metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
244
244
  metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=jzDHYmgU_XvLffZDazR_1viow_1qQFblx9UKyjtoM_0,3788
245
245
  metaflow/plugins/aws/step_functions/step_functions_deployer.py,sha256=JKYtDhKivtXUWPklprZFzkqezh14loGDmk8mNk6QtpI,3714
246
- metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py,sha256=zDWmrOeCEL2_uBbmdmXbVpHVTjswwjEL_rOux6MAcRI,7303
246
+ metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py,sha256=KWIPpE4ebOYyyXlRVtJF6XMnYcyjMFTxdzJWbP0m5PA,7719
247
247
  metaflow/plugins/azure/__init__.py,sha256=GuuhTVC-zSdyAf79a1wiERMq0Zts7fwVT7t9fAf234A,100
248
248
  metaflow/plugins/azure/azure_credential.py,sha256=JmdGEbVzgxy8ucqnQDdTTI_atyMX9WSZUw3qYOo7RhE,2174
249
249
  metaflow/plugins/azure/azure_exceptions.py,sha256=NnbwpUC23bc61HZjJmeXztY0tBNn_Y_VpIpDDuYWIZ0,433
@@ -328,7 +328,7 @@ metaflow/plugins/kubernetes/kube_utils.py,sha256=jdFMGbEmIow-oli26v31W9CmbZXigx0
328
328
  metaflow/plugins/kubernetes/kubernetes.py,sha256=3ugZTwMniF3VBZ3yzfIgby69KgaVw-i-Cq2g2FHef4A,31164
329
329
  metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=qtWTQqp8i-hTKAA0RcJ_qeOuD8TieN3B5vuyYdnvEP4,14425
330
330
  metaflow/plugins/kubernetes/kubernetes_client.py,sha256=tuvXP-QKpdeSmzVolB2R_TaacOr5DIb0j642eKcjsiM,6491
331
- metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=BB5E_pl39vVc6IhDZm2QYhr-S9BJMmky0pkK5DcED5E,33268
331
+ metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=htawUXm3SDfxzAcjluKrdfMez0o9Lf6ni5_rqK38YJY,33278
332
332
  metaflow/plugins/kubernetes/kubernetes_job.py,sha256=xhucNJR7EpM-XMsfY0nt-BASbjo_T4vL_-tmQ_xHL1U,33284
333
333
  metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=ZZU5vsBe67NmGuxgXw6clf7kKRST7867AW6_t3fCD5g,43065
334
334
  metaflow/plugins/kubernetes/spot_metadata_cli.py,sha256=an0nWCxgflmqIPBCBrlb4m3DereDFFJBLt-KKhqcHc8,1670
@@ -338,7 +338,7 @@ metaflow/plugins/metadata_providers/local.py,sha256=Z0CXaGZJbAkj4II3WspJi-uCCtSh
338
338
  metaflow/plugins/metadata_providers/service.py,sha256=WL3GkEQlQk0syjSZ6iOnBSb3nRGfeUye95ySvLnMwhg,22953
339
339
  metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
340
340
  metaflow/plugins/pypi/bootstrap.py,sha256=8EWBdwOp5moXkTfLadn3ZOtPXoGftjOFD-c2W_rn77c,14998
341
- metaflow/plugins/pypi/conda_decorator.py,sha256=fXG9EvImP4Eqle_Trhb3tQhs40xTc4cxL_r7r0BlJzo,14064
341
+ metaflow/plugins/pypi/conda_decorator.py,sha256=ie0ftOcozloj_qdASOxYvycb-Zr1GJHedzEDnp6cl2w,13984
342
342
  metaflow/plugins/pypi/conda_environment.py,sha256=MFWHUykFXpBhEuvxRku2FV5dtiWH_ECZgnoq1PoF9Ik,25134
343
343
  metaflow/plugins/pypi/micromamba.py,sha256=UltfY8NmLphfZ-AbpaMFIdxIeOXLdTYDrMrabvPrYVU,17352
344
344
  metaflow/plugins/pypi/parsers.py,sha256=gpOOG2Ph95wI73MWCAi7XjpK0gYhv5k5YIGBs73QPuE,8556
@@ -356,13 +356,13 @@ metaflow/plugins/uv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
356
356
  metaflow/plugins/uv/bootstrap.py,sha256=1UmNnnR7I1YcOtjdAmhuiU23-vj7NimUk3C9QillBaE,4380
357
357
  metaflow/plugins/uv/uv_environment.py,sha256=AYZICrBEq3Bv-taXktJwu9DhKFxNooPFwlcH379EYMs,2719
358
358
  metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
359
- metaflow/runner/click_api.py,sha256=w0-47ntQD4Dd-LiSJ9vPSjlvW5YBniNSmlf95lcXs5E,24026
360
- metaflow/runner/deployer.py,sha256=U-hwf4gVzwUlXgnkfTW3y1daGXvo5eP4HTQZwb-vS0g,11058
359
+ metaflow/runner/click_api.py,sha256=DSxa5A0C_IHNug7fZlLpD_N99F_skDcAjTRx5YRMylY,23756
360
+ metaflow/runner/deployer.py,sha256=VSthZi1tAVnWkebVbVfhLuvbVojDrZB4r6sdiGmrMfU,13906
361
361
  metaflow/runner/deployer_impl.py,sha256=zTING0_fwP44JcGo69DuNrVut5KqdBVzYOM7MYTZgIY,7049
362
362
  metaflow/runner/metaflow_runner.py,sha256=uo3BzcAfZ67VT_f-TPe5ZHiWHn6uuojWusOMGksvX14,18178
363
363
  metaflow/runner/nbdeploy.py,sha256=Sp5w-6nCZwjHaRBHWxi8udya-RYnJOB76KNLjB4L7Gs,4166
364
364
  metaflow/runner/nbrun.py,sha256=LhJu-Teoi7wTkNxg0kpNPVXFxH_9P4lvtp0ysMEIFJ8,7299
365
- metaflow/runner/subprocess_manager.py,sha256=x-MtPpGGMQUkIbQ_oNOuR-45b91DFvsCJ0SPoFc0dF4,23028
365
+ metaflow/runner/subprocess_manager.py,sha256=0nAXG1PM7KXSIpu6ECeNMv5EfrqCymlIS6k25crnIBA,22792
366
366
  metaflow/runner/utils.py,sha256=fU4vPazBdi6ATAUW_DaBAQeVslRwrLT8Pn9s5wav3gg,10350
367
367
  metaflow/sidecar/__init__.py,sha256=1mmNpmQ5puZCpRmmYlCOeieZ4108Su9XQ4_EqF1FGOU,131
368
368
  metaflow/sidecar/sidecar.py,sha256=EspKXvPPNiyRToaUZ51PS5TT_PzrBNAurn_wbFnmGr0,1334
@@ -405,16 +405,16 @@ metaflow/user_configs/config_options.py,sha256=d3hKA6WRPe21PdTl7sBnxIp5sE6zBpRtg
405
405
  metaflow/user_configs/config_parameters.py,sha256=Loa5wu3vIs0SLyGhbOo8b88nWgCuZ09k24EqC_lI7n4,20890
406
406
  metaflow/user_decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
407
407
  metaflow/user_decorators/common.py,sha256=0u9NRLQ95TfJCjWVEOGT4MJ9WoQgAMBM9kJ2gJGlVjk,5362
408
- metaflow/user_decorators/mutable_flow.py,sha256=icF7XFCS5FdlW3OEL68ZbQOtTPhLsUycKcFKVOeK_vU,19435
408
+ metaflow/user_decorators/mutable_flow.py,sha256=y2FCTQVjTTeiptsztD26jdkU_LY_Z_kzMjTkDajo2rc,19303
409
409
  metaflow/user_decorators/mutable_step.py,sha256=-BY0UDXf_RCAEnC5JlLzEXGdiw1KD9oSrSxS_SWaB9Y,16791
410
410
  metaflow/user_decorators/user_flow_decorator.py,sha256=2yDwZq9QGv9W-7kEuKwa8o4ZkTvuHJ5ESz7VVrGViAI,9890
411
411
  metaflow/user_decorators/user_step_decorator.py,sha256=JYNGXONWCpzwn-_bF5WiAkof4Ii9tRS4xdK8ojSxG6M,26007
412
- ob_metaflow-2.16.4.6.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
413
- ob_metaflow-2.16.4.6.data/data/share/metaflow/devtools/Tiltfile,sha256=I55XTG4RBnrMfDcYRtREXqqS8T9bF8agkZq0DlvdFLk,21404
414
- ob_metaflow-2.16.4.6.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
415
- ob_metaflow-2.16.4.6.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
416
- ob_metaflow-2.16.4.6.dist-info/METADATA,sha256=Vgm6oN_6xrgMpIxnfVcNFzObs8Mdwjm1TGHY6EsIEEs,5935
417
- ob_metaflow-2.16.4.6.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
418
- ob_metaflow-2.16.4.6.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
419
- ob_metaflow-2.16.4.6.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
420
- ob_metaflow-2.16.4.6.dist-info/RECORD,,
412
+ ob_metaflow-2.16.5.1.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
413
+ ob_metaflow-2.16.5.1.data/data/share/metaflow/devtools/Tiltfile,sha256=I55XTG4RBnrMfDcYRtREXqqS8T9bF8agkZq0DlvdFLk,21404
414
+ ob_metaflow-2.16.5.1.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
415
+ ob_metaflow-2.16.5.1.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
416
+ ob_metaflow-2.16.5.1.dist-info/METADATA,sha256=RP_gv_lojBowJ10b8dycYO2DUdOV3ipD_Y5hIs-NDs4,5935
417
+ ob_metaflow-2.16.5.1.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
418
+ ob_metaflow-2.16.5.1.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
419
+ ob_metaflow-2.16.5.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
420
+ ob_metaflow-2.16.5.1.dist-info/RECORD,,