ob-metaflow 2.16.8.2rc2__py2.py3-none-any.whl → 2.17.0.1__py2.py3-none-any.whl

metaflow/packaging_sys/backend.py

@@ -57,6 +57,15 @@ class PackagingBackend(ABC):
         """Open the archive from the given content."""
         pass
 
+    @classmethod
+    @abstractmethod
+    def cls_member_name(cls, member: Union[Any, str]) -> str:
+        """
+        Returns the name of the member as a string.
+        This is used to ensure consistent naming across different archive formats.
+        """
+        pass
+
     @classmethod
     @abstractmethod
     def cls_has_member(cls, archive: Any, name: str) -> bool:
@@ -72,14 +81,20 @@ class PackagingBackend(ABC):
     def cls_extract_members(
         cls,
         archive: Any,
-        members: Optional[List[str]] = None,
+        members: Optional[List[Any]] = None,
         dest_dir: str = ".",
     ) -> None:
         pass
 
     @classmethod
     @abstractmethod
-    def cls_list_members(cls, archive: Any) -> Optional[List[str]]:
+    def cls_list_names(cls, archive: Any) -> Optional[List[str]]:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def cls_list_members(cls, archive: Any) -> Optional[List[Any]]:
+        """List all members in the archive."""
         pass
 
     def has_member(self, name: str) -> bool:
@@ -93,17 +108,17 @@ class PackagingBackend(ABC):
         raise ValueError("Cannot get member from an uncreated archive")
 
     def extract_members(
-        self, members: Optional[List[str]] = None, dest_dir: str = "."
+        self, members: Optional[List[Any]] = None, dest_dir: str = "."
     ) -> None:
         if self._archive:
             self.cls_extract_members(self._archive, members, dest_dir)
         else:
             raise ValueError("Cannot extract from an uncreated archive")
 
-    def list_members(self) -> Optional[List[str]]:
+    def list_names(self) -> Optional[List[str]]:
         if self._archive:
-            return self.cls_list_members(self._archive)
-        raise ValueError("Cannot list members from an uncreated archive")
+            return self.cls_list_names(self._archive)
+        raise ValueError("Cannot list names from an uncreated archive")
 
     def __enter__(self):
         self.create()

metaflow/packaging_sys/tar_backend.py

@@ -1,7 +1,7 @@
 import tarfile
 
 from io import BytesIO
-from typing import IO, List, Optional, Union
+from typing import Any, IO, List, Optional, Union
 
 from .backend import PackagingBackend
 
@@ -56,6 +56,13 @@ class TarPackagingBackend(PackagingBackend):
     def cls_open(cls, content: IO[bytes]) -> tarfile.TarFile:
         return tarfile.open(fileobj=content, mode="r:gz")
 
+    @classmethod
+    def cls_member_name(cls, member: Union[tarfile.TarInfo, str]) -> str:
+        """
+        Returns the name of the member as a string.
+        """
+        return member.name if isinstance(member, tarfile.TarInfo) else member
+
     @classmethod
     def cls_has_member(cls, archive: tarfile.TarFile, name: str) -> bool:
         try:
@@ -76,11 +83,17 @@ class TarPackagingBackend(PackagingBackend):
     def cls_extract_members(
         cls,
         archive: tarfile.TarFile,
-        members: Optional[List[str]] = None,
+        members: Optional[List[Any]] = None,
         dest_dir: str = ".",
     ) -> None:
         archive.extractall(path=dest_dir, members=members)
 
     @classmethod
-    def cls_list_members(cls, archive: tarfile.TarFile) -> Optional[List[str]]:
+    def cls_list_members(
+        cls, archive: tarfile.TarFile
+    ) -> Optional[List[tarfile.TarInfo]]:
+        return archive.getmembers() or None
+
+    @classmethod
+    def cls_list_names(cls, archive: tarfile.TarFile) -> Optional[List[str]]:
         return archive.getnames() or None

metaflow/packaging_sys/v1.py

@@ -61,23 +61,25 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
         else:
             new_modules = []
 
-        self._modules = {
-            name: _ModuleInfo(
-                name,
-                set(
-                    Path(p).resolve().as_posix()
-                    for p in getattr(mod, "__path__", [mod.__file__])
-                ),
-                mod,
-                True,  # This is a Metaflow module (see filter below)
-            )
-            for (name, mod) in new_modules
-        }
-
-        # Filter the modules
-        self._modules = {
-            name: info for name, info in self._modules.items() if criteria(info.module)
-        }
+        self._modules = {}  # type: Dict[str, _ModuleInfo]
+        # We do this explicitly module by module to harden it against misbehaving
+        # modules like the one in:
+        # https://github.com/Netflix/metaflow/issues/2512
+        # We will silently ignore modules that are not well built.
+        for name, mod in new_modules:
+            try:
+                minfo = _ModuleInfo(
+                    name,
+                    set(
+                        Path(p).resolve().as_posix()
+                        for p in getattr(mod, "__path__", [mod.__file__])
+                    ),
+                    mod,
+                    True,  # This is a Metaflow module (see filter below)
+                )
+            except:
+                continue
+            self._modules[name] = minfo
 
         # Contain metadata information regarding the distributions packaged.
         # This allows Metaflow to "fake" distribution information when packaged
@@ -355,16 +357,14 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
                 )
                 yield json.dumps(self.create_mfcontent_info()).encode(
                     "utf-8"
-                ), os.path.join(self._code_dir, MFCONTENT_MARKER)
+                ), MFCONTENT_MARKER
             else:
                 for k in self._other_content.keys():
                     yield "<generated %s content>" % (os.path.basename(k)), k
                 yield "<generated %s content>" % (
                     os.path.basename(self._dist_info_file)
                 ), os.path.join(self._other_dir, self._dist_info_file)
-                yield "<generated %s content>" % MFCONTENT_MARKER, os.path.join(
-                    self._code_dir, MFCONTENT_MARKER
-                )
+                yield "<generated %s content>" % MFCONTENT_MARKER, MFCONTENT_MARKER
 
     def _metaflow_distribution_files(self) -> Generator[Tuple[str, str], None, None]:
         debug.package_exec("Including Metaflow from '%s'" % self._metaflow_root)

metaflow/plugins/argo/argo_client.py

@@ -58,21 +58,38 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
-    def get_workflow_templates(self):
+    def get_workflow_templates(self, page_size=100):
         client = self._client.get()
-        try:
-            return client.CustomObjectsApi().list_namespaced_custom_object(
-                group=self._group,
-                version=self._version,
-                namespace=self._namespace,
-                plural="workflowtemplates",
-            )["items"]
-        except client.rest.ApiException as e:
-            if e.status == 404:
-                return None
-            raise ArgoClientException(
-                json.loads(e.body)["message"] if e.body is not None else e.reason
-            )
+        continue_token = None
+
+        while True:
+            try:
+                params = {"limit": page_size}
+                if continue_token:
+                    params["_continue"] = continue_token
+
+                response = client.CustomObjectsApi().list_namespaced_custom_object(
+                    group=self._group,
+                    version=self._version,
+                    namespace=self._namespace,
+                    plural="workflowtemplates",
+                    **params,
+                )
+
+                for item in response.get("items", []):
+                    yield item
+
+                metadata = response.get("metadata", {})
+                continue_token = metadata.get("continue")
+
+                if not continue_token:
+                    break
+            except client.rest.ApiException as e:
+                if e.status == 404:
+                    return None
+                raise ArgoClientException(
+                    json.loads(e.body)["message"] if e.body is not None else e.reason
+                )
 
     def register_workflow_template(self, name, workflow_template):
         # Unfortunately, Kubernetes client does not handle optimistic

metaflow/plugins/argo/argo_workflows.py

@@ -216,23 +216,14 @@ class ArgoWorkflows(object):
         return name.replace(".", "-")
 
     @staticmethod
-    def list_templates(flow_name, all=False):
+    def list_templates(flow_name, all=False, page_size=100):
         client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
 
-        templates = client.get_workflow_templates()
-        if templates is None:
-            return []
-
-        template_names = [
-            template["metadata"]["name"]
-            for template in templates
-            if all
-            or flow_name
-            == template["metadata"]
-            .get("annotations", {})
-            .get("metaflow/flow_name", None)
-        ]
-        return template_names
+        for template in client.get_workflow_templates(page_size=page_size):
+            if all or flow_name == template["metadata"].get("annotations", {}).get(
+                "metaflow/flow_name", None
+            ):
+                yield template["metadata"]["name"]
 
     @staticmethod
     def delete(name):
@@ -271,6 +262,7 @@ class ArgoWorkflows(object):
                     flow_name=flow_name, run_id=name
                 )
             )
+        return True
 
     @staticmethod
     def get_workflow_status(flow_name, name):
@@ -965,11 +957,6 @@ class ArgoWorkflows(object):
                 dag_task = DAGTask(self._sanitize(node.name)).template(
                     self._sanitize(node.name)
                 )
-            if node.type == "split-switch":
-                raise ArgoWorkflowsException(
-                    "Deploying flows with switch statement "
-                    "to Argo Workflows is not supported currently."
-                )
             elif (
                 node.is_inside_foreach
                 and self.graph[node.in_funcs[0]].type == "foreach"
@@ -3291,8 +3278,8 @@ class ArgoWorkflows(object):
                     Trigger().template(
                         TriggerTemplate(self.name)
                         # Trigger a deployed workflow template
-                        .argo_workflow_trigger(
-                            ArgoWorkflowTrigger()
+                        .k8s_trigger(
+                            StandardK8STrigger()
                             .source(
                                 {
                                     "resource": {
@@ -4270,6 +4257,10 @@ class TriggerTemplate(object):
         self.payload = tree()
         self.payload["name"] = name
 
+    def k8s_trigger(self, k8s_trigger):
+        self.payload["k8s"] = k8s_trigger.to_json()
+        return self
+
     def argo_workflow_trigger(self, argo_workflow_trigger):
         self.payload["argoWorkflow"] = argo_workflow_trigger.to_json()
         return self
@@ -4344,3 +4335,57 @@ class TriggerParameter(object):
 
     def __str__(self):
         return json.dumps(self.payload, indent=4)
+
+
+class StandardK8STrigger(object):
+    # https://pkg.go.dev/github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1#StandardK8STrigger
+
+    def __init__(self):
+        tree = lambda: defaultdict(tree)
+        self.payload = tree()
+        self.payload["operation"] = "create"
+
+    def operation(self, operation):
+        self.payload["operation"] = operation
+        return self
+
+    def group(self, group):
+        self.payload["group"] = group
+        return self
+
+    def version(self, version):
+        self.payload["version"] = version
+        return self
+
+    def resource(self, resource):
+        self.payload["resource"] = resource
+        return self
+
+    def namespace(self, namespace):
+        self.payload["namespace"] = namespace
+        return self
+
+    def source(self, source):
+        self.payload["source"] = source
+        return self
+
+    def parameters(self, trigger_parameters):
+        if "parameters" not in self.payload:
+            self.payload["parameters"] = []
+        for trigger_parameter in trigger_parameters:
+            self.payload["parameters"].append(trigger_parameter.to_json())
+        return self
+
+    def live_object(self, live_object=True):
+        self.payload["liveObject"] = live_object
+        return self
+
+    def patch_strategy(self, patch_strategy):
+        self.payload["patchStrategy"] = patch_strategy
+        return self
+
+    def to_json(self):
+        return self.payload
+
+    def __str__(self):
+        return json.dumps(self.payload, indent=4)