metaflow 2.12.10__py2.py3-none-any.whl → 2.12.11__py2.py3-none-any.whl

metaflow/client/core.py

@@ -391,9 +391,9 @@ class MetaflowObject(object):
                     _object=obj,
                     _parent=self,
                     _namespace_check=self._namespace_check,
-                    _current_namespace=self._current_namespace
-                    if self._namespace_check
-                    else None,
+                    _current_namespace=(
+                        self._current_namespace if self._namespace_check else None
+                    ),
                 )
                 for obj in unfiltered_children
             ),
@@ -506,9 +506,9 @@ class MetaflowObject(object):
                 _object=obj,
                 _parent=self,
                 _namespace_check=self._namespace_check,
-                _current_namespace=self._current_namespace
-                if self._namespace_check
-                else None,
+                _current_namespace=(
+                    self._current_namespace if self._namespace_check else None
+                ),
             )
         else:
             raise KeyError(id)

metaflow/client/filecache.py

@@ -6,6 +6,8 @@ import time
 from tempfile import NamedTemporaryFile
 from hashlib import sha1
 
+from urllib.parse import urlparse
+
 from metaflow.datastore import FlowDataStore
 from metaflow.datastore.content_addressed_store import BlobCache
 from metaflow.exception import MetaflowException
@@ -83,7 +85,6 @@ class FileCache(object):
     def get_log_legacy(
         self, ds_type, location, logtype, attempt, flow_name, run_id, step_name, task_id
     ):
-
         ds_cls = self._get_datastore_storage_impl(ds_type)
         ds_root = ds_cls.path_join(*ds_cls.path_split(location)[:-5])
         cache_id = self._flow_ds_id(ds_type, ds_root, flow_name)
@@ -311,12 +312,24 @@ class FileCache(object):
 
     @staticmethod
     def _flow_ds_id(ds_type, ds_root, flow_name):
-        return ".".join([ds_type, ds_root, flow_name])
+        p = urlparse(ds_root)
+        sanitized_root = (p.netloc + p.path).replace("/", "_")
+        return ".".join([ds_type, sanitized_root, flow_name])
 
     @staticmethod
     def _task_ds_id(ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt):
+        p = urlparse(ds_root)
+        sanitized_root = (p.netloc + p.path).replace("/", "_")
         return ".".join(
-            [ds_type, ds_root, flow_name, run_id, step_name, task_id, str(attempt)]
+            [
+                ds_type,
+                sanitized_root,
+                flow_name,
+                run_id,
+                step_name,
+                task_id,
+                str(attempt),
+            ]
         )
 
     def _garbage_collect(self):

metaflow/cmd/develop/stub_generator.py

@@ -292,9 +292,11 @@ class StubGenerator:
             try:
                 potential_element = eval(
                     element,
-                    self._current_parent_module.__dict__
-                    if self._current_parent_module
-                    else None,
+                    (
+                        self._current_parent_module.__dict__
+                        if self._current_parent_module
+                        else None
+                    ),
                 )
                 if potential_element:
                     element = potential_element
@@ -555,14 +557,14 @@ class StubGenerator:
                         inspect.Parameter(
                             name=arg_name,
                             kind=inspect.Parameter.KEYWORD_ONLY,
-                            default=default
-                            if default_set
-                            else None
-                            if is_optional
-                            else inspect.Parameter.empty,
-                            annotation=Optional[type_name]
-                            if is_optional
-                            else type_name,
+                            default=(
+                                default
+                                if default_set
+                                else None if is_optional else inspect.Parameter.empty
+                            ),
+                            annotation=(
+                                Optional[type_name] if is_optional else type_name
+                            ),
                         )
                     )
                     if not default_set:
@@ -706,24 +708,31 @@ class StubGenerator:
             result = result + [
                 (
                     inspect.Signature(
-                        parameters=[
-                            inspect.Parameter(
-                                name="f",
-                                kind=inspect.Parameter.POSITIONAL_OR_KEYWORD,
-                                annotation=Optional[typing.Type[FlowSpecDerived]],
-                                default=None
-                                if no_arg_version
-                                else inspect.Parameter.empty,
-                            )
-                        ]
-                        + parameters
-                        if no_arg_version
-                        else [] + parameters,
-                        return_annotation=inspect.Signature.empty
-                        if no_arg_version
-                        else Callable[
-                            [typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]
-                        ],
+                        parameters=(
+                            [
+                                inspect.Parameter(
+                                    name="f",
+                                    kind=inspect.Parameter.POSITIONAL_OR_KEYWORD,
+                                    annotation=Optional[typing.Type[FlowSpecDerived]],
+                                    default=(
+                                        None
+                                        if no_arg_version
+                                        else inspect.Parameter.empty
+                                    ),
+                                )
+                            ]
+                            + parameters
+                            if no_arg_version
+                            else [] + parameters
+                        ),
+                        return_annotation=(
+                            inspect.Signature.empty
+                            if no_arg_version
+                            else Callable[
+                                [typing.Type[FlowSpecDerived]],
+                                typing.Type[FlowSpecDerived],
+                            ]
+                        ),
                     ),
                     "",
                 ),
@@ -732,24 +741,30 @@ class StubGenerator:
             result = result + [
                 (
                     inspect.Signature(
-                        parameters=[
-                            inspect.Parameter(
-                                name="f",
-                                kind=inspect.Parameter.POSITIONAL_OR_KEYWORD,
-                                annotation=Optional[MetaflowStepFunction],
-                                default=None
-                                if no_arg_version
-                                else inspect.Parameter.empty,
-                            )
-                        ]
-                        + parameters
-                        if no_arg_version
-                        else [] + parameters,
-                        return_annotation=inspect.Signature.empty
-                        if no_arg_version
-                        else typing.Callable[
-                            [MetaflowStepFunction], MetaflowStepFunction
-                        ],
+                        parameters=(
+                            [
+                                inspect.Parameter(
+                                    name="f",
+                                    kind=inspect.Parameter.POSITIONAL_OR_KEYWORD,
+                                    annotation=Optional[MetaflowStepFunction],
+                                    default=(
+                                        None
+                                        if no_arg_version
+                                        else inspect.Parameter.empty
+                                    ),
+                                )
+                            ]
+                            + parameters
+                            if no_arg_version
+                            else [] + parameters
+                        ),
+                        return_annotation=(
+                            inspect.Signature.empty
+                            if no_arg_version
+                            else typing.Callable[
+                                [MetaflowStepFunction], MetaflowStepFunction
+                            ]
+                        ),
                     ),
                     "",
                 ),

metaflow/datastore/content_addressed_store.py

@@ -133,7 +133,7 @@ class ContentAddressedStore(object):
                 load_paths.append((key, path))
 
         with self._storage_impl.load_bytes([p for _, p in load_paths]) as loaded:
-            for (path_key, file_path, meta) in loaded:
+            for path_key, file_path, meta in loaded:
                 key = self._storage_impl.path_split(path_key)[-1]
                 # At this point, we either return the object as is (if raw) or
                 # decode it according to the encoding version

metaflow/datastore/task_datastore.py

@@ -381,7 +381,7 @@ class TaskDataStore(object):
         # We assume that if we have one "old" style artifact, all of them are
         # like that which is an easy assumption to make since artifacts are all
         # stored by the same implementation of the datastore for a given task.
-        for (key, blob) in self._ca_store.load_blobs(to_load.keys()):
+        for key, blob in self._ca_store.load_blobs(to_load.keys()):
             names = to_load[key]
             for name in names:
                 # We unpickle everytime to have fully distinct objects (the user

metaflow/decorators.py

@@ -575,15 +575,13 @@ StepFlag = NewType("StepFlag", bool)
 @overload
 def step(
     f: Callable[[FlowSpecDerived], None]
-) -> Callable[[FlowSpecDerived, StepFlag], None]:
-    ...
+) -> Callable[[FlowSpecDerived, StepFlag], None]: ...
 
 
 @overload
 def step(
     f: Callable[[FlowSpecDerived, Any], None],
-) -> Callable[[FlowSpecDerived, Any, StepFlag], None]:
-    ...
+) -> Callable[[FlowSpecDerived, Any, StepFlag], None]: ...
 
 
 def step(

metaflow/extension_support/__init__.py

@@ -980,9 +980,9 @@ class _OrigLoader(Loader):
             if self._previously_loaded_module:
                 sys.modules[self._orig_name] = self._previously_loaded_module
             if self._previously_loaded_parent_module:
-                sys.modules[
-                    ".".join(self._orig_name.split(".")[:-1])
-                ] = self._previously_loaded_parent_module
+                sys.modules[".".join(self._orig_name.split(".")[:-1])] = (
+                    self._previously_loaded_parent_module
+                )
 
 
 class _LazyFinder(MetaPathFinder):

metaflow/extension_support/plugins.py

@@ -186,9 +186,9 @@ _plugin_categories = {
     "logging_sidecar": None,
     "monitor_sidecar": None,
     "aws_client_provider": lambda x: x.name,
-    "cli": lambda x: list(x.commands)[0]
-    if len(x.commands) == 1
-    else "too many commands",
+    "cli": lambda x: (
+        list(x.commands)[0] if len(x.commands) == 1 else "too many commands"
+    ),
 }
 
 

metaflow/metaflow_config.py

@@ -104,9 +104,11 @@ RETRY_WARNING_THRESHOLD = 3
 DATATOOLS_SUFFIX = from_conf("DATATOOLS_SUFFIX", "data")
 DATATOOLS_S3ROOT = from_conf(
     "DATATOOLS_S3ROOT",
-    os.path.join(DATASTORE_SYSROOT_S3, DATATOOLS_SUFFIX)
-    if DATASTORE_SYSROOT_S3
-    else None,
+    (
+        os.path.join(DATASTORE_SYSROOT_S3, DATATOOLS_SUFFIX)
+        if DATASTORE_SYSROOT_S3
+        else None
+    ),
 )
 
 TEMPDIR = from_conf("TEMPDIR", ".")
@@ -124,25 +126,31 @@ DATATOOLS_SESSION_VARS = from_conf("DATATOOLS_SESSION_VARS", {})
 # Similar to DATATOOLS_LOCALROOT, this is used ONLY by the IncludeFile's internal implementation.
 DATATOOLS_AZUREROOT = from_conf(
     "DATATOOLS_AZUREROOT",
-    os.path.join(DATASTORE_SYSROOT_AZURE, DATATOOLS_SUFFIX)
-    if DATASTORE_SYSROOT_AZURE
-    else None,
+    (
+        os.path.join(DATASTORE_SYSROOT_AZURE, DATATOOLS_SUFFIX)
+        if DATASTORE_SYSROOT_AZURE
+        else None
+    ),
 )
 # GS datatools root location
 # Note: we do not expose an actual datatools library for GS (like we do for S3)
 # Similar to DATATOOLS_LOCALROOT, this is used ONLY by the IncludeFile's internal implementation.
 DATATOOLS_GSROOT = from_conf(
     "DATATOOLS_GSROOT",
-    os.path.join(DATASTORE_SYSROOT_GS, DATATOOLS_SUFFIX)
-    if DATASTORE_SYSROOT_GS
-    else None,
+    (
+        os.path.join(DATASTORE_SYSROOT_GS, DATATOOLS_SUFFIX)
+        if DATASTORE_SYSROOT_GS
+        else None
+    ),
 )
 # Local datatools root location
 DATATOOLS_LOCALROOT = from_conf(
     "DATATOOLS_LOCALROOT",
-    os.path.join(DATASTORE_SYSROOT_LOCAL, DATATOOLS_SUFFIX)
-    if DATASTORE_SYSROOT_LOCAL
-    else None,
+    (
+        os.path.join(DATASTORE_SYSROOT_LOCAL, DATATOOLS_SUFFIX)
+        if DATASTORE_SYSROOT_LOCAL
+        else None
+    ),
 )
 
 # Secrets Backend - AWS Secrets Manager configuration
@@ -176,9 +184,11 @@ CARD_S3ROOT = from_conf(
 )
 CARD_AZUREROOT = from_conf(
     "CARD_AZUREROOT",
-    os.path.join(DATASTORE_SYSROOT_AZURE, CARD_SUFFIX)
-    if DATASTORE_SYSROOT_AZURE
-    else None,
+    (
+        os.path.join(DATASTORE_SYSROOT_AZURE, CARD_SUFFIX)
+        if DATASTORE_SYSROOT_AZURE
+        else None
+    ),
 )
 CARD_GSROOT = from_conf(
     "CARD_GSROOT",
@@ -238,6 +248,11 @@ DEFAULT_RUNTIME_LIMIT = from_conf("DEFAULT_RUNTIME_LIMIT", 5 * 24 * 60 * 60)
 ###
 UI_URL = from_conf("UI_URL")
 
+###
+# Capture error logs from argo
+###
+ARGO_WORKFLOWS_CAPTURE_ERROR_SCRIPT = from_conf("ARGO_WORKFLOWS_CAPTURE_ERROR_SCRIPT")
+
 # Contact information displayed when running the `metaflow` command.
 # Value should be a dictionary where:
 #  - key is a string describing contact method
@@ -306,9 +321,11 @@ SFN_EXECUTION_LOG_GROUP_ARN = from_conf("SFN_EXECUTION_LOG_GROUP_ARN")
 # Amazon S3 path for storing the results of AWS Step Functions Distributed Map
 SFN_S3_DISTRIBUTED_MAP_OUTPUT_PATH = from_conf(
     "SFN_S3_DISTRIBUTED_MAP_OUTPUT_PATH",
-    os.path.join(DATASTORE_SYSROOT_S3, "sfn_distributed_map_output")
-    if DATASTORE_SYSROOT_S3
-    else None,
+    (
+        os.path.join(DATASTORE_SYSROOT_S3, "sfn_distributed_map_output")
+        if DATASTORE_SYSROOT_S3
+        else None
+    ),
 )
 ###
 # Kubernetes configuration

metaflow/parameters.py

@@ -115,9 +115,9 @@ class DeployTimeField(object):
         self.parameter_name = parameter_name
         self.parameter_type = parameter_type
         self.return_str = return_str
-        self.print_representation = (
-            self.user_print_representation
-        ) = print_representation
+        self.print_representation = self.user_print_representation = (
+            print_representation
+        )
         if self.print_representation is None:
             self.print_representation = str(self.fun)
 

metaflow/plugins/airflow/airflow.py

@@ -399,17 +399,17 @@ class Airflow(object):
             "METAFLOW_CARD_GSROOT": CARD_GSROOT,
             "METAFLOW_S3_ENDPOINT_URL": S3_ENDPOINT_URL,
         }
-        env[
-            "METAFLOW_AZURE_STORAGE_BLOB_SERVICE_ENDPOINT"
-        ] = AZURE_STORAGE_BLOB_SERVICE_ENDPOINT
+        env["METAFLOW_AZURE_STORAGE_BLOB_SERVICE_ENDPOINT"] = (
+            AZURE_STORAGE_BLOB_SERVICE_ENDPOINT
+        )
         env["METAFLOW_DATASTORE_SYSROOT_AZURE"] = DATASTORE_SYSROOT_AZURE
         env["METAFLOW_CARD_AZUREROOT"] = CARD_AZUREROOT
         if DEFAULT_SECRETS_BACKEND_TYPE:
             env["METAFLOW_DEFAULT_SECRETS_BACKEND_TYPE"] = DEFAULT_SECRETS_BACKEND_TYPE
         if AWS_SECRETS_MANAGER_DEFAULT_REGION:
-            env[
-                "METAFLOW_AWS_SECRETS_MANAGER_DEFAULT_REGION"
-            ] = AWS_SECRETS_MANAGER_DEFAULT_REGION
+            env["METAFLOW_AWS_SECRETS_MANAGER_DEFAULT_REGION"] = (
+                AWS_SECRETS_MANAGER_DEFAULT_REGION
+            )
         if GCP_SECRET_MANAGER_PREFIX:
             env["METAFLOW_GCP_SECRET_MANAGER_PREFIX"] = GCP_SECRET_MANAGER_PREFIX
 

metaflow/plugins/airflow/airflow_utils.py

@@ -567,9 +567,11 @@ class AirflowTask(object):
         return cls(
             task_dict["name"],
             is_mapper_node=is_mapper_node,
-            operator_type=task_dict["operator_type"]
-            if "operator_type" in task_dict
-            else "kubernetes",
+            operator_type=(
+                task_dict["operator_type"]
+                if "operator_type" in task_dict
+                else "kubernetes"
+            ),
             flow_name=flow_name,
             flow_contains_foreach=flow_contains_foreach,
         ).set_operator_args(**op_args)