ob-metaflow 2.15.15.1__tar.gz → 2.15.18.1__tar.gz

{ob_metaflow-2.15.15.1/ob_metaflow.egg-info → ob_metaflow-2.15.18.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.15.15.1
+Version: 2.15.18.1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.15.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.18.1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/cli.py

@@ -1,5 +1,6 @@
 import functools
 import inspect
+import os
 import sys
 import traceback
 from datetime import datetime
@@ -242,6 +243,14 @@ def version(obj):
     type=click.Choice(["local"] + [m.TYPE for m in ENVIRONMENTS]),
     help="Execution environment type",
 )
+@click.option(
+    "--force-rebuild-environments/--no-force-rebuild-environments",
+    is_flag=True,
+    default=False,
+    hidden=True,
+    type=bool,
+    help="Explicitly rebuild the execution environments",
+)
 # See comment for --quiet
 @click.option(
     "--datastore",
@@ -300,6 +309,7 @@ def start(
     quiet=False,
     metadata=None,
     environment=None,
+    force_rebuild_environments=False,
     datastore=None,
     datastore_root=None,
     decospecs=None,
@@ -435,6 +445,8 @@ def start(
     ctx.obj.environment = [
         e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == environment
     ][0](ctx.obj.flow)
+    # set force rebuild flag for environments that support it.
+    ctx.obj.environment._force_rebuild = force_rebuild_environments
     ctx.obj.environment.validate_environment(ctx.obj.logger, datastore)
 
     ctx.obj.event_logger = LOGGING_SIDECARS[event_logger](
@@ -496,7 +508,7 @@ def start(
         ctx.obj.echo,
         ctx.obj.flow_datastore,
         {
-            k: ConfigValue(v)
+            k: ConfigValue(v) if v is not None else None
             for k, v in ctx.obj.flow.__class__._flow_state.get(
                 _FlowState.CONFIGS, {}
             ).items()
@@ -524,7 +536,7 @@ def start(
             decorators._attach_decorators(ctx.obj.flow, all_decospecs)
             decorators._init(ctx.obj.flow)
             # Regenerate graph if we attached more decorators
-            ctx.obj.flow.__class__._init_attrs()
+            ctx.obj.flow.__class__._init_graph()
             ctx.obj.graph = ctx.obj.flow._graph
 
         decorators._init_step_decorators(

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/cli_components/run_cmds.py

@@ -45,7 +45,7 @@ def before_run(obj, tags, decospecs):
         decorators._attach_decorators(obj.flow, all_decospecs)
         decorators._init(obj.flow)
         # Regenerate graph if we attached more decorators
-        obj.flow.__class__._init_attrs()
+        obj.flow.__class__._init_graph()
         obj.graph = obj.flow._graph
 
     obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint)

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/cmd/develop/stub_generator.py

@@ -488,9 +488,6 @@ class StubGenerator:
                 self._imports.add(name)
 
         def _add_to_typing_check(name, is_module=False):
-            # if name != self._current_module_name:
-            #    self._typing_imports.add(name)
-            #
             if name == "None":
                 return
             if is_module:
@@ -504,6 +501,24 @@ class StubGenerator:
                     # the current file
                     self._typing_imports.add(splits[0])
 
+        def _format_qualified_class_name(cls: type) -> str:
+            """Helper to format a class with its qualified module name"""
+            # Special case for NoneType - return None
+            if cls.__name__ == "NoneType":
+                return "None"
+
+            module = inspect.getmodule(cls)
+            if (
+                module
+                and module.__name__ != "builtins"
+                and module.__name__ != "__main__"
+            ):
+                module_name = self._get_module_name_alias(module.__name__)
+                _add_to_typing_check(module_name, is_module=True)
+                return f"{module_name}.{cls.__name__}"
+            else:
+                return cls.__name__
+
         if isinstance(element, str):
             # Special case for self referential things (particularly in a class)
             if element == self._current_name:
@@ -557,19 +572,15 @@ class StubGenerator:
                 return element.__name__
         elif isinstance(element, type(Ellipsis)):
             return "..."
-        # elif (
-        #    isinstance(element, typing._GenericAlias)
-        #    and hasattr(element, "_name")
-        #    and element._name in ("List", "Tuple", "Dict", "Set")
-        # ):
-        #    # 3.7 has these as _GenericAlias but they don't behave like the ones in 3.10
-        #    _add_to_import("typing")
-        #    return str(element)
         elif isinstance(element, typing._GenericAlias):
             # We need to check things recursively in __args__ if it exists
             args_str = []
             for arg in getattr(element, "__args__", []):
-                args_str.append(self._get_element_name_with_module(arg))
+                # Special handling for class objects in type arguments
+                if isinstance(arg, type):
+                    args_str.append(_format_qualified_class_name(arg))
+                else:
+                    args_str.append(self._get_element_name_with_module(arg))
 
             _add_to_import("typing")
             if element._name:
@@ -584,12 +595,15 @@ class StubGenerator:
                         args_str = [call_args, args_str[-1]]
                 return "typing.%s[%s]" % (element._name, ", ".join(args_str))
             else:
-                return "%s[%s]" % (element.__origin__, ", ".join(args_str))
+                # Handle the case where we have a generic type without a _name
+                origin = element.__origin__
+                if isinstance(origin, type):
+                    origin_str = _format_qualified_class_name(origin)
+                else:
+                    origin_str = str(origin)
+                return "%s[%s]" % (origin_str, ", ".join(args_str))
         elif isinstance(element, ForwardRef):
             f_arg = self._get_module_name_alias(element.__forward_arg__)
-            # if f_arg in ("Run", "Task"):  # HACK -- forward references in current.py
-            #    _add_to_import("metaflow")
-            #    f_arg = "metaflow.%s" % f_arg
             _add_to_typing_check(f_arg)
             return '"%s"' % f_arg
         elif inspect.getmodule(element) == inspect.getmodule(typing):

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/flowspec.py

@@ -87,6 +87,9 @@ class FlowSpecMeta(type):
         if name == "FlowSpec":
             return
 
+        cls._init_attrs()
+
+    def _init_attrs(cls):
         from .decorators import (
             DuplicateFlowDecoratorException,
         )  # Prevent circular import
@@ -103,6 +106,12 @@ class FlowSpecMeta(type):
         # Keys are _FlowState enum values
         cls._flow_state = {}
 
+        # Keep track if configs have been processed -- this is particularly applicable
+        # for the Runner/Deployer where calling multiple APIs on the same flow could
+        # cause the configs to be processed multiple times. For a given flow, once
+        # the configs have been processed, we do not process them again.
+        cls._configs_processed = False
+
         # We inherit stuff from our parent classes as well -- we need to be careful
         # in terms of the order; we will follow the MRO with the following rules:
         #  - decorators (cls._flow_decorators) will cause an error if they do not
@@ -127,10 +136,9 @@ class FlowSpecMeta(type):
                     cls._flow_state.setdefault(_FlowState.CONFIG_DECORATORS, []).extend(
                         base_configs
                     )
+        cls._init_graph()
 
-        cls._init_attrs()
-
-    def _init_attrs(cls):
+    def _init_graph(cls):
         # Graph and steps are specific to the class -- store here so we can access
         # in class method _process_config_decorators
         cls._graph = FlowGraph(cls)
@@ -225,6 +233,10 @@ class FlowSpec(metaclass=FlowSpecMeta):
     @classmethod
     def _process_config_decorators(cls, config_options, process_configs=True):
 
+        if cls._configs_processed:
+            return None
+        cls._configs_processed = True
+
         # Fast path for no user configurations
         if not process_configs or (
             not cls._flow_state.get(_FlowState.CONFIG_DECORATORS)
@@ -325,7 +337,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
         parameters.replace_flow_context(cls)
 
         # Re-calculate class level attributes after modifying the class
-        cls._init_attrs()
+        cls._init_graph()
         return cls
 
     def _set_constants(self, graph, kwargs, config_options):

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/metaflow_config.py

@@ -214,8 +214,6 @@ CARD_GSROOT = from_conf(
 )
 CARD_NO_WARNING = from_conf("CARD_NO_WARNING", False)
 
-SKIP_CARD_DUALWRITE = from_conf("SKIP_CARD_DUALWRITE", False)
-
 RUNTIME_CARD_RENDER_INTERVAL = from_conf("RUNTIME_CARD_RENDER_INTERVAL", 60)
 
 # Azure storage account URL
@@ -371,6 +369,8 @@ KUBERNETES_CONTAINER_IMAGE = from_conf(
 )
 # Image pull policy for container images
 KUBERNETES_IMAGE_PULL_POLICY = from_conf("KUBERNETES_IMAGE_PULL_POLICY", None)
+# Image pull secrets for container images
+KUBERNETES_IMAGE_PULL_SECRETS = from_conf("KUBERNETES_IMAGE_PULL_SECRETS", "")
 # Default container registry for K8S
 KUBERNETES_CONTAINER_REGISTRY = from_conf(
     "KUBERNETES_CONTAINER_REGISTRY", DEFAULT_CONTAINER_REGISTRY

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/argo/argo_workflows.py

@@ -2021,6 +2021,7 @@ class ArgoWorkflows(object):
                     namespace=resources["namespace"],
                     image=resources["image"],
                     image_pull_policy=resources["image_pull_policy"],
+                    image_pull_secrets=resources["image_pull_secrets"],
                     service_account=resources["service_account"],
                     secrets=(
                         [
@@ -2209,6 +2210,8 @@ class ArgoWorkflows(object):
                     .node_selectors(resources.get("node_selector"))
                     # Set tolerations
                     .tolerations(resources.get("tolerations"))
+                    # Set image pull secrets
+                    .image_pull_secrets(resources.get("image_pull_secrets"))
                     # Set container
                     .container(
                         # TODO: Unify the logic with kubernetes.py
@@ -3785,6 +3788,10 @@ class Template(object):
         self.payload["tolerations"] = tolerations
         return self
 
+    def image_pull_secrets(self, image_pull_secrets):
+        self.payload["image_pull_secrets"] = image_pull_secrets
+        return self
+
     def to_json(self):
         return self.payload
 

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -9,59 +9,16 @@ from metaflow.exception import MetaflowException
 from metaflow.plugins.argo.argo_client import ArgoClient
 from metaflow.metaflow_config import KUBERNETES_NAMESPACE
 from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
-from metaflow.runner.deployer import Deployer, DeployedFlow, TriggeredRun
+from metaflow.runner.deployer import (
+    Deployer,
+    DeployedFlow,
+    TriggeredRun,
+    generate_fake_flow_file_contents,
+)
 
 from metaflow.runner.utils import get_lower_level_group, handle_timeout, temporary_fifo
 
 
-def generate_fake_flow_file_contents(
-    flow_name: str, param_info: dict, project_name: Optional[str] = None
-):
-    params_code = ""
-    for _, param_details in param_info.items():
-        param_python_var_name = param_details["python_var_name"]
-        param_name = param_details["name"]
-        param_type = param_details["type"]
-        param_help = param_details["description"]
-        param_required = param_details["is_required"]
-
-        if param_type == "JSON":
-            params_code += (
-                f"    {param_python_var_name} = Parameter('{param_name}', "
-                f"type=JSONType, help='''{param_help}''', required={param_required})\n"
-            )
-        elif param_type == "FilePath":
-            is_text = param_details.get("is_text", True)
-            encoding = param_details.get("encoding", "utf-8")
-            params_code += (
-                f"    {param_python_var_name} = IncludeFile('{param_name}', "
-                f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
-                f"required={param_required})\n"
-            )
-        else:
-            params_code += (
-                f"    {param_python_var_name} = Parameter('{param_name}', "
-                f"type={param_type}, help='''{param_help}''', required={param_required})\n"
-            )
-
-    project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
-
-    contents = f"""\
-from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
-{project_decorator}class {flow_name}(FlowSpec):
-{params_code}
-    @step
-    def start(self):
-        self.next(self.end)
-    @step
-    def end(self):
-        pass
-if __name__ == '__main__':
-    {flow_name}()
-"""
-    return contents
-
-
 class ArgoWorkflowsTriggeredRun(TriggeredRun):
     """
     A class representing a triggered Argo Workflow execution.

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/aws/aws_client.py

@@ -35,6 +35,12 @@ class Boto3ClientProvider(object):
                 "Could not import module 'boto3'. Install boto3 first."
             )
 
+        # Convert dictionary config to Config object if needed
+        if "config" in client_params and not isinstance(
+            client_params["config"], Config
+        ):
+            client_params["config"] = Config(**client_params["config"])
+
         if module == "s3" and (
             "config" not in client_params or client_params["config"].retries is None
         ):

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/cards/card_datastore.py

@@ -16,7 +16,6 @@ from metaflow.metaflow_config import (
     CARD_SUFFIX,
     CARD_AZUREROOT,
     CARD_GSROOT,
-    SKIP_CARD_DUALWRITE,
 )
 import metaflow.metaflow_config as metaflow_config
 
@@ -231,23 +230,6 @@ class CardDatastore(object):
 
     def save_card(self, uuid, card_type, card_html, card_id=None, overwrite=True):
         card_file_name = card_type
-        # TEMPORARY_WORKAROUND: FIXME (LATER) : Fix the duplication of below block in a few months.
-        # Check file blame to understand the age of this temporary workaround.
-
-        # This function will end up saving cards at two locations.
-        # Thereby doubling the number of cards. (Which is a temporary fix)
-        # Why do this ? :
-        # When cards were introduced there was an assumption made about task-ids being unique.
-        # This assumption was incorrect.
-        # Only the pathspec needs to be unique but there is no such guarantees about task-ids.
-        # When task-ids are non-unique, card read would result in finding incorrect cards.
-        # This happens because cards were stored based on task-ids.
-        # If we immediately switch from storing based on task-ids to a step-name abstraction folder,
-        # then card reading will crash for many users.
-        # It would especially happen for users who are accessing cards created by a newer
-        # MF client from an older version of MF client.
-        # It will also easily end up breaking the metaflow-ui (which maybe using a client from an older version).
-        # Hence, we are writing cards to both paths so that we can introduce breaking changes later in the future.
         card_path_with_steps = self.get_card_location(
             self._get_card_write_path(),
             card_file_name,
@@ -255,24 +237,10 @@ class CardDatastore(object):
             card_id=card_id,
             suffix=CardNameSuffix.CARD,
         )
-        if SKIP_CARD_DUALWRITE:
-            self._backend.save_bytes(
-                [(card_path_with_steps, BytesIO(bytes(card_html, "utf-8")))],
-                overwrite=overwrite,
-            )
-        else:
-            card_path_without_steps = self.get_card_location(
-                self._get_card_read_path(with_steps=False),
-                card_file_name,
-                uuid,
-                card_id=card_id,
-                suffix=CardNameSuffix.CARD,
-            )
-            for cp in [card_path_with_steps, card_path_without_steps]:
-                self._backend.save_bytes(
-                    [(cp, BytesIO(bytes(card_html, "utf-8")))], overwrite=overwrite
-                )
-
+        self._backend.save_bytes(
+            [(card_path_with_steps, BytesIO(bytes(card_html, "utf-8")))],
+            overwrite=overwrite,
+        )
         return self.info_from_path(card_path_with_steps, suffix=CardNameSuffix.CARD)
 
     def _list_card_paths(self, card_type=None, card_hash=None, card_id=None):
@@ -283,6 +251,10 @@ class CardDatastore(object):
         )
 
         if len(card_paths_with_steps) == 0:
+            # The listing logic is reading the cards with steps and without steps
+            # because earlier versions of clients (ones that wrote cards before June 2022),
+            # would have written cards without steps. So as a fallback we will try to check for the
+            # cards without steps.
             card_paths_without_steps = self._backend.list_content(
                 [self._get_card_read_path(with_steps=False)]
             )

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/datatools/s3/s3op.py

@@ -131,7 +131,7 @@ def normalize_client_error(err):
     except ValueError:
         if error_code in ("AccessDenied", "AllAccessDisabled", "InvalidAccessKeyId"):
             return 403
-        if error_code == "NoSuchKey":
+        if error_code in ("NoSuchKey", "NoSuchBucket"):
             return 404
         if error_code == "InvalidRange":
             return 416

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/kubernetes/kubernetes.py

@@ -170,6 +170,7 @@ class Kubernetes(object):
         code_package_ds,
         docker_image,
         docker_image_pull_policy,
+        image_pull_secrets=None,
         step_cli=None,
         service_account=None,
         secrets=None,
@@ -206,6 +207,7 @@ class Kubernetes(object):
                 node_selector=node_selector,
                 image=docker_image,
                 image_pull_policy=docker_image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 cpu=cpu,
                 memory=memory,
                 disk=disk,
@@ -483,6 +485,7 @@ class Kubernetes(object):
         step_cli,
         docker_image,
         docker_image_pull_policy,
+        image_pull_secrets=None,
         service_account=None,
         secrets=None,
         node_selector=None,
@@ -529,6 +532,7 @@ class Kubernetes(object):
                 ),
                 image=docker_image,
                 image_pull_policy=docker_image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 cpu=cpu,
                 memory=memory,
                 disk=disk,

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -53,6 +53,12 @@ def kubernetes():
     default=None,
     help="Optional Docker Image Pull Policy for Kubernetes pod.",
 )
+@click.option(
+    "--image-pull-secrets",
+    default=None,
+    type=JSONTypeClass(),
+    multiple=False,
+)
 @click.option(
     "--service-account",
     help="IRSA requirement for Kubernetes pod.",
@@ -160,6 +166,7 @@ def step(
     executable=None,
     image=None,
     image_pull_policy=None,
+    image_pull_secrets=None,
     service_account=None,
     secrets=None,
     node_selector=None,
@@ -303,6 +310,7 @@ def step(
                 step_cli=step_cli,
                 docker_image=image,
                 docker_image_pull_policy=image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 service_account=service_account,
                 secrets=secrets,
                 node_selector=node_selector,

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -18,6 +18,7 @@ from metaflow.metaflow_config import (
     KUBERNETES_FETCH_EC2_METADATA,
     KUBERNETES_GPU_VENDOR,
     KUBERNETES_IMAGE_PULL_POLICY,
+    KUBERNETES_IMAGE_PULL_SECRETS,
     KUBERNETES_MEMORY,
     KUBERNETES_LABELS,
     KUBERNETES_ANNOTATIONS,
@@ -74,6 +75,10 @@ class KubernetesDecorator(StepDecorator):
         not, a default Docker image mapping to the current version of Python is used.
     image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
         If given, the imagePullPolicy to be applied to the Docker image of the step.
+    image_pull_secrets: List[str], default []
+        The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
+        Kubernetes image pull secrets to use when pulling container images
+        in Kubernetes.
     service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
         Kubernetes service account to use when launching pod in Kubernetes.
     secrets : List[str], optional, default None
@@ -141,6 +146,7 @@ class KubernetesDecorator(StepDecorator):
         "disk": "10240",
         "image": None,
         "image_pull_policy": None,
+        "image_pull_secrets": None,  # e.g., ["regcred"]
         "service_account": None,
         "secrets": None,  # e.g., mysecret
         "node_selector": None,  # e.g., kubernetes.io/os=linux
@@ -194,6 +200,10 @@ class KubernetesDecorator(StepDecorator):
             )
         if not self.attributes["image_pull_policy"] and KUBERNETES_IMAGE_PULL_POLICY:
             self.attributes["image_pull_policy"] = KUBERNETES_IMAGE_PULL_POLICY
+        if not self.attributes["image_pull_secrets"] and KUBERNETES_IMAGE_PULL_SECRETS:
+            self.attributes["image_pull_secrets"] = json.loads(
+                KUBERNETES_IMAGE_PULL_SECRETS
+            )
 
         if isinstance(self.attributes["node_selector"], str):
             self.attributes["node_selector"] = parse_kube_keyvalue_list(
@@ -494,6 +504,7 @@ class KubernetesDecorator(StepDecorator):
                         for key, val in v.items()
                     ]
                 elif k in [
+                    "image_pull_secrets",
                     "tolerations",
                     "persistent_volume_claims",
                     "labels",

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/kubernetes/kubernetes_job.py

@@ -235,8 +235,10 @@ class KubernetesJob(object):
                         )
                     ],
                     node_selector=self._kwargs.get("node_selector"),
-                    # TODO (savin): Support image_pull_secrets
-                    # image_pull_secrets=?,
+                    image_pull_secrets=[
+                        client.V1LocalObjectReference(secret)
+                        for secret in self._kwargs.get("image_pull_secrets") or []
+                    ],
                     # TODO (savin): Support preemption policies
                     # preemption_policy=?,
                     #

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -718,8 +718,11 @@ class JobSetSpec(object):
                                     )
                                 ],
                                 node_selector=self._kwargs.get("node_selector"),
-                                # TODO (savin): Support image_pull_secrets
-                                # image_pull_secrets=?,
+                                image_pull_secrets=[
+                                    client.V1LocalObjectReference(secret)
+                                    for secret in self._kwargs.get("image_pull_secrets")
+                                    or []
+                                ],
                                 # TODO (savin): Support preemption policies
                                 # preemption_policy=?,
                                 #

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/metadata_providers/service.py

@@ -72,14 +72,18 @@ class ServiceMetadataProvider(MetadataProvider):
     @classmethod
     def compute_info(cls, val):
         v = val.rstrip("/")
-        try:
-            resp = cls._session.get(
-                os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy()
-            )
-            resp.raise_for_status()
-        except:  # noqa E722
-            raise ValueError("Metaflow service [%s] unreachable." % v)
-        return v
+        for i in range(SERVICE_RETRY_COUNT):
+            try:
+                resp = cls._session.get(
+                    os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy()
+                )
+                resp.raise_for_status()
+            except:  # noqa E722
+                time.sleep(2 ** (i - 1))
+            else:
+                return v
+
+        raise ValueError("Metaflow service [%s] unreachable." % v)
 
     @classmethod
     def default_info(cls):

{ob_metaflow-2.15.15.1 → ob_metaflow-2.15.18.1}/metaflow/plugins/pypi/conda_environment.py

@@ -32,6 +32,7 @@ class CondaEnvironmentException(MetaflowException):
 class CondaEnvironment(MetaflowEnvironment):
     TYPE = "conda"
     _filecache = None
+    _force_rebuild = False
 
     def __init__(self, flow):
         self.flow = flow
@@ -71,7 +72,7 @@ class CondaEnvironment(MetaflowEnvironment):
         self.logger = make_thread_safe(logger)
 
         # TODO: Wire up logging
-        micromamba = Micromamba(self.logger)
+        micromamba = Micromamba(self.logger, self._force_rebuild)
         self.solvers = {"conda": micromamba, "pypi": Pip(micromamba, self.logger)}
 
     def init_environment(self, echo, only_steps=None):
@@ -107,7 +108,10 @@ class CondaEnvironment(MetaflowEnvironment):
             return (
                 id_,
                 (
-                    self.read_from_environment_manifest([id_, platform, type_])
+                    (
+                        not self._force_rebuild
+                        and self.read_from_environment_manifest([id_, platform, type_])
+                    )
                     or self.write_to_environment_manifest(
                         [id_, platform, type_],
                         self.solvers[type_].solve(id_, **environment),
@@ -153,7 +157,7 @@ class CondaEnvironment(MetaflowEnvironment):
             _meta = copy.deepcopy(local_packages)
             for id_, packages, _, _ in results:
                 for package in packages:
-                    if package.get("path"):
+                    if package.get("path") and not self._force_rebuild:
                         # Cache only those packages that manifest is unaware of
                         local_packages.pop(package["url"], None)
                     else:
@@ -186,7 +190,7 @@ class CondaEnvironment(MetaflowEnvironment):
             storage.save_bytes(
                 list_of_path_and_filehandle,
                 len_hint=len(list_of_path_and_filehandle),
-                # overwrite=True,
+                overwrite=self._force_rebuild,
             )
             for id_, packages, _, platform in results:
                 if id_ in dirty: