ob-metaflow 2.10.7.3__tar.gz → 2.10.9.1__tar.gz

{ob-metaflow-2.10.7.3/ob_metaflow.egg-info → ob-metaflow-2.10.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.10.7.3
+Version: 2.10.9.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/cards.py

@@ -6,6 +6,8 @@ from metaflow.plugins.cards.card_modules.components import (
     Image,
     Error,
     Markdown,
+    VegaChart,
+    ProgressBar,
 )
 from metaflow.plugins.cards.card_modules.basic import (
     DefaultCard,

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/decorators.py

@@ -129,7 +129,7 @@ class Decorator(object):
 
         attrs = {}
         # TODO: Do we really want to allow spaces in the names of attributes?!?
-        for a in re.split(""",(?=[\s\w]+=)""", deco_spec):
+        for a in re.split(r""",(?=[\s\w]+=)""", deco_spec):
             name, val = a.split("=", 1)
             try:
                 val_parsed = json.loads(val.strip().replace('\\"', '"'))

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/metaflow_config.py

@@ -165,6 +165,8 @@ CARD_NO_WARNING = from_conf("CARD_NO_WARNING", False)
 
 SKIP_CARD_DUALWRITE = from_conf("SKIP_CARD_DUALWRITE", False)
 
+RUNTIME_CARD_RENDER_INTERVAL = from_conf("RUNTIME_CARD_RENDER_INTERVAL", 60)
+
 # Azure storage account URL
 AZURE_STORAGE_BLOB_SERVICE_ENDPOINT = from_conf("AZURE_STORAGE_BLOB_SERVICE_ENDPOINT")
 

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/__init__.py

@@ -172,6 +172,8 @@ from .cards.card_modules.test_cards import (
     TestNonEditableCard,
     TestPathSpecCard,
     TestTimeoutCard,
+    TestRefreshCard,
+    TestRefreshComponentCard,
 )
 
 CARDS = [
@@ -188,5 +190,7 @@ CARDS = [
     TestNonEditableCard,
     BlankCard,
     DefaultCardJSON,
+    TestRefreshCard,
+    TestRefreshComponentCard,
 ]
 merge_lists(CARDS, MF_EXTERNAL_CARDS, "type")

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/airflow/airflow_cli.py

@@ -24,7 +24,7 @@ class IncorrectProductionToken(MetaflowException):
     headline = "Incorrect production token"
 
 
-VALID_NAME = re.compile("[^a-zA-Z0-9_\-\.]")
+VALID_NAME = re.compile(r"[^a-zA-Z0-9_\-\.]")
 
 
 def resolve_token(

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -29,7 +29,7 @@ from metaflow.util import get_username, to_bytes, to_unicode, version_parse
 
 from .argo_workflows import ArgoWorkflows
 
-VALID_NAME = re.compile("^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$")
+VALID_NAME = re.compile(r"^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$")
 
 
 class IncorrectProductionToken(MetaflowException):

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/aws/aws_utils.py

@@ -39,7 +39,7 @@ def get_ec2_instance_metadata():
 
 
 def get_docker_registry(image_uri):
-    """
+    r"""
     Explanation:
         (.+?(?:[:.].+?)\/)? - [GROUP 0] REGISTRY
             .+?                 - A registry must start with at least one character

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/aws/batch/batch.py

@@ -185,6 +185,7 @@ class Batch(object):
         env={},
         attrs={},
         host_volumes=None,
+        efs_volumes=None,
         use_tmpfs=None,
         tmpfs_tempdir=None,
         tmpfs_size=None,
@@ -232,6 +233,7 @@ class Batch(object):
                 efa,
                 memory=memory,
                 host_volumes=host_volumes,
+                efs_volumes=efs_volumes,
                 use_tmpfs=use_tmpfs,
                 tmpfs_tempdir=tmpfs_tempdir,
                 tmpfs_size=tmpfs_size,
@@ -341,6 +343,7 @@ class Batch(object):
         inferentia=None,
         efa=None,
         host_volumes=None,
+        efs_volumes=None,
         use_tmpfs=None,
         tmpfs_tempdir=None,
         tmpfs_size=None,
@@ -379,6 +382,7 @@ class Batch(object):
             env=env,
             attrs=attrs,
             host_volumes=host_volumes,
+            efs_volumes=efs_volumes,
             use_tmpfs=use_tmpfs,
             tmpfs_tempdir=tmpfs_tempdir,
             tmpfs_size=tmpfs_size,

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/aws/batch/batch_cli.py

@@ -153,6 +153,7 @@ def kill(ctx, run_id, user, my_runs):
 # TODO: Maybe remove it altogether since it's not used here
 @click.option("--ubf-context", default=None, type=click.Choice([None, "ubf_control"]))
 @click.option("--host-volumes", multiple=True)
+@click.option("--efs-volumes", multiple=True)
 @click.option(
     "--num-parallel",
     default=0,
@@ -184,6 +185,7 @@ def step(
     tmpfs_size=None,
     tmpfs_path=None,
     host_volumes=None,
+    efs_volumes=None,
     num_parallel=None,
     **kwargs
 ):
@@ -310,6 +312,7 @@ def step(
                 env=env,
                 attrs=attrs,
                 host_volumes=host_volumes,
+                efs_volumes=efs_volumes,
                 use_tmpfs=use_tmpfs,
                 tmpfs_tempdir=tmpfs_tempdir,
                 tmpfs_size=tmpfs_size,

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/aws/batch/batch_client.py

@@ -152,6 +152,7 @@ class BatchJob(object):
         efa,
         memory,
         host_volumes,
+        efs_volumes,
         use_tmpfs,
         tmpfs_tempdir,
         tmpfs_size,
@@ -274,19 +275,45 @@ class BatchJob(object):
                         }
                     )
 
-        if host_volumes:
+        if host_volumes or efs_volumes:
             job_definition["containerProperties"]["volumes"] = []
             job_definition["containerProperties"]["mountPoints"] = []
-            if isinstance(host_volumes, str):
-                host_volumes = [host_volumes]
-            for host_path in host_volumes:
-                name = host_path.replace("/", "_").replace(".", "_")
-                job_definition["containerProperties"]["volumes"].append(
-                    {"name": name, "host": {"sourcePath": host_path}}
-                )
-                job_definition["containerProperties"]["mountPoints"].append(
-                    {"sourceVolume": name, "containerPath": host_path}
-                )
+
+            if host_volumes:
+                if isinstance(host_volumes, str):
+                    host_volumes = [host_volumes]
+                for host_path in host_volumes:
+                    container_path = host_path
+                    if ":" in host_path:
+                        host_path, container_path = host_path.split(":", 1)
+                    name = host_path.replace("/", "_").replace(".", "_")
+                    job_definition["containerProperties"]["volumes"].append(
+                        {"name": name, "host": {"sourcePath": host_path}}
+                    )
+                    job_definition["containerProperties"]["mountPoints"].append(
+                        {"sourceVolume": name, "containerPath": container_path}
+                    )
+
+            if efs_volumes:
+                if isinstance(efs_volumes, str):
+                    efs_volumes = [efs_volumes]
+                for efs_id in efs_volumes:
+                    container_path = "/mnt/" + efs_id
+                    if ":" in efs_id:
+                        efs_id, container_path = efs_id.split(":", 1)
+                    name = "efs_" + efs_id
+                    job_definition["containerProperties"]["volumes"].append(
+                        {
+                            "name": name,
+                            "efsVolumeConfiguration": {
+                                "fileSystemId": efs_id,
+                                "transitEncryption": "ENABLED",
+                            },
+                        }
+                    )
+                    job_definition["containerProperties"]["mountPoints"].append(
+                        {"sourceVolume": name, "containerPath": container_path}
+                    )
 
         if use_tmpfs or (tmpfs_size and not use_tmpfs):
             if tmpfs_size:
@@ -401,6 +428,7 @@ class BatchJob(object):
         efa,
         memory,
         host_volumes,
+        efs_volumes,
         use_tmpfs,
         tmpfs_tempdir,
         tmpfs_size,
@@ -419,6 +447,7 @@ class BatchJob(object):
             efa,
             memory,
             host_volumes,
+            efs_volumes,
             use_tmpfs,
             tmpfs_tempdir,
             tmpfs_size,

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/aws/batch/batch_decorator.py

@@ -102,6 +102,7 @@ class BatchDecorator(StepDecorator):
         "inferentia": None,
         "efa": None,
         "host_volumes": None,
+        "efs_volumes": None,
         "use_tmpfs": False,
         "tmpfs_tempdir": True,
         "tmpfs_size": None,

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/aws/step_functions/step_functions.py

@@ -704,6 +704,7 @@ class StepFunctions(object):
                 env=env,
                 attrs=attrs,
                 host_volumes=resources["host_volumes"],
+                efs_volumes=resources["efs_volumes"],
             )
             .attempts(total_retries + 1)
         )

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/aws/step_functions/step_functions_cli.py

@@ -19,7 +19,7 @@ from metaflow.util import get_username, to_bytes, to_unicode, version_parse
 from .step_functions import StepFunctions
 from .production_token import load_token, store_token, new_token
 
-VALID_NAME = re.compile("[^a-zA-Z0-9_\-\.]")
+VALID_NAME = re.compile(r"[^a-zA-Z0-9_\-\.]")
 
 
 class IncorrectProductionToken(MetaflowException):

{ob-metaflow-2.10.7.3 → ob-metaflow-2.10.9.1}/metaflow/plugins/azure/azure_exceptions.py

@@ -10,4 +10,4 @@ class MetaflowAzureResourceError(MetaflowException):
 
 
 class MetaflowAzurePackageError(MetaflowException):
-    headline = "Missing required packages azure-identity and azure-storage-blob"
+    headline = "Missing required packages 'azure-identity' and 'azure-storage-blob'"