ob-metaflow 2.15.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/metaflow_config.py

@@ -21,6 +21,7 @@ if sys.platform == "darwin":
 
 # Path to the local directory to store artifacts for 'local' datastore.
 DATASTORE_LOCAL_DIR = ".metaflow"
+DATASTORE_SPIN_LOCAL_DIR = ".metaflow_spin"
 
 # Local configuration file (in .metaflow) containing overrides per-project
 LOCAL_CONFIG_FILE = "config.json"
@@ -47,6 +48,38 @@ DEFAULT_FROM_DEPLOYMENT_IMPL = from_conf(
     "DEFAULT_FROM_DEPLOYMENT_IMPL", "argo-workflows"
 )
 
+###
+# Spin configuration
+###
+# Essentially a whitelist of decorators that are allowed in Spin steps
+SPIN_ALLOWED_DECORATORS = from_conf(
+    "SPIN_ALLOWED_DECORATORS",
+    [
+        "conda",
+        "pypi",
+        "conda_base",
+        "pypi_base",
+        "environment",
+        "project",
+        "timeout",
+        "conda_env_internal",
+        "card",
+    ],
+)
+
+# Essentially a blacklist of decorators that are not allowed in Spin steps
+# Note: decorators not in either SPIN_ALLOWED_DECORATORS or SPIN_DISALLOWED_DECORATORS
+# are simply ignored in Spin steps
+SPIN_DISALLOWED_DECORATORS = from_conf(
+    "SPIN_DISALLOWED_DECORATORS",
+    [
+        "parallel",
+    ],
+)
+
+# Default value for persist option in spin command
+SPIN_PERSIST = from_conf("SPIN_PERSIST", False)
+
 ###
 # User configuration
 ###
@@ -57,6 +90,7 @@ USER = from_conf("USER")
 # Datastore configuration
 ###
 DATASTORE_SYSROOT_LOCAL = from_conf("DATASTORE_SYSROOT_LOCAL")
+DATASTORE_SYSROOT_SPIN = from_conf("DATASTORE_SYSROOT_SPIN")
 # S3 bucket and prefix to store artifacts for 's3' datastore.
 DATASTORE_SYSROOT_S3 = from_conf("DATASTORE_SYSROOT_S3")
 # Azure Blob Storage container and blob prefix
@@ -109,6 +143,9 @@ S3_WORKER_COUNT = from_conf("S3_WORKER_COUNT", 64)
 # top-level retries)
 S3_TRANSIENT_RETRY_COUNT = from_conf("S3_TRANSIENT_RETRY_COUNT", 20)
 
+# Whether to log transient retry messages to stdout
+S3_LOG_TRANSIENT_RETRIES = from_conf("S3_LOG_TRANSIENT_RETRIES", False)
+
 # S3 retry configuration used in the aws client
 # Use the adaptive retry strategy by default
 S3_CLIENT_RETRY_CONFIG = from_conf(
@@ -214,8 +251,6 @@ CARD_GSROOT = from_conf(
 )
 CARD_NO_WARNING = from_conf("CARD_NO_WARNING", False)
 
-SKIP_CARD_DUALWRITE = from_conf("SKIP_CARD_DUALWRITE", False)
-
 RUNTIME_CARD_RENDER_INTERVAL = from_conf("RUNTIME_CARD_RENDER_INTERVAL", 60)
 
 # Azure storage account URL
@@ -317,6 +352,8 @@ SERVICE_INTERNAL_URL = from_conf("SERVICE_INTERNAL_URL", SERVICE_URL)
 # in all Metaflow deployments. Hopefully, some day we can flip the
 # default to True.
 BATCH_EMIT_TAGS = from_conf("BATCH_EMIT_TAGS", False)
+# Default tags to add to AWS Batch jobs. These are in addition to the defaults set when BATCH_EMIT_TAGS is true.
+BATCH_DEFAULT_TAGS = from_conf("BATCH_DEFAULT_TAGS", {})
 
 ###
 # AWS Step Functions configuration
@@ -345,6 +382,8 @@ SFN_S3_DISTRIBUTED_MAP_OUTPUT_PATH = from_conf(
         else None
     ),
 )
+# Toggle for step command being part of the Step Function payload, or if it should be offloaded to S3
+SFN_COMPRESS_STATE_MACHINE = from_conf("SFN_COMPRESS_STATE_MACHINE", False)
 ###
 # Kubernetes configuration
 ###
@@ -371,6 +410,8 @@ KUBERNETES_CONTAINER_IMAGE = from_conf(
 )
 # Image pull policy for container images
 KUBERNETES_IMAGE_PULL_POLICY = from_conf("KUBERNETES_IMAGE_PULL_POLICY", None)
+# Image pull secrets for container images
+KUBERNETES_IMAGE_PULL_SECRETS = from_conf("KUBERNETES_IMAGE_PULL_SECRETS", "")
 # Default container registry for K8S
 KUBERNETES_CONTAINER_REGISTRY = from_conf(
     "KUBERNETES_CONTAINER_REGISTRY", DEFAULT_CONTAINER_REGISTRY
@@ -408,6 +449,9 @@ ARGO_EVENTS_INTERNAL_WEBHOOK_URL = from_conf(
     "ARGO_EVENTS_INTERNAL_WEBHOOK_URL", ARGO_EVENTS_WEBHOOK_URL
 )
 ARGO_EVENTS_WEBHOOK_AUTH = from_conf("ARGO_EVENTS_WEBHOOK_AUTH", "none")
+ARGO_EVENTS_SENSOR_NAMESPACE = from_conf(
+    "ARGO_EVENTS_SENSOR_NAMESPACE", KUBERNETES_NAMESPACE
+)
 
 ARGO_WORKFLOWS_UI_URL = from_conf("ARGO_WORKFLOWS_UI_URL")
 
@@ -450,10 +494,27 @@ CONDA_USE_FAST_INIT = from_conf("CONDA_USE_FAST_INIT", False)
 # Print out warning if escape hatch is not used for the target packages
 ESCAPE_HATCH_WARNING = from_conf("ESCAPE_HATCH_WARNING", True)
 
+###
+# Features
+###
+FEAT_ALWAYS_UPLOAD_CODE_PACKAGE = from_conf("FEAT_ALWAYS_UPLOAD_CODE_PACKAGE", False)
+###
+# Profile
+###
+PROFILE_FROM_START = from_conf("PROFILE_FROM_START", False)
 ###
 # Debug configuration
 ###
-DEBUG_OPTIONS = ["subcommand", "sidecar", "s3client", "tracing", "stubgen", "userconf"]
+DEBUG_OPTIONS = [
+    "subcommand",
+    "sidecar",
+    "s3client",
+    "tracing",
+    "stubgen",
+    "userconf",
+    "conda",
+    "package",
+]
 
 for typ in DEBUG_OPTIONS:
     vars()["DEBUG_%s" % typ.upper()] = from_conf("DEBUG_%s" % typ.upper(), False)
@@ -526,7 +587,7 @@ MAX_ATTEMPTS = 6
 # Feature flag (experimental features that are *explicitly* unsupported)
 
 # Process configs even when using the click_api for Runner/Deployer
-CLICK_API_PROCESS_CONFIG = from_conf("CLICK_API_PROCESS_CONFIG", False)
+CLICK_API_PROCESS_CONFIG = from_conf("CLICK_API_PROCESS_CONFIG", True)
 
 
 # PINNED_CONDA_LIBS are the libraries that metaflow depends on for execution
@@ -547,6 +608,7 @@ def get_pinned_conda_libs(python_version, datastore_type):
         pins["google-auth"] = ">=2.11.0"
         pins["google-cloud-secret-manager"] = ">=2.10.0"
         pins["simple-gcp-object-downloader"] = ">=0.1.0"
+        pins["packaging"] = ">=24.0"
     elif datastore_type == "local":
         pass
     else:

metaflow/metaflow_environment.py

@@ -1,3 +1,4 @@
+import json
 import os
 import platform
 import sys
@@ -8,6 +9,8 @@ from . import metaflow_git
 from metaflow.exception import MetaflowException
 from metaflow.extension_support import dump_module_info
 from metaflow.mflog import BASH_MFLOG, BASH_FLUSH_LOGS
+from metaflow.package import MetaflowPackage
+
 from . import R
 
 
@@ -49,8 +52,36 @@ class MetaflowEnvironment(object):
 
     def add_to_package(self):
         """
-        A list of tuples (file, arcname) to add to the job package.
-        `arcname` is an alternative name for the file in the job package.
+        Called to add custom files needed for this environment. This hook will be
+        called in the `MetaflowPackage` class where metaflow compiles the code package
+        tarball. This hook can return one of two things (the first is for backwards
+        compatibility -- move to the second):
+          - a generator yielding a tuple of `(file_path, arcname)` to add files to
+            the code package. `file_path` is the path to the file on the local filesystem
+            and `arcname` is the path relative to the packaged code.
+          - a generator yielding a tuple of `(content, arcname, type)` where:
+            - type is one of
+            ContentType.{USER_CONTENT, CODE_CONTENT, MODULE_CONTENT, OTHER_CONTENT}
+            - for USER_CONTENT:
+              - the file will be included relative to the directory containing the
+                user's flow file.
+              - content: path to the file to include
+              - arcname: path relative to the directory containing the user's flow file
+            - for CODE_CONTENT:
+              - the file will be included relative to the code directory in the package.
+                This will be the directory containing `metaflow`.
+              - content: path to the file to include
+              - arcname: path relative to the code directory in the package
+            - for MODULE_CONTENT:
+              - the module will be added to the code package as a python module. It will
+                be accessible as usual (import <module_name>)
+              - content: name of the module
+              - arcname: None (ignored)
+            - for OTHER_CONTENT:
+              - the file will be included relative to any other configuration/metadata
+                files for the flow
+              - content: path to the file to include
+              - arcname: path relative to the config directory in the package
         """
         return []
 
@@ -143,6 +174,7 @@ class MetaflowEnvironment(object):
                 "google-auth",
                 "simple-gcp-object-downloader",
                 "google-cloud-secret-manager",
+                "packaging",
             ],
         }
 
@@ -157,29 +189,63 @@ class MetaflowEnvironment(object):
         # skip pip installs if we know that packages might already be available
         return "if [ -z $METAFLOW_SKIP_INSTALL_DEPENDENCIES ]; then {}; fi".format(cmd)
 
-    def get_package_commands(self, code_package_url, datastore_type):
-        cmds = [
-            BASH_MFLOG,
-            BASH_FLUSH_LOGS,
-            "mflog 'Setting up task environment.'",
-            self._get_install_dependencies_cmd(datastore_type),
-            "mkdir metaflow",
-            "cd metaflow",
-            "mkdir .metaflow",  # mute local datastore creation log
-            "i=0; while [ $i -le 5 ]; do "
-            "mflog 'Downloading code package...'; "
-            + self._get_download_code_package_cmd(code_package_url, datastore_type)
-            + " && mflog 'Code package downloaded.' && break; "
-            "sleep 10; i=$((i+1)); "
-            "done",
-            "if [ $i -gt 5 ]; then "
-            "mflog 'Failed to download code package from %s "
-            "after 6 tries. Exiting...' && exit 1; "
-            "fi" % code_package_url,
-            "TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar",
-            "mflog 'Task is starting.'",
-            "flush_mflogs",
-        ]
+    def get_package_commands(
+        self, code_package_url, datastore_type, code_package_metadata=None
+    ):
+        # HACK: We want to keep forward compatibility with compute layers so that
+        # they can still call get_package_commands and NOT pass any metadata. If
+        # there is no additional information, we *assume* that it is the default
+        # used.
+        if code_package_metadata is None:
+            code_package_metadata = json.dumps(
+                {
+                    "version": 0,
+                    "archive_format": "tgz",
+                    "mfcontent_version": 1,
+                }
+            )
+
+        extra_exports = []
+        for k, v in MetaflowPackage.get_post_extract_env_vars(
+            code_package_metadata, dest_dir="$(pwd)"
+        ).items():
+            if k.endswith(":"):
+                # If the value ends with a colon, we override the existing value
+                extra_exports.append("export %s=%s" % (k[:-1], v))
+            else:
+                extra_exports.append(
+                    "export %s=%s:$(printenv %s)" % (k, v.replace('"', '\\"'), k)
+                )
+
+        cmds = (
+            [
+                BASH_MFLOG,
+                BASH_FLUSH_LOGS,
+                "mflog 'Setting up task environment.'",
+                self._get_install_dependencies_cmd(datastore_type),
+                "mkdir metaflow",
+                "cd metaflow",
+                "mkdir .metaflow",  # mute local datastore creation log
+                "i=0; while [ $i -le 5 ]; do "
+                "mflog 'Downloading code package...'; "
+                + self._get_download_code_package_cmd(code_package_url, datastore_type)
+                + " && mflog 'Code package downloaded.' && break; "
+                "sleep 10; i=$((i+1)); "
+                "done",
+                "if [ $i -gt 5 ]; then "
+                "mflog 'Failed to download code package from %s "
+                "after 6 tries. Exiting...' && exit 1; "
+                "fi" % code_package_url,
+            ]
+            + MetaflowPackage.get_extract_commands(
+                code_package_metadata, "job.tar", dest_dir="."
+            )
+            + extra_exports
+            + [
+                "mflog 'Task is starting.'",
+                "flush_mflogs",
+            ]
+        )
         return cmds
 
     def get_environment_info(self, include_ext_info=False):

metaflow/metaflow_profile.py

@@ -2,6 +2,24 @@ import time
 
 from contextlib import contextmanager
 
+from .metaflow_config import PROFILE_FROM_START
+
+init_time = None
+
+
+if PROFILE_FROM_START:
+
+    def from_start(msg: str):
+        global init_time
+        if init_time is None:
+            init_time = time.time()
+        print("From start: %s took %dms" % (msg, int((time.time() - init_time) * 1000)))
+
+else:
+
+    def from_start(_msg: str):
+        pass
+
 
 @contextmanager
 def profile(label, stats_dict=None):

metaflow/metaflow_version.py

@@ -11,7 +11,7 @@ import subprocess
 from os import path, name, environ, listdir
 
 from metaflow.extension_support import update_package_info
-from metaflow.info_file import CURRENT_DIRECTORY, read_info_file
+from metaflow.meta_files import read_info_file
 
 
 # True/False correspond to the value `public`` in get_version
@@ -133,6 +133,16 @@ def read_info_version():
     return None
 
 
+def make_public_version(version_string):
+    """
+    Takes a complex version string and returns a public, PEP 440-compliant version.
+    It removes local version identifiers (+...) and development markers (-...).
+    """
+    base_version = version_string.split("+", 1)[0]
+    public_version = base_version.split("-", 1)[0]
+    return public_version
+
+
 def get_version(public=False):
     """Tracks the version number.
 
@@ -170,6 +180,11 @@ def get_version(public=False):
     )  # Version info is cached in INFO file; includes extension info
 
     if version:
+        # If we have a version from the INFO file, use it directly.
+        # However, if we are asked for a public version, we parse it to make sure
+        # that no local information is included.
+        if public:
+            version = make_public_version(version)
         _version_cache[public] = version
         return version