ob-metaflow 2.15.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/util.py

@@ -4,12 +4,13 @@ import sys
 import tempfile
 import zlib
 import base64
+import re
+
 from functools import wraps
 from io import BytesIO
 from itertools import takewhile
-import re
+from typing import Dict, Any, Tuple, Optional, List, Generator
 
-from metaflow.exception import MetaflowUnknownUser, MetaflowInternalError
 
 try:
     # python2
@@ -162,6 +163,8 @@ def get_username():
 
 
 def resolve_identity_as_tuple():
+    from metaflow.exception import MetaflowUnknownUser
+
     prod_token = os.environ.get("METAFLOW_PRODUCTION_TOKEN")
     if prod_token:
         return "production", prod_token
@@ -177,6 +180,119 @@ def resolve_identity():
     return "%s:%s" % (identity_type, identity_value)
 
 
+def parse_spin_pathspec(pathspec: str, flow_name: str) -> Tuple:
+    """
+    Parse various pathspec formats for the spin command.
+
+    Parameters
+    ----------
+    pathspec : str
+        The pathspec string in one of the following formats:
+        - step_name (e.g., 'start')
+        - run_id/step_name (e.g., '221165/start')
+        - run_id/step_name/task_id (e.g., '221165/start/1350987')
+        - flow_name/run_id/step_name (e.g., 'ScalableFlow/221165/start')
+        - flow_name/run_id/step_name/task_id (e.g., 'ScalableFlow/221165/start/1350987')
+    flow_name : str
+        The name of the current flow.
+
+    Returns
+    -------
+    Tuple
+        A tuple of (step_name, full_pathspec_or_none)
+
+    Raises
+    ------
+    CommandException
+        If the pathspec format is invalid or flow name doesn't match.
+    """
+    from .exception import CommandException
+
+    parts = pathspec.split("/")
+
+    if len(parts) == 1:
+        # Just step name: 'start'
+        step_name = parts[0]
+        parsed_pathspec = None
+    elif len(parts) == 2:
+        # run_id/step_name: '221165/start'
+        run_id, step_name = parts
+        parsed_pathspec = f"{flow_name}/{run_id}/{step_name}"
+    elif len(parts) == 3:
+        # Could be run_id/step_name/task_id or flow_name/run_id/step_name
+        if parts[0] == flow_name:
+            # flow_name/run_id/step_name
+            _, run_id, step_name = parts
+            parsed_pathspec = f"{flow_name}/{run_id}/{step_name}"
+        else:
+            # run_id/step_name/task_id
+            run_id, step_name, task_id = parts
+            parsed_pathspec = f"{flow_name}/{run_id}/{step_name}/{task_id}"
+    elif len(parts) == 4:
+        # flow_name/run_id/step_name/task_id
+        parsed_flow_name, run_id, step_name, task_id = parts
+        if parsed_flow_name != flow_name:
+            raise CommandException(
+                f"Flow name '{parsed_flow_name}' in pathspec does not match current flow '{flow_name}'."
+            )
+        parsed_pathspec = pathspec
+    else:
+        raise CommandException(
+            f"Invalid pathspec format: '{pathspec}'. \n"
+            "Expected formats:\n"
+            "  - step_name (e.g., 'start')\n"
+            "  - run_id/step_name (e.g., '221165/start')\n"
+            "  - run_id/step_name/task_id (e.g., '221165/start/1350987')\n"
+            "  - flow_name/run_id/step_name (e.g., 'ScalableFlow/221165/start')\n"
+            "  - flow_name/run_id/step_name/task_id (e.g., 'ScalableFlow/221165/start/1350987')"
+        )
+
+    return step_name, parsed_pathspec
+
+
+def get_latest_task_pathspec(
+    flow_name: str, step_name: str, run_id: str = None
+) -> "metaflow.Task":
+    """
+    Returns a task pathspec from the latest run (or specified run) of the flow for the queried step.
+    If the queried step has several tasks, the task pathspec of the first task is returned.
+
+    Parameters
+    ----------
+    flow_name : str
+        The name of the flow.
+    step_name : str
+        The name of the step.
+    run_id : str, optional
+        The run ID to use. If None, uses the latest run.
+
+    Returns
+    -------
+    Task
+        A Metaflow Task instance containing the latest task for the queried step.
+
+    Raises
+    ------
+    MetaflowNotFound
+        If no task or run is found for the queried step.
+    """
+    from metaflow import Flow, Step
+    from metaflow.exception import MetaflowNotFound
+
+    if not run_id:
+        flow = Flow(flow_name)
+        run = flow.latest_run
+        if run is None:
+            raise MetaflowNotFound(f"No run found for flow {flow_name}")
+        run_id = run.id
+
+    try:
+        task = Step(f"{flow_name}/{run_id}/{step_name}").task
+        return task
+    except:
+        raise MetaflowNotFound(f"No task found for step {step_name} in run {run_id}")
+
+
 def get_latest_run_id(echo, flow_name):
     from metaflow.plugins.datastores.local_storage import LocalStorage
 
@@ -236,6 +352,8 @@ def get_object_package_version(obj):
 
 
 def compress_list(lst, separator=",", rangedelim=":", zlibmarker="!", zlibmin=500):
+    from metaflow.exception import MetaflowInternalError
+
     bad_items = [x for x in lst if separator in x or rangedelim in x or zlibmarker in x]
     if bad_items:
         raise MetaflowInternalError(
@@ -418,7 +536,7 @@ def to_pascalcase(obj):
     if isinstance(obj, dict):
         res = obj.__class__()
         for k in obj:
-            res[re.sub("([a-zA-Z])", lambda x: x.groups()[0].upper(), k, 1)] = (
+            res[re.sub("([a-zA-Z])", lambda x: x.groups()[0].upper(), k, count=1)] = (
                 to_pascalcase(obj[k])
             )
     elif isinstance(obj, (list, set, tuple)):
@@ -467,7 +585,79 @@ def to_pod(value):
     return str(value)
 
 
-if sys.version_info[:2] > (3, 5):
-    from metaflow._vendor.packaging.version import parse as version_parse
-else:
-    from distutils.version import LooseVersion as version_parse
+from metaflow._vendor.packaging.version import parse as version_parse
+
+
+def read_artifacts_module(file_path: str) -> Dict[str, Any]:
+    """
+    Read a Python module from the given file path and return its ARTIFACTS variable.
+
+    Parameters
+    ----------
+    file_path : str
+        The path to the Python file containing the ARTIFACTS variable.
+
+    Returns
+    -------
+    Dict[str, Any]
+        A dictionary containing the ARTIFACTS variable from the module.
+
+    Raises
+    -------
+    MetaflowInternalError
+        If the file cannot be read or does not contain the ARTIFACTS variable.
+    """
+    import importlib.util
+    import os
+
+    try:
+        module_name = os.path.splitext(os.path.basename(file_path))[0]
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        variables = vars(module)
+        if "ARTIFACTS" not in variables:
+            raise MetaflowInternalError(
+                f"Module {file_path} does not contain ARTIFACTS variable"
+            )
+        return variables.get("ARTIFACTS")
+    except Exception as e:
+        raise MetaflowInternalError(f"Error reading file {file_path}") from e
+
+
+# this is os.walk(follow_symlinks=True) with cycle detection
+def walk_without_cycles(
+    top_root: str,
+    exclude_dirs: Optional[List[str]] = None,
+) -> Generator[Tuple[str, List[str], List[str]], None, None]:
+    seen = set()
+
+    default_skip_dirs = ["__pycache__"]
+
+    def _recurse(root, skip_dirs):
+        for parent, dirs, files in os.walk(root):
+            dirs[:] = [d for d in dirs if d not in skip_dirs]
+            for d in dirs:
+                path = os.path.join(parent, d)
+                if os.path.islink(path):
+                    # Breaking loops: never follow the same symlink twice
+                    #
+                    # NOTE: this also means that links to sibling links are
+                    # not followed. In this case:
+                    #
+                    #   x -> y
+                    #   y -> oo
+                    #   oo/real_file
+                    #
+                    # real_file is only included twice, not three times
+                    reallink = os.path.realpath(path)
+                    if reallink not in seen:
+                        seen.add(reallink)
+                        for x in _recurse(path, default_skip_dirs):
+                            yield x
+            yield parent, dirs, files
+
+    skip_dirs = set(default_skip_dirs + (exclude_dirs or []))
+    for x in _recurse(top_root, skip_dirs):
+        skip_dirs = default_skip_dirs
+        yield x

metaflow/vendor.py

@@ -11,7 +11,6 @@ WHITELIST = {
     "README.txt",
     "__init__.py",
     "vendor_any.txt",
-    "vendor_v3_5.txt",
     "vendor_v3_6.txt",
     "vendor_v3_7.txt",
     "pip.LICENSE",
@@ -64,14 +63,29 @@ def find_vendored_libs(vendor_dir, whitelist, whitelist_dirs):
     return vendored_libs, paths
 
 
-def fetch_licenses(*info_dir, vendor_dir):
-    for file in chain.from_iterable(map(iter_subtree, info_dir)):
-        if "LICENSE" in file.name:
-            library = file.parent.name.split("-")[0]
-            shutil.copy(file, vendor_dir / ("%s.LICENSE" % library))
-        else:
+def fetch_licenses(*info_dirs, vendor_dir):
+    for dist_info in info_dirs:
+        metadata_file = dist_info / "METADATA"
+        if not metadata_file.exists():
+            continue
+
+        project_name = None
+        for line in metadata_file.read_text("utf-8").splitlines():
+            if line.startswith("Name: "):
+                project_name = line.split("Name: ", 1)[1].strip()
+                break
+        if not project_name:
             continue
 
+        for item in dist_info.iterdir():
+            if item.is_file() and re.search(r"(LICENSE|COPYING)", item.name, re.I):
+                shutil.copy(item, vendor_dir / f"{project_name}.LICENSE")
+            elif item.is_dir() and item.name.lower() == "licenses":
+                for license_file in item.iterdir():
+                    if license_file.is_file():
+                        dest_name = f"{project_name}.{license_file.name}"
+                        shutil.copy(license_file, vendor_dir / dest_name)
+
 
 def vendor(vendor_dir):
     # remove everything
@@ -109,6 +123,8 @@ def vendor(vendor_dir):
                 "-r",
                 "_vendor/vendor_%s.txt" % subdir,
                 "--no-compile",
+                "--no-binary",
+                ":all:",
             ]
         )
 

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.15.13.1"
+metaflow_version = "2.19.7.1rc0"

{ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/Makefile

@@ -31,6 +31,7 @@ MAKE_CMD := $(MAKE) -f "$(MKFILE_PATH)"
 MINIKUBE_CPUS ?= 4
 MINIKUBE_MEMORY ?= 6144
 MINIKUBE_DISK_SIZE ?= 20g
+WAIT_TIMEOUT ?= 300
 
 ifeq ($(shell uname), Darwin)
 	minikube_os = darwin
@@ -257,10 +258,11 @@ shell: setup-tilt
 		echo "🔎 Using $$user_shell for interactive session."; \
 		echo "🐍 If you installed Metaflow in a virtual environment, activate it now."; \
 		if [ -f "$(DEVTOOLS_DIR)/aws_config" ]; then \
-			env METAFLOW_HOME="$(DEVTOOLS_DIR)" \
+			env -u AWS_PROFILE \
+				-u AWS_SHARED_CREDENTIALS_FILE \
+				METAFLOW_HOME="$(DEVTOOLS_DIR)" \
 				METAFLOW_PROFILE=local \
 				AWS_CONFIG_FILE="$(DEVTOOLS_DIR)/aws_config" \
-				AWS_SHARED_CREDENTIALS_FILE= \
 				"$$user_shell" -i; \
 		else \
 			env METAFLOW_HOME="$(DEVTOOLS_DIR)" \
@@ -268,6 +270,15 @@ shell: setup-tilt
 				"$$user_shell" -i; \
 		fi'
 
+wait-until-ready:
+	@echo "Waiting for infrastructure to be ready. Timing out in $(WAIT_TIMEOUT) seconds..."
+	@timeout $(WAIT_TIMEOUT) bash -c 'while [ ! -f $(DEVTOOLS_DIR)/start.sh ]; do sleep 10; done; echo "Infra is Ready"' || (echo "Waiting for infra timed out"&&exit 1)
+# buffer to get the tilt api running
+	@timeout 120 bash -c 'while ! $(TILT) get session; do sleep 3;done'
+	@echo "Waiting for services to be ready. Timing out in $(WAIT_TIMEOUT) seconds..."
+# Need to wait for Tiltfile first, as other resources return 404 otherwise
+	@$(TILT) wait --for=condition=Ready "uiresource/(Tiltfile)" --timeout=$(WAIT_TIMEOUT)s
+	@$(TILT) wait --for=condition=Ready uiresource/generate-configs --timeout=$(WAIT_TIMEOUT)s
 
 # @echo '$(MAKE_CMD) create-dev-shell' >> $(DEVTOOLS_DIR)/start.sh
 # @echo 'rm -f /tmp/metaflow-devshell-*' >> $(DEVTOOLS_DIR)/start.sh

{ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/Tiltfile

@@ -14,6 +14,17 @@
 version_settings(constraint='>=0.22.2')
 allow_k8s_contexts('minikube')
 
+# Version configuration for components
+JOBSET_VERSION = os.getenv("JOBSET_VERSION", "v0.8.2")
+
+# Argo Workflows versions
+ARGO_WORKFLOWS_HELM_CHART_VERSION = os.getenv("ARGO_WORKFLOWS_HELM_CHART_VERSION", "0.45.2")  # Helm chart version
+ARGO_WORKFLOWS_IMAGE_TAG = os.getenv("ARGO_WORKFLOWS_IMAGE_TAG", "v3.6.0")  # Argo Workflows application version
+
+# Argo Events versions  
+ARGO_EVENTS_HELM_CHART_VERSION = os.getenv("ARGO_EVENTS_HELM_CHART_VERSION", "2.4.8")  # Helm chart version
+ARGO_EVENTS_IMAGE_TAG = os.getenv("ARGO_EVENTS_IMAGE_TAG", "v1.9.2")  # Argo Events application version
+
 components = {
     "metadata-service": ["postgresql"],
     "ui": ["postgresql", "minio"],
@@ -21,9 +32,10 @@ components = {
     "postgresql": [],
     "argo-workflows": [],
     "argo-events": ["argo-workflows"],
+    "jobset": [],
 }
 
-services_env = os.getenv("SERVICES", "").strip().lower()
+services_env = os.getenv("SERVICES", "all").strip().lower()
 
 if services_env:
     if services_env == "all":
@@ -172,6 +184,7 @@ if "postgresql" in enabled_components:
             'auth.username=metaflow',
             'auth.password=metaflow123',
             'auth.database=metaflow',
+            'image.repository=bitnamilegacy/postgresql',
             'primary.persistence.enabled=false',
             'primary.resources.requests.memory=128Mi',
             'primary.resources.requests.cpu=50m',
@@ -205,6 +218,7 @@ if "postgresql" in enabled_components:
 if "argo-workflows" in enabled_components:
     helm_remote(
         'argo-workflows',
+        version=ARGO_WORKFLOWS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -220,10 +234,25 @@ if "argo-workflows" in enabled_components:
             'controller.resources.requests.memory=128Mi',
             'controller.resources.requests.cpu=50m',
             'controller.resources.limits.memory=256Mi',
-            'controller.resources.limits.cpu=100m'
+            'controller.resources.limits.cpu=100m',
+            # Image version overrides
+            'images.tag=%s' % ARGO_WORKFLOWS_IMAGE_TAG,
         ]
     )
 
+    # This fixes issue described in: https://github.com/argoproj/argo-workflows/issues/10340
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'v1',
+        'kind': 'Secret',
+        'metadata': {
+            'name': 'default.service-account-token',
+            'annotations': {
+                'kubernetes.io/service-account.name': 'default'
+            }
+        },
+        'type': 'kubernetes.io/service-account-token'
+    }))
+
     k8s_yaml(encode_yaml({
         'apiVersion': 'rbac.authorization.k8s.io/v1',
         'kind': 'Role',
@@ -231,11 +260,23 @@ if "argo-workflows" in enabled_components:
             'name': 'argo-workflowtaskresults-role',
             'namespace': 'default'
         },
-        'rules': [{
+        'rules': [
+            {
             'apiGroups': ['argoproj.io'],
             'resources': ['workflowtaskresults'],
             'verbs': ['create', 'patch', 'get', 'list']
-        }]
+            },
+            {
+            'apiGroups': ['argoproj.io'],
+            'resources': ['workflowtasksets'],
+            'verbs': ['watch', 'list']
+            },
+            {
+            'apiGroups': ['argoproj.io'],
+            'resources': ['workflowtasksets/status'],
+            'verbs': ['patch']
+            },
+        ]
     }))
 
     k8s_yaml(encode_yaml({
@@ -282,6 +323,7 @@ if "argo-workflows" in enabled_components:
 if "argo-events" in enabled_components:
     helm_remote(
         'argo-events',
+        version=ARGO_EVENTS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -309,6 +351,8 @@ if "argo-events" in enabled_components:
             'configs.jetstream.versions[1].natsImage=nats:2.9.15',
             'configs.jetstream.versions[1].startCommand=/nats-server',
             'configs.jetstream.versions[1].version=2.9.15',
+            # Image version overrides
+            'global.image.tag=%s' % ARGO_EVENTS_IMAGE_TAG,
         ]
     )
 
@@ -516,6 +560,62 @@ if "argo-events" in enabled_components:
     config_resources.append('argo-events-controller-manager')
     config_resources.append('argo-events-webhook-eventsource-svc')
 
+#################################################
+# JOBSET
+#################################################
+if "jobset" in enabled_components:
+    # Apply JobSet manifests directly from GitHub releases
+    jobset_manifest_url = "https://github.com/kubernetes-sigs/jobset/releases/download/%s/manifests.yaml" % JOBSET_VERSION
+
+    cmd = "curl -sSL %s" % (jobset_manifest_url)
+    k8s_yaml(
+        local(
+            cmd,
+        )
+    )
+
+    k8s_resource(
+        'jobset-controller-manager',
+        labels=['jobset'],
+    )
+
+    metaflow_config["METAFLOW_KUBERNETES_JOBSET_ENABLED"] = "true"
+    
+    config_resources.append('jobset-controller-manager')
+
+    # ClusterRole for jobset operations
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'rbac.authorization.k8s.io/v1',
+        'kind': 'ClusterRole',
+        'metadata': {
+            'name': 'jobset-full-access'
+        },
+        'rules': [{
+            'apiGroups': ['jobset.x-k8s.io'],
+            'resources': ['jobsets'],
+            'verbs': ['*']
+        }]
+    }))
+
+    # ClusterRoleBinding for default service account to access jobsets
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'rbac.authorization.k8s.io/v1',
+        'kind': 'ClusterRoleBinding',
+        'metadata': {
+            'name': 'default-jobset-binding'
+        },
+        'subjects': [{
+            'kind': 'ServiceAccount',
+            'name': 'default',
+            'namespace': 'default'
+        }],
+        'roleRef': {
+            'kind': 'ClusterRole',
+            'name': 'jobset-full-access',
+            'apiGroup': 'rbac.authorization.k8s.io'
+        }
+    }))
+
 #################################################
 # METADATA SERVICE
 #################################################
@@ -530,7 +630,7 @@ if "metadata-service" in enabled_components:
             'metadatadb.database=metaflow',
             'metadatadb.host=postgresql',
             'image.repository=public.ecr.aws/outerbounds/metaflow_metadata_service',
-            'image.tag=2.4.13-2-g70af4ed',
+            'image.tag=2.5.0',
             'resources.requests.cpu=25m',
             'resources.requests.memory=64Mi',
             'resources.limits.cpu=50m',
@@ -568,7 +668,7 @@ if "ui" in enabled_components:
             'uiBackend.metaflowDatastoreSysRootS3=s3://metaflow-test',
             'uiBackend.metaflowS3EndpointURL=http://minio.default.svc.cluster.local:9000',
             'uiBackend.image.name=public.ecr.aws/outerbounds/metaflow_metadata_service',
-            'uiBackend.image.tag=2.4.13-2-g70af4ed',
+            'uiBackend.image.tag=2.5.0',
             'uiBackend.env[0].name=AWS_ACCESS_KEY_ID',
             'uiBackend.env[0].value=rootuser',
             'uiBackend.env[1].name=AWS_SECRET_ACCESS_KEY',
@@ -578,7 +678,7 @@ if "ui" in enabled_components:
             'uiBackend.resources.requests.memory=256Mi',
             'uiStatic.metaflowUIBackendURL=http://localhost:8083/api',
             'uiStatic.image.name=public.ecr.aws/outerbounds/metaflow_ui',
-            'uiStatic.image.tag=v1.3.13-5-g5dd049e',
+            'uiStatic.image.tag=v1.3.14',
             'uiStatic.resources.requests.cpu=25m',
             'uiStatic.resources.requests.memory=64Mi',
             'uiStatic.resources.limits.cpu=50m',

{ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/pick_services.sh

@@ -18,6 +18,7 @@ SERVICE_OPTIONS=(
     "ui"
     "argo-workflows"
     "argo-events"
+    "jobset"
 )
 
 gum style "$LOGO" \

{ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.15.13.1
+Version: 2.19.7.1rc0
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.13.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.19.7.1rc0; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description
@@ -85,4 +85,3 @@ We'd love to hear from you. Join our community [Slack workspace](http://slack.ou
 
 ## Contributing
 We welcome contributions to Metaflow. Please see our [contribution guide](https://docs.metaflow.org/introduction/contributing-to-metaflow) for more details.
-