metaflow 2.18.6__tar.gz → 2.19.7__tar.gz

{metaflow-2.18.6/metaflow.egg-info → metaflow-2.19.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.18.6
+Version: 2.19.7
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.6; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.19.7; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -101,4 +101,3 @@ We'd love to hear from you. Join our community [Slack workspace](http://slack.ou
 
 ## Contributing
 We welcome contributions to Metaflow. Please see our [contribution guide](https://docs.metaflow.org/introduction/contributing-to-metaflow) for more details.
-

{metaflow-2.18.6 → metaflow-2.19.7}/README.md

@@ -60,4 +60,3 @@ We'd love to hear from you. Join our community [Slack workspace](http://slack.ou
 
 ## Contributing
 We welcome contributions to Metaflow. Please see our [contribution guide](https://docs.metaflow.org/introduction/contributing-to-metaflow) for more details.
-

{metaflow-2.18.6 → metaflow-2.19.7}/devtools/Makefile

@@ -31,6 +31,7 @@ MAKE_CMD := $(MAKE) -f "$(MKFILE_PATH)"
 MINIKUBE_CPUS ?= 4
 MINIKUBE_MEMORY ?= 6144
 MINIKUBE_DISK_SIZE ?= 20g
+WAIT_TIMEOUT ?= 300
 
 ifeq ($(shell uname), Darwin)
 	minikube_os = darwin
@@ -269,6 +270,15 @@ shell: setup-tilt
 				"$$user_shell" -i; \
 		fi'
 
+wait-until-ready:
+	@echo "Waiting for infrastructure to be ready. Timing out in $(WAIT_TIMEOUT) seconds..."
+	@timeout $(WAIT_TIMEOUT) bash -c 'while [ ! -f $(DEVTOOLS_DIR)/start.sh ]; do sleep 10; done; echo "Infra is Ready"' || (echo "Waiting for infra timed out"&&exit 1)
+# buffer to get the tilt api running
+	@timeout 120 bash -c 'while ! $(TILT) get session; do sleep 3;done'
+	@echo "Waiting for services to be ready. Timing out in $(WAIT_TIMEOUT) seconds..."
+# Need to wait for Tiltfile first, as other resources return 404 otherwise
+	@$(TILT) wait --for=condition=Ready "uiresource/(Tiltfile)" --timeout=$(WAIT_TIMEOUT)s
+	@$(TILT) wait --for=condition=Ready uiresource/generate-configs --timeout=$(WAIT_TIMEOUT)s
 
 # @echo '$(MAKE_CMD) create-dev-shell' >> $(DEVTOOLS_DIR)/start.sh
 # @echo 'rm -f /tmp/metaflow-devshell-*' >> $(DEVTOOLS_DIR)/start.sh

{metaflow-2.18.6 → metaflow-2.19.7}/devtools/Tiltfile

@@ -14,6 +14,17 @@
 version_settings(constraint='>=0.22.2')
 allow_k8s_contexts('minikube')
 
+# Version configuration for components
+JOBSET_VERSION = os.getenv("JOBSET_VERSION", "v0.8.2")
+
+# Argo Workflows versions
+ARGO_WORKFLOWS_HELM_CHART_VERSION = os.getenv("ARGO_WORKFLOWS_HELM_CHART_VERSION", "0.45.2")  # Helm chart version
+ARGO_WORKFLOWS_IMAGE_TAG = os.getenv("ARGO_WORKFLOWS_IMAGE_TAG", "v3.6.0")  # Argo Workflows application version
+
+# Argo Events versions  
+ARGO_EVENTS_HELM_CHART_VERSION = os.getenv("ARGO_EVENTS_HELM_CHART_VERSION", "2.4.8")  # Helm chart version
+ARGO_EVENTS_IMAGE_TAG = os.getenv("ARGO_EVENTS_IMAGE_TAG", "v1.9.2")  # Argo Events application version
+
 components = {
     "metadata-service": ["postgresql"],
     "ui": ["postgresql", "minio"],
@@ -21,9 +32,10 @@ components = {
     "postgresql": [],
     "argo-workflows": [],
     "argo-events": ["argo-workflows"],
+    "jobset": [],
 }
 
-services_env = os.getenv("SERVICES", "").strip().lower()
+services_env = os.getenv("SERVICES", "all").strip().lower()
 
 if services_env:
     if services_env == "all":
@@ -172,6 +184,7 @@ if "postgresql" in enabled_components:
             'auth.username=metaflow',
             'auth.password=metaflow123',
             'auth.database=metaflow',
+            'image.repository=bitnamilegacy/postgresql',
             'primary.persistence.enabled=false',
             'primary.resources.requests.memory=128Mi',
             'primary.resources.requests.cpu=50m',
@@ -205,6 +218,7 @@ if "postgresql" in enabled_components:
 if "argo-workflows" in enabled_components:
     helm_remote(
         'argo-workflows',
+        version=ARGO_WORKFLOWS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -220,7 +234,9 @@ if "argo-workflows" in enabled_components:
             'controller.resources.requests.memory=128Mi',
             'controller.resources.requests.cpu=50m',
             'controller.resources.limits.memory=256Mi',
-            'controller.resources.limits.cpu=100m'
+            'controller.resources.limits.cpu=100m',
+            # Image version overrides
+            'images.tag=%s' % ARGO_WORKFLOWS_IMAGE_TAG,
         ]
     )
 
@@ -307,6 +323,7 @@ if "argo-workflows" in enabled_components:
 if "argo-events" in enabled_components:
     helm_remote(
         'argo-events',
+        version=ARGO_EVENTS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -334,6 +351,8 @@ if "argo-events" in enabled_components:
             'configs.jetstream.versions[1].natsImage=nats:2.9.15',
             'configs.jetstream.versions[1].startCommand=/nats-server',
             'configs.jetstream.versions[1].version=2.9.15',
+            # Image version overrides
+            'global.image.tag=%s' % ARGO_EVENTS_IMAGE_TAG,
         ]
     )
 
@@ -541,6 +560,62 @@ if "argo-events" in enabled_components:
     config_resources.append('argo-events-controller-manager')
     config_resources.append('argo-events-webhook-eventsource-svc')
 
+#################################################
+# JOBSET
+#################################################
+if "jobset" in enabled_components:
+    # Apply JobSet manifests directly from GitHub releases
+    jobset_manifest_url = "https://github.com/kubernetes-sigs/jobset/releases/download/%s/manifests.yaml" % JOBSET_VERSION
+
+    cmd = "curl -sSL %s" % (jobset_manifest_url)
+    k8s_yaml(
+        local(
+            cmd,
+        )
+    )
+
+    k8s_resource(
+        'jobset-controller-manager',
+        labels=['jobset'],
+    )
+
+    metaflow_config["METAFLOW_KUBERNETES_JOBSET_ENABLED"] = "true"
+    
+    config_resources.append('jobset-controller-manager')
+
+    # ClusterRole for jobset operations
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'rbac.authorization.k8s.io/v1',
+        'kind': 'ClusterRole',
+        'metadata': {
+            'name': 'jobset-full-access'
+        },
+        'rules': [{
+            'apiGroups': ['jobset.x-k8s.io'],
+            'resources': ['jobsets'],
+            'verbs': ['*']
+        }]
+    }))
+
+    # ClusterRoleBinding for default service account to access jobsets
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'rbac.authorization.k8s.io/v1',
+        'kind': 'ClusterRoleBinding',
+        'metadata': {
+            'name': 'default-jobset-binding'
+        },
+        'subjects': [{
+            'kind': 'ServiceAccount',
+            'name': 'default',
+            'namespace': 'default'
+        }],
+        'roleRef': {
+            'kind': 'ClusterRole',
+            'name': 'jobset-full-access',
+            'apiGroup': 'rbac.authorization.k8s.io'
+        }
+    }))
+
 #################################################
 # METADATA SERVICE
 #################################################
@@ -555,7 +630,7 @@ if "metadata-service" in enabled_components:
             'metadatadb.database=metaflow',
             'metadatadb.host=postgresql',
             'image.repository=public.ecr.aws/outerbounds/metaflow_metadata_service',
-            'image.tag=2.4.13-2-g70af4ed',
+            'image.tag=2.5.0',
             'resources.requests.cpu=25m',
             'resources.requests.memory=64Mi',
             'resources.limits.cpu=50m',

{metaflow-2.18.6 → metaflow-2.19.7}/devtools/pick_services.sh

@@ -18,6 +18,7 @@ SERVICE_OPTIONS=(
     "ui"
     "argo-workflows"
     "argo-events"
+    "jobset"
 )
 
 gum style "$LOGO" \

{metaflow-2.18.6 → metaflow-2.19.7}/metaflow/__init__.py

@@ -146,6 +146,7 @@ from .client import (
     metadata,
     get_metadata,
     default_metadata,
+    inspect_spin,
     Metaflow,
     Flow,
     Run,

{metaflow-2.18.6 → metaflow-2.19.7}/metaflow/cards.py

@@ -8,7 +8,11 @@ from metaflow.plugins.cards.card_modules.components import (
     Markdown,
     VegaChart,
     ProgressBar,
+    ValueBox,
     PythonCode,
+    EventsTimeline,
+    JSONViewer,
+    YAMLViewer,
 )
 from metaflow.plugins.cards.card_modules.basic import (
     DefaultCard,

{metaflow-2.18.6 → metaflow-2.19.7}/metaflow/cli.py

@@ -1,3 +1,4 @@
+import os
 import functools
 import inspect
 import os
@@ -7,7 +8,6 @@ from datetime import datetime
 
 import metaflow.tracing as tracing
 from metaflow._vendor import click
-from metaflow.system import _system_logger, _system_monitor
 
 from . import decorators, lint, metaflow_version, parameters, plugins
 from .cli_args import cli_args
@@ -27,6 +27,8 @@ from .metaflow_config import (
     DEFAULT_PACKAGE_SUFFIXES,
 )
 from .metaflow_current import current
+from .metaflow_profile import from_start
+from metaflow.system import _system_monitor, _system_logger
 from .metaflow_environment import MetaflowEnvironment
 from .packaging_sys import MetaflowCodeContent
 from .plugins import (
@@ -38,9 +40,9 @@ from .plugins import (
 )
 from .pylint_wrapper import PyLint
 from .R import metaflow_r_version, use_r
+from .util import get_latest_run_id, resolve_identity, decompress_list
 from .user_configs.config_options import LocalFileInput, config_options
 from .user_configs.config_parameters import ConfigValue
-from .util import get_latest_run_id, resolve_identity
 
 ERASE_TO_EOL = "\033[K"
 HIGHLIGHT = "red"
@@ -125,6 +127,8 @@ def logger(body="", system_msg=False, head="", bad=False, timestamp=True, nl=Tru
         "step": "metaflow.cli_components.step_cmd.step",
         "run": "metaflow.cli_components.run_cmds.run",
         "resume": "metaflow.cli_components.run_cmds.resume",
+        "spin": "metaflow.cli_components.run_cmds.spin",
+        "spin-step": "metaflow.cli_components.step_cmd.spin_step",
     },
 )
 def cli(ctx):
@@ -318,6 +322,13 @@ def version(obj):
     hidden=True,
     is_eager=True,
 )
+@click.option(
+    "--mode",
+    type=click.Choice(["spin"]),
+    default=None,
+    help="Execution mode for metaflow CLI commands. Use 'spin' to enable "
+    "spin metadata and spin datastore for executions",
+)
 @click.pass_context
 def start(
     ctx,
@@ -335,6 +346,7 @@ def start(
     local_config_file=None,
     config=None,
     config_value=None,
+    mode=None,
     **deco_options
 ):
     if quiet:
@@ -347,6 +359,7 @@ def start(
     if use_r():
         version = metaflow_r_version()
 
+    from_start("MetaflowCLI: Starting")
     echo("Metaflow %s" % version, fg="magenta", bold=True, nl=False)
     echo(" executing *%s*" % ctx.obj.flow.name, fg="magenta", nl=False)
     echo(" for *%s*" % resolve_identity(), fg="magenta")
@@ -366,6 +379,7 @@ def start(
     ctx.obj.check = functools.partial(_check, echo)
     ctx.obj.top_cli = cli
     ctx.obj.package_suffixes = package_suffixes.split(",")
+    ctx.obj.spin_mode = mode == "spin"
 
     ctx.obj.datastore_impl = [d for d in DATASTORES if d.TYPE == datastore][0]
 
@@ -472,19 +486,12 @@ def start(
     # set force rebuild flag for environments that support it.
     ctx.obj.environment._force_rebuild = force_rebuild_environments
     ctx.obj.environment.validate_environment(ctx.obj.logger, datastore)
-
     ctx.obj.event_logger = LOGGING_SIDECARS[event_logger](
         flow=ctx.obj.flow, env=ctx.obj.environment
     )
-    ctx.obj.event_logger.start()
-    _system_logger.init_system_logger(ctx.obj.flow.name, ctx.obj.event_logger)
-
     ctx.obj.monitor = MONITOR_SIDECARS[monitor](
         flow=ctx.obj.flow, env=ctx.obj.environment
     )
-    ctx.obj.monitor.start()
-    _system_monitor.init_system_monitor(ctx.obj.flow.name, ctx.obj.monitor)
-
     ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0](
         ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor
     )
@@ -498,6 +505,57 @@ def start(
     )
 
     ctx.obj.config_options = config_options
+    ctx.obj.is_spin = False
+    ctx.obj.skip_decorators = False
+
+    # Override values for spin steps, or if we are in spin mode
+    if (
+        hasattr(ctx, "saved_args")
+        and ctx.saved_args
+        and "spin" in ctx.saved_args[0]
+        or ctx.obj.spin_mode
+    ):
+        # To minimize side effects for spin, we will only use the following:
+        # - local metadata provider,
+        # - local datastore,
+        # - local environment,
+        # - null event logger,
+        # - null monitor
+        ctx.obj.is_spin = True
+        if "--skip-decorators" in ctx.saved_args:
+            ctx.obj.skip_decorators = True
+
+        ctx.obj.event_logger = LOGGING_SIDECARS["nullSidecarLogger"](
+            flow=ctx.obj.flow, env=ctx.obj.environment
+        )
+        ctx.obj.monitor = MONITOR_SIDECARS["nullSidecarMonitor"](
+            flow=ctx.obj.flow, env=ctx.obj.environment
+        )
+        # Use spin metadata, spin datastore, and spin datastore root
+        ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == "spin"][0](
+            ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor
+        )
+        ctx.obj.datastore_impl = [d for d in DATASTORES if d.TYPE == "spin"][0]
+        datastore_root = ctx.obj.datastore_impl.get_datastore_root_from_config(
+            ctx.obj.echo, create_on_absent=True
+        )
+        ctx.obj.datastore_impl.datastore_root = datastore_root
+
+        ctx.obj.flow_datastore = FlowDataStore(
+            ctx.obj.flow.name,
+            ctx.obj.environment,  # Same environment as run/resume
+            ctx.obj.metadata,  # local metadata
+            ctx.obj.event_logger,  # null event logger
+            ctx.obj.monitor,  # null monitor
+            storage_impl=ctx.obj.datastore_impl,
+        )
+
+    # Start event logger and monitor
+    ctx.obj.event_logger.start()
+    _system_logger.init_system_logger(ctx.obj.flow.name, ctx.obj.event_logger)
+
+    ctx.obj.monitor.start()
+    _system_monitor.init_system_monitor(ctx.obj.flow.name, ctx.obj.monitor)
 
     decorators._init(ctx.obj.flow)
 
@@ -512,9 +570,11 @@ def start(
         ctx.obj.logger,
         echo,
         deco_options,
+        ctx.obj.is_spin,
+        ctx.obj.skip_decorators,
     )
 
-    # In the case of run/resume, we will want to apply the TL decospecs
+    # In the case of run/resume/spin, we will want to apply the TL decospecs
     # *after* the run decospecs so that they don't take precedence. In other
     # words, for the same decorator, we want `myflow.py run --with foo` to
     # take precedence over any other `foo` decospec
@@ -542,11 +602,10 @@ def start(
     if (
         hasattr(ctx, "saved_args")
         and ctx.saved_args
-        and ctx.saved_args[0] not in ("run", "resume")
+        and ctx.saved_args[0] not in ("run", "resume", "spin")
     ):
-        # run/resume are special cases because they can add more decorators with --with,
+        # run/resume/spin are special cases because they can add more decorators with --with,
         # so they have to take care of themselves.
-
         all_decospecs = ctx.obj.tl_decospecs + list(
             ctx.obj.environment.decospecs() or []
         )
@@ -556,6 +615,9 @@ def start(
         # or a scheduler setting them up in their own way.
         if ctx.saved_args[0] not in ("step", "init"):
             all_decospecs += DEFAULT_DECOSPECS.split()
+        elif ctx.saved_args[0] == "spin-step":
+            # If we are in spin-args, we will not attach any decorators
+            all_decospecs = []
         if all_decospecs:
             decorators._attach_decorators(ctx.obj.flow, all_decospecs)
             decorators._init(ctx.obj.flow)
@@ -569,6 +631,9 @@ def start(
             ctx.obj.environment,
             ctx.obj.flow_datastore,
             ctx.obj.logger,
+            # The last two arguments are only used for spin steps
+            ctx.obj.is_spin,
+            ctx.obj.skip_decorators,
         )
 
         # Check the graph again (mutators may have changed it)