ob-metaflow 2.18.7.5__py2.py3-none-any.whl → 2.18.9.1__py2.py3-none-any.whl

metaflow/includefile.py

@@ -7,9 +7,10 @@ import json
 import os
 
 from hashlib import sha1
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, Optional, Union
 
 from metaflow._vendor import click
+from metaflow._vendor import yaml
 
 from .exception import MetaflowException
 from .parameters import (
@@ -20,7 +21,7 @@ from .parameters import (
 )
 
 from .plugins import DATACLIENTS
-from .user_configs.config_parameters import ConfigValue
+from .user_configs.config_options import ConfigInput
 from .util import get_username
 
 import functools
@@ -261,6 +262,12 @@ class IncludeFile(Parameter):
     show_default : bool, default True
         If True, show the default value in the help text. A value of None is equivalent
         to True.
+    parser : Union[str, Callable[[str], Any]], optional, default None
+        If a callable, it is a function that can parse the file contents
+        into any desired format. If a string, the string should refer to
+        a function (like "my_parser_package.my_parser.my_parser_function") which should
+        be able to parse the file contents. If the name starts with a ".", it is assumed
+        to be relative to "metaflow".
     """
 
     def __init__(
@@ -270,6 +277,7 @@ class IncludeFile(Parameter):
         is_text: Optional[bool] = None,
         encoding: Optional[str] = None,
         help: Optional[str] = None,
+        parser: Optional[Union[str, Callable[[str], Any]]] = None,
         **kwargs: Dict[str, str]
     ):
         self._includefile_overrides = {}
@@ -277,6 +285,7 @@ class IncludeFile(Parameter):
             self._includefile_overrides["is_text"] = is_text
         if encoding is not None:
             self._includefile_overrides["encoding"] = encoding
+        self._parser = parser
         # NOTA: Right now, there is an issue where these can't be overridden by config
         # in all circumstances. Ignoring for now.
         super(IncludeFile, self).__init__(
@@ -336,7 +345,20 @@ class IncludeFile(Parameter):
     def load_parameter(self, v):
         if v is None:
             return v
-        return v.decode(self.name, var_type="Parameter")
+
+        # Get the raw content from the file
+        content = v.decode(self.name, var_type="Parameter")
+        # If a parser is specified, use it to parse the content
+        if self._parser is not None:
+            try:
+                return ConfigInput._call_parser(self._parser, content)
+            except Exception as e:
+                raise MetaflowException(
+                    "Failed to parse content in parameter '%s' using parser: %s"
+                    % (self.name, str(e))
+                ) from e
+
+        return content
 
     @staticmethod
     def _eval_default(is_text, encoding, default_path):

metaflow/plugins/__init__.py

@@ -170,6 +170,7 @@ DEPLOYER_IMPL_PROVIDERS_DESC = [
 ]
 
 TL_PLUGINS_DESC = [
+    ("yaml_parser", ".parsers.yaml_parser"),
     ("requirements_txt_parser", ".pypi.parsers.requirements_txt_parser"),
     ("pyproject_toml_parser", ".pypi.parsers.pyproject_toml_parser"),
     ("conda_environment_yml_parser", ".pypi.parsers.conda_environment_yml_parser"),

metaflow/plugins/datatools/s3/s3.py

@@ -1385,6 +1385,9 @@ class S3(object):
             except OSError as e:
                 if e.errno == errno.ENOSPC:
                     raise MetaflowS3InsufficientDiskSpace(str(e))
+            except MetaflowException as ex:
+                # Re-raise Metaflow exceptions (including TimeoutException)
+                raise
             except Exception as ex:
                 error = str(ex)
             if tmp:

metaflow/plugins/datatools/s3/s3op.py

@@ -50,6 +50,7 @@ import metaflow.tracing as tracing
 from metaflow.metaflow_config import (
     S3_WORKER_COUNT,
 )
+from metaflow.exception import MetaflowException
 
 DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold
 DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1
@@ -287,6 +288,11 @@ def worker(result_file_name, queue, mode, s3config):
                                 result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
                             result_file.flush()
                             continue
+                    except MetaflowException:
+                        # Re-raise Metaflow exceptions (including TimeoutException)
+                        tmp.close()
+                        os.unlink(tmp.name)
+                        raise
                     except (SSLError, Exception) as e:
                         tmp.close()
                         os.unlink(tmp.name)
@@ -357,6 +363,9 @@ def worker(result_file_name, queue, mode, s3config):
                                 err = convert_to_client_error(e)
                                 handle_client_error(err, idx, result_file)
                                 continue
+                        except MetaflowException:
+                            # Re-raise Metaflow exceptions (including TimeoutException)
+                            raise
                         except (SSLError, Exception) as e:
                             # assume anything else is transient
                             result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
@@ -385,6 +394,10 @@ def convert_to_client_error(e):
 
 
 def handle_client_error(err, idx, result_file):
+    # Handle all MetaflowExceptions as fatal
+    if isinstance(err, MetaflowException):
+        raise err
+
     error_code = normalize_client_error(err)
     if error_code == 404:
         result_file.write("%d %d\n" % (idx, -ERROR_URL_NOT_FOUND))

metaflow/plugins/parsers.py

@@ -0,0 +1,16 @@
+from metaflow._vendor import yaml
+
+
+def yaml_parser(content: str) -> dict:
+    """
+    Parse YAML content to a dictionary.
+
+    Parameters
+    ----------
+    content : str
+
+    Returns
+    -------
+    dict
+    """
+    return yaml.safe_load(content)

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.18.7.5"
+metaflow_version = "2.18.9.1"

{ob_metaflow-2.18.7.5.data → ob_metaflow-2.18.9.1.data}/data/share/metaflow/devtools/Tiltfile

@@ -14,6 +14,17 @@
 version_settings(constraint='>=0.22.2')
 allow_k8s_contexts('minikube')
 
+# Version configuration for components
+JOBSET_VERSION = os.getenv("JOBSET_VERSION", "v0.8.2")
+
+# Argo Workflows versions
+ARGO_WORKFLOWS_HELM_CHART_VERSION = os.getenv("ARGO_WORKFLOWS_HELM_CHART_VERSION", "0.45.2")  # Helm chart version
+ARGO_WORKFLOWS_IMAGE_TAG = os.getenv("ARGO_WORKFLOWS_IMAGE_TAG", "v3.6.0")  # Argo Workflows application version
+
+# Argo Events versions  
+ARGO_EVENTS_HELM_CHART_VERSION = os.getenv("ARGO_EVENTS_HELM_CHART_VERSION", "2.4.8")  # Helm chart version
+ARGO_EVENTS_IMAGE_TAG = os.getenv("ARGO_EVENTS_IMAGE_TAG", "v1.9.2")  # Argo Events application version
+
 components = {
     "metadata-service": ["postgresql"],
     "ui": ["postgresql", "minio"],
@@ -21,9 +32,10 @@ components = {
     "postgresql": [],
     "argo-workflows": [],
     "argo-events": ["argo-workflows"],
+    "jobset": [],
 }
 
-services_env = os.getenv("SERVICES", "").strip().lower()
+services_env = os.getenv("SERVICES", "all").strip().lower()
 
 if services_env:
     if services_env == "all":
@@ -205,6 +217,7 @@ if "postgresql" in enabled_components:
 if "argo-workflows" in enabled_components:
     helm_remote(
         'argo-workflows',
+        version=ARGO_WORKFLOWS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -220,7 +233,9 @@ if "argo-workflows" in enabled_components:
             'controller.resources.requests.memory=128Mi',
             'controller.resources.requests.cpu=50m',
             'controller.resources.limits.memory=256Mi',
-            'controller.resources.limits.cpu=100m'
+            'controller.resources.limits.cpu=100m',
+            # Image version overrides
+            'images.tag=%s' % ARGO_WORKFLOWS_IMAGE_TAG,
         ]
     )
 
@@ -307,6 +322,7 @@ if "argo-workflows" in enabled_components:
 if "argo-events" in enabled_components:
     helm_remote(
         'argo-events',
+        version=ARGO_EVENTS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -334,6 +350,8 @@ if "argo-events" in enabled_components:
             'configs.jetstream.versions[1].natsImage=nats:2.9.15',
             'configs.jetstream.versions[1].startCommand=/nats-server',
             'configs.jetstream.versions[1].version=2.9.15',
+            # Image version overrides
+            'global.image.tag=%s' % ARGO_EVENTS_IMAGE_TAG,
         ]
     )
 
@@ -541,6 +559,62 @@ if "argo-events" in enabled_components:
     config_resources.append('argo-events-controller-manager')
     config_resources.append('argo-events-webhook-eventsource-svc')
 
+#################################################
+# JOBSET
+#################################################
+if "jobset" in enabled_components:
+    # Apply JobSet manifests directly from GitHub releases
+    jobset_manifest_url = "https://github.com/kubernetes-sigs/jobset/releases/download/%s/manifests.yaml" % JOBSET_VERSION
+
+    cmd = "curl -sSL %s" % (jobset_manifest_url)
+    k8s_yaml(
+        local(
+            cmd,
+        )
+    )
+
+    k8s_resource(
+        'jobset-controller-manager',
+        labels=['jobset'],
+    )
+
+    metaflow_config["METAFLOW_KUBERNETES_JOBSET_ENABLED"] = "true"
+    
+    config_resources.append('jobset-controller-manager')
+
+    # ClusterRole for jobset operations
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'rbac.authorization.k8s.io/v1',
+        'kind': 'ClusterRole',
+        'metadata': {
+            'name': 'jobset-full-access'
+        },
+        'rules': [{
+            'apiGroups': ['jobset.x-k8s.io'],
+            'resources': ['jobsets'],
+            'verbs': ['*']
+        }]
+    }))
+
+    # ClusterRoleBinding for default service account to access jobsets
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'rbac.authorization.k8s.io/v1',
+        'kind': 'ClusterRoleBinding',
+        'metadata': {
+            'name': 'default-jobset-binding'
+        },
+        'subjects': [{
+            'kind': 'ServiceAccount',
+            'name': 'default',
+            'namespace': 'default'
+        }],
+        'roleRef': {
+            'kind': 'ClusterRole',
+            'name': 'jobset-full-access',
+            'apiGroup': 'rbac.authorization.k8s.io'
+        }
+    }))
+
 #################################################
 # METADATA SERVICE
 #################################################

{ob_metaflow-2.18.7.5.data → ob_metaflow-2.18.9.1.data}/data/share/metaflow/devtools/pick_services.sh

@@ -18,6 +18,7 @@ SERVICE_OPTIONS=(
     "ui"
     "argo-workflows"
     "argo-events"
+    "jobset"
 )
 
 gum style "$LOGO" \

{ob_metaflow-2.18.7.5.dist-info → ob_metaflow-2.18.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.18.7.5
+Version: 2.18.9.1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.7.5; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.9.1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description
@@ -86,3 +86,4 @@ We'd love to hear from you. Join our community [Slack workspace](http://slack.ou
 ## Contributing
 We welcome contributions to Metaflow. Please see our [contribution guide](https://docs.metaflow.org/introduction/contributing-to-metaflow) for more details.
 
+

{ob_metaflow-2.18.7.5.dist-info → ob_metaflow-2.18.9.1.dist-info}/RECORD

@@ -12,7 +12,7 @@ metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
 metaflow/exception.py,sha256=_m9ZBJM0cooHRslDqfxCPQmkChqaTh6fGxp7HvISnYI,5161
 metaflow/flowspec.py,sha256=9wsO2_QoO_VHKusKdpslfbwQREOwf0fAzF-DSEA0iZ8,41968
 metaflow/graph.py,sha256=UOeClj-JeORRlZWOQMI1CirkrCpTvVWvRcSwODCajMg,19263
-metaflow/includefile.py,sha256=RtISGl1V48qjkJBakUZ9yPpHV102h7pOIFiKP8PLHpc,20927
+metaflow/includefile.py,sha256=NXERo_halboBCjvnS5iCjnBMyCMlQMnT2mRe6kxl2xU,21978
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=A2NdUq_MnQal_RUCMC8ZOSR0VYZGyi2mSgwPQB0UzQo,15343
 metaflow/meta_files.py,sha256=vlgJHI8GJUKzXoxdrVoH8yyCF5bhFgwYemUgnyd1wgM,342
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=g2SOU_CRzJLgDM_UGF9QDMANMAIHAsDRXE6S76_YzsY,14594
 metaflow/vendor.py,sha256=A82CGHfStZGDP5pQ5XzRjFkbN1ZC-vFmghXIrzMDDNg,5868
-metaflow/version.py,sha256=wr7-KAbKDrPvB4XEDBnfNF9mwf_ORPReiVhKweI1UQ4,30
+metaflow/version.py,sha256=VXnTFv7difhCqOiu2Rrnc4LOaj3sTVQlAoY6KrPgrWQ,30
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -198,7 +198,7 @@ metaflow/packaging_sys/distribution_support.py,sha256=VvikZBCH8N1TBZZ2Twk8jH1brm
 metaflow/packaging_sys/tar_backend.py,sha256=nFWuXiwYjWQkFdV2KaZ6gazNVvtY84Eqsh9txhU3pNY,3010
 metaflow/packaging_sys/utils.py,sha256=x8SVglJvY5mIAilS7MqZi2PpMr6IEyi6RCg3l8hN3G0,2972
 metaflow/packaging_sys/v1.py,sha256=kbNK0-pDAv3QJPZ789TE0UirGXcHbXkVQiyNT815H7A,20631
-metaflow/plugins/__init__.py,sha256=yFxjJOlnfap7tQMNgSgaso2tl_zr1BcWL7KoUKk4c9Y,8617
+metaflow/plugins/__init__.py,sha256=XBocLBSWEPAeeakhTb6HGuocykkVukmnUuQgg-gGWEk,8662
 metaflow/plugins/catch_decorator.py,sha256=vorlDA6MLB2yHSsEuBoNzAZbrJ6Vknj1qJO9vey2_AI,4523
 metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
 metaflow/plugins/debug_monitor.py,sha256=Md5X_sDOSssN9pt2D8YcaIjTK5JaQD55UAYTcF6xYF0,1099
@@ -207,6 +207,7 @@ metaflow/plugins/events_decorator.py,sha256=FULT_Iuue9KsLyNhHOL46uw8cdT-Viq8rmcg
 metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,11410
 metaflow/plugins/package_cli.py,sha256=GOxKJs9Wt0x9VtcTJ2EG_wj6pWnlS0XBFAyHpCzzuvs,2000
 metaflow/plugins/parallel_decorator.py,sha256=wtR_3eRIP3eV7fBIm15oouRjmHBFZ9OklxdaNvttLEQ,9702
+metaflow/plugins/parsers.py,sha256=y92kMEFmbBvzU7EjZSFgml3S6EOLwY5Qmb4Jwk7leUw,249
 metaflow/plugins/project_decorator.py,sha256=uhwsguEj7OM_E2OnY1ap3MoGocQHeywuJSa-qPuWn-U,7592
 metaflow/plugins/resources_decorator.py,sha256=AtoOwg4mHYHYthg-CAfbfam-QiT0ViuDLDoukoDvF6Q,1347
 metaflow/plugins/retry_decorator.py,sha256=tz_2Tq6GLg3vjDBZp0KKVTk3ADlCvqaWTSf7blmFdUw,1548
@@ -306,8 +307,8 @@ metaflow/plugins/datastores/s3_storage.py,sha256=CZdNqaKtxDXQbEg2YHyphph3hWcLIE5
 metaflow/plugins/datatools/__init__.py,sha256=ge4L16OBQLy2J_MMvoHg3lMfdm-MluQgRWoyZ5GCRnk,1267
 metaflow/plugins/datatools/local.py,sha256=FJvMOBcjdyhSPHmdLocBSiIT0rmKkKBmsaclxH75x08,4233
 metaflow/plugins/datatools/s3/__init__.py,sha256=14tr9fPjN3ULW5IOfKHeG7Uhjmgm7LMtQHfz1SFv-h8,248
-metaflow/plugins/datatools/s3/s3.py,sha256=3xrWD6pXoVRpuAQHyLGVya79UIE0S8AqAqe2prg4yMw,67182
-metaflow/plugins/datatools/s3/s3op.py,sha256=nDFgc5m0ud-uK4D2_vyweGpGoMjESxu3TlX86zA1MdQ,47561
+metaflow/plugins/datatools/s3/s3.py,sha256=b07QwYvfKvR1k0eZ_b5Hk4XQnQs3ORk6b0nvCwfZL8w,67324
+metaflow/plugins/datatools/s3/s3op.py,sha256=3GOtIV08gmAo7XE6NNnsSUxTTRIsvgPJn7CC0U5E7XY,48129
 metaflow/plugins/datatools/s3/s3tail.py,sha256=boQjQGQMI-bvTqcMP2y7uSlSYLcvWOy7J3ZUaF78NAA,2597
 metaflow/plugins/datatools/s3/s3util.py,sha256=FgRgaVmEq7-i2dV7q8XK5w5PfFt-xJjZa8WrK8IJfdI,3769
 metaflow/plugins/env_escape/__init__.py,sha256=tGNUZnmPvk52eNs__VK443b3CZ7ogEFTT-s9_n_HF8Q,8837
@@ -428,12 +429,12 @@ metaflow/user_decorators/mutable_flow.py,sha256=EywKTN3cnXPQF_s62wQaC4a4aH14j8oe
 metaflow/user_decorators/mutable_step.py,sha256=-BY0UDXf_RCAEnC5JlLzEXGdiw1KD9oSrSxS_SWaB9Y,16791
 metaflow/user_decorators/user_flow_decorator.py,sha256=2yDwZq9QGv9W-7kEuKwa8o4ZkTvuHJ5ESz7VVrGViAI,9890
 metaflow/user_decorators/user_step_decorator.py,sha256=4558NR8RJtN22OyTwCXO80bAMhMTaRGMoX12b1GMcPc,27232
-ob_metaflow-2.18.7.5.data/data/share/metaflow/devtools/Makefile,sha256=TT4TCq8ALSfqYyGqDPocN5oPcZe2FqoCZxmGO1LmyCc,13760
-ob_metaflow-2.18.7.5.data/data/share/metaflow/devtools/Tiltfile,sha256=Ty5p6AD3MwJAcAnOGv4yMz8fExAsnNQ11r8whK6uzzw,21381
-ob_metaflow-2.18.7.5.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
-ob_metaflow-2.18.7.5.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.18.7.5.dist-info/METADATA,sha256=7S0AlPeNebxn0NgvuCNDdr867lfbZuZHnf03RlrvslA,5935
-ob_metaflow-2.18.7.5.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-ob_metaflow-2.18.7.5.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
-ob_metaflow-2.18.7.5.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.18.7.5.dist-info/RECORD,,
+ob_metaflow-2.18.9.1.data/data/share/metaflow/devtools/Makefile,sha256=TT4TCq8ALSfqYyGqDPocN5oPcZe2FqoCZxmGO1LmyCc,13760
+ob_metaflow-2.18.9.1.data/data/share/metaflow/devtools/Tiltfile,sha256=8-VMYGdEBFp73Ur5BzQn7TxxkMBMhwJszsVhR_ftgBU,23892
+ob_metaflow-2.18.9.1.data/data/share/metaflow/devtools/pick_services.sh,sha256=PGjQeDIigFHeoQ0asmYNdYDPIOdeYy1UYvkw2wdN4zg,2209
+ob_metaflow-2.18.9.1.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.18.9.1.dist-info/METADATA,sha256=nlw2RXeC4Si_MUdE4HsupPx9ujhkoFYlyobwPy-jt8M,5936
+ob_metaflow-2.18.9.1.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+ob_metaflow-2.18.9.1.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
+ob_metaflow-2.18.9.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.18.9.1.dist-info/RECORD,,