metaflow 2.12.17__py2.py3-none-any.whl → 2.12.19__py2.py3-none-any.whl

metaflow/metaflow_environment.py

@@ -98,7 +98,7 @@ class MetaflowEnvironment(object):
             # Boto3 does not play well with passing None or an empty string to endpoint_url
             return "{python} -c '{script}'".format(
                 python=self._python(),
-                script='import boto3, os; ep=os.getenv(\\"METAFLOW_S3_ENDPOINT_URL\\"); boto3.client(\\"s3\\", **({endpoint_url:ep} if ep else {})).download_file(\\"%s\\", \\"%s\\", \\"job.tar\\")'
+                script='import boto3, os; ep=os.getenv(\\"METAFLOW_S3_ENDPOINT_URL\\"); boto3.client(\\"s3\\", **({\\"endpoint_url\\":ep} if ep else {})).download_file(\\"%s\\", \\"%s\\", \\"job.tar\\")'
                 % (bucket, s3_object),
             )
         elif datastore_type == "azure":

metaflow/plugins/kubernetes/kubernetes.py

@@ -698,7 +698,7 @@ class Kubernetes(object):
                     t = time.time()
                 time.sleep(update_delay(time.time() - start_time))
 
-        _make_prefix = lambda: b"[%s] " % util.to_bytes(self._job.id)
+        prefix = lambda: b"[%s] " % util.to_bytes(self._job.id)
 
         stdout_tail = get_log_tailer(stdout_location, self._datastore.TYPE)
         stderr_tail = get_log_tailer(stderr_location, self._datastore.TYPE)
@@ -708,7 +708,7 @@ class Kubernetes(object):
 
         # 2) Tail logs until the job has finished
         tail_logs(
-            prefix=_make_prefix(),
+            prefix=prefix(),
             stdout_tail=stdout_tail,
             stderr_tail=stderr_tail,
             echo=echo,

metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -571,10 +571,8 @@ class JobSetSpec(object):
                                 namespace=self._kwargs["namespace"],
                             ),
                             spec=client.V1PodSpec(
-                                ##  --- jobset require podspec deets start----
                                 subdomain=self._kwargs["subdomain"],
                                 set_hostname_as_fqdn=True,
-                                ##  --- jobset require podspec deets end ----
                                 # Timeout is set on the pod and not the job (important!)
                                 active_deadline_seconds=self._kwargs[
                                     "timeout_in_seconds"

metaflow/plugins/pypi/conda_decorator.py

@@ -50,6 +50,9 @@ class CondaStepDecorator(StepDecorator):
     # conda channels, users can specify channel::package as the package name.
 
     def __init__(self, attributes=None, statically_defined=False):
+        self._user_defined_attributes = (
+            attributes.copy() if attributes is not None else {}
+        )
         super(CondaStepDecorator, self).__init__(attributes, statically_defined)
 
         # Support legacy 'libraries=' attribute for the decorator.
@@ -59,6 +62,9 @@ class CondaStepDecorator(StepDecorator):
         }
         del self.attributes["libraries"]
 
+    def is_attribute_user_defined(self, name):
+        return name in self._user_defined_attributes
+
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
         # The init_environment hook for Environment creates the relevant virtual
         # environments. The step_init hook sets up the relevant state for that hook to
@@ -71,11 +77,16 @@ class CondaStepDecorator(StepDecorator):
 
         # Support flow-level decorator.
         if "conda_base" in self.flow._flow_decorators:
-            super_attributes = self.flow._flow_decorators["conda_base"][0].attributes
+            conda_base = self.flow._flow_decorators["conda_base"][0]
+            super_attributes = conda_base.attributes
             self.attributes["packages"] = {
                 **super_attributes["packages"],
                 **self.attributes["packages"],
             }
+            self._user_defined_attributes = {
+                **self._user_defined_attributes,
+                **conda_base._user_defined_attributes,
+            }
             self.attributes["python"] = (
                 self.attributes["python"] or super_attributes["python"]
             )
@@ -322,6 +333,9 @@ class CondaFlowDecorator(FlowDecorator):
     }
 
     def __init__(self, attributes=None, statically_defined=False):
+        self._user_defined_attributes = (
+            attributes.copy() if attributes is not None else {}
+        )
         super(CondaFlowDecorator, self).__init__(attributes, statically_defined)
 
         # Support legacy 'libraries=' attribute for the decorator.
@@ -333,6 +347,9 @@ class CondaFlowDecorator(FlowDecorator):
         if self.attributes["python"]:
             self.attributes["python"] = str(self.attributes["python"])
 
+    def is_attribute_user_defined(self, name):
+        return name in self._user_defined_attributes
+
     def flow_init(
         self, flow, graph, environment, flow_datastore, metadata, logger, echo, options
     ):

metaflow/plugins/pypi/conda_environment.py

@@ -185,7 +185,10 @@ class CondaEnvironment(MetaflowEnvironment):
         self.logger("Virtual environment(s) bootstrapped!")
 
     def executable(self, step_name, default=None):
-        step = next(step for step in self.flow if step.name == step_name)
+        step = next((step for step in self.flow if step.name == step_name), None)
+        if step is None:
+            # requesting internal steps e.g. _parameters
+            return super().executable(step_name, default)
         id_ = self.get_environment(step).get("id_")
         if id_:
             # bootstrap.py is responsible for ensuring the validity of this executable.

metaflow/plugins/pypi/pypi_decorator.py

@@ -40,7 +40,12 @@ class PyPIStepDecorator(StepDecorator):
 
         # Support flow-level decorator
         if "pypi_base" in self.flow._flow_decorators:
-            super_attributes = self.flow._flow_decorators["pypi_base"][0].attributes
+            pypi_base = self.flow._flow_decorators["pypi_base"][0]
+            super_attributes = pypi_base.attributes
+            self._user_defined_attributes = {
+                **self._user_defined_attributes,
+                **pypi_base._user_defined_attributes,
+            }
             self.attributes["packages"] = {
                 **super_attributes["packages"],
                 **self.attributes["packages"],
@@ -93,6 +98,12 @@ class PyPIStepDecorator(StepDecorator):
                     ),
                 )
             )
+        # TODO: This code snippet can be done away with by altering the constructor of
+        #       MetaflowEnvironment. A good first-task exercise.
+        # Avoid circular import
+        from metaflow.plugins.datastores.local_storage import LocalStorage
+
+        environment.set_local_root(LocalStorage.get_datastore_root_from_config(logger))
 
     def is_attribute_user_defined(self, name):
         return name in self._user_defined_attributes
@@ -117,6 +128,12 @@ class PyPIFlowDecorator(FlowDecorator):
     name = "pypi_base"
     defaults = {"packages": {}, "python": None, "disabled": None}
 
+    def __init__(self, attributes=None, statically_defined=False):
+        self._user_defined_attributes = (
+            attributes.copy() if attributes is not None else {}
+        )
+        super().__init__(attributes, statically_defined)
+
     def flow_init(
         self, flow, graph, environment, flow_datastore, metadata, logger, echo, options
     ):

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.17"
+metaflow_version = "2.12.19"

{metaflow-2.12.17.dist-info → metaflow-2.12.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.17
+Version: 2.12.19
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.17; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.19; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.17.dist-info → metaflow-2.12.19.dist-info}/RECORD

@@ -18,7 +18,7 @@ metaflow/lint.py,sha256=5rj1MlpluxyPTSINjtMoJ7viotyNzfjtBJSAihlAwMU,10870
 metaflow/metaflow_config.py,sha256=4PUd2-JWJs35SaobDgMg4RdpZaKjhEqPUFh2f0pjqnU,22853
 metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
 metaflow/metaflow_current.py,sha256=5Kri7fzj-rtIJVr5xh5kPKwZ0T73_4egZybzlDR-fgc,7136
-metaflow/metaflow_environment.py,sha256=bZxySBGuriqLjZSt0VvuLSuE3sMAyrD7wXwdyUdXO9g,8109
+metaflow/metaflow_environment.py,sha256=D7zHYqe8aLZVwJ20nx19vmsNW29Kf3PVE8hbBjVQin8,8115
 metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
 metaflow/metaflow_version.py,sha256=mPQ6g_3XjNdi0NrxDzwlW8ZH0nMyYpwqmJ04P7TIdP0,4774
 metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
@@ -35,7 +35,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=aRhCRpO5-SPvEInmdUUPKUQmAJLs0l3XtE3GQEGP7Rs,29
+metaflow/version.py,sha256=fKkvZn7TUtrxHS2I73qyoYIKvp6Psia2Jcc91r3fjck,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -279,22 +279,22 @@ metaflow/plugins/gcp/gs_tail.py,sha256=Jl_wvnzU7dub07A-DOAuP5FeccNIrPM-CeL1xKFs1
 metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRmvow,2030
 metaflow/plugins/gcp/includefile_support.py,sha256=vIDeR-MiJuUh-2S2pV7Z7FBkhIWwtHXaRrj76MWGRiY,3869
 metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/plugins/kubernetes/kubernetes.py,sha256=cr3TheUasxIBEwFZ3GEVbctaf8gW57BM5BDk80ikjPI,31063
+metaflow/plugins/kubernetes/kubernetes.py,sha256=sP5BDnjv-5cLm8i7E5YiJygWN4yNRQHt-hhOmysKm1k,31051
 metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=qBDdr1Lvtt-RO9pB-9_HTOPdzAmDvvJ0aiQ1OoCcrMU,10892
 metaflow/plugins/kubernetes/kubernetes_client.py,sha256=GKg-gT3qhXMRQV-sG1YyoOf3Z32NXr_wwEN2ytMVSEg,2471
 metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=CXStYHomuJJK_Yocpdo6OJadEQv5hDfSpO7GPL61ltw,25322
 metaflow/plugins/kubernetes/kubernetes_job.py,sha256=Cfkee8LbXC17jSXWoeNdomQRvF_8YSeXNg1gvxm6E_M,31806
-metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=OBmLtX-ZUDQdCCfftUmRMernfmTNMwdTxPoCAp_NmwE,40957
+metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=sCVuy-PRN60s5HWpnRvFmnljS41k70sBoPO3nOtl2ZE,40800
 metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7ITX12O8,22428
 metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
 metaflow/plugins/pypi/bootstrap.py,sha256=Tvc4_QKIx-A8j5Aq8ccWZrrxNM8csN40rK8HmxDx-Z8,5106
-metaflow/plugins/pypi/conda_decorator.py,sha256=fTJVbEfgOUtsDXIfnfsNk46sKeA9uTuTqGey9OFs9Ig,14738
-metaflow/plugins/pypi/conda_environment.py,sha256=erVvMgQhg6oBv4l6Yca72FKKoB0QEcitpTserPSPc44,19328
+metaflow/plugins/pypi/conda_decorator.py,sha256=D5B6xnBc2KUNPhaD0J062CGM1Jzo6D2bxhSU87KrmCM,15372
+metaflow/plugins/pypi/conda_environment.py,sha256=--q-8lypKupCdGsASpqABNpNqRxtQi6UCDgq8iHDFe4,19476
 metaflow/plugins/pypi/micromamba.py,sha256=67FiIZZz0Kig9EcN7bZLObsE6Z1MFyo4Dp93fd3Grcc,12178
 metaflow/plugins/pypi/pip.py,sha256=7B06mPOs5MvY33xbzPVYZlBr1iKMYaN-n8uulL9zSVg,13649
-metaflow/plugins/pypi/pypi_decorator.py,sha256=h5cAnxkWjmj4Ad4q0AkABKwhHQHYfeexy12yMaaLgXQ,6443
+metaflow/plugins/pypi/pypi_decorator.py,sha256=rDMbHl7r81Ye7-TuIlKAVJ_CDnfjl9jV44ZPws-UsTY,7229
 metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
 metaflow/plugins/pypi/utils.py,sha256=ds1Mnv_DaxGnLAYp7ozg_K6oyguGyNhvHfE-75Ia1YA,2836
 metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcwd-WpruE,310
@@ -344,9 +344,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-metaflow-2.12.17.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.12.17.dist-info/METADATA,sha256=w84PuAmNwt7w-PGFrUHmcNAj5BJFQ_26DkctyFTls2M,5906
-metaflow-2.12.17.dist-info/WHEEL,sha256=GUeE9LxUgRABPG7YM0jCNs9cBsAIx0YAkzCB88PMLgc,109
-metaflow-2.12.17.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.12.17.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.12.17.dist-info/RECORD,,
+metaflow-2.12.19.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.12.19.dist-info/METADATA,sha256=jMfQXXyRRJPzr4SJe-Y9028tXuVWJCMg7ZmXbEMswQI,5906
+metaflow-2.12.19.dist-info/WHEEL,sha256=O8iI_RpvXnU2dkdPNf_wSffaC9hi-VsSQKnuYEptk_s,109
+metaflow-2.12.19.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.12.19.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.12.19.dist-info/RECORD,,

{metaflow-2.12.17.dist-info → metaflow-2.12.19.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (74.1.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any