metaflow 2.15.9__py2.py3-none-any.whl → 2.15.11__py2.py3-none-any.whl

metaflow/includefile.py

@@ -459,7 +459,7 @@ class UploaderV2:
     @classmethod
     def encode_url(cls, url_type, url, **kwargs):
         return_value = {
-            "note": "Internal representation of IncludeFile(%s)" % url,
+            "note": "Internal representation of IncludeFile",
             "type": cls.file_type,
             "sub-type": url_type,
             "url": url,
@@ -472,7 +472,7 @@ class UploaderV2:
         r = UploaderV1.store(flow_name, path, is_text, encoding, handler, echo)
 
         # In V2, we store size for faster access
-        r["note"] = "Internal representation of IncludeFile(%s)" % path
+        r["note"] = "Internal representation of IncludeFile"
         r["type"] = cls.file_type
         r["sub-type"] = "uploaded"
         r["size"] = os.stat(path).st_size

metaflow/plugins/argo/argo_workflows.py

@@ -2732,7 +2732,7 @@ class ArgoWorkflows(object):
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": ":metaflow: Environment details"
+                    "text": "Environment details"
                 },
                 "fields": [
                     {
@@ -2750,7 +2750,7 @@ class ArgoWorkflows(object):
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": ":metaflow: Environment details"
+                    "text": "Environment details"
                 }
             }
 

metaflow/plugins/argo/argo_workflows_cli.py

@@ -40,6 +40,7 @@ unsupported_decorators = {
     "snowpark": "Step *%s* is marked for execution on Snowpark with Argo Workflows which isn't currently supported.",
     "slurm": "Step *%s* is marked for execution on Slurm with Argo Workflows which isn't currently supported.",
     "nvidia": "Step *%s* is marked for execution on Nvidia with Argo Workflows which isn't currently supported.",
+    "nvct": "Step *%s* is marked for execution on Nvct with Argo Workflows which isn't currently supported.",
 }
 
 

metaflow/plugins/datastores/gs_storage.py

@@ -119,7 +119,9 @@ class _GSRootClient(object):
                 blob.metadata = {"metaflow-user-attributes": json.dumps(metadata)}
             from google.cloud.storage.retry import DEFAULT_RETRY
 
-            blob.upload_from_filename(tmpfile, retry=DEFAULT_RETRY)
+            blob.upload_from_filename(
+                tmpfile, retry=DEFAULT_RETRY, timeout=(14400, 60)
+            )  # generous timeout for massive uploads. Use the same values as for Azure (connection_timeout, read_timeout)
         except Exception as e:
             process_gs_exception(e)
 

metaflow/plugins/datatools/s3/s3.py

@@ -1,3 +1,4 @@
+import errno
 import json
 import os
 import re
@@ -137,6 +138,10 @@ class MetaflowS3InvalidRange(MetaflowException):
     headline = "S3 invalid range"
 
 
+class MetaflowS3InsufficientDiskSpace(MetaflowException):
+    headline = "Insufficient disk space"
+
+
 class S3Object(object):
     """
     This object represents a path or an object in S3,
@@ -1377,8 +1382,10 @@ class S3(object):
                 elif error_code == "NoSuchBucket":
                     raise MetaflowS3URLException("Specified S3 bucket doesn't exist.")
                 error = str(err)
+            except OSError as e:
+                if e.errno == errno.ENOSPC:
+                    raise MetaflowS3InsufficientDiskSpace(str(e))
             except Exception as ex:
-                # TODO specific error message for out of disk space
                 error = str(ex)
             if tmp:
                 os.unlink(tmp.name)

metaflow/plugins/datatools/s3/s3op.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import errno
 import json
 import time
 import math
@@ -108,6 +109,7 @@ ERROR_VERIFY_FAILED = 9
 ERROR_LOCAL_FILE_NOT_FOUND = 10
 ERROR_INVALID_RANGE = 11
 ERROR_TRANSIENT = 12
+ERROR_OUT_OF_DISK_SPACE = 13
 
 
 def format_result_line(idx, prefix, url="", local=""):
@@ -277,6 +279,17 @@ def worker(result_file_name, queue, mode, s3config):
                             err = convert_to_client_error(e)
                             handle_client_error(err, idx, result_file)
                             continue
+                        except OSError as e:
+                            tmp.close()
+                            os.unlink(tmp.name)
+                            if e.errno == errno.ENOSPC:
+                                result_file.write(
+                                    "%d %d\n" % (idx, -ERROR_OUT_OF_DISK_SPACE)
+                                )
+                            else:
+                                result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
+                            result_file.flush()
+                            continue
                     except (SSLError, Exception) as e:
                         tmp.close()
                         os.unlink(tmp.name)
@@ -643,6 +656,8 @@ def exit(exit_code, url):
         msg = "Local file not found: %s" % url
     elif exit_code == ERROR_TRANSIENT:
         msg = "Transient error for url: %s" % url
+    elif exit_code == ERROR_OUT_OF_DISK_SPACE:
+        msg = "Out of disk space when downloading URL: %s" % url
     else:
         msg = "Unknown error"
     print("s3op failed:\n%s" % msg, file=sys.stderr)
@@ -1173,6 +1188,8 @@ def get(
             )
             if verify:
                 verify_info.append((url, sz))
+        elif sz == -ERROR_OUT_OF_DISK_SPACE:
+            exit(ERROR_OUT_OF_DISK_SPACE, url)
         elif sz == -ERROR_URL_ACCESS_DENIED:
             denied_url = url
             break

metaflow/plugins/pypi/conda_decorator.py

@@ -227,7 +227,8 @@ class CondaStepDecorator(StepDecorator):
         self.interpreter = (
             self.environment.interpreter(self.step)
             if not any(
-                decorator.name in ["batch", "kubernetes", "nvidia", "snowpark", "slurm"]
+                decorator.name
+                in ["batch", "kubernetes", "nvidia", "snowpark", "slurm", "nvct"]
                 for decorator in next(
                     step for step in self.flow if step.name == self.step
                 ).decorators

metaflow/plugins/pypi/conda_environment.py

@@ -326,6 +326,7 @@ class CondaEnvironment(MetaflowEnvironment):
                 "nvidia",
                 "snowpark",
                 "slurm",
+                "nvct",
             ]:
                 target_platform = getattr(decorator, "target_platform", "linux-64")
                 break

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.15.9"
+metaflow_version = "2.15.11"

{metaflow-2.15.9.dist-info → metaflow-2.15.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.15.9
+Version: 2.15.11
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.9; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.11; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.15.9.dist-info → metaflow-2.15.11.dist-info}/RECORD

@@ -12,7 +12,7 @@ metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
 metaflow/exception.py,sha256=_m9ZBJM0cooHRslDqfxCPQmkChqaTh6fGxp7HvISnYI,5161
 metaflow/flowspec.py,sha256=GgbTeUBtG3AmZwIF-prRFMsZqFYGVysd5xBS9IPIPBs,35953
 metaflow/graph.py,sha256=cdpnWr85aEj_rRn-7EjbndWjr_i8Dt3P7-oPUW0NNpI,12393
-metaflow/includefile.py,sha256=kWKDSlzVcRVNGG9PV5eB3o2ynrzqhVsfaLtkqjshn7Q,20948
+metaflow/includefile.py,sha256=RtISGl1V48qjkJBakUZ9yPpHV102h7pOIFiKP8PLHpc,20927
 metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=x4p6tnRzYqNNniCGXyrUW0WuYfTUgnaOMRivxvnxask,11661
@@ -37,7 +37,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=mJBkV5tShIyCsLDeM1zygQGeciQVMrVPm_qI8Oi33G0,14656
 metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
-metaflow/version.py,sha256=tTlVvolaB_v0CwYKrCJvwFACvGOTTmI-ceha9iDWSJE,28
+metaflow/version.py,sha256=V3MCZGYThdp_pU62PrdVdzGOfAFGKJNAcgmFrG7nuX0,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -210,8 +210,8 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=A1kI9rjVjCadDsBscZ2Wk8xRBI6GNgWV6SU7TyrdfrI,16530
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=_Mf5l1nJy2M1h-dsTdbsGAH3_sRI8pYg3djniutY6QA,186566
-metaflow/plugins/argo/argo_workflows_cli.py,sha256=Ka-ea4x19E6DrYgGm5ZwormxEbTGdun8fyHl-mh0tfc,38265
+metaflow/plugins/argo/argo_workflows.py,sha256=oX-e4o_CxTQP5DJ7zMQwckcobXZJz6hPxLs3fsHSnm8,186544
+metaflow/plugins/argo/argo_workflows_cli.py,sha256=X_GfJpc7jfP7DGuttl7U952767eBF6Ut45aWgoJzHVI,38375
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
 metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
 metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=lRRHUcpiyJZFltthxZoIp7aJWwy7pcdhaRm0etKN9es,14182
@@ -276,14 +276,14 @@ metaflow/plugins/cards/card_modules/chevron/tokenizer.py,sha256=lQU9OELUE9a5Xu4s
 metaflow/plugins/cards/card_viewer/viewer.html,sha256=qZJGzhZhQ1gugsknRP7zkAPPfUAtvemK1UKqXoGff5M,11593
 metaflow/plugins/datastores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/datastores/azure_storage.py,sha256=QvIUQGOZF1oKeRJXbl3RsV1MnO3OMfLzheoo0UNWn7E,16670
-metaflow/plugins/datastores/gs_storage.py,sha256=ED1xfXNDiPkqma5PEui8HCv0b4ImHK_rlUMQJD9UNes,9622
+metaflow/plugins/datastores/gs_storage.py,sha256=YEI62gElaJvACupFnY6ZEXuQLFWndi96upWFbnmc0VU,9782
 metaflow/plugins/datastores/local_storage.py,sha256=igrBDphhyu7EFIUj3BWcO7beiZbNnJLq--lF45UYSyI,4750
 metaflow/plugins/datastores/s3_storage.py,sha256=CZdNqaKtxDXQbEg2YHyphph3hWcLIE50puenm0WGVpk,5473
 metaflow/plugins/datatools/__init__.py,sha256=ge4L16OBQLy2J_MMvoHg3lMfdm-MluQgRWoyZ5GCRnk,1267
 metaflow/plugins/datatools/local.py,sha256=FJvMOBcjdyhSPHmdLocBSiIT0rmKkKBmsaclxH75x08,4233
 metaflow/plugins/datatools/s3/__init__.py,sha256=14tr9fPjN3ULW5IOfKHeG7Uhjmgm7LMtQHfz1SFv-h8,248
-metaflow/plugins/datatools/s3/s3.py,sha256=ThISU4XjXhDqAtfgH_PgtNp0OxF_bhzCgNozYIeMa7g,66993
-metaflow/plugins/datatools/s3/s3op.py,sha256=I7XkDJvVvvLt8SmXNSZjSNONIa2m3QlT5-ZL8g6erno,46936
+metaflow/plugins/datatools/s3/s3.py,sha256=3xrWD6pXoVRpuAQHyLGVya79UIE0S8AqAqe2prg4yMw,67182
+metaflow/plugins/datatools/s3/s3op.py,sha256=20XEOCCK_nAVm92ZOjLPTzLnTs9xrIX0Gj-ELxkuNPY,47742
 metaflow/plugins/datatools/s3/s3tail.py,sha256=boQjQGQMI-bvTqcMP2y7uSlSYLcvWOy7J3ZUaF78NAA,2597
 metaflow/plugins/datatools/s3/s3util.py,sha256=FgRgaVmEq7-i2dV7q8XK5w5PfFt-xJjZa8WrK8IJfdI,3769
 metaflow/plugins/env_escape/__init__.py,sha256=tGNUZnmPvk52eNs__VK443b3CZ7ogEFTT-s9_n_HF8Q,8837
@@ -330,8 +330,8 @@ metaflow/plugins/metadata_providers/local.py,sha256=Z0CXaGZJbAkj4II3WspJi-uCCtSh
 metaflow/plugins/metadata_providers/service.py,sha256=9j0db_EOGFdb49YTgr9q4EWqVAm1YUaW-Baj5gyRqIo,22809
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
 metaflow/plugins/pypi/bootstrap.py,sha256=SNONquX6QnTbu7htmhaQeVeZ2ofaFaUCDScRIrTTERc,14718
-metaflow/plugins/pypi/conda_decorator.py,sha256=piFcE4uGmWhhbGlxMK0GHd7BGEyqy6r9BFy8Mjoi80Q,15937
-metaflow/plugins/pypi/conda_environment.py,sha256=yeTPGuu38EQ8aYzXUbc6cLc9b2NMLLEnxS-C73PcDHk,22250
+metaflow/plugins/pypi/conda_decorator.py,sha256=N0HGiaS1mRsa6qT4eYzu2C3DHtas22QIXowW4vEl44M,15961
+metaflow/plugins/pypi/conda_environment.py,sha256=JuTfGfUML_AoeW4ASGkqPbm8OqtWsbtRwDNwrRTRqS4,22274
 metaflow/plugins/pypi/micromamba.py,sha256=LLJ2dGGOEyld07W8iI6dtE01h2Y1PQnBhU-dMBssZ3c,16502
 metaflow/plugins/pypi/parsers.py,sha256=gpOOG2Ph95wI73MWCAi7XjpK0gYhv5k5YIGBs73QPuE,8556
 metaflow/plugins/pypi/pip.py,sha256=H0cIy8odpZ-JTn4SwF0b74tuC3uRU7X8TdAQJ2kODG8,13971
@@ -393,12 +393,12 @@ metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 metaflow/user_configs/config_decorators.py,sha256=qCKVAvd0NKgaCxQ2OThes5-DYHXq6A1HqURubYNeFdw,20481
 metaflow/user_configs/config_options.py,sha256=m6jccSpzI4qUJ7vyYkYBIf8G3V0Caunxg_k7zg4Zlqg,21067
 metaflow/user_configs/config_parameters.py,sha256=oeJGVKu1ao_YQX6Lg6P2FEv5k5-_F4sARLlVpTW9ezM,15502
-metaflow-2.15.9.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
-metaflow-2.15.9.data/data/share/metaflow/devtools/Tiltfile,sha256=P5_rn_F3xYLN1_cEAQ9mNeS22HG2rb8beKIz2RIK6fU,20634
-metaflow-2.15.9.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
-metaflow-2.15.9.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.15.9.dist-info/METADATA,sha256=GGjvModROT2_4AhGn4h3nw8mEyOo0v-osUymrqE2y-A,6740
-metaflow-2.15.9.dist-info/WHEEL,sha256=Td9E1opt19FSuwsk_gcDwtsGPmyXw7uz9xQf-y2gvl8,109
-metaflow-2.15.9.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
-metaflow-2.15.9.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.15.9.dist-info/RECORD,,
+metaflow-2.15.11.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
+metaflow-2.15.11.data/data/share/metaflow/devtools/Tiltfile,sha256=P5_rn_F3xYLN1_cEAQ9mNeS22HG2rb8beKIz2RIK6fU,20634
+metaflow-2.15.11.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
+metaflow-2.15.11.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.15.11.dist-info/METADATA,sha256=aFy969IqHDb3_SQJx_gXBInl3KI_JmZn_kwzTAEaVpw,6742
+metaflow-2.15.11.dist-info/WHEEL,sha256=oSJJyWjO7Z2XSScFQUpXG1HL-N0sFMqqeKVVbZTPkWc,109
+metaflow-2.15.11.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
+metaflow-2.15.11.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.15.11.dist-info/RECORD,,

{metaflow-2.15.9.dist-info → metaflow-2.15.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.0)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any