metaflow 2.15.9__tar.gz → 2.15.11__tar.gz

{metaflow-2.15.9/metaflow.egg-info → metaflow-2.15.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.15.9
+Version: 2.15.11
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.9; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.11; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow/includefile.py

@@ -459,7 +459,7 @@ class UploaderV2:
     @classmethod
     def encode_url(cls, url_type, url, **kwargs):
         return_value = {
-            "note": "Internal representation of IncludeFile(%s)" % url,
+            "note": "Internal representation of IncludeFile",
             "type": cls.file_type,
             "sub-type": url_type,
             "url": url,
@@ -472,7 +472,7 @@ class UploaderV2:
         r = UploaderV1.store(flow_name, path, is_text, encoding, handler, echo)
 
         # In V2, we store size for faster access
-        r["note"] = "Internal representation of IncludeFile(%s)" % path
+        r["note"] = "Internal representation of IncludeFile"
         r["type"] = cls.file_type
         r["sub-type"] = "uploaded"
         r["size"] = os.stat(path).st_size

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow/plugins/argo/argo_workflows.py

@@ -2732,7 +2732,7 @@ class ArgoWorkflows(object):
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": ":metaflow: Environment details"
+                    "text": "Environment details"
                 },
                 "fields": [
                     {
@@ -2750,7 +2750,7 @@ class ArgoWorkflows(object):
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": ":metaflow: Environment details"
+                    "text": "Environment details"
                 }
             }
 

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -40,6 +40,7 @@ unsupported_decorators = {
     "snowpark": "Step *%s* is marked for execution on Snowpark with Argo Workflows which isn't currently supported.",
     "slurm": "Step *%s* is marked for execution on Slurm with Argo Workflows which isn't currently supported.",
     "nvidia": "Step *%s* is marked for execution on Nvidia with Argo Workflows which isn't currently supported.",
+    "nvct": "Step *%s* is marked for execution on Nvct with Argo Workflows which isn't currently supported.",
 }
 
 

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow/plugins/datastores/gs_storage.py

@@ -119,7 +119,9 @@ class _GSRootClient(object):
                 blob.metadata = {"metaflow-user-attributes": json.dumps(metadata)}
             from google.cloud.storage.retry import DEFAULT_RETRY
 
-            blob.upload_from_filename(tmpfile, retry=DEFAULT_RETRY)
+            blob.upload_from_filename(
+                tmpfile, retry=DEFAULT_RETRY, timeout=(14400, 60)
+            )  # generous timeout for massive uploads. Use the same values as for Azure (connection_timeout, read_timeout)
         except Exception as e:
             process_gs_exception(e)
 

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow/plugins/datatools/s3/s3.py

@@ -1,3 +1,4 @@
+import errno
 import json
 import os
 import re
@@ -137,6 +138,10 @@ class MetaflowS3InvalidRange(MetaflowException):
     headline = "S3 invalid range"
 
 
+class MetaflowS3InsufficientDiskSpace(MetaflowException):
+    headline = "Insufficient disk space"
+
+
 class S3Object(object):
     """
     This object represents a path or an object in S3,
@@ -1377,8 +1382,10 @@ class S3(object):
                 elif error_code == "NoSuchBucket":
                     raise MetaflowS3URLException("Specified S3 bucket doesn't exist.")
                 error = str(err)
+            except OSError as e:
+                if e.errno == errno.ENOSPC:
+                    raise MetaflowS3InsufficientDiskSpace(str(e))
             except Exception as ex:
-                # TODO specific error message for out of disk space
                 error = str(ex)
             if tmp:
                 os.unlink(tmp.name)

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow/plugins/datatools/s3/s3op.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import errno
 import json
 import time
 import math
@@ -108,6 +109,7 @@ ERROR_VERIFY_FAILED = 9
 ERROR_LOCAL_FILE_NOT_FOUND = 10
 ERROR_INVALID_RANGE = 11
 ERROR_TRANSIENT = 12
+ERROR_OUT_OF_DISK_SPACE = 13
 
 
 def format_result_line(idx, prefix, url="", local=""):
@@ -277,6 +279,17 @@ def worker(result_file_name, queue, mode, s3config):
                             err = convert_to_client_error(e)
                             handle_client_error(err, idx, result_file)
                             continue
+                        except OSError as e:
+                            tmp.close()
+                            os.unlink(tmp.name)
+                            if e.errno == errno.ENOSPC:
+                                result_file.write(
+                                    "%d %d\n" % (idx, -ERROR_OUT_OF_DISK_SPACE)
+                                )
+                            else:
+                                result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
+                            result_file.flush()
+                            continue
                     except (SSLError, Exception) as e:
                         tmp.close()
                         os.unlink(tmp.name)
@@ -643,6 +656,8 @@ def exit(exit_code, url):
         msg = "Local file not found: %s" % url
     elif exit_code == ERROR_TRANSIENT:
         msg = "Transient error for url: %s" % url
+    elif exit_code == ERROR_OUT_OF_DISK_SPACE:
+        msg = "Out of disk space when downloading URL: %s" % url
     else:
         msg = "Unknown error"
     print("s3op failed:\n%s" % msg, file=sys.stderr)
@@ -1173,6 +1188,8 @@ def get(
             )
             if verify:
                 verify_info.append((url, sz))
+        elif sz == -ERROR_OUT_OF_DISK_SPACE:
+            exit(ERROR_OUT_OF_DISK_SPACE, url)
         elif sz == -ERROR_URL_ACCESS_DENIED:
             denied_url = url
             break

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow/plugins/pypi/conda_decorator.py

@@ -227,7 +227,8 @@ class CondaStepDecorator(StepDecorator):
         self.interpreter = (
             self.environment.interpreter(self.step)
             if not any(
-                decorator.name in ["batch", "kubernetes", "nvidia", "snowpark", "slurm"]
+                decorator.name
+                in ["batch", "kubernetes", "nvidia", "snowpark", "slurm", "nvct"]
                 for decorator in next(
                     step for step in self.flow if step.name == self.step
                 ).decorators

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow/plugins/pypi/conda_environment.py

@@ -326,6 +326,7 @@ class CondaEnvironment(MetaflowEnvironment):
                 "nvidia",
                 "snowpark",
                 "slurm",
+                "nvct",
             ]:
                 target_platform = getattr(decorator, "target_platform", "linux-64")
                 break

metaflow-2.15.11/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.15.11"

{metaflow-2.15.9 → metaflow-2.15.11/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.15.9
+Version: 2.15.11
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.9; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.11; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.15.9 → metaflow-2.15.11}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.15.9
+metaflow-stubs==2.15.11

metaflow-2.15.9/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.15.9"