PyPI - anemoi-utils - Versions diffs - 0.4.6__tar.gz → 0.4.8__tar.gz - Mend

@@ -138,6 +138,8 @@ class BaseDownload(Loader):
         raise NotImplementedError
     def get_temporary_target(self, target, pattern):
+        if pattern is None:
+            return target
         dirname, basename = os.path.split(target)
         return pattern.format(dirname=dirname, basename=basename)
@@ -199,13 +201,13 @@ class Transfer:
             target = os.path.basename(source)
         temporary_target = {
-            False: "{dirname}/{basename}",
+            False: None,
             True: "{dirname}-downloading/{basename}",
             "-tmp/*": "{dirname}-tmp/{basename}",
             "*-tmp": "{dirname}/{basename}-tmp",
             "tmp-*": "{dirname}/tmp-{basename}",
         }.get(temporary_target, temporary_target)
-        assert isinstance(temporary_target, str), (type(temporary_target), temporary_target)
+        assert temporary_target is None or isinstance(temporary_target, str), (type(temporary_target), temporary_target)
         self.source = source
         self.target = target

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.4.6
+Version: 0.4.8
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
-__version__ = version = '0.4.6'
-__version_tuple__ = version_tuple = (0, 4, 6)
+__version__ = version = '0.4.8'
+__version_tuple__ = version_tuple = (0, 4, 8)

@@ -207,7 +207,7 @@ def load_any_dict_format(path) -> dict:
                 return tomllib.load(f)
     except (json.JSONDecodeError, yaml.YAMLError, tomllib.TOMLDecodeError) as e:
         LOG.warning(f"Failed to parse config file {path}", exc_info=e)
-        return ValueError(f"Failed to parse config file {path} [{e}]")
+        raise ValueError(f"Failed to parse config file {path} [{e}]")
     return open(path).read()

@@ -192,6 +192,9 @@ class S3Download(BaseDownload):
         _, _, bucket, key = source.split("/", 3)
         s3 = s3_client(bucket)
+        if key.endswith("/"):
+            return 0
         try:
             response = s3.head_object(Bucket=bucket, Key=key)
         except s3.exceptions.ClientError as e:

@@ -44,12 +44,16 @@ class SshBaseUpload(BaseUpload):
             hostnames = hostname.split("+")
             hostname = hostnames[random.randint(0, len(hostnames) - 1)]
+        if ".." in path.split("/"):
+            raise Exception("Path contains suspicious '..' : {target}")
         return hostname, path
     def get_temporary_target(self, target, pattern):
         hostname, path = self._parse_target(target)
-        dirname, basename = os.path.split(path)
-        path = pattern.format(dirname=dirname, basename=basename)
+        if pattern is not None:
+            dirname, basename = os.path.split(path)
+            path = pattern.format(dirname=dirname, basename=basename)
         return f"ssh://{hostname}:{path}"
     def rename_target(self, target, new_target):

anemoi-utils 0.4.6tar.gz → 0.4.8tar.gz

@@ -151,7 +151,7 @@ def test_transfer_local_to_s3_to_local(path):
 @pytest.mark.skipif(IN_CI, reason="Test requires ssh access to localhost")
-@pytest.mark.parametrize("path", ["directory", "directory/", "file"])
+@pytest.mark.parametrize("path", ["directory", "file"])
 @pytest.mark.parametrize("temporary_target", [True, False])
 def test_transfer_local_to_ssh(path, temporary_target):
     local = LOCAL_TEST_DATA + "/" + path