oasis-data-manager 0.1.1__tar.gz → 0.1.3__tar.gz

{oasis-data-manager-0.1.1/oasis_data_manager.egg-info → oasis-data-manager-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oasis-data-manager
-Version: 0.1.1
+Version: 0.1.3
 Summary: UNKNOWN
 Home-page: https://github.com/OasisLMF/OasisDataManager
 Author: Oasis LMF

oasis-data-manager-0.1.3/oasis_data_manager/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.1.3'

{oasis-data-manager-0.1.1 → oasis-data-manager-0.1.3}/oasis_data_manager/filestore/backends/aws_s3.py

@@ -178,6 +178,7 @@ class AwsS3Storage(BaseStorage):
             "s3_additional_kwargs": s3_additional_kwargs,
             "client_kwargs": {
                 "endpoint_url": self.endpoint_url,
+                "region_name": self.region_name,
             },
         }
 

{oasis-data-manager-0.1.1 → oasis-data-manager-0.1.3}/oasis_data_manager/filestore/backends/base.py

@@ -105,6 +105,8 @@ class BaseStorage(object):
         :return: filename string
         :rtype str
         """
+        if suffix.startswith('.'):
+            suffix = suffix[1:]
         return "{}.{}".format(uuid.uuid4().hex, suffix)
 
     def _is_valid_url(self, url):
@@ -246,6 +248,13 @@ class BaseStorage(object):
         :return: Absolute filepath to stored Object
         :rtype str
         """
+        # null ref given
+        if not reference:
+            if required:
+                raise MissingInputsException(reference)
+            else:
+                return None
+
         target = os.path.abspath(
             os.path.join(output_path, subdir) if subdir else output_path
         )
@@ -298,17 +307,23 @@ class BaseStorage(object):
         if not reference:
             return None
 
-        ext = "tar.gz" if not suffix else suffix
-        filename = filename if filename else self._get_unique_filename(ext)
-        storage_path = os.path.join(subdir, filename) if subdir else filename
+        if os.path.isfile(reference):
+            ext = "".join(Path(reference).suffixes) if not suffix else suffix
+            filename = filename if filename else self._get_unique_filename(ext)
+            storage_path = subdir if subdir else ''
+            self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+            storage_location = os.path.join(storage_path, filename)
 
-        self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+            self.logger.info("Store file: {} -> {}".format(reference, storage_location))
+            self.fs.put(reference, storage_location)
+            return storage_location
 
-        if os.path.isfile(reference):
-            self.logger.info("Store file: {} -> {}".format(reference, storage_path))
-            self.fs.put(reference, storage_path)
-            return storage_path
         elif os.path.isdir(reference):
+            ext = "tar.gz" if not suffix else suffix
+            filename = filename if filename else self._get_unique_filename(ext)
+            storage_path = os.path.join(subdir, filename) if subdir else filename
+            self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+
             self.logger.info("Store dir: {} -> {}".format(reference, storage_path))
             with tempfile.NamedTemporaryFile() as f:
                 self.compress(f.name, reference, arcname)

{oasis-data-manager-0.1.1 → oasis-data-manager-0.1.3/oasis_data_manager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oasis-data-manager
-Version: 0.1.1
+Version: 0.1.3
 Summary: UNKNOWN
 Home-page: https://github.com/OasisLMF/OasisDataManager
 Author: Oasis LMF

{oasis-data-manager-0.1.1 → oasis-data-manager-0.1.3}/oasis_data_manager.egg-info/requires.txt

@@ -1,16 +1,15 @@
 fastparquet
-fsspec
+fsspec>=2023.12.2
 pandas
-typing
 typing_extensions
 
 [extra]
 adlfs
 boto3
-dask
+dask<=2024.1.1
 dask-geopandas
 dask-sql
 distributed
 geopandas
 pyogrio
-s3fs>=2023.9.0
+s3fs>=2023.12.2

{oasis-data-manager-0.1.1 → oasis-data-manager-0.1.3}/requirements-package.in

@@ -1,5 +1,4 @@
 fastparquet
-fsspec
+fsspec>=2023.12.2
 pandas
-typing
 typing_extensions

oasis-data-manager-0.1.1/oasis_data_manager/__init__.py

@@ -1 +0,0 @@
-__version__ = '0.1.1'