oasis-data-manager 0.1.2__py2.py3-none-any.whl → 0.1.3__py2.py3-none-any.whl

oasis_data_manager/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.2'
+__version__ = '0.1.3'

oasis_data_manager/filestore/backends/aws_s3.py

@@ -178,6 +178,7 @@ class AwsS3Storage(BaseStorage):
             "s3_additional_kwargs": s3_additional_kwargs,
             "client_kwargs": {
                 "endpoint_url": self.endpoint_url,
+                "region_name": self.region_name,
             },
         }
 

oasis_data_manager/filestore/backends/base.py

@@ -105,6 +105,8 @@ class BaseStorage(object):
         :return: filename string
         :rtype str
         """
+        if suffix.startswith('.'):
+            suffix = suffix[1:]
         return "{}.{}".format(uuid.uuid4().hex, suffix)
 
     def _is_valid_url(self, url):
@@ -246,6 +248,13 @@ class BaseStorage(object):
         :return: Absolute filepath to stored Object
         :rtype str
         """
+        # null ref given
+        if not reference:
+            if required:
+                raise MissingInputsException(reference)
+            else:
+                return None
+
         target = os.path.abspath(
             os.path.join(output_path, subdir) if subdir else output_path
         )
@@ -298,17 +307,23 @@ class BaseStorage(object):
         if not reference:
             return None
 
-        ext = "tar.gz" if not suffix else suffix
-        filename = filename if filename else self._get_unique_filename(ext)
-        storage_path = os.path.join(subdir, filename) if subdir else filename
+        if os.path.isfile(reference):
+            ext = "".join(Path(reference).suffixes) if not suffix else suffix
+            filename = filename if filename else self._get_unique_filename(ext)
+            storage_path = subdir if subdir else ''
+            self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+            storage_location = os.path.join(storage_path, filename)
 
-        self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+            self.logger.info("Store file: {} -> {}".format(reference, storage_location))
+            self.fs.put(reference, storage_location)
+            return storage_location
 
-        if os.path.isfile(reference):
-            self.logger.info("Store file: {} -> {}".format(reference, storage_path))
-            self.fs.put(reference, storage_path)
-            return storage_path
         elif os.path.isdir(reference):
+            ext = "tar.gz" if not suffix else suffix
+            filename = filename if filename else self._get_unique_filename(ext)
+            storage_path = os.path.join(subdir, filename) if subdir else filename
+            self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+
             self.logger.info("Store dir: {} -> {}".format(reference, storage_path))
             with tempfile.NamedTemporaryFile() as f:
                 self.compress(f.name, reference, arcname)

{oasis_data_manager-0.1.2.dist-info → oasis_data_manager-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oasis-data-manager
-Version: 0.1.2
+Version: 0.1.3
 Summary: UNKNOWN
 Home-page: https://github.com/OasisLMF/OasisDataManager
 Author: Oasis LMF
@@ -18,18 +18,17 @@ License-File: LICENSE
 Requires-Dist: fastparquet
 Requires-Dist: fsspec >=2023.12.2
 Requires-Dist: pandas
-Requires-Dist: typing
 Requires-Dist: typing-extensions
 Provides-Extra: extra
 Requires-Dist: adlfs ; extra == 'extra'
 Requires-Dist: boto3 ; extra == 'extra'
-Requires-Dist: dask ; extra == 'extra'
+Requires-Dist: dask <=2024.1.1 ; extra == 'extra'
 Requires-Dist: dask-geopandas ; extra == 'extra'
 Requires-Dist: dask-sql ; extra == 'extra'
 Requires-Dist: distributed ; extra == 'extra'
 Requires-Dist: geopandas ; extra == 'extra'
 Requires-Dist: pyogrio ; extra == 'extra'
-Requires-Dist: s3fs >=2023.9.0 ; extra == 'extra'
+Requires-Dist: s3fs >=2023.12.2 ; extra == 'extra'
 
 UNKNOWN
 

{oasis_data_manager-0.1.2.dist-info → oasis_data_manager-0.1.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-oasis_data_manager/__init__.py,sha256=mdp2CftfqYbdKtP-eWv1z7rAUycYv6X1ntXSMUf8Kss,22
+oasis_data_manager/__init__.py,sha256=uZsygMXMKRw-7qhWojAjnpm8GFPXU92xW6XA8O5GwFY,22
 oasis_data_manager/config.py,sha256=_qx2Mu5n0Jx3W5SKCiqLr1SPdWLrbFv_B82r6Eosp_k,534
 oasis_data_manager/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/complex/complex.py,sha256=8oomV9WyLsa8sz8aMzlwv4naKCGOL3UdSlYQJxUFqCk,5382
@@ -17,12 +17,12 @@ oasis_data_manager/filestore/config.py,sha256=N0hSfPpRpej7uWGV54MEEH_0vvdpcgo3fZ
 oasis_data_manager/filestore/filestore.py,sha256=eaQGAer7Q9KM4B3bq9WmZAtjFdj9aRef_E3rI2i0dOk,2615
 oasis_data_manager/filestore/log.py,sha256=8l54LoOJiOG2pr4o93LzMocjH7dHcsOp14JWJ_MrqHQ,693
 oasis_data_manager/filestore/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oasis_data_manager/filestore/backends/aws_s3.py,sha256=te2rVQl6n4xr4X-xQv68gDNQDIdTh9ar-FNI4JUtQJg,10587
+oasis_data_manager/filestore/backends/aws_s3.py,sha256=uV8HfyJ_VuyCyl6oN0qIxpbhxhQ1aGAG4UfjyETyHP0,10636
 oasis_data_manager/filestore/backends/azure_abfs.py,sha256=Lo2aBBQsFPJJEb9svm1-W43Gz2g4LCaHlM-9mVXDhzg,5354
-oasis_data_manager/filestore/backends/base.py,sha256=aj__0KsmnVbXTHYSA_qlrIe5pxImSZ14dPMzskdNzJc,12971
+oasis_data_manager/filestore/backends/base.py,sha256=O-JmKpUKiXIsqR_PBmMT4_NNBYMzsp9r04q8u4iWjp0,13597
 oasis_data_manager/filestore/backends/local.py,sha256=MEX_CvwhsDfv9lvBjc8CdaDXaN53l9onQHmOgKjoJcg,1242
-oasis_data_manager-0.1.2.dist-info/LICENSE,sha256=qr-PXl5mSpeUk-A7RzYcH0dhR93hhgVK8SW9mzco0Ao,1517
-oasis_data_manager-0.1.2.dist-info/METADATA,sha256=R5rXMfHrg3pAZ6wjYTjqScEiMBYvv32fFrQamt9LjZM,1110
-oasis_data_manager-0.1.2.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-oasis_data_manager-0.1.2.dist-info/top_level.txt,sha256=qMC39T9UvDCPbNJLVtgu8h6f7c4KJYel7SnIpz62wsU,19
-oasis_data_manager-0.1.2.dist-info/RECORD,,
+oasis_data_manager-0.1.3.dist-info/LICENSE,sha256=qr-PXl5mSpeUk-A7RzYcH0dhR93hhgVK8SW9mzco0Ao,1517
+oasis_data_manager-0.1.3.dist-info/METADATA,sha256=P05T_lkfvsQzfKhjjYG3KE68jE44WxkJBRvj9w8aOUs,1100
+oasis_data_manager-0.1.3.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+oasis_data_manager-0.1.3.dist-info/top_level.txt,sha256=qMC39T9UvDCPbNJLVtgu8h6f7c4KJYel7SnIpz62wsU,19
+oasis_data_manager-0.1.3.dist-info/RECORD,,

{oasis_data_manager-0.1.2.dist-info → oasis_data_manager-0.1.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any