oasis-data-manager 0.1.6__py2.py3-none-any.whl → 0.2.0__py2.py3-none-any.whl

oasis_data_manager/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.6'
+__version__ = '0.2.0'

oasis_data_manager/filestore/backends/azure_abfs.py

@@ -114,24 +114,21 @@ class AzureABFSStorage(BaseStorage):
     def connection_string(self):
         if self._connection_string:
             return self._connection_string
-        else:
-            fsspec_storage_options = {
-                "anon": not self.account_key,
-                "account_name": self.account_name,
-                "account_key": self.account_key,
-                "use_ssl": self.azure_ssl,
-            }
-            fs = self.fsspec_filesystem_class(**fsspec_storage_options)
-
-            cs = ""
-            if self.endpoint_url:
-                cs += f"BlobEndpoint={self.endpoint_url};"
-            if fs.account_name:
-                cs += f"AccountName={fs.account_name};"
-            if fs.account_key:
-                cs += f"AccountKey={fs.account_key};"
-
-            return cs
+
+        cs_parts = [
+            f"DefaultEndpointsProtocol={'https' if self.azure_ssl else 'http'}",
+            f"AccountName={self.account_name}",
+            f"AccountKey={self.account_key}"
+        ]
+
+        if self.endpoint_url:
+            # Azurite requires the account name in the endpoint path
+            endpoint = self.endpoint_url.rstrip('/')
+            if self.account_name not in endpoint:
+                endpoint = f"{endpoint}/{self.account_name}"
+            cs_parts.append(f"BlobEndpoint={endpoint};")
+
+        return ";".join(cs_parts)
 
     def get_storage_url(self, filename=None, suffix="tar.gz", encode_params=True):
         filename = (

oasis_data_manager/filestore/backends/base.py

@@ -1,6 +1,4 @@
-import base64
 import contextlib
-import io
 import logging
 import os
 import shutil
@@ -16,6 +14,7 @@ import fsspec
 from fsspec.implementations.dirfs import DirFileSystem
 
 from oasis_data_manager.errors import OasisException
+import xxhash
 
 LOG_FILE_SUFFIX = "txt"
 ARCHIVE_FILE_SUFFIX = "tar.gz"
@@ -191,39 +190,93 @@ class BaseStorage(object):
         :return: Absolute filepath to stored Object
         :rtype str
         """
-        # null ref given
         if not reference:
             if required:
                 raise MissingInputsException(reference)
-            else:
-                return None
+            return None
 
-        # check if the file is in the cache, if so return that path
-        cache_filename = base64.b64encode(reference.encode()).decode()
-        if self.cache_root:
-            cached_file = os.path.join(self.cache_root, cache_filename)
-        else:
-            os.makedirs(os.path.dirname(no_cache_target), exist_ok=True)
-            cached_file = no_cache_target
-
-        if self.cache_root:
-            if os.path.exists(cached_file):
-                logging.info("Get from Cache: {}".format(reference))
-                return cached_file
-
-        if self._is_valid_url(reference):
-            # if the file is not in the path, and is a url download it to the cache
-            response = urlopen(reference)
-            fdata = response.read()
-
-            with io.open(cached_file, "w+b") as f:
-                f.write(fdata)
-                logging.info("Get from URL: {}".format(reference))
+        fs_protocol = getattr(self.fs.fs, "protocol", None)
+
+        # Normalize protocol into a tuple of strings
+        if isinstance(fs_protocol, str):
+            protocols = (fs_protocol,)
+        elif isinstance(fs_protocol, (list, tuple)):
+            protocols = tuple(fs_protocol)
         else:
-            # otherwise get it from the storage and add it to the cache
-            self.fs.get(reference, cached_file, recursive=True)
+            protocols = ()
+
+        enable_etag_cache = (
+            self.cache_root
+            and not self._is_valid_url(reference)
+            and any(p in ("s3", "s3a", "az", "abfs", "abfss") for p in protocols)
+        )
 
-        return cached_file
+        # No cache root configured, just return data
+        if not enable_etag_cache:
+            if not no_cache_target:
+                raise OasisException("Error: caching disabled for this filesystem and no_cache_target not provided")
+            Path(no_cache_target).parent.mkdir(parents=True, exist_ok=True)
+            if self._is_valid_url(reference):
+                with urlopen(reference, timeout=30) as r:
+                    data = r.read()
+                with open(no_cache_target, "wb") as f:
+                    f.write(data)
+                    logging.info("Get from URL: {}".format(reference))
+            else:
+                self.fs.get(reference, no_cache_target, recursive=True)
+                logging.info("Get from Filestore: {}".format(reference))
+            return no_cache_target
+
+        # Caching enabled
+        # Get metadata
+        try:
+            info = self.fs.info(reference)
+        except FileNotFoundError:
+            if required:
+                raise MissingInputsException(reference)
+            return None
+
+        # Raise error if type is not file
+        if info.get("type") == "directory":
+            raise OasisException(f"Directories are not supported in get_from_cache: {reference}")
+
+        remote_etag = info.get("ETag") or info.get("etag")
+        if remote_etag is None:
+            self.logger.warning(f"ETag missing for {reference} — skipping cache and returning fresh download")
+            if no_cache_target is not None:
+                dest_path = no_cache_target
+                Path(dest_path).parent.mkdir(parents=True, exist_ok=True)
+            else:
+                tmp = tempfile.NamedTemporaryFile(delete=False)
+                dest_path = tmp.name
+                tmp.close()
+            with self.fs.open(reference, "rb") as src, open(dest_path, "wb") as out:
+                shutil.copyfileobj(src, out)
+            return dest_path
+
+        # Create Cache dir
+        content_dir = Path(self.cache_root)
+        content_dir.mkdir(parents=True, exist_ok=True)
+
+        # Create reference hash for fast lookup
+        ref_hash = xxhash.xxh64(reference.encode()).hexdigest()
+        file_dir = content_dir / ref_hash
+        file_dir.mkdir(parents=True, exist_ok=True)
+        file_path = file_dir / "data"
+        etag_path = file_dir / "etag"
+
+        # Return if etag matches
+        if file_path.exists() and etag_path.exists():
+            cached_etag = etag_path.read_text()
+            if cached_etag == remote_etag:
+                return str(file_path)
+
+        # Redownload data and write etag
+        with self.fs.open(reference, "rb") as f, open(file_path, "wb") as out:
+            shutil.copyfileobj(f, out)
+        etag_path.write_text(remote_etag)
+
+        return str(file_path)
 
     def get(self, reference, output_path="", subdir="", required=False):
         """Retrieve stored object and stores it in the output path
@@ -386,7 +439,7 @@ class BaseStorage(object):
             self._fs = StrictRootDirFs(
                 path=self.root_dir,
                 fs=(
-                    self.fsspec_filesystem_class(**self.get_fsspec_storage_options())
+                    self.fsspec_filesystem_class(**self.get_fsspec_storage_options(), asynchronous=False)
                     if self.fsspec_filesystem_class
                     else None
                 ),

{oasis_data_manager-0.1.6.dist-info → oasis_data_manager-0.2.0.dist-info}/METADATA

@@ -1,12 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: oasis-data-manager
-Version: 0.1.6
-Summary: UNKNOWN
+Version: 0.2.0
 Home-page: https://github.com/OasisLMF/OasisDataManager
 Author: Oasis LMF
 Author-email: support@oasislmf.org
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -16,7 +13,8 @@ Description-Content-Type: text/markdown
 Requires-Dist: fastparquet
 Requires-Dist: fsspec>=2023.12.2
 Requires-Dist: pandas
-Requires-Dist: typing-extensions
+Requires-Dist: typing_extensions
+Requires-Dist: xxhash
 Provides-Extra: extra
 Requires-Dist: adlfs; extra == "extra"
 Requires-Dist: boto3; extra == "extra"
@@ -27,6 +25,12 @@ Requires-Dist: distributed; extra == "extra"
 Requires-Dist: geopandas==0.14.4; extra == "extra"
 Requires-Dist: pyogrio; extra == "extra"
 Requires-Dist: s3fs>=2023.12.2; extra == "extra"
-
-UNKNOWN
-
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python

{oasis_data_manager-0.1.6.dist-info → oasis_data_manager-0.2.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-oasis_data_manager/__init__.py,sha256=gW5NUxwGdPsiQjn0cOuuQT11pfthByI5DITDg_HMhLQ,22
+oasis_data_manager/__init__.py,sha256=FVHPBGkfhbQDi_z3v0PiKJrXXqXOx0vGW_1VaqNJi7U,22
 oasis_data_manager/config.py,sha256=_qx2Mu5n0Jx3W5SKCiqLr1SPdWLrbFv_B82r6Eosp_k,534
 oasis_data_manager/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/complex/complex.py,sha256=8oomV9WyLsa8sz8aMzlwv4naKCGOL3UdSlYQJxUFqCk,5382
@@ -19,10 +19,10 @@ oasis_data_manager/filestore/filestore.py,sha256=eaQGAer7Q9KM4B3bq9WmZAtjFdj9aRe
 oasis_data_manager/filestore/log.py,sha256=8l54LoOJiOG2pr4o93LzMocjH7dHcsOp14JWJ_MrqHQ,693
 oasis_data_manager/filestore/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/filestore/backends/aws_s3.py,sha256=PhzHHoAHDnBAskVfAm3Vj_18GZSyl9D0DF7qCod9_Ok,10567
-oasis_data_manager/filestore/backends/azure_abfs.py,sha256=IFHByChXB1AHbesCHZUHqIqbPvWi7X06rzREIc0BKeo,5315
-oasis_data_manager/filestore/backends/base.py,sha256=O-JmKpUKiXIsqR_PBmMT4_NNBYMzsp9r04q8u4iWjp0,13597
+oasis_data_manager/filestore/backends/azure_abfs.py,sha256=gSB0p3PdR9jV8TY3mqpM-Vveg9TOOOXr91qLXDaVxCs,5239
+oasis_data_manager/filestore/backends/base.py,sha256=mKi0IWWjzEsjy4o3KijFe58F2Yr1X9-SNJ8j13u1dhQ,15792
 oasis_data_manager/filestore/backends/local.py,sha256=MEX_CvwhsDfv9lvBjc8CdaDXaN53l9onQHmOgKjoJcg,1242
-oasis_data_manager-0.1.6.dist-info/METADATA,sha256=7Syo6SMizc0Y5Q2MA21J3pMwKfmfi1SoreWDNi-dWsE,1017
-oasis_data_manager-0.1.6.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
-oasis_data_manager-0.1.6.dist-info/top_level.txt,sha256=qMC39T9UvDCPbNJLVtgu8h6f7c4KJYel7SnIpz62wsU,19
-oasis_data_manager-0.1.6.dist-info/RECORD,,
+oasis_data_manager-0.2.0.dist-info/METADATA,sha256=Epm0t_ZiKUi1kscNeEQEt-imBtw5GshejFGBaAl945E,1178
+oasis_data_manager-0.2.0.dist-info/WHEEL,sha256=AeO2BvogYWm3eGaHCvhzmUYt8ia7KfURiHzO_1atlys,109
+oasis_data_manager-0.2.0.dist-info/top_level.txt,sha256=qMC39T9UvDCPbNJLVtgu8h6f7c4KJYel7SnIpz62wsU,19
+oasis_data_manager-0.2.0.dist-info/RECORD,,

{oasis_data_manager-0.1.6.dist-info → oasis_data_manager-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any