oasis-data-manager 0.1.2__py2.py3-none-any.whl → 0.1.4__py2.py3-none-any.whl

oasis_data_manager/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.2'
+__version__ = '0.1.4'

oasis_data_manager/df_reader/backends/base.py

@@ -64,11 +64,22 @@ class OasisReader:
     def _read(self):
         if not self.has_read:
             if hasattr(self.filename_or_buffer, "name"):
-                extension = pathlib.Path(self.filename_or_buffer.name).suffix
+                parts = pathlib.Path(self.filename_or_buffer.name).parts
             else:
-                extension = pathlib.Path(self.filename_or_buffer).suffix
+                parts = pathlib.Path(self.filename_or_buffer).parts
+
+            for part in parts:
+                for extension in [".parquet", ".pq"]:
+                    if part.endswith(extension):
+                        is_parquet = True
+                        break
+                else:
+                    continue  # if parquet extension is found the outer break will be called exiting the for with is_parquet = True
+                break
+            else:
+                is_parquet = False
 
-            if extension in [".parquet", ".pq"]:
+            if is_parquet:
                 self.has_read = True
                 self.read_parquet(*self.reader_args, **self.reader_kwargs)
             else:

oasis_data_manager/df_reader/backends/pyarrow.py

@@ -0,0 +1,73 @@
+import pyarrow.dataset as ds
+from functools import reduce
+
+from .base import OasisReader
+
+
+class OasisPyarrowReader(OasisReader):
+
+    def read_parquet(self, *args, **kwargs):
+        def pyarrow_dataset_filter(pandas_filter):
+            def pyarrow_dataset_single_filter(pandas_single_filter):
+                op_map = {
+                    "==": lambda x, y: ds.field(x) == y,
+                    "!=": lambda x, y: ds.field(x) != y,
+                    ">=": lambda x, y: ds.field(x) >= y,
+                    "<=": lambda x, y: ds.field(x) <= y,
+                    ">": lambda x, y: ds.field(x) > y,
+                    "<": lambda x, y: ds.field(x) < y,
+                    "in": lambda x, y: ds.field(x).isin(y),
+                    "not in": lambda x, y: ~ds.field(x).isin(y)
+                }
+
+                field_name, operator, value = pandas_single_filter
+                df_filter = op_map.get(operator, lambda x, y: None)(field_name, value)
+
+                return df_filter
+
+            def list_of_tuples(lot):
+                # Apply pyarrow_dataset_single_filter to each tuple in the list and combine using &
+                return reduce(lambda x, y: x & y, (pyarrow_dataset_single_filter(f) for f in lot))
+
+            def list_of_lists(lol):
+                # Convert each list in lol to a tuple of filters using list_of_tuples, then combine using |
+                return reduce(lambda x, y: x | y, (list_of_tuples(f) for f in lol))
+
+            # Check if all elements in pandas_filter are of the same type (either tuple or list)
+            element_types = {type(f) for f in pandas_filter}
+            if len(element_types) > 1:
+                raise ValueError("Mixing and matching tuples and lists")
+
+            # Process based on the single type found
+            if isinstance(pandas_filter[0], tuple):
+                new_filter = list_of_tuples(pandas_filter)
+            elif isinstance(pandas_filter[0], list):
+                new_filter = list_of_lists(pandas_filter)
+            else:
+                raise TypeError("Unsupported type in filter")
+
+            return new_filter
+
+        if 'filters' in kwargs:
+            ds_filter = pyarrow_dataset_filter(kwargs['filters'])
+        else:
+            ds_filter = None
+
+        if isinstance(self.filename_or_buffer, str):
+            if self.filename_or_buffer.startswith(
+                    "http://"
+            ) or self.filename_or_buffer.startswith("https://"):
+                dataset = ds.dataset(self.filename_or_buffer, partitioning='hive')
+                self.df = dataset.to_table(filter=ds_filter).to_pandas()
+            else:
+                _, uri = self.storage.get_storage_url(
+                    self.filename_or_buffer, encode_params=False
+                )
+
+                uri = uri.replace('file://', '')
+                dataset = ds.dataset(uri, partitioning='hive')
+                self.df = dataset.to_table(filter=ds_filter).to_pandas()
+
+        else:
+            dataset = ds.dataset(self.filename_or_buffer, partitioning='hive')
+            self.df = dataset.to_table(filter=ds_filter).to_pandas()

oasis_data_manager/df_reader/reader.py

@@ -6,6 +6,7 @@ __all__ = [
     'OasisDaskReader',
     'OasisDaskReaderCSV',
     'OasisDaskReaderParquet',
+    'OasisPyarrowReader',
 ]
 
 """
@@ -16,6 +17,11 @@ __all__ = [
 from .backends.base import OasisReader
 from .backends.pandas import OasisPandasReader, OasisPandasReaderCSV, OasisPandasReaderParquet
 
+try:
+    from .backends.pyarrow import OasisPyarrowReader
+except ModuleNotFoundError as e:
+    pass
+
 try:
     from .backends.dask import OasisDaskReader, OasisDaskReaderCSV, OasisDaskReaderParquet
 except ModuleNotFoundError as e:

oasis_data_manager/filestore/backends/aws_s3.py

@@ -178,6 +178,7 @@ class AwsS3Storage(BaseStorage):
             "s3_additional_kwargs": s3_additional_kwargs,
             "client_kwargs": {
                 "endpoint_url": self.endpoint_url,
+                "region_name": self.region_name,
             },
         }
 

oasis_data_manager/filestore/backends/base.py

@@ -105,6 +105,8 @@ class BaseStorage(object):
         :return: filename string
         :rtype str
         """
+        if suffix.startswith('.'):
+            suffix = suffix[1:]
         return "{}.{}".format(uuid.uuid4().hex, suffix)
 
     def _is_valid_url(self, url):
@@ -246,6 +248,13 @@ class BaseStorage(object):
         :return: Absolute filepath to stored Object
         :rtype str
         """
+        # null ref given
+        if not reference:
+            if required:
+                raise MissingInputsException(reference)
+            else:
+                return None
+
         target = os.path.abspath(
             os.path.join(output_path, subdir) if subdir else output_path
         )
@@ -298,17 +307,23 @@ class BaseStorage(object):
         if not reference:
             return None
 
-        ext = "tar.gz" if not suffix else suffix
-        filename = filename if filename else self._get_unique_filename(ext)
-        storage_path = os.path.join(subdir, filename) if subdir else filename
+        if os.path.isfile(reference):
+            ext = "".join(Path(reference).suffixes) if not suffix else suffix
+            filename = filename if filename else self._get_unique_filename(ext)
+            storage_path = subdir if subdir else ''
+            self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+            storage_location = os.path.join(storage_path, filename)
 
-        self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+            self.logger.info("Store file: {} -> {}".format(reference, storage_location))
+            self.fs.put(reference, storage_location)
+            return storage_location
 
-        if os.path.isfile(reference):
-            self.logger.info("Store file: {} -> {}".format(reference, storage_path))
-            self.fs.put(reference, storage_path)
-            return storage_path
         elif os.path.isdir(reference):
+            ext = "tar.gz" if not suffix else suffix
+            filename = filename if filename else self._get_unique_filename(ext)
+            storage_path = os.path.join(subdir, filename) if subdir else filename
+            self.fs.mkdirs(os.path.dirname(storage_path), exist_ok=True)
+
             self.logger.info("Store dir: {} -> {}".format(reference, storage_path))
             with tempfile.NamedTemporaryFile() as f:
                 self.compress(f.name, reference, arcname)

oasis_data_manager-0.1.4.dist-info/METADATA

@@ -0,0 +1,32 @@
+Metadata-Version: 2.1
+Name: oasis-data-manager
+Version: 0.1.4
+Summary: UNKNOWN
+Home-page: https://github.com/OasisLMF/OasisDataManager
+Author: Oasis LMF
+Author-email: support@oasislmf.org
+License: UNKNOWN
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.6
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: fastparquet
+Requires-Dist: fsspec>=2023.12.2
+Requires-Dist: pandas
+Requires-Dist: typing-extensions
+Provides-Extra: extra
+Requires-Dist: adlfs; extra == "extra"
+Requires-Dist: boto3; extra == "extra"
+Requires-Dist: dask>2024.1.1; extra == "extra"
+Requires-Dist: dask-geopandas; extra == "extra"
+Requires-Dist: dask-sql; extra == "extra"
+Requires-Dist: distributed; extra == "extra"
+Requires-Dist: geopandas==0.14.4; extra == "extra"
+Requires-Dist: pyogrio; extra == "extra"
+Requires-Dist: s3fs>=2023.12.2; extra == "extra"
+
+UNKNOWN
+

{oasis_data_manager-0.1.2.dist-info → oasis_data_manager-0.1.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-oasis_data_manager/__init__.py,sha256=mdp2CftfqYbdKtP-eWv1z7rAUycYv6X1ntXSMUf8Kss,22
+oasis_data_manager/__init__.py,sha256=aBEbDvx4LMg8A1TJJR6dEHu8rQODVin528hLS_EDvuA,22
 oasis_data_manager/config.py,sha256=_qx2Mu5n0Jx3W5SKCiqLr1SPdWLrbFv_B82r6Eosp_k,534
 oasis_data_manager/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/complex/complex.py,sha256=8oomV9WyLsa8sz8aMzlwv4naKCGOL3UdSlYQJxUFqCk,5382
@@ -6,23 +6,23 @@ oasis_data_manager/complex/examples.py,sha256=HlwOzJ2SVF9yE7ei9d2HWglUkYApiyQxwm
 oasis_data_manager/df_reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/df_reader/config.py,sha256=2xwWg5b6dnERUgQYWBjt2W64GdYUhXTtXCgrSPP60Mg,2507
 oasis_data_manager/df_reader/exceptions.py,sha256=9FV8n2eqrkTGpEt47GGs5k0eon2Y-Xz5K3wyc1R9fBs,102
-oasis_data_manager/df_reader/reader.py,sha256=14wuGTBKnIRslDMXsA3QjBiuvzcweRi29nM-V46pmLE,597
+oasis_data_manager/df_reader/reader.py,sha256=E_AV-DgZl4GFJibNS1GK7L4SXRh50DPEHlU8qYVwGWM,724
 oasis_data_manager/df_reader/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oasis_data_manager/df_reader/backends/base.py,sha256=cwvHzncinHfwwHW5hIjMQZ6KCVECxE3NixPT5SX7s-M,3057
+oasis_data_manager/df_reader/backends/base.py,sha256=gKaQKlMVWDftPJY6nhoA8IYwz-Csgq-xGm7iMV3tVBM,3461
 oasis_data_manager/df_reader/backends/dask.py,sha256=fE2vEAd5y7haUzi0oEUBE_4D5UL0jkmKyaAEaWnuFC4,6975
 oasis_data_manager/df_reader/backends/pandas.py,sha256=yTfULUunn0JLUqa_P5nazkGF6g6omaUY0litnPFITD4,3656
+oasis_data_manager/df_reader/backends/pyarrow.py,sha256=DCqJEgUJDHo0m_ke4WFFSOqPfPeskSO8Q5wMgihxqsI,3119
 oasis_data_manager/errors/__init__.py,sha256=9q_7nk5DNg1-WfQoBM4kw_Us34Y2szNkZwfE5-6_Rg0,687
 oasis_data_manager/filestore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/filestore/config.py,sha256=N0hSfPpRpej7uWGV54MEEH_0vvdpcgo3fZlx6m2muoY,3625
 oasis_data_manager/filestore/filestore.py,sha256=eaQGAer7Q9KM4B3bq9WmZAtjFdj9aRef_E3rI2i0dOk,2615
 oasis_data_manager/filestore/log.py,sha256=8l54LoOJiOG2pr4o93LzMocjH7dHcsOp14JWJ_MrqHQ,693
 oasis_data_manager/filestore/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oasis_data_manager/filestore/backends/aws_s3.py,sha256=te2rVQl6n4xr4X-xQv68gDNQDIdTh9ar-FNI4JUtQJg,10587
+oasis_data_manager/filestore/backends/aws_s3.py,sha256=uV8HfyJ_VuyCyl6oN0qIxpbhxhQ1aGAG4UfjyETyHP0,10636
 oasis_data_manager/filestore/backends/azure_abfs.py,sha256=Lo2aBBQsFPJJEb9svm1-W43Gz2g4LCaHlM-9mVXDhzg,5354
-oasis_data_manager/filestore/backends/base.py,sha256=aj__0KsmnVbXTHYSA_qlrIe5pxImSZ14dPMzskdNzJc,12971
+oasis_data_manager/filestore/backends/base.py,sha256=O-JmKpUKiXIsqR_PBmMT4_NNBYMzsp9r04q8u4iWjp0,13597
 oasis_data_manager/filestore/backends/local.py,sha256=MEX_CvwhsDfv9lvBjc8CdaDXaN53l9onQHmOgKjoJcg,1242
-oasis_data_manager-0.1.2.dist-info/LICENSE,sha256=qr-PXl5mSpeUk-A7RzYcH0dhR93hhgVK8SW9mzco0Ao,1517
-oasis_data_manager-0.1.2.dist-info/METADATA,sha256=R5rXMfHrg3pAZ6wjYTjqScEiMBYvv32fFrQamt9LjZM,1110
-oasis_data_manager-0.1.2.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-oasis_data_manager-0.1.2.dist-info/top_level.txt,sha256=qMC39T9UvDCPbNJLVtgu8h6f7c4KJYel7SnIpz62wsU,19
-oasis_data_manager-0.1.2.dist-info/RECORD,,
+oasis_data_manager-0.1.4.dist-info/METADATA,sha256=o5qfWy7Vc-J8yFesOYDlH6bWTPgtguPVWMt92sJqIo4,1017
+oasis_data_manager-0.1.4.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
+oasis_data_manager-0.1.4.dist-info/top_level.txt,sha256=qMC39T9UvDCPbNJLVtgu8h6f7c4KJYel7SnIpz62wsU,19
+oasis_data_manager-0.1.4.dist-info/RECORD,,

{oasis_data_manager-0.1.2.dist-info → oasis_data_manager-0.1.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

oasis_data_manager-0.1.2.dist-info/LICENSE

@@ -1,28 +0,0 @@
-BSD 3-Clause License
-
-Copyright (c) 2023, Oasis Loss Modelling Framework
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-   contributors may be used to endorse or promote products derived from
-   this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

oasis_data_manager-0.1.2.dist-info/METADATA

@@ -1,35 +0,0 @@
-Metadata-Version: 2.1
-Name: oasis-data-manager
-Version: 0.1.2
-Summary: UNKNOWN
-Home-page: https://github.com/OasisLMF/OasisDataManager
-Author: Oasis LMF
-Author-email: support@oasislmf.org
-License: BSD 3-Clause
-Platform: UNKNOWN
-Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.6
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: fastparquet
-Requires-Dist: fsspec >=2023.12.2
-Requires-Dist: pandas
-Requires-Dist: typing
-Requires-Dist: typing-extensions
-Provides-Extra: extra
-Requires-Dist: adlfs ; extra == 'extra'
-Requires-Dist: boto3 ; extra == 'extra'
-Requires-Dist: dask ; extra == 'extra'
-Requires-Dist: dask-geopandas ; extra == 'extra'
-Requires-Dist: dask-sql ; extra == 'extra'
-Requires-Dist: distributed ; extra == 'extra'
-Requires-Dist: geopandas ; extra == 'extra'
-Requires-Dist: pyogrio ; extra == 'extra'
-Requires-Dist: s3fs >=2023.9.0 ; extra == 'extra'
-
-UNKNOWN
-