oasis-data-manager 0.1.3__py2.py3-none-any.whl → 0.1.5__py2.py3-none-any.whl

oasis_data_manager/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.3'
+__version__ = '0.1.5'

oasis_data_manager/df_reader/backends/base.py

@@ -64,11 +64,22 @@ class OasisReader:
     def _read(self):
         if not self.has_read:
             if hasattr(self.filename_or_buffer, "name"):
-                extension = pathlib.Path(self.filename_or_buffer.name).suffix
+                parts = pathlib.Path(self.filename_or_buffer.name).parts
             else:
-                extension = pathlib.Path(self.filename_or_buffer).suffix
+                parts = pathlib.Path(self.filename_or_buffer).parts
+
+            for part in parts:
+                for extension in [".parquet", ".pq"]:
+                    if part.endswith(extension):
+                        is_parquet = True
+                        break
+                else:
+                    continue  # if parquet extension is found the outer break will be called exiting the for with is_parquet = True
+                break
+            else:
+                is_parquet = False
 
-            if extension in [".parquet", ".pq"]:
+            if is_parquet:
                 self.has_read = True
                 self.read_parquet(*self.reader_args, **self.reader_kwargs)
             else:

oasis_data_manager/df_reader/backends/pyarrow.py

@@ -0,0 +1,73 @@
+import pyarrow.dataset as ds
+from functools import reduce
+
+from .base import OasisReader
+
+
+class OasisPyarrowReader(OasisReader):
+
+    def read_parquet(self, *args, **kwargs):
+        def pyarrow_dataset_filter(pandas_filter):
+            def pyarrow_dataset_single_filter(pandas_single_filter):
+                op_map = {
+                    "==": lambda x, y: ds.field(x) == y,
+                    "!=": lambda x, y: ds.field(x) != y,
+                    ">=": lambda x, y: ds.field(x) >= y,
+                    "<=": lambda x, y: ds.field(x) <= y,
+                    ">": lambda x, y: ds.field(x) > y,
+                    "<": lambda x, y: ds.field(x) < y,
+                    "in": lambda x, y: ds.field(x).isin(y),
+                    "not in": lambda x, y: ~ds.field(x).isin(y)
+                }
+
+                field_name, operator, value = pandas_single_filter
+                df_filter = op_map.get(operator, lambda x, y: None)(field_name, value)
+
+                return df_filter
+
+            def list_of_tuples(lot):
+                # Apply pyarrow_dataset_single_filter to each tuple in the list and combine using &
+                return reduce(lambda x, y: x & y, (pyarrow_dataset_single_filter(f) for f in lot))
+
+            def list_of_lists(lol):
+                # Convert each list in lol to a tuple of filters using list_of_tuples, then combine using |
+                return reduce(lambda x, y: x | y, (list_of_tuples(f) for f in lol))
+
+            # Check if all elements in pandas_filter are of the same type (either tuple or list)
+            element_types = {type(f) for f in pandas_filter}
+            if len(element_types) > 1:
+                raise ValueError("Mixing and matching tuples and lists")
+
+            # Process based on the single type found
+            if isinstance(pandas_filter[0], tuple):
+                new_filter = list_of_tuples(pandas_filter)
+            elif isinstance(pandas_filter[0], list):
+                new_filter = list_of_lists(pandas_filter)
+            else:
+                raise TypeError("Unsupported type in filter")
+
+            return new_filter
+
+        if 'filters' in kwargs:
+            ds_filter = pyarrow_dataset_filter(kwargs['filters'])
+        else:
+            ds_filter = None
+
+        if isinstance(self.filename_or_buffer, str):
+            if self.filename_or_buffer.startswith(
+                    "http://"
+            ) or self.filename_or_buffer.startswith("https://"):
+                dataset = ds.dataset(self.filename_or_buffer, partitioning='hive')
+                self.df = dataset.to_table(filter=ds_filter).to_pandas()
+            else:
+                _, uri = self.storage.get_storage_url(
+                    self.filename_or_buffer, encode_params=False
+                )
+
+                uri = uri.replace('file://', '')
+                dataset = ds.dataset(uri, partitioning='hive')
+                self.df = dataset.to_table(filter=ds_filter).to_pandas()
+
+        else:
+            dataset = ds.dataset(self.filename_or_buffer, partitioning='hive')
+            self.df = dataset.to_table(filter=ds_filter).to_pandas()

oasis_data_manager/df_reader/reader.py

@@ -6,6 +6,7 @@ __all__ = [
     'OasisDaskReader',
     'OasisDaskReaderCSV',
     'OasisDaskReaderParquet',
+    'OasisPyarrowReader',
 ]
 
 """
@@ -16,6 +17,11 @@ __all__ = [
 from .backends.base import OasisReader
 from .backends.pandas import OasisPandasReader, OasisPandasReaderCSV, OasisPandasReaderParquet
 
+try:
+    from .backends.pyarrow import OasisPyarrowReader
+except ModuleNotFoundError as e:
+    pass
+
 try:
     from .backends.dask import OasisDaskReader, OasisDaskReaderCSV, OasisDaskReaderParquet
 except ModuleNotFoundError as e:

oasis_data_manager/filestore/backends/aws_s3.py

@@ -170,7 +170,6 @@ class AwsS3Storage(BaseStorage):
             s3_additional_kwargs["StorageClass"] = "REDUCED_REDUNDANCY"
 
         return {
-            "anon": not self.access_key and not self.security_token,
             "key": self.access_key,
             "secret": self.secret_key,
             "token": self.security_token,

{oasis_data_manager-0.1.3.dist-info → oasis_data_manager-0.1.5.dist-info}/METADATA

@@ -1,34 +1,32 @@
 Metadata-Version: 2.1
 Name: oasis-data-manager
-Version: 0.1.3
+Version: 0.1.5
 Summary: UNKNOWN
 Home-page: https://github.com/OasisLMF/OasisDataManager
 Author: Oasis LMF
 Author-email: support@oasislmf.org
-License: BSD 3-Clause
+License: UNKNOWN
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.6
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: fastparquet
-Requires-Dist: fsspec >=2023.12.2
+Requires-Dist: fsspec>=2023.12.2
 Requires-Dist: pandas
 Requires-Dist: typing-extensions
 Provides-Extra: extra
-Requires-Dist: adlfs ; extra == 'extra'
-Requires-Dist: boto3 ; extra == 'extra'
-Requires-Dist: dask <=2024.1.1 ; extra == 'extra'
-Requires-Dist: dask-geopandas ; extra == 'extra'
-Requires-Dist: dask-sql ; extra == 'extra'
-Requires-Dist: distributed ; extra == 'extra'
-Requires-Dist: geopandas ; extra == 'extra'
-Requires-Dist: pyogrio ; extra == 'extra'
-Requires-Dist: s3fs >=2023.12.2 ; extra == 'extra'
+Requires-Dist: adlfs; extra == "extra"
+Requires-Dist: boto3; extra == "extra"
+Requires-Dist: dask>2024.1.1; extra == "extra"
+Requires-Dist: dask-geopandas; extra == "extra"
+Requires-Dist: dask-sql; extra == "extra"
+Requires-Dist: distributed; extra == "extra"
+Requires-Dist: geopandas==0.14.4; extra == "extra"
+Requires-Dist: pyogrio; extra == "extra"
+Requires-Dist: s3fs>=2023.12.2; extra == "extra"
 
 UNKNOWN
 

{oasis_data_manager-0.1.3.dist-info → oasis_data_manager-0.1.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-oasis_data_manager/__init__.py,sha256=uZsygMXMKRw-7qhWojAjnpm8GFPXU92xW6XA8O5GwFY,22
+oasis_data_manager/__init__.py,sha256=DQiXHqQ3C29f_--ye7aVMDPnITrl9M0zMU42LYY48zE,22
 oasis_data_manager/config.py,sha256=_qx2Mu5n0Jx3W5SKCiqLr1SPdWLrbFv_B82r6Eosp_k,534
 oasis_data_manager/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/complex/complex.py,sha256=8oomV9WyLsa8sz8aMzlwv4naKCGOL3UdSlYQJxUFqCk,5382
@@ -6,23 +6,23 @@ oasis_data_manager/complex/examples.py,sha256=HlwOzJ2SVF9yE7ei9d2HWglUkYApiyQxwm
 oasis_data_manager/df_reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/df_reader/config.py,sha256=2xwWg5b6dnERUgQYWBjt2W64GdYUhXTtXCgrSPP60Mg,2507
 oasis_data_manager/df_reader/exceptions.py,sha256=9FV8n2eqrkTGpEt47GGs5k0eon2Y-Xz5K3wyc1R9fBs,102
-oasis_data_manager/df_reader/reader.py,sha256=14wuGTBKnIRslDMXsA3QjBiuvzcweRi29nM-V46pmLE,597
+oasis_data_manager/df_reader/reader.py,sha256=E_AV-DgZl4GFJibNS1GK7L4SXRh50DPEHlU8qYVwGWM,724
 oasis_data_manager/df_reader/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oasis_data_manager/df_reader/backends/base.py,sha256=cwvHzncinHfwwHW5hIjMQZ6KCVECxE3NixPT5SX7s-M,3057
+oasis_data_manager/df_reader/backends/base.py,sha256=gKaQKlMVWDftPJY6nhoA8IYwz-Csgq-xGm7iMV3tVBM,3461
 oasis_data_manager/df_reader/backends/dask.py,sha256=fE2vEAd5y7haUzi0oEUBE_4D5UL0jkmKyaAEaWnuFC4,6975
 oasis_data_manager/df_reader/backends/pandas.py,sha256=yTfULUunn0JLUqa_P5nazkGF6g6omaUY0litnPFITD4,3656
+oasis_data_manager/df_reader/backends/pyarrow.py,sha256=DCqJEgUJDHo0m_ke4WFFSOqPfPeskSO8Q5wMgihxqsI,3119
 oasis_data_manager/errors/__init__.py,sha256=9q_7nk5DNg1-WfQoBM4kw_Us34Y2szNkZwfE5-6_Rg0,687
 oasis_data_manager/filestore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oasis_data_manager/filestore/config.py,sha256=N0hSfPpRpej7uWGV54MEEH_0vvdpcgo3fZlx6m2muoY,3625
 oasis_data_manager/filestore/filestore.py,sha256=eaQGAer7Q9KM4B3bq9WmZAtjFdj9aRef_E3rI2i0dOk,2615
 oasis_data_manager/filestore/log.py,sha256=8l54LoOJiOG2pr4o93LzMocjH7dHcsOp14JWJ_MrqHQ,693
 oasis_data_manager/filestore/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oasis_data_manager/filestore/backends/aws_s3.py,sha256=uV8HfyJ_VuyCyl6oN0qIxpbhxhQ1aGAG4UfjyETyHP0,10636
+oasis_data_manager/filestore/backends/aws_s3.py,sha256=PhzHHoAHDnBAskVfAm3Vj_18GZSyl9D0DF7qCod9_Ok,10567
 oasis_data_manager/filestore/backends/azure_abfs.py,sha256=Lo2aBBQsFPJJEb9svm1-W43Gz2g4LCaHlM-9mVXDhzg,5354
 oasis_data_manager/filestore/backends/base.py,sha256=O-JmKpUKiXIsqR_PBmMT4_NNBYMzsp9r04q8u4iWjp0,13597
 oasis_data_manager/filestore/backends/local.py,sha256=MEX_CvwhsDfv9lvBjc8CdaDXaN53l9onQHmOgKjoJcg,1242
-oasis_data_manager-0.1.3.dist-info/LICENSE,sha256=qr-PXl5mSpeUk-A7RzYcH0dhR93hhgVK8SW9mzco0Ao,1517
-oasis_data_manager-0.1.3.dist-info/METADATA,sha256=P05T_lkfvsQzfKhjjYG3KE68jE44WxkJBRvj9w8aOUs,1100
-oasis_data_manager-0.1.3.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-oasis_data_manager-0.1.3.dist-info/top_level.txt,sha256=qMC39T9UvDCPbNJLVtgu8h6f7c4KJYel7SnIpz62wsU,19
-oasis_data_manager-0.1.3.dist-info/RECORD,,
+oasis_data_manager-0.1.5.dist-info/METADATA,sha256=Xmihv5tQoih-Ds0fUEFC8jM2VNX4eR23ER849lKftt4,1017
+oasis_data_manager-0.1.5.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
+oasis_data_manager-0.1.5.dist-info/top_level.txt,sha256=qMC39T9UvDCPbNJLVtgu8h6f7c4KJYel7SnIpz62wsU,19
+oasis_data_manager-0.1.5.dist-info/RECORD,,

{oasis_data_manager-0.1.3.dist-info → oasis_data_manager-0.1.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

oasis_data_manager-0.1.3.dist-info/LICENSE

@@ -1,28 +0,0 @@
-BSD 3-Clause License
-
-Copyright (c) 2023, Oasis Loss Modelling Framework
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-   contributors may be used to endorse or promote products derived from
-   this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.