oasis-data-manager 0.1.3__tar.gz → 0.1.5__tar.gz

{oasis-data-manager-0.1.3/oasis_data_manager.egg-info → oasis-data-manager-0.1.5}/PKG-INFO

@@ -1,21 +1,19 @@
 Metadata-Version: 2.1
 Name: oasis-data-manager
-Version: 0.1.3
+Version: 0.1.5
 Summary: UNKNOWN
 Home-page: https://github.com/OasisLMF/OasisDataManager
 Author: Oasis LMF
 Author-email: support@oasislmf.org
-License: BSD 3-Clause
+License: UNKNOWN
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.6
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 Provides-Extra: extra
-License-File: LICENSE
 
 UNKNOWN
 

oasis-data-manager-0.1.5/oasis_data_manager/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.1.5'

{oasis-data-manager-0.1.3 → oasis-data-manager-0.1.5}/oasis_data_manager/df_reader/backends/base.py

@@ -64,11 +64,22 @@ class OasisReader:
     def _read(self):
         if not self.has_read:
             if hasattr(self.filename_or_buffer, "name"):
-                extension = pathlib.Path(self.filename_or_buffer.name).suffix
+                parts = pathlib.Path(self.filename_or_buffer.name).parts
             else:
-                extension = pathlib.Path(self.filename_or_buffer).suffix
+                parts = pathlib.Path(self.filename_or_buffer).parts
+
+            for part in parts:
+                for extension in [".parquet", ".pq"]:
+                    if part.endswith(extension):
+                        is_parquet = True
+                        break
+                else:
+                    continue  # if parquet extension is found the outer break will be called exiting the for with is_parquet = True
+                break
+            else:
+                is_parquet = False
 
-            if extension in [".parquet", ".pq"]:
+            if is_parquet:
                 self.has_read = True
                 self.read_parquet(*self.reader_args, **self.reader_kwargs)
             else:

oasis-data-manager-0.1.5/oasis_data_manager/df_reader/backends/pyarrow.py

@@ -0,0 +1,73 @@
+import pyarrow.dataset as ds
+from functools import reduce
+
+from .base import OasisReader
+
+
+class OasisPyarrowReader(OasisReader):
+
+    def read_parquet(self, *args, **kwargs):
+        def pyarrow_dataset_filter(pandas_filter):
+            def pyarrow_dataset_single_filter(pandas_single_filter):
+                op_map = {
+                    "==": lambda x, y: ds.field(x) == y,
+                    "!=": lambda x, y: ds.field(x) != y,
+                    ">=": lambda x, y: ds.field(x) >= y,
+                    "<=": lambda x, y: ds.field(x) <= y,
+                    ">": lambda x, y: ds.field(x) > y,
+                    "<": lambda x, y: ds.field(x) < y,
+                    "in": lambda x, y: ds.field(x).isin(y),
+                    "not in": lambda x, y: ~ds.field(x).isin(y)
+                }
+
+                field_name, operator, value = pandas_single_filter
+                df_filter = op_map.get(operator, lambda x, y: None)(field_name, value)
+
+                return df_filter
+
+            def list_of_tuples(lot):
+                # Apply pyarrow_dataset_single_filter to each tuple in the list and combine using &
+                return reduce(lambda x, y: x & y, (pyarrow_dataset_single_filter(f) for f in lot))
+
+            def list_of_lists(lol):
+                # Convert each list in lol to a tuple of filters using list_of_tuples, then combine using |
+                return reduce(lambda x, y: x | y, (list_of_tuples(f) for f in lol))
+
+            # Check if all elements in pandas_filter are of the same type (either tuple or list)
+            element_types = {type(f) for f in pandas_filter}
+            if len(element_types) > 1:
+                raise ValueError("Mixing and matching tuples and lists")
+
+            # Process based on the single type found
+            if isinstance(pandas_filter[0], tuple):
+                new_filter = list_of_tuples(pandas_filter)
+            elif isinstance(pandas_filter[0], list):
+                new_filter = list_of_lists(pandas_filter)
+            else:
+                raise TypeError("Unsupported type in filter")
+
+            return new_filter
+
+        if 'filters' in kwargs:
+            ds_filter = pyarrow_dataset_filter(kwargs['filters'])
+        else:
+            ds_filter = None
+
+        if isinstance(self.filename_or_buffer, str):
+            if self.filename_or_buffer.startswith(
+                    "http://"
+            ) or self.filename_or_buffer.startswith("https://"):
+                dataset = ds.dataset(self.filename_or_buffer, partitioning='hive')
+                self.df = dataset.to_table(filter=ds_filter).to_pandas()
+            else:
+                _, uri = self.storage.get_storage_url(
+                    self.filename_or_buffer, encode_params=False
+                )
+
+                uri = uri.replace('file://', '')
+                dataset = ds.dataset(uri, partitioning='hive')
+                self.df = dataset.to_table(filter=ds_filter).to_pandas()
+
+        else:
+            dataset = ds.dataset(self.filename_or_buffer, partitioning='hive')
+            self.df = dataset.to_table(filter=ds_filter).to_pandas()

{oasis-data-manager-0.1.3 → oasis-data-manager-0.1.5}/oasis_data_manager/df_reader/reader.py

@@ -6,6 +6,7 @@ __all__ = [
     'OasisDaskReader',
     'OasisDaskReaderCSV',
     'OasisDaskReaderParquet',
+    'OasisPyarrowReader',
 ]
 
 """
@@ -16,6 +17,11 @@ __all__ = [
 from .backends.base import OasisReader
 from .backends.pandas import OasisPandasReader, OasisPandasReaderCSV, OasisPandasReaderParquet
 
+try:
+    from .backends.pyarrow import OasisPyarrowReader
+except ModuleNotFoundError as e:
+    pass
+
 try:
     from .backends.dask import OasisDaskReader, OasisDaskReaderCSV, OasisDaskReaderParquet
 except ModuleNotFoundError as e:

{oasis-data-manager-0.1.3 → oasis-data-manager-0.1.5}/oasis_data_manager/filestore/backends/aws_s3.py

@@ -170,7 +170,6 @@ class AwsS3Storage(BaseStorage):
             s3_additional_kwargs["StorageClass"] = "REDUCED_REDUNDANCY"
 
         return {
-            "anon": not self.access_key and not self.security_token,
             "key": self.access_key,
             "secret": self.secret_key,
             "token": self.security_token,

{oasis-data-manager-0.1.3 → oasis-data-manager-0.1.5/oasis_data_manager.egg-info}/PKG-INFO

@@ -1,21 +1,19 @@
 Metadata-Version: 2.1
 Name: oasis-data-manager
-Version: 0.1.3
+Version: 0.1.5
 Summary: UNKNOWN
 Home-page: https://github.com/OasisLMF/OasisDataManager
 Author: Oasis LMF
 Author-email: support@oasislmf.org
-License: BSD 3-Clause
+License: UNKNOWN
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.6
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 Provides-Extra: extra
-License-File: LICENSE
 
 UNKNOWN
 

{oasis-data-manager-0.1.3 → oasis-data-manager-0.1.5}/oasis_data_manager.egg-info/SOURCES.txt

@@ -1,5 +1,4 @@
 CHANGELOG.rst
-LICENSE
 MANIFEST.in
 README.md
 requirements-package.in
@@ -23,6 +22,7 @@ oasis_data_manager/df_reader/backends/__init__.py
 oasis_data_manager/df_reader/backends/base.py
 oasis_data_manager/df_reader/backends/dask.py
 oasis_data_manager/df_reader/backends/pandas.py
+oasis_data_manager/df_reader/backends/pyarrow.py
 oasis_data_manager/errors/__init__.py
 oasis_data_manager/filestore/__init__.py
 oasis_data_manager/filestore/config.py

{oasis-data-manager-0.1.3 → oasis-data-manager-0.1.5}/oasis_data_manager.egg-info/requires.txt

@@ -6,10 +6,10 @@ typing_extensions
 [extra]
 adlfs
 boto3
-dask<=2024.1.1
+dask>2024.1.1
 dask-geopandas
 dask-sql
 distributed
-geopandas
+geopandas==0.14.4
 pyogrio
 s3fs>=2023.12.2

{oasis-data-manager-0.1.3 → oasis-data-manager-0.1.5}/setup.cfg

@@ -2,6 +2,8 @@
 universal = 1
 
 [metadata]
+name = oasis-data-manager
+version = attr: oasis_data_manager.__version__
 description-file = README.md
 
 [tool:pytest]

{oasis-data-manager-0.1.3 → oasis-data-manager-0.1.5}/setup.py

@@ -39,14 +39,11 @@ setup(
     packages=find_packages(exclude=("tests", "tests.*", "tests.*.*")),
     include_package_data=True,
     package_data={
-        "": [
-            "LICENSE",
-        ],
+        "": [],
     },
     exclude_package_data={
         "": ["__pycache__", "*.py[co]"],
     },
-    license="BSD 3-Clause",
     description="",
     long_description="",
     long_description_content_type="text/markdown",
@@ -59,7 +56,6 @@ setup(
     extras_require=get_optional_requirements(),
     classifiers=[
         "Development Status :: 4 - Beta",
-        "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.6",

oasis-data-manager-0.1.3/LICENSE

@@ -1,28 +0,0 @@
-BSD 3-Clause License
-
-Copyright (c) 2023, Oasis Loss Modelling Framework
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-   contributors may be used to endorse or promote products derived from
-   this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

oasis-data-manager-0.1.3/oasis_data_manager/__init__.py

@@ -1 +0,0 @@
-__version__ = '0.1.3'