python-dwca-reader 0.15.1__tar.gz → 0.16.1__tar.gz

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/AUTHORS

@@ -5,6 +5,7 @@ Here is an inevitably incomplete list of MUCH-APPRECIATED CONTRIBUTORS -- people
 bugs, improved documentation, ...  and generally made this library much better:
 
     Dimitri Brosens
+    Scott Brown
     Ben Cail
     Julien Cigar
     Pedro Correia de Siracusa

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/CHANGES.txt

@@ -1,8 +1,21 @@
+v0.16.1 (2024-07-08)
+--------------------
+
+- Added official support for Python 3.12
+- Proper error message when trying to use an unsupported combination of Pandas option
+and archives with default values (issue #106).
+
+v0.16.0 (2023-11-13)
+--------------------
+
+- Dropped Python 3.5 and 3.6 support. Use Python 3.7 to 3.11 instead.
+- Added a star record iterator to DwCAReader (thanks to @csbrown)
+
 v0.15.1 (2023-01-17)
 --------------------
 
 - Workaround for errors with buggy GBIF downloads, see https://github.com/gbif/portal-feedback/issues/4533
-- Dropped Python 3.5
+
 
 v0.15.0 (2020-09-09)
 --------------------

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-dwca-reader
-Version: 0.15.1
+Version: 0.16.1
 Summary: A simple Python package to read Darwin Core Archive (DwC-A) files.
 Home-page: https://github.com/BelgianBiodiversityPlatform/python-dwca-reader
 Author: Nicolas Noé - Belgian Biodiversity Platform
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 License-File: LICENSE.txt
 License-File: AUTHORS

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/dwca/descriptors.py

@@ -108,7 +108,7 @@ class DataFileDescriptor(object):
         """
         file_encoding = "utf-8"
 
-        with io.open(datafile_path, 'rU', encoding=file_encoding) as datafile:
+        with io.open(datafile_path, 'r', encoding=file_encoding) as datafile:
             # Autodetect fields/lines termination
             dialect = csv.Sniffer().sniff(datafile.readline())
 

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/dwca/exceptions.py

@@ -14,4 +14,4 @@ class InvalidSimpleArchive(InvalidArchive):
 
 
 class NotADataFile(Exception):
-    """The file doesn't exists or is not a data file."""
+    """The file doesn't exist or is not a data file."""

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/dwca/files.py

@@ -6,7 +6,7 @@ from array import array
 from typing import List, Union, IO, Dict, Optional
 
 from dwca.descriptors import DataFileDescriptor
-from dwca.rows import CoreRow, ExtensionRow
+from dwca.rows import CoreRow, ExtensionRow, Row
 
 
 class CSVDataFile(object):
@@ -90,26 +90,15 @@ class CSVDataFile(object):
             core_id2: [8, 10] # Rows at position 8 and 10 references a Core Row whose ID is core_id2
             }
 
-        :raises: AttributeError if accessed on a core data file.
-
-        .. warning::
-
-            for permformance reasons, dictionary values are arrays('L') instead of regular python lists
-
         .. warning::
 
-            coreid_index is only available for extension data files.
+            for performance reasons, dictionary values are arrays('L') instead of regular python lists
 
         .. warning::
 
             Creating this index can be time and memory consuming for large archives, so it's created on the fly
             at first access.
         """
-        if self.file_descriptor.represents_corefile:
-            raise AttributeError(
-                "coreid_index is only available for extension data files"
-            )
-
         if self._coreid_index is None:
             self._coreid_index = self._build_coreid_index()
 
@@ -120,14 +109,18 @@ class CSVDataFile(object):
         index = {}  # type: Dict[str, array[int]]
 
         for position, row in enumerate(self):
-            tmp = ExtensionRow(row, position, self.file_descriptor)
-            index.setdefault(tmp.core_id, array('L')).append(position)
+            if self.file_descriptor.represents_corefile:
+              tmp = CoreRow(row, position, self.file_descriptor)
+              index.setdefault(tmp.id, array('L')).append(position)
+            else:
+              tmp = ExtensionRow(row, position, self.file_descriptor)
+              index.setdefault(tmp.core_id, array('L')).append(position)
 
         return index
 
     # TODO: For ExtensionRow and a specific field only, generalize ?
     # TODO: What happens if called on a Core Row?
-    def get_all_rows_by_coreid(self, core_id: int) -> List[ExtensionRow]:
+    def get_all_rows_by_coreid(self, core_id: int) -> List[Row]:
         """Return a list of :class:`dwca.rows.ExtensionRow` whose Core Id field match `core_id`."""
         if core_id not in self.coreid_index:
             return []

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/dwca/helpers.py

@@ -6,7 +6,7 @@ def remove_tree(path, retries=3, sleep=0.1):
     for i in range(retries):
         try:
             rmtree(path, False)
-        except WindowsError:
+        except OSError:
             time.sleep(sleep)
         else:
             break

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/dwca/read.py

@@ -10,6 +10,8 @@ from tempfile import mkdtemp
 from typing import List, Optional, Dict, Any, IO, Tuple
 from xml.etree.ElementTree import Element
 
+from pandas.io.parsers import TextFileReader
+
 import dwca.vendor
 from dwca.descriptors import ArchiveDescriptor, DataFileDescriptor, shorten_term
 from dwca.exceptions import RowNotFound, InvalidArchive, InvalidSimpleArchive, NotADataFile
@@ -186,7 +188,9 @@ class DwCAReader(object):
         .. note::
 
             Default values of Darwin Core Archive are supported: A column will be added to the DataFrame if a term has
-            a default value in the Metafile (but no corresponding column in the CSV Data File).
+            a default value in the Metafile (but no corresponding column in the CSV Data File). This is unfortunately
+            not supported in case the value returned by `pandas.read_csv()` is a `TextFileReader` (e.g. when using
+            `chunksize` or `iterator=True`).
         """
         datafile_descriptor = self.get_descriptor_for(relative_path)  # type: DataFileDescriptor
 
@@ -200,15 +204,25 @@ class DwCAReader(object):
         kwargs['header'] = None
         kwargs['names'] = datafile_descriptor.short_headers
 
-        df = read_csv(self.absolute_temporary_path(relative_path), **kwargs)
+        df_or_textreader = read_csv(self.absolute_temporary_path(relative_path), **kwargs)
 
         # Add a column for default values, if present in the file descriptor
         for field in datafile_descriptor.fields:
             field_default_value = field['default']
             if field_default_value is not None:
-                df[shorten_term(field['term'])] = field_default_value
+                if isinstance(df_or_textreader, TextFileReader):
+                    # I don't see how to assign default values to a TextFileReader, so
+                    # this is currently unsupported
+                    raise ValueError(
+                        "Pandas read_csv() was called with a chunksize or iterator=True, "
+                        "and therefore returns a TextFileReader instead of a DataFrame "
+                        "which is not supported in combination with default values of "
+                        "the archive."
+                    )
+
+                df_or_textreader[shorten_term(field['term'])] = field_default_value
 
-        return df
+        return df_or_textreader
 
     def orphaned_extension_rows(self) -> Dict[str, Dict[str, List[int]]]:
         """Return a dict of the orphaned extension rows.
@@ -434,6 +448,7 @@ class DwCAReader(object):
         except zipfile.BadZipfile:
             # Doesn't look like a valid zip, let's see if it's a tar archive (possibly compressed)
             try:
+                # TODO: Once we only support Python 3.12+, we should pass the filter="data" argument to extractall()
                 tarfile.open(self.archive_path, 'r:*').extractall(tmp_dir)
             except tarfile.ReadError:
                 raise InvalidArchive("The archive cannot be read. Is it a .zip or .tgz file?")

python-dwca-reader-0.16.1/dwca/star_record.py

@@ -0,0 +1,59 @@
+from dwca.files import CSVDataFile
+from typing import List
+from typing_extensions import Literal
+import itertools
+
+
+class StarRecordIterator(object):
+    """ Object used to iterate over multiple DWCA-files joined on the coreid
+    
+        :param files_to_join: a list of the `dwca.files.CSVDataFile`s we'd like to join.
+            May or may not include the core file (the core is not treated in a special way)
+        :param how: indicates the type of join.  "inner" and "outer" correspond vaguely to 
+            inner and full joins.  The outer join includes rows that don't match on all files,
+            however, it doesn't create null fields to fill in when rows are missing in files.
+            Attempts to conform to pandas.DataFrame.merge API.
+    """
+    def __init__(self, files_to_join: List[CSVDataFile], how: Literal["inner", "outer"] = "inner"):
+        self.files_to_join = files_to_join
+
+        # gather the coreids we want to join over.
+        self.common_core = set(self.files_to_join[0].coreid_index)
+        for data_file in self.files_to_join[1:]:
+            # inner join: coreid must be in all files
+            if how == "inner":
+                self.common_core &= set(data_file.coreid_index)
+            # outer join: coreid may be in any files
+            elif how == "outer":
+                self.common_core |= set(data_file.coreid_index)
+                
+        # initialize iterator variables
+        self._common_core_iterator = iter(self.common_core)
+        self._cross_product_iterator = iter([])
+
+    
+    def __next__(self):
+        # the next combination of rows matching this coreid
+        next_positions = next(self._cross_product_iterator, None)
+        # we finished all the combinations for this coreid
+        if not next_positions:
+            # get the next coreid
+            self._current_coreid = next(self._common_core_iterator)
+            self._files_with_current_coreid = [
+                csv_file for csv_file in self.files_to_join 
+                    if self._current_coreid in csv_file.coreid_index]
+            # this iterates over all combinations of rows matching a coreid from all files
+            self._cross_product_iterator = itertools.product(
+                *(
+                    csv_file.coreid_index[self._current_coreid] 
+                        for csv_file in self._files_with_current_coreid
+                ))
+            # go back and try to iterate over the rows for the new coreid
+            return next(self)
+        # zip up this combination of rows from all of the files.
+        return (
+            csv_file.get_row_by_position(position) for position, csv_file in zip(next_positions, self._files_with_current_coreid)
+        )
+
+    
+    def __iter__(self): return self

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/dwca/test/test_datafile.py

@@ -31,9 +31,13 @@ class TestCSVDataFile(unittest.TestCase):
         with DwCAReader(sample_data_path("dwca-2extensions.zip")) as dwca:
             extension_files = dwca.extension_files
 
+            core_txt = dwca.core_file
             description_txt = extension_files[0]
             vernacular_txt = extension_files[1]
 
+            expected_core = {'1': array('L', [0]), '2': array('L', [1]), '3': array('L', [2]), '4': array('L', [3])}
+            assert core_txt.coreid_index == expected_core
+
             expected_vernacular = {"1": array('L', [0, 1, 2]), "2": array('L', [3])}
             assert vernacular_txt.coreid_index == expected_vernacular
 

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/dwca/test/test_dwcareader.py

@@ -48,6 +48,26 @@ class TestPandasIntegration(unittest.TestCase):
             assert df["scientificName"].values.tolist() == \
                 ["tetraodon fluviatilis", "betta splendens"]
 
+    def test_pd_read_chunked_default_value(self):
+        """Pandas chuncksize should not be used with default values.
+
+        See: https://github.com/BelgianBiodiversityPlatform/python-dwca-reader/issues/106
+        """
+        with DwCAReader(sample_data_path("dwca-test-default.zip")) as dwca:
+            with pytest.raises(ValueError):
+                for chunk in dwca.pd_read("occurrence.txt", chunksize=1):
+                    pass
+
+    def test_pd_read_chunked(self):
+        """If no default values are available in the archive, chunksize should work.
+
+        See: https://github.com/BelgianBiodiversityPlatform/python-dwca-reader/issues/106
+        """
+        with DwCAReader(sample_data_path("dwca-simple-test-archive.zip")) as dwca:
+            for chunk in dwca.pd_read("occurrence.txt", chunksize=2):
+                assert isinstance(chunk, pd.DataFrame)
+
+
     def test_pd_read_no_data_files(self):
         with DwCAReader(sample_data_path("dwca-simple-test-archive.zip")) as dwca:
             with pytest.raises(NotADataFile):

python-dwca-reader-0.16.1/dwca/test/test_star_record.py

@@ -0,0 +1,54 @@
+from dwca.read import DwCAReader
+from dwca.rows import CoreRow
+from dwca.star_record import StarRecordIterator
+from .helpers import sample_data_path
+import unittest
+
+class TestStarRecordIterator(unittest.TestCase):
+
+    def test_inner_join(self):
+
+        expected_inner_join = frozenset({
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 0, 'VernacularName')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 1, 'VernacularName')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 2, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 0, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 1, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 2, 'VernacularName')})
+        })
+
+        with DwCAReader(sample_data_path("dwca-2extensions.zip")) as dwca:
+            star_records = StarRecordIterator(dwca.extension_files + [dwca.core_file], how="inner")
+            stars = []
+            for star_record in star_records:
+                rows = []
+                for row in star_record:
+                    rows.append((row.id if isinstance(row, CoreRow) else row.core_id, row.position, row.rowtype.split('/')[-1]))
+                stars.append(frozenset(rows))
+
+            assert frozenset(stars) == expected_inner_join
+    
+    def test_outer_join(self):
+
+        expected_outer_join = frozenset({
+            frozenset({('4', 2, 'Description'), ('4', 3, 'Taxon')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 0, 'VernacularName')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 1, 'VernacularName')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 2, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 0, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 1, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 2, 'VernacularName')}),
+            frozenset({('3', 2, 'Taxon')}),
+            frozenset({('2', 1, 'Taxon'), ('2', 3, 'VernacularName')}) 
+        })
+
+        with DwCAReader(sample_data_path("dwca-2extensions.zip")) as dwca:
+            star_records = StarRecordIterator(dwca.extension_files + [dwca.core_file], how="outer")
+            stars = []
+            for star_record in star_records:
+                rows = []
+                for row in star_record:
+                    rows.append((row.id if isinstance(row, CoreRow) else row.core_id, row.position, row.rowtype.split('/')[-1]))
+                stars.append(frozenset(rows))
+
+            assert frozenset(stars) == expected_outer_join

python-dwca-reader-0.16.1/dwca/version.py

@@ -0,0 +1 @@
+__version__ = '0.16.1'

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/python_dwca_reader.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-dwca-reader
-Version: 0.15.1
+Version: 0.16.1
 Summary: A simple Python package to read Darwin Core Archive (DwC-A) files.
 Home-page: https://github.com/BelgianBiodiversityPlatform/python-dwca-reader
 Author: Nicolas Noé - Belgian Biodiversity Platform
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 License-File: LICENSE.txt
 License-File: AUTHORS

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/python_dwca_reader.egg-info/SOURCES.txt

@@ -15,6 +15,7 @@ dwca/helpers.py
 dwca/minibench.py
 dwca/read.py
 dwca/rows.py
+dwca/star_record.py
 dwca/vendor.py
 dwca/version.py
 dwca/darwincore/__init__.py
@@ -27,6 +28,7 @@ dwca/test/test_datafile.py
 dwca/test/test_descriptors.py
 dwca/test/test_dwcareader.py
 dwca/test/test_rows.py
+dwca/test/test_star_record.py
 python_dwca_reader.egg-info/PKG-INFO
 python_dwca_reader.egg-info/SOURCES.txt
 python_dwca_reader.egg-info/dependency_links.txt

python-dwca-reader-0.16.1/requirements-dev.txt

@@ -0,0 +1,4 @@
+pandas
+mock==2.0.0
+pytest
+typing-extensions

{python-dwca-reader-0.15.1 → python-dwca-reader-0.16.1}/setup.py

@@ -18,6 +18,7 @@ setup(
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: Implementation :: PyPy',
     ]
 )

python-dwca-reader-0.15.1/dwca/version.py

@@ -1 +0,0 @@
-__version__ = '0.15.1'

python-dwca-reader-0.15.1/requirements-dev.txt

@@ -1,3 +0,0 @@
-pandas
-mock==2.0.0
-pytest