python-dwca-reader 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

dwca/descriptors.py

@@ -108,7 +108,7 @@ class DataFileDescriptor(object):
         """
         file_encoding = "utf-8"
 
-        with io.open(datafile_path, 'rU', encoding=file_encoding) as datafile:
+        with io.open(datafile_path, 'r', encoding=file_encoding) as datafile:
             # Autodetect fields/lines termination
             dialect = csv.Sniffer().sniff(datafile.readline())
 

dwca/exceptions.py

@@ -14,4 +14,4 @@ class InvalidSimpleArchive(InvalidArchive):
 
 
 class NotADataFile(Exception):
-    """The file doesn't exists or is not a data file."""
+    """The file doesn't exist or is not a data file."""

dwca/files.py

@@ -6,7 +6,7 @@ from array import array
 from typing import List, Union, IO, Dict, Optional
 
 from dwca.descriptors import DataFileDescriptor
-from dwca.rows import CoreRow, ExtensionRow
+from dwca.rows import CoreRow, ExtensionRow, Row
 
 
 class CSVDataFile(object):
@@ -90,26 +90,15 @@ class CSVDataFile(object):
             core_id2: [8, 10] # Rows at position 8 and 10 references a Core Row whose ID is core_id2
             }
 
-        :raises: AttributeError if accessed on a core data file.
-
-        .. warning::
-
-            for permformance reasons, dictionary values are arrays('L') instead of regular python lists
-
         .. warning::
 
-            coreid_index is only available for extension data files.
+            for performance reasons, dictionary values are arrays('L') instead of regular python lists
 
         .. warning::
 
             Creating this index can be time and memory consuming for large archives, so it's created on the fly
             at first access.
         """
-        if self.file_descriptor.represents_corefile:
-            raise AttributeError(
-                "coreid_index is only available for extension data files"
-            )
-
         if self._coreid_index is None:
             self._coreid_index = self._build_coreid_index()
 
@@ -120,14 +109,18 @@ class CSVDataFile(object):
         index = {}  # type: Dict[str, array[int]]
 
         for position, row in enumerate(self):
-            tmp = ExtensionRow(row, position, self.file_descriptor)
-            index.setdefault(tmp.core_id, array('L')).append(position)
+            if self.file_descriptor.represents_corefile:
+              tmp = CoreRow(row, position, self.file_descriptor)
+              index.setdefault(tmp.id, array('L')).append(position)
+            else:
+              tmp = ExtensionRow(row, position, self.file_descriptor)
+              index.setdefault(tmp.core_id, array('L')).append(position)
 
         return index
 
     # TODO: For ExtensionRow and a specific field only, generalize ?
     # TODO: What happens if called on a Core Row?
-    def get_all_rows_by_coreid(self, core_id: int) -> List[ExtensionRow]:
+    def get_all_rows_by_coreid(self, core_id: int) -> List[Row]:
         """Return a list of :class:`dwca.rows.ExtensionRow` whose Core Id field match `core_id`."""
         if core_id not in self.coreid_index:
             return []

dwca/helpers.py

@@ -6,7 +6,7 @@ def remove_tree(path, retries=3, sleep=0.1):
     for i in range(retries):
         try:
             rmtree(path, False)
-        except WindowsError:
+        except OSError:
             time.sleep(sleep)
         else:
             break

dwca/read.py

@@ -10,6 +10,8 @@ from tempfile import mkdtemp
 from typing import List, Optional, Dict, Any, IO, Tuple
 from xml.etree.ElementTree import Element
 
+from pandas.io.parsers import TextFileReader
+
 import dwca.vendor
 from dwca.descriptors import ArchiveDescriptor, DataFileDescriptor, shorten_term
 from dwca.exceptions import RowNotFound, InvalidArchive, InvalidSimpleArchive, NotADataFile
@@ -186,7 +188,9 @@ class DwCAReader(object):
         .. note::
 
             Default values of Darwin Core Archive are supported: A column will be added to the DataFrame if a term has
-            a default value in the Metafile (but no corresponding column in the CSV Data File).
+            a default value in the Metafile (but no corresponding column in the CSV Data File). This is unfortunately
+            not supported in case the value returned by `pandas.read_csv()` is a `TextFileReader` (e.g. when using
+            `chunksize` or `iterator=True`).
         """
         datafile_descriptor = self.get_descriptor_for(relative_path)  # type: DataFileDescriptor
 
@@ -200,15 +204,25 @@ class DwCAReader(object):
         kwargs['header'] = None
         kwargs['names'] = datafile_descriptor.short_headers
 
-        df = read_csv(self.absolute_temporary_path(relative_path), **kwargs)
+        df_or_textreader = read_csv(self.absolute_temporary_path(relative_path), **kwargs)
 
         # Add a column for default values, if present in the file descriptor
         for field in datafile_descriptor.fields:
             field_default_value = field['default']
             if field_default_value is not None:
-                df[shorten_term(field['term'])] = field_default_value
+                if isinstance(df_or_textreader, TextFileReader):
+                    # I don't see how to assign default values to a TextFileReader, so
+                    # this is currently unsupported
+                    raise ValueError(
+                        "Pandas read_csv() was called with a chunksize or iterator=True, "
+                        "and therefore returns a TextFileReader instead of a DataFrame "
+                        "which is not supported in combination with default values of "
+                        "the archive."
+                    )
+
+                df_or_textreader[shorten_term(field['term'])] = field_default_value
 
-        return df
+        return df_or_textreader
 
     def orphaned_extension_rows(self) -> Dict[str, Dict[str, List[int]]]:
         """Return a dict of the orphaned extension rows.
@@ -434,6 +448,7 @@ class DwCAReader(object):
         except zipfile.BadZipfile:
             # Doesn't look like a valid zip, let's see if it's a tar archive (possibly compressed)
             try:
+                # TODO: Once we only support Python 3.12+, we should pass the filter="data" argument to extractall()
                 tarfile.open(self.archive_path, 'r:*').extractall(tmp_dir)
             except tarfile.ReadError:
                 raise InvalidArchive("The archive cannot be read. Is it a .zip or .tgz file?")

dwca/star_record.py

@@ -0,0 +1,59 @@
+from dwca.files import CSVDataFile
+from typing import List
+from typing_extensions import Literal
+import itertools
+
+
+class StarRecordIterator(object):
+    """ Object used to iterate over multiple DWCA-files joined on the coreid
+    
+        :param files_to_join: a list of the `dwca.files.CSVDataFile`s we'd like to join.
+            May or may not include the core file (the core is not treated in a special way)
+        :param how: indicates the type of join.  "inner" and "outer" correspond vaguely to 
+            inner and full joins.  The outer join includes rows that don't match on all files,
+            however, it doesn't create null fields to fill in when rows are missing in files.
+            Attempts to conform to pandas.DataFrame.merge API.
+    """
+    def __init__(self, files_to_join: List[CSVDataFile], how: Literal["inner", "outer"] = "inner"):
+        self.files_to_join = files_to_join
+
+        # gather the coreids we want to join over.
+        self.common_core = set(self.files_to_join[0].coreid_index)
+        for data_file in self.files_to_join[1:]:
+            # inner join: coreid must be in all files
+            if how == "inner":
+                self.common_core &= set(data_file.coreid_index)
+            # outer join: coreid may be in any files
+            elif how == "outer":
+                self.common_core |= set(data_file.coreid_index)
+                
+        # initialize iterator variables
+        self._common_core_iterator = iter(self.common_core)
+        self._cross_product_iterator = iter([])
+
+    
+    def __next__(self):
+        # the next combination of rows matching this coreid
+        next_positions = next(self._cross_product_iterator, None)
+        # we finished all the combinations for this coreid
+        if not next_positions:
+            # get the next coreid
+            self._current_coreid = next(self._common_core_iterator)
+            self._files_with_current_coreid = [
+                csv_file for csv_file in self.files_to_join 
+                    if self._current_coreid in csv_file.coreid_index]
+            # this iterates over all combinations of rows matching a coreid from all files
+            self._cross_product_iterator = itertools.product(
+                *(
+                    csv_file.coreid_index[self._current_coreid] 
+                        for csv_file in self._files_with_current_coreid
+                ))
+            # go back and try to iterate over the rows for the new coreid
+            return next(self)
+        # zip up this combination of rows from all of the files.
+        return (
+            csv_file.get_row_by_position(position) for position, csv_file in zip(next_positions, self._files_with_current_coreid)
+        )
+
+    
+    def __iter__(self): return self

dwca/test/test_datafile.py

@@ -31,9 +31,13 @@ class TestCSVDataFile(unittest.TestCase):
         with DwCAReader(sample_data_path("dwca-2extensions.zip")) as dwca:
             extension_files = dwca.extension_files
 
+            core_txt = dwca.core_file
             description_txt = extension_files[0]
             vernacular_txt = extension_files[1]
 
+            expected_core = {'1': array('L', [0]), '2': array('L', [1]), '3': array('L', [2]), '4': array('L', [3])}
+            assert core_txt.coreid_index == expected_core
+
             expected_vernacular = {"1": array('L', [0, 1, 2]), "2": array('L', [3])}
             assert vernacular_txt.coreid_index == expected_vernacular
 

dwca/test/test_dwcareader.py

@@ -48,6 +48,26 @@ class TestPandasIntegration(unittest.TestCase):
             assert df["scientificName"].values.tolist() == \
                 ["tetraodon fluviatilis", "betta splendens"]
 
+    def test_pd_read_chunked_default_value(self):
+        """Pandas chuncksize should not be used with default values.
+
+        See: https://github.com/BelgianBiodiversityPlatform/python-dwca-reader/issues/106
+        """
+        with DwCAReader(sample_data_path("dwca-test-default.zip")) as dwca:
+            with pytest.raises(ValueError):
+                for chunk in dwca.pd_read("occurrence.txt", chunksize=1):
+                    pass
+
+    def test_pd_read_chunked(self):
+        """If no default values are available in the archive, chunksize should work.
+
+        See: https://github.com/BelgianBiodiversityPlatform/python-dwca-reader/issues/106
+        """
+        with DwCAReader(sample_data_path("dwca-simple-test-archive.zip")) as dwca:
+            for chunk in dwca.pd_read("occurrence.txt", chunksize=2):
+                assert isinstance(chunk, pd.DataFrame)
+
+
     def test_pd_read_no_data_files(self):
         with DwCAReader(sample_data_path("dwca-simple-test-archive.zip")) as dwca:
             with pytest.raises(NotADataFile):

dwca/test/test_star_record.py

@@ -0,0 +1,54 @@
+from dwca.read import DwCAReader
+from dwca.rows import CoreRow
+from dwca.star_record import StarRecordIterator
+from .helpers import sample_data_path
+import unittest
+
+class TestStarRecordIterator(unittest.TestCase):
+
+    def test_inner_join(self):
+
+        expected_inner_join = frozenset({
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 0, 'VernacularName')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 1, 'VernacularName')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 2, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 0, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 1, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 2, 'VernacularName')})
+        })
+
+        with DwCAReader(sample_data_path("dwca-2extensions.zip")) as dwca:
+            star_records = StarRecordIterator(dwca.extension_files + [dwca.core_file], how="inner")
+            stars = []
+            for star_record in star_records:
+                rows = []
+                for row in star_record:
+                    rows.append((row.id if isinstance(row, CoreRow) else row.core_id, row.position, row.rowtype.split('/')[-1]))
+                stars.append(frozenset(rows))
+
+            assert frozenset(stars) == expected_inner_join
+    
+    def test_outer_join(self):
+
+        expected_outer_join = frozenset({
+            frozenset({('4', 2, 'Description'), ('4', 3, 'Taxon')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 0, 'VernacularName')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 1, 'VernacularName')}),
+            frozenset({('1', 0, 'Description'), ('1', 0, 'Taxon'), ('1', 2, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 0, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 1, 'VernacularName')}),
+            frozenset({('1', 1, 'Description'), ('1', 0, 'Taxon'), ('1', 2, 'VernacularName')}),
+            frozenset({('3', 2, 'Taxon')}),
+            frozenset({('2', 1, 'Taxon'), ('2', 3, 'VernacularName')}) 
+        })
+
+        with DwCAReader(sample_data_path("dwca-2extensions.zip")) as dwca:
+            star_records = StarRecordIterator(dwca.extension_files + [dwca.core_file], how="outer")
+            stars = []
+            for star_record in star_records:
+                rows = []
+                for row in star_record:
+                    rows.append((row.id if isinstance(row, CoreRow) else row.core_id, row.position, row.rowtype.split('/')[-1]))
+                stars.append(frozenset(rows))
+
+            assert frozenset(stars) == expected_outer_join

dwca/version.py

@@ -1 +1 @@
-__version__ = '0.15.1'
+__version__ = '0.16.1'

{python_dwca_reader-0.15.1.dist-info → python_dwca_reader-0.16.1.dist-info}/AUTHORS

@@ -5,6 +5,7 @@ Here is an inevitably incomplete list of MUCH-APPRECIATED CONTRIBUTORS -- people
 bugs, improved documentation, ...  and generally made this library much better:
 
     Dimitri Brosens
+    Scott Brown
     Ben Cail
     Julien Cigar
     Pedro Correia de Siracusa

{python_dwca_reader-0.15.1.dist-info → python_dwca_reader-0.16.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-dwca-reader
-Version: 0.15.1
+Version: 0.16.1
 Summary: A simple Python package to read Darwin Core Archive (DwC-A) files.
 Home-page: https://github.com/BelgianBiodiversityPlatform/python-dwca-reader
 Author: Nicolas Noé - Belgian Biodiversity Platform
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 License-File: LICENSE.txt
 License-File: AUTHORS

python_dwca_reader-0.16.1.dist-info/RECORD

@@ -0,0 +1,28 @@
+dwca/__init__.py,sha256=Jzb_RfgvXCrm_SQ4AfeGVi1N36YybxnM5mpyxrnihgI,33
+dwca/descriptors.py,sha256=tjbS8ul_FMUP_frCBG4q635Ui3htEy7ffJEXN9CtSk0,13118
+dwca/exceptions.py,sha256=3TyIelM2rrnFPH97isxQGv0ICRLebT_bXMnXOhuIg30,380
+dwca/files.py,sha256=EgOTbzyyH5mCeILqOc30wL9FdjXklHgosf0JKtSZnBU,6535
+dwca/helpers.py,sha256=3o6L4yTqtVIsFEl9PMpDoyqITKXvS-ErTGVvjWIlFZA,245
+dwca/minibench.py,sha256=9_kXsT4bKjxgb5NtYBVgePp1yFK8KkH7h5okxh3XVBY,1539
+dwca/read.py,sha256=5ITKQYyvPzL7LUzefgT6YVThxYg_iFGsdG80zkeV09Q,22228
+dwca/rows.py,sha256=BW81_eY4beCEygGzlfhoKyi5RFy2KIRAaG1dME0rXx8,9111
+dwca/star_record.py,sha256=swNQIv5jeBRDxKzW_t5FVJ9LdvaSNINF17sWESisx2k,2763
+dwca/vendor.py,sha256=Qcg1I0qh4Q-nZ_n5zui7loDOq59QsxFHKa2ywmaRcpA,98
+dwca/version.py,sha256=_5Udfe3jX_Ju2NuNb0dUgUDu7GADMpIFGMOs0TQl5TI,23
+dwca/darwincore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dwca/darwincore/build_dc_terms_list.py,sha256=F-MT5UAJYtDnmz0-SbD7rInypQak-u56hoCSXICaBjQ,1464
+dwca/darwincore/terms.py,sha256=SnNnWHzpe99FndzByVzTt8xrOOi43MYvEMZunoUL95Q,8093
+dwca/darwincore/utils.py,sha256=7fA2mI1Pghz_zW5FP9D8e38iZ2AK2XrrqdxPBCrd8eE,845
+dwca/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dwca/test/helpers.py,sha256=_ymEel-HudB9g8DhwsKqVXh_KnXFiYaHaDV1u9bjBvk,156
+dwca/test/test_datafile.py,sha256=LKYk_jZavahR5tZmA9v3tabHnw8Z0HgBPqOnDVLsIXE,7073
+dwca/test/test_descriptors.py,sha256=t8M8buEHAbEwjbiAsEuZgIiF1kSpidzbqG9hti-KipA,25888
+dwca/test/test_dwcareader.py,sha256=0EiSUYNnjk0pRGLBY4XICbBeEbiuutwR-rDW3DkwCcY,43642
+dwca/test/test_rows.py,sha256=J125mYdVu9zxkauOzaV2DLgSldLBPJwKNDkMuPwWuRc,1637
+dwca/test/test_star_record.py,sha256=Y9nm1WlN4JB6xcmEviyPEQPrvUeP5O4wpbbWI1BAVGQ,2829
+python_dwca_reader-0.16.1.dist-info/AUTHORS,sha256=5qn6Kxn1rBj2J7M4LHSwQiQHhVOs5ejilGssEmEhct4,528
+python_dwca_reader-0.16.1.dist-info/LICENSE.txt,sha256=2Rc1BeoBQjG30Abb8zL34Wd9MrxVBRftyXiltvK2hJg,1506
+python_dwca_reader-0.16.1.dist-info/METADATA,sha256=YOHyFkTEiNE5WDk3tgCHnYeLQbvTj3rt5RSToFk1_8M,1551
+python_dwca_reader-0.16.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+python_dwca_reader-0.16.1.dist-info/top_level.txt,sha256=wUMEHbrB4ck4vyfybf4X2bv-HLgIU8uen_C5bMw8P1E,5
+python_dwca_reader-0.16.1.dist-info/RECORD,,

{python_dwca_reader-0.15.1.dist-info → python_dwca_reader-0.16.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: bdist_wheel (0.41.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

python_dwca_reader-0.15.1.dist-info/RECORD

@@ -1,26 +0,0 @@
-dwca/__init__.py,sha256=Jzb_RfgvXCrm_SQ4AfeGVi1N36YybxnM5mpyxrnihgI,33
-dwca/descriptors.py,sha256=cBSrl4LgyclUbRLfWNXtcKnCpIL0j_f5qgLQe8TSleI,13119
-dwca/exceptions.py,sha256=91dAQNReolSWU3FnHWWZULFh5a9TLiN9OeT092vPwZk,381
-dwca/files.py,sha256=uehcELqd9go3oTuuS078nDN0FHu3g1vidYEuS-oglTw,6662
-dwca/helpers.py,sha256=yOl74SlrjyKlj7swES-iwOpYXT-5fvLeZGDYN--vcM4,250
-dwca/minibench.py,sha256=9_kXsT4bKjxgb5NtYBVgePp1yFK8KkH7h5okxh3XVBY,1539
-dwca/read.py,sha256=_P0s4iIO5CzINZKO4CKAS6JS-IAW2bV67040ZywcKE8,21269
-dwca/rows.py,sha256=BW81_eY4beCEygGzlfhoKyi5RFy2KIRAaG1dME0rXx8,9111
-dwca/vendor.py,sha256=Qcg1I0qh4Q-nZ_n5zui7loDOq59QsxFHKa2ywmaRcpA,98
-dwca/version.py,sha256=8WhJAZouJDJEIY8zYgYIOv2VtMy_b_0q-sscUgsm7U0,23
-dwca/darwincore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dwca/darwincore/build_dc_terms_list.py,sha256=F-MT5UAJYtDnmz0-SbD7rInypQak-u56hoCSXICaBjQ,1464
-dwca/darwincore/terms.py,sha256=SnNnWHzpe99FndzByVzTt8xrOOi43MYvEMZunoUL95Q,8093
-dwca/darwincore/utils.py,sha256=7fA2mI1Pghz_zW5FP9D8e38iZ2AK2XrrqdxPBCrd8eE,845
-dwca/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dwca/test/helpers.py,sha256=_ymEel-HudB9g8DhwsKqVXh_KnXFiYaHaDV1u9bjBvk,156
-dwca/test/test_datafile.py,sha256=FzOE-_HaiAEzK-dU5b5fVjFzYenkEnKNnxdlAKZSu20,6859
-dwca/test/test_descriptors.py,sha256=t8M8buEHAbEwjbiAsEuZgIiF1kSpidzbqG9hti-KipA,25888
-dwca/test/test_dwcareader.py,sha256=OeQMX5oYuqCR-BKcMQE1wXQVIN0sTlXOI99OqC9DvV8,42766
-dwca/test/test_rows.py,sha256=J125mYdVu9zxkauOzaV2DLgSldLBPJwKNDkMuPwWuRc,1637
-python_dwca_reader-0.15.1.dist-info/AUTHORS,sha256=YnTWgnQixoNypqkXEsqPGiMJ2tfbaapCNvu4ja-A2Tg,512
-python_dwca_reader-0.15.1.dist-info/LICENSE.txt,sha256=2Rc1BeoBQjG30Abb8zL34Wd9MrxVBRftyXiltvK2hJg,1506
-python_dwca_reader-0.15.1.dist-info/METADATA,sha256=LE142udk9M_L1Qddka8T1zGKi8C15dr8Semir9B8bFY,1500
-python_dwca_reader-0.15.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-python_dwca_reader-0.15.1.dist-info/top_level.txt,sha256=wUMEHbrB4ck4vyfybf4X2bv-HLgIU8uen_C5bMw8P1E,5
-python_dwca_reader-0.15.1.dist-info/RECORD,,