python-dwca-reader 0.16.4__py3-none-any.whl → 0.17.1__py3-none-any.whl

dwca/rows.py

@@ -1,11 +1,9 @@
 """Objects that represents data rows coming from DarwinCore Archives."""
 
 import csv
-import sys
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 from dwca.descriptors import DataFileDescriptor
-from dwca.exceptions import InvalidArchive
 
 
 class Row(object):
@@ -14,6 +12,8 @@ class Row(object):
     This class is intended to be subclassed rather than used directly.
     """
 
+    __slots__ = ("descriptor", "position", "rowtype", "raw_fields", "data")
+
     # Common ground for __str__ between subclasses
     def _build_str(self, source_str, id_str):
         txt = (
@@ -52,33 +52,50 @@ class Row(object):
     def __init__(
         self, csv_line: str, position: int, datafile_descriptor: DataFileDescriptor
     ) -> None:
-        #: An instance of :class:`dwca.descriptors.DataFileDescriptor` describing the originating
-        #: data file.
+        self._populate(
+            csv_line_to_fields(
+                csv_line,
+                line_ending=datafile_descriptor.lines_terminated_by,
+                field_ending=datafile_descriptor.fields_terminated_by,
+                fields_enclosed_by=datafile_descriptor.fields_enclosed_by,
+            ),
+            position,
+            datafile_descriptor,
+        )
+
+    @classmethod
+    def from_fields(
+        cls,
+        raw_fields: List[str],
+        position: int,
+        datafile_descriptor: DataFileDescriptor,
+    ) -> "Row":
+        """Build a Row from an already-split data row.
+
+        This is the constructor used by the streaming engine. :meth:`__init__`, which takes a
+        raw CSV line, is kept for backwards compatibility.
+        """
+        row = cls.__new__(cls)
+        row._populate(raw_fields, position, datafile_descriptor)
+        return row
+
+    def _populate(self, raw_fields, position, datafile_descriptor) -> None:
+        #: An instance of :class:`dwca.descriptors.DataFileDescriptor` describing the
+        #: originating data file.
         self.descriptor = datafile_descriptor  # type: DataFileDescriptor
 
-        #: The row position/index (starting at 0) in the source data file. This can be used, for example with
-        #: :meth:`dwca.read.DwCAReader.get_corerow_by_position` or :meth:`dwca.files.CSVDataFile.get_row_by_position`.
+        #: The row position/index (starting at 0) in the source data file. This can be used,
+        #: for example with :meth:`dwca.read.DwCAReader.get_corerow_by_position` or
+        #: :meth:`dwca.files.CSVDataFile.get_row_by_position`.
         self.position = position  # type: int
 
-        #: The csv line type as stated in the archive descriptor.
-        #: (or None if the archive has no descriptor). Examples:
-        #: http://rs.tdwg.org/dwc/terms/Occurrence,
+        #: The csv line type as stated in the archive descriptor (or None if the archive has
+        #: no descriptor). Examples: http://rs.tdwg.org/dwc/terms/Occurrence,
         #: http://rs.gbif.org/terms/1.0/VernacularName, ...
         self.rowtype = self.descriptor.type  # type: Optional[str]
 
-        # self.raw_fields is a list of the csv_line's content
-        #:
-        self.raw_fields = csv_line_to_fields(
-            csv_line,
-            line_ending=self.descriptor.lines_terminated_by,
-            field_ending=self.descriptor.fields_terminated_by,
-            fields_enclosed_by=self.descriptor.fields_enclosed_by,
-        )
-
-        # TODO: raw_fields is a new property: to test
-
-        # TODO: Consistency check ?? self.raw_fields length should be :
-        # num of self.raw_fields described in core_meta + 2 (id and \n)
+        #: A list of the row's raw (unmapped) field values.
+        self.raw_fields = raw_fields
 
         #: A dict containing the Row data, such as::
         #:
@@ -90,29 +107,11 @@ class Row(object):
         #:
         #:      myrow.data['http://rs.tdwg.org/dwc/terms/locality']  # => "Brussels"
         #:
-        #: .. note:: The :func:`dwca.darwincore.utils.qualname` helper is available to make such calls less verbose.
-        self.data = {}  # type: Dict[str, str]
-
-        for field_descriptor in self.descriptor.fields:
-            try:
-                column_index = int(field_descriptor["index"])
-                field_row_value = self.raw_fields[column_index]
-            except TypeError:
-                # int() argument must be a string... We don't have an index for this field
-                field_row_value = None
-            except IndexError:
-                msg = (
-                    "The descriptor references a non-existent field (index={i})".format(
-                        i=column_index
-                    )
-                )
-                raise InvalidArchive(msg)
-
-            field_default_value = field_descriptor["default"]
-
-            self.data[field_descriptor["term"]] = (
-                field_row_value or field_default_value or ""
-            )
+        #: .. note:: The :func:`dwca.darwincore.utils.qualname` helper is available to make
+        #:    such calls less verbose.
+        self.data = datafile_descriptor.field_plan.build_data(
+            raw_fields
+        )  # type: Dict[str, str]
 
 
 class CoreRow(Row):
@@ -127,14 +126,14 @@ class CoreRow(Row):
         id_str = "Row id: " + str(self.id)
         return super(CoreRow, self)._build_str("Core file", id_str)
 
-    def __init__(
-        self, csv_line: str, position: int, datafile_descriptor: DataFileDescriptor
-    ) -> None:
-        super(CoreRow, self).__init__(csv_line, position, datafile_descriptor)
+    __slots__ = ("id", "source_metadata", "extension_data_files", "_extensions")
 
-        if self.descriptor.id_index is not None:
+    def _populate(self, raw_fields, position, datafile_descriptor) -> None:
+        super(CoreRow, self)._populate(raw_fields, position, datafile_descriptor)
+
+        if datafile_descriptor.id_index is not None:
             #: The row id
-            self.id = self.raw_fields[self.descriptor.id_index]
+            self.id = raw_fields[datafile_descriptor.id_index]
         else:
             self.id = None
 
@@ -183,25 +182,42 @@ class CoreRow(Row):
     # Should these 3 be factorized ? How ? Mixin ? Parent class ?
     def __key(self):
         """Return a tuple representing the row. Common ground between equality and hash."""
+        # A CoreRow obtained without going through DwCAReader iteration (e.g.
+        # CSVDataFile.get_row_by_position() called directly) never had link_extension_files()
+        # / link_source_metadata() called on it, so self.extensions and self.source_metadata
+        # don't exist. Fall back to None for those rows instead of letting the attribute
+        # access raise AttributeError. This doesn't change equality for linked rows (both
+        # attributes are always present after DwCAReader.next() links them).
+        extensions = self.extensions if hasattr(self, "extension_data_files") else None
+        source_metadata = (
+            self.source_metadata if hasattr(self, "source_metadata") else None
+        )
+
         return (
             self.descriptor,
             self.id,
             self.data,
-            self.extensions,
-            self.source_metadata,
+            extensions,
+            source_metadata,
             self.rowtype,
             self.raw_fields,
             self.position,
         )
 
     def __eq__(self, other):
-        return self.__key() == other.__key()
+        # The isinstance test is required, not just defensive: __key is name-mangled, so
+        # other.__key() means other._CoreRow__key(), which an ExtensionRow (or any non-row)
+        # doesn't have. Without it, comparing to anything else raises AttributeError.
+        if not isinstance(other, CoreRow):
+            return NotImplemented
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
+        return self.__key() == other.__key()
 
     def __hash__(self):
-        return hash(self.__key())
+        # __key() embeds the data dict, the raw field list and the lazily-loaded extensions,
+        # none of which are hashable. Equal rows still hash equally because this is a subset
+        # of the equality key.
+        return hash((self.descriptor, self.id, self.rowtype, self.position))
 
 
 class ExtensionRow(Row):
@@ -215,13 +231,13 @@ class ExtensionRow(Row):
         id_str = "Core row id: " + str(self.core_id)
         return super(ExtensionRow, self)._build_str("Extension file", id_str)
 
-    def __init__(
-        self, csv_line: str, position: int, datafile_descriptor: DataFileDescriptor
-    ) -> None:
-        super(ExtensionRow, self).__init__(csv_line, position, datafile_descriptor)
+    __slots__ = ("core_id",)
+
+    def _populate(self, raw_fields, position, datafile_descriptor) -> None:
+        super(ExtensionRow, self)._populate(raw_fields, position, datafile_descriptor)
 
         #: The id of the core row this extension row is referring to.
-        self.core_id = self.raw_fields[datafile_descriptor.coreid_index]
+        self.core_id = raw_fields[datafile_descriptor.coreid_index]
 
     def __key(self):
         """Return a tuple representing the row. Common ground between equality and hash."""
@@ -235,13 +251,15 @@ class ExtensionRow(Row):
         )
 
     def __eq__(self, other):
-        return self.__key() == other.__key()
+        # See the note on CoreRow.__eq__: __key is name-mangled, so this test is what keeps
+        # comparisons against a CoreRow (or anything else) from raising AttributeError.
+        if not isinstance(other, ExtensionRow):
+            return NotImplemented
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
+        return self.__key() == other.__key()
 
     def __hash__(self):
-        return hash(self.__key())
+        return hash((self.descriptor, self.core_id, self.rowtype, self.position))
 
 
 def csv_line_to_fields(csv_line, line_ending, field_ending, fields_enclosed_by):
@@ -249,16 +267,25 @@ def csv_line_to_fields(csv_line, line_ending, field_ending, fields_enclosed_by):
 
     Return a list of fields. Content is not trimmed.
     """
-    csv_line = csv_line.rstrip(line_ending)
-    raw_fields = []
+    # A carriage return is stripped alongside the declared terminator: archives routinely
+    # declare "\n" while the file itself has CRLF line endings. This matches both the csv
+    # module's own behavior and CSVDataFile._iter_field_lists, so the two access paths agree.
+    csv_line = csv_line.rstrip(line_ending + "\r")
 
     if fields_enclosed_by == "":
-        opts = {"quoting": csv.QUOTE_NONE}
-    else:
-        opts = {"quoting": csv.QUOTE_ALL, "quotechar": fields_enclosed_by}
-
-    for row in csv.reader([csv_line], delimiter=field_ending, **opts):
-        for f in row:
-            field = f.strip(fields_enclosed_by)
-            raw_fields.append(field)
-    return raw_fields
+        # No enclosure: the line is simply split on the separator. This also keeps any
+        # quote character that happens to appear in the content.
+        return csv_line.split(field_ending)
+
+    # The csv module unwraps the enclosure and un-doubles escaped quote characters itself.
+    # Stripping the quote character afterwards would eat a legitimate leading or trailing
+    # one from the field's own content.
+    for row in csv.reader(
+        [csv_line],
+        delimiter=field_ending,
+        quotechar=fields_enclosed_by,
+        quoting=csv.QUOTE_MINIMAL,
+    ):
+        return row
+
+    return []

dwca/star_record.py

@@ -1,13 +1,12 @@
 from dwca.files import CSVDataFile
-from typing import List
-from typing_extensions import Literal
+from typing import List, Literal
 import itertools
 
 
 class StarRecordIterator(object):
     """Object used to iterate over multiple DWCA-files joined on the coreid
 
-    :param files_to_join: a list of the `dwca.files.CSVDataFile`s we'd like to join.
+    :param files_to_join: a list of the :class:`dwca.files.CSVDataFile` objects to join.
         May or may not include the core file (the core is not treated in a special way)
     :param how: indicates the type of join.  "inner" and "outer" correspond vaguely to
         inner and full joins.  The outer join includes rows that don't match on all files,

dwca/test/archive_builder.py

@@ -0,0 +1,188 @@
+"""Build synthetic Darwin Core Archives for tests.
+
+Produces a directory-based archive (DwCAReader reads directories directly, so nothing
+needs zipping). Every parameter that the Metafile can express is explicit, which is what
+lets the characterization tests cover encodings, terminators, quoting and header counts
+without committing more binary sample files.
+"""
+
+import os
+import shutil
+import tempfile
+from xml.sax.saxutils import quoteattr
+
+TERM_PREFIX = "http://rs.tdwg.org/dwc/terms/"
+
+
+def temp_archive_dir(test_case):
+    """Return a temporary directory that is removed when `test_case` finishes.
+
+    Several tests in the suite count the entries in the system temporary directory, so
+    leaking one per test would eventually make them flaky.
+    """
+    directory = tempfile.mkdtemp()
+    test_case.addCleanup(shutil.rmtree, directory, ignore_errors=True)
+    return directory
+
+METAFILE_TEMPLATE = """<archive xmlns="http://rs.tdwg.org/dwc/text/">
+  <core encoding={encoding} fieldsTerminatedBy={fields_terminated_by} \
+linesTerminatedBy={lines_terminated_by} fieldsEnclosedBy={fields_enclosed_by} \
+ignoreHeaderLines={ignore_header_lines} rowType="http://rs.tdwg.org/dwc/terms/Occurrence">
+    <files><location>occurrence.txt</location></files>
+    <id index="{id_index}" />
+{fields}
+  </core>
+{extension}
+</archive>
+"""
+
+EXTENSION_TEMPLATE = """  <extension encoding={encoding} fieldsTerminatedBy={fields_terminated_by} \
+linesTerminatedBy={lines_terminated_by} fieldsEnclosedBy={fields_enclosed_by} \
+ignoreHeaderLines={ignore_header_lines} rowType="http://rs.gbif.org/terms/1.0/VernacularName">
+    <files><location>extension.txt</location></files>
+    <coreid index="0" />
+{fields}
+  </extension>"""
+
+
+def _escape(raw):
+    # The Metafile stores separators escaped, e.g. the tab character as the two
+    # characters backslash-t. Mirror what real archives contain.
+    escaped = (
+        raw.replace("\\", "\\\\")
+        .replace("\t", "\\t")
+        .replace("\n", "\\n")
+        .replace("\r", "\\r")
+    )
+    return quoteattr(escaped)
+
+
+def _field_tags(terms, defaults, indent="    "):
+    lines = []
+    for index, term in enumerate(terms):
+        default = (defaults or {}).get(term)
+        if default is None:
+            lines.append(
+                '{i}<field index="{n}" term="{t}"/>'.format(i=indent, n=index, t=term)
+            )
+        else:
+            lines.append(
+                '{i}<field index="{n}" term="{t}" default={d}/>'.format(
+                    i=indent, n=index, t=term, d=quoteattr(default)
+                )
+            )
+    for term, default in (defaults or {}).items():
+        if term not in terms:
+            lines.append(
+                '{i}<field term="{t}" default={d}/>'.format(
+                    i=indent, t=term, d=quoteattr(default)
+                )
+            )
+    return "\n".join(lines)
+
+
+def _render_rows(rows, fields_terminated_by, lines_terminated_by, fields_enclosed_by):
+    out = []
+    for row in rows:
+        if fields_enclosed_by:
+            cells = [
+                fields_enclosed_by
+                + cell.replace(fields_enclosed_by, fields_enclosed_by * 2)
+                + fields_enclosed_by
+                for cell in row
+            ]
+        else:
+            cells = list(row)
+        out.append(fields_terminated_by.join(cells))
+    return lines_terminated_by.join(out) + (lines_terminated_by if out else "")
+
+
+def build_archive(
+    directory,
+    rows,
+    columns=None,
+    encoding="utf-8",
+    lines_terminated_by="\n",
+    fields_terminated_by="\t",
+    fields_enclosed_by="",
+    ignore_header_lines=0,
+    header_rows=(),
+    id_index=0,
+    terms=None,
+    defaults=None,
+    extension=None,
+    trailing_newline=True,
+    raw_payload=None,
+):
+    """Write a Darwin Core Archive into `directory` and return its path.
+
+    :param rows: list of lists of str, the data rows.
+    :param terms: list of full term URIs, one per column. Defaults to term0, term1, ...
+    :param defaults: dict term -> default value. A term not present in `terms` becomes a
+        default-only field (no index attribute), as the standard allows.
+    :param extension: list of rows for an extension data file, or None. Column 0 is the coreid.
+    :param raw_payload: bytes written verbatim as the core data file instead of `rows`.
+        Used to build inputs no well-formed writer would produce (undecodable bytes, ragged
+        rows, blank lines).
+    """
+    if columns is None:
+        columns = max([len(r) for r in rows] + [1]) if rows else 1
+    if terms is None:
+        terms = [TERM_PREFIX + "term" + str(i) for i in range(columns)]
+
+    metafile = METAFILE_TEMPLATE.format(
+        encoding=quoteattr(encoding),
+        fields_terminated_by=_escape(fields_terminated_by),
+        lines_terminated_by=_escape(lines_terminated_by),
+        fields_enclosed_by=_escape(fields_enclosed_by),
+        ignore_header_lines=quoteattr(str(ignore_header_lines)),
+        id_index=id_index,
+        fields=_field_tags(terms, defaults),
+        extension=(
+            ""
+            if extension is None
+            else EXTENSION_TEMPLATE.format(
+                encoding=quoteattr(encoding),
+                fields_terminated_by=_escape(fields_terminated_by),
+                lines_terminated_by=_escape(lines_terminated_by),
+                fields_enclosed_by=_escape(fields_enclosed_by),
+                ignore_header_lines=quoteattr(str(ignore_header_lines)),
+                fields=_field_tags(
+                    [TERM_PREFIX + "vernacularName"], None, indent="    "
+                ).replace('index="0"', 'index="1"'),
+            )
+        ),
+    )
+
+    with open(os.path.join(directory, "meta.xml"), "w", encoding="utf-8") as f:
+        f.write(metafile)
+
+    data_path = os.path.join(directory, "occurrence.txt")
+    if raw_payload is not None:
+        with open(data_path, "wb") as f:
+            f.write(raw_payload)
+    else:
+        payload = _render_rows(
+            list(header_rows) + list(rows),
+            fields_terminated_by,
+            lines_terminated_by,
+            fields_enclosed_by,
+        )
+        if not trailing_newline and payload.endswith(lines_terminated_by):
+            payload = payload[: -len(lines_terminated_by)]
+        with open(data_path, "w", encoding=encoding, newline="") as f:
+            f.write(payload)
+
+    if extension is not None:
+        payload = _render_rows(
+            list(header_rows) + list(extension),
+            fields_terminated_by,
+            lines_terminated_by,
+            fields_enclosed_by,
+        )
+        with open(
+            os.path.join(directory, "extension.txt"), "w", encoding=encoding, newline=""
+        ) as f:
+            f.write(payload)
+
+    return directory