python-dwca-reader 0.16.3__py3-none-any.whl → 0.17.0__py3-none-any.whl

dwca/descriptors.py

@@ -12,6 +12,7 @@ import csv
 import io
 import os
 import re
+from operator import itemgetter
 import xml.etree.ElementTree as ET
 from typing import Optional, List, Dict, Set
 from xml.etree.ElementTree import Element
@@ -210,6 +211,66 @@ class DataFileDescriptor(object):
             fields_terminated_by=fields_terminated_by,
         )
 
+    def __key(self):
+        """Return a tuple describing the data file layout. Common ground between equality and hash.
+
+        Deliberately excludes `raw_element`: it's an xml.etree.ElementTree.Element, which
+        compares by identity, so including it would make two descriptors built from the same
+        metafile section never compare equal. `lines_to_ignore` stands in for the only part of
+        `raw_element` that affects parsing (the ignoreHeaderLines attribute).
+
+        `created_from_file` is excluded too: it records where the descriptor came from, not how
+        the file is laid out, and its only behavioral effect is already covered by
+        `lines_to_ignore`. `represents_extension` is excluded as it's just the negation of
+        `represents_corefile`.
+        """
+        return (
+            self.file_location,
+            self.file_encoding,
+            self.type,
+            self.id_index,
+            self.coreid_index,
+            self.represents_corefile,
+            self.fields,
+            self.lines_terminated_by,
+            self.fields_enclosed_by,
+            self.fields_terminated_by,
+            self.lines_to_ignore,
+        )
+
+    def __eq__(self, other):
+        if not isinstance(other, DataFileDescriptor):
+            return NotImplemented
+
+        return self.__key() == other.__key()
+
+    def __hash__(self):
+        # __key() embeds `fields`, a list of dicts, which isn't hashable. Equal descriptors
+        # still hash equally because this is a subset of the equality key.
+        return hash(
+            (
+                self.file_location,
+                self.file_encoding,
+                self.type,
+                self.id_index,
+                self.coreid_index,
+                self.represents_corefile,
+                self.lines_terminated_by,
+                self.fields_enclosed_by,
+                self.fields_terminated_by,
+                self.lines_to_ignore,
+            )
+        )
+
+    @property
+    def field_plan(self) -> "FieldPlan":
+        """A cached :class:`FieldPlan` turning a split data row into a term -> value dict."""
+        plan = self.__dict__.get("_field_plan")
+        if plan is None:
+            plan = FieldPlan(self.fields, self.id_index, self.coreid_index)
+            self.__dict__["_field_plan"] = plan
+        return plan
+
     @property
     def terms(self) -> Set[str]:
         """Return a Python set containing all the Darwin Core terms appearing in file."""
@@ -229,9 +290,9 @@ class DataFileDescriptor(object):
         columns = {}
 
         for f in self.fields:
-            if f[
-                "index"
-            ]:  # Some (default values for example) don't have a corresponding col.
+            # Some fields (those carrying only a default value) have no column. Note the
+            # explicit None test: index 0 is a valid column and must not be dropped.
+            if f["index"] is not None:
                 columns[f["index"]] = f["term"]
 
         # In addition to DwC terms, we may also have id (Core) or core_id (Extensions) columns
@@ -264,6 +325,181 @@ class DataFileDescriptor(object):
         return int(self.raw_element.get("ignoreHeaderLines", 0))
 
 
+class FieldPlan(object):
+    """Precomputed mapping from a split CSV line to a term -> value dict.
+
+    Built once per :class:`DataFileDescriptor` (see its ``field_plan`` property) so that
+    parsing a row costs one C-level ``dict(zip(...))`` instead of a Python loop over the
+    field descriptors.
+    """
+
+    __slots__ = (
+        "_fields",
+        "_id_index",
+        "_coreid_index",
+        "_contiguous_terms",
+        "_terms",
+        "_getter",
+        "_single_column",
+        "_defaults",
+        "_constants",
+        "_ordered",
+        "required_columns",
+    )
+
+    def __init__(self, fields, id_index=None, coreid_index=None):
+        self._fields = fields
+        self._id_index = id_index
+        self._coreid_index = coreid_index
+
+        indexed = [f for f in fields if f["index"] is not None]
+        indexes = [f["index"] for f in indexed]
+
+        #: Number of columns a data row must have for this plan to apply.
+        self.required_columns = max(indexes) + 1 if indexes else 0
+
+        # Terms without a column: the value is always the default.
+        self._constants = tuple(
+            (f["term"], f["default"] or "") for f in fields if f["index"] is None
+        )
+        # Indexed terms that also carry a default, used when the cell is empty (issue #80).
+        self._defaults = tuple(
+            (f["term"], f["default"]) for f in indexed if f["default"]
+        )
+
+        # When some terms have no column, the fast paths below would append them after the
+        # mapped ones and change the key order of Row.data (which is user-visible through
+        # str(row)). Those archives use an ordered path instead; they are rare, and the
+        # ordered path is still free of the per-field try/except and int() it replaces.
+        self._ordered = (
+            tuple((f["term"], f["index"], f["default"]) for f in fields)
+            if self._constants
+            else None
+        )
+
+        if indexes and sorted(indexes) == list(range(len(indexes))):
+            # Fast path: the indexed fields cover columns 0..n-1 exactly once.
+            ordered = sorted(indexed, key=lambda f: f["index"])
+            self._contiguous_terms = tuple(f["term"] for f in ordered)
+            self._terms = None
+            self._getter = None
+            self._single_column = False
+        else:
+            self._contiguous_terms = None
+            self._terms = tuple(f["term"] for f in indexed)
+            self._getter = itemgetter(*indexes) if indexes else None
+            # itemgetter with a single argument returns a scalar, not a tuple.
+            self._single_column = len(indexes) == 1
+
+    def build_data(self, raw_fields):
+        """Return the term -> value dict for an already-split data row."""
+        if len(raw_fields) < self.required_columns:
+            self._raise_missing_column(raw_fields)
+
+        if self._ordered is not None:
+            data = {}
+            for term, index, default in self._ordered:
+                value = raw_fields[index] if index is not None else None
+                data[term] = value or default or ""
+            return data
+
+        if self._contiguous_terms is not None:
+            data = dict(zip(self._contiguous_terms, raw_fields))
+        elif self._getter is not None:
+            values = self._getter(raw_fields)
+            if self._single_column:
+                values = (values,)
+            data = dict(zip(self._terms, values))
+        else:
+            data = {}
+
+        for term, default in self._defaults:
+            if not data[term]:
+                data[term] = default
+        for term, constant in self._constants:
+            data[term] = constant
+
+        return data
+
+    def term_getter(self, terms):
+        """Return a callable mapping a split data row to a tuple of values for `terms`.
+
+        :raises ValueError: if any requested term is absent from the data file.
+        """
+        # `terms` is iterated three times below (missing, indexes, defaults). A generator or
+        # other one-shot iterable would be exhausted after the first pass, silently turning
+        # every later pass empty rather than raising - so normalise to a list once up front.
+        terms = list(terms)
+
+        by_term = {f["term"]: f for f in self._fields}
+
+        # "id" and "coreid" name the archive's key columns. These are the names this library
+        # already uses for them in headers(), short_headers() and pd_read(). A declared term of
+        # the same name wins, which matters for metafile-less archives whose terms are raw CSV
+        # header names, and keeps this consistent with row.data.
+        if "id" not in by_term and self._id_index is not None:
+            by_term["id"] = {"term": "id", "index": self._id_index, "default": None}
+        if "coreid" not in by_term and self._coreid_index is not None:
+            by_term["coreid"] = {"term": "coreid", "index": self._coreid_index, "default": None}
+
+        missing = [term for term in terms if term not in by_term]
+        if missing:
+            raise ValueError(
+                "These terms are not in this data file: {t}".format(
+                    t=", ".join(sorted(missing))
+                )
+            )
+
+        indexes = tuple(by_term[term]["index"] for term in terms)
+        defaults = tuple(by_term[term]["default"] for term in terms)
+
+        if indexes and all(i is not None for i in indexes) and not any(defaults):
+            # Fast path: every term maps to a column and none has a default, so the whole
+            # tuple comes out of a single C-level call.
+            getter = itemgetter(*indexes)
+            required = max(indexes) + 1
+
+            if len(indexes) == 1:
+                # itemgetter with a single argument returns a scalar, not a tuple.
+                def get_one(raw_fields):
+                    if len(raw_fields) < required:
+                        self._raise_missing_column(raw_fields)
+                    return (getter(raw_fields),)
+
+                return get_one
+
+            def get_many(raw_fields):
+                if len(raw_fields) < required:
+                    self._raise_missing_column(raw_fields)
+                return getter(raw_fields)
+
+            return get_many
+
+        required = max([i for i in indexes if i is not None] or [-1]) + 1
+
+        def get_general(raw_fields):
+            if len(raw_fields) < required:
+                self._raise_missing_column(raw_fields)
+            return tuple(
+                (raw_fields[index] if index is not None else None) or default or ""
+                for index, default in zip(indexes, defaults)
+            )
+
+        return get_general
+
+    def _raise_missing_column(self, raw_fields):
+        # Slow path: report the same index the old per-field loop would have reported.
+        for field in self._fields:
+            index = field["index"]
+            if index is not None and index >= len(raw_fields):
+                raise InvalidArchive(
+                    "The descriptor references a non-existent field (index={i})".format(
+                        i=index
+                    )
+                )
+        raise InvalidArchive("The descriptor references a non-existent field")
+
+
 class ArchiveDescriptor(object):
     """Class used to encapsulate the whole Metafile (`meta.xml`)."""