python-dwca-reader 0.16.2__py3-none-any.whl → 0.16.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dwca/darwincore/build_dc_terms_list.py +8 -6
- dwca/darwincore/terms.py +171 -1
- dwca/darwincore/utils.py +1 -1
- dwca/descriptors.py +104 -81
- dwca/files.py +5 -5
- dwca/minibench.py +8 -6
- dwca/read.py +81 -39
- dwca/rows.py +85 -47
- dwca/star_record.py +27 -19
- dwca/test/helpers.py +1 -1
- dwca/test/test_datafile.py +8 -3
- dwca/test/test_descriptors.py +144 -90
- dwca/test/test_dwcareader.py +95 -43
- dwca/test/test_rows.py +8 -4
- dwca/test/test_star_record.py +117 -27
- dwca/vendor.py +1 -0
- dwca/version.py +1 -1
- {python_dwca_reader-0.16.2.dist-info → python_dwca_reader-0.16.3.dist-info}/METADATA +1 -1
- python_dwca_reader-0.16.3.dist-info/RECORD +28 -0
- python_dwca_reader-0.16.2.dist-info/RECORD +0 -28
- {python_dwca_reader-0.16.2.dist-info → python_dwca_reader-0.16.3.dist-info}/AUTHORS +0 -0
- {python_dwca_reader-0.16.2.dist-info → python_dwca_reader-0.16.3.dist-info}/LICENSE.txt +0 -0
- {python_dwca_reader-0.16.2.dist-info → python_dwca_reader-0.16.3.dist-info}/WHEEL +0 -0
- {python_dwca_reader-0.16.2.dist-info → python_dwca_reader-0.16.3.dist-info}/top_level.txt +0 -0
|
@@ -11,12 +11,14 @@
|
|
|
11
11
|
import argparse
|
|
12
12
|
import xml.etree.ElementTree as ET
|
|
13
13
|
|
|
14
|
-
parser = argparse.ArgumentParser(
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
parser = argparse.ArgumentParser(
|
|
15
|
+
description="Generate a list of qualnames "
|
|
16
|
+
"of Darwin Core terms from XML "
|
|
17
|
+
"description files."
|
|
18
|
+
)
|
|
17
19
|
|
|
18
20
|
# Required positional argument: one or more XML files to be read
|
|
19
|
-
parser.add_argument(
|
|
21
|
+
parser.add_argument("source_xml", nargs="+", type=argparse.FileType("r"))
|
|
20
22
|
parser.parse_args()
|
|
21
23
|
args = parser.parse_args()
|
|
22
24
|
|
|
@@ -26,11 +28,11 @@ qualnames = set()
|
|
|
26
28
|
for source_file in args.source_xml:
|
|
27
29
|
root = ET.parse(source_file).getroot()
|
|
28
30
|
# First, extract the RowType itself... (Occcurrence, Taxon, ...)
|
|
29
|
-
qualnames.add(root.get(
|
|
31
|
+
qualnames.add(root.get("rowType"))
|
|
30
32
|
|
|
31
33
|
# Store each qualname found in any tag to our set
|
|
32
34
|
for ch in root.iter():
|
|
33
|
-
qn = ch.get(
|
|
35
|
+
qn = ch.get("qualName")
|
|
34
36
|
if qn:
|
|
35
37
|
qualnames.add(qn)
|
|
36
38
|
|
dwca/darwincore/terms.py
CHANGED
|
@@ -1 +1,171 @@
|
|
|
1
|
-
TERMS = [
|
|
1
|
+
TERMS = [
|
|
2
|
+
"http://rs.tdwg.org/dwc/terms/occurrenceDetails",
|
|
3
|
+
"http://rs.tdwg.org/dwc/terms/highestBiostratigraphicZone",
|
|
4
|
+
"http://rs.tdwg.org/dwc/terms/associatedTaxa",
|
|
5
|
+
"http://rs.tdwg.org/dwc/terms/latestEpochOrHighestSeries",
|
|
6
|
+
"http://rs.tdwg.org/dwc/terms/informationWithheld",
|
|
7
|
+
"http://rs.tdwg.org/dwc/terms/minimumDistanceAboveSurfaceInMeters",
|
|
8
|
+
"http://rs.tdwg.org/dwc/terms/behavior",
|
|
9
|
+
"http://rs.tdwg.org/dwc/terms/verbatimLongitude",
|
|
10
|
+
"http://rs.tdwg.org/dwc/terms/countryCode",
|
|
11
|
+
"http://rs.tdwg.org/dwc/terms/associatedSequences",
|
|
12
|
+
"http://rs.tdwg.org/dwc/terms/habitat",
|
|
13
|
+
"http://rs.tdwg.org/dwc/terms/islandGroup",
|
|
14
|
+
"http://rs.tdwg.org/dwc/terms/nomenclaturalCode",
|
|
15
|
+
"http://purl.org/dc/terms/license",
|
|
16
|
+
"http://rs.tdwg.org/dwc/terms/earliestEonOrLowestEonothem",
|
|
17
|
+
"http://rs.tdwg.org/dwc/terms/year",
|
|
18
|
+
"http://rs.tdwg.org/dwc/terms/taxonID",
|
|
19
|
+
"http://rs.tdwg.org/dwc/terms/continent",
|
|
20
|
+
"http://rs.tdwg.org/dwc/terms/nameAccordingTo",
|
|
21
|
+
"http://rs.tdwg.org/dwc/terms/latestEraOrHighestErathem",
|
|
22
|
+
"http://rs.tdwg.org/dwc/terms/eventID",
|
|
23
|
+
"http://rs.tdwg.org/dwc/terms/nameAccordingToID",
|
|
24
|
+
"http://rs.tdwg.org/dwc/terms/decimalLongitude",
|
|
25
|
+
"http://rs.tdwg.org/dwc/terms/sampleSizeValue",
|
|
26
|
+
"http://rs.tdwg.org/dwc/terms/identifiedBy",
|
|
27
|
+
"http://rs.tdwg.org/dwc/terms/latestEonOrHighestEonothem",
|
|
28
|
+
"http://rs.tdwg.org/dwc/terms/catalogNumber",
|
|
29
|
+
"http://rs.tdwg.org/dwc/terms/eventRemarks",
|
|
30
|
+
"http://rs.tdwg.org/dwc/terms/taxonRank",
|
|
31
|
+
"http://rs.tdwg.org/dwc/terms/verbatimDepth",
|
|
32
|
+
"http://rs.tdwg.org/dwc/terms/originalNameUsageID",
|
|
33
|
+
"http://purl.org/dc/terms/rightsHolder",
|
|
34
|
+
"http://rs.tdwg.org/dwc/terms/lowestBiostratigraphicZone",
|
|
35
|
+
"http://purl.org/dc/terms/bibliographicCitation",
|
|
36
|
+
"http://rs.tdwg.org/dwc/terms/island",
|
|
37
|
+
"http://rs.tdwg.org/dwc/terms/associatedMedia",
|
|
38
|
+
"http://rs.tdwg.org/dwc/terms/group",
|
|
39
|
+
"http://rs.tdwg.org/dwc/terms/endDayOfYear",
|
|
40
|
+
"http://rs.tdwg.org/dwc/terms/individualCount",
|
|
41
|
+
"http://rs.tdwg.org/dwc/terms/occurrenceRemarks",
|
|
42
|
+
"http://rs.tdwg.org/dwc/terms/disposition",
|
|
43
|
+
"http://rs.tdwg.org/dwc/terms/collectionID",
|
|
44
|
+
"http://rs.tdwg.org/dwc/terms/sampleSizeUnit",
|
|
45
|
+
"http://rs.tdwg.org/dwc/terms/earliestPeriodOrLowestSystem",
|
|
46
|
+
"http://rs.tdwg.org/dwc/terms/county",
|
|
47
|
+
"http://rs.tdwg.org/dwc/terms/eventDate",
|
|
48
|
+
"http://rs.tdwg.org/dwc/terms/earliestEpochOrLowestSeries",
|
|
49
|
+
"http://rs.tdwg.org/dwc/terms/formation",
|
|
50
|
+
"http://rs.tdwg.org/dwc/terms/Taxon",
|
|
51
|
+
"http://purl.org/dc/terms/references",
|
|
52
|
+
"http://purl.org/dc/terms/language",
|
|
53
|
+
"http://rs.tdwg.org/dwc/terms/decimalLatitude",
|
|
54
|
+
"http://rs.tdwg.org/dwc/terms/startDayOfYear",
|
|
55
|
+
"http://rs.tdwg.org/dwc/terms/fieldNotes",
|
|
56
|
+
"http://rs.tdwg.org/dwc/terms/municipality",
|
|
57
|
+
"http://rs.tdwg.org/dwc/terms/namePublishedInYear",
|
|
58
|
+
"http://rs.tdwg.org/dwc/terms/associatedReferences",
|
|
59
|
+
"http://rs.tdwg.org/dwc/terms/georeferenceVerificationStatus",
|
|
60
|
+
"http://rs.tdwg.org/dwc/terms/datasetName",
|
|
61
|
+
"http://rs.tdwg.org/dwc/terms/locality",
|
|
62
|
+
"http://rs.tdwg.org/dwc/terms/identificationID",
|
|
63
|
+
"http://purl.org/dc/terms/type",
|
|
64
|
+
"http://rs.tdwg.org/dwc/terms/scientificNameID",
|
|
65
|
+
"http://rs.tdwg.org/dwc/terms/verbatimLatitude",
|
|
66
|
+
"http://rs.tdwg.org/dwc/terms/occurrenceID",
|
|
67
|
+
"http://rs.tdwg.org/dwc/terms/verbatimTaxonRank",
|
|
68
|
+
"http://rs.tdwg.org/dwc/terms/verbatimSRS",
|
|
69
|
+
"http://rs.tdwg.org/dwc/terms/order",
|
|
70
|
+
"http://rs.tdwg.org/dwc/terms/georeferenceProtocol",
|
|
71
|
+
"http://rs.tdwg.org/dwc/terms/locationAccordingTo",
|
|
72
|
+
"http://rs.tdwg.org/dwc/terms/originalNameUsage",
|
|
73
|
+
"http://rs.tdwg.org/dwc/terms/geodeticDatum",
|
|
74
|
+
"http://rs.tdwg.org/dwc/terms/Occurrence",
|
|
75
|
+
"http://rs.tdwg.org/dwc/terms/establishmentMeans",
|
|
76
|
+
"http://rs.tdwg.org/dwc/terms/parentNameUsage",
|
|
77
|
+
"http://rs.tdwg.org/dwc/terms/taxonRemarks",
|
|
78
|
+
"http://rs.tdwg.org/dwc/terms/footprintSRS",
|
|
79
|
+
"http://rs.tdwg.org/dwc/terms/collectionCode",
|
|
80
|
+
"http://rs.tdwg.org/dwc/terms/class",
|
|
81
|
+
"http://rs.tdwg.org/dwc/terms/family",
|
|
82
|
+
"http://rs.tdwg.org/dwc/terms/footprintWKT",
|
|
83
|
+
"http://rs.tdwg.org/dwc/terms/georeferenceRemarks",
|
|
84
|
+
"http://rs.tdwg.org/dwc/terms/lithostratigraphicTerms",
|
|
85
|
+
"http://rs.tdwg.org/dwc/terms/geologicalContextID",
|
|
86
|
+
"http://rs.tdwg.org/dwc/terms/dateIdentified",
|
|
87
|
+
"http://rs.tdwg.org/dwc/terms/higherGeographyID",
|
|
88
|
+
"http://rs.tdwg.org/dwc/terms/pointRadiusSpatialFit",
|
|
89
|
+
"http://rs.tdwg.org/dwc/terms/dynamicProperties",
|
|
90
|
+
"http://rs.tdwg.org/dwc/terms/verbatimLocality",
|
|
91
|
+
"http://purl.org/dc/terms/modified",
|
|
92
|
+
"http://rs.tdwg.org/dwc/terms/month",
|
|
93
|
+
"http://rs.tdwg.org/dwc/terms/earliestEraOrLowestErathem",
|
|
94
|
+
"http://rs.tdwg.org/dwc/terms/samplingProtocol",
|
|
95
|
+
"http://rs.tdwg.org/dwc/terms/infraspecificEpithet",
|
|
96
|
+
"http://rs.tdwg.org/dwc/terms/footprintSpatialFit",
|
|
97
|
+
"http://rs.tdwg.org/dwc/terms/namePublishedIn",
|
|
98
|
+
"http://rs.tdwg.org/dwc/terms/georeferenceSources",
|
|
99
|
+
"http://rs.tdwg.org/dwc/terms/identificationRemarks",
|
|
100
|
+
"http://rs.tdwg.org/dwc/terms/taxonConceptID",
|
|
101
|
+
"http://purl.org/dc/terms/accessRights",
|
|
102
|
+
"http://rs.tdwg.org/dwc/terms/lifeStage",
|
|
103
|
+
"http://rs.tdwg.org/dwc/terms/acceptedNameUsage",
|
|
104
|
+
"http://rs.tdwg.org/dwc/terms/verbatimEventDate",
|
|
105
|
+
"http://rs.tdwg.org/dwc/terms/typeStatus",
|
|
106
|
+
"http://rs.tdwg.org/dwc/terms/maximumDepthInMeters",
|
|
107
|
+
"http://rs.tdwg.org/dwc/terms/preparations",
|
|
108
|
+
"http://rs.tdwg.org/dwc/terms/georeferencedDate",
|
|
109
|
+
"http://rs.tdwg.org/dwc/terms/institutionID",
|
|
110
|
+
"http://rs.tdwg.org/dwc/terms/verbatimCoordinates",
|
|
111
|
+
"http://rs.tdwg.org/dwc/terms/vernacularName",
|
|
112
|
+
"http://rs.tdwg.org/dwc/terms/bed",
|
|
113
|
+
"http://rs.tdwg.org/dwc/terms/materialSampleID",
|
|
114
|
+
"http://rs.tdwg.org/dwc/terms/maximumElevationInMeters",
|
|
115
|
+
"http://purl.org/dc/terms/rights",
|
|
116
|
+
"http://rs.tdwg.org/dwc/terms/scientificName",
|
|
117
|
+
"http://rs.tdwg.org/dwc/terms/identificationReferences",
|
|
118
|
+
"http://rs.tdwg.org/dwc/terms/day",
|
|
119
|
+
"http://rs.tdwg.org/dwc/terms/fieldNumber",
|
|
120
|
+
"http://rs.tdwg.org/dwc/terms/parentNameUsageID",
|
|
121
|
+
"http://rs.tdwg.org/dwc/terms/acceptedNameUsageID",
|
|
122
|
+
"http://rs.tdwg.org/dwc/terms/identificationQualifier",
|
|
123
|
+
"http://rs.tdwg.org/dwc/terms/dataGeneralizations",
|
|
124
|
+
"http://rs.tdwg.org/dwc/terms/sex",
|
|
125
|
+
"http://rs.tdwg.org/dwc/terms/previousIdentifications",
|
|
126
|
+
"http://rs.tdwg.org/dwc/terms/georeferencedBy",
|
|
127
|
+
"http://rs.tdwg.org/dwc/terms/latestPeriodOrHighestSystem",
|
|
128
|
+
"http://rs.tdwg.org/dwc/terms/kingdom",
|
|
129
|
+
"http://rs.tdwg.org/dwc/terms/basisOfRecord",
|
|
130
|
+
"http://rs.tdwg.org/dwc/terms/institutionCode",
|
|
131
|
+
"http://rs.tdwg.org/dwc/terms/verbatimCoordinateSystem",
|
|
132
|
+
"http://rs.tdwg.org/dwc/terms/waterBody",
|
|
133
|
+
"http://rs.tdwg.org/dwc/terms/phylum",
|
|
134
|
+
"http://rs.tdwg.org/dwc/terms/ownerInstitutionCode",
|
|
135
|
+
"http://rs.tdwg.org/dwc/terms/latestAgeOrHighestStage",
|
|
136
|
+
"http://rs.tdwg.org/dwc/terms/occurrenceStatus",
|
|
137
|
+
"http://rs.tdwg.org/dwc/terms/higherClassification",
|
|
138
|
+
"http://purl.org/dc/terms/source",
|
|
139
|
+
"http://rs.tdwg.org/dwc/terms/recordedBy",
|
|
140
|
+
"http://rs.tdwg.org/dwc/terms/datasetID",
|
|
141
|
+
"http://rs.tdwg.org/dwc/terms/minimumElevationInMeters",
|
|
142
|
+
"http://rs.tdwg.org/dwc/terms/individualID",
|
|
143
|
+
"http://rs.tdwg.org/dwc/terms/samplingEffort",
|
|
144
|
+
"http://rs.tdwg.org/dwc/terms/Event",
|
|
145
|
+
"http://rs.tdwg.org/dwc/terms/member",
|
|
146
|
+
"http://rs.tdwg.org/dwc/terms/scientificNameAuthorship",
|
|
147
|
+
"http://rs.tdwg.org/dwc/terms/locationID",
|
|
148
|
+
"http://rs.tdwg.org/dwc/terms/higherGeography",
|
|
149
|
+
"http://rs.tdwg.org/dwc/terms/genus",
|
|
150
|
+
"http://rs.tdwg.org/dwc/terms/associatedOccurrences",
|
|
151
|
+
"http://rs.tdwg.org/dwc/terms/verbatimElevation",
|
|
152
|
+
"http://rs.tdwg.org/dwc/terms/maximumDistanceAboveSurfaceInMeters",
|
|
153
|
+
"http://rs.tdwg.org/dwc/terms/minimumDepthInMeters",
|
|
154
|
+
"http://rs.tdwg.org/dwc/terms/stateProvince",
|
|
155
|
+
"http://rs.tdwg.org/dwc/terms/nomenclaturalStatus",
|
|
156
|
+
"http://rs.tdwg.org/dwc/terms/taxonomicStatus",
|
|
157
|
+
"http://rs.tdwg.org/dwc/terms/specificEpithet",
|
|
158
|
+
"http://rs.tdwg.org/dwc/terms/parentEventID",
|
|
159
|
+
"http://rs.tdwg.org/dwc/terms/subgenus",
|
|
160
|
+
"http://rs.tdwg.org/dwc/terms/coordinatePrecision",
|
|
161
|
+
"http://rs.tdwg.org/dwc/terms/eventTime",
|
|
162
|
+
"http://rs.tdwg.org/dwc/terms/country",
|
|
163
|
+
"http://rs.tdwg.org/dwc/terms/coordinateUncertaintyInMeters",
|
|
164
|
+
"http://rs.tdwg.org/dwc/terms/identificationVerificationStatus",
|
|
165
|
+
"http://rs.tdwg.org/dwc/terms/otherCatalogNumbers",
|
|
166
|
+
"http://rs.tdwg.org/dwc/terms/earliestAgeOrLowestStage",
|
|
167
|
+
"http://rs.tdwg.org/dwc/terms/recordNumber",
|
|
168
|
+
"http://rs.tdwg.org/dwc/terms/reproductiveCondition",
|
|
169
|
+
"http://rs.tdwg.org/dwc/terms/locationRemarks",
|
|
170
|
+
"http://rs.tdwg.org/dwc/terms/namePublishedInID",
|
|
171
|
+
]
|
dwca/darwincore/utils.py
CHANGED
dwca/descriptors.py
CHANGED
|
@@ -28,21 +28,21 @@ class DataFileDescriptor(object):
|
|
|
28
28
|
* :meth:`make_from_file` (created by analyzing the data file)
|
|
29
29
|
"""
|
|
30
30
|
|
|
31
|
-
def __init__(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
created_from_file: bool,
|
|
34
|
+
raw_element: Element,
|
|
35
|
+
represents_corefile: bool,
|
|
36
|
+
datafile_type: Optional[str],
|
|
37
|
+
file_location: str,
|
|
38
|
+
file_encoding: str,
|
|
39
|
+
id_index: int,
|
|
40
|
+
coreid_index: int,
|
|
41
|
+
fields: List[Dict],
|
|
42
|
+
lines_terminated_by: str,
|
|
43
|
+
fields_enclosed_by: str,
|
|
44
|
+
fields_terminated_by: str,
|
|
45
|
+
) -> None:
|
|
46
46
|
#: True if this descriptor was created by analyzing the data file.
|
|
47
47
|
self.created_from_file = created_from_file
|
|
48
48
|
#: The <section> element describing the data file, from the metafile. None if the
|
|
@@ -108,7 +108,7 @@ class DataFileDescriptor(object):
|
|
|
108
108
|
"""
|
|
109
109
|
file_encoding = "utf-8"
|
|
110
110
|
|
|
111
|
-
with io.open(datafile_path,
|
|
111
|
+
with io.open(datafile_path, "r", encoding=file_encoding) as datafile:
|
|
112
112
|
# Autodetect fields/lines termination
|
|
113
113
|
dialect = csv.Sniffer().sniff(datafile.readline())
|
|
114
114
|
|
|
@@ -126,21 +126,26 @@ class DataFileDescriptor(object):
|
|
|
126
126
|
columns = next(dr)
|
|
127
127
|
|
|
128
128
|
fields = [
|
|
129
|
-
{
|
|
129
|
+
{"index": i, "term": column, "default": None}
|
|
130
|
+
for i, column in enumerate(columns)
|
|
130
131
|
]
|
|
131
132
|
|
|
132
|
-
return cls(
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
133
|
+
return cls(
|
|
134
|
+
created_from_file=True,
|
|
135
|
+
raw_element=None, # No metafile, so no XML section to store
|
|
136
|
+
represents_corefile=True, # In archives w/o metafiles, there's only core data
|
|
137
|
+
datafile_type=None, # No metafile => no rowType information
|
|
138
|
+
file_location=os.path.basename(
|
|
139
|
+
datafile_path
|
|
140
|
+
), # datafile_path also has the dir
|
|
141
|
+
file_encoding=file_encoding,
|
|
142
|
+
id_index=None,
|
|
143
|
+
coreid_index=None,
|
|
144
|
+
fields=fields,
|
|
145
|
+
lines_terminated_by=lines_terminated_by,
|
|
146
|
+
fields_enclosed_by=fields_enclosed_by,
|
|
147
|
+
fields_terminated_by=fields_terminated_by,
|
|
148
|
+
)
|
|
144
149
|
|
|
145
150
|
@classmethod
|
|
146
151
|
def make_from_metafile_section(cls, section_tag):
|
|
@@ -149,56 +154,66 @@ class DataFileDescriptor(object):
|
|
|
149
154
|
:param section_tag: The XML Element section containing details about the data file.
|
|
150
155
|
:type section_tag: :class:`xml.etree.ElementTree.Element`
|
|
151
156
|
"""
|
|
152
|
-
if section_tag.tag ==
|
|
153
|
-
id_index = int(section_tag.find(
|
|
157
|
+
if section_tag.tag == "core":
|
|
158
|
+
id_index = int(section_tag.find("id").get("index"))
|
|
154
159
|
coreid_index = None
|
|
155
160
|
else:
|
|
156
161
|
id_index = None
|
|
157
|
-
coreid_index = int(section_tag.find(
|
|
162
|
+
coreid_index = int(section_tag.find("coreid").get("index"))
|
|
158
163
|
|
|
159
164
|
fields = []
|
|
160
|
-
for field_tag in section_tag.findall(
|
|
161
|
-
default = field_tag.get(
|
|
165
|
+
for field_tag in section_tag.findall("field"):
|
|
166
|
+
default = field_tag.get("default", None)
|
|
162
167
|
|
|
163
168
|
# Default fields don't have an index attribute
|
|
164
|
-
index =
|
|
165
|
-
|
|
166
|
-
fields.append(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
169
|
+
index = int(field_tag.get("index")) if field_tag.get("index") else None
|
|
170
|
+
|
|
171
|
+
fields.append(
|
|
172
|
+
{"term": field_tag.get("term"), "index": index, "default": default}
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
file_encoding = section_tag.get("encoding")
|
|
176
|
+
|
|
177
|
+
lines_terminated_by = _decode_xml_attribute(
|
|
178
|
+
raw_element=section_tag,
|
|
179
|
+
attribute_name="linesTerminatedBy",
|
|
180
|
+
default_value="\n",
|
|
181
|
+
encoding=file_encoding,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
fields_terminated_by = _decode_xml_attribute(
|
|
185
|
+
raw_element=section_tag,
|
|
186
|
+
attribute_name="fieldsTerminatedBy",
|
|
187
|
+
default_value="\t",
|
|
188
|
+
encoding=file_encoding,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
fields_enclosed_by = _decode_xml_attribute(
|
|
192
|
+
raw_element=section_tag,
|
|
193
|
+
attribute_name="fieldsEnclosedBy",
|
|
194
|
+
default_value="",
|
|
195
|
+
encoding=file_encoding,
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
return cls(
|
|
199
|
+
created_from_file=False,
|
|
200
|
+
raw_element=section_tag,
|
|
201
|
+
represents_corefile=(section_tag.tag == "core"),
|
|
202
|
+
datafile_type=section_tag.get("rowType"),
|
|
203
|
+
file_location=section_tag.find("files").find("location").text,
|
|
204
|
+
file_encoding=file_encoding,
|
|
205
|
+
id_index=id_index,
|
|
206
|
+
coreid_index=coreid_index,
|
|
207
|
+
fields=fields,
|
|
208
|
+
lines_terminated_by=lines_terminated_by,
|
|
209
|
+
fields_enclosed_by=fields_enclosed_by,
|
|
210
|
+
fields_terminated_by=fields_terminated_by,
|
|
211
|
+
)
|
|
197
212
|
|
|
198
213
|
@property
|
|
199
214
|
def terms(self) -> Set[str]:
|
|
200
215
|
"""Return a Python set containing all the Darwin Core terms appearing in file."""
|
|
201
|
-
return set([f[
|
|
216
|
+
return set([f["term"] for f in self.fields])
|
|
202
217
|
|
|
203
218
|
@property
|
|
204
219
|
def headers(self) -> List[str]:
|
|
@@ -214,14 +229,16 @@ class DataFileDescriptor(object):
|
|
|
214
229
|
columns = {}
|
|
215
230
|
|
|
216
231
|
for f in self.fields:
|
|
217
|
-
if f[
|
|
218
|
-
|
|
232
|
+
if f[
|
|
233
|
+
"index"
|
|
234
|
+
]: # Some (default values for example) don't have a corresponding col.
|
|
235
|
+
columns[f["index"]] = f["term"]
|
|
219
236
|
|
|
220
237
|
# In addition to DwC terms, we may also have id (Core) or core_id (Extensions) columns
|
|
221
238
|
if self.id_index is not None:
|
|
222
|
-
columns[self.id_index] =
|
|
239
|
+
columns[self.id_index] = "id"
|
|
223
240
|
if self.coreid_index is not None:
|
|
224
|
-
columns[self.coreid_index] =
|
|
241
|
+
columns[self.coreid_index] = "coreid"
|
|
225
242
|
|
|
226
243
|
return [columns[f] for f in sorted(columns.keys())]
|
|
227
244
|
|
|
@@ -244,7 +261,7 @@ class DataFileDescriptor(object):
|
|
|
244
261
|
# Single-file archives always have a header line with DwC terms
|
|
245
262
|
return 1
|
|
246
263
|
|
|
247
|
-
return int(self.raw_element.get(
|
|
264
|
+
return int(self.raw_element.get("ignoreHeaderLines", 0))
|
|
248
265
|
|
|
249
266
|
|
|
250
267
|
class ArchiveDescriptor(object):
|
|
@@ -255,29 +272,35 @@ class ArchiveDescriptor(object):
|
|
|
255
272
|
files_to_ignore = []
|
|
256
273
|
|
|
257
274
|
# Let's drop the XML namespace to avoid prefixes
|
|
258
|
-
metaxml_content = re.sub(' xmlns="[^"]+"',
|
|
275
|
+
metaxml_content = re.sub(' xmlns="[^"]+"', "", metaxml_content, count=1)
|
|
259
276
|
|
|
260
277
|
#: A :class:`xml.etree.ElementTree.Element` instance containing the complete Archive Descriptor.
|
|
261
278
|
self.raw_element = ET.fromstring(metaxml_content) # type: Element
|
|
262
279
|
|
|
263
280
|
#: The path (relative to archive root) of the (scientific) metadata of the archive.
|
|
264
|
-
self.metadata_filename = self.raw_element.get(
|
|
281
|
+
self.metadata_filename = self.raw_element.get("metadata", None)
|
|
265
282
|
|
|
266
283
|
#: An instance of :class:`dwca.descriptors.DataFileDescriptor` describing the core data file.
|
|
267
|
-
raw_core_element = self.raw_element.find(
|
|
268
|
-
self.core = DataFileDescriptor.make_from_metafile_section(
|
|
284
|
+
raw_core_element = self.raw_element.find("core")
|
|
285
|
+
self.core = DataFileDescriptor.make_from_metafile_section(
|
|
286
|
+
raw_core_element
|
|
287
|
+
) # type: DataFileDescriptor
|
|
269
288
|
|
|
270
289
|
#: A list of :class:`dwca.descriptors.DataFileDescriptor` instances describing each of the archive's extension
|
|
271
290
|
#: data files.
|
|
272
291
|
self.extensions = [] # type: List[DataFileDescriptor]
|
|
273
|
-
for extension_tag in self.raw_element.findall(
|
|
274
|
-
location_tag = extension_tag.find(
|
|
292
|
+
for extension_tag in self.raw_element.findall("extension"): # type: Element
|
|
293
|
+
location_tag = extension_tag.find("./files/location")
|
|
275
294
|
if location_tag is not None:
|
|
276
295
|
extension_filename = location_tag.text
|
|
277
296
|
if extension_filename not in files_to_ignore:
|
|
278
|
-
self.extensions.append(
|
|
297
|
+
self.extensions.append(
|
|
298
|
+
DataFileDescriptor.make_from_metafile_section(extension_tag)
|
|
299
|
+
)
|
|
279
300
|
else:
|
|
280
|
-
raise InvalidArchive(
|
|
301
|
+
raise InvalidArchive(
|
|
302
|
+
"An extension file is referenced in Metafile, but its path is not specified."
|
|
303
|
+
)
|
|
281
304
|
|
|
282
305
|
#: A list of extension (types) in use in the archive.
|
|
283
306
|
#:
|
dwca/files.py
CHANGED
|
@@ -79,7 +79,7 @@ class CSVDataFile(object):
|
|
|
79
79
|
raise StopIteration
|
|
80
80
|
|
|
81
81
|
@property
|
|
82
|
-
def coreid_index(self)
|
|
82
|
+
def coreid_index(self) -> Dict[str, array]:
|
|
83
83
|
"""An index of the core rows referenced by this data file.
|
|
84
84
|
|
|
85
85
|
It is a Python dict such as:
|
|
@@ -110,11 +110,11 @@ class CSVDataFile(object):
|
|
|
110
110
|
|
|
111
111
|
for position, row in enumerate(self):
|
|
112
112
|
if self.file_descriptor.represents_corefile:
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
tmp = CoreRow(row, position, self.file_descriptor)
|
|
114
|
+
index.setdefault(tmp.id, array("L")).append(position)
|
|
115
115
|
else:
|
|
116
|
-
|
|
117
|
-
|
|
116
|
+
tmp = ExtensionRow(row, position, self.file_descriptor)
|
|
117
|
+
index.setdefault(tmp.core_id, array("L")).append(position)
|
|
118
118
|
|
|
119
119
|
return index
|
|
120
120
|
|
dwca/minibench.py
CHANGED
|
@@ -8,12 +8,12 @@ import resource
|
|
|
8
8
|
from dwca.read import DwCAReader
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
def sizeof_fmt(num, suffix=
|
|
12
|
-
for unit in [
|
|
11
|
+
def sizeof_fmt(num, suffix="B"):
|
|
12
|
+
for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
|
|
13
13
|
if abs(num) < 1024.0:
|
|
14
14
|
return "%3.1f%s%s" % (num, unit, suffix)
|
|
15
15
|
num /= 1024.0
|
|
16
|
-
return "%.1f%s%s" % (num,
|
|
16
|
+
return "%.1f%s%s" % (num, "Yi", suffix)
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def show_memory_usage():
|
|
@@ -27,8 +27,9 @@ def show_memory_usage():
|
|
|
27
27
|
# print "Memory before:"
|
|
28
28
|
# show_memory_usage()
|
|
29
29
|
|
|
30
|
+
|
|
30
31
|
def test():
|
|
31
|
-
with DwCAReader(
|
|
32
|
+
with DwCAReader("dwca-florabank1-occurrences") as dwca:
|
|
32
33
|
# print "Time after open:"
|
|
33
34
|
# print time.ctime()
|
|
34
35
|
|
|
@@ -39,7 +40,7 @@ def test():
|
|
|
39
40
|
for row in dwca:
|
|
40
41
|
# tmp = row.data[qn('locality')]
|
|
41
42
|
i = i + 1
|
|
42
|
-
if
|
|
43
|
+
if i % 100000 == 0:
|
|
43
44
|
print("in loop mem: ")
|
|
44
45
|
show_memory_usage()
|
|
45
46
|
|
|
@@ -53,7 +54,8 @@ def test():
|
|
|
53
54
|
# show_memory_usage()
|
|
54
55
|
|
|
55
56
|
|
|
56
|
-
if
|
|
57
|
+
if __name__ == "__main__":
|
|
57
58
|
from timeit import Timer
|
|
59
|
+
|
|
58
60
|
t = Timer("test()", "from __main__ import test")
|
|
59
61
|
print(t.timeit(number=3))
|