pywmdr 0.2.dev4__py3-none-any.whl → 0.2.dev5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pywmdr/__init__.py +2 -0
- pywmdr/cli_options.py +7 -0
- pywmdr/migrations.py +417 -0
- pywmdr/pygeoapi_plugin.py +103 -0
- {pywmdr-0.2.dev4.dist-info → pywmdr-0.2.dev5.dist-info}/METADATA +11 -1
- {pywmdr-0.2.dev4.dist-info → pywmdr-0.2.dev5.dist-info}/RECORD +10 -9
- {pywmdr-0.2.dev4.dist-info → pywmdr-0.2.dev5.dist-info}/WHEEL +0 -0
- {pywmdr-0.2.dev4.dist-info → pywmdr-0.2.dev5.dist-info}/entry_points.txt +0 -0
- {pywmdr-0.2.dev4.dist-info → pywmdr-0.2.dev5.dist-info}/licenses/LICENSE.md +0 -0
- {pywmdr-0.2.dev4.dist-info → pywmdr-0.2.dev5.dist-info}/top_level.txt +0 -0
pywmdr/__init__.py
CHANGED
|
@@ -28,6 +28,7 @@ import click
|
|
|
28
28
|
from pywmdr.ets import ets
|
|
29
29
|
from pywmdr.bundle import bundle
|
|
30
30
|
from pywmdr.util import get_package_version
|
|
31
|
+
from pywmdr.migrations import migrate
|
|
31
32
|
|
|
32
33
|
__version__ = get_package_version()
|
|
33
34
|
|
|
@@ -40,3 +41,4 @@ def cli():
|
|
|
40
41
|
|
|
41
42
|
cli.add_command(bundle)
|
|
42
43
|
cli.add_command(ets)
|
|
44
|
+
cli.add_command(migrate)
|
pywmdr/cli_options.py
CHANGED
|
@@ -25,6 +25,13 @@ import sys
|
|
|
25
25
|
import click
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
OPTION_OUTPUT = click.option(
|
|
29
|
+
'--output',
|
|
30
|
+
'-o',
|
|
31
|
+
type=click.File('w', encoding='utf-8'),
|
|
32
|
+
help='Name of output file')
|
|
33
|
+
|
|
34
|
+
|
|
28
35
|
def OPTION_VERBOSITY(f):
|
|
29
36
|
logging_options = ['ERROR', 'WARNING', 'INFO', 'DEBUG']
|
|
30
37
|
|
pywmdr/migrations.py
ADDED
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
##############################################################################
|
|
2
|
+
#
|
|
3
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
4
|
+
# or more contributor license agreements. See the NOTICE file
|
|
5
|
+
# distributed with this work for additional information
|
|
6
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
7
|
+
# to you under the Apache License, Version 2.0 (the
|
|
8
|
+
# "License"); you may not use this file except in compliance
|
|
9
|
+
# with the License. You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing,
|
|
14
|
+
# software distributed under the License is distributed on an
|
|
15
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
16
|
+
# KIND, either express or implied. See the License for the
|
|
17
|
+
# specific language governing permissions and limitations
|
|
18
|
+
# under the License.
|
|
19
|
+
#
|
|
20
|
+
###############################################################################
|
|
21
|
+
|
|
22
|
+
import datetime
|
|
23
|
+
import json
|
|
24
|
+
import logging
|
|
25
|
+
import re
|
|
26
|
+
|
|
27
|
+
from lxml import etree
|
|
28
|
+
|
|
29
|
+
import click
|
|
30
|
+
|
|
31
|
+
from pywmdr import cli_options
|
|
32
|
+
from pywmdr.util import urlopen_
|
|
33
|
+
|
|
34
|
+
LOGGER = logging.getLogger(__name__)
|
|
35
|
+
|
|
36
|
+
NAMESPACES = {
|
|
37
|
+
'gmd': 'http://www.isotc211.org/2005/gmd',
|
|
38
|
+
'gco': 'http://www.isotc211.org/2005/gco',
|
|
39
|
+
'gml': 'http://www.opengis.net/gml/3.2',
|
|
40
|
+
'om': 'http://www.opengis.net/om/2.0',
|
|
41
|
+
'wmdr': 'http://def.wmo.int/wmdr/2017',
|
|
42
|
+
'xlink': 'http://www.w3.org/1999/xlink'
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def xpath_(node: etree.Element, xpath: str) -> str | None:
|
|
47
|
+
"""
|
|
48
|
+
Helper function to derive a value from an XPath
|
|
49
|
+
|
|
50
|
+
:param node: `etree.Element`
|
|
51
|
+
:param xpath: `str` of XPath expression
|
|
52
|
+
|
|
53
|
+
:returns: `str` of matching XPath value or `None`
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
try:
|
|
57
|
+
val = node.xpath(xpath, namespaces=NAMESPACES)[0]
|
|
58
|
+
if hasattr(val, 'text'): # node
|
|
59
|
+
return val.text
|
|
60
|
+
else: # attribute
|
|
61
|
+
return val
|
|
62
|
+
except IndexError:
|
|
63
|
+
return None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class Migration:
|
|
67
|
+
def __init__(self, record: str):
|
|
68
|
+
"""
|
|
69
|
+
Initializer
|
|
70
|
+
|
|
71
|
+
:param record: `str` of WMDR1 XML record
|
|
72
|
+
|
|
73
|
+
:returns: `pywmdr.migrations.Migration`
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
self.record = record
|
|
77
|
+
self.warnings = []
|
|
78
|
+
|
|
79
|
+
def migrate(self) -> dict:
|
|
80
|
+
"""
|
|
81
|
+
Migrate a WMDR1 XML to WMDR2
|
|
82
|
+
|
|
83
|
+
:returns: `dict` of WMDR2
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
created = datetime.datetime.now(datetime.UTC).strftime(
|
|
87
|
+
'%Y-%m-%dT%H:%M:%SZ').replace('+00:00', 'Z')
|
|
88
|
+
|
|
89
|
+
wmdr2 = {
|
|
90
|
+
'id': None,
|
|
91
|
+
'type': 'Feature',
|
|
92
|
+
'conformsTo': ['http://wigos.wmo.int/spec/wmdr/2/conf/core'],
|
|
93
|
+
'time': None,
|
|
94
|
+
'geometry': None,
|
|
95
|
+
'properties': {
|
|
96
|
+
'type': 'facility',
|
|
97
|
+
'created': created,
|
|
98
|
+
'contacts': []
|
|
99
|
+
},
|
|
100
|
+
'links': []
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if isinstance(self.record, str):
|
|
104
|
+
record_ = self.record.encode('utf-8')
|
|
105
|
+
elif isinstance(self.record, bytes):
|
|
106
|
+
record_ = self.record
|
|
107
|
+
|
|
108
|
+
tree = etree.fromstring(record_)
|
|
109
|
+
|
|
110
|
+
facility = tree.xpath('//wmdr:facility', namespaces=NAMESPACES)[0]
|
|
111
|
+
|
|
112
|
+
val = xpath_(facility, './/wmdr:ObservingFacility/gml:identifier')
|
|
113
|
+
wmdr2['id'] = val
|
|
114
|
+
|
|
115
|
+
val = xpath_(facility, './/wmdr:geospatialLocation//gml:Point/gml:pos')
|
|
116
|
+
|
|
117
|
+
if val is not None:
|
|
118
|
+
coordinates = [float(x) for x in val.split()]
|
|
119
|
+
wmdr2['geometry'] = {
|
|
120
|
+
'type': 'Point',
|
|
121
|
+
'coordinates': [coordinates[1], coordinates[0]]
|
|
122
|
+
}
|
|
123
|
+
if len(coordinates) == 3:
|
|
124
|
+
wmdr2['geometry']['coordinates'].append(coordinates[2])
|
|
125
|
+
|
|
126
|
+
val = xpath_(facility, './/wmdr:ObservingFacility/gml:name')
|
|
127
|
+
wmdr2['properties']['title'] = val
|
|
128
|
+
|
|
129
|
+
val = xpath_(facility, './/wmdr:ObservingFacility/wmdr:description//wmdr:description') # noqa
|
|
130
|
+
if val is not None:
|
|
131
|
+
wmdr2['properties']['description'] = val
|
|
132
|
+
|
|
133
|
+
val = xpath_(facility, './/wmdr:facilityType/@xlink:href')
|
|
134
|
+
wmdr2['properties']['facilityType'] = {
|
|
135
|
+
'id': val.split('/')[-1],
|
|
136
|
+
'url': val
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
val = xpath_(facility, './/wmdr:wmoRegion/@xlink:href')
|
|
140
|
+
if val is not None:
|
|
141
|
+
wmdr2['properties']['wmoRegion'] = {
|
|
142
|
+
'id': val.split('/')[-1],
|
|
143
|
+
'url': val
|
|
144
|
+
}
|
|
145
|
+
else:
|
|
146
|
+
wmdr2['properties']['wmoRegion'] = {
|
|
147
|
+
'id': 'unknown',
|
|
148
|
+
'url': 'http://codes.wmo.int/wmdr/WMORegion/unknown'
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
val = xpath_(facility, './/wmdr:dateEstablished')
|
|
152
|
+
wmdr2['time'] = {
|
|
153
|
+
'interval': [val]
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
val = xpath_(facility, './/wmdr:dateClosed')
|
|
157
|
+
if val is not None:
|
|
158
|
+
wmdr2['time']['interval'].append(val)
|
|
159
|
+
else:
|
|
160
|
+
wmdr2['time']['interval'].append('..')
|
|
161
|
+
|
|
162
|
+
for ci_rp in tree.xpath('//gmd:CI_ResponsibleParty', namespaces=NAMESPACES): # noqa
|
|
163
|
+
contact2 = self.ci_responsibleparty_to_contact(ci_rp)
|
|
164
|
+
if contact2 is not None:
|
|
165
|
+
wmdr2['properties']['contacts'].append(contact2)
|
|
166
|
+
|
|
167
|
+
if not wmdr2['properties']['contacts']:
|
|
168
|
+
wmdr2['properties'].pop('contacts')
|
|
169
|
+
if contact2 is not None:
|
|
170
|
+
wmdr2['properties']['contacts'].append(contact2)
|
|
171
|
+
|
|
172
|
+
for link in facility.xpath('.//wmdr:onlineResource//gmd:URL', namespaces=NAMESPACES): # noqa
|
|
173
|
+
wmdr2['links'].append({
|
|
174
|
+
'rel': 'related',
|
|
175
|
+
'href': link.text
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
territories = tree.xpath('//wmdr:territory', namespaces=NAMESPACES)
|
|
179
|
+
if territories:
|
|
180
|
+
wmdr2['properties']['territories'] = []
|
|
181
|
+
for territory in territories:
|
|
182
|
+
territory_ = {}
|
|
183
|
+
territory_name = xpath_(
|
|
184
|
+
territory, './/wmdr:territoryName/@xlink:href')
|
|
185
|
+
|
|
186
|
+
if territory_name is not None:
|
|
187
|
+
territory_['territory'] = {
|
|
188
|
+
'id': territory_name.split('/')[-1],
|
|
189
|
+
'url': territory_name
|
|
190
|
+
}
|
|
191
|
+
begin = xpath_(
|
|
192
|
+
territory, './/wmdr:validPeriod//gml:beginPosition')
|
|
193
|
+
|
|
194
|
+
territory_['dates'] = [begin]
|
|
195
|
+
|
|
196
|
+
end = xpath_(territory, './/wmdr:validPeriod//gml:endPosition') # noqa
|
|
197
|
+
|
|
198
|
+
if end is None:
|
|
199
|
+
territory_['dates'].append('..')
|
|
200
|
+
else:
|
|
201
|
+
territory_['dates'].append(end)
|
|
202
|
+
wmdr2['properties']['territories'].append(territory_)
|
|
203
|
+
|
|
204
|
+
observations = tree.xpath('//wmdr:observation/wmdr:ObservingCapability', namespaces=NAMESPACES) # noqa
|
|
205
|
+
|
|
206
|
+
if observations:
|
|
207
|
+
wmdr2['properties']['observations'] = []
|
|
208
|
+
|
|
209
|
+
for observation in observations:
|
|
210
|
+
observation_ = {
|
|
211
|
+
'programAffiliations': [],
|
|
212
|
+
'configurations': []
|
|
213
|
+
}
|
|
214
|
+
observed_geometry = xpath_(
|
|
215
|
+
observation, './/om:type/@xlink:href')
|
|
216
|
+
|
|
217
|
+
if observed_geometry is not None:
|
|
218
|
+
observation_['observedGeometry'] = {
|
|
219
|
+
'id': observed_geometry.split('/')[-1],
|
|
220
|
+
'url': observed_geometry
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
observed_property = xpath_(
|
|
224
|
+
observation, './/om:observedProperty/@xlink:href')
|
|
225
|
+
|
|
226
|
+
observation_['observedProperty'] = {
|
|
227
|
+
'id': observed_property.split('/')[-1],
|
|
228
|
+
'url': observed_property
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
program_affiliations = observation.xpath(
|
|
232
|
+
'./wmdr:programAffiliation/@xlink:href',
|
|
233
|
+
namespaces=NAMESPACES)
|
|
234
|
+
if program_affiliations:
|
|
235
|
+
for program_affiliation in program_affiliations:
|
|
236
|
+
observation_['programAffiliations'].append({
|
|
237
|
+
'programAffiliation': {
|
|
238
|
+
'id': program_affiliation.split('/')[-1],
|
|
239
|
+
'url': program_affiliation
|
|
240
|
+
}})
|
|
241
|
+
|
|
242
|
+
deployments = observation.xpath(
|
|
243
|
+
'.//wmdr:Deployment', namespaces=NAMESPACES)
|
|
244
|
+
|
|
245
|
+
for deployment in deployments:
|
|
246
|
+
configuration_ = {
|
|
247
|
+
'id': None,
|
|
248
|
+
'time': {
|
|
249
|
+
'interval': []
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
configuration_['id'] = xpath_(deployment, './/@gml:id')
|
|
253
|
+
|
|
254
|
+
begin = xpath_(
|
|
255
|
+
deployment, './/wmdr:validPeriod//gml:beginPosition')
|
|
256
|
+
|
|
257
|
+
configuration_['time']['interval'].append(begin)
|
|
258
|
+
|
|
259
|
+
end = xpath_(
|
|
260
|
+
deployment, './/wmdr:validPeriod//gml:endPosition')
|
|
261
|
+
|
|
262
|
+
if end is None:
|
|
263
|
+
configuration_['time']['interval'].append('..')
|
|
264
|
+
else:
|
|
265
|
+
configuration_['time']['interval'].append(end)
|
|
266
|
+
|
|
267
|
+
observing_method = xpath_(
|
|
268
|
+
deployment, './/wmdr:observingMethod/@xlink:href')
|
|
269
|
+
if observing_method is not None:
|
|
270
|
+
configuration_['observingMethod'] = {
|
|
271
|
+
'id': observing_method.split('/')[-1],
|
|
272
|
+
'url': observing_method
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
observation_['configurations'].append(configuration_)
|
|
276
|
+
|
|
277
|
+
if not observation_['configurations']:
|
|
278
|
+
msg = f"No configurations found for observation ({observation_['observedProperty']['url']})" # noqa
|
|
279
|
+
self.warnings.append(msg)
|
|
280
|
+
observation_.pop('configurations')
|
|
281
|
+
|
|
282
|
+
wmdr2['properties']['observations'].append(observation_)
|
|
283
|
+
|
|
284
|
+
wmdr2['links'].append({
|
|
285
|
+
'rel': 'original',
|
|
286
|
+
'type': 'application/xml',
|
|
287
|
+
'title': f"{wmdr2['properties']['title']} as WMDR1",
|
|
288
|
+
'href': f"https://oscar.wmo.int/oai/provider?verb=GetRecord&metadataPrefix=wmdr&identifier={wmdr2['id']}" # noqa
|
|
289
|
+
})
|
|
290
|
+
|
|
291
|
+
return wmdr2
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def ci_responsibleparty_to_contact(self, node: etree.Element) -> tuple[dict, list[str]]: # noqa
|
|
295
|
+
"""
|
|
296
|
+
Helper function to transform ISO gmd:CI_ResponsibleParty to a
|
|
297
|
+
WMDR2 contact
|
|
298
|
+
|
|
299
|
+
:param node: `etree.Element` of gmd:CI_ResponsibleParty
|
|
300
|
+
|
|
301
|
+
:returns: `dict` of WMDR2 contact object, or `None`
|
|
302
|
+
"""
|
|
303
|
+
|
|
304
|
+
val = None
|
|
305
|
+
contact = {}
|
|
306
|
+
|
|
307
|
+
val = xpath_(node, './/gmd:role/gmd:CI_RoleCode/@codeListValue')
|
|
308
|
+
if val is None:
|
|
309
|
+
LOGGER.debug('No role found; skipping')
|
|
310
|
+
return
|
|
311
|
+
else:
|
|
312
|
+
contact['roles'] = [val]
|
|
313
|
+
|
|
314
|
+
val = xpath_(node, './/gmd:individualName/gco:CharacterString')
|
|
315
|
+
if val is not None:
|
|
316
|
+
contact['name'] = val
|
|
317
|
+
|
|
318
|
+
val = xpath_(node, './/gmd:organisationName/gco:CharacterString')
|
|
319
|
+
if val is not None:
|
|
320
|
+
contact['organization'] = val
|
|
321
|
+
|
|
322
|
+
phones = node.xpath('.//gmd:voice/gco:CharacterString',
|
|
323
|
+
namespaces=NAMESPACES)
|
|
324
|
+
if phones:
|
|
325
|
+
contact['phones'] = []
|
|
326
|
+
for phone in phones:
|
|
327
|
+
contact['phones'].append({'value': phone.text})
|
|
328
|
+
|
|
329
|
+
emails = node.xpath('.//gmd:electronicMailAddress/gco:CharacterString',
|
|
330
|
+
namespaces=NAMESPACES)
|
|
331
|
+
if emails:
|
|
332
|
+
contact['emails'] = []
|
|
333
|
+
for email in emails:
|
|
334
|
+
# validate we have email address
|
|
335
|
+
# (something at something dot something, no white space)
|
|
336
|
+
email_valid = re.fullmatch(r'[^@\s]+@[^@\s]+\.[^@\s]+', email.text) is not None # noqa
|
|
337
|
+
email_address = email.text.strip()
|
|
338
|
+
if not email_valid:
|
|
339
|
+
msg = f'Invalid email: {email_address}'
|
|
340
|
+
self.warnings.append(msg)
|
|
341
|
+
contact['emails'].append({'value': email_address})
|
|
342
|
+
|
|
343
|
+
val = xpath_(node, './/gmd:contactInstructions/gco:CharacterString')
|
|
344
|
+
if val is not None:
|
|
345
|
+
contact['contactInstructions'] = val
|
|
346
|
+
|
|
347
|
+
try:
|
|
348
|
+
val = node.xpath('.//gmd:address', namespaces=NAMESPACES)[0]
|
|
349
|
+
except IndexError:
|
|
350
|
+
val = None
|
|
351
|
+
|
|
352
|
+
if val is not None:
|
|
353
|
+
address = {}
|
|
354
|
+
|
|
355
|
+
val2 = xpath_(val, './/gmd:deliveryPoint/gco:CharacterString')
|
|
356
|
+
if val2 is not None:
|
|
357
|
+
address['deliveryPoint'] = [val2]
|
|
358
|
+
|
|
359
|
+
val2 = xpath_(val, './/gmd:city/gco:CharacterString')
|
|
360
|
+
if val2 is not None:
|
|
361
|
+
address['city'] = val2
|
|
362
|
+
|
|
363
|
+
val2 = xpath_(val, './/gmd:postalCode/gco:CharacterString')
|
|
364
|
+
if val2 is not None:
|
|
365
|
+
address['postalCode'] = val2
|
|
366
|
+
|
|
367
|
+
val2 = xpath_(val, './/gmd:country/gco:CharacterString')
|
|
368
|
+
if val2 is not None:
|
|
369
|
+
address['country'] = val2
|
|
370
|
+
|
|
371
|
+
val2 = xpath_(val, './/gmd:onlineResource//gmd:URL')
|
|
372
|
+
if val2 is not None:
|
|
373
|
+
address['links'] = [{
|
|
374
|
+
'href': val2
|
|
375
|
+
}]
|
|
376
|
+
|
|
377
|
+
if address:
|
|
378
|
+
contact['addresses'] = [address]
|
|
379
|
+
|
|
380
|
+
return contact
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
@click.command()
|
|
384
|
+
@click.pass_context
|
|
385
|
+
@click.argument('file_or_url')
|
|
386
|
+
@cli_options.OPTION_OUTPUT
|
|
387
|
+
@cli_options.OPTION_VERBOSITY
|
|
388
|
+
def migrate(ctx, file_or_url, output, verbosity):
|
|
389
|
+
"""migrate a WMDR1 record to WMDR2 core"""
|
|
390
|
+
|
|
391
|
+
click.echo(f'Opening {file_or_url}')
|
|
392
|
+
|
|
393
|
+
if file_or_url.startswith('http'):
|
|
394
|
+
content = urlopen_(file_or_url).read()
|
|
395
|
+
else:
|
|
396
|
+
with open(file_or_url) as fh:
|
|
397
|
+
content = fh.read()
|
|
398
|
+
|
|
399
|
+
click.echo(f'Migrating {file_or_url} from WMDR1 to WMDR2')
|
|
400
|
+
|
|
401
|
+
try:
|
|
402
|
+
m = Migration(content)
|
|
403
|
+
content = m.migrate()
|
|
404
|
+
if output is not None:
|
|
405
|
+
json.dump(content, output, indent=4)
|
|
406
|
+
else:
|
|
407
|
+
click.echo(json.dumps(content, indent=4))
|
|
408
|
+
|
|
409
|
+
if m.warnings:
|
|
410
|
+
click.echo('\nWARNINGS:')
|
|
411
|
+
for w in m.warnings:
|
|
412
|
+
click.echo(f'- {w}')
|
|
413
|
+
except Exception as err:
|
|
414
|
+
import traceback
|
|
415
|
+
print(traceback.format_exc())
|
|
416
|
+
raise click.ClickException(err)
|
|
417
|
+
ctx.exit(1)
|
pywmdr/pygeoapi_plugin.py
CHANGED
|
@@ -37,11 +37,18 @@
|
|
|
37
37
|
# processor:
|
|
38
38
|
# name: pywmdr.pygeoapi_plugin.WMDR2ETSProcessor
|
|
39
39
|
#
|
|
40
|
+
# pywmdr-wmdr2-migrate:
|
|
41
|
+
# type: process
|
|
42
|
+
# processor:
|
|
43
|
+
# name: pywmdr.pygeoapi_plugin.WMDR1toWMDR2Processor
|
|
44
|
+
#
|
|
45
|
+
#
|
|
40
46
|
# 3. (re)start pygeoapi
|
|
41
47
|
#
|
|
42
48
|
# The resulting processes will be available at the following endpoints:
|
|
43
49
|
#
|
|
44
50
|
# /processes/pywmdr-record-validate
|
|
51
|
+
# /processes/pywmdr-wmdr2-migrate
|
|
45
52
|
#
|
|
46
53
|
# Note that pygeoapi's OpenAPI/Swagger interface (at /openapi) will also
|
|
47
54
|
# provide a developer-friendly interface to test and run requests
|
|
@@ -52,6 +59,8 @@ import logging
|
|
|
52
59
|
|
|
53
60
|
from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError
|
|
54
61
|
|
|
62
|
+
from pywmdr.bundle import WMDR2_FILES
|
|
63
|
+
from pywmdr.migrations import Migration
|
|
55
64
|
from pywmdr.wmdr2.ets import WMDR2TestSuite
|
|
56
65
|
from pywmdr.util import get_package_version, THISDIR, urlopen_
|
|
57
66
|
|
|
@@ -63,6 +72,11 @@ with (THISDIR / 'resources' / 'ets-report.json').open() as fh:
|
|
|
63
72
|
with (THISDIR / 'resources' / '0-20008-0-THE.json').open() as fh:
|
|
64
73
|
EXAMPLE_WMDR2 = json.load(fh)
|
|
65
74
|
|
|
75
|
+
with (THISDIR / 'resources' / '0-20000-0-87582.xml').open() as fh:
|
|
76
|
+
EXAMPLE_WMDR1 = fh.read()
|
|
77
|
+
|
|
78
|
+
with (WMDR2_FILES / 'wmdr2-bundled.json').open() as fh:
|
|
79
|
+
WMDR2_SCHEMA = json.load(fh)
|
|
66
80
|
|
|
67
81
|
PROCESS_WMDR2_ETS = {
|
|
68
82
|
'version': get_package_version(),
|
|
@@ -113,6 +127,55 @@ PROCESS_WMDR2_ETS = {
|
|
|
113
127
|
}
|
|
114
128
|
|
|
115
129
|
|
|
130
|
+
PROCESS_MIGRATE = {
|
|
131
|
+
'version': get_package_version(),
|
|
132
|
+
'id': 'pywmdr-wmdr2-migrate',
|
|
133
|
+
'title': {
|
|
134
|
+
'en': 'WMDR1 to WMDR2 migration utility'
|
|
135
|
+
},
|
|
136
|
+
'description': {
|
|
137
|
+
'en': 'Migrate a WMDR1 XML into WMDR2'
|
|
138
|
+
},
|
|
139
|
+
'keywords': ['wigos', 'wmdr2', 'wmdr1', 'migrate', 'metadata'],
|
|
140
|
+
'links': [{
|
|
141
|
+
'type': 'text/html',
|
|
142
|
+
'rel': 'about',
|
|
143
|
+
'title': 'information',
|
|
144
|
+
'href': 'https://github.com/wmo-im/wmdr2',
|
|
145
|
+
'hreflang': 'en-US'
|
|
146
|
+
}],
|
|
147
|
+
'jobControlOptions': ['sync-execute', 'async-execute'],
|
|
148
|
+
'inputs': {
|
|
149
|
+
'record': {
|
|
150
|
+
'title': 'WMD1 record',
|
|
151
|
+
'description': 'WMDR1 record (can be inline or remote link)',
|
|
152
|
+
'schema': {
|
|
153
|
+
'type': ['object', 'string']
|
|
154
|
+
},
|
|
155
|
+
'minOccurs': 1,
|
|
156
|
+
'maxOccurs': 1,
|
|
157
|
+
'metadata': None,
|
|
158
|
+
'keywords': ['wmdr1']
|
|
159
|
+
}
|
|
160
|
+
},
|
|
161
|
+
'outputs': {
|
|
162
|
+
'result': {
|
|
163
|
+
'title': 'WMDR1 to WMDR2 migration result',
|
|
164
|
+
'description': 'WMDR1 to WMDR2 migration result',
|
|
165
|
+
'schema': {
|
|
166
|
+
'contentMediaType': 'application/geo+json',
|
|
167
|
+
**WMDR2_SCHEMA
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
'example': {
|
|
172
|
+
'inputs': {
|
|
173
|
+
'record': EXAMPLE_WMDR2
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
|
|
116
179
|
class WMDR2ETSProcessor(BaseProcessor):
|
|
117
180
|
"""WMDR2 ETS"""
|
|
118
181
|
|
|
@@ -151,3 +214,43 @@ class WMDR2ETSProcessor(BaseProcessor):
|
|
|
151
214
|
|
|
152
215
|
def __repr__(self):
|
|
153
216
|
return '<WMDR2ETSProcessor>'
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
class WMDR2MigrateProcessor(BaseProcessor):
|
|
220
|
+
"""Migrate processor"""
|
|
221
|
+
|
|
222
|
+
def __init__(self, processor_def):
|
|
223
|
+
"""
|
|
224
|
+
Initialize object
|
|
225
|
+
|
|
226
|
+
:param processor_def: provider definition
|
|
227
|
+
|
|
228
|
+
:returns: pywmdr.pygeoapi_plugin.WMDR2MigrateProcessor
|
|
229
|
+
"""
|
|
230
|
+
|
|
231
|
+
super().__init__(processor_def, PROCESS_MIGRATE)
|
|
232
|
+
|
|
233
|
+
def execute(self, data, outputs=None):
|
|
234
|
+
|
|
235
|
+
response = None
|
|
236
|
+
mimetype = 'application/json'
|
|
237
|
+
record = data.get('record')
|
|
238
|
+
|
|
239
|
+
if record is None:
|
|
240
|
+
msg = 'Missing record'
|
|
241
|
+
LOGGER.error(msg)
|
|
242
|
+
raise ProcessorExecuteError(msg)
|
|
243
|
+
|
|
244
|
+
if isinstance(record, str) and record.startswith('http'):
|
|
245
|
+
LOGGER.debug('Record is a link')
|
|
246
|
+
record = json.loads(urlopen_(record).read())
|
|
247
|
+
else:
|
|
248
|
+
LOGGER.debug('Record is inline')
|
|
249
|
+
|
|
250
|
+
LOGGER.debug('Running ETS against record')
|
|
251
|
+
response = Migration(record).migrate()
|
|
252
|
+
|
|
253
|
+
return mimetype, response
|
|
254
|
+
|
|
255
|
+
def __repr__(self):
|
|
256
|
+
return '<WMDR2MigrateProcessor>'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pywmdr
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.dev5
|
|
4
4
|
Summary: A Python implementation of the test suite for WIGOS Metadata Record
|
|
5
5
|
Author-email: Tom Kralidis <tomkralidis@gmail.com>
|
|
6
6
|
Maintainer-email: Tom Kralidis <tomkralidis@gmail.com>
|
|
@@ -23,6 +23,7 @@ Description-Content-Type: text/markdown
|
|
|
23
23
|
License-File: LICENSE.md
|
|
24
24
|
Requires-Dist: click
|
|
25
25
|
Requires-Dist: jsonschema
|
|
26
|
+
Requires-Dist: lxml
|
|
26
27
|
Requires-Dist: rfc3339-validator
|
|
27
28
|
Requires-Dist: rfc3987
|
|
28
29
|
Requires-Dist: shapely
|
|
@@ -95,6 +96,15 @@ pywmdr ets validate https://example.org/path/to/file.json --verbosity DEBUG
|
|
|
95
96
|
|
|
96
97
|
# write results to logfile
|
|
97
98
|
pywmdr ets validate https://example.org/path/to/file.json --verbosity DEBUG --logfile /tmp/foo.txt
|
|
99
|
+
|
|
100
|
+
# migrate WMDR1 to WMDR2 (file on disk)
|
|
101
|
+
pywmdr migrate /path/to/file.xml --verbosity DEBUG
|
|
102
|
+
|
|
103
|
+
# migrate WMDR1 to WMDR2 (URL)
|
|
104
|
+
pywmdr migrate https://example.org/path/to/file.xml --verbosity DEBUG
|
|
105
|
+
|
|
106
|
+
# migrate WMDR1 to WMDR2 (URL), write results to file
|
|
107
|
+
pywmdr migrate https://example.org/path/to/file.xml --output file.json
|
|
98
108
|
```
|
|
99
109
|
|
|
100
110
|
## Using the API
|
|
@@ -1,18 +1,19 @@
|
|
|
1
|
-
pywmdr/__init__.py,sha256=
|
|
1
|
+
pywmdr/__init__.py,sha256=O_2G6cZKo3-Y8bupQ3NUaF35TxS9urMDRxWD0CN6AeA,1381
|
|
2
2
|
pywmdr/bundle.py,sha256=VgkDwwsftqbpBXKJORwjzl6oqST5LVB-6K0vCKOZRqo,3190
|
|
3
|
-
pywmdr/cli_options.py,sha256=
|
|
3
|
+
pywmdr/cli_options.py,sha256=BCNGnzm525keOj8UpKnXnhtunfBiNmdcuGtXGoq1iz8,1667
|
|
4
4
|
pywmdr/codelists.py,sha256=UMYqy4Tk8GLulqMjE3xouZMWCUpnW1qHbuwZ1CiACpA,3694
|
|
5
5
|
pywmdr/errors.py,sha256=Uu2J6vmTjaWuNJaW-mtdZ_KCztc9x4wjaLRVDZP6Lk0,1176
|
|
6
6
|
pywmdr/ets.py,sha256=I1JeheXMCsPtRXjf5ay8I3MZFrrsDtj5k0XT68XCOOY,2076
|
|
7
|
-
pywmdr/
|
|
7
|
+
pywmdr/migrations.py,sha256=FxhJYLQE6_BXLpEMZAWQRu1nIGDBGlt88HkcpbPWUwc,14328
|
|
8
|
+
pywmdr/pygeoapi_plugin.py,sha256=Ls6PqsVgfPNzcWS-ff7jZOxhLTSc9CjyPL6lgXgMRPo,7286
|
|
8
9
|
pywmdr/util.py,sha256=LE_Ds6OuG7vNQxjFQcogWiPD5CNQyHhqIXeW6TNXJZE,2639
|
|
9
10
|
pywmdr/resources/0-20008-0-THE.json,sha256=luAjCrxbnwq4x0nwqm2vyx8d5rXmkMeGwVCQ2w-gaXg,5873
|
|
10
11
|
pywmdr/resources/ets-report.json,sha256=VwZohaD_EloSlwhYXxnj08Z3ckfbFYudLc2dWwxvlus,2243
|
|
11
12
|
pywmdr/wmdr2/__init__.py,sha256=VK88WG3wpt5RqksGgutMnr02OhUlksWcoAnoeSRRWic,1035
|
|
12
13
|
pywmdr/wmdr2/ets.py,sha256=8M5zyubMfiQvArl_-LoGlH8dhDiY-fIAmkIsub-N3JM,7817
|
|
13
|
-
pywmdr-0.2.
|
|
14
|
-
pywmdr-0.2.
|
|
15
|
-
pywmdr-0.2.
|
|
16
|
-
pywmdr-0.2.
|
|
17
|
-
pywmdr-0.2.
|
|
18
|
-
pywmdr-0.2.
|
|
14
|
+
pywmdr-0.2.dev5.dist-info/licenses/LICENSE.md,sha256=Q8E0TXZpH75ydPcip7DXr21Y3uwJNgQUFv3ipMXwPkw,839
|
|
15
|
+
pywmdr-0.2.dev5.dist-info/METADATA,sha256=1sCFM6Zi6au4wNJ9HKV1HwAZERDHYYb_k0ar6ND_FSM,5444
|
|
16
|
+
pywmdr-0.2.dev5.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
17
|
+
pywmdr-0.2.dev5.dist-info/entry_points.txt,sha256=Ga7GlkDlThWwprQDc5OawyWtV_RT9FXilgXjwKacynU,38
|
|
18
|
+
pywmdr-0.2.dev5.dist-info/top_level.txt,sha256=_Zdw0CS7tVYk-CJZmu9xEqIzOuY98RsI6CUMh8Kh818,7
|
|
19
|
+
pywmdr-0.2.dev5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|