emso-metadata-harmonizer 1.1.0.dev1__tar.gz → 1.1.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {emso_metadata_harmonizer-1.1.0.dev1/src/emso_metadata_harmonizer.egg-info → emso_metadata_harmonizer-1.1.0.dev2}/PKG-INFO +1 -1
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/__init__.py +5 -1
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/dataset_generator.py +3 -3
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/erddap/datasets_xml.py +1 -3
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/erddap_config.py +2 -1
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/emso.py +10 -14
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/resource_manager.py +15 -15
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/tests.py +6 -6
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/utils.py +16 -13
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/vocabularies.py +8 -12
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/waterframe.py +7 -10
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/report.py +2 -1
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2/src/emso_metadata_harmonizer.egg-info}/PKG-INFO +1 -1
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/LICENSE +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/README.md +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/pypi.md +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/pyproject.toml +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/setup.cfg +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/erddap/__init__.py +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/erddap/erddap.py +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/__init__.py +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/constants.py +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/dataset.py +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/metadata_templates.py +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer/metadata/xmlutils.py +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer.egg-info/SOURCES.txt +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer.egg-info/dependency_links.txt +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer.egg-info/requires.txt +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/src/emso_metadata_harmonizer.egg-info/top_level.txt +0 -0
- {emso_metadata_harmonizer-1.1.0.dev1 → emso_metadata_harmonizer-1.1.0.dev2}/tests/test_metadata_harmonizer.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: emso-metadata-harmonizer
|
|
3
|
-
Version: 1.1.0.
|
|
3
|
+
Version: 1.1.0.dev2
|
|
4
4
|
Summary: Toolbox for creating/assessing EMSO-compliant NetCDF datasets and integrate them into ERDDAP services
|
|
5
5
|
Author-email: Enoc Martínez <enoc.martinez@upc.edu>
|
|
6
6
|
Maintainer-email: Enoc Martínez <enoc.martinez@upc.edu>
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
1
3
|
from .erddap_config import erddap_config
|
|
2
4
|
from .erddap import ERDDAP
|
|
3
5
|
from .metadata.waterframe import WaterFrame
|
|
@@ -6,4 +8,6 @@ from .dataset_generator import generate_dataset
|
|
|
6
8
|
from .metadata.waterframe import WaterFrame
|
|
7
9
|
from .metadata import setup_log
|
|
8
10
|
|
|
9
|
-
|
|
11
|
+
logging.getLogger("emso_metadata_harmonizer").addHandler(logging.NullHandler())
|
|
12
|
+
|
|
13
|
+
__version__ = "1.1.0dev2"
|
|
@@ -13,13 +13,13 @@ import logging
|
|
|
13
13
|
import rich
|
|
14
14
|
import pandas as pd
|
|
15
15
|
import yaml
|
|
16
|
-
|
|
16
|
+
|
|
17
17
|
from .metadata import EmsoMetadata
|
|
18
18
|
from .metadata.dataset import load_data
|
|
19
|
-
from .metadata.utils import assert_type,
|
|
19
|
+
from .metadata.utils import assert_type, EMH_LOGGER_NAME
|
|
20
20
|
from .metadata.waterframe import WaterFrame, get_coordinates_from_dataframe
|
|
21
21
|
|
|
22
|
-
logger = logging.getLogger(
|
|
22
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
23
23
|
|
|
24
24
|
global_elements = (
|
|
25
25
|
# Array with attribute_name, type, mandatory (True, False), additional_checks
|
|
@@ -54,12 +54,10 @@ def generate_erddap_dataset(wf: WaterFrame, directory, dataset_id, recursive: bo
|
|
|
54
54
|
assert isinstance(wf, WaterFrame), f"Expected WaterFrame (got {type(wf)})"
|
|
55
55
|
assert isinstance(directory, str), f"Expected str for directory (got {type(directory)})"
|
|
56
56
|
|
|
57
|
-
log = logging.getLogger()
|
|
58
|
-
|
|
59
57
|
vocab = wf.vocabulary
|
|
60
58
|
if mapping:
|
|
61
59
|
# convert from array to a dict with source as the key
|
|
62
|
-
|
|
60
|
+
logger.info(f"Using user-supplied variable mapping")
|
|
63
61
|
var_mapping = {d["source"]: d for d in mapping["mapping"]["variables"]}
|
|
64
62
|
attr_mapping = mapping["mapping"]["attributes"]
|
|
65
63
|
else:
|
|
@@ -15,8 +15,9 @@ import rich
|
|
|
15
15
|
|
|
16
16
|
from .erddap.datasets_xml import generate_erddap_dataset, add_dataset
|
|
17
17
|
from .metadata.waterframe import WaterFrame
|
|
18
|
+
from .metadata.utils import EMH_LOGGER_NAME
|
|
18
19
|
|
|
19
|
-
logger = logging.getLogger(
|
|
20
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
20
21
|
|
|
21
22
|
def erddap_config(file: str, dataset_id: str, source_path: str, output: str = "", datasets_xml_file: str = "",
|
|
22
23
|
mapping: dict={}, filename_regex=".*", recursive=False):
|
|
@@ -12,14 +12,14 @@ created: 3/3/23
|
|
|
12
12
|
import logging
|
|
13
13
|
import os
|
|
14
14
|
from .vocabularies import GEMET, GCMD, SeaDataNetVocabulary, EuroSciVoc, Keyword, OSO
|
|
15
|
-
from .utils import get_file_list, assert_type
|
|
15
|
+
from .utils import get_file_list, assert_type, EMH_LOGGER_NAME
|
|
16
16
|
from .resource_manager import ResourceManager, process_markdown_file, load_json
|
|
17
17
|
|
|
18
18
|
# Specifications version used when the caller does not ask for one. "develop" keeps the behaviour of previous
|
|
19
19
|
# releases; set it to "latest" to follow the newest published tag instead.
|
|
20
20
|
emso_version = "develop"
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
23
23
|
|
|
24
24
|
user_defined_specs_file = "" # used to overload online specifications, used for development only
|
|
25
25
|
|
|
@@ -94,7 +94,7 @@ class EmsoMetadata:
|
|
|
94
94
|
:param version: EMSO Metadata Specifications version. Empty uses the module default (see emso_version)
|
|
95
95
|
:param max_threads: size of the download thread pool
|
|
96
96
|
"""
|
|
97
|
-
|
|
97
|
+
logger.info("Loading EMSO Metadata resources...")
|
|
98
98
|
|
|
99
99
|
# All downloading, caching and parsing is delegated to the ResourceManager, which resolves the
|
|
100
100
|
# requested version against manifest.json and fetches everything in parallel.
|
|
@@ -137,7 +137,7 @@ class EmsoMetadata:
|
|
|
137
137
|
# ==== Load all SDN vocabularies ==== #
|
|
138
138
|
# Every file was already downloaded and parsed by the ResourceManager, so this is just a re-shuffle
|
|
139
139
|
for vocab in list(self.sdn_vocabs.keys()):
|
|
140
|
-
|
|
140
|
+
logger.debug(f" loading SDN vocabulary {vocab}")
|
|
141
141
|
df = self.resource_manager.get(vocab, "csv")
|
|
142
142
|
self.sdn_vocabs[vocab] = df
|
|
143
143
|
self.sdn_vocabs_narrower[vocab] = self.resource_manager.get(vocab, "narrower")
|
|
@@ -149,15 +149,15 @@ class EmsoMetadata:
|
|
|
149
149
|
self.sdn_vocabs_uris[vocab] = df["uri"].values
|
|
150
150
|
|
|
151
151
|
# ==== Load Copernicus Variables ==== #
|
|
152
|
-
|
|
152
|
+
logger.debug(f" loading Copernicus Parameters")
|
|
153
153
|
tables = self.resource_manager.get("Copernicus Parameters", "md")
|
|
154
154
|
|
|
155
155
|
self.copernicus_variables = tables["Copernicus variables"]["variable name"].to_list()
|
|
156
|
-
|
|
156
|
+
logger.debug(f" loading EDMO codes")
|
|
157
157
|
self.edmo_codes = self.resource_manager.get("EDMO", "csv")
|
|
158
158
|
|
|
159
159
|
if user_defined_specs_file:
|
|
160
|
-
|
|
160
|
+
logger.warning(f"Using custom specifications file: {user_defined_specs_file}")
|
|
161
161
|
tables = process_markdown_file(user_defined_specs_file)
|
|
162
162
|
else:
|
|
163
163
|
tables = self.resource_manager.get("EMSO_Metadata_Specifications", "md")
|
|
@@ -325,8 +325,6 @@ class EmsoMetadata:
|
|
|
325
325
|
"""
|
|
326
326
|
Search in vocab <vocab_id> for the element with matching uri and return element identified by key
|
|
327
327
|
"""
|
|
328
|
-
log = logging.getLogger()
|
|
329
|
-
|
|
330
328
|
uri = self.harmonize_sdn_uri(uri)
|
|
331
329
|
__allowed_keys = ["prefLabel", "id", "definition", "altLabel"]
|
|
332
330
|
if key not in __allowed_keys:
|
|
@@ -335,7 +333,7 @@ class EmsoMetadata:
|
|
|
335
333
|
position = self.vocab_row(vocab_id, "uri", uri)
|
|
336
334
|
if position is None:
|
|
337
335
|
#raise LookupError(f"Could not get {key} for '{uri}' in vocab {vocab_id}")
|
|
338
|
-
|
|
336
|
+
logger.warning(f"Could not get {key} for '{uri}' in vocab {vocab_id}")
|
|
339
337
|
return
|
|
340
338
|
|
|
341
339
|
return self.sdn_vocabs[vocab_id][key].values[position]
|
|
@@ -380,7 +378,6 @@ class EmsoMetadata:
|
|
|
380
378
|
:param target_vocab: id of the vocabulary terms that we want to find
|
|
381
379
|
:returns: list with matches
|
|
382
380
|
"""
|
|
383
|
-
log = logging.getLogger()
|
|
384
381
|
__valid_relations = ["narrower", "broader", "related"]
|
|
385
382
|
uri = self.harmonize_sdn_uri(uri)
|
|
386
383
|
|
|
@@ -397,7 +394,7 @@ class EmsoMetadata:
|
|
|
397
394
|
try:
|
|
398
395
|
uri_relations = relations[uri]
|
|
399
396
|
except KeyError:
|
|
400
|
-
|
|
397
|
+
logger.warning(f"relation {relation} for {uri} not found!")
|
|
401
398
|
return ""
|
|
402
399
|
|
|
403
400
|
if type(uri_relations) is str: # make sure it's a list
|
|
@@ -413,10 +410,9 @@ class EmsoMetadata:
|
|
|
413
410
|
"""
|
|
414
411
|
The same as get relations but throws an error if more than one element are found
|
|
415
412
|
"""
|
|
416
|
-
log = logging.getLogger()
|
|
417
413
|
results = self.get_relations(vocab_id, uri, relation, target_vocab)
|
|
418
414
|
if len(results) == 0:
|
|
419
|
-
|
|
415
|
+
logger.warning(f"Could not find relation {relation} for {uri}")
|
|
420
416
|
return ""
|
|
421
417
|
elif len(results) != 1:
|
|
422
418
|
raise LookupError(f"Expected 1 value, got {len(results)}")
|
|
@@ -34,9 +34,9 @@ from concurrent.futures import ThreadPoolExecutor
|
|
|
34
34
|
import pandas as pd
|
|
35
35
|
import requests
|
|
36
36
|
|
|
37
|
-
from .utils import download_file, get_file_md5
|
|
37
|
+
from .utils import download_file, get_file_md5, EMH_LOGGER_NAME
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
40
40
|
|
|
41
41
|
DEFAULT_CACHE_DIR = ".emso"
|
|
42
42
|
|
|
@@ -190,10 +190,10 @@ class Resource:
|
|
|
190
190
|
could not be established.
|
|
191
191
|
"""
|
|
192
192
|
if not force and self.is_cached(known_md5=known_md5):
|
|
193
|
-
|
|
193
|
+
logger.debug(f" {self.label} is up to date")
|
|
194
194
|
return self.md5 if self.md5 else known_md5
|
|
195
195
|
|
|
196
|
-
|
|
196
|
+
logger.info(f" downloading {self.label} to {self.filename}...")
|
|
197
197
|
os.makedirs(os.path.dirname(self.filename) or ".", exist_ok=True)
|
|
198
198
|
download_file(self.url, self.filename)
|
|
199
199
|
self.downloaded = True
|
|
@@ -202,7 +202,7 @@ class Resource:
|
|
|
202
202
|
if self.md5 and local_md5 != self.md5:
|
|
203
203
|
# The vocabulary snapshot tracks a mutable branch: a mismatch means it moved between publishing the
|
|
204
204
|
# manifest and this download, not that the file is corrupt.
|
|
205
|
-
|
|
205
|
+
logger.warning(f"{self.label}: md5 mismatch (expected {self.md5}, got {local_md5}). "
|
|
206
206
|
f"The upstream snapshot probably moved; the downloaded file is used anyway.")
|
|
207
207
|
return local_md5
|
|
208
208
|
|
|
@@ -218,7 +218,7 @@ class Resource:
|
|
|
218
218
|
elif extension == ".json":
|
|
219
219
|
self.dict = load_json(self.filename)
|
|
220
220
|
else:
|
|
221
|
-
|
|
221
|
+
logger.debug(f" {self.label}: extension '{extension}' not parsed, keeping file as-is")
|
|
222
222
|
|
|
223
223
|
def process(self, force=False, known_md5=None):
|
|
224
224
|
"""
|
|
@@ -231,7 +231,7 @@ class Resource:
|
|
|
231
231
|
return local_md5
|
|
232
232
|
except Exception as e:
|
|
233
233
|
self.error = e
|
|
234
|
-
|
|
234
|
+
logger.error(f"Could not process {self.label}: {e}")
|
|
235
235
|
return None
|
|
236
236
|
|
|
237
237
|
def get(self):
|
|
@@ -282,7 +282,7 @@ class ResourceManager:
|
|
|
282
282
|
self.vocabularies = self.get_vocabularies()
|
|
283
283
|
|
|
284
284
|
self.version = self.resolve_version(version)
|
|
285
|
-
|
|
285
|
+
logger.info(f"Using EMSO Metadata Specifications {self.version}")
|
|
286
286
|
|
|
287
287
|
self.resources = {} # name -> {key: Resource}
|
|
288
288
|
self.__build_resources()
|
|
@@ -302,7 +302,7 @@ class ResourceManager:
|
|
|
302
302
|
if cached and not self.force_update:
|
|
303
303
|
age = time.time() - os.path.getmtime(filename)
|
|
304
304
|
if age < manifest_max_age:
|
|
305
|
-
|
|
305
|
+
logger.debug(f"{what} is recent enough ({age / 3600:.1f} h), not checking upstream")
|
|
306
306
|
return cached
|
|
307
307
|
|
|
308
308
|
for url in urls:
|
|
@@ -311,7 +311,7 @@ class ResourceManager:
|
|
|
311
311
|
response.raise_for_status()
|
|
312
312
|
document = response.json()
|
|
313
313
|
except (requests.exceptions.RequestException, ValueError) as e:
|
|
314
|
-
|
|
314
|
+
logger.debug(f"Could not fetch {what} from {url}: {e}")
|
|
315
315
|
continue
|
|
316
316
|
|
|
317
317
|
os.makedirs(os.path.dirname(filename) or ".", exist_ok=True)
|
|
@@ -319,11 +319,11 @@ class ResourceManager:
|
|
|
319
319
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
320
320
|
json.dump(document, f, indent=2)
|
|
321
321
|
os.replace(tmp, filename) # single atomic write
|
|
322
|
-
|
|
322
|
+
logger.debug(f"{what} updated from {url}")
|
|
323
323
|
return document
|
|
324
324
|
|
|
325
325
|
if cached:
|
|
326
|
-
|
|
326
|
+
logger.warning(f"Could not reach any {what} URL, using the cached copy")
|
|
327
327
|
os.utime(filename, None) # back off instead of retrying on every call
|
|
328
328
|
return cached
|
|
329
329
|
|
|
@@ -402,7 +402,7 @@ class ResourceManager:
|
|
|
402
402
|
thread pool is the right tool here.
|
|
403
403
|
"""
|
|
404
404
|
resources = self.__all_resources()
|
|
405
|
-
|
|
405
|
+
logger.info(f"Loading {len(resources)} EMSO metadata resources ({self.max_threads} threads)...")
|
|
406
406
|
t = time.time()
|
|
407
407
|
|
|
408
408
|
def worker(resource):
|
|
@@ -425,7 +425,7 @@ class ResourceManager:
|
|
|
425
425
|
self.__save_state()
|
|
426
426
|
|
|
427
427
|
downloaded = sum(1 for r in resources if r.downloaded)
|
|
428
|
-
|
|
428
|
+
logger.info(f"Loaded {len(resources)} resources in {time.time() - t:.02f} s "
|
|
429
429
|
f"({downloaded} downloaded, {len(resources) - downloaded} from cache)")
|
|
430
430
|
|
|
431
431
|
def __save_state(self):
|
|
@@ -435,7 +435,7 @@ class ResourceManager:
|
|
|
435
435
|
json.dump(self.__state, f, indent=2)
|
|
436
436
|
os.replace(tmp, self.__state_file)
|
|
437
437
|
except OSError as e:
|
|
438
|
-
|
|
438
|
+
logger.debug(f"Could not write the cache state file: {e}") # purely an optimisation, never fatal
|
|
439
439
|
|
|
440
440
|
# ------------------------------------------------------------------ accessors
|
|
441
441
|
def __contains__(self, name):
|
|
@@ -74,7 +74,7 @@ class EmsoMetadataTester:
|
|
|
74
74
|
error = False
|
|
75
75
|
for test in all_tests:
|
|
76
76
|
if test not in self.implemented_tests.keys():
|
|
77
|
-
|
|
77
|
+
logger.error(f"ERROR test {test} not implemented!")
|
|
78
78
|
error = True
|
|
79
79
|
if error:
|
|
80
80
|
raise ValueError("Some tests are not implemented")
|
|
@@ -222,7 +222,7 @@ class EmsoMetadataTester:
|
|
|
222
222
|
|
|
223
223
|
if attribute in metadata.keys():
|
|
224
224
|
if test_name not in self.implemented_tests.keys():
|
|
225
|
-
|
|
225
|
+
logger.error(f"Test '{test_name}' not implemented!")
|
|
226
226
|
|
|
227
227
|
else:
|
|
228
228
|
implemented = True
|
|
@@ -257,7 +257,7 @@ class EmsoMetadataTester:
|
|
|
257
257
|
try:
|
|
258
258
|
p, m = test_method(v, args) # apply test method
|
|
259
259
|
except Exception as e:
|
|
260
|
-
|
|
260
|
+
logger.error(f"Error when executing test '{test_name}' with arguments '{args}' and value '{v}'")
|
|
261
261
|
raise e
|
|
262
262
|
if not m:
|
|
263
263
|
m = "ok" # instead of empty message just leave ok
|
|
@@ -340,7 +340,7 @@ class EmsoMetadataTester:
|
|
|
340
340
|
multiple = row["Multiple"]
|
|
341
341
|
annotation = row["annotations"]
|
|
342
342
|
if not test_name:
|
|
343
|
-
|
|
343
|
+
logger.warning(f"WARNING: test for {attribute} not implemented!")
|
|
344
344
|
continue
|
|
345
345
|
|
|
346
346
|
args = []
|
|
@@ -382,7 +382,7 @@ class EmsoMetadataTester:
|
|
|
382
382
|
else:
|
|
383
383
|
dataset_id = global_attr["title"]
|
|
384
384
|
|
|
385
|
-
|
|
385
|
+
logger.info(f"==== Validating dataset '{CYN}{global_attr['title']}{RST}' ====")
|
|
386
386
|
|
|
387
387
|
# Test global attributes
|
|
388
388
|
if "global" in variable_filter or not variable_filter:
|
|
@@ -445,7 +445,7 @@ class EmsoMetadataTester:
|
|
|
445
445
|
# ------------ EDMO -------- #
|
|
446
446
|
def edmo_code(self, value, args):
|
|
447
447
|
if type(value) == str:
|
|
448
|
-
|
|
448
|
+
logger.warning("EDMO code should be integer! converting from string to int")
|
|
449
449
|
try:
|
|
450
450
|
value = int(value)
|
|
451
451
|
except ValueError:
|
|
@@ -33,6 +33,10 @@ NRM = "\x1B[0m"
|
|
|
33
33
|
PRL = "\033[95m"
|
|
34
34
|
RST = "\033[0m"
|
|
35
35
|
|
|
36
|
+
EMH_LOGGER_NAME = "emso_metadata_harmonizer"
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
39
|
+
|
|
36
40
|
|
|
37
41
|
def group_metadata_variables(metadata):
|
|
38
42
|
"""
|
|
@@ -147,12 +151,11 @@ def download_file(url: str, filename: str, headers: Optional[Dict[str, str]] = N
|
|
|
147
151
|
if chunk:
|
|
148
152
|
file.write(chunk)
|
|
149
153
|
except (HTTPError, requests.exceptions.ConnectionError) as e:
|
|
150
|
-
log = logging.getLogger()
|
|
151
154
|
if alternative:
|
|
152
|
-
|
|
155
|
+
logger.warning(f"Failed to fetch URL, using alternative {alternative}")
|
|
153
156
|
download_file(alternative, filename, headers=headers, chunk_size=chunk_size, alternative="")
|
|
154
157
|
else:
|
|
155
|
-
|
|
158
|
+
logger.error(f"Failed to fetch URL {url}")
|
|
156
159
|
raise e
|
|
157
160
|
|
|
158
161
|
|
|
@@ -227,23 +230,21 @@ def get_dir_list(dir_name):
|
|
|
227
230
|
|
|
228
231
|
|
|
229
232
|
class LoggerSuperclass:
|
|
230
|
-
def __init__(self,
|
|
233
|
+
def __init__(self, name: str, colour=NRM):
|
|
231
234
|
"""
|
|
232
235
|
SuperClass that defines logging as class methods adding a heading name
|
|
233
236
|
"""
|
|
234
237
|
self._logger_name = name
|
|
235
|
-
self.
|
|
236
|
-
if not logger:
|
|
237
|
-
self.__logger = logging # if not assign the generic module
|
|
238
|
+
self.logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
238
239
|
self._log_colour = colour
|
|
239
240
|
|
|
240
241
|
def warning(self, *args):
|
|
241
242
|
mystr = YEL + "[%s] " % self._logger_name + str(*args) + RST
|
|
242
|
-
self.
|
|
243
|
+
self.logger.warning(mystr)
|
|
243
244
|
|
|
244
245
|
def error(self, *args, exception: any = False):
|
|
245
246
|
mystr = "[%s] " % self._logger_name + str(*args)
|
|
246
|
-
self.
|
|
247
|
+
self.logger.error(RED + mystr + RST)
|
|
247
248
|
if exception:
|
|
248
249
|
if isinstance(exception, bool):
|
|
249
250
|
raise ValueError(mystr)
|
|
@@ -252,14 +253,16 @@ class LoggerSuperclass:
|
|
|
252
253
|
|
|
253
254
|
def debug(self, *args):
|
|
254
255
|
mystr = self._log_colour + "[%s] " % self._logger_name + str(*args) + RST
|
|
255
|
-
self.
|
|
256
|
+
self.logger.debug(mystr)
|
|
256
257
|
|
|
257
258
|
def info(self, *args):
|
|
258
259
|
mystr = self._log_colour + "[%s] " % self._logger_name + str(*args) + RST
|
|
259
|
-
self.
|
|
260
|
+
self.logger.info(mystr)
|
|
260
261
|
|
|
261
262
|
def setLevel(self, level):
|
|
262
|
-
self.
|
|
263
|
+
self.logger.setLevel(level)
|
|
264
|
+
|
|
265
|
+
|
|
263
266
|
|
|
264
267
|
|
|
265
268
|
def setup_log(name, path="log", log_level="debug"):
|
|
@@ -298,7 +301,7 @@ def setup_log(name, path="log", log_level="debug"):
|
|
|
298
301
|
if not filename.endswith(".log"):
|
|
299
302
|
filename += ".log"
|
|
300
303
|
|
|
301
|
-
logger = logging.getLogger()
|
|
304
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
302
305
|
logger.setLevel(level)
|
|
303
306
|
log_formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)-7s: %(message)s',
|
|
304
307
|
datefmt='%Y-%m-%d %H:%M:%S')
|
|
@@ -9,15 +9,15 @@ structures, so the harmonizer needs no RDF toolchain and no downloads of its own
|
|
|
9
9
|
"""
|
|
10
10
|
import logging
|
|
11
11
|
import pandas as pd
|
|
12
|
-
from .utils import LoggerSuperclass, BLU
|
|
12
|
+
from .utils import LoggerSuperclass, BLU, EMH_LOGGER_NAME
|
|
13
13
|
|
|
14
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
class GenericVocabulary(LoggerSuperclass):
|
|
17
18
|
def __init__(self, code, name, uri) -> None:
|
|
18
19
|
|
|
19
|
-
|
|
20
|
-
LoggerSuperclass.__init__(self, log, code, colour=BLU)
|
|
20
|
+
LoggerSuperclass.__init__(self, code, colour=BLU)
|
|
21
21
|
|
|
22
22
|
self.labels = [] # labels as they are
|
|
23
23
|
self.labels_lc = [] # labels as lower case for easier comparison
|
|
@@ -204,8 +204,6 @@ class OSO(GenericVocabulary):
|
|
|
204
204
|
def get_uri_from_name(self, name, cls):
|
|
205
205
|
assert cls in ["rfs", "sites", "platforms"]
|
|
206
206
|
|
|
207
|
-
log = logging.getLogger()
|
|
208
|
-
|
|
209
207
|
if cls == "rfs":
|
|
210
208
|
df = self.rfs
|
|
211
209
|
elif cls == "sites":
|
|
@@ -215,17 +213,15 @@ class OSO(GenericVocabulary):
|
|
|
215
213
|
try:
|
|
216
214
|
uri = df.loc[df["label"] == name]["uri"].values[0]
|
|
217
215
|
except (KeyError, IndexError):
|
|
218
|
-
|
|
216
|
+
self.error(f"ERROR: OSO does not have any '{cls}' with label '{name}', valid names:")
|
|
219
217
|
for a in df['label'].unique():
|
|
220
|
-
|
|
218
|
+
self.error(f" - '{a}'")
|
|
221
219
|
return ""
|
|
222
220
|
return str(uri)
|
|
223
221
|
|
|
224
222
|
def get_name_from_uri(self, uri, cls):
|
|
225
223
|
assert cls in ["rfs", "sites", "platforms"]
|
|
226
224
|
|
|
227
|
-
log = logging.getLogger()
|
|
228
|
-
|
|
229
225
|
if cls == "rfs":
|
|
230
226
|
df = self.rfs
|
|
231
227
|
elif cls == "sites":
|
|
@@ -235,9 +231,9 @@ class OSO(GenericVocabulary):
|
|
|
235
231
|
try:
|
|
236
232
|
uri = df.loc[df["uri"] == uri]["label"].values[0]
|
|
237
233
|
except (KeyError, IndexError):
|
|
238
|
-
|
|
234
|
+
self.error(f"ERROR: OSO does not have any '{cls}' with uri '{uri}', valid uris:")
|
|
239
235
|
for a in df['uri'].unique():
|
|
240
|
-
|
|
236
|
+
self.info(f" - '{a}'")
|
|
241
237
|
return ""
|
|
242
238
|
return str(uri)
|
|
243
239
|
|
|
@@ -288,7 +284,7 @@ class Keyword:
|
|
|
288
284
|
}
|
|
289
285
|
r = mapping.get(title, "undefined")
|
|
290
286
|
if r == "undefined":
|
|
291
|
-
|
|
287
|
+
logger.warning(f"Could not find type for '{self.name}' ({self.vocab_code})")
|
|
292
288
|
return r
|
|
293
289
|
|
|
294
290
|
def __repr__(self) -> str:
|
|
@@ -26,7 +26,7 @@ from .metadata_templates import time_valid_names, depth_valid_names, latitude_va
|
|
|
26
26
|
from . import init_emso_metadata
|
|
27
27
|
from .constants import iso_time_format
|
|
28
28
|
from .metadata_templates import dimension_metadata, quality_control_metadata
|
|
29
|
-
from .utils import LoggerSuperclass, CYN
|
|
29
|
+
from .utils import LoggerSuperclass, CYN, EMH_LOGGER_NAME
|
|
30
30
|
from .vocabularies import Keyword
|
|
31
31
|
|
|
32
32
|
try:
|
|
@@ -37,7 +37,7 @@ except ImportError:
|
|
|
37
37
|
|
|
38
38
|
emso = None # Global variable used to avoid duplicated EMSO metadata objects in different waterframes
|
|
39
39
|
|
|
40
|
-
logger = logging.getLogger(
|
|
40
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
# Make sure that we have all the coordinates
|
|
@@ -89,7 +89,7 @@ class WaterFrame(LoggerSuperclass):
|
|
|
89
89
|
self._time, self._depth, self._latitude, self._longitude, self._sensor_id, self._platform_id = \
|
|
90
90
|
get_coordinates_from_dataframe(df)
|
|
91
91
|
|
|
92
|
-
LoggerSuperclass.__init__(self,
|
|
92
|
+
LoggerSuperclass.__init__(self, "WF", colour=CYN)
|
|
93
93
|
global emso
|
|
94
94
|
if not emso:
|
|
95
95
|
emso = init_emso_metadata()
|
|
@@ -888,7 +888,6 @@ class WaterFrame(LoggerSuperclass):
|
|
|
888
888
|
|
|
889
889
|
@staticmethod
|
|
890
890
|
def from_erddap(url, dataset_id, protocol="tabledap", permissive=True, data_from="") -> "WaterFrame":
|
|
891
|
-
logger = logging.getLogger()
|
|
892
891
|
url = f"{url}/{protocol}/{dataset_id}.nc"
|
|
893
892
|
|
|
894
893
|
if data_from and isinstance(data_from, pd.Timestamp):
|
|
@@ -1339,11 +1338,10 @@ class OperationalTester:
|
|
|
1339
1338
|
success = len(self.errors) == 0
|
|
1340
1339
|
|
|
1341
1340
|
if self.quiet:
|
|
1342
|
-
log = logging.getLogger()
|
|
1343
1341
|
if success:
|
|
1344
|
-
|
|
1342
|
+
logger.info("Operational tests: ✅ the NetCDF file is operationally sound!")
|
|
1345
1343
|
else:
|
|
1346
|
-
|
|
1344
|
+
logger.info("Operational tests: ❌ the NetCDF file is not operationally valid")
|
|
1347
1345
|
|
|
1348
1346
|
report = {
|
|
1349
1347
|
"valid": success,
|
|
@@ -1496,11 +1494,10 @@ class KeywordTester:
|
|
|
1496
1494
|
|
|
1497
1495
|
if self.quiet:
|
|
1498
1496
|
# stdout is disabled, so report the outcome through the logger
|
|
1499
|
-
log = logging.getLogger()
|
|
1500
1497
|
if self.errors or self.warnings:
|
|
1501
|
-
|
|
1498
|
+
logger.info("Keywords check: ❌ errors encountered")
|
|
1502
1499
|
else:
|
|
1503
|
-
|
|
1500
|
+
logger.info("Keywords check: ✅ passed")
|
|
1504
1501
|
|
|
1505
1502
|
return self.errors, self.warnings
|
|
1506
1503
|
|
|
@@ -24,8 +24,9 @@ from .metadata import EmsoMetadata
|
|
|
24
24
|
from .metadata.utils import threadify
|
|
25
25
|
from .metadata.dataset import get_netcdf_metadata
|
|
26
26
|
from .metadata.tests import EmsoMetadataTester
|
|
27
|
+
from .metadata.utils import EMH_LOGGER_NAME
|
|
27
28
|
|
|
28
|
-
logger = logging.getLogger(
|
|
29
|
+
logger = logging.getLogger(EMH_LOGGER_NAME)
|
|
29
30
|
|
|
30
31
|
def metadata_results_to_json(df: pd.DataFrame)-> dict:
|
|
31
32
|
report = {"ERRORS": {}, "WARNINGS": {}}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: emso-metadata-harmonizer
|
|
3
|
-
Version: 1.1.0.
|
|
3
|
+
Version: 1.1.0.dev2
|
|
4
4
|
Summary: Toolbox for creating/assessing EMSO-compliant NetCDF datasets and integrate them into ERDDAP services
|
|
5
5
|
Author-email: Enoc Martínez <enoc.martinez@upc.edu>
|
|
6
6
|
Maintainer-email: Enoc Martínez <enoc.martinez@upc.edu>
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|