pywmdr 0.2.dev1__tar.gz → 0.2.dev3__tar.gz

{pywmdr-0.2.dev1/pywmdr.egg-info → pywmdr-0.2.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pywmdr
-Version: 0.2.dev1
+Version: 0.2.dev3
 Summary: A Python implementation of the test suite for WIGOS Metadata Record
 Author-email: Tom Kralidis <tomkralidis@gmail.com>
 Maintainer-email: Tom Kralidis <tomkralidis@gmail.com>

{pywmdr-0.2.dev1 → pywmdr-0.2.dev3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pywmdr"
-version = "0.2.dev1"
+version = "0.2.dev3"
 description = "A Python implementation of the test suite for WIGOS Metadata Record"
 readme = "README.md"
 requires-python = ">=3.12"

{pywmdr-0.2.dev1 → pywmdr-0.2.dev3}/pywmdr/bundle.py

@@ -19,6 +19,8 @@
 #
 ###############################################################################
 
+import csv
+import io
 import logging
 from pathlib import Path
 import shutil
@@ -38,6 +40,8 @@ TEMPDIR2 = Path(tempfile.TemporaryDirectory().name)
 
 WMDR2_FILES = get_userdir() / 'wmdr2'
 WMDR2_FILES_TEMP = TEMPDIR2 / 'wmdr2'
+WMDS_DIR = get_userdir() / 'wmds'
+WMDS_DIR_TEMP = TEMPDIR2 / 'wmds'
 
 
 @click.group()
@@ -61,6 +65,21 @@ def sync(ctx, verbosity):
     with json_schema.open('wb') as fh:
         fh.write(urlopen_(f'{WMDR2_SCHEMA}').read())
 
+    LOGGER.debug('Caching codelists')
+    LOGGER.debug(f'Downloading WMDS codelists to {WMDS_DIR_TEMP}')
+    WMDS_DIR_TEMP.mkdir(parents=True, exist_ok=True)
+    WMDS_CODELISTS_LIST = 'https://codes.wmo.int/wmdr?_format=csv&_view=with_metadata&status=valid'  # noqa
+    codelists = urlopen_(WMDS_CODELISTS_LIST).read().decode('utf-8')
+    codelists_fh = io.StringIO(codelists.strip())
+    reader = csv.reader(codelists_fh)
+    for row in reader:
+        codelist_url = f'https://codes.wmo.int/wmdr/{row[1]}?_format=csv&_view=with_metadata&status=valid'  # noqa
+        LOGGER.debug(f'Downloading {codelist_url}')
+        codelist_file = WMDS_DIR_TEMP / f'{row[1]}.csv'
+        LOGGER.debug(f'Writing codelist {codelist_file}')
+        with codelist_file.open('wb') as fh:
+            fh.write(urlopen_(codelist_url).read())
+
     LOGGER.debug(f'Removing {USERDIR}')
     if USERDIR.exists():
         shutil.rmtree(USERDIR)

pywmdr-0.2.dev3/pywmdr/codelists.py

@@ -0,0 +1,122 @@
+##############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+import csv
+import logging
+
+from pywmdr.util import get_userdir
+
+LOGGER = logging.getLogger(__name__)
+
+CODELISTS_MAP = {
+    'facilityType': {
+        'codelists': ['FacilityType.csv']
+    },
+    'wmoRegion': {
+        'codelists': ['WMORegion.csv']
+    },
+    'territory': {
+        'codelists': ['TerritoryName.csv']
+    },
+    'observedGeometry': {
+        'codelists': ['Geometry.csv']
+    },
+    'observedProperty': {
+        'codelists': [
+            'ObservedVariableAtmosphere.csv',
+            'ObservedVariableEarth.csv',
+            'ObservedVariableOcean.csv',
+            'ObservedVariableOuterSpace.csv',
+            'ObservedVariableTerrestrial.csv'
+        ]
+    },
+    'programAffiliation': {
+        'codelists': ['ProgramAffiliation.csv']
+    },
+    'reportingStatus': {
+        'codelists': ['ReportingStatus.csv']
+    },
+    'observingMethod': {
+        'codelists': [
+            'ObservingMethodAtmosphere.csv',
+            'ObservingMethodTerrestrial.csv',
+            'ObservingMethodOcean.csv'
+        ]
+    },
+    'operatingStatus': {
+        'codelists': ['InstrumentOperatingStatus.csv']
+    },
+    'sourceOfObservation': {
+        'codelists': ['SourceOfObservation.csv']
+    },
+    'referenceSurface': {
+        'codelists': ['ReferenceSurfaceType.csv']
+    },
+    'unit': {
+        'codelists': ['unit.csv']
+    },
+}
+
+
+class WMDSCodelists:
+    def __init__(self):
+        """initializer"""
+
+        self.codelists = {}
+        for key, value in CODELISTS_MAP.items():
+            self.codelists[key] = []
+            for codelist in value['codelists']:
+                filename = get_userdir() / 'wmds' / codelist
+                with filename.open() as fh:
+                    reader = csv.DictReader(fh)
+                    for row in reader:
+                        self.codelists[key].append(row['@notation'])
+
+    def is_valid(self, pname: str, pvalue: dict | None) -> bool:
+        """
+        Helper function to determine whether a concept is valid
+        (WMDS codelist or null)
+
+        :param pname: name of property/object
+        :param value: value of property/object
+
+        :returns: `bool` of whether a concept is valid
+        """
+
+        if pname not in self.codelists:
+            msg = f'{pname} not found in codelists'
+            LOGGER.error(msg)
+            raise ValueError(msg)
+
+        if pvalue is None:
+            LOGGER.debug('Value is null')
+            return True
+
+        LOGGER.debug(f'Property name: {pname}')
+        LOGGER.debug(f'Property value: {pvalue}')
+        LOGGER.debug(f'Codelist: {self.codelists[pname]}')
+        if pvalue['id'] in self.codelists[pname]:
+            return True
+
+        return False
+
+    def __repr__(self):
+        return '<WMDSCodelists>'

{pywmdr-0.2.dev1 → pywmdr-0.2.dev3}/pywmdr/pygeoapi_plugin.py

@@ -60,7 +60,7 @@ LOGGER = logging.getLogger(__name__)
 with (THISDIR / 'resources' / 'ets-report.json').open() as fh:
     ETS_REPORT_SCHEMA = json.load(fh)
 
-with (THISDIR / 'resources' / '20250504_0-20008-0-NRB.json').open() as fh:
+with (THISDIR / 'resources' / '0-20008-0-THE.json').open() as fh:
     EXAMPLE_WMDR2 = json.load(fh)
 
 

pywmdr-0.2.dev3/pywmdr/resources/0-20008-0-THE.json

@@ -0,0 +1,176 @@
+{
+    "type": "Feature",
+    "id": "0-20008-0-THE",
+    "geometry": {
+        "type": "Point",
+        "coordinates": [
+            22.956,
+            40.634,
+            60
+        ]
+    },
+    "time": {
+        "interval": [
+            "1982-03-13",
+            ".."
+        ]
+    },
+    "conformsTo": [
+        "http://wigos.wmo.int/spec/wmdr/2/conf/core"
+    ],
+    "properties": {
+        "type": "facility",
+        "facilityType": {
+            "id": "landFixed"
+        },
+        "title": "Thessaloniki",
+        "created": "2024-12-11T00:00:00Z",
+        "updated": "2024-12-11T00:00:00Z",
+        "description": "The Brewer spectrophotometers and all radiation instruments are located on the roof of the 4-storey building of the Physics Dept., of the Aristotle University of Thessaloniki. The Campus is located in the city center about 500 m east of the seashore. The highest elevation of about 1100 m is at a distance of about 15 km towards the east. Thessaloniki is the second largest city of Greece with about 1000000 inhabitants in the greater metropolitan area.",
+        "contacts": [
+            {
+                "identifier": "contact:aristotle-university-of-thessaloniki--ed7b7e4a",
+                "organization": "Aristotle University of Thessaloniki",
+                "roles": [
+                    "owner"
+                ],
+                "links": [
+                    {
+                        "rel": "about",
+                        "href": "https://lapweb.physics.auth.gr",
+                        "type": "text/html"
+                    }
+                ]
+            },
+            {
+                "identifier": "contact:clima@hnms.gr",
+                "name": "Mrs M. Michelaraki",
+                "organization": "Hellenic National Meteorological Service",
+                "roles": [
+                    "pointOfContact"
+                ],
+                "links": [
+                    {
+                        "rel": "about",
+                        "href": "https://www.emy.gr",
+                        "type": "text/html"
+                    }
+                ]
+            },
+            {
+                "identifier": "contact:abais@auth.gr",
+                "name": "Alkiviadis Bais",
+                "organization": "Aristotle University of Thessaloniki",
+                "roles": [
+                    "principalInvestigator"
+                ],
+                "links": [
+                    {
+                        "rel": "about",
+                        "href": "https://lapweb.physics.auth.gr",
+                        "type": "text/html"
+                    }
+                ]
+            },
+            {
+                "identifier": "contact:balis@auth.gr",
+                "name": "Dimitris Balis",
+                "organization": "Aristotle University of Thessaloniki",
+                "roles": [
+                    "principalInvestigator"
+                ],
+                "links": [
+                    {
+                        "rel": "about",
+                        "href": "https://lapweb.physics.auth.gr",
+                        "type": "text/html"
+                    }
+                ]
+            }
+        ],
+        "wmoRegion": {
+            "id": "europe"
+        },
+        "territories": [
+            {
+                "dates": [
+                    "2011-02-16",
+                    ".."
+                ],
+                "territory": {"id": "GRC"}
+            }
+        ],
+        "observations": [
+            {
+                "observedProperty": {
+                    "id": "12164"
+                },
+                "programAffiliations": [
+                    {
+                        "dates": [
+                            "1982-03-13",
+                            ".."
+                        ],
+                        "programAffiliation": {
+                            "id": "GAWregional"
+                        },
+                        "reportingStatus": {
+                            "id": "operational"
+                        }
+                    },
+                    {
+                        "dates": [
+                            "2008-01-01",
+                            "2024-12-11"
+                        ],
+                        "programAffiliation": {
+                            "id": "GALION"
+                        },
+                        "reportingStatus": {
+                            "id": "operational"
+                        }
+                    },
+                    {
+                        "dates": [
+                            "2003-06-01",
+                            "2024-12-11"
+                        ],
+                        "programAffiliation": {
+                            "id": "AERONET"
+                        },
+                        "reportingStatus": {
+                            "id": "operational"
+                        }
+                    }
+                ],
+                "configurations": [
+                    {
+                        "id": "configuration-456",
+                        "time": {
+                            "interval": [
+                                "1982-03-13",
+                                ".."
+                            ]
+                        },
+                        "observingMethod": {
+                            "id": "78"
+                        },
+                        "sourceOfObservation": {
+                            "id": "automaticReading"
+                        },
+                        "instrument": {
+                            "id": "instrument-789"
+                        }
+                    }
+                ]
+            }
+        ]
+    },
+    "links": [
+        {
+            "href": "https://lapweb.physics.auth.gr/en/infrastructure/#Infrastructure",
+            "rel": "about",
+            "type": "text/html"
+        }
+    ]
+}

{pywmdr-0.2.dev1 → pywmdr-0.2.dev3}/pywmdr/resources/ets-report.json

@@ -1,5 +1,5 @@
 {
-    "$schema: https://json-schema.org/draft/2020-12/schema",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
     "$id": "https://raw.githubusercontent.com/wmo-im/pywmdr/refs/heads/main/pywmdr/resources/ets-report.json",
     "type": "object",
     "title": "WMDR2 Executable Test Suite report",

{pywmdr-0.2.dev1 → pywmdr-0.2.dev3}/pywmdr/wmdr2/ets.py

@@ -27,6 +27,7 @@ from jsonschema import FormatChecker
 from jsonschema.validators import Draft202012Validator
 
 from pywmdr.bundle import WMDR2_FILES
+from pywmdr.codelists import WMDSCodelists
 from pywmdr.errors import TestSuiteError
 from pywmdr.util import get_current_datetime_rfc3339, get_package_version
 
@@ -42,7 +43,7 @@ def gen_test_id(test_id: str) -> str:
     :returns: test identifier as URI
     """
 
-    return f'http://wis.wmo.int/spec/wmdr/2/conf/core/{test_id}'
+    return f'http://wigos.wmo.int/spec/wmdr/2/conf/core/{test_id}'
 
 
 class WMDR2TestSuite:
@@ -57,6 +58,7 @@ class WMDR2TestSuite:
 
         self.version = get_package_version()
         self.errors = []
+        self.codelists = WMDSCodelists()
         self.record = record
 
     def run_tests(self):
@@ -144,6 +146,73 @@ class WMDR2TestSuite:
 
         return status
 
+    def test_requirement_codelisted_values(self):
+        """
+        Test property values against required codelists
+        """
+
+        validation_errors = []
+
+        status = {
+            'id': gen_test_id('codelisted_values'),
+            'code': 'PASSED'
+        }
+
+        codelisted_values = []
+
+        LOGGER.debug('Validating against codelisted values')
+
+        codelisted_values.append(
+            ('facilityType', [self.record['properties']['facilityType']]))
+
+        codelisted_values.append(
+            ('wmoRegion', [self.record['properties']['wmoRegion']]))
+
+        for territory in self.record['properties'].get('territories', []):
+            codelisted_values.append(('territory', [territory['territory']]))
+
+        for observation in self.record['properties']['observations']:
+            codelisted_values.append(
+                ('observedGeometry', [observation.get('observedGeometry')]))
+            codelisted_values.append(
+                ('observedProperty', [observation['observedProperty']]))
+
+            for program_affiliation in observation.get('programAffiliations'):
+                codelisted_values.append(
+                    ('programAffiliation', [program_affiliation['programAffiliation']]))  # noqa
+                codelisted_values.append(
+                    ('reportingStatus', [program_affiliation.get('reportingStatus')]))  # noqa
+
+            for configuration in observation['configurations']:
+                codelisted_values.append(
+                    ('observingMethod', [configuration.get('observingMethod')]))  # noqa
+                codelisted_values.append(
+                    ('operatingStatus', [configuration.get('operatingStatus')]))  # noqa
+                codelisted_values.append(
+                    ('sourceOfObservation', [configuration.get('sourceOfObservation')]))  # noqa
+
+                vertical_distance = configuration.get('verticalDistance')
+
+                if vertical_distance is not None:
+                    codelisted_values.append(
+                        ('unit', [vertical_distance['unit']]))
+                    codelisted_values.append(
+                        ('referenceSurface', [vertical_distance['referenceSurface']]))  # noqa
+
+        for codelisted_value in codelisted_values:
+            LOGGER.debug(f'Testing {codelisted_value[0]} against {codelisted_value[1]}')  # noqa
+            for codelisted_value2 in codelisted_value[1]:
+                if not self.codelists.is_valid(codelisted_value[0], codelisted_value2):  # noqa
+                    msg = f'Invalid codelist value for {codelisted_value[0]}'
+                    validation_errors.append(msg)
+
+        if validation_errors:
+            status['code'] = 'FAILED'
+            status['message'] = f'{len(validation_errors)} error(s)'
+            status['errors'] = validation_errors
+
+        return status
+
     def raise_for_status(self):
         """
         Raise error if one or more failures were found during validation.

{pywmdr-0.2.dev1 → pywmdr-0.2.dev3/pywmdr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pywmdr
-Version: 0.2.dev1
+Version: 0.2.dev3
 Summary: A Python implementation of the test suite for WIGOS Metadata Record
 Author-email: Tom Kralidis <tomkralidis@gmail.com>
 Maintainer-email: Tom Kralidis <tomkralidis@gmail.com>

{pywmdr-0.2.dev1 → pywmdr-0.2.dev3}/pywmdr.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ pyproject.toml
 pywmdr/__init__.py
 pywmdr/bundle.py
 pywmdr/cli_options.py
+pywmdr/codelists.py
 pywmdr/errors.py
 pywmdr/ets.py
 pywmdr/pygeoapi_plugin.py
@@ -14,8 +15,9 @@ pywmdr.egg-info/dependency_links.txt
 pywmdr.egg-info/entry_points.txt
 pywmdr.egg-info/requires.txt
 pywmdr.egg-info/top_level.txt
-pywmdr/resources/20250504_0-20008-0-NRB.json
+pywmdr/resources/0-20008-0-THE.json
 pywmdr/resources/ets-report.json
 pywmdr/wmdr2/__init__.py
 pywmdr/wmdr2/ets.py
+tests/test_codelists.py
 tests/test_wmdr_ets.py

pywmdr-0.2.dev3/tests/test_codelists.py

@@ -0,0 +1,45 @@
+##############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+import pytest
+
+from pywmdr.codelists import WMDSCodelists
+
+
+@pytest.mark.parametrize("pname, pvalue, expected, raise_", [
+    ('unit', {'id': 'metres'}, False, False),
+    ('unit', None, True, False),
+    ('iunit', {'id': 'metres'}, False, True),
+    ('programAffiliation', {'id': 'GBON'}, True, False),
+    ('programAffiliation', {'id': '404'}, False, False)
+])
+def test_codelists_is_valid(pname, pvalue, expected, raise_):
+    """Simple tests for testing codelist value validity"""
+
+    codelists = WMDSCodelists()
+
+    if raise_:
+        with pytest.raises(ValueError):
+            assert codelists.is_valid(pname, pvalue)
+    elif expected:
+        assert codelists.is_valid(pname, pvalue)
+    else:
+        assert not codelists.is_valid(pname, pvalue)

{pywmdr-0.2.dev1 → pywmdr-0.2.dev3}/tests/test_wmdr_ets.py

@@ -28,11 +28,13 @@ from pywmdr.wmdr2.ets import WMDR2TestSuite
 from .util import get_test_file_path
 
 
-@pytest.mark.parametrize("filename, failed, passed, skipped, warnings_", [
-    # ('wmdr2-passing.json', 0, 1, 0, 0)
+@pytest.mark.parametrize("filename, passed, failed, skipped, warnings_", [
+    ('wmdr2-passing.json', 2, 0, 0, 0),
+    ('wmdr2-failing-invalid.json', 1, 1, 0, 0),
+    ('wmdr2-failing-codelisted-value.json', 1, 1, 0, 0)
 ])
-def test_ets(filename, failed, passed, skipped, warnings_):
-    """Simple tests for a passing record"""
+def test_ets(filename, passed, failed, skipped, warnings_):
+    """Simple tests for ETS validation"""
 
     with get_test_file_path(filename).open() as fh:
         data = json.load(fh)
@@ -45,7 +47,7 @@ def test_ets(filename, failed, passed, skipped, warnings_):
 
     codes = [r['code'] for r in results['tests']]
 
-    assert codes.count('FAILED') == failed
     assert codes.count('PASSED') == passed
+    assert codes.count('FAILED') == failed
     assert codes.count('SKIPPED') == skipped
     assert codes.count('WARNINGS') == warnings_

pywmdr-0.2.dev1/pywmdr/resources/20250504_0-20008-0-NRB.json

@@ -1,94 +0,0 @@
-{
-    "id": "urn:wmo:md:ke-meteo:0-20008-0-NRB",
-    "type": "Feature",
-    "conformsTo": [
-        "http://wis.wmo.int/spec/wmdr/2/conf/core"
-    ],
-    "geometry": {
-        "type": "Point",
-        "coordinates": [
-            36.75919,
-            -1.30169,
-            1795.0
-        ]
-    },
-    "properties": {
-        "created": "1996-01-01T00:00:00Z",
-        "wmo:dataPolicy": "core",
-        "contacts": [
-            {
-                "name": "World Meteorological Organization WMO and Federal Office for Meteorology and Climatology MeteoSwiss",
-                "organization": "World Meteorological Organization WMO and Federal Office for Meteorology and Climatology MeteoSwiss",
-                "emails": [
-                    {
-                        "value": "oscar@wmo.int"
-                    }
-                ],
-                "addresses": [
-                    {
-                        "deliveryPoint": [
-                            "7bis, avenue de la Paix"
-                        ],
-                        "city": "Geneva",
-                        "postalCode": "CH-1211",
-                        "country": "Switzerland"
-                    }
-                ],
-                "contactInstructions": "email",
-                "links": [
-                    {
-                        "rel": "canonical",
-                        "type": "text/html",
-                        "href": "https://oscar.wmo.int/surface"
-                    }
-                ],
-                "roles": [
-                    "host"
-                ]
-            },
-            {
-                "name": "Waweru,Amos,Mr",
-                "organization": "Kenyan Meteorological Department",
-                "phones": [
-                    {
-                        "value": "+254723521586"
-                    }
-                ],
-                "emails": [
-                    {
-                        "value": "waweru_k@yahoo.com"
-                    },
-                    {
-                        "value": "amoskamau1969@gmail.com"
-                    }
-                ],
-                "addresses": [
-                    {
-                        "deliveryPoint": [
-                            "Kenya Meteorological Department 30259\nNairobi \nKenya"
-                        ],
-                        "city": "Nairobi",
-                        "administrativeArea": "NB",
-                        "postalCode": "00100",
-                        "country": "Kenya"
-                    }
-                ],
-                "contactInstructions": "africa",
-                "roles": [
-                    "processor",
-                    "producer"
-                ]
-            }
-        ]
-    },
-    "links": [
-        {
-            "rel": "license",
-            "href": "https://creativecommons.org/licenses/by/4.0/"
-        },
-        {
-            "rel": "stations",
-            "href": "http://www.meteo.go.ke/obsv/ozone.html"
-        }
-    ]
-}