envlib 0.1.0__py3-none-any.whl

envlib/vocabularies/__init__.py

@@ -0,0 +1,336 @@
+"""Controlled vocabularies for envlib metadata fields.
+
+Each vocabulary is a bundled JSON file in this package directory. A user-level
+overlay directory (``~/.envlib/vocabularies/``) takes precedence per-file when
+present — :func:`refresh` writes there, never into the installed package.
+
+Vocabulary semantics:
+
+- Upstream-sourced lists (``variable`` membership, ``standard_name``) are
+  refreshable from their APIs via :func:`refresh`.
+- The envlib-curated ``(variable, feature) -> CF standard_name`` mapping inside
+  ``variable.json`` is hand-maintained and NEVER regenerated by refresh —
+  refresh only reports new/removed upstream terms for manual curation.
+- envlib-defined vocabularies (``feature``, ``method``, ``processing_level``,
+  ``aggregation_statistic``, ``frequency_interval``, ``license``) have no
+  upstream source; refresh does not touch them.
+"""
+
+from __future__ import annotations
+
+import builtins
+import datetime
+import json
+import pathlib
+import re
+import xml.etree.ElementTree as ET
+
+import urllib3
+
+BUNDLE_DIR = pathlib.Path(__file__).parent
+USER_DIR = pathlib.Path('~/.envlib/vocabularies').expanduser()
+
+FIELDS = (
+    'feature',
+    'variable',
+    'method',
+    'processing_level',
+    'aggregation_statistic',
+    'frequency_interval',
+    'license',
+    'standard_name',
+)
+
+ODM2_VARIABLENAME_URL = 'http://vocabulary.odm2.org/api/v1/variablename/?format=json'
+CF_STANDARD_NAME_XML_URL = 'https://cfconventions.org/Data/cf-standard-names/current/src/cf-standard-name-table.xml'
+
+_HTTP_OK = 200
+
+# Per-field parsed-JSON cache and derived lookup cache (lowercased input -> canonical value).
+_cache: dict = {}
+_lookup_cache: dict = {}
+
+
+def clear_cache():
+    """Clear the in-memory vocabulary caches (called automatically by :func:`refresh`)."""
+    _cache.clear()
+    _lookup_cache.clear()
+
+
+def _vocab_path(field: str) -> pathlib.Path:
+    user_path = USER_DIR / f'{field}.json'
+    if user_path.exists():
+        return user_path
+    return BUNDLE_DIR / f'{field}.json'
+
+
+def _check_field(field: str):
+    if field not in FIELDS:
+        msg = f'Unknown vocabulary field {field!r}; valid fields are {FIELDS}.'
+        raise ValueError(msg)
+
+
+def _load(field: str) -> dict:
+    _check_field(field)
+    if field not in _cache:
+        with open(_vocab_path(field), encoding='utf-8') as f:
+            _cache[field] = json.load(f)
+    return _cache[field]
+
+
+def _names(field: str) -> builtins.list[str]:
+    data = _load(field)
+    if field == 'standard_name':
+        return builtins.list(data['names'].keys())
+    if field == 'variable':
+        return builtins.list(data['entries'].keys())
+    return [entry['name'] for entry in data['entries']]
+
+
+def _lookup(field: str) -> dict:
+    """Map of lowercased accepted input -> canonical value (includes input aliases)."""
+    if field not in _lookup_cache:
+        table = {name.lower(): name for name in _names(field)}
+        if field == 'frequency_interval':
+            for entry in _load(field)['entries']:
+                for alias in entry.get('aliases', []):
+                    table[alias.lower()] = entry['name']
+        _lookup_cache[field] = table
+    return _lookup_cache[field]
+
+
+def list(field: str) -> builtins.list[str]:  # noqa: A001 - public API name set by the design plan
+    """Return the valid canonical values for a vocabulary field.
+
+    Args:
+        field: One of the envlib vocabulary fields (e.g. ``'variable'``, ``'feature'``).
+
+    Returns:
+        List of canonical values. For ``frequency_interval`` this is the canonical
+        codes only — input aliases (``'24h'``, ``'60min'``) are not included.
+    """
+    return _names(field)
+
+
+def canonical(field: str, value: str) -> str:
+    """Resolve user input to the canonical vocabulary value.
+
+    Matching is case-insensitive after whitespace stripping; ``frequency_interval``
+    additionally resolves the closed input-alias table (``'24h'`` -> ``'day'``,
+    ``'60min'`` -> ``'1h'``). Raises ``ValueError`` when the value is not in the
+    vocabulary.
+    """
+    _check_field(field)
+    if not isinstance(value, str):
+        msg = f'{field} value must be a str, not {type(value).__name__}.'
+        raise TypeError(msg)
+    key = value.strip().lower()
+    table = _lookup(field)
+    if key not in table:
+        msg = f'{value!r} is not a valid {field}; see envlib.vocabularies.list({field!r}).'
+        raise ValueError(msg)
+    return table[key]
+
+
+def is_valid(field: str, value: str) -> bool:
+    """Whether ``value`` is acceptable input for the vocabulary field (aliases included)."""
+    try:
+        canonical(field, value)
+    except (ValueError, TypeError):
+        return False
+    return True
+
+
+def frequency_entry(code: str) -> dict:
+    """Return the frequency_interval table entry for a canonical or alias code.
+
+    The entry dict has keys ``name`` (canonical code), ``kind`` (``'fixed'`` |
+    ``'calendar'``), ``seconds`` (int, or None for calendar codes), ``aliases``.
+    """
+    canon = canonical('frequency_interval', code)
+    for entry in _load('frequency_interval')['entries']:
+        if entry['name'] == canon:
+            return entry
+    msg = f'frequency_interval table entry missing for {canon!r} (corrupt vocabulary file?).'
+    raise RuntimeError(msg)
+
+
+def get_cf_standard_names(variable: str, feature: str) -> builtins.list[str] | None:
+    """CF standard_name candidates envlib considers for a (variable, feature) pair.
+
+    Args:
+        variable: envlib variable (CV member; raises ``ValueError`` if unknown).
+        feature: envlib feature (CV member; raises ``ValueError`` if unknown).
+
+    Returns:
+        Ordered candidate list — the FIRST entry is the curated default that
+        envlib auto-populates at validate/register time; an empty list means
+        "curated: no applicable standard name" (common for freshwater
+        water-quality variables); ``None`` means the pair is not yet curated
+        (envlib warns but does not block).
+    """
+    var = canonical('variable', variable)
+    feat = canonical('feature', feature)
+    entry = _load('variable')['entries'][var]
+    cf = entry.get('cf')
+    if cf is None:
+        return None
+    candidates = cf.get(feat)
+    if candidates is None:
+        return None
+    return [c['standard_name'] for c in candidates]
+
+
+###################################################
+# Refresh from upstream sources
+
+
+def _fetch(url: str) -> bytes:
+    resp = urllib3.request('GET', url, timeout=urllib3.Timeout(total=120))
+    if resp.status != _HTTP_OK:
+        msg = f'GET {url} returned HTTP {resp.status}.'
+        raise ConnectionError(msg)
+    return resp.data
+
+
+def _utc_today() -> str:
+    return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d')
+
+
+def _write_vocab(target_dir: pathlib.Path, field: str, data: dict):
+    path = target_dir / f'{field}.json'
+    with open(path, 'w', encoding='utf-8') as f:
+        json.dump(data, f, indent=1, ensure_ascii=False)
+        f.write('\n')
+
+
+def _snake_case(term: str) -> str:
+    s = re.sub(r'(?<=[a-z0-9])([A-Z])', r'_\1', term)
+    s = re.sub(r'_+', '_', s)
+    return s.lower()
+
+
+def _refresh_standard_name(target_dir: pathlib.Path) -> dict:
+    # The CF table ships as XML from cfconventions.org over HTTPS; content is not
+    # attacker-controlled in the threat model here, so stdlib parsing is acceptable.
+    root = ET.fromstring(_fetch(CF_STANDARD_NAME_XML_URL))  # noqa: S314
+    version = root.findtext('version_number')
+    last_modified = root.findtext('last_modified')
+    names = {}
+    for entry in root.findall('entry'):
+        names[entry.get('id')] = entry.findtext('canonical_units') or ''
+    aliases = {}
+    for alias in root.findall('alias'):
+        aliases[alias.get('id')] = alias.findtext('entry_id')
+    data = {
+        '_meta': {
+            'vocabulary': 'standard_name',
+            'source': f'CF standard name table v{version} ({CF_STANDARD_NAME_XML_URL})',
+            'cf_version': version,
+            'cf_last_modified': last_modified,
+            'updated': _utc_today(),
+        },
+        'names': dict(sorted(names.items())),
+        'aliases': dict(sorted(aliases.items())),
+    }
+    _write_vocab(target_dir, 'standard_name', data)
+    return {'cf_version': version, 'names': len(names), 'aliases': len(aliases)}
+
+
+def _refresh_variable(target_dir: pathlib.Path) -> dict:
+    current = json.loads(json.dumps(_load('variable')))  # deep copy of the effective file
+    entries = current['entries']
+    overrides = current.get('odm2_spelling_overrides', {})
+
+    payload = json.loads(_fetch(ODM2_VARIABLENAME_URL))
+    # The ODM2 API wraps results as {'meta': {...}, 'objects': [...]}.
+    objects = payload['objects'] if isinstance(payload, dict) else payload
+    if isinstance(payload, dict):
+        total = payload.get('meta', {}).get('total_count')
+        if total is not None and total > len(objects):
+            msg = f'ODM2 API returned {len(objects)} of {total} terms (pagination?); refusing a partial refresh.'
+            raise RuntimeError(msg)
+    fetched_terms = [item['term'] for item in objects]
+
+    added = []
+    mismatched = []
+    conflicts_with_extension = []
+    name_sources: dict = {}
+    for term in fetched_terms:
+        name = overrides.get(term, _snake_case(term))
+        name_sources.setdefault(name, []).append(term)
+        entry = entries.get(name)
+        if entry is None:
+            entries[name] = {'source': 'odm2', 'odm2_term': term}
+            added.append(name)
+        elif entry.get('source') == 'envlib':
+            # ODM2 grew a term colliding with an envlib extension — manual curation
+            # decides whether to promote the extension; refresh never edits it.
+            conflicts_with_extension.append({'name': name, 'odm2_term': term})
+        elif entry.get('odm2_term') is None:
+            entry['odm2_term'] = term
+        elif entry['odm2_term'] != term:
+            mismatched.append({'name': name, 'existing': entry['odm2_term'], 'fetched': term})
+
+    collisions = {name: terms for name, terms in name_sources.items() if len(terms) > 1}
+    fetched_set = set(fetched_terms)
+    removed = [
+        name
+        for name, entry in entries.items()
+        if entry.get('source') == 'odm2' and entry.get('odm2_term') not in fetched_set
+    ]
+
+    current['entries'] = dict(sorted(entries.items()))
+    current['_meta']['updated'] = _utc_today()
+    _write_vocab(target_dir, 'variable', current)
+    return {
+        'total': len(entries),
+        'added': sorted(added),
+        'removed': sorted(removed),  # reported only — refresh never deletes entries
+        'collisions': collisions,
+        'mismatched_odm2_terms': mismatched,
+        'conflicts_with_extension': conflicts_with_extension,
+    }
+
+
+def refresh(field: str | None = None, _target_dir=None) -> dict:
+    """Refresh upstream-sourced vocabularies from their APIs.
+
+    Fetches the ODM2 variablename list and/or the current CF standard-name table
+    and writes updated vocabulary files to the user overlay directory
+    (``~/.envlib/vocabularies/``) — never into the installed package. For
+    ``variable``, only ODM2-sourced membership is updated (additively — upstream
+    removals are reported, never deleted); curated ``cf`` mappings and envlib
+    extensions are never touched.
+
+    Args:
+        field: ``'variable'``, ``'standard_name'``, or None for both.
+        _target_dir: Internal/dev hook — write somewhere other than the user
+            overlay dir (used to generate the bundled files).
+
+    Returns:
+        Report dict keyed by refreshed field (counts, added/removed terms,
+        snake-case collisions, extension conflicts).
+    """
+    refreshable = ('standard_name', 'variable')
+    fields: tuple[str, ...]
+    if field is None:
+        fields = refreshable
+    elif field in refreshable:
+        fields = (field,)
+    else:
+        _check_field(field)
+        msg = f'{field!r} has no upstream source to refresh from.'
+        raise ValueError(msg)
+
+    target_dir = pathlib.Path(_target_dir).expanduser() if _target_dir is not None else USER_DIR
+    target_dir.mkdir(parents=True, exist_ok=True)
+
+    report = {}
+    for f in fields:
+        if f == 'standard_name':
+            report[f] = _refresh_standard_name(target_dir)
+        else:
+            report[f] = _refresh_variable(target_dir)
+    clear_cache()
+    return report

envlib/vocabularies/aggregation_statistic.json

@@ -0,0 +1,20 @@
+{
+  "_meta": {
+    "vocabulary": "aggregation_statistic",
+    "source": "statistical subset of the CF Conventions cell_methods vocabulary; no upstream refresh",
+    "updated": "2026-07-07"
+  },
+  "entries": [
+    {"name": "point", "description": "Instantaneous sampling; no aggregation over the interval."},
+    {"name": "mean", "description": "Arithmetic mean over the aggregation interval."},
+    {"name": "sum", "description": "Sum (accumulation) over the aggregation interval."},
+    {"name": "maximum", "description": "Maximum value within the aggregation interval."},
+    {"name": "minimum", "description": "Minimum value within the aggregation interval."},
+    {"name": "median", "description": "Median value within the aggregation interval."},
+    {"name": "mode", "description": "Most frequent value within the aggregation interval."},
+    {"name": "mid_range", "description": "Average of the maximum and minimum within the aggregation interval."},
+    {"name": "variance", "description": "Variance of values within the aggregation interval."},
+    {"name": "standard_deviation", "description": "Standard deviation of values within the aggregation interval."},
+    {"name": "range", "description": "Maximum minus minimum within the aggregation interval."}
+  ]
+}

envlib/vocabularies/feature.json

@@ -0,0 +1,19 @@
+{
+  "_meta": {
+    "vocabulary": "feature",
+    "source": "envlib-defined, mapped to ENVO URIs (looked up via EBI OLS4, 2026-07-07; envo_label records the ENVO term when it differs from the envlib name)",
+    "updated": "2026-07-07"
+  },
+  "entries": [
+    {"name": "atmosphere", "description": "Ambient air, weather, and climate.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_01000267"},
+    {"name": "waterway", "description": "Rivers, streams, and canals.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_00000029", "envo_label": "watercourse"},
+    {"name": "still_water", "description": "Lakes, reservoirs, and ponds.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_01000617", "envo_label": "lentic water body"},
+    {"name": "ocean", "description": "Open ocean and seas.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_00000015"},
+    {"name": "groundwater", "description": "Aquifers and subsurface water.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_01001004"},
+    {"name": "glacier", "description": "Glaciers and ice sheets.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_00000133"},
+    {"name": "wetland", "description": "Swamps, marshes, and bogs.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_01001209", "envo_label": "wetland ecosystem"},
+    {"name": "soil", "description": "Soil and subsurface earth.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_00001998"},
+    {"name": "coastline", "description": "Coastal zones and estuaries.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_00000303", "envo_label": "sea coast"},
+    {"name": "land", "description": "General land surface and terrain.", "envo_uri": "http://purl.obolibrary.org/obo/ENVO_01001785"}
+  ]
+}

envlib/vocabularies/frequency_interval.json

@@ -0,0 +1,21 @@
+{
+  "_meta": {
+    "vocabulary": "frequency_interval",
+    "source": "envlib-defined frequency codes (finalized 2026-07-05); no upstream refresh. Only canonical codes are stored, hashed, displayed, and query-matched; aliases are accepted input synonyms only. Admission rule for future fixed codes: the duration must divide 24 h evenly. None (not an entry) is the value for irregular cadences.",
+    "updated": "2026-07-07"
+  },
+  "entries": [
+    {"name": "1min", "kind": "fixed", "seconds": 60, "aliases": []},
+    {"name": "5min", "kind": "fixed", "seconds": 300, "aliases": []},
+    {"name": "10min", "kind": "fixed", "seconds": 600, "aliases": []},
+    {"name": "15min", "kind": "fixed", "seconds": 900, "aliases": []},
+    {"name": "30min", "kind": "fixed", "seconds": 1800, "aliases": []},
+    {"name": "1h", "kind": "fixed", "seconds": 3600, "aliases": ["60min"]},
+    {"name": "3h", "kind": "fixed", "seconds": 10800, "aliases": []},
+    {"name": "6h", "kind": "fixed", "seconds": 21600, "aliases": []},
+    {"name": "12h", "kind": "fixed", "seconds": 43200, "aliases": []},
+    {"name": "day", "kind": "fixed", "seconds": 86400, "aliases": ["24h"]},
+    {"name": "month", "kind": "calendar", "seconds": null, "aliases": []},
+    {"name": "year", "kind": "calendar", "seconds": null, "aliases": []}
+  ]
+}

envlib/vocabularies/license.json

@@ -0,0 +1,63 @@
+{
+  "_meta": {
+    "vocabulary": "license",
+    "source": "curated SPDX subset + envlib extensions for common non-SPDX open-access data licenses; open-access licenses only; no upstream refresh",
+    "updated": "2026-07-07"
+  },
+  "entries": [
+    {
+      "name": "CC-BY-4.0",
+      "title": "Creative Commons Attribution 4.0 International",
+      "source": "spdx",
+      "url": "https://spdx.org/licenses/CC-BY-4.0.html"
+    },
+    {
+      "name": "CC-BY-SA-4.0",
+      "title": "Creative Commons Attribution Share Alike 4.0 International",
+      "source": "spdx",
+      "url": "https://spdx.org/licenses/CC-BY-SA-4.0.html"
+    },
+    {
+      "name": "CC-BY-NC-4.0",
+      "title": "Creative Commons Attribution Non Commercial 4.0 International",
+      "source": "spdx",
+      "url": "https://spdx.org/licenses/CC-BY-NC-4.0.html"
+    },
+    {
+      "name": "CC-BY-NC-SA-4.0",
+      "title": "Creative Commons Attribution Non Commercial Share Alike 4.0 International",
+      "source": "spdx",
+      "url": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.html"
+    },
+    {
+      "name": "CC0-1.0",
+      "title": "Creative Commons Zero v1.0 Universal (public domain dedication)",
+      "source": "spdx",
+      "url": "https://spdx.org/licenses/CC0-1.0.html"
+    },
+    {
+      "name": "ODbL-1.0",
+      "title": "Open Data Commons Open Database License v1.0",
+      "source": "spdx",
+      "url": "https://spdx.org/licenses/ODbL-1.0.html"
+    },
+    {
+      "name": "CC-BY-3.0",
+      "title": "Creative Commons Attribution 3.0 Unported",
+      "source": "spdx",
+      "url": "https://spdx.org/licenses/CC-BY-3.0.html"
+    },
+    {
+      "name": "CC-BY-SA-3.0",
+      "title": "Creative Commons Attribution Share Alike 3.0 Unported",
+      "source": "spdx",
+      "url": "https://spdx.org/licenses/CC-BY-SA-3.0.html"
+    },
+    {
+      "name": "Copernicus-1.0",
+      "title": "Licence to Use Copernicus Products (ECMWF/Copernicus, e.g. ERA5, C3S/CAMS products)",
+      "source": "envlib",
+      "url": "https://apps.ecmwf.int/datasets/licences/copernicus/"
+    }
+  ]
+}

envlib/vocabularies/method.json

@@ -0,0 +1,37 @@
+{
+  "_meta": {
+    "vocabulary": "method",
+    "source": "envlib-defined (ported from tethys for migration continuity); no upstream refresh",
+    "updated": "2026-07-07"
+  },
+  "entries": [
+    {
+      "name": "derivation",
+      "description": "A method for creating results by simple calculations from field activities, sample analyses, or sensor recordings."
+    },
+    {
+      "name": "estimation",
+      "description": "A method for creating results by rough approximation or professional judgement. Does not use an analytical or numerical model."
+    },
+    {
+      "name": "field_activity",
+      "description": "A method for creating results by performing an activity in the field at or on a sampling feature. Includes manually-operated instruments (e.g., handheld probes, manual gauging, field surveys) - anything requiring a human at the measurement location."
+    },
+    {
+      "name": "simulation",
+      "description": "A method for creating results by running an analytical or numerical model. Generally more complex and/or more uncertain than derivations. Used for climate model runs, reanalyses (e.g., ERA5), and other model outputs."
+    },
+    {
+      "name": "sample_analysis",
+      "description": "A method for ex situ analysis of a sample using an instrument, typically in a laboratory, for the purpose of measuring properties of a sample."
+    },
+    {
+      "name": "sensor_recording",
+      "description": "A method for creating results by independent, automated sensor measurements without direct human interaction at the sensor during the measurement. Includes in-situ data loggers and remote sensing."
+    },
+    {
+      "name": "forecast",
+      "description": "A type of simulation to predict the future. Kept distinct from simulation so consumers can filter forecasts from historical model runs."
+    }
+  ]
+}

envlib/vocabularies/processing_level.json

@@ -0,0 +1,21 @@
+{
+  "_meta": {
+    "vocabulary": "processing_level",
+    "source": "envlib-defined; no upstream refresh",
+    "updated": "2026-07-07"
+  },
+  "entries": [
+    {
+      "name": "raw",
+      "description": "As collected or generated; no quality control applied."
+    },
+    {
+      "name": "preliminary",
+      "description": "Automatically screened or near-real-time; subject to revision (e.g., ERA5T, USGS provisional, telemetered data with automated checks)."
+    },
+    {
+      "name": "quality_controlled",
+      "description": "Vetted and considered settled (e.g., consolidated ERA5, USGS approved, a council's validated archive)."
+    }
+  ]
+}