libinsitu 1.7.1__py3-none-any.whl

libinsitu/__init__.py

@@ -0,0 +1,6 @@
+from .common import *
+from .catalog import *
+from .qc.qc_utils import *
+from .qc.graphs import *
+from .cli.transform import dataframe_to_netcdf
+from ._version import __version__

libinsitu/_version.py

@@ -0,0 +1,10 @@
+from importlib.metadata import version, PackageNotFoundError
+import sys
+
+if "pytest" in sys.modules:
+    __version__ = "TEST"
+else:
+    try:
+        __version__ = version("libinsitu")
+    except PackageNotFoundError:
+        __version__ = "DEV"

libinsitu/catalog.py

@@ -0,0 +1,155 @@
+# Handling fetch and parsing of Thredds server
+
+import xml.etree.ElementTree as ET
+
+from typing import Dict
+from urllib.parse import urljoin, urlsplit
+import pprint
+
+from requests import HTTPError, Session
+
+from libinsitu import parallel_map
+from libinsitu.log import info
+
+NS={
+    "thredds" : "http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0",
+    "xlink" : "http://www.w3.org/1999/xlink"}
+
+XLINK = "{http://www.w3.org/1999/xlink}"
+
+
+class AutoRepr :
+    def __repr__(self):
+        return pprint.pformat(self.__dict__)
+
+class Dataset(AutoRepr) :
+    def __init__(self, id, name, size) :
+        self.id = id
+        self.name = name
+        self.size = size
+        self.services = dict() # Dict of service type => URL
+
+class Catalog(AutoRepr) :
+    def __init__(self, id, name, description, url, link) :
+        self.id = id
+        self.name = name
+        self.description = description
+        self.url = url
+        self.link = link
+        self.authorized = None
+        self.datasets : Dict[str, Dataset]= dict()
+        self.catalogs : Dict[str, Catalog] = dict() # Dict of sub catalogs
+        self.services = dict() # Dict of service type => URL
+
+def parse_services(catalog, base):
+
+    res = dict()
+    for service in catalog.findall('.//thredds:service', NS):
+        name = service.get("name")
+        url = service.get("base")
+        if url != "":
+            res[name] = base + url
+    return res
+
+def parse_catalog(el, url, id=None, name=None) :
+    """Parse catalog from 'Dataset' element of root catalog or datasetREf/metadata element """
+
+    if id is None :
+        name = el.attrib['name']
+        id = el.attrib['ID']
+
+    # find doc
+    metadata = el.find('thredds:metadata', NS)  # Optionnal nested node "metadata"
+    if metadata is None:
+        metadata = el
+    docs = metadata.findall('thredds:documentation', NS)
+    description = None
+    link = None
+    for doc in docs:
+        if doc.get("type") == "summary":
+            description = doc.text
+        else:
+            link_ = doc.get('{%s}href' % NS["xlink"])
+            if link_ is not None:
+                link = link_
+    return Catalog(id, name, description, url, link)
+
+def extract_sub(url, subCatEl) :
+    id = subCatEl.attrib[XLINK + "title"]
+    href = subCatEl.attrib[XLINK + "href"]
+    sub_url = urljoin(url, href)
+    return (id, sub_url)
+
+def fetch_catalog(url, session=None, recursive=True, parallel=True) :
+
+    info("Fetching : %s" % url)
+
+    if session is None :
+        session = Session()
+
+    base = base_url(url)
+    xml = http_get(url, session)
+
+    catalogEl = ET.fromstring(xml)
+    datasetEl = catalogEl.find('thredds:dataset', NS)
+
+    catalog = parse_catalog(datasetEl, url)
+    catalog.services = parse_services(catalogEl, base)
+
+    # Parse or fetch sub catalogs
+
+    sub_elements = list(datasetEl.findall("thredds:catalogRef", NS))
+
+    def fetch_rec(subCatEl) :
+        id, sub_url = extract_sub(url, subCatEl)
+
+        try:
+            sub_catalog = fetch_catalog(sub_url, session, parallel=parallel)
+            return (id, sub_catalog)
+        except HTTPError as e:
+            if e.response.status_code == 401:
+                print("URL %s not authorized." % e.request.url)
+            else:
+                raise e
+
+    if recursive :
+        # Common loop for sequential of parallel fetch
+        for id, sub_catalog in parallel_map(fetch_rec, sub_elements, parallel):
+            catalog.catalogs[id] = sub_catalog
+    else :
+        for subCatEl in sub_elements :
+            id, sub_url = extract_sub(url, subCatEl)
+            catalog.catalogs[id] = parse_catalog(subCatEl, sub_url, id, id)
+
+    # Datasets
+    datasets = datasetEl.findall("thredds:dataset", NS)
+
+    for dataset_el in datasets :
+
+        uri = dataset_el.attrib["urlPath"]
+        id = dataset_el.attrib["ID"]
+        name = dataset_el.attrib["name"]
+        size = float(dataset_el.find("thredds:dataSize", NS).text)
+
+        dataset = Dataset(id, name, size)
+        dataset.services = {key: base + uri for key, base in catalog.services.items()}
+
+        catalog.datasets[dataset.name] = dataset
+
+    return catalog
+
+def base_url(url) :
+    parts = urlsplit(url)
+    return  "%s://%s" % (parts.scheme, parts.netloc)
+
+
+
+def http_get(url, session) :
+
+    #print("fetching : %s" % url)
+
+    res = session.get(url)
+    res.raise_for_status()
+
+    res.encoding = res.apparent_encoding
+    return res.text

libinsitu/cdl.py

@@ -0,0 +1,336 @@
+import re
+from copy import deepcopy
+from typing import Dict, List, Any
+from libinsitu import STATION_NAME_VAR, DefaultDict, read_str, readSingleVar
+from libinsitu.common import parse_value, DATA_VARS, read_res, CDL_PATH, LONGITUDE_VAR, LATITUDE_VAR, ELEVATION_VAR, \
+    fill_str, TIME_VAR
+from libinsitu.log import info, warning
+import numpy as np
+from enum import Enum
+
+VALUE_ATTR = "_value"
+FILL_VALUE_ATTR = "_FillValue"
+SYSTEM_ATTRIBUTES = [FILL_VALUE_ATTR, VALUE_ATTR]
+
+class Variable :
+    def __init__(self, name:str, type:str, dimensions:List[str]):
+        self.type = type
+        self.dimensions = dimensions
+        self.name = name
+        self.attributes = {}
+
+class CDL :
+    def __init__(self):
+        self.dimensions : Dict[str, int] = {}
+        self.variables : Dict[str, Variable] = {}
+        self.global_attributes = {}
+
+class CDLType:
+    CHAR = "char"
+    STRING = "string"
+    FLOAT = "float"
+    DOUBLE = "double"
+    INT = "int"
+    SHORT = "short"
+    UINT = "uint"
+
+class NetCDFType :
+    CHAR = "c"
+    STRING = str
+    FLOAT = "f4"
+    DOUBLE = "f8"
+    INT = "i4"
+    SHORT = "i2"
+    UINT = "u4"
+
+# CDL to NetCDF types
+CDL_TYPES_MAP = {
+    CDLType.CHAR : NetCDFType.CHAR,
+    CDLType.STRING: NetCDFType.STRING,
+    CDLType.FLOAT: NetCDFType.FLOAT,
+    CDLType.DOUBLE: NetCDFType.DOUBLE,
+    CDLType.INT: NetCDFType.INT,
+    CDLType.SHORT: NetCDFType.SHORT,
+    CDLType.UINT: NetCDFType.UINT}
+
+
+
+# Cache to CDL
+_CDL:CDL = None
+
+def init_cdl(properties=DefaultDict(lambda : "-"), custom_cdl = None) :
+    global _CDL
+    # Read CDL from resource or custom file
+    if custom_cdl is None:
+        cdl_file = read_res(CDL_PATH)
+    else:
+        info("Using custom CDL file %s" % custom_cdl)
+        cdl_file = open(custom_cdl, "r")
+    _CDL = parse_cdl(cdl_file, properties)
+
+def get_cdl(init=False) :
+    if _CDL is None:
+        if not init :
+            raise Exception("No CDL set yet")
+        else:
+            init_cdl(DefaultDict(lambda : "-"))
+    return _CDL
+
+
+
+def replace_placeholders(strval, attributes) :
+
+    def repl(m) :
+        key = m.group().strip("{").strip("}")
+
+        mandatory = False
+
+        if "!" in key :
+            mandatory = True
+            key = key.replace("!", "")
+
+        if not key in attributes :
+            if mandatory:
+                raise Exception(f"Missing  mandatory attribute '{key}'")
+            else:
+                warning("Key : '%s' not found in attributes, using empty string instead" % key)
+                return ""
+
+        res = attributes[key]
+        return "" if res is None else str(res)
+
+    return re.sub(r'{\!?\w+}', repl, strval)
+
+def parse_cdl(lines, attributes=dict()) :
+    """ Parse CDL file """
+
+    res = CDL()
+
+    section = None
+    curr_var = None
+
+    for line in lines :
+        line = line.strip()
+
+        # Skip comments
+        if line.startswith("#") or len(line) == 0:
+            continue
+
+        if "#" in line :
+            line = line.split("#")[0]
+            line = line.strip()
+
+        # key = value
+        if "=" in line :
+            line = line.strip(";")
+            key, val = line.split("=", 1)
+            key = key.strip()
+            val = parse_value(val.strip(), split=True)
+
+            if isinstance(val, str):
+                val = replace_placeholders(val.strip(), attributes)
+
+            # Defining dimension
+            if section == "dimensions" :
+
+                dim = 0 if val == "UNLIMITED" else int(val)
+                res.dimensions[key] = dim
+
+            elif section == "variables" :
+                varname, attrname = key.split(":")
+
+                if varname == "*" :
+                    varname = curr_var
+
+                if varname == "" :
+                    res.global_attributes[attrname] = val
+                else :
+                    res.variables[varname].attributes[attrname] = val
+            else :
+                raise Exception("Assignement outside any section : %s" % line)
+
+
+        # Skip start or end
+        elif "{" in line or "}" in line :
+            continue
+
+        # Change section
+        elif ":" in line :
+            section = line.strip(":").strip()
+            continue
+
+        elif ";" in line :
+            # New var
+            line = line.strip(";").strip()
+            type, var = line.split()
+
+            # Transform CDL type to NetCDF type
+            type = CDL_TYPES_MAP[type]
+
+            dims=[]
+            if "(" in var :
+                var, dims = var.split("(")
+                dims = dims.strip(")").strip().split(",")
+            res.variables[var] = Variable(var, type, dims)
+            curr_var = var
+        else :
+            raise Exception("Bad line : %s" %line)
+
+    return res
+
+def update_attributes(dest_var, attrs:Dict[str, Any], dry_run=False, delete=False) :
+
+    dry_prefix = "would " if dry_run else ""
+
+    existing_attrs = set(dest_var.ncattrs())
+    new_attrs = set(attrs.keys())
+
+    extra_attrs = existing_attrs - new_attrs
+
+    if delete :
+        for attrname in extra_attrs :
+            info(dry_prefix + "delete attribute %s#%s" % (dest_var.name, attrname))
+            if not dry_run :
+                dest_var.delncattr(attrname)
+
+    for key, val in attrs.items() :
+        oldval = None if not key in existing_attrs else dest_var.getncattr(key)
+
+        if key in SYSTEM_ATTRIBUTES :
+            # Do not update system attributes
+            continue
+
+        if oldval != val :
+
+            if (val is None or val == "") and not delete :
+                continue
+
+            info(dry_prefix + "update attribute %s#%s %s -> %s" % (dest_var.name, key, oldval, val))
+
+            if not dry_run:
+                dest_var.setncattr(key, val)
+
+def update_value(dest_var, vardef:Variable, dry_run=False):
+    """"Update single value from _value attribute"""
+
+    # No _value attribute ?
+    if not VALUE_ATTR in vardef.attributes :
+        return
+
+    value = vardef.attributes[VALUE_ATTR]
+
+    if len(vardef.dimensions) > 0:
+        raise Exception(f"Variable {vardef.name} : _value attribute only supported on non zero dimention variables")
+
+    if vardef.type == NetCDFType.STRING :
+        old_val = read_str(dest_var)
+    else:
+        old_val = readSingleVar(dest_var)
+
+    prefix = "would " if dry_run else ""
+    info(f'{prefix}replace value {old_val} => {value} in {vardef.name}')
+
+    if dry_run:
+        return
+
+    # Skip empty value
+    if value is None or value == "":
+        return
+
+    dest_var[0] = value
+
+
+
+def cmp_var(var,  vardef:Variable) :
+    return var.dtype == vardef.type and var.dimensions == tuple(vardef.dimensions)
+
+def create_or_replace_var(ncfile, vardef:Variable, dry_run=False) :
+
+    if vardef.name in ncfile.variables:
+
+        var = ncfile.variables[vardef.name]
+
+        if cmp_var(var, vardef):
+            # Same var -> skipping
+            return
+
+        # Different vars
+        if dry_run:
+            info("Would replace var : %s" % vardef.name)
+        else:
+            info("Replacing var : %s" % vardef.name)
+            del ncfile.variables[vardef.name]
+
+    if dry_run :
+        info("Would add variable : %s" % vardef.name)
+        return
+
+    least_significant_digit = vardef.attributes.get("least_significant_digit", None)
+    fill_value = vardef.attributes.get(FILL_VALUE_ATTR, None)
+
+    info("Adding variable '%s'. Precision:%s" %  (vardef.name, least_significant_digit))
+
+    ncfile.createVariable(
+        vardef.name, vardef.type, vardef.dimensions,
+        zlib=vardef.type is not str,
+        complevel=9,
+        least_significant_digit=least_significant_digit,
+        fill_value=fill_value)
+
+def initVar(ncfile, vardef:Variable, dry_run=False, delete_attrs=False) :
+
+    create_or_replace_var(ncfile, vardef, dry_run)
+
+    var = ncfile.variables[vardef.name]
+
+    # Update attributes
+    update_attributes(var, vardef.attributes, dry_run, delete_attrs)
+
+    # Update single value if any
+    update_value(var, vardef, dry_run=dry_run)
+
+def cdl2netcdf(ncfile, cdl: CDL, dry_run=False, delete_attrs=False) :
+    """Init NetCDF file from a CDL"""
+
+    for dimname, dim in cdl.dimensions.items() :
+
+        # Already there, skipping
+        if not dimname in ncfile.dimensions and not dry_run :
+            info("Adding dimension '%s'", dimname)
+            ncfile.createDimension(dimname, dim)
+
+    for varname, vardef in cdl.variables.items() :
+        initVar(ncfile, vardef, dry_run, delete_attrs)
+
+    # Update global attributes
+    update_attributes(ncfile, cdl.global_attributes, dry_run, delete_attrs)
+
+
+def init_nc(netcdf, properties, data_vars=DATA_VARS, dry_run=False, delete_attrs=False, custom_cdl=None) :
+
+    init_cdl(properties, custom_cdl)
+
+    cdl = get_cdl()
+
+    # Ensures all requested data vars are defined
+    missing_vars = set(data_var for data_var in data_vars if data_var not in cdl.variables)
+    if len(missing_vars) > 0 :
+        raise Exception("Unknown data vars : %s" % missing_vars)
+
+    filtered_cdl = deepcopy(cdl)
+
+    # Filter data vars (variables with "time" dimension)
+    # Also adds the "Time" variable
+    filtered_cdl.variables = dict()
+    for key, var in cdl.variables.items() :
+        if "time" in var.dimensions and not key in data_vars + [TIME_VAR] :
+            continue
+        filtered_cdl.variables[key] = var
+
+    cdl2netcdf(netcdf, filtered_cdl, dry_run, delete_attrs)
+
+    return cdl
+
+
+
+