cetk 0.0.4.dev0__py3-none-any.whl

cetk/edb/importers/timevar_import.py

@@ -0,0 +1,115 @@
+import numpy as np
+from django.db import IntegrityError
+
+from cetk.edb.models import ColdstartTimevar, FlowTimevar, Timevar
+
+from .utils import import_error, worksheet_to_dataframe
+
+
+def import_timevarsheet(workbook, validation=False, sheetname="Timevar"):
+    return_message = []
+    timevar_data = workbook[sheetname].values
+    df_timevar = worksheet_to_dataframe(timevar_data)
+    if sheetname == "Timevar":
+        keyname = "emission"
+    if sheetname == "FlowTimevar":
+        keyname = "flow"
+    if sheetname == "ColdstartTimevar":
+        keyname = "coldstart"
+    timevar_dict = {keyname: {}}
+
+    # NB this only works if Excel file has exact same format
+    nr_timevars = (len(df_timevar["ID"]) + 1) // 27
+    for i in range(nr_timevars):
+        label = df_timevar["ID"][i * 27]
+        typeday = np.asarray(
+            df_timevar[
+                [
+                    "monday",
+                    "tuesday",
+                    "wednesday",
+                    "thursday",
+                    "friday",
+                    "saturday",
+                    "sunday",
+                ]
+            ][i * 27 : i * 27 + 24]
+        )
+        month = np.asarray(df_timevar.iloc[i * 27 + 25, 2:14])
+        typeday_str = np.array2string(typeday).replace("\n", "").replace(" ", ", ")
+        month_str = np.array2string(month).replace("\n", "").replace(" ", ", ")
+        timevar_dict[keyname].update(
+            {label: {"typeday": typeday_str, "month": month_str}}
+        )
+    tv, return_append = import_timevars(
+        timevar_dict, overwrite=True, validation=validation
+    )
+    return_message += return_append
+    return_dict = {"timevar": {"updated or created": len(tv[keyname])}}
+    return return_dict, return_message
+
+
+def import_timevars(timevar_data, overwrite=False, validation=False):
+    """import time-variation profiles."""
+
+    # Timevar instances must not be created by bulk_create as the save function
+    # is overloaded to calculate the normation constant.
+    def make_timevar(data, timevarclass, subname=None, validation=False):
+        retdict = {}
+        for name, timevar_data in data.items():
+            try:
+                typeday = timevar_data["typeday"]
+                month = timevar_data["month"]
+                if type(typeday) == list:
+                    typeday = str(typeday)
+                if type(month) == list:
+                    month = str(month)
+                if overwrite:
+                    newobj, _ = timevarclass.objects.update_or_create(
+                        name=name,
+                        defaults={"typeday": typeday, "month": month},
+                    )
+                else:
+                    try:
+                        newobj = timevarclass.objects.create(
+                            name=name, typeday=typeday, month=month
+                        )
+                    except IntegrityError:
+                        raise IntegrityError(
+                            f"{timevarclass.__name__} {name} "
+                            f"already exist in inventory."
+                        )
+                retdict[name] = newobj
+            except KeyError:
+                return_message.append(
+                    import_error(
+                        f"Invalid specification of timevar {name}"
+                        f", are 'typeday' and 'month' given?",
+                        validation=validation,
+                    )
+                )
+        return retdict
+
+    return_message = []
+    timevars = {}
+    for vartype, subdict in timevar_data.items():
+        if vartype == "emission":
+            timevars.setdefault("emission", {}).update(
+                make_timevar(timevar_data[vartype], Timevar)
+            )
+        elif vartype == "flow":
+            timevars.setdefault("flow", {}).update(
+                make_timevar(timevar_data[vartype], FlowTimevar)
+            )
+        elif vartype == "coldstart":
+            timevars.setdefault("coldstart", {}).update(
+                make_timevar(timevar_data[vartype], ColdstartTimevar)
+            )
+        else:
+            return_message.append(
+                import_error(
+                    f"invalid time-variation type '{vartype}' specified",
+                    validation=validation,
+                )
+            )
+    return timevars, return_message

cetk/edb/importers/utils.py

@@ -0,0 +1,92 @@
+import pandas as pd
+
+from cetk.edb.cache import cache_queryset
+from cetk.edb.models import Settings
+
+
+class ValidationError(Exception):
+    """Error while validating emission data."""
+
+    pass
+
+
+class EmptySheet(Exception):
+    """Error trying to read data from an empty sheet."""
+
+    pass
+
+
+def nan2None(d):
+    out = d.copy()
+    for k, val in out.items():
+        out[k] = val if pd.notna(k) else val
+    return out
+
+
+def import_error(message, return_message="", validation=False):
+    """import error management"""
+    if not validation:
+        raise ValidationError(f"VALIDATION: {message}")
+    else:
+        return_message += f"VALIDATION: {message}\n"
+    return return_message
+
+
+def import_row_error(msg, row_nr, validation=False):
+    """import error managment with row nr"""
+    return import_error(f"{msg}, on row {row_nr}", validation=validation)
+
+
+def import_row_substance_error(msg, row_nr, substance, validation=False):
+    """import error managment with row nr"""
+    return import_error(
+        f"{msg}, for '{substance}' on row {row_nr}", validation=validation
+    )
+
+
+def cache_codeset(code_set):
+    """return dict {code: activitycode} for codeset."""
+    if code_set is None:
+        return {}
+    return cache_queryset(code_set.codes.all(), "code")
+
+
+def cache_codesets():
+    """
+    return list of dictionaries with activity-codes
+    for all code-sets in Settings.
+    """
+    settings = Settings.get_current()
+    code_sets = {}
+    if settings.codeset1 is not None:
+        code_sets[settings.codeset1.slug] = cache_codeset(settings.codeset1)
+    if settings.codeset2 is not None:
+        code_sets[settings.codeset2.slug] = cache_codeset(settings.codeset2)
+    if settings.codeset3 is not None:
+        code_sets[settings.codeset3.slug] = cache_codeset(settings.codeset3)
+    return code_sets
+
+
+def worksheet_to_dataframe(data):
+    data = list(data)
+    if not data:
+        raise EmptySheet("Sheet is empty")
+    df = pd.DataFrame(data)
+    # Set the first row as the header
+    df.columns = df.iloc[0]
+    df = df[1:].reset_index(drop=True)
+    # Remove completely empty rows
+    df = df.dropna(how="all")
+    # Remove completely empty columns without a header
+    df = df.loc[:, ~(df.isna().all() & df.columns.isna())]
+    return df
+
+
+def get_substance_emission_columns(df):
+    """return columns with substance emissions."""
+    return [col for col in df.columns if col.startswith("subst:")]
+
+
+def get_activity_rate_columns(df):
+    """return columns with activity rates."""
+    return [col for col in df.columns if col.startswith("act:")]

cetk/edb/importers/validation.py

@@ -0,0 +1,207 @@
+import os
+from os.path import basename, splitext
+
+import pandas as pd
+import rasterio as rio
+
+from cetk.edb.units import emission_unit_to_si
+
+
+def with_rownr(msg, row_nr):
+    return f"{msg}, at row {row_nr}"
+
+
+def with_rownr_and_substance(msg, row_nr, substance):
+    return f"{msg}, for '{substance}' at row {row_nr}"
+
+
+def validate_columns(
+    df, required_cols, code_sets=None, substances=None, activities=None
+):
+    messages = []
+    missing_cols = [
+        col for col in required_cols if col not in df.columns and df.index.name != col
+    ]
+    if len(missing_cols) > 0:
+        messages.append(
+            f"GridSource: Missing required columns: '{', '.join(missing_cols)}'"
+        )
+
+    if code_sets is not None:
+        for codeset_slug in code_sets:
+            col_name = f"activitycode_{codeset_slug}"
+            if col_name not in df.columns:
+                messages.append(
+                    f"GridSource: Missing column for code-set '{codeset_slug}'"
+                )
+    if substances is not None:
+        expected_substances = {
+            col[6:] for col in df.columns if col.startswith("subst:")
+        }
+        invalid_substances = expected_substances.difference(substances.keys())
+        if len(invalid_substances) > 0:
+            messages.append(f"GridSource: Invalid substances: {invalid_substances}")
+
+    if activities is not None:
+        expected_activities = {
+            col[4:].strip() for col in df.columns if col.startswith("act:")
+        }
+        invalid_activities = expected_activities.difference(activities.keys())
+        if len(invalid_activities) > 0:
+            messages.append(f"GridSource: Invalid activities: {invalid_activities}")
+    return messages
+
+
+def validate_unit(values, row_nr):
+    """validate substance emission unit."""
+    messages = []
+    unit = values["emission_unit"]
+    try:
+        emission_unit_to_si(1.0, unit)
+    except (KeyError, ValueError) as err:
+        messages.append(with_rownr(f"GridSource: invalid unit, {err}", row_nr))
+    return messages
+
+
+def validate_activitycodes(values, code_sets, row_nr, src=None):
+    """validate and set activity codes."""
+    messages = []
+    # get activitycodes
+    for code_ind, (codeset_slug, codes) in enumerate(code_sets.items(), 1):
+        code_attribute = f"activitycode{code_ind}"
+        code_col = f"activitycode_{codeset_slug}"
+        code = values[code_col]
+        if code is None:
+            messages.append(f"GridSource: missing value in column '{code_col}'")
+        try:
+            activitycode = codes[code]
+        except KeyError:
+            messages.append(f"GridSource: unknown code '{code}' in column '{code_col}'")
+        if src is not None:
+            setattr(src, code_attribute, activitycode)
+    return [with_rownr(msg, row_nr) for msg in messages]
+
+
+def validate_timevar(values, timevars, row_nr, src=None):
+    """validate and set timevar."""
+    timevar_name = values["timevar"]
+    messages = []
+    if pd.notna(timevar_name):
+        try:
+            timevar = timevars[timevar_name]
+        except KeyError:
+            messages.append(f"GridSource: timevar '{timevar_name}' does not exist")
+    else:
+        timevar = None
+    if src is not None:
+        src.timevar = timevar
+    return [with_rownr(msg, row_nr) for msg in messages]
+
+
+def data_to_raster(raster_name, raster_path, datadir, substance):
+    if raster_path is not None:
+        # generate raster name form path
+        if raster_name is None:
+            raster_name = splitext(basename(raster_path))[0]
+        # if placeholder is used for substance in path
+        # the raster-name will be given a suffix
+
+        if "{subst}" in raster_path:
+            if "{subst}" not in raster_name:
+                raster_name += "-{subst}"
+            rpath = raster_path.format(subst=substance)
+            rname = raster_name.format(subst=substance)
+        else:
+            rpath = raster_path
+            rname = raster_name
+
+        if not os.path.isabs(rpath):
+            rpath = os.path.join(datadir, rpath)
+    else:
+        rpath = None
+        rname = (
+            raster_name.format(subst=substance)
+            if "{subst}" in raster_name
+            else raster_name
+        )
+    return rname, rpath
+
+
+def validate_raster(
+    values: dict,
+    raster_names: list,
+    datadir: str,
+    raster_dict: dict,
+    row_nr: int,
+    substance: str,
+):
+    messages = []
+    rasters_without_path = []
+    raster_path = values["path"] if not pd.isna(values["path"]) else None
+    raster_name = values["rastername"] if not pd.isna(values["rastername"]) else None
+    if raster_name is None and raster_path is None:
+        messages.append("neither rastername nor path specified")
+    rname, rpath = data_to_raster(raster_name, raster_path, datadir, substance)
+    if rpath is not None:
+        if not os.path.exists(rpath):
+            messages.append(f"raster with path {rpath} does not exist")
+        elif rname not in raster_dict:
+            with rio.open(rpath, "r") as dset:
+                try:
+                    _ = int(dset.crs.to_authority()[1])
+                except ValueError:
+                    messages.append(
+                        "raster coordinate reference system has no epsg code"
+                    )
+                rdata = dset.read()
+            raster_dict.setdefault(rname, {})["sum"] = rdata[rdata != dset.nodata].sum()
+            raster_dict[rname]["path"] = rpath
+    else:
+        rasters_without_path.append(rname)
+
+    for rname in rasters_without_path:
+        if rname not in raster_names and rname not in raster_dict:
+            messages.append(
+                f"raster '{rname}' not found in database, and no path specified"
+            )
+
+    return [with_rownr_and_substance(msg, row_nr, substance) for msg in messages]
+
+
+def validate_emission(values, row_nr, substance):
+    """validate emission value"""
+    messages = []
+    emis_value = values[f"subst:{substance}"]
+    if emis_value != "sum":
+        try:
+            val = float(emis_value)
+            if val < 0:
+                messages.append(
+                    with_rownr_and_substance("emission value < 0", row_nr, substance)
+                )
+        except ValueError:
+            messages.append(
+                with_rownr_and_substance(
+                    f"Invalid emission value '{emis_value}'", row_nr, substance
+                )
+            )
+    return messages
+
+
+def validate_activity(values, row_nr, activity):
+    """validate emission value"""
+    messages = []
+    rate = values[f"act:{activity}"]
+    if rate != "sum":
+        try:
+            if float(rate) < 0:
+                messages.append(
+                    with_rownr_and_substance("activity rate < 0", row_nr, activity)
+                )
+        except ValueError:
+            messages.append(
+                with_rownr_and_substance(
+                    f"invalid activity rate '{rate}'", row_nr, activity
+                )
+            )
+    return messages

cetk/edb/ltreefield.py

@@ -0,0 +1,76 @@
+"""A django field definition for Postgres hierarchical ltree field.
+
+Source: https://github.com/whitglint/ltreefield
+license: MIT, https://github.com/whitglint/ltreefield/blob/master/LICENSE
+"""
+
+from django.db import models
+from django.db.models import Lookup
+
+
+class LtreeField(models.CharField):
+    description = "SQLite alternative to ltree (up to %(max_length)s)"
+
+    def __init__(self, *args, **kwargs):
+        kwargs["max_length"] = 256
+        super().__init__(*args, **kwargs)
+
+    # def db_type(self, connection):
+    #    return "ltree"
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        del kwargs["max_length"]
+        return name, path, args, kwargs
+
+
+class AncestorOrEqual(Lookup):
+    lookup_name = "aore"
+
+    def as_sql(self, qn, connection):
+        lhs, lhs_params = self.process_lhs(qn, connection)
+        rhs, rhs_params = self.process_rhs(qn, connection)
+        params = lhs_params + rhs_params
+        if len(rhs_params) > 1:
+            raise ValueError("This alternative for LTree is not implemented")
+        elif len(rhs_params[0]) > 2:
+            return (
+                '%s like %s or %s like %s or %s like ""  '
+                % (lhs, rhs, lhs, rhs_params[0][:-2], lhs),
+                params,
+            )
+        else:
+            return '%s like %s or %s like "" ' % (lhs, rhs, lhs), params
+
+
+LtreeField.register_lookup(AncestorOrEqual)
+
+
+class DescendantOrEqual(Lookup):
+    lookup_name = "dore"
+
+    def as_sql(self, qn, connection):
+        lhs, lhs_params = self.process_lhs(qn, connection)
+        rhs, rhs_params = self.process_rhs(qn, connection)
+        params = lhs_params + rhs_params
+        if len(params) > 1:
+            raise ValueError("This alternative for LTree is not implemented")
+        else:
+            params = [params[0] + "%"]
+            return "%s like %s" % (lhs, rhs), params
+
+
+LtreeField.register_lookup(DescendantOrEqual)
+
+
+class Match(Lookup):
+    lookup_name = "match"
+
+    def as_sql(self, qn, connection):
+        lhs, lhs_params = self.process_lhs(qn, connection)
+        rhs, rhs_params = self.process_rhs(qn, connection)
+        params = lhs_params + rhs_params
+        return "%s like %s" % (lhs, rhs), params
+
+
+LtreeField.register_lookup(Match)