cetk 0.0.4.dev0__py3-none-any.whl

cetk/edb/importers/activity_import.py

@@ -0,0 +1,279 @@
+"""import activities and emission-factors."""
+
+from django.db import IntegrityError
+
+from cetk.edb.cache import cache_queryset
+from cetk.edb.models import (
+    Activity,
+    AreaSource,
+    AreaSourceActivity,
+    EmissionFactor,
+    Facility,
+    GridSource,
+    GridSourceActivity,
+    PointSource,
+    PointSourceActivity,
+    Substance,
+)
+from cetk.edb.units import activity_ef_unit_to_si
+
+from .utils import import_error, worksheet_to_dataframe
+
+
+def import_emissionfactorsheet(workbook, validation):
+    return_dict = {}
+    return_message = []
+    activities = cache_queryset(
+        Activity.objects.prefetch_related("emissionfactors").all(), "name"
+    )
+    data = workbook["EmissionFactor"].values
+    df_activity = worksheet_to_dataframe(data)
+
+    activity_names = df_activity["activity_name"]
+    update_activities = {}
+    create_activities = {}
+    drop_emfacs = []
+    for row_nr, activity_name in enumerate(activity_names):
+        try:
+            activity = activities[activity_name]
+            if activity_name not in update_activities.keys():
+                if df_activity["activity_unit"][row_nr] != activity.unit:
+                    # check if all sources with this activity are updated.
+                    error_message = (
+                        "Changing activity rate unit for activity "
+                        f"{activity_name} without updating all sources"
+                        " in inventory with this activity. Define as new"
+                        " activity by giving new name, or update all"
+                        " sources with this activity simultaneously."
+                    )
+                    pointsource_ids = [
+                        psa.source_id
+                        for psa in PointSourceActivity.objects.filter(
+                            activity_id=activity.id
+                        )
+                    ]
+                    if len(pointsource_ids) > 0:
+                        pointsources = [
+                            PointSource.objects.get(id=i) for i in pointsource_ids
+                        ]
+                        facility_names = [
+                            Facility.objects.get(id=source.facility_id).name
+                            for source in pointsources
+                        ]
+                        source_names = [source.name for source in pointsources]
+                        if "PointSource" in workbook.sheetnames:
+                            new_pointsources = worksheet_to_dataframe(
+                                workbook["PointSource"].values
+                            )
+                            for facility, source in zip(facility_names, source_names):
+                                row_exists = (
+                                    (new_pointsources["source_name"] == source)
+                                    & (new_pointsources["facility_name"] == facility)
+                                ).any()
+                                if not row_exists:
+                                    return_message.append(
+                                        import_error(
+                                            error_message, validation=validation
+                                        )
+                                    )
+                        else:
+                            return_message.append(
+                                import_error(error_message, validation=validation)
+                            )
+                    areasource_ids = [
+                        psa.source_id
+                        for psa in AreaSourceActivity.objects.filter(
+                            activity_id=activity.id
+                        )
+                    ]
+                    if len(areasource_ids) > 0:
+                        areasources = [
+                            AreaSource.objects.get(id=i) for i in areasource_ids
+                        ]
+                        facility_names = [
+                            Facility.objects.get(id=source.facility_id).name
+                            for source in areasources
+                        ]
+                        source_names = [source.name for source in areasources]
+                        if "AreaSource" in workbook.sheetnames:
+                            new_areasources = worksheet_to_dataframe(
+                                workbook["AreaSource"].values
+                            )
+                            for facility, source in zip(facility_names, source_names):
+                                row_exists = (
+                                    (new_areasources["source_name"] == source)
+                                    & (new_areasources["facility_name"] == facility)
+                                ).any()
+                                if not row_exists:
+                                    return_message.append(
+                                        import_error(
+                                            error_message, validation=validation
+                                        )
+                                    )
+                        else:
+                            return_message.append(
+                                import_error(error_message, validation=validation)
+                            )
+                    gridsource_ids = [
+                        psa.source_id
+                        for psa in GridSourceActivity.objects.filter(
+                            activity_id=activity.id
+                        )
+                    ]
+                    if len(gridsource_ids) > 0:
+                        gridsources = [
+                            GridSource.objects.get(id=i) for i in gridsource_ids
+                        ]
+                        source_names = [source.name for source in gridsources]
+                        if "GridSource" in workbook.sheetnames:
+                            update_sources = [
+                                name
+                                in worksheet_to_dataframe(
+                                    workbook["GridSource"].values
+                                )["name"]
+                                for name in gridsources
+                            ]
+                            if not all(update_sources):
+                                return_message.append(
+                                    import_error(
+                                        error_message,
+                                        validation=validation,
+                                    )
+                                )
+                        else:
+                            return_message.append(
+                                import_error(error_message, validation=validation)
+                            )
+                    setattr(activity, "unit", df_activity["activity_unit"][row_nr])
+                update_activities[activity_name] = activity
+                drop_emfacs += list(activities[activity_name].emissionfactors.all())
+            elif (
+                df_activity["activity_unit"][row_nr]
+                != update_activities[activity_name].unit
+            ):
+                return_message.append(
+                    import_error(
+                        f"conflicting units for activity '{activity_name}'",
+                        validation=validation,
+                    )
+                )
+        except KeyError:
+            if activity_name not in create_activities:
+                activity = Activity(
+                    name=activity_name, unit=df_activity["activity_unit"][row_nr]
+                )
+                create_activities[activity_name] = activity
+            else:
+                if (
+                    df_activity["activity_unit"][row_nr]
+                    != create_activities[activity_name].unit
+                ):
+                    return_message.append(
+                        import_error(
+                            "multiple rows for the same activity " + str(activity_name),
+                            validation=validation,
+                        )
+                    )
+    Activity.objects.bulk_create(create_activities.values())
+    Activity.objects.bulk_update(
+        update_activities.values(),
+        [
+            "name",
+            "unit",
+        ],
+    )
+    # drop existing emfacs of activities that will be updated
+    EmissionFactor.objects.filter(pk__in=[inst.id for inst in drop_emfacs]).delete()
+    return_dict.update(
+        {
+            "activity": {
+                "updated": len(update_activities),
+                "created": len(create_activities),
+            }
+        }
+    )
+    data = workbook["EmissionFactor"].values
+    df_emfac = worksheet_to_dataframe(data)
+    substances = cache_queryset(Substance.objects.all(), "slug")
+    activities = cache_queryset(Activity.objects.all(), "name")
+    # unique together activity_name and substance
+    emissionfactors = cache_queryset(
+        EmissionFactor.objects.all(), ["activity", "substance"]
+    )
+    update_emfacs = []
+    create_emfacs = []
+    for row_nr in range(len(df_emfac)):
+        activity_name = df_emfac.iloc[row_nr]["activity_name"]
+        try:
+            activity = activities[activity_name]
+            subst = df_emfac.iloc[row_nr]["substance"]
+            try:
+                substance = substances[subst]
+                factor = df_emfac.iloc[row_nr]["factor"]
+                factor_unit = df_emfac.iloc[row_nr]["emissionfactor_unit"]
+                activity_quantity_unit, time_unit = activity.unit.split("/")
+                mass_unit, factor_quantity_unit = factor_unit.split("/")
+                if activity_quantity_unit != factor_quantity_unit:
+                    # emission factor and activity need to have the same unit
+                    # for quantity, eg GJ, m3 "pellets", number of produces bottles
+                    return_message.append(
+                        import_error(
+                            "Units for emission factor and activity rate for"
+                            f" '{activity_name}'"
+                            " are inconsistent, convert units before importing.",
+                            validation=validation,
+                        )
+                    )
+                else:
+                    factor = activity_ef_unit_to_si(factor, factor_unit)
+                try:
+                    emfac = emissionfactors[(activity, substance)]
+                    setattr(emfac, "activity", activity)
+                    setattr(emfac, "substance", substance)
+                    setattr(emfac, "factor", factor)
+                    update_emfacs.append(emfac)
+                except KeyError:
+                    emfac = EmissionFactor(
+                        activity=activity, substance=substance, factor=factor
+                    )
+                    create_emfacs.append(emfac)
+            except KeyError:
+                if subst == "PM2.5":
+                    substance = substances["PM25"]
+                else:
+                    return_message.append(
+                        import_error(
+                            f"unknown substance '{subst}'"
+                            f" for emission factor on row '{row_nr}'",
+                            validation=validation,
+                        )
+                    )
+        except KeyError:
+            return_message.append(
+                import_error(
+                    f"unknown activity '{activity_name}'"
+                    f" for emission factor on row '{row_nr}'",
+                    validation=validation,
+                )
+            )
+    try:
+        EmissionFactor.objects.bulk_create(create_emfacs)
+    except IntegrityError:
+        return_message.append(
+            import_error(
+                "Two emission factors for same activity and substance are given.",
+                validation=validation,
+            )
+        )
+    EmissionFactor.objects.bulk_update(
+        update_emfacs, ["activity", "substance", "factor"]
+    )
+    return_dict.update(
+        {
+            "emission_factors": {
+                "updated": len(update_emfacs),
+                "created": len(create_emfacs),
+            }
+        }
+    )
+    return return_dict, return_message

cetk/edb/importers/codeset_import.py

@@ -0,0 +1,124 @@
+from cetk.edb.models import ActivityCode, CodeSet, VerticalDist
+
+from .utils import import_error, worksheet_to_dataframe
+
+
+def import_codesetsheet(workbook, validation):
+    return_message = []
+    return_dict = {}
+    nr_codesets = CodeSet.objects.count()
+    data = workbook["CodeSet"].values
+    df_codeset = worksheet_to_dataframe(data)
+    slugs = df_codeset["slug"]
+    update_codesets = []
+    create_codesets = {}
+    for row_nr, slug in enumerate(slugs):
+        try:
+            codeset = CodeSet.objects.get(slug=slug)
+            setattr(codeset, "name", df_codeset["name"][row_nr])
+            setattr(codeset, "description", df_codeset["description"][row_nr])
+            update_codesets.append(codeset)
+        except CodeSet.DoesNotExist:
+            if nr_codesets + len(create_codesets) < 3:
+                codeset = CodeSet(
+                    name=df_codeset["name"][row_nr],
+                    slug=slug,
+                    description=df_codeset["description"][row_nr],
+                )
+                if slug not in create_codesets:
+                    create_codesets[slug] = codeset
+            else:
+                return_message.append(
+                    import_error(
+                        "Trying to import a new codeset," " but can have maximum 3.",
+                        validation=validation,
+                    )
+                )
+
+    CodeSet.objects.bulk_create(create_codesets.values())
+    CodeSet.objects.bulk_update(
+        update_codesets,
+        [
+            "name",
+            "description",
+        ],
+    )
+    return_dict = {
+        "codeset": {
+            "updated": len(update_codesets),
+            "created": len(create_codesets),
+        }
+    }
+    return return_dict, return_message
+
+
+def import_activitycodesheet(workbook, validation):
+    return_message = []
+    data = workbook["ActivityCode"].values
+    df_activitycode = worksheet_to_dataframe(data)
+    update_activitycodes = []
+    create_activitycodes = {}
+    for row_key, row in df_activitycode.iterrows():
+        row_dict = row.to_dict()
+        try:
+            codeset = CodeSet.objects.get(slug=row_dict["codeset_slug"])
+            if "vertical_distribution_slug" in row_dict:
+                if row_dict["vertical_distribution_slug"] is not None:
+                    try:
+                        vdist = VerticalDist.objects.get(
+                            slug=row_dict["vertical_distribution_slug"]
+                        )
+                        vdist_id = vdist.id
+                    except VerticalDist.DoesNotExist:
+                        return_message.append(
+                            import_error(
+                                f"Trying to import an activity-code "
+                                "from row '{row_key}' but "
+                                "Vertical Distribution "
+                                f"'{row_dict['vertical_distribution_slug']}'"
+                                " is not defined.",
+                                validation=validation,
+                            )
+                        )
+                else:
+                    vdist_id = None
+            else:
+                vdist_id = None
+            try:
+                activitycode = ActivityCode.objects.get(
+                    code_set_id=codeset.id, code=row_dict["activitycode"]
+                )
+                setattr(activitycode, "label", row_dict["label"])
+                setattr(activitycode, "vertical_dist_id", vdist_id)
+                update_activitycodes.append(activitycode)
+            except ActivityCode.DoesNotExist:
+                activitycode = ActivityCode(
+                    code=row_dict["activitycode"],
+                    label=row_dict["label"],
+                    code_set_id=codeset.id,
+                    vertical_dist_id=vdist_id,
+                )
+                create_activitycodes[row_dict["activitycode"]] = activitycode
+        except CodeSet.DoesNotExist:
+            return_message.append(
+                import_error(
+                    f"Trying to import an activity code from row '{row_key}'"
+                    f"but CodeSet '{row_dict['codeset_slug']}' is not defined.",
+                    validation=validation,
+                )
+            )
+    ActivityCode.objects.bulk_create(create_activitycodes.values())
+    ActivityCode.objects.bulk_update(
+        update_activitycodes,
+        [
+            "label",
+            "vertical_dist_id",
+        ],
+    )
+    return_dict = {
+        "activitycode": {
+            "updated": len(update_activitycodes),
+            "created": len(create_activitycodes),
+        }
+    }
+    return return_dict, return_message

cetk/edb/importers/gridsource_import.py

@@ -0,0 +1,256 @@
+import logging
+from os.path import dirname
+
+import numpy as np
+import pandas as pd
+import rasterio as rio
+from openpyxl import load_workbook
+
+from cetk.edb.cache import cache_queryset
+from cetk.edb.models import (
+    Activity,
+    GridSource,
+    Substance,
+    Timevar,
+    drop_gridsource_raster,
+    list_gridsource_rasters,
+    write_gridsource_raster,
+)
+from cetk.edb.units import activity_rate_unit_to_si, emission_unit_to_si
+
+from .utils import (
+    cache_codesets,
+    get_activity_rate_columns,
+    get_substance_emission_columns,
+    nan2None,
+    worksheet_to_dataframe,
+)
+from .validation import (
+    data_to_raster,
+    validate_activity,
+    validate_activitycodes,
+    validate_columns,
+    validate_emission,
+    validate_raster,
+    validate_timevar,
+    validate_unit,
+    with_rownr_and_substance,
+)
+
+REQUIRED_COLUMNS_GRID = ["name", "rastername", "timevar", "path", "emission_unit"]
+
+
+log = logging.getLogger(__name__)
+
+
+def read_csv(filepath, encoding=None):
+    with open(filepath, encoding=encoding or "utf-8") as csvfile:
+        log.debug("reading point-sources from csv-file")
+        df = pd.read_csv(
+            csvfile,
+            sep=";",
+            skip_blank_lines=True,
+            comment="#",
+            dtype=np._str,
+        )
+        for col in REQUIRED_COLUMNS_GRID:
+            if col not in df.columns:
+                raise ImportError(f"Missing required column '{col}'")
+    return df
+
+
+def read_xlsx(filepath):
+    try:
+        workbook = load_workbook(filename=filepath, data_only=True, read_only=True)
+    except Exception as exc:
+        raise ImportError(f"could not read workbook: {exc}")
+    if "GridSource" not in workbook:
+        raise ImportError("no 'GridSource' sheet found")
+
+    worksheet = workbook["GridSource"]
+    data = worksheet.values
+    df = worksheet_to_dataframe(data)
+    df = df.astype(dtype="string")
+    workbook.close()
+    return df
+
+
+def read_import_file(filepath, encoding=None):
+    """read import file."""
+    if filepath.suffix == ".csv":
+        df = read_csv(filepath, encoding=encoding)
+    elif filepath.suffix == ".xlsx":
+        df = read_xlsx(filepath)
+    else:
+        raise ImportError("only xlsx and csv files are supported for import")
+    if "name" not in df.columns:
+        raise ImportError("Missing column 'name' for in GridSource sheet")
+    df = df.set_index("name")
+    return df
+
+
+def validate_gridsources(df, timevars, code_sets, raster_names, datadir):
+    """validate grid-sources in dataframe but do not write to database.
+    returns (rasters, messages)
+    where rasters is a nested dict with all gridsource rasters
+    and messages is a list of all validation error messages.
+    """
+    row_nr = 2
+    rasters = {}
+    messages = []
+    act_cols = get_activity_rate_columns(df)
+    subst_cols = get_substance_emission_columns(df)
+    for _, row in df.iterrows():
+        row_dict = nan2None(row.to_dict())
+        messages += validate_activitycodes(row_dict, code_sets, row_nr=row_nr)
+        messages += validate_timevar(row_dict, timevars, row_nr=row_nr)
+        messages += validate_unit(row_dict, row_nr)
+        for col in filter(lambda x: row_dict[x] is not None, subst_cols):
+            subst_slug = col[6:]
+            messages += validate_raster(
+                row_dict, raster_names, datadir, rasters, row_nr, subst_slug
+            )
+            messages += validate_emission(row_dict, row_nr, subst_slug)
+            # if sum is specified instead of an emission total, the emission value
+            # is calculated as the sum of the emission raster
+            if row_dict[col] == "sum" and row_dict["path"] is None:
+                messages.append(
+                    with_rownr_and_substance(
+                        "raster path required for emission specified by 'sum'",
+                        row_nr,
+                        subst_slug,
+                    )
+                )
+        for col in filter(lambda x: row_dict[x] is not None, act_cols):
+            act_name = col[4:].strip()
+            messages += validate_raster(
+                row_dict, raster_names, datadir, rasters, row_nr, subst_slug
+            )
+            messages += validate_activity(row_dict, row_nr, act_name)
+            # if sum is specified instead of an emission total, the emission value
+            # is calculated as the sum of the emission raster
+            if row_dict[col] == "sum" and row_dict["path"] is None:
+                messages.append(
+                    with_rownr_and_substance(
+                        "raster path required for activity rate specified by 'sum'",
+                        row_nr,
+                        act_name,
+                    )
+                )
+        row_nr += 1
+    return rasters, messages
+
+
+def create_or_update_rasters(rasters, raster_names):
+    """
+    Write all rasters with a path specified to the database.
+    Existing rasters will be replaced.
+    """
+    for raster_name, raster_data in rasters.items():
+        if raster_data["path"] is not None:
+            if raster_name in raster_names:
+                drop_gridsource_raster(raster_name)
+            with rio.open(raster_data["path"]) as raster:
+                write_gridsource_raster(raster, raster_name)
+
+
+def import_gridsources(filepath, encoding=None):
+    """validate and/or import grid-sources from file."""
+
+    substances = cache_queryset(Substance.objects.all(), "slug")
+    timevars = cache_queryset(Timevar.objects.all(), "name")
+    code_sets = cache_codesets()
+    activities = cache_queryset(Activity.objects.all(), "name")
+    raster_names = list_gridsource_rasters()
+    datadir = dirname(filepath)
+
+    try:
+        df = read_import_file(filepath, encoding)
+    except ImportError as err:
+        return {}, [f"{err}"]
+
+    messages = []
+    messages += validate_columns(
+        df,
+        REQUIRED_COLUMNS_GRID,
+        substances=substances,
+        activities=activities,
+        code_sets=code_sets,
+    )
+    if len(messages) > 0:
+        messages += ["Could not validate gridsources, due to error in columns."]
+        return {}, messages
+
+    row_nr = 2
+    rasters, messages_sources = validate_gridsources(
+        df, timevars, code_sets, raster_names, datadir
+    )
+    messages += messages_sources
+    if len(messages) > 0:
+        return {}, messages
+
+    create_or_update_rasters(rasters, raster_names)
+    nr_created_sources = 0
+    nr_updated_sources = 0
+
+    subst_cols = get_substance_emission_columns(df)
+    act_cols = get_activity_rate_columns(df)
+    for name, row in df.iterrows():
+        row_dict = nan2None(row.to_dict())
+        src, created = GridSource.objects.get_or_create(name=name)
+        if created:
+            nr_created_sources += 1
+        else:
+            nr_updated_sources += 1
+        # set tags
+        tag_keys = [key for key in row_dict.keys() if key.startswith("tag:")]
+        src.tags = {
+            key[4:]: row_dict[key] for key in tag_keys if row_dict[key] is not None
+        }
+        validate_activitycodes(row_dict, code_sets, row_nr, src)
+        validate_timevar(row_dict, timevars, row_nr, src)
+        src.save()
+
+        # remove any existing emissions
+        if not created:
+            src.substances.all().delete()
+            src.activities.all().delete()
+
+        for col in filter(lambda x: row_dict[x] is not None, subst_cols):
+            subst = col[6:]
+            # if sum is specified instead of an emission total,
+            # the emission value is calculated as the sum of the raster
+            rname, rpath = data_to_raster(
+                row_dict["rastername"], row_dict["path"], datadir, subst
+            )
+            emis = {"substance": substances[subst], "raster": rname}
+            emis_value = row_dict[col]
+            unit = row_dict["emission_unit"]
+            if emis_value == "sum":
+                emis["value"] = emission_unit_to_si(rasters[rname]["sum"], unit)
+            else:
+                emis["value"] = emission_unit_to_si(float(emis_value), unit)
+            src.substances.create(**emis)
+
+        for col in filter(lambda x: row_dict[x] is not None, act_cols):
+            activity_name = col[4:]
+            rname, rpath = data_to_raster(
+                row_dict["rastername"], row_dict["path"], datadir, subst
+            )
+            emis = {"activity": activities[activity_name], "raster": rname}
+            rate = row_dict[col]
+            if rate == "sum":
+                emis["rate"] = activity_rate_unit_to_si(
+                    rasters[rname]["sum"], emis["activity"].unit
+                )
+            else:
+                emis["rate"] = activity_rate_unit_to_si(
+                    float(rate), emis["activity"].unit
+                )
+            src.activities.create(**emis)
+
+        row_nr += 1
+    db_updates = {
+        "gridsources": {"updated": nr_updated_sources, "created": nr_created_sources}
+    }
+    return db_updates, messages