cetk 0.0.4.dev0__py3-none-any.whl

cetk/emissions/calc.py

@@ -0,0 +1,148 @@
+"""Calculate emissions."""
+
+import pandas as pd
+from django.db import connection
+
+from cetk.edb.const import DEFAULT_EMISSION_UNIT
+from cetk.edb.models import CodeSet, Settings, Substance
+from cetk.edb.units import emis_conversion_factor_from_si
+from cetk.emissions.queries import (
+    create_aggregate_emis_query,
+    create_source_emis_query,
+    create_used_substances_query,
+)
+
+
+def get_used_substances():
+    """return list of substances with emissions or emission factors."""
+    sql = create_used_substances_query()
+    cur = connection.cursor()
+    return [Substance.objects.get(slug=rec[0]) for rec in cur.execute(sql).fetchall()]
+
+
+def calculate_source_emissions(
+    sourcetype,
+    substances=None,
+    srid=None,
+    name=None,
+    ids=None,
+    tags=None,
+    polygon=None,
+    unit=DEFAULT_EMISSION_UNIT,
+):
+    settings = Settings.get_current()
+    srid = srid or settings.srid
+    sql = create_source_emis_query(
+        sourcetype=sourcetype,
+        srid=srid,
+        substances=substances,
+        name=name,
+        ids=ids,
+        tags=tags,
+        polygon=polygon,
+    )
+    cur = connection.cursor()
+    cur.execute(sql)
+    return cur
+
+
+def calculate_source_emissions_df(
+    sourcetype,
+    substances=None,
+    name=None,
+    ids=None,
+    tags=None,
+    polygon=None,
+    unit=DEFAULT_EMISSION_UNIT,
+):
+    cur = calculate_source_emissions(
+        sourcetype=sourcetype,
+        substances=substances,
+        name=name,
+        ids=ids,
+        tags=tags,
+        polygon=polygon,
+        unit=unit,
+    )
+    df = pd.DataFrame(cur.fetchall(), columns=[col[0] for col in cur.description])
+    if sourcetype == "grid":
+        df.set_index(["source_id", "substance", "raster"], inplace=True)
+    elif sourcetype == "road":
+        df.set_index(["source_id", "substance", "vehicle"], inplace=True)
+        traffic_work = df.loc[(slice(None), "traffic_work"), :]["emis"]
+        traffic_work.name = "traffic_work"
+        traffic_work.index = traffic_work.index.droplevel("substance")
+        df.drop(level="substance", labels=["traffic_work"], inplace=True)
+        df = df.join(traffic_work)
+        df = df.reorder_levels(["source_id", "substance", "vehicle"])
+    else:
+        df.set_index(["source_id", "substance"], inplace=True)
+    df.loc[:, "emis"] *= emis_conversion_factor_from_si(unit)
+    df.sort_index(inplace=True)
+    return df
+
+
+def aggregate_emissions(
+    substances=None,
+    sourcetypes=None,
+    codeset=None,
+    polygon=None,
+    tags=None,
+    point_ids=None,
+    area_ids=None,
+    unit=DEFAULT_EMISSION_UNIT,
+    name=None,
+):
+
+    settings = Settings.get_current()
+    codeset_index = None if codeset is None else settings.get_codeset_index(codeset)
+    sql = create_aggregate_emis_query(
+        substances=substances,
+        sourcetypes=sourcetypes,
+        codeset_index=codeset_index,
+        polygon=polygon,
+        name=name,
+        tags=tags,
+        point_ids=point_ids,
+        area_ids=area_ids,
+    )
+    cur = connection.cursor()
+    cur.execute(sql)
+    df = pd.DataFrame(cur.fetchall(), columns=[col[0] for col in cur.description])
+    if codeset is not None:
+        try:
+            codeset = CodeSet.objects.get(slug=codeset)
+        except CodeSet.DoesNotExist:
+            raise ValueError(f"Codeset {codeset} does not exist, choose valid slug.")
+    if codeset is not None:
+        # add code labels to dataframe
+        df.insert(1, "activity", "")
+        code_labels = dict(codeset.codes.values_list("code", "label"))
+        for ind in df.index:
+            code = df.loc[ind, "activitycode"]
+            if code is not None:
+                df.loc[ind, "activity"] = code_labels[code]
+        # add to index (to remain also after pivoting)
+        df.set_index(["activitycode", "activity"], inplace=True)
+        df = df.pivot(columns="substance")
+    else:
+        df.insert(1, "activity", "total")
+        df.set_index(["activity"], inplace=True)
+        df = df.pivot(columns="substance")
+    df.columns = df.columns.set_names(["quantity", "substance"])
+    if "traffic_work" in df.columns.get_level_values("substance"):
+        # set quantity to traffic if substance is traffic_work
+        df.columns = pd.MultiIndex.from_tuples(
+            [
+                (q, s) if s != "traffic_work" else ("traffic", "veh*km")
+                for q, s in df.columns
+            ]
+        )
+        df.columns.names = ["quantity", "substance"]
+        # convert from veh*m to veh*km
+        df.loc[:, ("traffic", "veh*km")] /= 1000.0
+
+    # convert emission unit
+    if len(df) > 0:
+        df.loc[:, ("emission", slice(None))] *= emis_conversion_factor_from_si(unit)
+    return df

cetk/emissions/filters.py

@@ -0,0 +1,179 @@
+"""Functions to dynamically create SQL where clauses for emission calculations."""
+
+from collections.abc import Iterable
+
+from django.contrib.gis.geos import Polygon
+
+from cetk.edb.models import Settings, get_gridsource_raster, list_gridsource_rasters
+
+
+def create_name_where_clause(name):
+    """create SQL to filter sources by source name"""
+    return f"sources.name = '{name}'"
+
+
+def create_ids_where_clause(ids):
+    """create SQL to filter sources by source id's
+    args:
+        ids: iterable of source id's
+    """
+    source_ids = ",".join(ids)
+    return f"sources.id IN ({source_ids})"
+
+
+def create_polygon_where_clause(polygon):
+    """create SQL to filter sources by polygon
+    args:
+        sourcetype: "grid", "point" or "area"
+        polygon: a polygon in Polygon or EWKT-format
+    """
+    polygon_ewkt = polygon.ewkt if isinstance(polygon, Polygon) else polygon
+    settings = Settings.get_current()
+    sql = f"""\
+       ST_Intersects(
+         ST_transform(sources.geom, {settings.srid}),
+         ST_Transform(GeomFromEWKT('{polygon_ewkt}'), {settings.srid})
+       )
+    """
+    return sql
+
+
+def create_tag_where_clause(tags):
+    """create SQL to filter sources on tags
+    args:
+        tags: a dictionary with key value pairs
+
+    To exclude all sources with a specific tag, specify operator as part of value:
+    "tag": "!=value"
+    """
+    conds = []
+    for tag, cond in tags.items():
+        if cond.startswith("!="):
+            val = cond[2:].strip()
+            conds.append(
+                f"(sources.tags IS NULL OR NOT sources.tags ? {tag} OR sources.tags->>{tag} != {val})"  # noqa
+            )
+        else:
+            if cond.startswith("="):
+                val = cond[1:].strip()
+            else:
+                val = cond
+            conds.append(f"sources.tags->>%{tag} = {val}")
+    return " AND ".join(conds)
+
+
+def create_ef_substance_where_clause(substances):
+    """filter to only include specified substances.
+    args:
+        substances: list of Substance model instances
+    """
+
+    if not isinstance(substances, Iterable):
+        substance_ids = str(substances.id)
+    else:
+        substance_ids = ",".join([str(s.id) for s in substances])
+    return f"ef.substance_id IN ({substance_ids})"
+
+
+# def create_activitycode_where_clauses(ac1, ac2, ac3, first_cond=True):
+#     """filter to only include specified activitycodes.
+#     args:
+#         ac1: iterable with activitycode instances
+#         ac2: iterable with activitycode instances
+#         ac3: iterable with activitycode instances
+
+#     kwargs:
+#         first_cond: True adds WHERE, False adds AND
+
+#     returns sql, params
+
+#     returned where clause includes place-holders for codes named:
+#     ac1_code_1, ac1_code_2, ac2_code_1, ac2_code_2 etc.
+#     """
+
+#     # ac1, ac2, ac3 can be either a single code or an iterable
+#     sql = ""
+#     params = {}
+#     if ac1 is not None:
+#         if not isinstance(ac1, Iterable):
+#             ac1 = [ac1]
+#         else:
+#             ac1 = list(ac1)
+#         if not first_cond:
+#             sql += " AND ("
+#         else:
+#             sql += " WHERE ("
+#             first_cond = False
+#         sql += " OR ".join(
+#             [f"ac1.code <@ %(ac1_code_{i})s" for i, ac in enumerate(ac1)]
+#         )
+#         sql += ")"
+#         params.update({f"ac1_code_{i}": ac.code for i, ac in enumerate(ac1)})
+
+#     if ac2 is not None:
+#         if not isinstance(ac2, Iterable):
+#             ac2 = [ac2]
+#         else:
+#             ac2 = list(ac2)
+#         if not first_cond:
+#             sql += " AND ("
+#         else:
+#             sql += " WHERE ("
+#             first_cond = False
+#         sql += " OR ".join(
+#             [f"ac2.code <@ %(ac2_code_{i})s" for i, ac in enumerate(ac2)]
+#         )
+#         sql += ")"
+#         params.update({f"ac2_code_{i}": ac.code for i, ac in enumerate(ac2)})
+
+#     if ac3 is not None:
+#         if not isinstance(ac3, Iterable):
+#             ac3 = [ac3]
+#         else:
+#             ac3 = list(ac3)
+
+#         if not first_cond:
+#             sql += " AND ("
+#         else:
+#             sql += " WHERE ("
+#             first_cond = False
+
+#         sql += " OR ".join(
+#             [f"ac3.code <@ %(ac3_code_{i})s" for i, ac in enumerate(ac3)]
+#         )
+#         sql += ")"
+#         params.update({f"ac3_code_{i}": ac.code for i, ac in enumerate(ac3)})
+
+#     return sql, params
+
+
+def create_substance_emis_where_clause(substances):
+    """filter to only include emissions of specified substances.
+    args:
+        substances: list of Substance model instances
+    """
+    substance_ids = (
+        str(substances.id)
+        if not isinstance(substances, Iterable)
+        else ",".join([str(s.id) for s in substances])
+    )
+    return f"emis.substance_id IN ({substance_ids})"
+
+
+def create_raster_share_in_polygon_sql(polygon):
+    """create inline sql table for fractions of rasters inside polygon."""
+    rasters = {}
+    for raster_name in list_gridsource_rasters():
+        if polygon is not None:
+            raster_data, metadata = get_gridsource_raster(raster_name, clip_by=polygon)
+            rasters[raster_name] = raster_data.sum()
+        else:
+            rasters[raster_name] = 1.0
+    if len(rasters) > 0:
+        sql = "\nUNION ALL\n".join(
+            f"SELECT '{name}' as name, {total} as total"
+            for name, total in rasters.items()
+        )
+    else:
+        sql = """SELECT "dummy" as name, 1.0 as total"""
+    return sql

cetk/emissions/gridsource_emissions.sql

@@ -0,0 +1,52 @@
+/*
+Calculate emissions.
+Substance emissions and activity emissions are calculated separately and
+are then aggregated by source, raster and substance.
+ */
+WITH sources as (
+  SELECT sources.id as source_id, name as source_name, tags, timevar_id,
+       ac1.code as ac1, ac2.code as ac2, ac3.code as ac3
+  FROM edb_gridsource as sources
+       LEFT JOIN edb_activitycode as ac1 ON ac1.id = sources.activitycode1_id
+       LEFT JOIN edb_activitycode as ac2 ON ac2.id = sources.activitycode2_id
+       LEFT JOIN edb_activitycode as ac3 ON ac3.id = sources.activitycode3_id
+       {source_filters}
+)
+SELECT sources.source_id, aggr_emis.substance_id, aggr_emis.raster, source_name, ac1, ac2, ac3,
+  sources.timevar_id, substances.slug as substance, aggr_emis.emis
+FROM
+(
+  SELECT substance_id, source_id, raster, SUM(emis) as emis
+  FROM
+  (
+    SELECT
+      sources.source_id,
+      emis.substance_id,
+      emis.raster,
+      emis.value as emis
+    FROM edb_gridsourcesubstance as emis
+	 JOIN sources ON sources.source_id = emis.source_id
+     WHERE emis.value > 0
+	 {emis_substance_filter}
+    UNION ALL
+    SELECT
+      sources.source_id,
+      ef_subst.substance_id,
+      act.raster,
+      act.rate * ef_subst.factor as emis
+      FROM edb_gridsourceactivity as act
+	   JOIN sources ON act.source_id=sources.source_id
+	   JOIN (
+	     SELECT *
+	       FROM edb_emissionfactor as ef
+		    JOIN edb_activity ON edb_activity.id=ef.activity_id
+	      WHERE ef.factor > 0
+		    {ef_substance_filter}
+	   ) as ef_subst ON act.activity_id=ef_subst.activity_id
+     WHERE act.rate > 0
+  ) all_emis
+   GROUP BY substance_id, source_id, raster
+) aggr_emis
+  JOIN sources ON aggr_emis.source_id = sources.source_id
+  JOIN substances ON aggr_emis.substance_id = substances.id
+ ORDER BY sources.source_id

cetk/emissions/pointsource_emissions.sql

@@ -0,0 +1,53 @@
+/*
+Calculate emissions, optionally filtered by a polygon.
+Substance emissions and activity emissions are calculated separately and
+are then aggregated by source, raster and substance.
+ */
+WITH sources as (
+  SELECT sources.id as source_id, name as source_name, tags, timevar_id,
+       ac1.code as ac1, ac2.code as ac2, ac3.code as ac3,
+       ST_Transform(geom, {srid}) as geom
+  FROM edb_pointsource as sources
+       LEFT JOIN edb_activitycode as ac1 ON ac1.id = sources.activitycode1_id
+       LEFT JOIN edb_activitycode as ac2 ON ac2.id = sources.activitycode2_id
+       LEFT JOIN edb_activitycode as ac3 ON ac3.id = sources.activitycode3_id
+       {source_filters}
+)
+SELECT sources.source_id, aggr_emis.substance_id, source_name, ac1, ac2, ac3,
+  sources.geom as geom,
+  ST_AsText(sources.geom) as wkt,
+  sources.timevar_id, substances.slug as substance, aggr_emis.emis
+FROM
+(
+  SELECT substance_id, source_id, SUM(emis) as emis
+  FROM
+  (
+    SELECT
+      sources.source_id,
+      emis.substance_id as substance_id,
+      emis.value as emis
+    FROM edb_pointsourcesubstance as emis
+	 JOIN sources ON sources.source_id = emis.source_id
+     WHERE emis.value > 0
+	 {emis_substance_filter}
+    UNION ALL
+    SELECT
+      sources.source_id,
+      ef_subst.substance_id as substance_id,
+      act.rate * ef_subst.factor as emis
+      FROM edb_pointsourceactivity as act
+	   JOIN sources ON act.source_id=sources.source_id
+	   JOIN (
+	     SELECT *
+	       FROM edb_emissionfactor as ef
+		    JOIN edb_activity ON edb_activity.id=ef.activity_id
+	      WHERE ef.factor > 0
+		    {ef_substance_filter}
+	   ) as ef_subst ON act.activity_id=ef_subst.activity_id
+     WHERE act.rate > 0
+  ) all_emis
+   GROUP BY substance_id, source_id
+) aggr_emis
+  JOIN sources ON aggr_emis.source_id = sources.source_id
+  JOIN substances ON aggr_emis.substance_id = substances.id
+ ORDER BY sources.source_id

cetk/emissions/queries.py

@@ -0,0 +1,221 @@
+"""Functions to create queries for emissions."""
+
+from collections.abc import Sequence
+from importlib import resources
+
+from cetk.edb.models import Settings, Substance
+from cetk.emissions.filters import (
+    create_ef_substance_where_clause,
+    create_ids_where_clause,
+    create_name_where_clause,
+    create_polygon_where_clause,
+    create_raster_share_in_polygon_sql,
+    create_substance_emis_where_clause,
+    create_tag_where_clause,
+)
+
+
+def load_sql(filename):
+    return resources.files("cetk.emissions").joinpath(filename).read_text()
+
+
+def create_source_emis_query(
+    sourcetype,
+    srid=None,
+    name=None,
+    ids=None,
+    tags=None,
+    polygon=None,
+    substances=None,
+    #    ac1=None,
+    #    ac2=None,
+    #    ac3=None,
+):
+    """
+        Create sql for emissions grouped by source, timevar & substance
+
+        Emissions are returned in kg/s.
+
+        optional arguments:
+            srid: sources will be transformed to this srid, default is inventory srid
+            ids: sequence of source id's for which to include sources
+            name: filter by source name(accepts regexp)
+            tags: filter by tags(specify as dictionary)
+            polygon: only include sources within polygon (EWKT or Polygon)
+            substances: iterable of substance model instances(default is all)
+    #        ac1: iterable of activitycode instances
+    #        ac2: iterable of activitycode instances
+    #        ac3: iterable of activitycode instances
+    """
+
+    sql = load_sql(f"{sourcetype}source_emissions.sql")
+    source_filters = []
+    if tags is not None:
+        source_filters.append(create_tag_where_clause(tags))
+    if ids is not None:
+        source_filters.append(create_ids_where_clause(ids))
+    if name is not None:
+        source_filters.append(create_name_where_clause(name))
+    if polygon is not None and sourcetype != "grid":
+        source_filters.append(create_polygon_where_clause(polygon))
+    if len(source_filters) > 0:
+        source_filter_sql = "WHERE " + " AND ".join(source_filters)
+    else:
+        source_filter_sql = ""
+
+    traffic_work = Substance.objects.get(slug="traffic_work")
+    if substances is not None:
+        emis_subst_filter = " AND " + create_substance_emis_where_clause(substances)
+        if sourcetype == "road":
+            ef_subst_filter = " AND " + create_ef_substance_where_clause(
+                substances + [traffic_work]
+            )
+        else:
+            ef_subst_filter = " AND " + create_ef_substance_where_clause(substances)
+    else:
+        ef_subst_filter = "AND 1=1"
+        emis_subst_filter = ""
+
+    # replace place-holders by generated sql
+    sql = sql.format(
+        srid=srid,
+        source_filters=source_filter_sql,
+        ef_substance_filter=ef_subst_filter,
+        emis_substance_filter=emis_subst_filter,
+        traffic_work_subst_id=traffic_work.id,
+    )
+    return sql
+
+
+def create_aggregate_emis_query(
+    substances=None,
+    sourcetypes=None,
+    codeset_index=None,
+    polygon=None,
+    tags=None,
+    name=None,
+    point_ids=None,
+    area_ids=None,
+    grid_ids=None,
+    road_ids=None,
+    raster_share_in_polygon=None,
+):
+    sql = load_sql("aggregate_emissions.sql")
+    traffic_work_subst_id = Substance.objects.values_list("id", flat=True).get(
+        slug="traffic_work"
+    )
+
+    if isinstance(substances, Substance):
+        substances = [substances]
+
+    sourcetypes = sourcetypes or ("point", "area", "grid", "road")
+    if not isinstance(sourcetypes, Sequence):
+        sourcetypes = [sourcetypes]
+
+    settings = Settings.get_current()
+
+    # adapt query to group by requested code-set
+    if codeset_index is not None:
+        if codeset_index not in (1, 2, 3):
+            raise ValueError("Invalid code-set index, must be in range 1-3")
+        ac_groupby = f"ac{codeset_index},"
+        ac_column = f"ac{codeset_index} as activitycode,"
+    else:
+        ac_groupby = ""
+        ac_column = ""
+
+    # general source filters
+    source_filters = []
+    if tags is not None:
+        source_filters.append(create_tag_where_clause(tags))
+    if name is not None:
+        source_filters.append(create_name_where_clause(name))
+
+    # point source filters
+    point_source_filters = source_filters.copy()
+    if polygon is not None:
+        point_source_filters.append(create_polygon_where_clause(polygon))
+    if point_ids is not None:
+        point_source_filters.append(create_ids_where_clause(point_ids))
+    if "point" not in sourcetypes:
+        point_source_filters.append("1=0")
+
+    if len(point_source_filters) > 0:
+        point_source_filter_sql = "WHERE " + " AND ".join(point_source_filters)
+    else:
+        point_source_filter_sql = ""
+
+    # area source filters
+    area_source_filters = source_filters.copy()
+    if polygon is not None:
+        area_source_filters.append(create_polygon_where_clause(polygon))
+    if area_ids is not None:
+        area_source_filters.append(create_ids_where_clause(area_ids))
+    if "area" not in sourcetypes:
+        area_source_filters.append("1=0")
+
+    if len(area_source_filters) > 0:
+        area_source_filter_sql = "WHERE " + " AND ".join(area_source_filters)
+    else:
+        area_source_filter_sql = ""
+
+    # road source filters
+    road_source_filters = ["sources.aadt > 0"] + source_filters
+    if polygon is not None:
+        road_source_filters.append(create_polygon_where_clause(polygon))
+    if road_ids is not None:
+        road_source_filters.append(create_ids_where_clause(road_ids))
+    if "road" not in sourcetypes:
+        road_source_filters.append("1=0")
+    if len(road_source_filters) > 0:
+        road_source_filter_sql = "WHERE " + " AND ".join(road_source_filters)
+    else:
+        road_source_filter_sql = ""
+
+    if substances is not None:
+        emis_subst_filter = " AND " + create_substance_emis_where_clause(substances)
+        ef_subst_filter = " AND " + create_ef_substance_where_clause(substances)
+    else:
+        emis_subst_filter = ""
+        ef_subst_filter = ""
+
+    # grid source filters
+    grid_source_filters = source_filters.copy()
+
+    if grid_ids is not None:
+        grid_source_filters.append(create_ids_where_clause(grid_ids))
+    if "grid" not in sourcetypes:
+        grid_source_filters.append("1=0")
+    if len(grid_source_filters) > 0:
+        grid_source_filter_sql = "WHERE " + " AND ".join(grid_source_filters)
+    else:
+        grid_source_filter_sql = ""
+
+    raster_share_sql = create_raster_share_in_polygon_sql(polygon)
+    if substances is not None:
+        substances_id = [s.id for s in substances]
+    else:
+        substances_id = [s.id for s in Substance.objects.all()]
+    sql = sql.format(
+        srid=settings.srid,
+        ac_column=ac_column,
+        ac_groupby=ac_groupby,
+        point_source_filter=point_source_filter_sql,
+        area_source_filter=area_source_filter_sql,
+        grid_source_filter=grid_source_filter_sql,
+        road_source_filter=road_source_filter_sql,
+        emis_substance_filter=emis_subst_filter,
+        ef_substance_filter=ef_subst_filter,
+        raster_share_sql=raster_share_sql,
+        traffic_work_subst_id=traffic_work_subst_id,
+        substances=substances_id,
+    )
+    return sql
+
+
+def create_used_substances_query():
+    """
+    Create query to retrieve all substances for which there are emissions
+    in the inventory
+    """
+    return load_sql("used_substances.sql")