cetk 0.0.4.dev0__py3-none-any.whl

cetk/edb/migrations/0002_data.py

@@ -0,0 +1,156 @@
+# Generated by Django 1.11.11 on 2018-03-28 08:36
+from __future__ import unicode_literals
+
+from django.db import migrations
+from django.utils.text import slugify
+
+# slug, name and long_name of all substances
+# these are also used to create corresponding emission and concentration params
+SUBSTANCES = (
+    ("activity", "activity", "Activity"),
+    ("As", "As", "Arsenic"),
+    ("BC", "BC", "Black Carbon"),
+    ("BaP", "BaP", "Benzo[a]pyrene"),
+    ("Cd", "Cd", "Cadmium"),
+    ("Cr", "Cr", "Chromium"),
+    ("Cu", "Cu", "Copper"),
+    ("C6H6", "Benzene", "Benzene"),
+    ("CH4", "Methane", "Methane"),
+    ("CO", "CO", "Carbon monoxide"),
+    ("CO2", "CO2", "Carbon dioxide"),
+    (
+        "Dioxin",
+        "Dioxin",
+        "Dioxin",
+    ),
+    ("HC", "Hydrocarbons", "Hydrocarbons"),
+    ("HCB", "HCB", "Hexachlorobenzene"),
+    ("HFC", "HFC", "Hydrofluorocarbons"),
+    ("Hg", "Hg", "Mercury"),
+    ("N2O", "N2O", "Nitrous oxide"),
+    ("NH3", "NH3", "Ammonia"),
+    ("Ni", "Ni", "Nickel"),
+    ("NMHC", "NMHC", "Non-methane hydrocarbons"),
+    ("NMVOC", "NMVOC", "Non-methane volatile organic compound"),
+    ("NOx", "NOx", "Nitrogen oxides (as NO2)"),
+    ("NO2", "NO2", "Nitrogen dioxide"),
+    ("NO", "NO", "Nitrogen monooxide"),
+    ("O3", "Ozone", "Ozone"),
+    ("PAH4", "PAH4", "Sum of 4 polycyclic aromatic hydrocarbons"),
+    ("Pb", "Pb", "Lead"),
+    ("PFC", "PFC", "Perfluorocarbons"),
+    ("PM10", "PM10", "Particulate matter < 10 micrometers in diameter"),
+    ("PM25", "PM2.5", "Particulate matter < 2.5 micrometers in diameter"),
+    ("PM10resusp", "PM10resusp", "Resuspended particles < 10 micrometers in diameter"),
+    (
+        "PM10wear",
+        "PM10wear",
+        "Particles < 10 micrometers in diameter from road, tyre and break wear",
+    ),
+    (
+        "PM25resusp",
+        "PM2.5resusp",
+        "Resuspended particles < 2.5 micrometers in diameter",
+    ),
+    (
+        "PM25wear",
+        "PM2.5wear",
+        "Particles < 2.5 micrometers in diameter from road, tyre and break wear",
+    ),
+    ("PN", "PN", "Particle Number"),
+    ("Se", "Se", "Selenium"),
+    ("PCB", "PCB", "Polychlorinated biphenyls"),
+    ("SF6", "SF6", "Sulfur Hexafluoride"),
+    ("SO2", "SO2", "Sulphur dioxide"),
+    ("SOx", "SOx", "Sulphur oxides (as SO2)"),
+    ("traffic_work", "traffic work", "Traffic work"),
+    ("TSP", "TSP", "Total Suspended Particles"),
+    ("Zn", "Zn", "Zinc"),
+)
+
+# quantity, slug, name
+PARAMETERS = (
+    ("temperature", "dtmp", "Air temperature difference"),
+    ("temperature", "temp", "Air temperature"),
+    ("wind speed", "wspd", "Wind speed"),
+    ("wind direction", "wdir", "Wind direction"),
+    ("radiation", "netr", "Net radiation"),
+    ("radiation", "glob", "Global radiation"),
+    ("cloud cover", "tcc", "Total Cloud Cover"),
+    ("pressure", "pres", "Pressure"),
+    ("pressure", "psea", "Pressure at sea level"),
+    ("relative humidity", "rhum", "Relative Humidity"),
+    ("1 hour precipitation", "1h_prec", "1 hour Precipitation"),
+    ("1 hour snow", "1h_snow", "1 hour Snow"),
+    ("mixing height", "zi", "Mixing height"),
+    ("albedo", "albedo", "Albedo"),
+    ("roughness length", "z0", "Roughness length"),
+    ("traffic condition", "traffic_condition", "Traffic condition"),
+    ("landuse class", "landuse_class", "Landuse class"),
+    ("length", "lmo", "Monin Obukhov Length"),
+    ("height", "height", "height"),
+    ("height", "z_sl", "surface layer height"),
+    ("heat flux", "h_sens", "Sensible heat flux"),
+    ("scale", "theta_star", "Potential temperature scale"),
+    ("scale", "u_star", "Friction velocity"),
+    ("scale", "h_star", "heat flux scale"),
+    ("scale", "w_star", "convective velocity scale"),
+    ("dummy", "dummy", "dummy"),
+)
+
+
+def auto_name_parameter(instance):
+    quantity = instance.quantity.capitalize()
+    if instance.substance is not None:
+        instance.name = f"{quantity} {instance.substance.name}"
+    else:
+        instance.name = quantity
+
+
+def auto_slug_parameter(instance):
+    if instance.substance is not None:
+        quantity = slugify(instance.quantity)
+        instance.slug = f"{quantity}_{instance.substance.slug}"
+    else:
+        instance.slug = slugify(instance.name)
+
+
+def create_parameters(apps, schema_editor):
+    Substance = apps.get_model("edb", "Substance")
+    Parameter = apps.get_model("edb", "Parameter")
+
+    # create substances and related parameters
+    for slug, name, long_name in SUBSTANCES:
+        substance = Substance.objects.create(slug=slug, name=name, long_name=long_name)
+
+        # traffic work is created as 'traffic' parameter instead of 'emission'
+        if slug == "traffic_work":
+            Parameter.objects.create(
+                quantity="traffic",
+                slug="traffic_work",
+                name="Traffic work",
+                substance=substance,
+            )
+            continue
+
+        # save is not called in migrations
+        # and model methods are not available
+        emis = Parameter(quantity="emission", substance=substance)
+        auto_name_parameter(emis)
+        auto_slug_parameter(emis)
+        emis.save()
+
+        conc = Parameter(quantity="concentration", substance=substance)
+        auto_name_parameter(conc)
+        auto_slug_parameter(conc)
+        conc.save()
+
+    # create generic parameters
+    for quantity, slug, name in PARAMETERS:
+        Parameter.objects.create(quantity=quantity, name=name, slug=slug)
+
+
+class Migration(migrations.Migration):
+    dependencies = [("edb", "0001_initial")]
+
+    operations = [migrations.RunPython(create_parameters)]

cetk/edb/migrations/0003_create_gpkg_tables.py

@@ -0,0 +1,14 @@
+# Generated by Django 4.2.4 on 2023-11-02 10:19
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("edb", "0002_data"),
+    ]
+
+    operations = [
+        migrations.RunSQL("select gpkgCreateBaseTables()"),
+        migrations.RunSQL("select gpkgInsertEpsgSRID(4979)"),
+    ]

cetk/edb/models/__init__.py

@@ -0,0 +1,49 @@
+from .common_models import Parameter, Settings, Substance  # noqa
+from .fleets import Fleet, FleetMember, FleetMemberFuel  # noqa: F401
+from .gridsource_models import (  # noqa
+    GridSource,
+    GridSourceActivity,
+    GridSourceSubstance,
+    drop_gridsource_raster,
+    get_gridsource_raster,
+    list_gridsource_rasters,
+    write_gridsource_raster,
+)
+from .road_classes import (  # noqa: F401
+    PrefetchRoadClassAttributes,
+    RoadAttribute,
+    RoadAttributeValue,
+    RoadClass,
+    get_valid_road_attribute_values,
+)
+from .road_models import (  # noqa: F401
+    VELOCITY_CHOICES,
+    CongestionProfile,
+    RoadSource,
+    TrafficSituation,
+    Vehicle,
+    VehicleEF,
+    VehicleFuel,
+    VehicleFuelComb,
+    default_congestion_profile_data,
+)
+from .source_models import (  # noqa
+    Activity,
+    ActivityCode,
+    AreaSource,
+    AreaSourceActivity,
+    AreaSourceSubstance,
+    CodeSet,
+    EmissionFactor,
+    Facility,
+    PointSource,
+    PointSourceActivity,
+    PointSourceSubstance,
+    VerticalDist,
+)
+from .timevar_models import (  # noqa
+    ColdstartTimevar,
+    FlowTimevar,
+    Timevar,
+    timevar_to_series,
+)

cetk/edb/models/base.py

@@ -0,0 +1,52 @@
+"""Abstract base models for cetk."""
+
+from django.contrib.gis.db import models
+from django.utils.text import slugify
+from django.utils.translation import gettext_lazy as _
+
+
+class BaseNamedModel(models.Model):
+    """Base class for models with a name and a slug field."""
+
+    class Meta:
+        abstract = True
+
+    def clean(self):
+        if not self.slug and self.name:
+            self.slug = slugify(self.name)
+
+    def __str__(self):
+        return self.name
+
+
+class NamedModelManager(models.Manager):
+    """Database manager for named models."""
+
+    def get_by_natural_key(self, slug):
+        """Return a model instance given its slug."""
+        return self.get(slug=slug)
+
+
+class NaturalKeyManager(models.Manager):
+
+    """Database manager for models with natural key."""
+
+    def get_by_natural_key(self, *key):
+        """Return a model instance given its natural key."""
+        return self.get(**dict(zip(self.model.natural_key_fields, key)))
+
+
+class NamedModel(BaseNamedModel):
+    """A model with a unique name and slug."""
+
+    name = models.CharField(_("name"), max_length=64, unique=True)
+    slug = models.SlugField(_("slug"), max_length=64, unique=True)
+
+    objects = NamedModelManager()
+
+    class Meta:
+        abstract = True
+
+    def natural_key(self):
+        """Return the natural key (the slug) for this model instance."""
+        return (self.slug,)

cetk/edb/models/common_models.py

@@ -0,0 +1,144 @@
+from django.contrib.gis.db import models
+from django.contrib.gis.geos import GEOSGeometry
+from django.core.exceptions import NON_FIELD_ERRORS, ValidationError
+from django.utils.text import slugify
+from django.utils.translation import gettext_lazy as _
+
+from cetk.edb.const import WGS84_SRID
+from cetk.edb.models.base import NamedModel
+from cetk.edb.models.source_models import CodeSet
+
+
+class SettingsManager(models.Manager):
+    # makes sure always just one instance of settings exists
+    def get_queryset(self):
+        return super().get_queryset()[:1]
+
+
+DEFAULT_EXTENT = "Polygon((-25.0 35.0, -25.0 70.0, 40.0 70.0, 40.0 35.0, -25.0 35.0))"
+
+
+class Substance(NamedModel):
+    """A chemical substance."""
+
+    long_name = models.CharField(_("long name"), max_length=64)
+
+    class Meta:
+        db_table = "substances"
+        default_related_name = "substances"
+
+
+class Parameter(NamedModel):
+    """A parameter."""
+
+    quantity = models.CharField(_("physical quantity"), max_length=30)
+    substance = models.ForeignKey(
+        Substance, on_delete=models.CASCADE, null=True, verbose_name=_("substance")
+    )
+
+    class Meta:
+        db_table = "parameters"
+        default_related_name = "parameters"
+
+    def validate_unique(self, *args, **kwargs):
+        """Avoid duplicate emission or conc. parameters for a substance."""
+        super().validate_unique(*args, **kwargs)
+        if self.quantity in ("emission", "concentration"):
+            duplicates = type(self).objects.filter(
+                quantity=self.quantity, substance=self.substance
+            )
+            if duplicates.exists():
+                raise ValidationError(
+                    {
+                        NON_FIELD_ERRORS: [
+                            f"A parameter for {self.quantity} of {self.substance} "
+                            f"already exist"
+                        ]
+                    }
+                )
+
+    def _auto_name(self):
+        """Auto-generate a name."""
+        quantity = self.quantity.capitalize()
+        if self.substance is not None:
+            self.name = f"{quantity} {self.substance.name}"
+        else:
+            self.name = quantity
+
+    def _auto_slug(self):
+        """Auto-generate a slug."""
+        if self.substance is not None:
+            quantity = slugify(self.quantity)
+            self.slug = f"{quantity}_{self.substance.slug}"
+        else:
+            self.slug = slugify(self.name)
+
+    def save(self, *args, **kwargs):
+        """Overloads save to auto-generate name and slug if missing."""
+        if self.name is None:
+            self._auto_name()
+        if self.slug is None:
+            self._auto_slug()
+        super().save(*args, **kwargs)
+
+
+class Settings(models.Model):
+    """Inventory specific database settings, replaces gadgets Domain and Inventory."""
+
+    srid = models.IntegerField(
+        _("SRID"), help_text=_("Spatial reference system identifier")
+    )
+    extent = models.PolygonField(_("extent"), geography=True)
+
+    timezone = models.CharField(_("timezone"), max_length=64)
+
+    # some functionality in Gadget only works for one out of the three codesets.
+    codeset1 = models.ForeignKey(
+        CodeSet, null=True, on_delete=models.SET_NULL, related_name="+"
+    )
+    codeset2 = models.ForeignKey(
+        CodeSet, null=True, on_delete=models.SET_NULL, related_name="+"
+    )
+    codeset3 = models.ForeignKey(
+        CodeSet, null=True, on_delete=models.SET_NULL, related_name="+"
+    )
+    objects = SettingsManager()
+
+    class Meta:
+        db_table = "settings"
+        default_related_name = "settings"
+
+    def get_current():
+        # Retrieve the settings, if exist
+        return Settings.objects.get_or_create(
+            defaults={
+                "srid": 3006,
+                "timezone": "Europe/Stockholm",
+                "extent": GEOSGeometry(DEFAULT_EXTENT, WGS84_SRID),
+                "codeset1": CodeSet.objects.filter(id=1).first(),
+                "codeset2": CodeSet.objects.filter(id=2).first(),
+                "codeset3": CodeSet.objects.filter(id=3).first(),
+            }
+        )[0]
+
+    def get_codeset_index(self, codeset):
+        """Return index of a specific codeset."""
+        if type(codeset) == str:
+            codeset_slug = codeset
+        elif type(codeset) == CodeSet:
+            codeset_slug = codeset.slug
+        else:
+            raise ValueError(f"codeset '{codeset}' is not of valid type")
+        # if self.codeset1 is not None:
+        #     if codeset_slug == self.codeset1.slug:
+        #         return 1
+        # if self.codeset2 is not None:
+        #     if codeset_slug == self.codeset2.slug:
+        #         return 2
+        # if self.codeset3 is not None:
+        #     if codeset_slug == self.codeset3.slug:
+        #         return 3
+        if len(CodeSet.objects.filter(slug=codeset_slug)) > 0:
+            return CodeSet.objects.filter(slug=codeset_slug).first().id
+        else:
+            raise ValueError(f"codeset '{codeset}' not found in inventory settings")

cetk/edb/models/fleets.py

@@ -0,0 +1,160 @@
+"""Database models to describe road fleets."""
+
+import itertools
+
+from django.db import models
+from django.db.models import Q
+
+from cetk.edb.const import CHAR_FIELD_LENGTH
+
+from .road_models import Vehicle, VehicleFuel, VehicleFuelComb
+
+
+class FleetManager(models.Manager):
+    """Custom database manager for the Fleet model."""
+
+    use_in_migrations = True
+
+    def bulk_create_from_dicts(self, data):
+        vehicles = {v.name: v for v in Vehicle.objects.all()}
+        fuels = {f.name: f for f in VehicleFuel.objects.all()}
+        validcombinations = {
+            (vfc.vehicle.name, vfc.fuel.name)
+            for vfc in VehicleFuelComb.objects.select_related("vehicle", "fuel")
+        }
+        for f in data:
+            for m in f["members"]:
+                for fu in m["fuels"]:
+                    # breakpoint()
+                    if (m["vehicle"], fu["fuel"]) not in validcombinations:
+                        raise ValueError(
+                            f"{m['vehicle']}/{fu['fuel']} is not a valid vehicle/fuel"
+                            + " combination"
+                        )
+
+        fleets = [
+            self.model(
+                name=f["name"],
+                default_heavy_vehicle_share=f["default_heavy_vehicle_share"],
+                tags=f.get("tags", {}),
+            )
+            for f in data
+        ]
+        self.model.objects.bulk_create(fleets)
+        fleets = self.model.objects.all()
+
+        membermodel = self.model._meta.get_field("vehicles").related_model
+        members = [
+            membermodel(
+                fraction=m["fraction"],
+                coldstart_fraction=m.get("coldstart_fraction", 0),
+                vehicle=vehicles[m["vehicle"]],
+                timevar=m["timevar"],
+                coldstart_timevar=m.get("coldstart_timevar"),
+                fleet=fleet,
+            )
+            for fleet, f in zip(fleets, data)
+            for m in f["members"]
+        ]
+        membermodel.objects.bulk_create(members)
+        members = membermodel.objects.all()
+
+        memberfuelmodel = membermodel._meta.get_field("fuels").related_model
+        memberfuels = [
+            memberfuelmodel(
+                fraction=fu["fraction"], fuel=fuels[fu["fuel"]], fleet_member=member
+            )
+            for member, m in zip(
+                members, itertools.chain.from_iterable(f["members"] for f in data)
+            )
+            for fu in m["fuels"]
+        ]
+        memberfuelmodel.objects.bulk_create(memberfuels)
+
+        return fleets
+
+
+class Fleet(models.Model):
+    """A composition of vehicles."""
+
+    name = models.CharField(max_length=CHAR_FIELD_LENGTH, unique=True)
+    default_heavy_vehicle_share = models.FloatField()
+    tags = models.JSONField(blank=True, default=dict)
+
+    objects = FleetManager()
+
+    class Meta:
+        default_related_name = "fleets"
+        constraints = [
+            models.CheckConstraint(
+                check=Q(default_heavy_vehicle_share__range=(0, 1)),
+                name="fleet_default_heavy_vehicle_share_between_0_and_1",
+            ),
+        ]
+
+    def __str__(self):
+        """Unicode representation of fleet."""
+        return self.name
+
+    @property
+    def default_light_vehicle_share(self):
+        """Default share of light vehicles in fleet."""
+        return 1 - self.default_heavy_vehicle_share
+
+
+class FleetMember(models.Model):
+    """A member vehicle of a fleet."""
+
+    # share represented by vehicle (sums to 1.0 for the fleet)
+    fraction = models.FloatField(default=0.0)
+    coldstart_fraction = models.FloatField(default=0.0)
+    vehicle = models.ForeignKey("Vehicle", on_delete=models.PROTECT, related_name="+")
+    timevar = models.ForeignKey(
+        "FlowTimevar",
+        on_delete=models.SET_NULL,
+        related_name="+",
+        null=True,
+        blank=True,
+    )
+    coldstart_timevar = models.ForeignKey(
+        "ColdstartTimevar",
+        on_delete=models.SET_NULL,
+        related_name="+",
+        null=True,
+        blank=True,
+    )
+    fleet = models.ForeignKey(Fleet, on_delete=models.CASCADE, related_name="vehicles")
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=("fleet", "vehicle"), name="fleetmember_unique_vehicle_in_fleet"
+            ),
+        ]
+
+    def __str__(self):
+        """Unicode representation of fleet member vehicle."""
+        return self.vehicle.name
+
+
+class FleetMemberFuel(models.Model):
+    """A fuel used by a fleet member."""
+
+    # share of fleet member represented by fuel (sums to 1.0 for fleet member)
+    fraction = models.FloatField(default=0.0)
+    fuel = models.ForeignKey("VehicleFuel", on_delete=models.PROTECT, related_name="+")
+    fleet_member = models.ForeignKey(
+        FleetMember, on_delete=models.CASCADE, related_name="fuels"
+    )
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=("fleet_member", "fuel"),
+                name="fleetmemberfuel_unique_fuel_in_fleetmember",
+            ),
+        ]
+
+    def __str__(self):
+        """Unicode representation of fleet member fuel."""
+        return self.fuel.name