cetk 0.0.6.dev0__tar.gz → 0.0.7.dev0__tar.gz

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cetk
-Version: 0.0.6.dev0
+Version: 0.0.7.dev0
 Summary: CLAIRs Emission ToolKit cetk
 Author: Christian Asker, Mattias Jakobsson, David Segersson, Eef van Dongen
 Maintainer-email: FO Luft <foluftadmin@smhi.se>
@@ -18,7 +18,7 @@ Requires-Dist: numpy>=1.20.3
 Requires-Dist: openpyxl>=3.0.7
 Requires-Dist: pandas>=2.0
 Requires-Dist: pyogrio>=0.4.2
-Requires-Dist: rastafari>=0.4
+Requires-Dist: rastafari2>=0.4
 Requires-Dist: rasterio>=1.3
 Requires-Dist: shapely>=2.0
 Dynamic: license-file

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/pyproject.toml

@@ -31,7 +31,7 @@ dependencies = [       # Release date  Comment
   "openpyxl >=3.0.7",  # 2021-03-09    required by pandas 2.0.0
   "pandas >=2.0",      # 2023-04-03    cetk import test fails with earlier versions
   "pyogrio >=0.4.2",   # 2022-10-06    cetk export test fails with earlier versions
-  "rastafari >=0.4",
+  "rastafari2 >=0.4",
   "rasterio >=1.3",    # 2022-07-06    rasterio.CRS is missing in earlier versions
   "shapely >=2.0",     # 2022-12-12    cetk calc --rasterize fails with earlier versions
 ]

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/__init__.py

@@ -1,6 +1,6 @@
 """cetk, a python library for editing Clair emission inventories offline."""
 
-__version__ = "0.0.6.dev"
+__version__ = "0.0.7.dev"
 
 import os
 

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/edb/const.py

@@ -3,7 +3,7 @@ DEFAULT_SRID = 3857
 CHAR_FIELD_LENGTH = 100
 CODE_FIELD_LENGTH = 50
 NODATA = -9999.0
-DEFAULT_EMISSION_UNIT = "kg/year"
+DEFAULT_EMISSION_UNIT = "kg/s"
 
 # sheet names which are valid for data import
 SHEET_NAMES = [

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/edb/models/fleets.py

@@ -25,7 +25,6 @@ class FleetManager(models.Manager):
         for f in data:
             for m in f["members"]:
                 for fu in m["fuels"]:
-                    # breakpoint()
                     if (m["vehicle"], fu["fuel"]) not in validcombinations:
                         raise ValueError(
                             f"{m['vehicle']}/{fu['fuel']} is not a valid vehicle/fuel"

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/edb/rasterize/rasterizer.py

@@ -318,15 +318,7 @@ class EmissionRasterizer:
         """Get querysets for emissions."""
 
         sourcetypes = sourcetypes or SOURCETYPES
-        # not used; srid = Settings.get_current().srid
-
-        # if ac1 is not None or ac2 is not None or ac3 is not None:
-        #     breakpoint()
-        #     # raise ValueError(
-        #     #     "filtering for ac not implemented in source_emissions_query yet!"
-        #     # )
-        # if activity codes are given as strings, get the corresponding
-        # activity code instances
+
         if ac1 is not None and len(ac1) > 0 and isinstance(ac1[0], str):
             # TODO why code__in= not code= ?
             codeset1 = Settings.get_current().codeset1
@@ -708,10 +700,12 @@ class EmissionRasterizer:
                         name=var_name,
                         unit=self.unit,
                         instance=self.instance,
+                        substance=substance.slug,
                         cell_methods=cell_methods,
                         parameter=param.name,
                         time=time,
                         chunksizes=chunking,
+                        quantity="emission",
                     )
                     subst_vars["field2d"] = {"emission": var_name}
             else:
@@ -890,6 +884,7 @@ class EmissionRasterizer:
         # rasterizing chunks and writing to dataset
         chunk_begin = begin
         chunk_end = begin
+        chunk_time_ind = 0
         while chunk_end < end:
             if min_time_chunksize == 1e9:
                 # means no result_files exist
@@ -934,13 +929,19 @@ class EmissionRasterizer:
                                 substance,
                                 emis_chunk,
                                 timestamps=[chunk_begin, chunk_end],
+                                chunk_time_ind=chunk_time_ind,
                             )
                         # update time-span of variable and dataset
 
             # update chunk time interval
+            chunk_time_ind = (
+                chunk_time_ind
+                + int((chunk_end - chunk_begin).total_seconds() / 3600)
+                + 1
+            )
             chunk_begin = chunk_end + datetime.timedelta(hours=1)
 
-    def set_data(self, dset, substance, data, timestamps=None):
+    def set_data(self, dset, substance, data, timestamps=None, chunk_time_ind=0):
         """Add chunk of data to variable."""
 
         var = dset[f"emission_{substance.slug}"]
@@ -951,12 +952,12 @@ class EmissionRasterizer:
         else:
             time_var = dset["time"]
             # map_oriented, np.fliplr flips 3d data along second dimension
-            var[:, :, :] = np.fliplr(data)
+            var[chunk_time_ind:, :, :] = np.fliplr(data)
             times = pd.date_range(timestamps[0], timestamps[1], freq=self.time_step)
             hours_since_1970 = (
                 times - pd.to_datetime("1970-01-01").tz_localize(self.timezone)
             ).total_seconds() / 3600
-            time_var[:] = hours_since_1970
+            time_var[chunk_time_ind:] = hours_since_1970
 
     def _timeseries_emis(self, substance, begin, end, sourcetype):
         """Get emission timeseries data chunk for sources.
@@ -1398,7 +1399,7 @@ class EmissionRasterizer:
             self.log.info(
                 f"No exact match for EPSG and spatial ref, using best guess: {epsg}"
             )
-            grid_mapping.srid = epsg
+        grid_mapping.srid = epsg
         grid_mapping.crs_wkt = crs.to_wkt()  # for CF Conventions 1.7
         return grid_mapping
 
@@ -1409,6 +1410,7 @@ def create_variable(
     name: str,
     unit: Optional[str] = None,
     quantity: Optional[str] = None,
+    substance: Optional[str] = None,
     instance: Optional[str] = None,
     cell_methods: Optional[str] = None,
     parameter: Optional[str] = None,
@@ -1437,6 +1439,8 @@ def create_variable(
         var.parameter = parameter
     if quantity is not None:
         var.quantity = quantity
+    if substance is not None:
+        var.substance = substance
     if instance is not None:
         var.instance = instance
     if cell_methods is not None:

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/emissions/filters.py

@@ -89,8 +89,6 @@ def create_activitycode_where_clauses(ac1, ac2, ac3):
     """
 
     sql_list = []  # return list of sql strings
-    # TODO
-    # breakpoint()
 
     # ac1, ac2, ac3 can be either a single code or an iterable
     if ac1 is not None:

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/emissions/queries.py

@@ -50,6 +50,7 @@ def create_source_emis_query(
     """
 
     sql = load_sql(f"{sourcetype}source_emissions.sql")
+    ac_filter = ""
     source_filters = []
     if tags is not None:
         source_filters.append(create_tag_where_clause(tags))
@@ -60,7 +61,11 @@ def create_source_emis_query(
     if polygon is not None and sourcetype != "grid":
         source_filters.append(create_polygon_where_clause(polygon))
     if ac1 is not None or ac2 is not None or ac3 is not None:
-        source_filters += create_activitycode_where_clauses(ac1, ac2, ac3)
+        if sourcetype != "road":
+            source_filters += create_activitycode_where_clauses(ac1, ac2, ac3)
+        else:
+            ac_filter = create_activitycode_where_clauses(ac1, ac2, ac3)
+            ac_filter = "AND " + ac_filter[0]
 
     if len(source_filters) > 0:
         source_filter_sql = "WHERE " + " AND ".join(source_filters)
@@ -81,13 +86,23 @@ def create_source_emis_query(
         emis_subst_filter = ""
 
     # replace place-holders by generated sql
-    sql = sql.format(
-        srid=srid,
-        source_filters=source_filter_sql,
-        ef_substance_filter=ef_subst_filter,
-        emis_substance_filter=emis_subst_filter,
-        traffic_work_subst_id=traffic_work.id,
-    )
+    if sourcetype != "road":
+        sql = sql.format(
+            srid=srid,
+            source_filters=source_filter_sql,
+            ef_substance_filter=ef_subst_filter,
+            emis_substance_filter=emis_subst_filter,
+            traffic_work_subst_id=traffic_work.id,
+        )
+    else:
+        sql = sql.format(
+            srid=srid,
+            source_filters=source_filter_sql,
+            ef_substance_filter=ef_subst_filter,
+            emis_substance_filter=emis_subst_filter,
+            traffic_work_subst_id=traffic_work.id,
+            ac_filter=ac_filter,
+        )
     return sql
 
 

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/emissions/roadsource_emissions.sql

@@ -65,7 +65,7 @@ fleet_veh as (
     LEFT JOIN edb_activitycode as ac1 ON vfc.activitycode1_id=ac1.id
     LEFT JOIN edb_activitycode as ac2 ON vfc.activitycode2_id=ac2.id
     LEFT JOIN edb_activitycode as ac3 ON vfc.activitycode3_id=ac3.id
-    WHERE veh_ef.substance_id != {traffic_work_subst_id}
+    WHERE veh_ef.substance_id != {traffic_work_subst_id} {ac_filter}
     GROUP BY fm.fleet_id, fm.id, veh_ef.traffic_situation_id,
       fm.vehicle_id, veh_ef.substance_id
     UNION
@@ -89,6 +89,7 @@ fleet_veh as (
     LEFT JOIN edb_activitycode as ac1 ON vfc.activitycode1_id=ac1.id
     LEFT JOIN edb_activitycode as ac2 ON vfc.activitycode2_id=ac2.id
     LEFT JOIN edb_activitycode as ac3 ON vfc.activitycode3_id=ac3.id
+    WHERE 1=1 {ac_filter}
   ) as ef
   JOIN edb_fleetmember fm ON ef.fleetmember_id = fm.id
   JOIN edb_vehicle veh ON ef.vehicle_id=veh.id
@@ -101,7 +102,8 @@ fleet_veh as (
      ef.stopngo_ef != 0 OR
      ef.heavy_ef != 0 OR
      ef.saturated_ef != 0
-   ) {ef_substance_filter}
+   )
+   {ef_substance_filter}
 ),
 /*
 Calculate weight of each LOS (Level Of Service) for all valid combinations

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/emissions/views.py

@@ -79,7 +79,7 @@ def create_emission_table(sourcetype, substances=None, unit=DEFAULT_EMISSION_UNI
     """
     cur = connection.cursor()
     cur.execute(f"DROP TABLE IF EXISTS {sourcetype}source_emissions")
-    # breakpoint()
+
     cur.execute(table_sql)
     cur.execute(
         f"""

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk/tools/cetk_command.py

@@ -293,6 +293,7 @@ class Editor(object):
                     srid=srid,
                     basename=basename,
                 )
+                kwargs = {f"ac{codeset_index}": code}
                 rasterizer = EmissionRasterizer(output, nx=nx, ny=ny)
                 rasterizer.process(
                     substances,
@@ -304,14 +305,8 @@ class Editor(object):
                     area_ids=area_ids,
                     grid_ids=grid_ids,
                     road_ids=road_ids,
-                    ac1=code,
+                    **kwargs,
                 )
-                # except Exception as err:
-                #     print(f"could not rasterize emissions: {str(err)}")
-                #     breakpoint()
-                #     continue
-                #     log.error(f"could not rasterize emissions: {str(err)}")
-                #     sys.exit(1)
         else:
             try:
                 output = Output(

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cetk
-Version: 0.0.6.dev0
+Version: 0.0.7.dev0
 Summary: CLAIRs Emission ToolKit cetk
 Author: Christian Asker, Mattias Jakobsson, David Segersson, Eef van Dongen
 Maintainer-email: FO Luft <foluftadmin@smhi.se>
@@ -18,7 +18,7 @@ Requires-Dist: numpy>=1.20.3
 Requires-Dist: openpyxl>=3.0.7
 Requires-Dist: pandas>=2.0
 Requires-Dist: pyogrio>=0.4.2
-Requires-Dist: rastafari>=0.4
+Requires-Dist: rastafari2>=0.4
 Requires-Dist: rasterio>=1.3
 Requires-Dist: shapely>=2.0
 Dynamic: license-file

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/src/cetk.egg-info/requires.txt

@@ -6,6 +6,6 @@ numpy>=1.20.3
 openpyxl>=3.0.7
 pandas>=2.0
 pyogrio>=0.4.2
-rastafari>=0.4
+rastafari2>=0.4
 rasterio>=1.3
 shapely>=2.0

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/tests/edb/test_common_models.py

@@ -34,7 +34,6 @@ class TestActivityCodes:
         ac1 = code_set.codes.get(code="1")
         ac13 = code_set.codes.get(code="1.3")
         ac131 = code_set.codes.get(code="1.3.1")
-        # breakpoint()
         assert ac13 in list(ac1.get_children())
         assert ac131 not in list(ac1.get_children())
         assert ac131 in list(ac13.get_children())

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/tests/emissions/test_aggregate_emissions.py

@@ -155,5 +155,5 @@ def test_aggregate_emissions_all_sourcetypes(
     assert df.index[0] == ("3", "Diffuse sources")
     assert df.columns[0] == ("emission", "SOx")
     # reference emission: "seconds of year" * "grid fraction in polygon"
-    ref_emis = 1.0 * 365.25 * 24 * 3600 * raster_share
+    ref_emis = 1.0 * raster_share
     assert ref_emis == pytest.approx(df.iloc[0, 0], 1e-5)

{cetk-0.0.6.dev0 → cetk-0.0.7.dev0}/tests/tools/test_calculate.py

@@ -45,8 +45,12 @@ def test_aggregate(inventory, tmpdir):
     assert np.all(df.columns.levels[0] == ["emission"])
     assert np.all(df.columns.levels[1] == ["CO", "NMVOC", "NOx", "PM10", "PM25", "SOx"])
     assert df.index.names == ["activity"]
-    assert df.loc["total", ("emission", "NOx")].item() == pytest.approx(45108)
-    assert df.loc["total", ("emission", "PM25")].item() == pytest.approx(5.411264e6)
+    assert df.loc["total", ("emission", "NOx")].item() == pytest.approx(
+        45108 / (365.25 * 24 * 3600)
+    )
+    assert df.loc["total", ("emission", "PM25")].item() == pytest.approx(
+        5.411264e6 / (365.25 * 24 * 3600)
+    )
 
 
 @pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")