hestia-earth-models 0.65.4__py3-none-any.whl → 0.65.5__py3-none-any.whl

hestia_earth/models/hestia/seed_emissions.py

@@ -18,7 +18,7 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list, flatten, list_sum, safe_parse_float
 from hestia_earth.utils.emission import cycle_emissions_in_system_boundary
 
-from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table, debugValues
 from hestia_earth.models.utils import _omit, group_by
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.site import valid_site_type
@@ -54,7 +54,7 @@ REQUIREMENTS = {
             "siteType": ["cropland", "glass or high accessible cover"],
             "country": {"@type": "Term", "termType": "region"}
         },
-        "emissions": [{"@type": "Emission"}]
+        "emissions": [{"@type": "Emission", "value": ""}]
     }
 }
 RETURNS = {
@@ -86,11 +86,24 @@ def _emission(term_id: str, value: float, input: dict):
     return emission
 
 
-def _run(cycle: dict, economicValueShare: float, total_yield: float, seed_input: dict, grouped_emissions: dict):
-    term = seed_input.get('term', {})
+def _run_emission(
+    cycle: dict, economicValueShare: float, total_yield: float, seed_input: dict, term_id: str, emission_value: float
+):
+    input_term = seed_input.get('term', {})
+    input_term_id = input_term.get('@id')
     seed_value = list_sum(seed_input.get('value'))
+    value = emission_value * economicValueShare / 100 / total_yield * seed_value
+    debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                value=value,
+                coefficient=1,
+                input=input_term_id)
+
+    return _emission(term_id, value, input_term)
+
+
+def _run(cycle: dict, economicValueShare: float, total_yield: float, seed_input: dict, grouped_emissions: dict):
     return [
-        _emission(term_id, emission_value * economicValueShare / 100 / total_yield * seed_value, term)
+        _run_emission(cycle, economicValueShare, total_yield, seed_input, term_id, emission_value)
         for term_id, emission_value in grouped_emissions.items()
     ]
 
@@ -102,12 +115,12 @@ def _filter_emissions(cycle: dict):
         {
             'id': i.get('term', {}).get('@id'),
             'group-id': get_lookup_value(i.get('term', {}), LOOKUPS['emission'], model=MODEL, model_key=MODEL_KEY),
-            'value': list_sum(i.get('value'), 0)
+            'value': list_sum(i.get('value'))
         }
         for i in cycle.get('emissions', [])
         if all([
             i.get('term', {}).get('@id') in required_emission_term_ids,
-            list_sum(i.get('value'), 0) > 0
+            len(i.get('value', [])) > 0
         ])
     ]
     emission_ids = set([v.get('id') for v in emissions])
@@ -120,7 +133,7 @@ def _filter_emissions(cycle: dict):
             'id': group_id,
             'emissions': list(filter(
                 lambda id: id in required_emission_term_ids,
-                list(lookup[lookup[column_name('inputProductionGroupId')] == group_id].termid)
+                set(list(lookup[lookup[column_name('inputProductionGroupId')] == group_id].termid))
             ))
         }
         for group_id in group_ids
@@ -129,8 +142,8 @@ def _filter_emissions(cycle: dict):
         {
             'id': group.get('id'),
             'total-emissions': len(group.get('emissions', [])),
-            'included-emissions': len([v in emission_ids for v in group.get('emissions', [])]),
-            'missing-emissions': '-'.join([v for v in group.get('emissions', []) if v not in emission_ids])
+            'included-emissions': len(list(filter(lambda v: v in emission_ids, group.get('emissions', [])))),
+            'missing-emissions': '-'.join(list(filter(lambda v: v not in emission_ids, group.get('emissions', []))))
         }
         for group in emissions_per_group
     ]
@@ -154,14 +167,14 @@ def _evs(product: dict):
     ) or product.get('economicValueShare')
 
 
-def _yield(country_id: str, end_year: int, product: dict):
+def _faostat_yield(country_id: str, end_year: int, product: dict):
     grouping = get_crop_grouping_faostat_production(MODEL, product.get('term', {}))
     return safe_parse_float(extract_grouped_data_closest_date(get_table_value(
         download_lookup('region-crop-cropGroupingFaostatProduction-yield.csv'),
         'termid',
         country_id,
         column_name(grouping)
-    ), end_year)) or list_sum(product.get('value'))
+    ), end_year))
 
 
 def _group_seed_inputs(inputs: list):
@@ -186,10 +199,11 @@ def _should_run(cycle: dict):
         {
             'product': product.get('term', {}).get('@id'),
             'seed-id': get_lookup_value(
-                product.get('term', {}), 'correspondingSeedTermIds', model=MODEL, key=MODEL_KEY) or None,
+                product.get('term', {}), 'correspondingSeedTermIds', model=MODEL, model_key=MODEL_KEY) or None,
             'economicValueShare': _evs(product),
-            'yield': _yield(country_id, end_year, product),
-            'landCover-id': get_landCover_term_id(product.get('term', {}), model=MODEL, key=MODEL_KEY)
+            'FAOSTAT-yield': _faostat_yield(country_id, end_year, product),
+            'product-yield': list_sum(product.get('value')),
+            'landCover-id': get_landCover_term_id(product.get('term', {}), model=MODEL, model_key=MODEL_KEY)
         }
         for product in crop_products
     ]
@@ -197,7 +211,7 @@ def _should_run(cycle: dict):
         value for value in crop_products if all([
             value.get('seed-id'),
             value.get('economicValueShare'),
-            value.get('yield'),
+            value.get('FAOSTAT-yield') or value.get('product-yield'),
             value.get('landCover-id'),
         ])
     ]
@@ -209,7 +223,7 @@ def _should_run(cycle: dict):
         {
             'input': i,
             'is-corresponding-seed': i.get('term', {}).get('@id') in seed_term_ids,
-            'input-value': i.get('value'),
+            'input-value': list_sum(i.get('value'), default=None),
             'has-linked-impact-assessment': bool(i.get('impactAssessment')),
             'is-fromCycle': i.get('fromCycle', False),
             'is-producedInCycle': i.get('producedInCycle', False),
@@ -221,7 +235,7 @@ def _should_run(cycle: dict):
         v.get('input') for v in seed_inputs
         if all([
             v.get('is-corresponding-seed', False),
-            list_sum(v.get('input-value') or [-1]) > 0,
+            v.get('input-value') or -1 > 0,
             not v.get('has-linked-impact-assessment'),
             not v.get('is-fromCycle'),
             not v.get('is-producedInCycle'),
@@ -230,12 +244,12 @@ def _should_run(cycle: dict):
 
     crop_land_cover_ids = list(set([p.get('landCover-id') for p in valid_crop_products]))
     total_economicValueShare = list_sum([p.get('economicValueShare') for p in valid_crop_products])
-    total_yield = list_sum([p.get('yield') for p in valid_crop_products])
+    total_yield = list_sum([p.get('FAOSTAT-yield') or p.get('product-yield') for p in valid_crop_products])
 
     emissions, emissions_per_group = _filter_emissions(cycle)
     # group emissions with the same group-id
     grouped_emissions = reduce(
-        lambda p, c: p | {c.get('group-id'): p.get(c.get('group-id'), 0) + c.get('value', 0)},
+        lambda p, c: p | {c.get('group-id'): p.get(c.get('group-id'), 0) + (c.get('value') or 0)},
         emissions,
         {}
     )
@@ -266,7 +280,7 @@ def _should_run(cycle: dict):
                         emissions_per_group=log_as_table(emissions_per_group),
                         **_omit(seed_input, 'input'))
 
-        logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY)
+        logShouldRun(cycle, MODEL, term_id, should_run, methodTier=TIER, model_key=MODEL_KEY)
 
     return should_run, total_economicValueShare, total_yield, grouped_seed_inputs, grouped_emissions
 

hestia_earth/models/hestia/utils.py

@@ -0,0 +1,48 @@
+from hestia_earth.schema import TermTermType
+
+from hestia_earth.models.utils.term import get_lookup_value
+from . import MODEL
+
+IPCC_LAND_USE_CATEGORY_ANNUAL = "Annual crops"
+IPCC_LAND_USE_CATEGORY_PERENNIAL = "Perennial crops"
+TOTAL_CROPLAND = "Cropland"
+ANNUAL_CROPLAND = "Arable land"
+FOREST_LAND = "Forest land"
+OTHER_LAND = "Other land"
+PERMANENT_CROPLAND = "Permanent crops"
+PERMANENT_PASTURE = "Permanent meadows and pastures"
+TOTAL_AGRICULTURAL_CHANGE = "Total agricultural change"
+ALL_LAND_USE_TERMS = [
+    FOREST_LAND,
+    TOTAL_CROPLAND,
+    ANNUAL_CROPLAND,
+    PERMANENT_CROPLAND,
+    PERMANENT_PASTURE,
+    OTHER_LAND
+]
+
+# Mapping from Land use terms to Management node terms.
+# land use term: (@id, name)
+LAND_USE_TERMS_FOR_TRANSFORMATION = {
+    FOREST_LAND: ("forest", "Forest"),
+    ANNUAL_CROPLAND: ("annualCropland", "Annual cropland"),
+    PERMANENT_CROPLAND: ("permanentCropland", "Permanent cropland"),
+    PERMANENT_PASTURE: ("permanentPasture", "Permanent pasture"),
+    OTHER_LAND: ("otherLand", OTHER_LAND)  # Not used yet
+}
+
+
+def crop_ipcc_land_use_category(
+    crop_term_id: str,
+    lookup_term_type: str = TermTermType.LANDCOVER.value
+) -> str:
+    """
+    Looks up the crop in the lookup.
+    Returns the IPCC_LAND_USE_CATEGORY.
+    """
+    return get_lookup_value(
+        lookup_term={"@id": crop_term_id, "type": "Term", "termType": lookup_term_type},
+        column='IPCC_LAND_USE_CATEGORY',
+        model=MODEL,
+        term={"@id": crop_term_id, "type": "Term", "termType": lookup_term_type}
+    )

hestia_earth/models/impact_assessment/emissions.py

@@ -4,9 +4,11 @@ Emissions
 Creates an [Indicator](https://hestia.earth/schema/Indicator) for every [Emission](https://hestia.earth/schema/Emission)
 contained within the [ImpactAssesment.cycle](https://hestia.earth/schema/ImpactAssessment#cycle).
 It does this by dividing the Emission amount by the Product amount, and applying an allocation between co-products.
+Note: for any Emission in the system boundary that does not exist in the Cycle, it will log the model as failed,
+so that we know the Emission is missing.
 """
-
 from hestia_earth.utils.tools import list_sum
+from hestia_earth.utils.emission import cycle_emissions_in_system_boundary
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.impact_assessment import get_product, convert_value_from_cycle
@@ -38,10 +40,12 @@ RETURNS = {
 MODEL_KEY = 'emissions'
 
 
-def _indicator(product: dict):
+def _indicator(impact_assessment: dict, product: dict):
     def run(emission: dict):
         term_id = emission.get('term', {}).get('@id')
-        value = convert_value_from_cycle(product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id)
+        value = convert_value_from_cycle(
+            impact_assessment, product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id
+        )
 
         indicator = _new_indicator(emission.get('term', {}), emission.get('methodModel'))
         indicator['value'] = value
@@ -57,13 +61,23 @@ def _indicator(product: dict):
     return run
 
 
+def _log_missing_emissions(impact_assessment: dict, emissions: list):
+    term_ids = cycle_emissions_in_system_boundary(cycle=impact_assessment.get('cycle', {}))
+    emission_ids = list(map(lambda e: e.get('term', {}).get('@id'), emissions))
+    missing_term_ids = [term_id for term_id in term_ids if term_id not in emission_ids]
+    for term_id in missing_term_ids:
+        logRequirements(impact_assessment, model=MODEL, term=term_id,
+                        present_in_cycle=False)
+        logShouldRun(impact_assessment, MODEL, term_id, should_run=False)
+
+
 def _should_run_emission(impact_assessment: dict):
     product = get_product(impact_assessment)
 
     def exec(emission: dict):
         term_id = emission.get('term', {}).get('@id')
         has_value = convert_value_from_cycle(
-            product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id
+            impact_assessment, product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id
         ) is not None
         not_deleted = emission.get('deleted', False) is not True
 
@@ -91,4 +105,5 @@ def run(impact_assessment: dict):
     should_run, product = _should_run(impact_assessment)
     emissions = impact_assessment.get('cycle', {}).get(MODEL_KEY, []) if should_run else []
     emissions = list(filter(_should_run_emission(impact_assessment), emissions))
-    return list(map(_indicator(product), emissions))
+    _log_missing_emissions(impact_assessment, emissions)
+    return list(map(_indicator(impact_assessment, product), emissions))

hestia_earth/models/ipcc2019/co2ToAirCarbonStockChange_utils.py

@@ -47,7 +47,6 @@ _NOMINAL_ERROR = 75
 Carbon stock measurements without an associated `sd` should be assigned a nominal error of 75% (2*sd as a percentage of
 the mean).
 """
-_DEFAULT_YEARS_SINCE_LUC_EVENT = 999
 _TRANSITION_PERIOD_YEARS = 20
 _TRANSITION_PERIOD_DAYS = 20 * YEAR  # 20 years in days
 _VALID_DATE_FORMATS = {
@@ -96,6 +95,7 @@ class _InventoryKey(Enum):
     LAND_USE_SUMMARY = "land-use-summary"
     LAND_USE_CHANGE_EVENT = "luc-event"
     YEARS_SINCE_LUC_EVENT = "years-since-luc-event"
+    YEARS_SINCE_INVENTORY_START = "years-since-inventory-start"
 
 
 CarbonStock = NamedTuple("CarbonStock", [
@@ -611,7 +611,6 @@ def _create_compile_inventory_function(
         )
 
         inventory = _squash_inventory(
-            cycle_id,
             cycle_inventory,
             carbon_stock_inventory,
             land_use_inventory,
@@ -1008,14 +1007,20 @@ def _compile_land_use_inventory(
         is_luc_event = detect_land_use_change_func(land_use_summary, prev_land_use_summary)
 
         time_delta = current_year - prev_year
-        prev_years_since_luc_event = result.get(prev_year, {}).get(_InventoryKey.YEARS_SINCE_LUC_EVENT, 0)
+        prev_years_since_luc_event = (
+            result.get(prev_year, {}).get(_InventoryKey.YEARS_SINCE_LUC_EVENT, _TRANSITION_PERIOD_YEARS)
+        )
+        prev_years_since_inventory_start = result.get(prev_year, {}).get(_InventoryKey.YEARS_SINCE_INVENTORY_START, 0)
+
         years_since_luc_event = time_delta if is_luc_event else prev_years_since_luc_event + time_delta
+        years_since_inventory_start = prev_years_since_inventory_start + time_delta
 
         update_dict = {
             current_year: {
                 _InventoryKey.LAND_USE_SUMMARY: land_use_summary,
                 _InventoryKey.LAND_USE_CHANGE_EVENT: is_luc_event,
                 _InventoryKey.YEARS_SINCE_LUC_EVENT: years_since_luc_event,
+                _InventoryKey.YEARS_SINCE_INVENTORY_START: years_since_inventory_start
             }
         }
         return result | update_dict
@@ -1030,9 +1035,7 @@ def _compile_land_use_inventory(
             start_year: {
                 _InventoryKey.LAND_USE_SUMMARY: summarise_land_use_func(
                     land_cover_nodes_by_year.get(start_year, [])
-                ),
-                _InventoryKey.LAND_USE_CHANGE_EVENT: False,
-                _InventoryKey.YEARS_SINCE_LUC_EVENT: _DEFAULT_YEARS_SINCE_LUC_EVENT
+                )
             }
         }
     ) if should_run else {}
@@ -1062,7 +1065,6 @@ def _sorted_merge(*sources: Union[dict, list[dict]]) -> dict:
 
 
 def _squash_inventory(
-    cycle_id: str,
     cycle_inventory: dict,
     carbon_stock_inventory: dict,
     land_use_inventory: dict,
@@ -1075,8 +1077,6 @@ def _squash_inventory(
 
     Parameters
     ----------
-    cycle_id : str
-        The unique identifier of the cycle being processed.
 
     cycle_inventory : dict
         A dictionary representing the share of emissions for each cycle, grouped by year.
@@ -1162,12 +1162,7 @@ def _squash_inventory(
     )))
 
     def should_run_group(method: MeasurementMethodClassification, year: int) -> bool:
-        carbon_stock_inventory_group = carbon_stock_inventory.get(method, {}).get(year, {})
-        share_of_emissions_group = cycle_inventory.get(year, {})
-
-        has_emission = _InventoryKey.CO2_EMISSION in carbon_stock_inventory_group.keys()
-        is_relevant_for_cycle = cycle_id in share_of_emissions_group.get(_InventoryKey.SHARE_OF_EMISSION, {}).keys()
-        return all([has_emission, is_relevant_for_cycle])
+        return _InventoryKey.CO2_EMISSION in carbon_stock_inventory.get(method, {}).get(year, {}).keys()
 
     def squash(result: dict, year: int) -> dict:
         update_dict = next(
@@ -1175,7 +1170,10 @@ def _squash_inventory(
                 {
                     year: {
                         **_get_land_use_change_data(year, land_use_inventory),
-                        **reduce(merge, [carbon_stock_inventory[method][year], cycle_inventory[year]], dict())
+                        **reduce(merge, [
+                            carbon_stock_inventory.get(method, {}).get(year, {}),
+                            cycle_inventory.get(year, {})
+                        ], dict())
                     }
                 } for method in measurement_method_ranking if should_run_group(method, year)
             ),
@@ -1205,15 +1203,22 @@ def _get_land_use_change_data(
         )
     )
 
-    delta_time = closest_inventory_year - year if closest_inventory_year else 0
-    inventory_data = land_use_inventory.get(closest_inventory_year, {})
+    time_delta = closest_inventory_year - year if closest_inventory_year else 0
+    prev_years_since_luc_event = (
+        land_use_inventory.get(closest_inventory_year, {}).get(_InventoryKey.YEARS_SINCE_LUC_EVENT)
+    )
+    prev_years_since_inventory_start = (
+        land_use_inventory.get(closest_inventory_year, {}).get(_InventoryKey.YEARS_SINCE_INVENTORY_START)
+    )
 
-    years_since_luc_event = (
-        inventory_data.get(_InventoryKey.YEARS_SINCE_LUC_EVENT, _DEFAULT_YEARS_SINCE_LUC_EVENT) - delta_time
+    years_since_luc_event = prev_years_since_luc_event - time_delta if prev_years_since_luc_event else None
+    years_since_inventory_start = (
+        prev_years_since_inventory_start - time_delta if prev_years_since_inventory_start else None
     )
 
     return {
-        _InventoryKey.YEARS_SINCE_LUC_EVENT: years_since_luc_event
+        _InventoryKey.YEARS_SINCE_LUC_EVENT: years_since_luc_event,
+        _InventoryKey.YEARS_SINCE_INVENTORY_START: years_since_inventory_start
     }
 
 
@@ -1408,25 +1413,54 @@ def create_run_function(
         """
         data = inventory[year]
         years_since_luc_event = data[_InventoryKey.YEARS_SINCE_LUC_EVENT]
-        emission_term_id = (
-            land_use_change_emission_term_id if years_since_luc_event <= _TRANSITION_PERIOD_YEARS
-            else management_change_emission_term_id
-        )
+        years_since_inventory_start = data[_InventoryKey.YEARS_SINCE_INVENTORY_START]
+
+        is_luc_emission = bool(years_since_luc_event) and years_since_luc_event <= _TRANSITION_PERIOD_YEARS
+        is_data_complete = bool(years_since_inventory_start) and years_since_inventory_start > _TRANSITION_PERIOD_YEARS
+
+        if is_luc_emission:
+            # If LUC emission allocate emissions to land use change AND add corresponding zero emission to management
+            emission_term_id = land_use_change_emission_term_id
+            zero_emission_term_id = management_change_emission_term_id
+        elif is_data_complete:
+            # If management emission && data complete allocate emissions to management change AND add corresponding
+            # zero emission to management
+            emission_term_id = management_change_emission_term_id
+            zero_emission_term_id = land_use_change_emission_term_id
+        else:
+            # If management emission, but not data complete allocate emissions to management change only
+            emission_term_id = management_change_emission_term_id
+            zero_emission_term_id = None
 
         rescaled_emission = _rescale_carbon_stock_change_emission(
             data[_InventoryKey.CO2_EMISSION], data[_InventoryKey.SHARE_OF_EMISSION][cycle_id]
         )
 
+        zero_emission = CarbonStockChangeEmission(
+            value=array(0),
+            start_date=rescaled_emission.start_date,
+            end_date=rescaled_emission.end_date,
+            method=rescaled_emission.method
+        ) if zero_emission_term_id else None
+
         previous_emission = result.get(emission_term_id)
+        previous_zero_emission = result.get(zero_emission_term_id)
 
-        update_dict = {
+        emission_dict = {
             emission_term_id: (
                 _add_carbon_stock_change_emissions(previous_emission, rescaled_emission) if previous_emission
                 else rescaled_emission
             )
         }
 
-        return result | update_dict
+        zero_emission_dict = {
+            zero_emission_term_id: (
+                _add_carbon_stock_change_emissions(previous_zero_emission, zero_emission) if previous_zero_emission
+                else zero_emission
+            )
+        } if zero_emission_term_id else {}
+
+        return result | emission_dict | zero_emission_dict
 
     def run(cycle_id: str, cycle_start_date: str, cycle_end_date: str, inventory: dict) -> list[dict]:
         """
@@ -1447,9 +1481,13 @@ def create_run_function(
         list[dict]
             A list of [Emission](https://www.hestia.earth/schema/Emission) nodes containing model results.
         """
+
+        def should_run_year(year: int) -> bool:
+            return cycle_id in inventory.get(year, {}).get(_InventoryKey.SHARE_OF_EMISSION, {}).keys()
+
         assigned_emissions = reduce(
             lambda result, year: reduce_emissions(result, year, cycle_id, inventory),
-            inventory.keys(),
+            (year for year in inventory.keys() if should_run_year(year)),
             {}
         )