hestia-earth-models 0.65.3__py3-none-any.whl → 0.65.5__py3-none-any.whl

hestia_earth/models/hestia/resourceUse_utils.py

@@ -0,0 +1,200 @@
+"""
+Resource Use
+
+Provides common code for land transformation models.
+"""
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.tools import to_precision
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.blank_node import _gapfill_datestr, DatestrGapfillMode, DatestrFormat, _str_dates_match
+from hestia_earth.models.utils.impact_assessment import get_site
+from hestia_earth.models.utils.indicator import _new_indicator
+from .utils import (
+    LAND_USE_TERMS_FOR_TRANSFORMATION,
+    crop_ipcc_land_use_category,
+)
+from . import MODEL
+
+_MAXIMUM_OFFSET_DAYS = 365 * 2
+_OUTPUT_SIGNIFICANT_DIGITS = 3
+_RESOURCE_USE_TERM_ID = 'landOccupationDuringCycle'
+
+
+def _new_indicator_with_value(
+    term_id: str,
+    land_cover_id: str,
+    previous_land_cover_id: str,
+    value: float
+) -> dict:
+    indicator = _new_indicator(
+        term=term_id,
+        model=MODEL,
+        land_cover_id=land_cover_id,
+        previous_land_cover_id=previous_land_cover_id
+    )
+    indicator["value"] = to_precision(number=value, digits=_OUTPUT_SIGNIFICANT_DIGITS) if value != 0 else 0
+    return indicator
+
+
+def _gap_filled_date_obj(date_str: str) -> datetime:
+    return datetime.strptime(
+        _gapfill_datestr(datestr=date_str, mode=DatestrGapfillMode.MIDDLE),
+        DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND.value
+    )
+
+
+def _should_run_close_date_found(
+    ia_end_date_str: str,
+    management_nodes: list,
+    historic_date_offset: int
+) -> tuple[bool, str]:
+    historic_ia_date_obj = (
+        _gap_filled_date_obj(ia_end_date_str) - relativedelta(years=historic_date_offset)
+        if ia_end_date_str else None
+    )
+    # Calculate all distances in days which are less than MAXIMUM_OFFSET_DAYS from historic date
+    # Assumption: if there are two dates are equidistant from the target chose the second.
+    filtered_dates = {
+        abs((_gap_filled_date_obj(node.get("endDate")) - historic_ia_date_obj).days): node.get("endDate")
+        for node in management_nodes
+        if node.get("term", {}).get("termType", "") == TermTermType.LANDCOVER.value and
+        abs((_gap_filled_date_obj(node.get("endDate")) - historic_ia_date_obj).days) <= _MAXIMUM_OFFSET_DAYS
+    }
+    nearest_date = filtered_dates[min(filtered_dates.keys())] if filtered_dates else ""
+
+    return nearest_date != "", nearest_date
+
+
+def should_run(
+    impact_assessment: dict,
+    site: dict,
+    term_id: str,
+    historic_date_offset: int
+) -> tuple[bool, dict, str]:
+    relevant_emission_resource_use = [
+        node for node in impact_assessment.get("emissionsResourceUse", [])
+        if node.get("term", {}).get("@id", "") == _RESOURCE_USE_TERM_ID and node.get("value", -1) >= 0
+    ]
+
+    filtered_management_nodes = [
+        node for node in site.get("management", [])
+        if node.get("value", -1) >= 0 and node.get("term", {}).get("termType", "") == TermTermType.LANDCOVER.value
+    ]
+    current_node_index = next(
+        (i for i, node in enumerate(filtered_management_nodes)
+         if _str_dates_match(node.get("endDate", ""), impact_assessment.get("endDate", ""))),
+        None
+    )
+    current_node = filtered_management_nodes.pop(current_node_index) if current_node_index is not None else None
+
+    close_date_found, closest_date_str = _should_run_close_date_found(
+        ia_end_date_str=impact_assessment.get("endDate", ""),
+        management_nodes=filtered_management_nodes,
+        historic_date_offset=historic_date_offset
+    )
+
+    logRequirements(
+        log_node=impact_assessment,
+        model=MODEL,
+        term_id=term_id,
+        site=site
+    )
+
+    should_run_result = all([
+        relevant_emission_resource_use != [],
+        current_node,
+        close_date_found
+    ])
+    logShouldRun(site, MODEL, term=term_id, should_run=should_run_result)
+
+    return should_run_result, current_node, closest_date_str
+
+
+def _get_land_occupation_for_land_use_type(impact_assessment: dict, ipcc_land_use_category: str) -> float:
+    """
+    Returns the sum of all land occupation for the specified land_use_category.
+    """
+    return sum(
+        node.get("value", 0) for node in impact_assessment.get("emissionsResourceUse", [])
+        if node.get("term", {}).get("@id", "") == _RESOURCE_USE_TERM_ID
+        and crop_ipcc_land_use_category(node.get("landCover", {}).get("@id", "")) == ipcc_land_use_category
+    )
+
+
+def _calculate_indicator_value(
+    impact_assessment: dict,
+    management_nodes: list,
+    ipcc_land_use_category: str,
+    previous_land_cover_id: str,
+    historic_date_offset: int
+) -> float:
+    """
+    Land transformation from [land type] previous management nodes
+     = (Land occupation, during Cycle * Historic Site Percentage Area [land type] / 100) / HISTORIC_DATE_OFFSET
+    """
+    land_occupation_for_cycle = _get_land_occupation_for_land_use_type(
+        impact_assessment=impact_assessment,
+        ipcc_land_use_category=ipcc_land_use_category
+    )
+    historical_land_use = sum(
+        node.get("value", 0) for node in management_nodes
+        if node.get("term", {}).get("@id", "") == previous_land_cover_id
+    )
+    return ((land_occupation_for_cycle * historical_land_use) / 100) / historic_date_offset
+
+
+def _run_calculate_transformation(
+    term_id: str,
+    current_node: dict,
+    closest_date_str: str,
+    impact_assessment: dict,
+    site: dict,
+    historic_date_offset: int
+) -> list:
+    """
+    Calculate land transformation for all land use categories.
+    """
+    indicators = [
+        _new_indicator_with_value(
+            term_id=term_id,
+            land_cover_id=current_node.get("term", {}).get("@id"),
+            previous_land_cover_id=previous_land_cover_id,
+            value=_calculate_indicator_value(
+                impact_assessment=impact_assessment,
+                management_nodes=[
+                    node for node in site.get("management", [])
+                    if _str_dates_match(node.get("endDate", ""), closest_date_str)
+                ],
+                ipcc_land_use_category=crop_ipcc_land_use_category(current_node.get("term", {}).get("@id", "")),
+                previous_land_cover_id=previous_land_cover_id,
+                historic_date_offset=historic_date_offset
+            )
+        ) for previous_land_cover_id in [t[0] for t in LAND_USE_TERMS_FOR_TRANSFORMATION.values()]
+    ]
+
+    return indicators
+
+
+def run_resource_use(
+    impact_assessment: dict,
+    historic_date_offset: int,
+    term_id: str
+) -> list:
+    site = get_site(impact_assessment)
+    _should_run, current_node, closest_date_str = should_run(
+        impact_assessment=impact_assessment,
+        site=site,
+        term_id=term_id,
+        historic_date_offset=historic_date_offset
+    )
+    return _run_calculate_transformation(
+        term_id=term_id,
+        current_node=current_node,
+        closest_date_str=closest_date_str,
+        site=site,
+        impact_assessment=impact_assessment,
+        historic_date_offset=historic_date_offset
+    ) if _should_run else []

hestia_earth/models/hestia/seed_emissions.py

@@ -18,8 +18,8 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list, flatten, list_sum, safe_parse_float
 from hestia_earth.utils.emission import cycle_emissions_in_system_boundary
 
-from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
-from hestia_earth.models.utils import _omit
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table, debugValues
+from hestia_earth.models.utils import _omit, group_by
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.site import valid_site_type
 from hestia_earth.models.utils.cycle import cycle_end_year
@@ -54,7 +54,7 @@ REQUIREMENTS = {
             "siteType": ["cropland", "glass or high accessible cover"],
             "country": {"@type": "Term", "termType": "region"}
         },
-        "emissions": [{"@type": "Emission"}]
+        "emissions": [{"@type": "Emission", "value": ""}]
     }
 }
 RETURNS = {
@@ -86,11 +86,24 @@ def _emission(term_id: str, value: float, input: dict):
     return emission
 
 
-def _run(cycle: dict, economicValueShare: float, total_yield: float, seed_input: dict, grouped_emissions: dict):
-    term = seed_input.get('term', {})
+def _run_emission(
+    cycle: dict, economicValueShare: float, total_yield: float, seed_input: dict, term_id: str, emission_value: float
+):
+    input_term = seed_input.get('term', {})
+    input_term_id = input_term.get('@id')
     seed_value = list_sum(seed_input.get('value'))
+    value = emission_value * economicValueShare / 100 / total_yield * seed_value
+    debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                value=value,
+                coefficient=1,
+                input=input_term_id)
+
+    return _emission(term_id, value, input_term)
+
+
+def _run(cycle: dict, economicValueShare: float, total_yield: float, seed_input: dict, grouped_emissions: dict):
     return [
-        _emission(term_id, emission_value * economicValueShare / 100 / total_yield * seed_value, term)
+        _run_emission(cycle, economicValueShare, total_yield, seed_input, term_id, emission_value)
         for term_id, emission_value in grouped_emissions.items()
     ]
 
@@ -102,12 +115,12 @@ def _filter_emissions(cycle: dict):
         {
             'id': i.get('term', {}).get('@id'),
             'group-id': get_lookup_value(i.get('term', {}), LOOKUPS['emission'], model=MODEL, model_key=MODEL_KEY),
-            'value': list_sum(i.get('value'), 0)
+            'value': list_sum(i.get('value'))
         }
         for i in cycle.get('emissions', [])
         if all([
             i.get('term', {}).get('@id') in required_emission_term_ids,
-            list_sum(i.get('value'), 0) > 0
+            len(i.get('value', [])) > 0
         ])
     ]
     emission_ids = set([v.get('id') for v in emissions])
@@ -120,7 +133,7 @@ def _filter_emissions(cycle: dict):
             'id': group_id,
             'emissions': list(filter(
                 lambda id: id in required_emission_term_ids,
-                list(lookup[lookup[column_name('inputProductionGroupId')] == group_id].termid)
+                set(list(lookup[lookup[column_name('inputProductionGroupId')] == group_id].termid))
             ))
         }
         for group_id in group_ids
@@ -129,8 +142,8 @@ def _filter_emissions(cycle: dict):
         {
             'id': group.get('id'),
             'total-emissions': len(group.get('emissions', [])),
-            'included-emissions': len([v in emission_ids for v in group.get('emissions', [])]),
-            'missing-emissions': '-'.join([v for v in group.get('emissions', []) if v not in emission_ids])
+            'included-emissions': len(list(filter(lambda v: v in emission_ids, group.get('emissions', [])))),
+            'missing-emissions': '-'.join(list(filter(lambda v: v not in emission_ids, group.get('emissions', []))))
         }
         for group in emissions_per_group
     ]
@@ -154,23 +167,18 @@ def _evs(product: dict):
     ) or product.get('economicValueShare')
 
 
-def _yield(country_id: str, end_year: int, product: dict):
+def _faostat_yield(country_id: str, end_year: int, product: dict):
     grouping = get_crop_grouping_faostat_production(MODEL, product.get('term', {}))
     return safe_parse_float(extract_grouped_data_closest_date(get_table_value(
         download_lookup('region-crop-cropGroupingFaostatProduction-yield.csv'),
         'termid',
         country_id,
         column_name(grouping)
-    ), end_year)) or list_sum(product.get('value'))
+    ), end_year))
 
 
 def _group_seed_inputs(inputs: list):
-    def _group_by(group: dict, input: dict):
-        term_id = input.get('term', {}).get('@id')
-        group[term_id] = group.get(term_id, []) + [input]
-        return group
-
-    grouped_inputs = reduce(_group_by, inputs, {})
+    grouped_inputs = group_by(inputs, ['term.@id'])
     return [
         inputs[0] | {'value': flatten([v.get('value') for v in inputs])}
         for inputs in grouped_inputs.values()
@@ -191,10 +199,11 @@ def _should_run(cycle: dict):
         {
             'product': product.get('term', {}).get('@id'),
             'seed-id': get_lookup_value(
-                product.get('term', {}), 'correspondingSeedTermIds', model=MODEL, key=MODEL_KEY) or None,
+                product.get('term', {}), 'correspondingSeedTermIds', model=MODEL, model_key=MODEL_KEY) or None,
             'economicValueShare': _evs(product),
-            'yield': _yield(country_id, end_year, product),
-            'landCover-id': get_landCover_term_id(product.get('term', {}), model=MODEL, key=MODEL_KEY)
+            'FAOSTAT-yield': _faostat_yield(country_id, end_year, product),
+            'product-yield': list_sum(product.get('value')),
+            'landCover-id': get_landCover_term_id(product.get('term', {}), model=MODEL, model_key=MODEL_KEY)
         }
         for product in crop_products
     ]
@@ -202,7 +211,7 @@ def _should_run(cycle: dict):
         value for value in crop_products if all([
             value.get('seed-id'),
             value.get('economicValueShare'),
-            value.get('yield'),
+            value.get('FAOSTAT-yield') or value.get('product-yield'),
             value.get('landCover-id'),
         ])
     ]
@@ -214,7 +223,7 @@ def _should_run(cycle: dict):
         {
             'input': i,
             'is-corresponding-seed': i.get('term', {}).get('@id') in seed_term_ids,
-            'input-value': i.get('value'),
+            'input-value': list_sum(i.get('value'), default=None),
             'has-linked-impact-assessment': bool(i.get('impactAssessment')),
             'is-fromCycle': i.get('fromCycle', False),
             'is-producedInCycle': i.get('producedInCycle', False),
@@ -226,7 +235,7 @@ def _should_run(cycle: dict):
         v.get('input') for v in seed_inputs
         if all([
             v.get('is-corresponding-seed', False),
-            list_sum(v.get('input-value') or [-1]) > 0,
+            v.get('input-value') or -1 > 0,
             not v.get('has-linked-impact-assessment'),
             not v.get('is-fromCycle'),
             not v.get('is-producedInCycle'),
@@ -235,12 +244,12 @@ def _should_run(cycle: dict):
 
     crop_land_cover_ids = list(set([p.get('landCover-id') for p in valid_crop_products]))
     total_economicValueShare = list_sum([p.get('economicValueShare') for p in valid_crop_products])
-    total_yield = list_sum([p.get('yield') for p in valid_crop_products])
+    total_yield = list_sum([p.get('FAOSTAT-yield') or p.get('product-yield') for p in valid_crop_products])
 
     emissions, emissions_per_group = _filter_emissions(cycle)
     # group emissions with the same group-id
     grouped_emissions = reduce(
-        lambda p, c: p | {c.get('group-id'): p.get(c.get('group-id'), 0) + c.get('value', 0)},
+        lambda p, c: p | {c.get('group-id'): p.get(c.get('group-id'), 0) + (c.get('value') or 0)},
         emissions,
         {}
     )
@@ -271,7 +280,7 @@ def _should_run(cycle: dict):
                         emissions_per_group=log_as_table(emissions_per_group),
                         **_omit(seed_input, 'input'))
 
-        logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY)
+        logShouldRun(cycle, MODEL, term_id, should_run, methodTier=TIER, model_key=MODEL_KEY)
 
     return should_run, total_economicValueShare, total_yield, grouped_seed_inputs, grouped_emissions
 

hestia_earth/models/hestia/utils.py

@@ -0,0 +1,48 @@
+from hestia_earth.schema import TermTermType
+
+from hestia_earth.models.utils.term import get_lookup_value
+from . import MODEL
+
+IPCC_LAND_USE_CATEGORY_ANNUAL = "Annual crops"
+IPCC_LAND_USE_CATEGORY_PERENNIAL = "Perennial crops"
+TOTAL_CROPLAND = "Cropland"
+ANNUAL_CROPLAND = "Arable land"
+FOREST_LAND = "Forest land"
+OTHER_LAND = "Other land"
+PERMANENT_CROPLAND = "Permanent crops"
+PERMANENT_PASTURE = "Permanent meadows and pastures"
+TOTAL_AGRICULTURAL_CHANGE = "Total agricultural change"
+ALL_LAND_USE_TERMS = [
+    FOREST_LAND,
+    TOTAL_CROPLAND,
+    ANNUAL_CROPLAND,
+    PERMANENT_CROPLAND,
+    PERMANENT_PASTURE,
+    OTHER_LAND
+]
+
+# Mapping from Land use terms to Management node terms.
+# land use term: (@id, name)
+LAND_USE_TERMS_FOR_TRANSFORMATION = {
+    FOREST_LAND: ("forest", "Forest"),
+    ANNUAL_CROPLAND: ("annualCropland", "Annual cropland"),
+    PERMANENT_CROPLAND: ("permanentCropland", "Permanent cropland"),
+    PERMANENT_PASTURE: ("permanentPasture", "Permanent pasture"),
+    OTHER_LAND: ("otherLand", OTHER_LAND)  # Not used yet
+}
+
+
+def crop_ipcc_land_use_category(
+    crop_term_id: str,
+    lookup_term_type: str = TermTermType.LANDCOVER.value
+) -> str:
+    """
+    Looks up the crop in the lookup.
+    Returns the IPCC_LAND_USE_CATEGORY.
+    """
+    return get_lookup_value(
+        lookup_term={"@id": crop_term_id, "type": "Term", "termType": lookup_term_type},
+        column='IPCC_LAND_USE_CATEGORY',
+        model=MODEL,
+        term={"@id": crop_term_id, "type": "Term", "termType": lookup_term_type}
+    )

hestia_earth/models/impact_assessment/emissions.py

@@ -4,9 +4,11 @@ Emissions
 Creates an [Indicator](https://hestia.earth/schema/Indicator) for every [Emission](https://hestia.earth/schema/Emission)
 contained within the [ImpactAssesment.cycle](https://hestia.earth/schema/ImpactAssessment#cycle).
 It does this by dividing the Emission amount by the Product amount, and applying an allocation between co-products.
+Note: for any Emission in the system boundary that does not exist in the Cycle, it will log the model as failed,
+so that we know the Emission is missing.
 """
-
 from hestia_earth.utils.tools import list_sum
+from hestia_earth.utils.emission import cycle_emissions_in_system_boundary
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.impact_assessment import get_product, convert_value_from_cycle
@@ -38,10 +40,12 @@ RETURNS = {
 MODEL_KEY = 'emissions'
 
 
-def _indicator(product: dict):
+def _indicator(impact_assessment: dict, product: dict):
     def run(emission: dict):
         term_id = emission.get('term', {}).get('@id')
-        value = convert_value_from_cycle(product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id)
+        value = convert_value_from_cycle(
+            impact_assessment, product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id
+        )
 
         indicator = _new_indicator(emission.get('term', {}), emission.get('methodModel'))
         indicator['value'] = value
@@ -57,13 +61,23 @@ def _indicator(product: dict):
     return run
 
 
+def _log_missing_emissions(impact_assessment: dict, emissions: list):
+    term_ids = cycle_emissions_in_system_boundary(cycle=impact_assessment.get('cycle', {}))
+    emission_ids = list(map(lambda e: e.get('term', {}).get('@id'), emissions))
+    missing_term_ids = [term_id for term_id in term_ids if term_id not in emission_ids]
+    for term_id in missing_term_ids:
+        logRequirements(impact_assessment, model=MODEL, term=term_id,
+                        present_in_cycle=False)
+        logShouldRun(impact_assessment, MODEL, term_id, should_run=False)
+
+
 def _should_run_emission(impact_assessment: dict):
     product = get_product(impact_assessment)
 
     def exec(emission: dict):
         term_id = emission.get('term', {}).get('@id')
         has_value = convert_value_from_cycle(
-            product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id
+            impact_assessment, product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id
         ) is not None
         not_deleted = emission.get('deleted', False) is not True
 
@@ -91,4 +105,5 @@ def run(impact_assessment: dict):
     should_run, product = _should_run(impact_assessment)
     emissions = impact_assessment.get('cycle', {}).get(MODEL_KEY, []) if should_run else []
     emissions = list(filter(_should_run_emission(impact_assessment), emissions))
-    return list(map(_indicator(product), emissions))
+    _log_missing_emissions(impact_assessment, emissions)
+    return list(map(_indicator(impact_assessment, product), emissions))