hestia-earth-models 0.71.0__py3-none-any.whl → 0.72.1__py3-none-any.whl

hestia_earth/models/config/Site.json

@@ -88,6 +88,14 @@
         "runStrategy": "add_blank_node_if_missing",
         "mergeStrategy": "list",
         "stage": 1
+      },
+      {
+        "key": "measurements",
+        "model": "hestia",
+        "value": "histosol",
+        "runStrategy": "add_blank_node_if_missing",
+        "mergeStrategy": "list",
+        "stage": 1
       }
     ],
     [

hestia_earth/models/environmentalFootprintV3_1/environmentalFootprintSingleOverallScore.py

@@ -1,8 +1,7 @@
-from typing import List, Optional, Tuple
-
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum
+from typing import List, Optional, Tuple
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table, debugValues
 from hestia_earth.models.utils.blank_node import get_lookup_value
@@ -89,6 +88,23 @@ def _indicator_factors(impact_assessment: dict, indicator: dict):
     }
 
 
+def _map_input_ids(value: dict) -> set[str]:
+    return set(map(lambda i: i.get('@id'), value.get('inputs', [])))
+
+
+def _count_duplicate_indicators(reference_indicator: dict, indicators: list) -> int:
+    """
+    Counts the number of `reference_indicator` indicators found in a list of indicators.
+    Uses indicator.term.@id and indicator.inputs to determine uniqueness.
+    """
+    return sum([
+        1
+        for i in indicators
+        if (i["term"]["@id"] == reference_indicator["term"]["@id"]) and (
+                    _map_input_ids(i) == _map_input_ids(reference_indicator))
+    ])
+
+
 def _indicator(value: float) -> dict:
     indicator = _new_indicator(TERM_ID, MODEL)
     indicator['value'] = value
@@ -115,7 +131,7 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list[dict]]:
     processed_indicators = [{
         "indicator": indicator['term']['@id'],
         "valid-indicator": _valid_indicator(indicator),
-        "one-indicator-for-category": sum(1 for i in indicators if i['term']['@id'] == indicator['term']['@id']) == 1,
+        "one-indicator-for-category": _count_duplicate_indicators(indicator, indicators) == 1,
         "indicator-pef-category": indicator.get('term', {}).get('@id'),
     } | _indicator_factors(impact_assessment, indicator) for indicator in indicators]
 

hestia_earth/models/geospatialDatabase/histosol.py

@@ -1,7 +1,7 @@
-from hestia_earth.schema import MeasurementMethodClassification, TermTermType
+from hestia_earth.schema import MeasurementMethodClassification
 
 from hestia_earth.models.log import logRequirements, logShouldRun
-from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.measurement import _new_measurement, total_other_soilType_value
 from hestia_earth.models.utils.source import get_source
 from .utils import download, has_geospatial_data, should_download
 from . import MODEL
@@ -14,7 +14,13 @@ REQUIREMENTS = {
             {"region": {"@type": "Term", "termType": "region"}}
         ],
         "none": {
-            "measurements": [{"@type": "Measurement", "value": "", "term.termType": "soilType"}]
+            "measurements": [{
+                "@type": "Measurement",
+                "value": "100",
+                "depthUpper": "0",
+                "depthLower": "30",
+                "term.termType": "soilType"
+            }]
         }
     }
 }
@@ -50,17 +56,18 @@ def _run(site: dict):
 
 
 def _should_run(site: dict):
-    measurements = site.get('measurements', [])
-    no_soil_type = all([m.get('term', {}).get('termType') != TermTermType.SOILTYPE.value for m in measurements])
     contains_geospatial_data = has_geospatial_data(site)
     below_max_area_size = should_download(TERM_ID, site)
 
+    total_measurements_value = total_other_soilType_value(site.get('measurements', []), TERM_ID)
+
     logRequirements(site, model=MODEL, term=TERM_ID,
                     contains_geospatial_data=contains_geospatial_data,
                     below_max_area_size=below_max_area_size,
-                    no_soil_type=no_soil_type)
+                    total_soilType_measurements_value=total_measurements_value,
+                    total_soilType_measurements_value_is_0=total_measurements_value == 0)
 
-    should_run = all([contains_geospatial_data, below_max_area_size, no_soil_type])
+    should_run = all([contains_geospatial_data, below_max_area_size, total_measurements_value == 0])
     logShouldRun(site, MODEL, TERM_ID, should_run)
     return should_run
 

hestia_earth/models/hestia/histosol.py

@@ -0,0 +1,53 @@
+from hestia_earth.schema import MeasurementMethodClassification
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.measurement import _new_measurement, total_other_soilType_value
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "measurements": [{
+            "@type": "Measurement",
+            "value": "100",
+            "depthUpper": "0",
+            "depthLower": "30",
+            "term.termType": "soilType"
+        }]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "0",
+        "depthUpper": "0",
+        "depthLower": "30",
+        "methodClassification": "modelled using other measurements"
+    }]
+}
+LOOKUPS = {
+    "soilType": "sumMax100Group"
+}
+TERM_ID = 'histosol'
+
+
+def _measurement():
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = [0]
+    measurement['depthUpper'] = 0
+    measurement['depthLower'] = 30
+    measurement['methodClassification'] = MeasurementMethodClassification.MODELLED_USING_OTHER_MEASUREMENTS.value
+    return measurement
+
+
+def _should_run(site: dict):
+    total_measurements_value = total_other_soilType_value(site.get('measurements', []), TERM_ID)
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    total_soilType_measurements_value=total_measurements_value,
+                    total_soilType_measurements_value_is_100=total_measurements_value == 100)
+
+    should_run = all([total_measurements_value == 100])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+    return should_run
+
+
+def run(site: dict): return [_measurement()] if _should_run(site) else []

hestia_earth/models/hestia/landCover.py

@@ -148,7 +148,7 @@ def cap_values(dictionary: dict, lower_limit: float = 0, upper_limit: float = 1)
 
 def site_area_sum_to_100(dict_of_percentages: dict):
     return False if dict_of_percentages == {} else (
-        math.isclose(sum(dict_of_percentages.values()), 1.0, rel_tol=0.01) or
+        math.isclose(sum(dict_of_percentages.values()), 1.0, rel_tol=0.05) or
         math.isclose(sum(dict_of_percentages.values()), 0.0, rel_tol=0.01)
     )
 
@@ -581,6 +581,14 @@ def _get_year_from_landCover(node: dict):
     return int(date[:4])
 
 
+def _scale_site_area_errors(site_area: dict) -> dict:
+    """Redistribute the result of any rounding error in proportion to the other land use types."""
+    # Positive errors would not have been capped, so won't be missing.
+    negative_errors = [v for v in site_area.values() if v < 0.0]
+    return {k: v + negative_errors[0] * v for k, v in site_area.items()} \
+        if negative_errors and abs(negative_errors[0]) < 1 and all([v < 1 for v in site_area.values()]) else site_area
+
+
 def _should_run_historical_land_use_change(site: dict, nodes: list, land_use_type: str) -> tuple[bool, dict]:
     # Assume a single management node for single-cropping.
     return _should_run_historical_land_use_change_single_crop(
@@ -728,7 +736,7 @@ def _should_run_historical_land_use_change_single_crop(
         if land_type != land_use_type
     }
     site_area[land_use_type] = 1 - sum(site_area.values())
-    capped_site_area = cap_values(dictionary=site_area, lower_limit=0, upper_limit=1)
+    capped_site_area = cap_values(dictionary=_scale_site_area_errors(site_area))
 
     sum_of_site_areas_is_100 = site_area_sum_to_100(capped_site_area)
     site_type_allowed = site.get("siteType") in SITE_TYPES

hestia_earth/models/ipcc2019/co2ToAirUreaHydrolysis.py

@@ -2,7 +2,8 @@ from hestia_earth.schema import EmissionMethodTier
 from hestia_earth.utils.tools import list_sum, safe_parse_float, non_empty_list
 from hestia_earth.utils.model import find_term_match
 
-from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils import multiply_values
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.term import get_urea_terms
@@ -36,66 +37,65 @@ def _emission(value: float):
     return emission
 
 
-def _urea_input_value(cycle: dict):
-    def exec(data: dict):
-        term_id = data.get('id')
-        values = data.get('values')
-        coeff = safe_parse_float(get_term_lookup(term_id, LOOKUPS['inorganicFertiliser'][2]), 1)
-        debugValues(cycle, model=MODEL, term=TERM_ID,
-                    product=term_id,
-                    coefficient=coeff)
-        return list_sum(values) * coeff
-    return exec
+def _urea_emission_factor(term_id: str):
+    return safe_parse_float(get_term_lookup(term_id, LOOKUPS['inorganicFertiliser'][2]), None)
 
 
-def _run(cycle: dict, urea_values: list):
-    value = list_sum(list(map(_urea_input_value(cycle), urea_values)))
+def _run(urea_values: list):
+    value = list_sum([v.get('value') * v.get('factor') for v in urea_values if v.get('value')])
     return [_emission(value)]
 
 
-def _get_urea_values(cycle: dict, inputs: list, term_id: str):
+def _get_urea_value(cycle: dict, inputs: list, term_id: str):
     inputs = list(filter(lambda i: i.get('term', {}).get('@id') == term_id, inputs))
     values = [list_sum(i.get('value'), 0) for i in inputs if len(i.get('value', [])) > 0]
-    return [0] if len(inputs) == 0 and _is_term_type_complete(cycle, 'fertiliser') else values
+    return list_sum(values, default=None)
 
 
 def _should_run(cycle: dict):
+    is_fertiliser_complete = _is_term_type_complete(cycle, 'fertiliser')
     inputs = cycle.get('inputs', [])
     term_ids = get_urea_terms()
 
     country_id = cycle.get('site', {}).get('country', {}).get('@id')
     urea_share = get_country_breakdown(MODEL, TERM_ID, country_id, LOOKUPS['inorganicFertiliser'][0])
     uan_share = get_country_breakdown(MODEL, TERM_ID, country_id, LOOKUPS['inorganicFertiliser'][1])
-    urea_unspecified_as_n = list_sum(find_term_match(inputs, UNSPECIFIED_TERM_ID).get('value', []))
+    urea_unspecified_as_n = list_sum(find_term_match(inputs, UNSPECIFIED_TERM_ID).get('value', []), default=None)
 
     urea_values = [
         {
-            'id': id,
-            'values': _get_urea_values(cycle, inputs, id)
-        } for id in term_ids
+            'id': term_id,
+            'value': _get_urea_value(cycle, inputs, term_id),
+            'factor': _urea_emission_factor(term_id)
+        } for term_id in term_ids
     ] + non_empty_list([
         {
             'id': 'ureaKgN',
-            'values': [urea_unspecified_as_n * urea_share]
-        } if urea_share is not None else None,
+            'value': multiply_values([urea_unspecified_as_n, urea_share]),
+            'factor': _urea_emission_factor('ureaKgN')
+        },
         {
             'id': 'ureaAmmoniumNitrateKgN',
-            'values': [urea_unspecified_as_n * uan_share]
-        } if urea_share is not None else None
-    ] if urea_unspecified_as_n > 0 else [])
-    has_urea_value = any([len(data.get('values')) > 0 for data in urea_values])
+            'value': multiply_values([urea_unspecified_as_n, uan_share]),
+            'factor': _urea_emission_factor('ureaAmmoniumNitrateKgN')
+        }
+    ] if urea_unspecified_as_n is not None else [])
+
+    has_urea_value = any([data.get('value') is not None for data in urea_values])
 
     logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    is_term_type_fertiliser_complete=is_fertiliser_complete,
                     has_urea_value=has_urea_value,
+                    urea_values=log_as_table(urea_values),
                     urea_unspecified_as_n=urea_unspecified_as_n,
                     urea_share=urea_share,
                     uan_share=uan_share)
 
-    should_run = all([has_urea_value])
+    should_run = has_urea_value or is_fertiliser_complete
     logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
     return should_run, urea_values
 
 
 def run(cycle: dict):
     should_run, urea_values = _should_run(cycle)
-    return _run(cycle, urea_values) if should_run else []
+    return _run(urea_values) if should_run else []

hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_1.py

@@ -3,17 +3,18 @@ from functools import reduce
 from numpy import empty_like, random, vstack
 from numpy.typing import NDArray
 from pydash.objects import merge
-from typing import Callable, Optional, Union
+from typing import Callable, Literal, Optional, Union
 
 from hestia_earth.schema import MeasurementMethodClassification, SiteSiteType, TermTermType
-from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.blank_node import get_node_value
+from hestia_earth.utils.model import find_term_match, filter_list_term_type
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.utils import split_on_condition
 from hestia_earth.models.utils.array_builders import gen_seed
 from hestia_earth.models.utils.blank_node import (
-    cumulative_nodes_match, cumulative_nodes_lookup_match, cumulative_nodes_term_match, node_lookup_match,
-    node_term_match, group_nodes_by_year, validate_start_date_end_date
+    cumulative_nodes_match, cumulative_nodes_lookup_match, cumulative_nodes_term_match, group_by_term,
+    node_lookup_match, node_term_match, group_nodes_by_year, validate_start_date_end_date
 )
 from hestia_earth.models.utils.ecoClimateZone import EcoClimateZone, get_eco_climate_zone_value
 from hestia_earth.models.utils.descriptive_stats import calc_descriptive_stats
@@ -1031,8 +1032,8 @@ def _assign_ipcc_soil_category(
     IpccSoilCategory
         The assigned IPCC soil category.
     """
-    soil_types = filter_list_term_type(measurement_nodes, TermTermType.SOILTYPE)
-    usda_soil_types = filter_list_term_type(measurement_nodes, TermTermType.USDASOILTYPE)
+    soil_types = _get_soil_type_measurements(measurement_nodes, TermTermType.SOILTYPE)
+    usda_soil_types = _get_soil_type_measurements(measurement_nodes, TermTermType.USDASOILTYPE)
 
     clay_content = get_node_value(find_term_match(measurement_nodes, _CLAY_CONTENT_TERM_ID))
     sand_content = get_node_value(find_term_match(measurement_nodes, _SAND_CONTENT_TERM_ID))
@@ -1053,6 +1054,20 @@ def _assign_ipcc_soil_category(
     ) if len(soil_types) > 0 or len(usda_soil_types) > 0 else default
 
 
+def _get_soil_type_measurements(
+    nodes: list[dict], term_type: Literal[TermTermType.SOILTYPE, TermTermType.USDASOILTYPE]
+) -> list[dict]:
+    grouped = group_by_term(filter_list_term_type(nodes, term_type))
+
+    def depth_distance(node):
+        upper, lower = node.get("depthUpper", 0), node.get("depthLower", 100)
+        return abs(upper - DEPTH_UPPER) + abs(lower - DEPTH_LOWER)
+
+    return non_empty_list(
+        min(nodes_, key=depth_distance) for key in grouped if (nodes_ := grouped.get(key, []))
+    )
+
+
 def _check_soil_category(
     *,
     key: IpccSoilCategory,
@@ -1461,7 +1476,7 @@ Value: Corresponding decision tree for IPCC management categories based on land
 """
 
 _IPCC_LAND_USE_CATEGORY_TO_DEFAULT_IPCC_MANAGEMENT_CATEGORY = {
-    IpccLandUseCategory.GRASSLAND: IpccManagementCategory.NOMINALLY_MANAGED,
+    IpccLandUseCategory.GRASSLAND: IpccManagementCategory.UNKNOWN,
     IpccLandUseCategory.ANNUAL_CROPS_WET: IpccManagementCategory.UNKNOWN,
     IpccLandUseCategory.ANNUAL_CROPS: IpccManagementCategory.UNKNOWN
 }