hestia-earth-models 0.74.12__py3-none-any.whl → 0.74.14__py3-none-any.whl

hestia_earth/models/config/ImpactAssessment.json

@@ -339,7 +339,8 @@
       "runStrategy": "always",
       "mergeStrategy": "list",
       "mergeArgs": {
-        "replaceThreshold": ["value", 0.01]
+        "replaceThreshold": ["value", 0.01],
+        "sameMethodModel": true
       },
       "stage": 1
     },
@@ -962,7 +963,8 @@
       "runStrategy": "add_blank_node_if_missing",
       "mergeStrategy": "list",
       "mergeArgs": {
-        "replaceThreshold": ["value", 0.01]
+        "replaceThreshold": ["value", 0.01],
+        "sameMethodModel": true
       },
       "stage": 1
     },

hestia_earth/models/ecoalimV9/cycle.py

@@ -12,7 +12,10 @@ from hestia_earth.models.utils.background_emissions import (
     parse_term_id,
     process_input_mappings
 )
-from .utils import get_input_mappings, extract_input_mapping
+from .utils import (
+    get_input_mappings, extract_input_mapping,
+    CUTOFF_KEY, get_cutoff_lookup, cutoff_value, filter_blank_nodes_cutoff
+)
 from . import MODEL
 
 REQUIREMENTS = {
@@ -63,7 +66,14 @@ MODEL_KEY = 'cycle'
 TIER = EmissionMethodTier.BACKGROUND.value
 
 
-def _emission(term_id: str, value: float, input: dict, country_id: str = None, key_id: str = None):
+def _emission(
+    term_id: str,
+    value: float,
+    input: dict,
+    country_id: str = None,
+    key_id: str = None,
+    cutoff_lookup=None
+):
     emission = _new_emission(term=term_id, model=MODEL, value=value, country_id=country_id, key_id=key_id)
     emission['methodTier'] = TIER
     emission['inputs'] = [input.get('term')]
@@ -71,10 +81,11 @@ def _emission(term_id: str, value: float, input: dict, country_id: str = None, k
         emission['operation'] = input.get('operation')
     if input.get('animal'):
         emission['animals'] = [input.get('animal')]
-    return emission
+    cutoff = cutoff_value(cutoff_lookup, term_id=term_id, country_id=country_id, key_id=key_id)
+    return emission | ({CUTOFF_KEY: value * cutoff} if cutoff is not None else {})
 
 
-def _run_input(cycle: dict):
+def _run_input(cycle: dict, cutoff_lookup=None):
     no_gap_filled_background_emissions_func = no_gap_filled_background_emissions(cycle)
     log_missing_emissions_func = log_missing_emissions(cycle, model=MODEL, methodTier=TIER)
 
@@ -122,6 +133,7 @@ def _run_input(cycle: dict):
                 input=input,
                 country_id=values[0].get('country'),
                 key_id=values[0].get('key'),
+                cutoff_lookup=cutoff_lookup
             )
             for term_id, values in results.items()
         ]
@@ -131,4 +143,7 @@ def _run_input(cycle: dict):
 def run(cycle: dict):
     inputs = get_background_inputs(cycle)
     grouped_inputs = group_by_keys(inputs, ['term', 'operation', 'animal'])
-    return flatten(map(_run_input(cycle), grouped_inputs.values()))
+    # used to calculate the global coefficient
+    cutoff_lookup = get_cutoff_lookup(term_type=TermTermType.EMISSION)
+    emissions = flatten(map(_run_input(cycle, cutoff_lookup), grouped_inputs.values()))
+    return emissions if cutoff_lookup is None else filter_blank_nodes_cutoff(emissions)

hestia_earth/models/ecoalimV9/utils.py

@@ -1,7 +1,8 @@
+import os
 from functools import lru_cache
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import download_lookup, column_name
-from hestia_earth.utils.tools import non_empty_list
+from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.tools import non_empty_list, safe_parse_float, omit
 
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.background_emissions import convert_background_lookup
@@ -35,3 +36,52 @@ def _build_lookup(term_type: str):
 def ecoalim_values(mapping: str, term_type: TermTermType):
     data = _build_lookup(term_type.value)
     return list(data[mapping].items())
+
+
+CUTOFF_KEY = 'cutoff_coeff'
+_CUTOFF_MAX_PERCENTAGE = int(os.getenv('ECOALIM_CUTOFF_MAX_PERCENT', '99'))
+
+
+def get_cutoff_lookup(term_type: TermTermType):
+    filename = f"ecoalim-{term_type.value}-cutoff.csv"
+    return download_lookup(filename, keep_in_memory=False) if _CUTOFF_MAX_PERCENTAGE else None
+
+
+def _cutoff_id(term_id: str, country_id: str = None, key_id: str = None):
+    return term_id + (
+        f"+inputs[{key_id}]" if key_id else ''
+    ) + (
+        f"+country[{country_id}]" if country_id else ''
+    )
+
+
+def cutoff_value(cutoff_lookup, term_id: str, country_id: str = None, key_id: str = None):
+    cutoff_id = _cutoff_id(term_id=term_id, country_id=country_id, key_id=key_id)
+    return None if cutoff_lookup is None else safe_parse_float(
+        get_table_value(cutoff_lookup, 'termid', cutoff_id, 'percentage'),
+        default=None
+    )
+
+
+def filter_blank_nodes_cutoff(blank_nodes: list):
+    # use the generic contibution of the blank node towards EF Score to remove the lowest percentage
+    total_contributions = sum([
+        v.get(CUTOFF_KEY, 0) for v in blank_nodes
+    ])
+    blank_nodes_with_contributions = sorted([
+        (
+            v,
+            v.get(CUTOFF_KEY, 0) * 100 / total_contributions
+        )
+        for v in blank_nodes
+    ], key=lambda v: v[1], reverse=True)
+
+    sum_contributions = 0
+    filtered_blank_nodes = []
+    for blank_node, contribution in blank_nodes_with_contributions:
+        sum_contributions = sum_contributions + contribution
+        if sum_contributions > _CUTOFF_MAX_PERCENTAGE:
+            break
+        filtered_blank_nodes.append(omit(blank_node, [CUTOFF_KEY]))
+
+    return filtered_blank_nodes

hestia_earth/models/environmentalFootprintV3_1/environmentalFootprintSingleOverallScore.py

@@ -58,8 +58,7 @@ def _get_factor(term: dict, column: str) -> Optional[float]:
 
 
 def _get_pef_method_model(term: dict) -> List[str]:
-    entries = get_lookup_value(term, method_model_colum, model=MODEL, term=TERM_ID) or ''
-    return entries.split(";")
+    return (get_lookup_value(term, method_model_colum, skip_debug=True) or '').split(";")
 
 
 def _indicator_factors(impact_assessment: dict, indicator: dict):
@@ -110,9 +109,11 @@ def _run(indicators: List[dict]):
     return _indicator(value=list_sum([indicator["weighted-value"] for indicator in indicators]))
 
 
-def _valid_indicator(indicator: Optional[dict]) -> bool:
-    value = None if indicator is None else _node_value(indicator)
-    return isinstance(value, (int, float)) and _is_a_PEF_indicator(indicator)
+def _valid_indicator(indicator: Optional[dict]):
+    return isinstance(_node_value(indicator), (int, float))
+
+
+def _log_indicators(indicators: list): return ';'.join([v['indicator'] for v in indicators])
 
 
 def _should_run(impact_assessment: dict) -> Tuple[bool, list[dict]]:
@@ -123,25 +124,33 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list[dict]]:
     ]
     has_pef_indicators = bool(indicators)
 
-    processed_indicators = [{
-        "indicator": indicator['term']['@id'],
-        "valid-indicator": _valid_indicator(indicator),
-        "one-indicator-for-category": _count_duplicate_indicators(indicator, indicators) == 1,
-        "indicator-pef-category": indicator.get('term', {}).get('@id'),
-    } | _indicator_factors(impact_assessment, indicator) for indicator in indicators]
+    processed_indicators = [
+        {
+            "indicator": indicator['term']['@id'],
+            "methodModel": indicator.get('methodModel', {}).get("@id"),
+            "valid-value": _valid_indicator(indicator),
+            "count-indicators": _count_duplicate_indicators(indicator, indicators),
+            "PEF-category": indicator.get('term', {}).get('@id')
+        } | _indicator_factors(impact_assessment, indicator)
+        for indicator in indicators
+    ]
+
+    duplicate_indicators = [v for v in processed_indicators if v['count-indicators'] > 1]
+
+    invalid_indicators = [v for v in processed_indicators if not v['valid-value']]
+    valid_indicators = [v for v in processed_indicators if v['valid-value']]
 
-    no_duplicate_indicators = all([indicator['one-indicator-for-category'] for indicator in processed_indicators])
-    valid_indicators = [indicator for indicator in processed_indicators if indicator['valid-indicator']]
-    all_indicators_valid = all([indicator['valid-indicator'] for indicator in processed_indicators])
+    all_indicators_valid = len(valid_indicators) == len(processed_indicators)
 
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     has_pef_indicators=has_pef_indicators,
-                    no_duplicate_indicators=no_duplicate_indicators,
+                    all_indicators=log_as_table(processed_indicators),
                     all_indicators_valid=all_indicators_valid,
-                    processed_indicators=log_as_table(processed_indicators),
-                    )
+                    duplicate_indicators=_log_indicators(duplicate_indicators),
+                    valid_indicators=_log_indicators(valid_indicators),
+                    invalid_indicators=_log_indicators(invalid_indicators))
 
-    should_run = all([has_pef_indicators, all_indicators_valid, no_duplicate_indicators])
+    should_run = all([has_pef_indicators, all_indicators_valid, not duplicate_indicators])
     logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
     return should_run, valid_indicators
 

hestia_earth/models/hestia/landCover.py

@@ -1,13 +1,12 @@
-from datetime import datetime, timedelta
+import os
 from hestia_earth.schema import SiteSiteType, TermTermType
 from hestia_earth.utils.model import filter_list_term_type
-from hestia_earth.utils.tools import to_precision, omit, pick, non_empty_list
+from hestia_earth.utils.tools import to_precision, omit, pick, non_empty_list, flatten
 from hestia_earth.utils.blank_node import group_by_keys
 
 from hestia_earth.models.log import logRequirements, log_as_table, logShouldRun
-from hestia_earth.models.utils.constant import DAYS_IN_YEAR
 from hestia_earth.models.utils.management import _new_management
-from hestia_earth.models.utils.blank_node import DatestrFormat, _gapfill_datestr, DatestrGapfillMode
+from hestia_earth.models.utils.blank_node import _gapfill_datestr, DatestrGapfillMode, condense_nodes
 from .utils import (
     LAND_USE_TERMS_FOR_TRANSFORMATION,
     IPCC_LAND_USE_CATEGORY_PERENNIAL,
@@ -79,6 +78,7 @@ LOOKUPS = {
 }
 MODEL_KEY = 'landCover'
 
+_FALLBACK_COMPUTE_DATA = os.getenv('MODEL_LAND_COVER_FALLBACK_COMPUTE_DATA', 'true') == 'true'
 _BUILDING_SITE_TYPES = [
     SiteSiteType.AGRI_FOOD_PROCESSOR.value,
     SiteSiteType.ANIMAL_HOUSING.value,
@@ -91,11 +91,23 @@ _ALLOWED_SITE_TYPES = [
     SiteSiteType.PERMANENT_PASTURE.value
 ]
 _DEFAULT_WINDOW_IN_YEARS = 20
-_DATE_TOLERANCE_IN_YEARS = 1
 _ALLOWED_LAND_USE_TYPES = [ANNUAL_CROPLAND, PERMANENT_CROPLAND, PERMANENT_PASTURE, TOTAL_CROPLAND]
 _COMPLETE_CHANGES_OTHER_LAND = {k: 0 for k in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()} | {OTHER_LAND: 1}
 
 
+def _run(values: list):
+    blank_nodes = [
+        _new_management(
+            term=value.get("term-id"),
+            value=value.get("percentage"),
+            model=MODEL,
+            start_date=f"{value['year'] - _DEFAULT_WINDOW_IN_YEARS}-01-01",
+            end_date=f"{value['year'] - _DEFAULT_WINDOW_IN_YEARS}-12-31",
+        ) for value in values
+    ]
+    return condense_nodes(blank_nodes)
+
+
 def _should_group_landCover(management_node: dict):
     return any(
         bool(_get_lookup_with_cache(
@@ -107,7 +119,7 @@ def _should_group_landCover(management_node: dict):
 
 
 def _get_land_use_term_from_node(node: dict):
-    return _get_lookup_with_cache(lookup_term=node.get("term", {}), column=LOOKUPS.get("landCover")[1])
+    return _get_lookup_with_cache(lookup_term=node.get("term", {}), column='FAOSTAT_LAND_AREA_CATEGORY')
 
 
 def _date_strip(date: str): return date[:10] if date else None
@@ -116,25 +128,6 @@ def _date_strip(date: str): return date[:10] if date else None
 def _date_year(date: str): return int(date[:4]) if date else None
 
 
-def _no_prior_land_cover_data(nodes: list, reference_date: str) -> bool:
-    """
-    Returns true if there are no nodes whose start & end dates the target_node falls within,
-    including a tolerance.
-    """
-    target_date = (
-        datetime.strptime(reference_date, DatestrFormat.YEAR_MONTH_DAY.value)
-        - timedelta(days=_DEFAULT_WINDOW_IN_YEARS * DAYS_IN_YEAR)
-    )
-    tolerance = timedelta(days=_DATE_TOLERANCE_IN_YEARS * DAYS_IN_YEAR)
-    previous_nodes = [
-        node for node in nodes
-        if datetime.strptime(node.get("startDate"), DatestrFormat.YEAR_MONTH_DAY.value) - tolerance
-        < target_date <
-        datetime.strptime(node.get("endDate"), DatestrFormat.YEAR_MONTH_DAY.value) + tolerance
-    ]
-    return len(previous_nodes) == 0
-
-
 def _collect_land_use_types(nodes: list) -> list:
     """Look up the land use type from management nodes."""
     return [
@@ -151,13 +144,11 @@ def _collect_land_use_types(nodes: list) -> list:
 def _site_area_valid(site_area: dict): return site_area and all([v is not None for v in site_area.values()])
 
 
-def _extend_site_area(site: dict, management_nodes: list, land_use_node: dict) -> list:
+def _extend_site_area(site: dict, existing_years: set, land_use_node: dict) -> list:
     reference_year = land_use_node['year']
+    target_year = land_use_node['year'] - _DEFAULT_WINDOW_IN_YEARS
 
-    has_no_prior_land_cover_data = _no_prior_land_cover_data(
-        nodes=management_nodes,
-        reference_date=f"{land_use_node['year']}-06-01"
-    )
+    has_no_prior_land_cover_data = target_year not in existing_years
 
     site_area_from_lookups = get_site_area_from_lookups(
         country_id=site.get("country", {}).get("@id"),
@@ -171,7 +162,7 @@ def _extend_site_area(site: dict, management_nodes: list, land_use_node: dict) -
         term=land_use_node['term'],
         land_use_type=land_use_node['landUseType'],
         reference_year=reference_year
-    ) if not has_siteArea_from_lookups else ({}, False, {})
+    ) if not has_siteArea_from_lookups and _FALLBACK_COMPUTE_DATA else ({}, False, {})
 
     is_perenial = crop_ipcc_land_use_category(land_use_node['landCover-id']) == IPCC_LAND_USE_CATEGORY_PERENNIAL
 
@@ -225,6 +216,9 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
         if not _should_group_landCover(node)
     ])
 
+    # get all existing dates that we should not add again
+    existing_years = set(non_empty_list(flatten(map(_years_from_node, management_nodes))))
+
     # get the Management `landCover` nodes that are "landUse" nodes
     land_use_nodes = [
         node for node in management_nodes
@@ -240,7 +234,7 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
 
     # add metadata
     land_use_nodes = sorted([
-        node if is_site_building else _extend_site_area(site, management_nodes, node)
+        node if is_site_building else _extend_site_area(site, existing_years, node)
         for node in land_use_nodes
     ], key=lambda n: n['year'])
 
@@ -252,7 +246,7 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
             'siteArea': _COMPLETE_CHANGES_OTHER_LAND,
             'site-area-from-lookup': {}
         }
-    ] if is_site_building else land_use_nodes
+    ] if land_use_nodes and is_site_building else land_use_nodes
 
     # create list of land use nodes with all data
     land_use_nodes = sorted([
@@ -268,7 +262,7 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
         for land_type, ratio in node.get('siteArea').items()
     ], key=lambda n: '-'.join([str(n['year']), n['term-id']]))
 
-    _log_land_use_nodes(site, land_use_nodes)
+    land_use_nodes and _log_land_use_nodes(site, land_use_nodes)
 
     # filter valid values
     valid_land_use_nodes = [
@@ -314,12 +308,4 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
 
 def run(site: dict) -> list:
     should_run, values = _should_run(site=site)
-    return [
-        _new_management(
-            term=value.get("term-id"),
-            value=value.get("percentage"),
-            model=MODEL,
-            start_date=f"{value['year'] - _DEFAULT_WINDOW_IN_YEARS}-01-01",
-            end_date=f"{value['year'] - _DEFAULT_WINDOW_IN_YEARS}-12-31",
-        ) for value in values
-    ] if should_run else []
+    return _run(values) if should_run else []

hestia_earth/models/hestia/landCover_utils.py

@@ -1,4 +1,3 @@
-import functools
 import math
 from functools import lru_cache
 from collections import defaultdict
@@ -39,6 +38,25 @@ def get_land_use_terms():
     return [v[0] for v in LAND_USE_TERMS_FOR_TRANSFORMATION.values()]
 
 
+@lru_cache()
+def _get_immutable_lookup(term_id: str, term_type: str, col: str):
+    new_term = {"@id": term_id, "termType": term_type} if term_type and term_id else {}
+    return get_lookup_value(
+        lookup_term=new_term,
+        column=col,
+        skip_debug=False,
+        model=MODEL,
+        term=term_id
+    )
+
+
+def _get_lookup_with_cache(lookup_term: dict, column: str):
+    return _get_immutable_lookup(term_id=lookup_term.get("@id"), term_type=lookup_term.get("termType"), col=column)
+
+
+def _get_faostat_name(term: dict) -> str: return _get_lookup_with_cache(term, "cropGroupingFaostatArea")
+
+
 def _is_missing_or_none(value) -> bool:
     return value is None or _is_missing_value(value)
 
@@ -68,26 +86,6 @@ def _cap_values(dictionary: dict, lower_limit: float = 0, upper_limit: float = 1
     return {key: min([upper_limit, max([lower_limit, value])]) for key, value in dictionary.items()}
 
 
-def _get_lookup_with_cache(lookup_term, column):
-    """Wrapper to get_lookup_value which pulls out the immutable parts of the term to allow caching."""
-    @functools.cache
-    def _get_immutable_lookup(term_id: str, term_type: str, col: str):
-        new_term = {"@id": term_id, "termType": term_type} if term_type and term_id else {}
-        return get_lookup_value(
-            lookup_term=new_term,
-            column=col,
-            skip_debug=False,
-            model=MODEL,
-            term=term_id
-        )
-
-    return _get_immutable_lookup(
-        term_id=lookup_term.get("@id"),
-        term_type=lookup_term.get("termType"),
-        col=column
-    )
-
-
 def _get_changes(country_id: str, reference_year: int) -> tuple[dict, list]:
     """
     For each entry in ALL_LAND_USE_TERMS, creates a key: value in output dictionary, also TOTAL
@@ -349,11 +347,6 @@ def _get_shares_of_expansion(
     })
 
 
-def _get_faostat_name(term: dict) -> str:
-    """For landCover terms, find the cropGroupingFaostatArea name for the landCover id."""
-    return _get_lookup_with_cache(term, "cropGroupingFaostatArea")
-
-
 def _get_most_common_or_alphabetically_first(crop_terms: list) -> str:
     histogram = {term: crop_terms.count(term) for term in crop_terms}
     max_freq = max(histogram.values())
@@ -363,7 +356,8 @@ def _get_most_common_or_alphabetically_first(crop_terms: list) -> str:
 
 def _get_complete_faostat_to_crop_mapping() -> dict:
     """Returns mapping in the format: {faostat_name: IPCC_LAND_USE_CATEGORY, ...}"""
-    lookup = download_lookup("crop.csv")
+    term_type = TermTermType.CROP.value
+    lookup = download_lookup(f"{term_type}.csv")
     mappings = defaultdict(list)
     for crop_term_id in [row[0] for row in lookup]:
         key = column_name(
@@ -746,24 +740,25 @@ def get_site_area_from_lookups(country_id: str, reference_year: int, term: dict)
     """
     lookup_prefix = 'region-crop-cropGroupingFAOSTAT-landCover'
     lookup_column = _get_faostat_name(term)
+    raw_region_data = {
+        land_type: get_region_lookup_value(
+            lookup_name=f"{lookup_prefix}-{_get_land_cover_lookup_suffix(land_type)}.csv",
+            term_id=country_id,
+            column=lookup_column,
+            model=MODEL,
+            model_key=MODEL_KEY
+        ) if lookup_column else None
+        for land_type in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
+    }
+    parsed_region_data = {
+        land_type: safe_parse_float(
+            value=extract_grouped_data(data=value, key=str(reference_year)),
+            default=None
+        )
+        for land_type, value in raw_region_data.items()
+    }
+    # Divide by 100 to match site_area ratios
     return {
-        # Divide by 100 to match site_area ratios
         land_type: value / 100 if value is not None else value
-        for land_type, value in
-        {
-            land_type: safe_parse_float(
-                value=extract_grouped_data(
-                    data=get_region_lookup_value(
-                        lookup_name=f"{lookup_prefix}-{_get_land_cover_lookup_suffix(land_type)}.csv",
-                        term_id=country_id,
-                        column=lookup_column,
-                        model=MODEL,
-                        key=MODEL_KEY
-                    ),
-                    key=str(reference_year)
-                ),
-                default=None
-            )
-            for land_type in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
-        }.items()
+        for land_type, value in parsed_region_data.items()
     }

hestia_earth/models/ipcc2019/ch4ToAirEntericFermentation.py

@@ -74,7 +74,8 @@ LOOKUPS = {
     ],
     "crop-property": ["neutralDetergentFibreContent", "energyContentHigherHeatingValue"],
     "forage-property": ["neutralDetergentFibreContent", "energyContentHigherHeatingValue"],
-    "processedFood-property": ["neutralDetergentFibreContent", "energyContentHigherHeatingValue"]
+    "processedFood-property": ["neutralDetergentFibreContent", "energyContentHigherHeatingValue"],
+    "feedFoodAdditive": ["hasEnergyContent"]
 }
 RETURNS = {
     "Emission": [{