hestia-earth-models 0.74.4__py3-none-any.whl → 0.74.6__py3-none-any.whl

hestia_earth/models/cml2001Baseline/abioticResourceDepletionMineralsAndMetals.py

@@ -80,7 +80,6 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
                     "input-term-type": input.get('termType'),
                     "indicator-term-id": resource_indicator['term']['@id'],
                     "indicator-is-valid": _valid_resource_indicator(resource_indicator),
-                    "input": input,
                     "indicator-input-is-valid": _valid_input(input),
                     "value": _node_value(resource_indicator),
                     "coefficient": get_table_value(

hestia_earth/models/config/Cycle.json

@@ -1264,6 +1264,21 @@
         },
         "stage": 2
       },
+      {
+        "key": "emissions",
+        "model": "ipcc2019",
+        "value": "co2ToAirBiocharStockChange",
+        "runStrategy": "add_blank_node_if_missing",
+        "runArgs": {
+          "runNonMeasured": true,
+          "runNonAddedTerm": true
+        },
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
+      },
       {
         "key": "emissions",
         "model": "ipcc2019",

hestia_earth/models/config/ImpactAssessment.json

@@ -82,7 +82,7 @@
       "key": "emissionsResourceUse",
       "model": "pooreNemecek2018",
       "value": "landOccupationDuringCycle",
-      "runStrategy": "always",
+      "runStrategy": "add_blank_node_if_missing",
       "mergeStrategy": "list",
       "mergeArgs": {
         "replaceThreshold": ["value", 0.01]
@@ -225,6 +225,14 @@
       },
       "stage": 1
     },
+    {
+      "key": "cache_emissionsResourceUse",
+      "model": "impact_assessment",
+      "value": "pre_checks.cache_emissionsResourceUse",
+      "runStrategy": "always",
+      "mergeStrategy": "default",
+      "stage": 1
+    },
     {
       "key": "impacts",
       "model": "ipcc2021",

hestia_earth/models/cycle/animal/input/hestiaAggregatedData.py

@@ -34,8 +34,8 @@ MODEL_ID = 'hestiaAggregatedData'
 MODEL_KEY = 'impactAssessment'
 
 
-def _run_animal_input(cycle: dict, input: dict):
-    inputs = link_inputs_to_impact(MODEL_ID, cycle, [input])
+def _run_animal_input(cycle: dict, animal: dict, input: dict):
+    inputs = link_inputs_to_impact(MODEL_ID, cycle, [input], animalId=animal.get('animalId'))
     return inputs[0] if inputs else input
 
 
@@ -43,7 +43,7 @@ def _run_animal(cycle: dict, animal: dict):
     return animal | {
         'inputs': [
             (
-                _run_animal_input(cycle, input) if should_link_input_to_impact(cycle)(input) else input
+                _run_animal_input(cycle, animal, input) if should_link_input_to_impact(cycle)(input) else input
             ) for input in animal.get('inputs', [])
         ]
     }

hestia_earth/models/cycle/completeness/seed.py

@@ -36,7 +36,7 @@ def run(cycle: dict):
     site_type = cycle.get('site', {}).get('siteType')
     site_type_allowed = site_type in ALLOWED_SITE_TYPES
 
-    has_seed = find_term_match(cycle.get('inputs', []), 'seed', None)
+    has_seed = find_term_match(cycle.get('inputs', []), 'seed', None) is not None
 
     product = find_primary_product(cycle) or {}
     term_id = product.get('term', {}).get('@id')

hestia_earth/models/cycle/input/hestiaAggregatedData.py

@@ -1,12 +1,13 @@
-from hestia_earth.schema import TermTermType
+from hestia_earth.schema import TermTermType, NodeType
 from hestia_earth.utils.model import find_primary_product, linked_node, filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
+from hestia_earth.models.data.hestiaAggregatedData import DEFAULT_COUNTRY_ID, find_closest_impact_id
 from hestia_earth.models.utils.crop import valid_site_type
-from hestia_earth.models.utils.term import get_lookup_value, get_generic_crop, download_term
+from hestia_earth.models.utils.term import get_lookup_value, get_generic_crop
 from hestia_earth.models.utils.aggregated import (
-    should_link_input_to_impact, link_inputs_to_impact, find_closest_impact, aggregated_end_date
+    should_link_input_to_impact, link_inputs_to_impact, aggregated_end_date
 )
 
 REQUIREMENTS = {
@@ -56,29 +57,37 @@ MODEL_KEY = 'impactAssessment'
 
 
 def _run_seed(cycle: dict, primary_product: dict, seed_input: dict, product_term_id: str):
-    product = download_term(product_term_id, TermTermType.SEED)
     country = seed_input.get('country')
+    country_id = (country or {}).get('@id')
+
+    primary_product_id = primary_product.get('term', {}).get('@id')
+    default_product_id = get_generic_crop().get('@id')
+
     # to avoid double counting seed => aggregated impact => seed, we need to get the impact of the previous decade
     # if the data does not exist, use the aggregated impact of generic crop instead
-    date = aggregated_end_date(cycle.get('endDate'))
-    match_end_date = [{'match': {'endDate': date - 10}}]
-    default_product = get_generic_crop()
-
-    impact = (
-        find_closest_impact(cycle, date, product, country, must_queries=match_end_date) or
-        find_closest_impact(cycle, date, primary_product.get('term', {}), country, must_queries=match_end_date) or
-        find_closest_impact(cycle, date, default_product, country)
+    date = aggregated_end_date(cycle.get('endDate')) - 10
+
+    impact_id = (
+        find_closest_impact_id(product_id=product_term_id, country_id=country_id, year=date) or
+        find_closest_impact_id(product_id=product_term_id, country_id=DEFAULT_COUNTRY_ID, year=date) or
+        find_closest_impact_id(product_id=primary_product_id, country_id=country_id, year=date) or
+        find_closest_impact_id(product_id=primary_product_id, country_id=DEFAULT_COUNTRY_ID, year=date) or
+        find_closest_impact_id(product_id=default_product_id, country_id=country_id, year=date) or
+        find_closest_impact_id(product_id=default_product_id, country_id=DEFAULT_COUNTRY_ID, year=date)
     )
 
-    search_by_product_term_id = (product or primary_product or default_product).get('@id')
-    search_by_country_id = (country or {}).get('@id') or 'region-world'
+    search_by_product_term_id = product_term_id or primary_product_id or default_product_id
+    search_by_country_id = country_id or DEFAULT_COUNTRY_ID
     debugValues(cycle, model=MODEL_ID, term=seed_input.get('term', {}).get('@id'), key=MODEL_KEY,
                 search_by_product_term_id=search_by_product_term_id,
                 search_by_country_id=search_by_country_id,
                 search_by_end_date=str(date),
-                impact_assessment_id_found=(impact or {}).get('@id'))
+                impact_assessment_id_found=impact_id)
 
-    return seed_input | {MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True} if impact else None
+    return seed_input | {
+            MODEL_KEY: linked_node({'@type': NodeType.IMPACTASSESSMENT.value, '@id': impact_id}),
+            'impactAssessmentIsProxy': True
+        } if impact_id else None
 
 
 def _should_run_seed(cycle: dict):

hestia_earth/models/data/hestiaAggregatedData/__init__.py

@@ -0,0 +1,73 @@
+import os
+import json
+from datetime import datetime
+from hestia_earth.utils.storage._s3_client import _load_from_bucket
+from hestia_earth.utils.api import _safe_get_request
+
+_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+_FILENAME = 'hestiaAggregatedData.json'
+FILEPATH = os.path.join(_CURRENT_DIR, _FILENAME)
+_CACHED_DATA = {}
+DEFAULT_COUNTRY_ID = 'region-world'
+
+
+def _today(): return datetime.now().strftime('%Y-%m-%d')
+
+
+def _download_data():
+    try:
+        return json.loads(_load_from_bucket('hestia-data', os.path.join('data', _FILENAME)))
+    except Exception:
+        return _safe_get_request(f"https://hestia.earth/data/{_FILENAME}")
+
+
+def _load_data():
+    data = None
+
+    if os.path.exists(FILEPATH):
+        with open(FILEPATH, 'r') as f:
+            data = json.load(f)
+
+    is_data_valid = data and data['date'] == _today()
+
+    if not is_data_valid:
+        data = _download_data()
+        with open(FILEPATH, 'w') as f:
+            f.write(json.dumps(data))
+
+    return data
+
+
+def _get_data():
+    global _CACHED_DATA  # noqa: F824
+    if not _CACHED_DATA or _CACHED_DATA['date'] != _today():
+        _CACHED_DATA = _load_data()
+    return _CACHED_DATA
+
+
+def _get_closest_id(data: dict, year: int):
+    available_years = [int(y) for y in data.keys() if int(y) <= year]
+    return data[str(sorted(available_years, reverse=True)[0])] if available_years else None
+
+
+def find_closest_impact_id(product_id: str, country_id: str, year: int):
+    """
+    Find the `@id` of the closest ImpactAssessment to the target year.
+
+    Parameters
+    ----------
+    product_id : str
+        The `@id` of the product (Term).
+    country_id : str
+        The `@id` of the country (Term).
+    year : int
+        The target year.
+
+    Returns
+    -------
+    str
+        The `@id` as a string if found.
+    """
+    data = _get_data()
+    values = data.get(product_id, {}).get(country_id, {})
+    return _get_closest_id(data=values, year=year)

hestia_earth/models/environmentalFootprintV3_1/scarcityWeightedWaterUse.py

@@ -33,7 +33,7 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_country_value(MODEL, TERM_ID, impact_assessment, f"{list(LOOKUPS.keys())[0]}.csv",
-                                 country_fallback=True)
+                                 default_world_value=True)
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     value=value)
     logShouldRun(impact_assessment, MODEL, TERM_ID, value is not None)

hestia_earth/models/environmentalFootprintV3_1/soilQualityIndexLandOccupation.py

@@ -7,7 +7,7 @@ from hestia_earth.utils.tools import list_sum
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.indicator import _new_indicator
 from hestia_earth.models.utils.landCover import get_pef_grouping
-from hestia_earth.models.utils.lookup import fallback_country, _node_value, get_region_lookup_value
+from hestia_earth.models.utils.lookup import _node_value, get_region_lookup_value
 from . import MODEL
 from ..utils.impact_assessment import get_country_id
 
@@ -68,7 +68,6 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
         'area-by-year-is-valid': _node_value(indicator) is not None and _node_value(
             indicator) >= 0,
         'area-unit-is-valid': indicator.get('term', {}).get("units") == "m2*year",
-        'used-country': fallback_country(get_country_id(impact_assessment, blank_node=indicator), [LOOKUP]),
         'pef-grouping': get_pef_grouping(indicator.get('landCover', {}).get("@id"))
 
     } for indicator in land_occupation_indicators]
@@ -79,10 +78,10 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
                 model=MODEL,
                 term=TERM_ID,
                 lookup_name=LOOKUP,
-                term_id=indicator['used-country'],
-                column=indicator['pef-grouping']
-            ),
-            "using-fallback-country-region-world-CFs": indicator['used-country'] != indicator['country-id']
+                term_id=indicator['country-id'],
+                column=indicator['pef-grouping'],
+                fallback_world=True
+            )
         } for indicator in found_land_occupation_indicators
     ]
 

hestia_earth/models/environmentalFootprintV3_1/soilQualityIndexLandTransformation.py

@@ -8,7 +8,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.impact_assessment import get_country_id
 from hestia_earth.models.utils.indicator import _new_indicator
 from hestia_earth.models.utils.landCover import get_pef_grouping
-from hestia_earth.models.utils.lookup import fallback_country, _node_value, get_region_lookup_value
+from hestia_earth.models.utils.lookup import _node_value, get_region_lookup_value
 from . import MODEL
 
 REQUIREMENTS = {
@@ -104,31 +104,28 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
             'value-is-valid': (
                 _node_value(indicator) is not None and
                 _node_value(indicator) >= 0
-            ),
-            'lookup-country': fallback_country(
-                get_country_id(impact_assessment, blank_node=indicator),
-                [from_lookup_file, to_lookup_file]
-            ),
+            )
         } for indicator in resource_uses
     ]
 
     found_transformations_with_coefficient = [
         transformation | {
-            'using-fallback-country-region-world-CFs': transformation['lookup-country'] != transformation['country-id'],
             'factor-from': get_region_lookup_value(
                 model=MODEL,
                 term=TERM_ID,
                 lookup_name=from_lookup_file,
-                term_id=transformation['lookup-country'],
-                column=get_pef_grouping(transformation['land-cover-id-from'])) if
-            transformation['land-cover-id-from'] else None,
+                term_id=transformation['country-id'],
+                column=get_pef_grouping(transformation['land-cover-id-from']),
+                fallback_world=True
+            ) if transformation['land-cover-id-from'] else None,
             'factor-to': get_region_lookup_value(
                 model=MODEL,
                 term=TERM_ID,
                 lookup_name=to_lookup_file,
-                term_id=transformation['lookup-country'],
-                column=get_pef_grouping(transformation['land-cover-id-to'])) if
-            transformation['land-cover-id-to'] else None
+                term_id=transformation['country-id'],
+                column=get_pef_grouping(transformation['land-cover-id-to']),
+                fallback_world=True
+            ) if transformation['land-cover-id-to'] else None
         } for transformation in found_transformations
     ]
 

hestia_earth/models/fantkeEtAl2016/damageToHumanHealthParticulateMatterFormation.py

@@ -28,7 +28,7 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_emission_lookup_value(
-        model=MODEL, term_id=TERM_ID, impact=impact_assessment, lookup_col=LOOKUPS['emission'], grouped_key='default'
+        model=MODEL, term_id=TERM_ID, impact=impact_assessment, lookup_col=LOOKUPS['emission'], group_key='default'
     )
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     value=value)

hestia_earth/models/hestia/default_resourceUse.py

@@ -1,4 +1,4 @@
-from hestia_earth.schema import IndicatorMethodTier, TermTermType
+from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import flatten, safe_parse_float
 
 from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
@@ -8,6 +8,8 @@ from hestia_earth.models.utils.indicator import _new_indicator
 from hestia_earth.models.utils.background_emissions import no_gap_filled_background_emissions
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.input import unique_background_inputs
+from hestia_earth.models.utils.impact_assessment import get_product
+from hestia_earth.models.utils.crop import get_landCover_term_id
 from . import MODEL
 
 REQUIREMENTS = {
@@ -29,8 +31,7 @@ REQUIREMENTS = {
 RETURNS = {
     "Indicator": [{
         "value": "",
-        "inputs": "",
-        "methodTier": "background"
+        "inputs": ""
     }]
 }
 LOOKUPS = {
@@ -38,14 +39,12 @@ LOOKUPS = {
     "organicFertiliser": "backgroundEmissionsResourceUseDefaultValue"
 }
 MODEL_KEY = 'default_resourceUse'
-TIER = IndicatorMethodTier.BACKGROUND.value
 
 
-def _indicator(term_id: str, value: float, input: dict):
-    indicator = _new_indicator(term_id, MODEL)
+def _indicator(term_id: str, value: float, input: dict, land_cover_id: str):
+    indicator = _new_indicator(term_id, MODEL, land_cover_id)
     indicator['value'] = value
     indicator['inputs'] = [input]
-    indicator['methodTier'] = TIER
     return indicator
 
 
@@ -53,7 +52,7 @@ def _default_value(input: dict):
     return safe_parse_float(get_lookup_value(input.get('term', {}), LOOKUPS['organicFertiliser']), default=None)
 
 
-def _run_input(impact: dict):
+def _run_input(impact: dict, land_cover_id: str):
     required_resourceUse_term_ids = background_emissions_in_system_boundary(impact, TermTermType.RESOURCEUSE)
 
     def run(input: dict):
@@ -62,14 +61,14 @@ def _run_input(impact: dict):
         value = input.get('default-value-from-lookup')
 
         for emission_id in required_resourceUse_term_ids:
-            logShouldRun(impact, MODEL, term_id, True, methodTier=TIER, model_key=MODEL_KEY, emission_id=emission_id)
+            logShouldRun(impact, MODEL, term_id, True, model_key=MODEL_KEY, emission_id=emission_id)
             debugValues(impact, model=MODEL, term=emission_id,
                         value=value,
                         coefficient=1,
                         input=term_id)
 
         return [
-            _indicator(term_id, value, input_term) for term_id in required_resourceUse_term_ids
+            _indicator(term_id, value, input_term, land_cover_id) for term_id in required_resourceUse_term_ids
         ]
 
     return run
@@ -95,18 +94,21 @@ def _should_run(impact: dict):
         ])
     ]
 
-    should_run = all([bool(valid_inputs)])
+    landCover_term_id = get_landCover_term_id(get_product(impact).get("term", {}))
+
+    should_run = all([bool(valid_inputs), landCover_term_id])
 
     for input in inputs:
         term_id = input.get('input').get('term', {}).get('@id')
 
         logRequirements(impact, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                        **_omit(input, ['input', 'input-value']))
-        logShouldRun(impact, MODEL, term_id, should_run, methodTier=TIER, model_key=MODEL_KEY)
+                        **_omit(input, ['input', 'input-value']),
+                        landCover_term_id=landCover_term_id)
+        logShouldRun(impact, MODEL, term_id, should_run, model_key=MODEL_KEY)
 
-    return should_run, valid_inputs
+    return should_run, valid_inputs, landCover_term_id
 
 
 def run(impact: dict):
-    should_run, grouped_inputs = _should_run(impact)
-    return flatten(map(_run_input(impact), grouped_inputs)) if should_run else []
+    should_run, grouped_inputs, landCover_term_id = _should_run(impact)
+    return flatten(map(_run_input(impact, landCover_term_id), grouped_inputs)) if should_run else []

hestia_earth/models/hestia/landCover.py

@@ -750,6 +750,22 @@ def _should_run_historical_land_use_change_total_cropland(site: dict, nodes: lis
     return all([should_run_annual, should_run_permanent]), scaled_results
 
 
+def _log_all_terms(site: dict, land_use_type: str, country_id: str, changes: dict, missing_changes: list):
+    for land_use_term, _ in LAND_USE_TERMS_FOR_TRANSFORMATION.values():
+        logRequirements(
+            log_node=site,
+            model=MODEL,
+            term=land_use_term,
+            model_key=MODEL_KEY,
+            land_use_type=land_use_type,
+            country_id=country_id,
+            changes=log_as_table(changes),
+            missing_changes=log_as_table(missing_changes)
+        )
+
+        logShouldRun(site, MODEL, land_use_term, False, model_key=MODEL_KEY)
+
+
 def _should_run_historical_land_use_change_single_crop(
     site: dict,
     term: dict,
@@ -909,6 +925,14 @@ def _should_run_historical_land_use_change_single_crop(
 
     should_run = all([len(missing_changes) == 0, country_id, site_type_allowed, sum_of_site_areas_is_100])
     logShouldRun(site, MODEL, term.get("@id"), should_run, model_key=MODEL_KEY)
+    if not should_run:
+        _log_all_terms(
+            site=site,
+            land_use_type=land_use_type,
+            country_id=country_id,
+            changes=changes,
+            missing_changes=missing_changes
+        )
 
     return should_run, capped_site_area