hestia-earth-models 0.74.4__tar.gz → 0.74.6__tar.gz

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_models
-Version: 0.74.4
+Version: 0.74.6
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/cml2001Baseline/abioticResourceDepletionMineralsAndMetals.py

@@ -80,7 +80,6 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
                     "input-term-type": input.get('termType'),
                     "indicator-term-id": resource_indicator['term']['@id'],
                     "indicator-is-valid": _valid_resource_indicator(resource_indicator),
-                    "input": input,
                     "indicator-input-is-valid": _valid_input(input),
                     "value": _node_value(resource_indicator),
                     "coefficient": get_table_value(

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/config/Cycle.json

@@ -1264,6 +1264,21 @@
         },
         "stage": 2
       },
+      {
+        "key": "emissions",
+        "model": "ipcc2019",
+        "value": "co2ToAirBiocharStockChange",
+        "runStrategy": "add_blank_node_if_missing",
+        "runArgs": {
+          "runNonMeasured": true,
+          "runNonAddedTerm": true
+        },
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
+      },
       {
         "key": "emissions",
         "model": "ipcc2019",

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/config/ImpactAssessment.json

@@ -82,7 +82,7 @@
       "key": "emissionsResourceUse",
       "model": "pooreNemecek2018",
       "value": "landOccupationDuringCycle",
-      "runStrategy": "always",
+      "runStrategy": "add_blank_node_if_missing",
       "mergeStrategy": "list",
       "mergeArgs": {
         "replaceThreshold": ["value", 0.01]
@@ -225,6 +225,14 @@
       },
       "stage": 1
     },
+    {
+      "key": "cache_emissionsResourceUse",
+      "model": "impact_assessment",
+      "value": "pre_checks.cache_emissionsResourceUse",
+      "runStrategy": "always",
+      "mergeStrategy": "default",
+      "stage": 1
+    },
     {
       "key": "impacts",
       "model": "ipcc2021",

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/cycle/animal/input/hestiaAggregatedData.py

@@ -34,8 +34,8 @@ MODEL_ID = 'hestiaAggregatedData'
 MODEL_KEY = 'impactAssessment'
 
 
-def _run_animal_input(cycle: dict, input: dict):
-    inputs = link_inputs_to_impact(MODEL_ID, cycle, [input])
+def _run_animal_input(cycle: dict, animal: dict, input: dict):
+    inputs = link_inputs_to_impact(MODEL_ID, cycle, [input], animalId=animal.get('animalId'))
     return inputs[0] if inputs else input
 
 
@@ -43,7 +43,7 @@ def _run_animal(cycle: dict, animal: dict):
     return animal | {
         'inputs': [
             (
-                _run_animal_input(cycle, input) if should_link_input_to_impact(cycle)(input) else input
+                _run_animal_input(cycle, animal, input) if should_link_input_to_impact(cycle)(input) else input
             ) for input in animal.get('inputs', [])
         ]
     }

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/cycle/completeness/seed.py

@@ -36,7 +36,7 @@ def run(cycle: dict):
     site_type = cycle.get('site', {}).get('siteType')
     site_type_allowed = site_type in ALLOWED_SITE_TYPES
 
-    has_seed = find_term_match(cycle.get('inputs', []), 'seed', None)
+    has_seed = find_term_match(cycle.get('inputs', []), 'seed', None) is not None
 
     product = find_primary_product(cycle) or {}
     term_id = product.get('term', {}).get('@id')

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/cycle/input/hestiaAggregatedData.py

@@ -1,12 +1,13 @@
-from hestia_earth.schema import TermTermType
+from hestia_earth.schema import TermTermType, NodeType
 from hestia_earth.utils.model import find_primary_product, linked_node, filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
+from hestia_earth.models.data.hestiaAggregatedData import DEFAULT_COUNTRY_ID, find_closest_impact_id
 from hestia_earth.models.utils.crop import valid_site_type
-from hestia_earth.models.utils.term import get_lookup_value, get_generic_crop, download_term
+from hestia_earth.models.utils.term import get_lookup_value, get_generic_crop
 from hestia_earth.models.utils.aggregated import (
-    should_link_input_to_impact, link_inputs_to_impact, find_closest_impact, aggregated_end_date
+    should_link_input_to_impact, link_inputs_to_impact, aggregated_end_date
 )
 
 REQUIREMENTS = {
@@ -56,29 +57,37 @@ MODEL_KEY = 'impactAssessment'
 
 
 def _run_seed(cycle: dict, primary_product: dict, seed_input: dict, product_term_id: str):
-    product = download_term(product_term_id, TermTermType.SEED)
     country = seed_input.get('country')
+    country_id = (country or {}).get('@id')
+
+    primary_product_id = primary_product.get('term', {}).get('@id')
+    default_product_id = get_generic_crop().get('@id')
+
     # to avoid double counting seed => aggregated impact => seed, we need to get the impact of the previous decade
     # if the data does not exist, use the aggregated impact of generic crop instead
-    date = aggregated_end_date(cycle.get('endDate'))
-    match_end_date = [{'match': {'endDate': date - 10}}]
-    default_product = get_generic_crop()
-
-    impact = (
-        find_closest_impact(cycle, date, product, country, must_queries=match_end_date) or
-        find_closest_impact(cycle, date, primary_product.get('term', {}), country, must_queries=match_end_date) or
-        find_closest_impact(cycle, date, default_product, country)
+    date = aggregated_end_date(cycle.get('endDate')) - 10
+
+    impact_id = (
+        find_closest_impact_id(product_id=product_term_id, country_id=country_id, year=date) or
+        find_closest_impact_id(product_id=product_term_id, country_id=DEFAULT_COUNTRY_ID, year=date) or
+        find_closest_impact_id(product_id=primary_product_id, country_id=country_id, year=date) or
+        find_closest_impact_id(product_id=primary_product_id, country_id=DEFAULT_COUNTRY_ID, year=date) or
+        find_closest_impact_id(product_id=default_product_id, country_id=country_id, year=date) or
+        find_closest_impact_id(product_id=default_product_id, country_id=DEFAULT_COUNTRY_ID, year=date)
     )
 
-    search_by_product_term_id = (product or primary_product or default_product).get('@id')
-    search_by_country_id = (country or {}).get('@id') or 'region-world'
+    search_by_product_term_id = product_term_id or primary_product_id or default_product_id
+    search_by_country_id = country_id or DEFAULT_COUNTRY_ID
     debugValues(cycle, model=MODEL_ID, term=seed_input.get('term', {}).get('@id'), key=MODEL_KEY,
                 search_by_product_term_id=search_by_product_term_id,
                 search_by_country_id=search_by_country_id,
                 search_by_end_date=str(date),
-                impact_assessment_id_found=(impact or {}).get('@id'))
+                impact_assessment_id_found=impact_id)
 
-    return seed_input | {MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True} if impact else None
+    return seed_input | {
+            MODEL_KEY: linked_node({'@type': NodeType.IMPACTASSESSMENT.value, '@id': impact_id}),
+            'impactAssessmentIsProxy': True
+        } if impact_id else None
 
 
 def _should_run_seed(cycle: dict):

hestia_earth_models-0.74.6/hestia_earth/models/data/hestiaAggregatedData/__init__.py

@@ -0,0 +1,73 @@
+import os
+import json
+from datetime import datetime
+from hestia_earth.utils.storage._s3_client import _load_from_bucket
+from hestia_earth.utils.api import _safe_get_request
+
+_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+_FILENAME = 'hestiaAggregatedData.json'
+FILEPATH = os.path.join(_CURRENT_DIR, _FILENAME)
+_CACHED_DATA = {}
+DEFAULT_COUNTRY_ID = 'region-world'
+
+
+def _today(): return datetime.now().strftime('%Y-%m-%d')
+
+
+def _download_data():
+    try:
+        return json.loads(_load_from_bucket('hestia-data', os.path.join('data', _FILENAME)))
+    except Exception:
+        return _safe_get_request(f"https://hestia.earth/data/{_FILENAME}")
+
+
+def _load_data():
+    data = None
+
+    if os.path.exists(FILEPATH):
+        with open(FILEPATH, 'r') as f:
+            data = json.load(f)
+
+    is_data_valid = data and data['date'] == _today()
+
+    if not is_data_valid:
+        data = _download_data()
+        with open(FILEPATH, 'w') as f:
+            f.write(json.dumps(data))
+
+    return data
+
+
+def _get_data():
+    global _CACHED_DATA  # noqa: F824
+    if not _CACHED_DATA or _CACHED_DATA['date'] != _today():
+        _CACHED_DATA = _load_data()
+    return _CACHED_DATA
+
+
+def _get_closest_id(data: dict, year: int):
+    available_years = [int(y) for y in data.keys() if int(y) <= year]
+    return data[str(sorted(available_years, reverse=True)[0])] if available_years else None
+
+
+def find_closest_impact_id(product_id: str, country_id: str, year: int):
+    """
+    Find the `@id` of the closest ImpactAssessment to the target year.
+
+    Parameters
+    ----------
+    product_id : str
+        The `@id` of the product (Term).
+    country_id : str
+        The `@id` of the country (Term).
+    year : int
+        The target year.
+
+    Returns
+    -------
+    str
+        The `@id` as a string if found.
+    """
+    data = _get_data()
+    values = data.get(product_id, {}).get(country_id, {})
+    return _get_closest_id(data=values, year=year)

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/environmentalFootprintV3_1/scarcityWeightedWaterUse.py

@@ -33,7 +33,7 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_country_value(MODEL, TERM_ID, impact_assessment, f"{list(LOOKUPS.keys())[0]}.csv",
-                                 country_fallback=True)
+                                 default_world_value=True)
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     value=value)
     logShouldRun(impact_assessment, MODEL, TERM_ID, value is not None)

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/environmentalFootprintV3_1/soilQualityIndexLandOccupation.py

@@ -7,7 +7,7 @@ from hestia_earth.utils.tools import list_sum
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.indicator import _new_indicator
 from hestia_earth.models.utils.landCover import get_pef_grouping
-from hestia_earth.models.utils.lookup import fallback_country, _node_value, get_region_lookup_value
+from hestia_earth.models.utils.lookup import _node_value, get_region_lookup_value
 from . import MODEL
 from ..utils.impact_assessment import get_country_id
 
@@ -68,7 +68,6 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
         'area-by-year-is-valid': _node_value(indicator) is not None and _node_value(
             indicator) >= 0,
         'area-unit-is-valid': indicator.get('term', {}).get("units") == "m2*year",
-        'used-country': fallback_country(get_country_id(impact_assessment, blank_node=indicator), [LOOKUP]),
         'pef-grouping': get_pef_grouping(indicator.get('landCover', {}).get("@id"))
 
     } for indicator in land_occupation_indicators]
@@ -79,10 +78,10 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
                 model=MODEL,
                 term=TERM_ID,
                 lookup_name=LOOKUP,
-                term_id=indicator['used-country'],
-                column=indicator['pef-grouping']
-            ),
-            "using-fallback-country-region-world-CFs": indicator['used-country'] != indicator['country-id']
+                term_id=indicator['country-id'],
+                column=indicator['pef-grouping'],
+                fallback_world=True
+            )
         } for indicator in found_land_occupation_indicators
     ]
 

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/environmentalFootprintV3_1/soilQualityIndexLandTransformation.py

@@ -8,7 +8,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.impact_assessment import get_country_id
 from hestia_earth.models.utils.indicator import _new_indicator
 from hestia_earth.models.utils.landCover import get_pef_grouping
-from hestia_earth.models.utils.lookup import fallback_country, _node_value, get_region_lookup_value
+from hestia_earth.models.utils.lookup import _node_value, get_region_lookup_value
 from . import MODEL
 
 REQUIREMENTS = {
@@ -104,31 +104,28 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
             'value-is-valid': (
                 _node_value(indicator) is not None and
                 _node_value(indicator) >= 0
-            ),
-            'lookup-country': fallback_country(
-                get_country_id(impact_assessment, blank_node=indicator),
-                [from_lookup_file, to_lookup_file]
-            ),
+            )
         } for indicator in resource_uses
     ]
 
     found_transformations_with_coefficient = [
         transformation | {
-            'using-fallback-country-region-world-CFs': transformation['lookup-country'] != transformation['country-id'],
             'factor-from': get_region_lookup_value(
                 model=MODEL,
                 term=TERM_ID,
                 lookup_name=from_lookup_file,
-                term_id=transformation['lookup-country'],
-                column=get_pef_grouping(transformation['land-cover-id-from'])) if
-            transformation['land-cover-id-from'] else None,
+                term_id=transformation['country-id'],
+                column=get_pef_grouping(transformation['land-cover-id-from']),
+                fallback_world=True
+            ) if transformation['land-cover-id-from'] else None,
             'factor-to': get_region_lookup_value(
                 model=MODEL,
                 term=TERM_ID,
                 lookup_name=to_lookup_file,
-                term_id=transformation['lookup-country'],
-                column=get_pef_grouping(transformation['land-cover-id-to'])) if
-            transformation['land-cover-id-to'] else None
+                term_id=transformation['country-id'],
+                column=get_pef_grouping(transformation['land-cover-id-to']),
+                fallback_world=True
+            ) if transformation['land-cover-id-to'] else None
         } for transformation in found_transformations
     ]
 

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/fantkeEtAl2016/damageToHumanHealthParticulateMatterFormation.py

@@ -28,7 +28,7 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_emission_lookup_value(
-        model=MODEL, term_id=TERM_ID, impact=impact_assessment, lookup_col=LOOKUPS['emission'], grouped_key='default'
+        model=MODEL, term_id=TERM_ID, impact=impact_assessment, lookup_col=LOOKUPS['emission'], group_key='default'
     )
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     value=value)

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/hestia/default_resourceUse.py

@@ -1,4 +1,4 @@
-from hestia_earth.schema import IndicatorMethodTier, TermTermType
+from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import flatten, safe_parse_float
 
 from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
@@ -8,6 +8,8 @@ from hestia_earth.models.utils.indicator import _new_indicator
 from hestia_earth.models.utils.background_emissions import no_gap_filled_background_emissions
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.input import unique_background_inputs
+from hestia_earth.models.utils.impact_assessment import get_product
+from hestia_earth.models.utils.crop import get_landCover_term_id
 from . import MODEL
 
 REQUIREMENTS = {
@@ -29,8 +31,7 @@ REQUIREMENTS = {
 RETURNS = {
     "Indicator": [{
         "value": "",
-        "inputs": "",
-        "methodTier": "background"
+        "inputs": ""
     }]
 }
 LOOKUPS = {
@@ -38,14 +39,12 @@ LOOKUPS = {
     "organicFertiliser": "backgroundEmissionsResourceUseDefaultValue"
 }
 MODEL_KEY = 'default_resourceUse'
-TIER = IndicatorMethodTier.BACKGROUND.value
 
 
-def _indicator(term_id: str, value: float, input: dict):
-    indicator = _new_indicator(term_id, MODEL)
+def _indicator(term_id: str, value: float, input: dict, land_cover_id: str):
+    indicator = _new_indicator(term_id, MODEL, land_cover_id)
     indicator['value'] = value
     indicator['inputs'] = [input]
-    indicator['methodTier'] = TIER
     return indicator
 
 
@@ -53,7 +52,7 @@ def _default_value(input: dict):
     return safe_parse_float(get_lookup_value(input.get('term', {}), LOOKUPS['organicFertiliser']), default=None)
 
 
-def _run_input(impact: dict):
+def _run_input(impact: dict, land_cover_id: str):
     required_resourceUse_term_ids = background_emissions_in_system_boundary(impact, TermTermType.RESOURCEUSE)
 
     def run(input: dict):
@@ -62,14 +61,14 @@ def _run_input(impact: dict):
         value = input.get('default-value-from-lookup')
 
         for emission_id in required_resourceUse_term_ids:
-            logShouldRun(impact, MODEL, term_id, True, methodTier=TIER, model_key=MODEL_KEY, emission_id=emission_id)
+            logShouldRun(impact, MODEL, term_id, True, model_key=MODEL_KEY, emission_id=emission_id)
             debugValues(impact, model=MODEL, term=emission_id,
                         value=value,
                         coefficient=1,
                         input=term_id)
 
         return [
-            _indicator(term_id, value, input_term) for term_id in required_resourceUse_term_ids
+            _indicator(term_id, value, input_term, land_cover_id) for term_id in required_resourceUse_term_ids
         ]
 
     return run
@@ -95,18 +94,21 @@ def _should_run(impact: dict):
         ])
     ]
 
-    should_run = all([bool(valid_inputs)])
+    landCover_term_id = get_landCover_term_id(get_product(impact).get("term", {}))
+
+    should_run = all([bool(valid_inputs), landCover_term_id])
 
     for input in inputs:
         term_id = input.get('input').get('term', {}).get('@id')
 
         logRequirements(impact, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                        **_omit(input, ['input', 'input-value']))
-        logShouldRun(impact, MODEL, term_id, should_run, methodTier=TIER, model_key=MODEL_KEY)
+                        **_omit(input, ['input', 'input-value']),
+                        landCover_term_id=landCover_term_id)
+        logShouldRun(impact, MODEL, term_id, should_run, model_key=MODEL_KEY)
 
-    return should_run, valid_inputs
+    return should_run, valid_inputs, landCover_term_id
 
 
 def run(impact: dict):
-    should_run, grouped_inputs = _should_run(impact)
-    return flatten(map(_run_input(impact), grouped_inputs)) if should_run else []
+    should_run, grouped_inputs, landCover_term_id = _should_run(impact)
+    return flatten(map(_run_input(impact, landCover_term_id), grouped_inputs)) if should_run else []

{hestia_earth_models-0.74.4 → hestia_earth_models-0.74.6}/hestia_earth/models/hestia/landCover.py

@@ -750,6 +750,22 @@ def _should_run_historical_land_use_change_total_cropland(site: dict, nodes: lis
     return all([should_run_annual, should_run_permanent]), scaled_results
 
 
+def _log_all_terms(site: dict, land_use_type: str, country_id: str, changes: dict, missing_changes: list):
+    for land_use_term, _ in LAND_USE_TERMS_FOR_TRANSFORMATION.values():
+        logRequirements(
+            log_node=site,
+            model=MODEL,
+            term=land_use_term,
+            model_key=MODEL_KEY,
+            land_use_type=land_use_type,
+            country_id=country_id,
+            changes=log_as_table(changes),
+            missing_changes=log_as_table(missing_changes)
+        )
+
+        logShouldRun(site, MODEL, land_use_term, False, model_key=MODEL_KEY)
+
+
 def _should_run_historical_land_use_change_single_crop(
     site: dict,
     term: dict,
@@ -909,6 +925,14 @@ def _should_run_historical_land_use_change_single_crop(
 
     should_run = all([len(missing_changes) == 0, country_id, site_type_allowed, sum_of_site_areas_is_100])
     logShouldRun(site, MODEL, term.get("@id"), should_run, model_key=MODEL_KEY)
+    if not should_run:
+        _log_all_terms(
+            site=site,
+            land_use_type=land_use_type,
+            country_id=country_id,
+            changes=changes,
+            missing_changes=missing_changes
+        )
 
     return should_run, capped_site_area