hestia-earth-models 0.70.6__tar.gz → 0.71.0__tar.gz

{hestia_earth_models-0.70.6 → hestia_earth_models-0.71.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_models
-Version: 0.70.6
+Version: 0.71.0
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

hestia_earth_models-0.71.0/hestia_earth/models/cache_nodes.py

@@ -0,0 +1,157 @@
+import os
+from functools import reduce
+from hestia_earth.schema import NodeType
+from hestia_earth.utils.tools import current_time_ms, flatten
+from hestia_earth.earth_engine import init_gee
+
+from .log import logger
+from .utils import CACHE_KEY
+from .utils.site import years_from_cycles
+from .utils.source import CACHE_SOURCES_KEY, find_sources
+from .cache_sites import run as cache_sites
+
+CACHE_RELATED_KEY = 'related'
+CACHE_NESTED_KEY = 'nested'
+
+_CACHE_BATCH_SIZE = int(os.getenv('CACHE_SITES_BATCH_SIZE', '5000'))
+_ENABLE_CACHE_YEARS = os.getenv('ENABLE_CACHE_YEARS', 'true') == 'true'
+_ENABLE_CACHE_RELATED_NODES = os.getenv('ENABLE_CACHE_RELATED_NODES', 'true') == 'true'
+_CACHE_NODE_TYPES = [
+    NodeType.SITE.value,
+    NodeType.CYCLE.value,
+    NodeType.IMPACTASSESSMENT.value
+]
+
+
+def _pop_items(values: list, nb_items: int):
+    if len(values) < nb_items:
+        removed_items = values[:]  # Get a copy of the entire array
+        values.clear()  # Remove all items from the original array
+    else:
+        removed_items = values[:nb_items]  # Get the first N items
+        del values[:nb_items]  # Remove the first N items from the original array
+
+    return removed_items
+
+
+def _filter_by_type(nodes: list, type: str): return [n for n in nodes if n.get('@type', n.get('type')) == type]
+
+
+def _node_key(node: dict): return '/'.join([node.get('type', node.get('@type')), node.get('id', node.get('@id'))])
+
+
+def _years_from_cycles(nodes: dict): return years_from_cycles(_filter_by_type(nodes, NodeType.CYCLE.value))
+
+
+def _linked_node(data: dict): return {'type': data.get('type'), 'id': data.get('id')}
+
+
+def _find_nested_nodes(data) -> list[dict]:
+    if isinstance(data, dict):
+        if data.get('type') in _CACHE_NODE_TYPES and data.get('id'):
+            return [_linked_node(data)]
+        return flatten(_find_nested_nodes(list(data.values())))
+    if isinstance(data, list):
+        return flatten(map(_find_nested_nodes, data))
+    return []
+
+
+def _nested_nodes(node_keys: list[str]):
+    def exec(group: dict, node: dict):
+        nested_nodes = _find_nested_nodes(list(node.values()))
+
+        for nested_node in nested_nodes:
+            group_id = _node_key(nested_node)
+            group[group_id] = group.get(group_id, {})
+            group[group_id][CACHE_RELATED_KEY] = group.get(group_id, {}).get(CACHE_RELATED_KEY, []) + [
+                _linked_node(node)
+            ]
+
+            # cache nodes that current node refers (nesting)
+            if group_id in node_keys:
+                group_id = _node_key(node)
+                group[group_id] = group.get(group_id, {})
+                group[group_id][CACHE_NESTED_KEY] = group.get(group_id, {}).get(CACHE_NESTED_KEY, []) + [
+                    _linked_node(nested_node)
+                ]
+
+        return group
+    return exec
+
+
+def _cache_related_nodes(nodes: list):
+    # only cache nodes included in the file
+    nodes_keys = list(map(_node_key, nodes))
+    # for each node, compile list of nested nodes
+    nested_nodes_mapping = reduce(_nested_nodes(nodes_keys), nodes, {})
+
+    def cache_related_node(node: dict):
+        nodes_mapping = nested_nodes_mapping.get(_node_key(node), {})
+        related_nodes = nodes_mapping.get(CACHE_RELATED_KEY) or []
+        nested_nodes = nodes_mapping.get(CACHE_NESTED_KEY) or []
+        # save in cache
+        cached_data = node.get(CACHE_KEY, {}) | {
+            CACHE_RELATED_KEY: related_nodes,
+            CACHE_NESTED_KEY: nested_nodes
+        }
+        return node | {CACHE_KEY: cached_data}
+
+    return list(map(cache_related_node, nodes))
+
+
+def _cache_sources(nodes: list):
+    sources = find_sources()
+    return [
+        n | ({
+            CACHE_KEY: n.get(CACHE_KEY, {}) | {CACHE_SOURCES_KEY: sources}
+        } if n.get('type', n.get('@type')) in _CACHE_NODE_TYPES else {})
+        for n in nodes
+    ]
+
+
+def _safe_cache_sites(sites: list, years: list):
+    try:
+        return cache_sites(sites, years)
+    except Exception as e:
+        logger.error(f"An error occured while caching nodes on EE: {str(e)}")
+        if 'exceeded' in str(e):
+            logger.debug('Fallback to caching sites one by one')
+            # run one by one in case the batching does not work
+            return flatten([cache_sites([site], years) for site in sites])
+        else:
+            raise e
+
+
+def _cache_sites(nodes: list, batch_size: int = _CACHE_BATCH_SIZE):
+    start = current_time_ms()
+
+    # build list of nodes by key to update as sites are processed
+    nodes_mapping = {_node_key(n): n for n in nodes}
+
+    years = _years_from_cycles(nodes) if _ENABLE_CACHE_YEARS else []
+    sites = _filter_by_type(nodes, 'Site')
+
+    while len(sites) > 0:
+        batch_values = _pop_items(sites, batch_size)
+        logger.info(f"Processing {len(batch_values)} sites / {len(sites)} remaining.")
+        results = _safe_cache_sites(batch_values, years)
+        for result in results:
+            nodes_mapping[_node_key(result)] = result
+
+    logger.info(f"Done caching sites in {current_time_ms() - start} ms")
+
+    # replace original sites with new cached sites
+    return list(nodes_mapping.values())
+
+
+def run(nodes: list):
+    init_gee()
+
+    # cache sites data
+    cached_nodes = _cache_sites(nodes)
+
+    # cache related nodes
+    cached_nodes = _cache_related_nodes(cached_nodes) if _ENABLE_CACHE_RELATED_NODES else cached_nodes
+
+    # cache sources
+    return _cache_sources(cached_nodes)

{hestia_earth_models-0.70.6 → hestia_earth_models-0.71.0}/hestia_earth/models/cache_sites.py

@@ -81,7 +81,7 @@ def _run_values(
         site_cache = merge(
             site.get(CACHE_KEY, {}),
             {CACHE_GEOSPATIAL_KEY: cached_data},
-            ({CACHE_YEARS_KEY: list(set(cached_value(site, CACHE_YEARS_KEY, []) + years))} if years else {})
+            ({CACHE_YEARS_KEY: sorted(list(set(cached_value(site, CACHE_YEARS_KEY, []) + years)))} if years else {})
         )
         return merge(site, {CACHE_KEY: site_cache})
 

{hestia_earth_models-0.70.6 → hestia_earth_models-0.71.0}/hestia_earth/models/config/Cycle.json

@@ -2054,36 +2054,6 @@
         },
         "stage": 2
       },
-      {
-        "key": "emissions",
-        "model": "ipcc2006",
-        "value": "n2OToAirOrganicSoilCultivationDirect",
-        "runStrategy": "add_blank_node_if_missing",
-        "runArgs": {
-          "runNonMeasured": true,
-          "runNonAddedTerm": true
-        },
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 2
-      },
-      {
-        "key": "emissions",
-        "model": "ipcc2006",
-        "value": "co2ToAirOrganicSoilCultivation",
-        "runStrategy": "add_blank_node_if_missing",
-        "runArgs": {
-          "runNonMeasured": true,
-          "runNonAddedTerm": true
-        },
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 2
-      },
       {
         "key": "emissions",
         "model": "ipcc2006",

{hestia_earth_models-0.70.6 → hestia_earth_models-0.71.0}/hestia_earth/models/data/ecoinventV3/__init__.py

@@ -5,17 +5,19 @@ from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logger
 
-CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-_FILEPATH = os.getenv('ECOINVENT_V3_FILEPATH', f"{os.path.join(CURRENT_DIR, 'ecoinventV3_excerpt')}.csv")
+_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+_ENV_NAME = 'ECOINVENT_V3_FILEPATH'
 
 
 @lru_cache()
 def _get_file():
-    if not os.path.exists(_FILEPATH):
-        logger.warning('Ecoinvent file not found. Please make sure to set env variable "ECOINVENT_V3_FILEPATH".')
+    filepath = os.getenv(_ENV_NAME, f"{os.path.join(_CURRENT_DIR, 'ecoinventV3_excerpt')}.csv")
+
+    if not os.path.exists(filepath):
+        logger.warning('Ecoinvent file not found. Please make sure to set env variable "%s".', _ENV_NAME)
         return None
 
-    return load_lookup(filepath=_FILEPATH, keep_in_memory=True)
+    return load_lookup(filepath=filepath, keep_in_memory=True)
 
 
 def ecoinventV3_emissions(ecoinventName: str):

{hestia_earth_models-0.70.6 → hestia_earth_models-0.71.0}/hestia_earth/models/ecoinventV3/__init__.py

@@ -5,7 +5,11 @@ from hestia_earth.utils.tools import flatten, list_sum
 from hestia_earth.models.log import debugValues, logShouldRun, logRequirements
 from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
 from hestia_earth.models.utils.emission import _new_emission
-from hestia_earth.models.utils.background_emissions import get_background_inputs, no_gap_filled_background_emissions
+from hestia_earth.models.utils.background_emissions import (
+    get_background_inputs,
+    no_gap_filled_background_emissions,
+    log_missing_emissions
+)
 from hestia_earth.models.utils.blank_node import group_by_keys
 from hestia_earth.models.utils.pesticideAI import get_pesticides_from_inputs
 from hestia_earth.models.utils.fertiliser import get_fertilisers_from_inputs
@@ -47,6 +51,7 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
+    "emission": "inputProductionGroupId",
     "electricity": "ecoinventMapping",
     "fuel": "ecoinventMapping",
     "inorganicFertiliser": "ecoinventMapping",
@@ -97,6 +102,7 @@ def _add_emission(cycle: dict, input: dict):
 
 def _run_input(cycle: dict):
     no_gap_filled_background_emissions_func = no_gap_filled_background_emissions(cycle)
+    log_missing_emissions_func = log_missing_emissions(cycle, model=MODEL, methodTier=TIER)
 
     def run(inputs: list):
         input = inputs[0]
@@ -118,6 +124,7 @@ def _run_input(cycle: dict):
         logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER)
 
         grouped_emissions = reduce(_add_emission(cycle, input), mappings, {}) if should_run else {}
+        log_missing_emissions_func(input_term_id, list(grouped_emissions.keys()))
         return [
             _emission(term_id, value * input_value, input)
             for term_id, value in grouped_emissions.items()

{hestia_earth_models-0.70.6 → hestia_earth_models-0.71.0}/hestia_earth/models/hestia/aboveGroundCropResidue.py

@@ -105,9 +105,9 @@ def _run(cycle: dict, total_values: list):
         term_id = model.get('product')
         value = _run_model(model, cycle, total_value)
         debugValues(cycle, model=MODEL, term=term_id,
-                    total_value=total_value,
-                    remaining_value=remaining_value,
-                    value=value)
+                    total_above_ground_crop_residue=total_value,
+                    remaining_crop_residue_value=remaining_value,
+                    allocated_value=value)
 
         if value == 0:
             values.extend([_product(term_id, value)])

{hestia_earth_models-0.70.6 → hestia_earth_models-0.71.0}/hestia_earth/models/hestia/seed_emissions.py

@@ -98,6 +98,23 @@ def _run(cycle: dict, economicValueShare: float, total_yield: float, seed_input:
     ]
 
 
+def _map_group_emissions(group_id: str, required_emission_term_ids: list, emission_ids: list):
+    lookup = download_lookup('emission.csv')
+    emissions = list(filter(
+        lambda id: id in required_emission_term_ids,
+        find_term_ids_by(lookup, column_name('inputProductionGroupId'), group_id)
+    ))
+    included_emissions = list(filter(lambda v: v in emission_ids, emissions))
+    missing_emissions = list(filter(lambda v: v not in emission_ids, emissions))
+    return {
+        'id': group_id,
+        'total-emissions': len(emissions),
+        'included-emissions': len(included_emissions),
+        'missing-emissions': '-'.join(missing_emissions),
+        'is-valid': len(emissions) == len(included_emissions)
+    }
+
+
 def _filter_emissions(cycle: dict):
     required_emission_term_ids = cycle_emissions_in_system_boundary(cycle)
 
@@ -117,31 +134,12 @@ def _filter_emissions(cycle: dict):
     group_ids = set([v.get('group-id') for v in emissions if v.get('group-id')])
 
     # for each group, get the list of all required emissions
-    lookup = download_lookup('emission.csv')
     emissions_per_group = [
-        {
-            'id': group_id,
-            'emissions': list(filter(
-                lambda id: id in required_emission_term_ids,
-                find_term_ids_by(lookup, column_name('inputProductionGroupId'), group_id)
-            ))
-        }
+        _map_group_emissions(group_id, required_emission_term_ids, emission_ids)
         for group_id in group_ids
     ]
-    emissions_per_group = [
-        {
-            'id': group.get('id'),
-            'total-emissions': len(group.get('emissions', [])),
-            'included-emissions': len(list(filter(lambda v: v in emission_ids, group.get('emissions', [])))),
-            'missing-emissions': '-'.join(list(filter(lambda v: v not in emission_ids, group.get('emissions', []))))
-        }
-        for group in emissions_per_group
-    ]
     # only keep groups that have all emissions present in the Cycle
-    valid_groups = list(filter(
-        lambda group: group.get('total-emissions') == group.get('included-emissions'),
-        emissions_per_group
-    ))
+    valid_groups = list(filter(lambda group: group.get('is-valid'), emissions_per_group))
     valid_group_ids = set([v.get('id') for v in valid_groups])
 
     # finally, only return emissions which groups are valid
@@ -273,6 +271,12 @@ def _should_run(cycle: dict):
 
         logShouldRun(cycle, MODEL, term_id, should_run, methodTier=TIER, model_key=MODEL_KEY)
 
+        # log missing emissions to show in the logs
+        for group in emissions_per_group:
+            if not group.get('is-valid'):
+                logShouldRun(cycle, MODEL, term_id, False,
+                             methodTier=TIER, model_key=MODEL_KEY, emission_id=group.get('id'))
+
     return should_run, total_economicValueShare, total_yield, grouped_seed_inputs, grouped_emissions