hestia-earth-models 0.73.6__py3-none-any.whl → 0.73.8__py3-none-any.whl

hestia_earth/models/utils/background_emissions.py

@@ -1,9 +1,9 @@
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
-from hestia_earth.utils.tools import flatten
+from hestia_earth.utils.tools import flatten, non_empty_list
 from hestia_earth.utils.emission import cycle_emissions_in_system_boundary
 
-from hestia_earth.models.log import logShouldRun
+from hestia_earth.models.log import logShouldRun, debugValues
 from . import is_from_model
 from .term import get_lookup_value
 
@@ -58,23 +58,30 @@ def no_gap_filled_background_emissions(
     return check_input
 
 
-def all_background_emission_term_ids(cycle: dict):
-    term_ids = cycle_emissions_in_system_boundary(cycle)
+def all_background_emission_term_ids(node: dict, termType: TermTermType):
+    term_ids = cycle_emissions_in_system_boundary(node, termType=termType)
     return list(set([
-        get_lookup_value({'termType': TermTermType.EMISSION.value, '@id': term_id}, 'inputProductionGroupId')
+        get_lookup_value({'termType': termType.value, '@id': term_id}, 'inputProductionGroupId')
         for term_id in term_ids
     ]))
 
 
-def log_missing_emissions(cycle: dict, **log_args):
-    all_emission_term_ids = all_background_emission_term_ids(cycle)
+def log_missing_emissions(node: dict, termType: TermTermType = TermTermType.EMISSION, **log_args):
+    all_emission_term_ids = all_background_emission_term_ids(node, termType)
 
     def log_input(input_term_id: str, included_emission_term_ids: list, **extra_log_args):
-        missing_emission_term_ids = [
+        missing_emission_term_ids = non_empty_list([
             term_id for term_id in all_emission_term_ids if term_id not in included_emission_term_ids
-        ]
+        ])
         for emission_id in missing_emission_term_ids:
-            logShouldRun(cycle, term=input_term_id, should_run=False, emission_id=emission_id,
+            # debug value on the emission itself so it appears for the input
+            debugValues(node, term=emission_id,
+                        value=None,
+                        coefficient=None,
+                        input=input_term_id,
+                        **log_args,
+                        **extra_log_args)
+            logShouldRun(node, term=input_term_id, should_run=False, emission_id=emission_id,
                          **log_args,
                          **extra_log_args)
     return log_input

hestia_earth/models/utils/emission.py

@@ -2,7 +2,7 @@ from collections.abc import Iterable
 from typing import Optional, Union
 from hestia_earth.schema import EmissionMethodTier, SchemaType, TermTermType
 from hestia_earth.utils.model import linked_node
-
+from hestia_earth.utils.emission import cycle_emissions_in_system_boundary, emissions_in_system_boundary
 
 from . import flatten_args
 from .term import download_term
@@ -13,20 +13,22 @@ from .constant import Units, get_atomic_conversion
 EMISSION_METHOD_TIERS = [e.value for e in EmissionMethodTier]
 
 
-def _new_emission(term, model=None):
+def _new_emission(term, model=None, country_id: str = None, key_id: str = None):
     node = {'@type': SchemaType.EMISSION.value}
     node['term'] = linked_node(term if isinstance(term, dict) else download_term(term, TermTermType.EMISSION))
+    if country_id:
+        node['country'] = linked_node(download_term(country_id, TermTermType.REGION))
+    if key_id:
+        node['key'] = linked_node(download_term(key_id))
     return include_methodModel(node, model)
 
 
-def get_nh3_no3_nox_to_n(cycle: dict, nh3_term_id: str, no3_term_id: str, nox_term_id: str, allow_none: bool = False):
-    default_value = 0 if allow_none else None
-
-    nh3 = find_terms_value(cycle.get('emissions', []), nh3_term_id, default=default_value)
+def get_nh3_no3_nox_to_n(cycle: dict, nh3_term_id: str = None, no3_term_id: str = None, nox_term_id: str = None):
+    nh3 = find_terms_value(cycle.get('emissions', []), nh3_term_id, default=None)
     nh3 = None if nh3 is None else nh3 / get_atomic_conversion(Units.KG_NH3, Units.TO_N)
-    no3 = find_terms_value(cycle.get('emissions', []), no3_term_id, default=default_value)
+    no3 = find_terms_value(cycle.get('emissions', []), no3_term_id, default=None)
     no3 = None if no3 is None else no3 / get_atomic_conversion(Units.KG_NO3, Units.TO_N)
-    nox = find_terms_value(cycle.get('emissions', []), nox_term_id, default=default_value)
+    nox = find_terms_value(cycle.get('emissions', []), nox_term_id, default=None)
     nox = None if nox is None else nox / get_atomic_conversion(Units.KG_NOX, Units.TO_N)
 
     return (nh3, no3, nox)
@@ -100,3 +102,11 @@ def to_emission_method_tier(method: Union[EmissionMethodTier, str]) -> Optional[
 def filter_emission_inputs(emission: dict, term_type: TermTermType):
     inputs = emission.get('inputs', [])
     return [i for i in inputs if i.get('termType') == term_type.value]
+
+
+def background_emissions_in_system_boundary(node: dict, term_type: TermTermType = TermTermType.EMISSION):
+    term_ids = (
+        cycle_emissions_in_system_boundary(node, term_type) if term_type == TermTermType.EMISSION else
+        emissions_in_system_boundary(term_type)
+    )
+    return [id for id in term_ids if 'InputsProduction' in id]

hestia_earth/models/utils/impact_assessment.py

@@ -4,7 +4,7 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum, safe_parse_date
 
 from hestia_earth.models.log import debugValues
-from .lookup import all_factor_value, _region_factor_value, _aware_factor_value, fallback_country
+from .lookup import all_factor_value, region_factor_value, aware_factor_value, fallback_country
 from .product import find_by_product
 from .site import region_level_1_id
 
@@ -190,7 +190,7 @@ def impact_country_value(
         blank_nodes=blank_nodes,
         grouped_key=group_key,
         default_no_values=default_no_values,
-        factor_value_func=_region_factor_value
+        factor_value_func=region_factor_value
     )
 
 
@@ -231,7 +231,7 @@ def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, grou
         blank_nodes=blank_nodes,
         grouped_key=group_key,
         default_no_values=None,
-        factor_value_func=_aware_factor_value
+        factor_value_func=aware_factor_value
     )
 
 

hestia_earth/models/utils/indicator.py

@@ -5,7 +5,10 @@ from .method import include_methodModel
 from .term import download_term
 
 
-def _new_indicator(term, model=None, land_cover_id: str = None, previous_land_cover_id: str = None):
+def _new_indicator(
+    term: dict, model=None,
+    land_cover_id: str = None, previous_land_cover_id: str = None, country_id: str = None, key_id: str = None
+):
     node = {'@type': SchemaType.INDICATOR.value}
     node['term'] = linked_node(term if isinstance(term, dict) else download_term(
         term, TermTermType.CHARACTERISEDINDICATOR)
@@ -14,4 +17,8 @@ def _new_indicator(term, model=None, land_cover_id: str = None, previous_land_co
         node['landCover'] = linked_node(download_term(land_cover_id, TermTermType.LANDCOVER))
     if previous_land_cover_id:
         node['previousLandCover'] = linked_node(download_term(previous_land_cover_id, TermTermType.LANDCOVER))
+    if country_id:
+        node['country'] = linked_node(download_term(country_id, TermTermType.REGION))
+    if key_id:
+        node['key'] = linked_node(download_term(key_id))
     return include_methodModel(node, model)

hestia_earth/models/utils/lookup.py

@@ -1,4 +1,4 @@
-import functools
+from functools import lru_cache
 from typing import Optional, List
 from hestia_earth.utils.lookup import (
     download_lookup,
@@ -19,16 +19,20 @@ def _node_value(node):
 
 
 def _factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, grouped_key: Optional[str] = None):
-    def get_value(data: dict):
-        node_term_id = data.get('term', {}).get('@id')
-        grouped_data_key = grouped_key or data.get('methodModel', {}).get('@id')
-        value = _node_value(data)
+    @lru_cache()
+    def get_coefficient(node_term_id: str, grouped_data_key: str):
         coefficient = get_region_lookup_value(lookup_name, node_term_id, lookup_col, model=model, term=term_id)
         # value is either a number or matching between a model and a value (restrict value to specific model only)
-        coefficient = safe_parse_float(
+        return safe_parse_float(
             extract_grouped_data(coefficient, grouped_data_key),
             default=None
         ) if ':' in str(coefficient) else safe_parse_float(coefficient, default=None)
+
+    def get_value(data: dict):
+        node_term_id = data.get('term', {}).get('@id')
+        grouped_data_key = grouped_key or data.get('methodModel', {}).get('@id')
+        value = _node_value(data)
+        coefficient = get_coefficient(node_term_id, grouped_data_key)
         if value is not None and coefficient is not None:
             if model:
                 debugValues(data, model=model, term=term_id,
@@ -40,7 +44,15 @@ def _factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, g
     return get_value
 
 
-def _region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):
+def region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):
+    @lru_cache()
+    def get_coefficient(node_term_id: str, region_term_id: str):
+        coefficient = get_region_lookup_value(lookup_name, region_term_id, node_term_id, model=model, term=term_id)
+        return safe_parse_float(
+            extract_grouped_data(coefficient, group_key) if group_key else coefficient,
+            default=None
+        )
+
     def get_value(data: dict):
         node_term_id = data.get('term', {}).get('@id')
         value = _node_value(data)
@@ -48,11 +60,7 @@ def _region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term
         region_term_id = (
             (data.get('region') or data.get('country') or {'@id': lookup_term_id}).get('@id')
         ) if lookup_term_id.startswith('GADM-') else lookup_term_id
-        coefficient = get_region_lookup_value(lookup_name, region_term_id, node_term_id, model=model, term=term_id)
-        coefficient = safe_parse_float(
-            extract_grouped_data(coefficient, group_key) if group_key else coefficient,
-            default=None
-        )
+        coefficient = get_coefficient(node_term_id, region_term_id)
         if value is not None and coefficient is not None:
             debugValues(data, model=model, term=term_id,
                         node=node_term_id,
@@ -62,20 +70,24 @@ def _region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term
     return get_value
 
 
-def _aware_factor_value(model: str, term_id: str, lookup_name: str, aware_id: str, group_key: str = None):
+def aware_factor_value(model: str, term_id: str, lookup_name: str, aware_id: str, group_key: str = None):
     lookup = download_lookup(lookup_name, False)  # avoid saving in memory as there could be many different files used
     lookup_col = column_name('awareWaterBasinId')
 
+    @lru_cache()
+    def get_coefficient(node_term_id: str):
+        coefficient = _get_single_table_value(lookup, lookup_col, int(aware_id), column_name(node_term_id))
+        return safe_parse_float(
+            extract_grouped_data(coefficient, group_key),
+            default=None
+        ) if group_key else coefficient
+
     def get_value(data: dict):
         node_term_id = data.get('term', {}).get('@id')
         value = _node_value(data)
 
         try:
-            coefficient = _get_single_table_value(lookup, lookup_col, int(aware_id), column_name(node_term_id))
-            coefficient = safe_parse_float(
-                extract_grouped_data(coefficient, group_key),
-                default=None
-            ) if group_key else coefficient
+            coefficient = get_coefficient(node_term_id)
             if value is not None and coefficient is not None:
                 debugValues(data, model=model, term=term_id,
                             node=node_term_id,
@@ -159,13 +171,18 @@ def fallback_country(country_id: str, lookups: List[str]) -> str:
     return country_id if country_id and is_in_lookup(country_id) else fallback_id if is_in_lookup(fallback_id) else None
 
 
-@functools.cache
-def get_region_lookup_value(lookup_name: str, term_id: str, column: str, **log_args):
+def get_region_lookup(lookup_name: str, term_id: str):
     # for performance, try to load the region specific lookup if exists
-    lookup = (
+    return (
         download_lookup(lookup_name.replace('region-', f"{term_id}-"))
         if lookup_name and lookup_name.startswith('region-') else None
     ) or download_lookup(lookup_name)
+
+
+@lru_cache()
+def get_region_lookup_value(lookup_name: str, term_id: str, column: str, **log_args):
+    # for performance, try to load the region specific lookup if exists
+    lookup = get_region_lookup(lookup_name, term_id)
     value = get_table_value(lookup, 'termid', term_id, column_name(column))
     debugMissingLookup(lookup_name, 'termid', term_id, column, value, **log_args)
     return value

hestia_earth/models/utils/productivity.py

@@ -17,5 +17,5 @@ PRODUCTIVITY_KEY = {
 
 
 def get_productivity(country: dict, default: PRODUCTIVITY = PRODUCTIVITY.HIGH):
-    hdi = safe_parse_float(get_lookup_value(country, 'hdi'), default=None)
+    hdi = safe_parse_float(get_lookup_value(country, 'HDI'), default=None)
     return next((key for key in PRODUCTIVITY_KEY if hdi and PRODUCTIVITY_KEY[key](hdi)), default)

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.73.6'
+VERSION = '0.73.8'

hestia_earth/orchestrator/strategies/merge/merge_list.py

@@ -1,7 +1,7 @@
 import pydash
 from datetime import datetime
 from hestia_earth.schema import UNIQUENESS_FIELDS
-from hestia_earth.utils.tools import safe_parse_date
+from hestia_earth.utils.tools import safe_parse_date, flatten
 
 from hestia_earth.orchestrator.utils import _non_empty_list, update_node_version
 from .merge_node import merge as merge_node
@@ -27,39 +27,6 @@ def _has_property(value: dict, key: str):
 def _values_have_property(values: list, key: str): return any([_has_property(v, key) for v in values])
 
 
-def _match_list_el(source: list, dest: list, key: str):
-    src_value = sorted(_non_empty_list([pydash.objects.get(x, key) for x in source]))
-    dest_value = sorted(_non_empty_list([pydash.objects.get(x, key) for x in dest]))
-    return src_value == dest_value
-
-
-def _get_value(data: dict, key: str, merge_args: dict = {}):
-    value = pydash.objects.get(data, key)
-    date = safe_parse_date(value) if key in ['startDate', 'endDate'] else None
-    return datetime.strftime(date, merge_args.get('matchDatesFormat', '%Y-%m-%d')) if date else value
-
-
-def _match_el(source: dict, dest: dict, keys: list, merge_args: dict = {}):
-    def match(key: str):
-        keys = key.split('.')
-        src_value = _get_value(source, key, merge_args)
-        dest_value = _get_value(dest, key, merge_args)
-        is_list = len(keys) >= 2 and (
-            isinstance(pydash.objects.get(source, keys[0]), list) or
-            isinstance(pydash.objects.get(dest, keys[0]), list)
-        )
-        return _match_list_el(
-            pydash.objects.get(source, keys[0], []),
-            pydash.objects.get(dest, keys[0], []),
-            '.'.join(keys[1:])
-        ) if is_list else src_value == dest_value
-
-    source_properties = [p for p in keys if _has_property(source, p)]
-    dest_properties = [p for p in keys if _has_property(dest, p)]
-
-    return all(map(match, source_properties)) if source_properties == dest_properties else False
-
-
 def _handle_local_property(values: list, properties: list, local_id: str):
     # Handle "impactAssessment.@id" if present in the data
     existing_id = local_id.replace('.id', '.@id')
@@ -76,38 +43,58 @@ def _handle_local_property(values: list, properties: list, local_id: str):
     return properties
 
 
-def _find_match_el_index(values: list, el: dict, same_methodModel: bool, model: dict, node_type: str, merge_args: dict):
-    """
-    Find an element in the values that match the new element, based on the unique properties.
-    To find a matching element:
+def _get_value(data: dict, key: str, merge_args: dict = {}):
+    value = pydash.objects.get(data, key)
+    date = safe_parse_date(value) if key in ['startDate', 'endDate'] else None
+    return datetime.strftime(date, merge_args.get('matchDatesFormat', '%Y-%m-%d')) if date else value
+
+
+def _value_index_key(value: dict, properties: list, merge_args: dict = {}):
+    def property_value(key: str):
+        keys = key.split('.')
+        prop_value = _get_value(value, key, merge_args)
+        is_list = len(keys) >= 2 and isinstance(pydash.objects.get(value, keys[0]), list)
+        return sorted(_non_empty_list([
+            pydash.objects.get(x, '.'.join(keys[1:]))
+            for x in pydash.objects.get(value, keys[0], [])
+        ])) if is_list else prop_value
+
+    source_properties = [p for p in properties if _has_property(value, p)]
+    return '-'.join(map(str, flatten(map(property_value, source_properties))))
+
+
+def _build_matching_properties(values: list, model: dict = {}, merge_args: dict = {}, node_type: str = ''):
+    # only merge node if it has the same `methodModel`
+    same_methodModel = merge_args.get('sameMethodModel', False)
 
-    1. Update list of properties to handle `methodModel.@id` and `impactAssessment.@id`
-    2. Filter values that have the same unique properties as el
-    3. Make sure all shared unique properties are identical
-    """
     properties = _matching_properties(model, node_type)
     properties = list(set(properties + [_METHOD_MODEL_KEY])) if same_methodModel else [
         p for p in properties if p != _METHOD_MODEL_KEY
     ]
-    properties = _handle_local_property(values, properties, 'impactAssessment.id')
+    return _handle_local_property(values, properties, 'impactAssessment.id')
 
-    return next(
-        (i for i in range(len(values)) if _match_el(values[i], el, properties, merge_args)),
-        None
-    ) if properties else None
 
+def merge(source: list, new_values: list, version: str, model: dict = {}, merge_args: dict = {}, node_type: str = ''):
+    source = [] if source is None else source
 
-def merge(source: list, merge_with: list, version: str, model: dict = {}, merge_args: dict = {}, node_type: str = ''):
-    source = source if source is not None else []
-
-    # only merge node if it has the same `methodModel`
-    same_methodModel = merge_args.get('sameMethodModel', False)
     # only merge if the
     skip_same_term = merge_args.get('skipSameTerm', False)
 
-    for el in _non_empty_list(merge_with):
-        source_index = _find_match_el_index(source, el, same_methodModel, model, node_type, merge_args)
+    # build list of properties used to do the matching
+    properties = _build_matching_properties(source, model, merge_args, node_type)
+
+    source_index_keys = {
+        _value_index_key(value, properties, merge_args): index
+        for index, value in enumerate(source)
+    } if properties else None
+
+    for el in _non_empty_list(new_values):
+        new_value_index_key = _value_index_key(el, properties, merge_args)
+        source_index = source_index_keys.get(new_value_index_key) if source_index_keys else None
         if source_index is None:
+            # add to index keys for next elements
+            if source_index_keys:
+                source_index_keys[new_value_index_key] = len(source)
             source.append(update_node_version(version, el))
         elif not skip_same_term:
             source[source_index] = merge_node(source[source_index], el, version, model, merge_args)

{hestia_earth_models-0.73.6.dist-info → hestia_earth_models-0.73.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.73.6
+Version: 0.73.8
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -11,8 +11,8 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hestia-earth-schema==33.*
-Requires-Dist: hestia-earth-utils>=0.14.9
+Requires-Dist: hestia-earth-schema<34.0.0,>=33.4.0
+Requires-Dist: hestia-earth-utils>=0.15.1
 Requires-Dist: python-dateutil>=2.8.1
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0