hestia-earth-models 0.73.7__py3-none-any.whl → 0.74.0__py3-none-any.whl

hestia_earth/models/utils/__init__.py

@@ -43,6 +43,9 @@ def _omit(values: dict, keys: list) -> dict: return {k: v for k, v in values.ite
 def _include(value: dict, keys: list) -> dict: return {k: v for k, v in value.items() if k in keys}
 
 
+def unique_values(values: list, key='@id'): return list({v[key]: v for v in values}.values())
+
+
 def _run_in_serie(data: dict, models: list): return reduce(lambda prev, model: model(prev), models, data)
 
 

hestia_earth/models/utils/background_emissions.py

@@ -1,16 +1,19 @@
-from hestia_earth.schema import TermTermType
+from functools import reduce
+from typing import Callable, Tuple
+from hestia_earth.schema import TermTermType, EmissionMethodTier
+from hestia_earth.utils.lookup import _is_missing_value, lookup_columns
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
-from hestia_earth.utils.tools import flatten
+from hestia_earth.utils.tools import flatten, non_empty_list, safe_parse_float
 from hestia_earth.utils.emission import cycle_emissions_in_system_boundary
 
-from hestia_earth.models.log import logShouldRun
-from . import is_from_model
+from hestia_earth.models.log import logShouldRun, logRequirements, debugValues
+from . import is_from_model, _omit
 from .term import get_lookup_value
 
 
 def _animal_inputs(animal: dict):
     inputs = animal.get('inputs', [])
-    return [(input | {'animal': animal.get('term', {})}) for input in inputs]
+    return [(input | {'animalId': animal['animalId'], 'animal': animal.get('term', {})}) for input in inputs]
 
 
 def _should_run_input(products: list):
@@ -58,23 +61,127 @@ def no_gap_filled_background_emissions(
     return check_input
 
 
-def all_background_emission_term_ids(cycle: dict):
-    term_ids = cycle_emissions_in_system_boundary(cycle)
-    return list(set([
-        get_lookup_value({'termType': TermTermType.EMISSION.value, '@id': term_id}, 'inputProductionGroupId')
+def _all_background_emission_term_ids(node: dict, termType: TermTermType):
+    term_ids = cycle_emissions_in_system_boundary(node, termType=termType)
+    background_ids = list(set([
+        get_lookup_value({'termType': termType.value, '@id': term_id}, 'inputProductionGroupId')
         for term_id in term_ids
     ]))
+    # make sure input production emission is itself in the system boundary
+    return [term_id for term_id in background_ids if term_id in term_ids]
 
 
-def log_missing_emissions(cycle: dict, **log_args):
-    all_emission_term_ids = all_background_emission_term_ids(cycle)
+def log_missing_emissions(node: dict, termType: TermTermType = TermTermType.EMISSION, **log_args):
+    all_emission_term_ids = _all_background_emission_term_ids(node, termType)
 
     def log_input(input_term_id: str, included_emission_term_ids: list, **extra_log_args):
-        missing_emission_term_ids = [
+        missing_emission_term_ids = non_empty_list([
             term_id for term_id in all_emission_term_ids if term_id not in included_emission_term_ids
-        ]
+        ])
+
         for emission_id in missing_emission_term_ids:
-            logShouldRun(cycle, term=input_term_id, should_run=False, emission_id=emission_id,
+            # debug value on the emission itself so it appears for the input
+            debugValues(node, term=emission_id,
+                        value=None,
+                        coefficient=None,
+                        input=input_term_id,
+                        **log_args,
+                        **extra_log_args)
+            logRequirements(node, term=input_term_id,
+                            emission_id=emission_id,
+                            has_emission_factor=False,
+                            **log_args,
+                            **extra_log_args)
+            logShouldRun(node, term=input_term_id, should_run=False,
+                         emission_id=emission_id,
                          **log_args,
                          **extra_log_args)
     return log_input
+
+
+_KEY_TO_FIELD = {
+    'inputs': 'key'
+}
+
+
+def _key_to_field(key: str): return _KEY_TO_FIELD.get(key) or key
+
+
+def _values_from_column(index_column: str, column: str, value: str):
+    values = column.split('+')
+    term_id = values[0]
+    value = safe_parse_float(value, default=None)
+    return {
+        term_id: {
+            'value': value
+        } | {
+            _key_to_field(v.split('[')[0]): v.split('[')[1][:-1] for v in values[1:]
+        }
+    } if all([
+        column != index_column,
+        not column.startswith('ecoinvent'),
+        not column.startswith('ecoalim'),
+        not _is_missing_value(value)
+    ]) else {}
+
+
+def convert_background_lookup(lookup, index_column: str):
+    columns = lookup_columns(lookup)
+    return {
+        row[index_column]: reduce(
+            lambda prev, curr: prev | _values_from_column(index_column, curr, row[curr]),
+            columns,
+            {}
+        )
+        for row in lookup
+    }
+
+
+def parse_term_id(term_id: str): return term_id.split('-')[0]
+
+
+def join_term_id(term_id: str, data: dict):
+    return '-'.join(non_empty_list([term_id] + list(_omit(data, ['value']).values())))
+
+
+def _process_mapping(
+    node: dict,
+    input: dict,
+    term_type: TermTermType,
+    extract_mapping: Callable[[Tuple, TermTermType], Tuple[dict, float]],
+    **log_args
+) -> dict:
+    input_term_id = input.get('term', {}).get('@id')
+    operation_term_id = input.get('operation', {}).get('@id')
+    animal_term_id = input.get('animal', {}).get('@id')
+
+    def add(prev: dict, mapping: Tuple):
+        values, coefficient = extract_mapping(mapping, term_type)
+        for term_id, data in values:
+            # log run on each node so we know it did run
+            logShouldRun(node, term=input_term_id, should_run=True,
+                         methodTier=EmissionMethodTier.BACKGROUND.value,
+                         emission_id=term_id,
+                         **log_args)
+            debugValues(node, term=term_id,
+                        value=data.get('value'),
+                        coefficient=coefficient,
+                        input=input_term_id,
+                        operation=operation_term_id,
+                        animal=animal_term_id,
+                        **log_args)
+            group_id = join_term_id(term_id, data)
+            prev[group_id] = prev.get(group_id, []) + [data | {'coefficient': coefficient}]
+        return prev
+    return add
+
+
+def process_input_mappings(
+    node: dict,
+    input: dict,
+    mappings: list,
+    term_type: TermTermType,
+    extract_mapping: Callable[[tuple, TermTermType], Tuple[dict, float]],
+    **log_args
+):
+    return reduce(_process_mapping(node, input, term_type, extract_mapping, **log_args), mappings, {})

hestia_earth/models/utils/blank_node.py

@@ -111,17 +111,6 @@ def properties_logs(blank_nodes: list, properties: Union[dict, list]):
     return log_as_table(logs)
 
 
-def group_by_keys(group_keys: list = ['term']):
-    def run(group: dict, node: dict):
-        group_key = '-'.join(non_empty_list([
-            node.get(v, {}).get('@id') if isinstance(node.get(v), dict) else node.get(v)
-            for v in group_keys
-        ]))
-        group[group_key] = group.get(group_key, []) + [node]
-        return group
-    return run
-
-
 def _module_term_id(term_id: str, module):
     term_id_str = term_id.split('.')[-1] if '.' in term_id else term_id
     return getattr(module, 'TERM_ID', term_id_str).split(',')[0]
@@ -1337,6 +1326,7 @@ def get_inputs_from_properties(input: dict, term_types: Union[TermTermType, List
         {
             'term': p.get('key'),
             'value': [(p.get('value') / 100) * (p.get('share', 100) / 100) * input_value],
+            # for grouping
             'parent': term
         } for p in (properties or []) if all([p.get('key'), p.get('value')])
     ]) if input_value > 0 else []

hestia_earth/models/utils/emission.py

@@ -2,7 +2,7 @@ from collections.abc import Iterable
 from typing import Optional, Union
 from hestia_earth.schema import EmissionMethodTier, SchemaType, TermTermType
 from hestia_earth.utils.model import linked_node
-
+from hestia_earth.utils.emission import cycle_emissions_in_system_boundary, emissions_in_system_boundary
 
 from . import flatten_args
 from .term import download_term
@@ -13,20 +13,22 @@ from .constant import Units, get_atomic_conversion
 EMISSION_METHOD_TIERS = [e.value for e in EmissionMethodTier]
 
 
-def _new_emission(term, model=None):
+def _new_emission(term, model=None, country_id: str = None, key_id: str = None):
     node = {'@type': SchemaType.EMISSION.value}
     node['term'] = linked_node(term if isinstance(term, dict) else download_term(term, TermTermType.EMISSION))
+    if country_id:
+        node['country'] = linked_node(download_term(country_id, TermTermType.REGION))
+    if key_id:
+        node['key'] = linked_node(download_term(key_id))
     return include_methodModel(node, model)
 
 
-def get_nh3_no3_nox_to_n(cycle: dict, nh3_term_id: str, no3_term_id: str, nox_term_id: str, allow_none: bool = False):
-    default_value = 0 if allow_none else None
-
-    nh3 = find_terms_value(cycle.get('emissions', []), nh3_term_id, default=default_value)
+def get_nh3_no3_nox_to_n(cycle: dict, nh3_term_id: str = None, no3_term_id: str = None, nox_term_id: str = None):
+    nh3 = find_terms_value(cycle.get('emissions', []), nh3_term_id, default=None)
     nh3 = None if nh3 is None else nh3 / get_atomic_conversion(Units.KG_NH3, Units.TO_N)
-    no3 = find_terms_value(cycle.get('emissions', []), no3_term_id, default=default_value)
+    no3 = find_terms_value(cycle.get('emissions', []), no3_term_id, default=None)
     no3 = None if no3 is None else no3 / get_atomic_conversion(Units.KG_NO3, Units.TO_N)
-    nox = find_terms_value(cycle.get('emissions', []), nox_term_id, default=default_value)
+    nox = find_terms_value(cycle.get('emissions', []), nox_term_id, default=None)
     nox = None if nox is None else nox / get_atomic_conversion(Units.KG_NOX, Units.TO_N)
 
     return (nh3, no3, nox)
@@ -100,3 +102,11 @@ def to_emission_method_tier(method: Union[EmissionMethodTier, str]) -> Optional[
 def filter_emission_inputs(emission: dict, term_type: TermTermType):
     inputs = emission.get('inputs', [])
     return [i for i in inputs if i.get('termType') == term_type.value]
+
+
+def background_emissions_in_system_boundary(node: dict, term_type: TermTermType = TermTermType.EMISSION):
+    term_ids = (
+        cycle_emissions_in_system_boundary(node, term_type) if term_type == TermTermType.EMISSION else
+        emissions_in_system_boundary(term_type)
+    )
+    return [id for id in term_ids if 'InputsProduction' in id]

hestia_earth/models/utils/feedipedia.py

@@ -47,7 +47,7 @@ def _map_properties(lookup, term_id: str, column_prefix: str):
     return {'value': value, 'sd': sd, 'min': min, 'max': max}
 
 
-def rescale_properties_from_dryMatter(model: str, node: dict, blank_nodes: list):
+def rescale_properties_from_dryMatter(model: str, node: dict, blank_nodes: list, **log_args):
     properties = get_feedipedia_properties()
     # download all to save time
     term_types = [blank_node.get('term', {}).get('termType') for blank_node in blank_nodes]
@@ -78,7 +78,7 @@ def rescale_properties_from_dryMatter(model: str, node: dict, blank_nodes: list)
             ])
         ])
         for prop in new_properties:
-            logShouldRun(node, model, term_id, True, property=prop.get('term', {}).get('@id'))
+            logShouldRun(node, model, term_id, True, property=prop.get('term', {}).get('@id'), **log_args)
         return (
             blank_node | {'properties': merge_blank_nodes(all_properties, new_properties)}
         ) if new_properties else blank_node

hestia_earth/models/utils/impact_assessment.py

@@ -4,7 +4,7 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum, safe_parse_date
 
 from hestia_earth.models.log import debugValues
-from .lookup import all_factor_value, _region_factor_value, _aware_factor_value, fallback_country
+from .lookup import all_factor_value, region_factor_value, aware_factor_value, fallback_country
 from .product import find_by_product
 from .site import region_level_1_id
 
@@ -190,7 +190,7 @@ def impact_country_value(
         blank_nodes=blank_nodes,
         grouped_key=group_key,
         default_no_values=default_no_values,
-        factor_value_func=_region_factor_value
+        factor_value_func=region_factor_value
     )
 
 
@@ -219,10 +219,8 @@ def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, grou
     blank_nodes = impact.get('emissionsResourceUse', [])
     site = get_site(impact)
     aware_id = site.get('awareWaterBasinId')
-    if aware_id is None:
-        return None
 
-    return all_factor_value(
+    return None if aware_id is None else all_factor_value(
         logs_model=model,
         logs_term_id=term_id,
         node=impact,
@@ -231,7 +229,7 @@ def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, grou
         blank_nodes=blank_nodes,
         grouped_key=group_key,
         default_no_values=None,
-        factor_value_func=_aware_factor_value
+        factor_value_func=aware_factor_value
     )
 
 

hestia_earth/models/utils/indicator.py

@@ -5,7 +5,10 @@ from .method import include_methodModel
 from .term import download_term
 
 
-def _new_indicator(term, model=None, land_cover_id: str = None, previous_land_cover_id: str = None):
+def _new_indicator(
+    term: dict, model=None,
+    land_cover_id: str = None, previous_land_cover_id: str = None, country_id: str = None, key_id: str = None
+):
     node = {'@type': SchemaType.INDICATOR.value}
     node['term'] = linked_node(term if isinstance(term, dict) else download_term(
         term, TermTermType.CHARACTERISEDINDICATOR)
@@ -14,4 +17,8 @@ def _new_indicator(term, model=None, land_cover_id: str = None, previous_land_co
         node['landCover'] = linked_node(download_term(land_cover_id, TermTermType.LANDCOVER))
     if previous_land_cover_id:
         node['previousLandCover'] = linked_node(download_term(previous_land_cover_id, TermTermType.LANDCOVER))
+    if country_id:
+        node['country'] = linked_node(download_term(country_id, TermTermType.REGION))
+    if key_id:
+        node['key'] = linked_node(download_term(key_id))
     return include_methodModel(node, model)

hestia_earth/models/utils/lookup.py

@@ -19,16 +19,20 @@ def _node_value(node):
 
 
 def _factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, grouped_key: Optional[str] = None):
-    def get_value(data: dict):
-        node_term_id = data.get('term', {}).get('@id')
-        grouped_data_key = grouped_key or data.get('methodModel', {}).get('@id')
-        value = _node_value(data)
+    @lru_cache()
+    def get_coefficient(node_term_id: str, grouped_data_key: str):
         coefficient = get_region_lookup_value(lookup_name, node_term_id, lookup_col, model=model, term=term_id)
         # value is either a number or matching between a model and a value (restrict value to specific model only)
-        coefficient = safe_parse_float(
+        return safe_parse_float(
             extract_grouped_data(coefficient, grouped_data_key),
             default=None
         ) if ':' in str(coefficient) else safe_parse_float(coefficient, default=None)
+
+    def get_value(data: dict):
+        node_term_id = data.get('term', {}).get('@id')
+        grouped_data_key = grouped_key or data.get('methodModel', {}).get('@id')
+        value = _node_value(data)
+        coefficient = get_coefficient(node_term_id, grouped_data_key)
         if value is not None and coefficient is not None:
             if model:
                 debugValues(data, model=model, term=term_id,
@@ -40,7 +44,15 @@ def _factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, g
     return get_value
 
 
-def _region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):
+def region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):
+    @lru_cache()
+    def get_coefficient(node_term_id: str, region_term_id: str):
+        coefficient = get_region_lookup_value(lookup_name, region_term_id, node_term_id, model=model, term=term_id)
+        return safe_parse_float(
+            extract_grouped_data(coefficient, group_key) if group_key else coefficient,
+            default=None
+        )
+
     def get_value(data: dict):
         node_term_id = data.get('term', {}).get('@id')
         value = _node_value(data)
@@ -48,11 +60,7 @@ def _region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term
         region_term_id = (
             (data.get('region') or data.get('country') or {'@id': lookup_term_id}).get('@id')
         ) if lookup_term_id.startswith('GADM-') else lookup_term_id
-        coefficient = get_region_lookup_value(lookup_name, region_term_id, node_term_id, model=model, term=term_id)
-        coefficient = safe_parse_float(
-            extract_grouped_data(coefficient, group_key) if group_key else coefficient,
-            default=None
-        )
+        coefficient = get_coefficient(node_term_id, region_term_id)
         if value is not None and coefficient is not None:
             debugValues(data, model=model, term=term_id,
                         node=node_term_id,
@@ -62,26 +70,30 @@ def _region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term
     return get_value
 
 
-def _aware_factor_value(model: str, term_id: str, lookup_name: str, aware_id: str, group_key: str = None):
+def aware_factor_value(model: str, term_id: str, lookup_name: str, aware_id: str, group_key: str = None):
     lookup = download_lookup(lookup_name, False)  # avoid saving in memory as there could be many different files used
     lookup_col = column_name('awareWaterBasinId')
 
+    @lru_cache()
+    def get_coefficient(node_term_id: str):
+        coefficient = _get_single_table_value(lookup, lookup_col, int(aware_id), column_name(node_term_id))
+        return safe_parse_float(
+            extract_grouped_data(coefficient, group_key),
+            default=None
+        ) if group_key else coefficient
+
     def get_value(data: dict):
         node_term_id = data.get('term', {}).get('@id')
         value = _node_value(data)
 
         try:
-            coefficient = _get_single_table_value(lookup, lookup_col, int(aware_id), column_name(node_term_id))
-            coefficient = safe_parse_float(
-                extract_grouped_data(coefficient, group_key),
-                default=None
-            ) if group_key else coefficient
+            coefficient = get_coefficient(node_term_id)
             if value is not None and coefficient is not None:
                 debugValues(data, model=model, term=term_id,
                             node=node_term_id,
                             value=value,
                             coefficient=coefficient)
-        except ValueError:  # factor does not exist
+        except Exception:  # factor does not exist
             coefficient = None
 
         return {'id': node_term_id, 'value': value, 'coefficient': coefficient}

hestia_earth/models/utils/productivity.py

@@ -17,5 +17,5 @@ PRODUCTIVITY_KEY = {
 
 
 def get_productivity(country: dict, default: PRODUCTIVITY = PRODUCTIVITY.HIGH):
-    hdi = safe_parse_float(get_lookup_value(country, 'hdi'), default=None)
+    hdi = safe_parse_float(get_lookup_value(country, 'HDI'), default=None)
     return next((key for key in PRODUCTIVITY_KEY if hdi and PRODUCTIVITY_KEY[key](hdi)), default)

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.73.7'
+VERSION = '0.74.0'

hestia_earth/orchestrator/log.py

@@ -3,6 +3,8 @@ import sys
 import logging
 
 LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
+_EXTENDED_LOGS = os.getenv('LOG_EXTENDED', 'true') == 'true'
+_LOG_DATE_FORMAT = os.getenv('LOG_DATE_FORMAT', '%Y-%m-%dT%H:%M:%S%z')
 
 # disable root logger
 root_logger = logging.getLogger()
@@ -25,9 +27,12 @@ def log_to_file(filepath: str):
         Path of the file.
     """
     formatter = logging.Formatter(
-        '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", '
-        '"filename": "%(filename)s", "message": "%(message)s"}',
-        '%Y-%m-%dT%H:%M:%S%z')
+        '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", "message": "%(message)s"}',
+        _LOG_DATE_FORMAT
+    ) if _EXTENDED_LOGS else logging.Formatter(
+        '{"logger": "%(name)s", "message": "%(message)s"}',
+        _LOG_DATE_FORMAT
+    )
     handler = logging.FileHandler(filepath, encoding='utf-8')
     handler.setFormatter(formatter)
     handler.setLevel(logging.getLevelName(LOG_LEVEL))

hestia_earth/orchestrator/strategies/merge/merge_list.py

@@ -1,7 +1,7 @@
 import pydash
 from datetime import datetime
 from hestia_earth.schema import UNIQUENESS_FIELDS
-from hestia_earth.utils.tools import safe_parse_date
+from hestia_earth.utils.tools import safe_parse_date, flatten
 
 from hestia_earth.orchestrator.utils import _non_empty_list, update_node_version
 from .merge_node import merge as merge_node
@@ -27,39 +27,6 @@ def _has_property(value: dict, key: str):
 def _values_have_property(values: list, key: str): return any([_has_property(v, key) for v in values])
 
 
-def _match_list_el(source: list, dest: list, key: str):
-    src_value = sorted(_non_empty_list([pydash.objects.get(x, key) for x in source]))
-    dest_value = sorted(_non_empty_list([pydash.objects.get(x, key) for x in dest]))
-    return src_value == dest_value
-
-
-def _get_value(data: dict, key: str, merge_args: dict = {}):
-    value = pydash.objects.get(data, key)
-    date = safe_parse_date(value) if key in ['startDate', 'endDate'] else None
-    return datetime.strftime(date, merge_args.get('matchDatesFormat', '%Y-%m-%d')) if date else value
-
-
-def _match_el(source: dict, dest: dict, keys: list, merge_args: dict = {}):
-    def match(key: str):
-        keys = key.split('.')
-        src_value = _get_value(source, key, merge_args)
-        dest_value = _get_value(dest, key, merge_args)
-        is_list = len(keys) >= 2 and (
-            isinstance(pydash.objects.get(source, keys[0]), list) or
-            isinstance(pydash.objects.get(dest, keys[0]), list)
-        )
-        return _match_list_el(
-            pydash.objects.get(source, keys[0], []),
-            pydash.objects.get(dest, keys[0], []),
-            '.'.join(keys[1:])
-        ) if is_list else src_value == dest_value
-
-    source_properties = [p for p in keys if _has_property(source, p)]
-    dest_properties = [p for p in keys if _has_property(dest, p)]
-
-    return all(map(match, source_properties)) if source_properties == dest_properties else False
-
-
 def _handle_local_property(values: list, properties: list, local_id: str):
     # Handle "impactAssessment.@id" if present in the data
     existing_id = local_id.replace('.id', '.@id')
@@ -76,38 +43,58 @@ def _handle_local_property(values: list, properties: list, local_id: str):
     return properties
 
 
-def _find_match_el_index(values: list, el: dict, same_methodModel: bool, model: dict, node_type: str, merge_args: dict):
-    """
-    Find an element in the values that match the new element, based on the unique properties.
-    To find a matching element:
+def _get_value(data: dict, key: str, merge_args: dict = {}):
+    value = pydash.objects.get(data, key)
+    date = safe_parse_date(value) if key in ['startDate', 'endDate'] else None
+    return datetime.strftime(date, merge_args.get('matchDatesFormat', '%Y-%m-%d')) if date else value
+
+
+def _value_index_key(value: dict, properties: list, merge_args: dict = {}):
+    def property_value(key: str):
+        keys = key.split('.')
+        prop_value = _get_value(value, key, merge_args)
+        is_list = len(keys) >= 2 and isinstance(pydash.objects.get(value, keys[0]), list)
+        return sorted(_non_empty_list([
+            pydash.objects.get(x, '.'.join(keys[1:]))
+            for x in pydash.objects.get(value, keys[0], [])
+        ])) if is_list else prop_value
+
+    source_properties = [p for p in properties if _has_property(value, p)]
+    return '-'.join(map(str, flatten(map(property_value, source_properties))))
+
+
+def _build_matching_properties(values: list, model: dict = {}, merge_args: dict = {}, node_type: str = ''):
+    # only merge node if it has the same `methodModel`
+    same_methodModel = merge_args.get('sameMethodModel', False)
 
-    1. Update list of properties to handle `methodModel.@id` and `impactAssessment.@id`
-    2. Filter values that have the same unique properties as el
-    3. Make sure all shared unique properties are identical
-    """
     properties = _matching_properties(model, node_type)
     properties = list(set(properties + [_METHOD_MODEL_KEY])) if same_methodModel else [
         p for p in properties if p != _METHOD_MODEL_KEY
     ]
-    properties = _handle_local_property(values, properties, 'impactAssessment.id')
+    return _handle_local_property(values, properties, 'impactAssessment.id')
 
-    return next(
-        (i for i in range(len(values)) if _match_el(values[i], el, properties, merge_args)),
-        None
-    ) if properties else None
 
+def merge(source: list, new_values: list, version: str, model: dict = {}, merge_args: dict = {}, node_type: str = ''):
+    source = [] if source is None else source
 
-def merge(source: list, merge_with: list, version: str, model: dict = {}, merge_args: dict = {}, node_type: str = ''):
-    source = source if source is not None else []
-
-    # only merge node if it has the same `methodModel`
-    same_methodModel = merge_args.get('sameMethodModel', False)
     # only merge if the
     skip_same_term = merge_args.get('skipSameTerm', False)
 
-    for el in _non_empty_list(merge_with):
-        source_index = _find_match_el_index(source, el, same_methodModel, model, node_type, merge_args)
+    # build list of properties used to do the matching
+    properties = _build_matching_properties(source, model, merge_args, node_type)
+
+    source_index_keys = {
+        _value_index_key(value, properties, merge_args): index
+        for index, value in enumerate(source)
+    } if properties else None
+
+    for el in _non_empty_list(new_values):
+        new_value_index_key = _value_index_key(el, properties, merge_args)
+        source_index = source_index_keys.get(new_value_index_key) if source_index_keys else None
         if source_index is None:
+            # add to index keys for next elements
+            if source_index_keys:
+                source_index_keys[new_value_index_key] = len(source)
             source.append(update_node_version(version, el))
         elif not skip_same_term:
             source[source_index] = merge_node(source[source_index], el, version, model, merge_args)

{hestia_earth_models-0.73.7.dist-info → hestia_earth_models-0.74.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.73.7
+Version: 0.74.0
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -11,8 +11,8 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hestia-earth-schema==33.*
-Requires-Dist: hestia-earth-utils>=0.14.9
+Requires-Dist: hestia-earth-schema<34.0.0,>=33.5.0
+Requires-Dist: hestia-earth-utils>=0.15.1
 Requires-Dist: python-dateutil>=2.8.1
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0