hestia-earth-models 0.74.4__py3-none-any.whl → 0.74.5__py3-none-any.whl

hestia_earth/models/utils/lookup.py

@@ -5,8 +5,7 @@ from hestia_earth.utils.lookup import (
     get_table_value,
     column_name,
     extract_grouped_data,
-    _get_single_table_value,
-    lookup_term_ids
+    _get_single_table_value
 )
 from hestia_earth.utils.tools import list_sum, safe_parse_float, non_empty_list
 
@@ -18,59 +17,99 @@ def _node_value(node):
     return list_sum(value, default=None) if isinstance(value, list) else value
 
 
-def _factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, grouped_key: Optional[str] = None):
+def _log_value_coeff(log_node: dict, value: float, coefficient: float, **log_args):
+    if value is not None and coefficient:
+        debugValues(log_node, value=value, coefficient=coefficient, **log_args)
+
+
+def _factor_value(
+    log_node: dict,
+    model: str,
+    term_id: str,
+    lookup_name: str,
+    lookup_col: str,
+    group_key: Optional[str] = None,
+    default_world_value: Optional[bool] = False
+):
     @lru_cache()
     def get_coefficient(node_term_id: str, grouped_data_key: str):
-        coefficient = get_region_lookup_value(lookup_name, node_term_id, lookup_col, model=model, term=term_id)
+        coefficient = get_region_lookup_value(
+            lookup_name=lookup_name,
+            term_id=node_term_id,
+            column=lookup_col,
+            fallback_world=default_world_value,
+            model=model, term=term_id
+        )
         # value is either a number or matching between a model and a value (restrict value to specific model only)
         return safe_parse_float(
             extract_grouped_data(coefficient, grouped_data_key),
             default=None
         ) if ':' in str(coefficient) else safe_parse_float(coefficient, default=None)
 
-    def get_value(data: dict):
-        node_term_id = data.get('term', {}).get('@id')
-        grouped_data_key = grouped_key or data.get('methodModel', {}).get('@id')
-        value = _node_value(data)
+    def get_value(blank_node: dict):
+        node_term_id = blank_node.get('term', {}).get('@id')
+        grouped_data_key = group_key or blank_node.get('methodModel', {}).get('@id')
+        value = _node_value(blank_node)
         coefficient = get_coefficient(node_term_id, grouped_data_key)
-        if value is not None and coefficient is not None:
-            if model:
-                debugValues(data, model=model, term=term_id,
-                            node=node_term_id,
-                            operation=data.get('operation', {}).get('@id'),
-                            value=value,
-                            coefficient=coefficient)
+        if model:
+            _log_value_coeff(log_node=log_node, value=value, coefficient=coefficient,
+                             model=model,
+                             term=term_id,
+                             node=node_term_id,
+                             operation=blank_node.get('operation', {}).get('@id'))
         return {'id': node_term_id, 'value': value, 'coefficient': coefficient}
     return get_value
 
 
-def region_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):
+def region_factor_value(
+    log_node: dict,
+    model: str,
+    term_id: str,
+    lookup_name: str,
+    lookup_term_id: str,
+    group_key: Optional[str] = None,
+    default_world_value: Optional[bool] = False
+):
     @lru_cache()
     def get_coefficient(node_term_id: str, region_term_id: str):
-        coefficient = get_region_lookup_value(lookup_name, region_term_id, node_term_id, model=model, term=term_id)
+        coefficient = get_region_lookup_value(
+            lookup_name=lookup_name,
+            term_id=region_term_id,
+            column=node_term_id,
+            fallback_world=default_world_value,
+            model=model, term=term_id
+        )
         return safe_parse_float(
             extract_grouped_data(coefficient, group_key) if group_key else coefficient,
             default=None
         )
 
-    def get_value(data: dict):
-        node_term_id = data.get('term', {}).get('@id')
-        value = _node_value(data)
+    def get_value(blank_node: dict):
+        node_term_id = blank_node.get('term', {}).get('@id')
+        value = _node_value(blank_node)
         # when getting data for a `region`, we can try to get the `region` on the node first, in case it is set
         region_term_id = (
-            (data.get('region') or data.get('country') or {'@id': lookup_term_id}).get('@id')
+            (blank_node.get('region') or blank_node.get('country') or {'@id': lookup_term_id}).get('@id')
         ) if lookup_term_id.startswith('GADM-') else lookup_term_id
         coefficient = get_coefficient(node_term_id, region_term_id)
-        if value is not None and coefficient is not None:
-            debugValues(data, model=model, term=term_id,
-                        node=node_term_id,
-                        value=value,
-                        coefficient=coefficient)
+        _log_value_coeff(log_node=log_node, value=value, coefficient=coefficient,
+                         model=model,
+                         term=term_id,
+                         node=node_term_id,
+                         operation=blank_node.get('operation', {}).get('@id'))
         return {'id': node_term_id, 'region-id': region_term_id, 'value': value, 'coefficient': coefficient}
     return get_value
 
 
-def aware_factor_value(model: str, term_id: str, lookup_name: str, aware_id: str, group_key: str = None):
+def aware_factor_value(
+    log_node: dict,
+    model: str,
+    term_id: str,
+    lookup_name: str,
+    aware_id: str,
+    group_key: Optional[str] = None,
+    default_world_value: Optional[bool] = False
+):
     lookup = download_lookup(lookup_name, False)  # avoid saving in memory as there could be many different files used
     lookup_col = column_name('awareWaterBasinId')
 
@@ -82,17 +121,16 @@ def aware_factor_value(model: str, term_id: str, lookup_name: str, aware_id: str
             default=None
         ) if group_key else coefficient
 
-    def get_value(data: dict):
-        node_term_id = data.get('term', {}).get('@id')
-        value = _node_value(data)
+    def get_value(blank_node: dict):
+        node_term_id = blank_node.get('term', {}).get('@id')
+        value = _node_value(blank_node)
 
         try:
             coefficient = get_coefficient(node_term_id)
-            if value is not None and coefficient is not None:
-                debugValues(data, model=model, term=term_id,
-                            node=node_term_id,
-                            value=value,
-                            coefficient=coefficient)
+            _log_value_coeff(log_node=log_node, value=value, coefficient=coefficient,
+                             model=model,
+                             term=term_id,
+                             node=node_term_id)
         except Exception:  # factor does not exist
             coefficient = None
 
@@ -101,17 +139,21 @@ def aware_factor_value(model: str, term_id: str, lookup_name: str, aware_id: str
 
 
 def all_factor_value(
-    logs_model: str,
-    logs_term_id: str,
-    node: dict,
+    log_model: str,
+    log_term_id: str,
+    log_node: dict,
     lookup_name: str,
     lookup_col: str,
     blank_nodes: List[dict],
-    grouped_key: Optional[str] = None,
+    group_key: Optional[str] = None,
     default_no_values=0,
-    factor_value_func=_factor_value
+    factor_value_func=_factor_value,
+    default_world_value: bool = False
 ):
-    values = list(map(factor_value_func(logs_model, logs_term_id, lookup_name, lookup_col, grouped_key), blank_nodes))
+    values = list(map(
+        factor_value_func(log_node, log_model, log_term_id, lookup_name, lookup_col, group_key, default_world_value),
+        blank_nodes
+    ))
 
     has_values = len(values) > 0
     missing_values = set([
@@ -129,11 +171,11 @@ def all_factor_value(
             row_value=debug_values[1] if len(debug_values) == 2 else debug_values[0],
             col=debug_values[0] if len(debug_values) == 2 else lookup_col,
             value=None,
-            model=logs_model,
-            term=logs_term_id
+            model=log_model,
+            term=log_term_id
         )
 
-    debugValues(node, model=logs_model, term=logs_term_id,
+    debugValues(log_node, model=log_model, term=log_term_id,
                 all_with_factors=all_with_factors,
                 missing_lookup_factor=log_as_table([
                     {
@@ -144,7 +186,7 @@ def all_factor_value(
                     for v in missing_values
                 ]),
                 has_values=has_values,
-                values_used=log_as_table(values))
+                values_used=log_as_table([v for v in values if v.get('coefficient')]))
 
     values = [float((v.get('value') or 0) * (v.get('coefficient') or 0)) for v in values]
 
@@ -152,37 +194,20 @@ def all_factor_value(
     return None if not all_with_factors else (list_sum(values) if has_values else default_no_values)
 
 
-def _country_in_lookup(country_id: str):
-    def in_lookup(lookup_name: str):
-        return (
-            download_lookup(lookup_name.replace('region', country_id)) is not None or
-            country_id in lookup_term_ids(download_lookup(lookup_name))
-        )
-    return in_lookup
-
-
-def fallback_country(country_id: str, lookups: List[str]) -> str:
-    """
-    Given a country `@id`, and lookup tables, checks if a location can be used in lookup file
-    else fallback to the default "region-world".
-    """
-    is_in_lookup = lambda v: all(map(_country_in_lookup(v), lookups))  # noqa: E731
-    fallback_id = 'region-world'
-    return country_id if country_id and is_in_lookup(country_id) else fallback_id if is_in_lookup(fallback_id) else None
-
-
 def get_region_lookup(lookup_name: str, term_id: str):
     # for performance, try to load the region specific lookup if exists
     return (
-        download_lookup(lookup_name.replace('region-', f"{term_id}-"))
+        download_lookup(lookup_name.replace('region-', f"{term_id}-"), build_index=True)
         if lookup_name and lookup_name.startswith('region-') else None
-    ) or download_lookup(lookup_name)
+    ) or download_lookup(lookup_name, build_index=True)
 
 
 @lru_cache()
-def get_region_lookup_value(lookup_name: str, term_id: str, column: str, **log_args):
+def get_region_lookup_value(lookup_name: str, term_id: str, column: str, fallback_world: bool = False, **log_args):
     # for performance, try to load the region specific lookup if exists
     lookup = get_region_lookup(lookup_name, term_id)
     value = get_table_value(lookup, 'termid', term_id, column_name(column))
+    if value is None and fallback_world:
+        return get_region_lookup_value(lookup_name, 'region-world', column, **log_args)
     debugMissingLookup(lookup_name, 'termid', term_id, column, value, **log_args)
     return value

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.74.4'
+VERSION = '0.74.5'

hestia_earth/orchestrator/models/__init__.py

@@ -9,10 +9,10 @@ from copy import deepcopy
 from hestia_earth.utils.tools import non_empty_list, current_time_ms
 
 from hestia_earth.models.version import VERSION
-from ..log import logger
-from ..utils import get_required_model_param, _snakecase
-from ..strategies.run import should_run
-from ..strategies.merge import merge
+from hestia_earth.orchestrator.log import logger
+from hestia_earth.orchestrator.utils import get_required_model_param, _snakecase, reset_index
+from hestia_earth.orchestrator.strategies.run import should_run
+from hestia_earth.orchestrator.strategies.merge import merge
 
 
 def _memory_usage():
@@ -29,6 +29,23 @@ def _max_workers(type: str):
         return None
 
 
+# do not deep copy to improve performance, only set on low risk keys
+_SKIP_DEEPCOPY_KEYS = [
+    'emissions',
+    'emissionsResourceUse'
+]
+
+
+def _node_copy(node: dict):
+    skip_keys = [key for key in _SKIP_DEEPCOPY_KEYS if key in node and isinstance(node[key], list)]
+    new_node = deepcopy(node | {
+        key: [] for key in skip_keys
+    }) | {
+        key: node[key] for key in skip_keys
+    }
+    return new_node
+
+
 def _list_except_item(list, item):
     idx = list.index(item)
     return list[:idx] + list[idx+1:]
@@ -66,22 +83,40 @@ def _import_model(name: str):
 
 def _run_pre_checks(data: dict):
     node_type = _snakecase(data.get('@type', data.get('type')))
+
+    now = current_time_ms()
+    memory_usage = _memory_usage()
+
     try:
         pre_checks = _import_model('.'.join([node_type, 'pre_checks'])).get('run')
         logger.info('running pre checks for %s', node_type)
-        return pre_checks(data)
+        data = pre_checks(data)
     except Exception:
-        return data
+        pass
+
+    logger.info('model_model=%s, model_value=%s, time=%s, memory_used=%s',
+                node_type, 'pre_checks', current_time_ms() - now, _memory_usage() - memory_usage)
+
+    return data
 
 
 def _run_post_checks(data: dict):
     node_type = _snakecase(data.get('@type', data.get('type')))
+
+    now = current_time_ms()
+    memory_usage = _memory_usage()
+
     try:
         post_checks = _import_model('.'.join([node_type, 'post_checks'])).get('run')
         logger.info('running post checks for %s', node_type)
-        return post_checks(data)
+        data = post_checks(data)
     except Exception:
-        return data
+        pass
+
+    logger.info('model_model=%s, model_value=%s, time=%s, memory_used=%s',
+                node_type, 'post_checks', current_time_ms() - now, _memory_usage() - memory_usage)
+
+    return data
 
 
 def _run_model(data: dict, model: dict, all_models: list):
@@ -108,7 +143,7 @@ def _run(data: dict, model: dict, all_models: list):
 def _run_serie(data: dict, models: list, stage: Union[int, List[int]] = None):
     return reduce(
         lambda prev, m: merge(
-            prev, _run_parallel(prev, m, models) if isinstance(m, list) else [_run(deepcopy(prev), m, models)]
+            prev, _run_parallel(prev, m, models) if isinstance(m, list) else [_run(_node_copy(prev), m, models)]
         ),
         _filter_models_stage(models, stage=stage),
         data
@@ -120,7 +155,7 @@ def _run_parallel(data: dict, model: list, all_models: list):
 
     max_workers = _max_workers(data.get('@type', data.get('type')))
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        futures = [executor.submit(_run, deepcopy(data), m, all_models) for m in model]
+        futures = [executor.submit(_run, _node_copy(data), m, all_models) for m in model]
 
     for future in concurrent.futures.as_completed(futures):
         results.append(future.result())
@@ -129,6 +164,8 @@ def _run_parallel(data: dict, model: list, all_models: list):
 
 
 def run(data: dict, models: list, stage: Union[int, List[int]] = None):
+    # make sure we reset before recalculating the node
+    reset_index()
     # run pre-checks if exist
     data = _run_pre_checks(data)
     data = _run_serie(data, models, stage=stage)

hestia_earth/orchestrator/models/transformations.py

@@ -7,7 +7,7 @@ from hestia_earth.models.utils.transformation import previous_transformation
 from hestia_earth.models.utils.product import find_by_product
 
 from . import run as run_node, _import_model
-from hestia_earth.orchestrator.utils import new_practice, _filter_by_keys
+from hestia_earth.orchestrator.utils import new_practice, _filter_by_keys, reset_index
 
 
 def _full_completeness():
@@ -94,6 +94,8 @@ def _run_transformation(cycle: dict, models: list):
         transformation = _add_excreta_inputs(previous, transformation)
         transformation = _apply_transformation_share(previous, transformation)
         transformation = _run_models(cycle, transformation, models)
+        # reset the index between 2 transformations, as they dont share the same values
+        reset_index()
         return transformations + [transformation]
     return run
 

hestia_earth/orchestrator/strategies/merge/__init__.py

@@ -1,5 +1,4 @@
 from functools import reduce
-import pydash
 
 from hestia_earth.orchestrator.utils import _non_empty, _non_empty_list, update_node_version
 from . import merge_append
@@ -34,7 +33,7 @@ def _merge_result(data: dict, result: dict):
     node_type = data.get('type', data.get('@type'))
     values = [values] if not isinstance(values, list) and merge_type == 'list' else values
     new_value = _STRATEGIES[merge_type](current, values, version, model, merge_args, node_type)
-    new_data = pydash.objects.merge({}, data, {key: new_value})
+    new_data = data | {key: new_value}
     return update_node_version(version, new_data, data)
 
 

hestia_earth/orchestrator/strategies/merge/merge_list.py

@@ -3,7 +3,12 @@ from datetime import datetime
 from hestia_earth.schema import UNIQUENESS_FIELDS
 from hestia_earth.utils.tools import safe_parse_date, flatten
 
-from hestia_earth.orchestrator.utils import _non_empty_list, update_node_version
+from hestia_earth.orchestrator.utils import (
+    _non_empty_list,
+    update_node_version,
+    wrap_index,
+    update_index
+)
 from .merge_node import merge as merge_node
 
 _METHOD_MODEL_KEY = 'methodModel.@id'
@@ -74,7 +79,14 @@ def _build_matching_properties(values: list, model: dict = {}, merge_args: dict
     return _handle_local_property(values, properties, 'impactAssessment.id')
 
 
-def merge(source: list, new_values: list, version: str, model: dict = {}, merge_args: dict = {}, node_type: str = ''):
+def merge(
+    source: list,
+    new_values: list,
+    version: str,
+    model: dict = {},
+    merge_args: dict = {},
+    node_type: str = ''
+):
     source = [] if source is None else source
 
     # only merge if the
@@ -83,19 +95,30 @@ def merge(source: list, new_values: list, version: str, model: dict = {}, merge_
     # build list of properties used to do the matching
     properties = _build_matching_properties(source, model, merge_args, node_type)
 
-    source_index_keys = {
-        _value_index_key(value, properties, merge_args): index
-        for index, value in enumerate(source)
-    } if properties else None
+    # store previous identical index to speed merging
+    index_key = '-'.join([node_type, model.get('key', '')])
+    # when the subkey changes, we need to completely rebuild the index
+    index_sub_key = '-'.join(properties + [str(merge_args)])
+
+    def build_index():
+        return {
+            _value_index_key(value, properties, merge_args): index
+            for index, value in enumerate(source)
+        } if properties else None
+
+    source_index_keys = wrap_index(index_key, index_sub_key, build_index)
 
     for el in _non_empty_list(new_values):
         new_value_index_key = _value_index_key(el, properties, merge_args)
         source_index = source_index_keys.get(new_value_index_key) if source_index_keys else None
         if source_index is None:
             # add to index keys for next elements
-            if source_index_keys:
-                source_index_keys[new_value_index_key] = len(source)
+            source_index_keys = source_index_keys or {}
+            source_index_keys[new_value_index_key] = len(source)
             source.append(update_node_version(version, el))
         elif not skip_same_term:
             source[source_index] = merge_node(source[source_index], el, version, model, merge_args)
+
+    update_index(index_key, index_sub_key, source_index_keys)
+
     return source

hestia_earth/orchestrator/utils.py

@@ -6,6 +6,35 @@ from functools import reduce
 EXCLUDED_VERSION_KEYS = [
     '@type'
 ]
+_memory = {}
+
+
+def wrap_index(key: str, sub_key: str, func):
+    global _memory  # noqa: F824
+    memory_data = _memory.get(key, {})
+    data = (memory_data.get('data') if memory_data.get('sub_key') == sub_key else None) or func()
+    _memory[key] = {
+        'sub_key': sub_key,
+        'data': data
+    }
+    return data
+
+
+def update_index(key: str, sub_key: str, data):
+    global _memory  # noqa: F824
+    _memory[key] = {
+        'sub_key': sub_key,
+        'data': data
+    }
+    return data
+
+
+def reset_index():
+    """
+    Reset the merging index between different nodes.
+    """
+    global _memory
+    _memory = {}
 
 
 def get_required_model_param(model, key: str):

{hestia_earth_models-0.74.4.dist-info → hestia_earth_models-0.74.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.74.4
+Version: 0.74.5
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -12,8 +12,7 @@ Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hestia-earth-schema<34.0.0,>=33.5.0
-Requires-Dist: hestia-earth-utils>=0.15.1
-Requires-Dist: python-dateutil>=2.8.1
+Requires-Dist: hestia-earth-utils>=0.15.3
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0
 Requires-Dist: pydash