hestia-earth-models 0.66.0__py3-none-any.whl → 0.67.1__py3-none-any.whl

hestia_earth/models/site/management.py

@@ -110,7 +110,7 @@ _PRACTICES_TERM_TYPES = [
     TermTermType.LANDUSEMANAGEMENT,
     TermTermType.SYSTEM
 ]
-_PRACTICES_COMPLETENESS_MAPPING = COMPLETENESS_MAPPING.get(SchemaType.PRACTICE.value)
+_PRACTICES_COMPLETENESS_MAPPING = COMPLETENESS_MAPPING.get(SchemaType.PRACTICE.value, {})
 _ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
 _INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
 _ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
@@ -242,7 +242,7 @@ def _get_relevant_items(cycle: dict, item_name: str, term_types: List[TermTermTy
         term_type for term_type in term_types
         if any([
             not completeness_mapping.get(term_type.value),
-            cycle.get('completeness').get(completeness_mapping.get(term_type.value), False)
+            cycle.get('completeness', {}).get(completeness_mapping.get(term_type.value), False)
         ])
     ]
     blank_nodes = filter_list_term_type(cycle.get(item_name, []), complete_term_types)

hestia_earth/models/utils/__init__.py

@@ -165,6 +165,12 @@ def last_day_of_month(year: int, month: int):
     )
 
 
+def current_date(): return datetime.datetime.now().date().strftime('%Y-%m-%d')
+
+
+def current_year(): return int(current_date()[:4])
+
+
 def flatten_args(args) -> list:
     """
     Flatten the input args into a single list.

hestia_earth/models/utils/aggregated.py

@@ -6,7 +6,8 @@ from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.tools import safe_parse_date, non_empty_list
 
 from hestia_earth.models.log import debugValues, logShouldRun
-from hestia_earth.models.utils.cycle import is_organic
+from . import current_year
+from .cycle import is_organic
 
 MODEL_KEY = 'impactAssessment'
 MATCH_WORLD_QUERY = {'match': {'country.name.keyword': {'query': 'World', 'boost': 1}}}
@@ -14,7 +15,7 @@ MATCH_WORLD_QUERY = {'match': {'country.name.keyword': {'query': 'World', 'boost
 
 def aggregated_end_date(end_date: str):
     year = safe_parse_date(end_date).year
-    return round(math.floor(year / 10) * 10) + 9
+    return min([round(math.floor(year / 10) * 10) + 9, current_year()])
 
 
 def _match_region_country(region: dict, country: dict):
@@ -36,8 +37,7 @@ def _match_region_country(region: dict, country: dict):
     }
 
 
-def find_closest_impact(cycle: dict, end_date: str, input: dict, region: dict, country: dict, must_queries=[]):
-    term = input.get('term', {})
+def find_closest_impact(cycle: dict, end_date: str, term: dict, region: dict, country: dict, must_queries=[]):
     query = {
         'bool': {
             'must': non_empty_list([
@@ -74,21 +74,24 @@ def find_closest_impact(cycle: dict, end_date: str, input: dict, region: dict, c
 
 def _link_input_to_impact(model: str, cycle: dict, date: int):
     def run(input: dict):
-        term_id = input.get('term', {}).get('@id')
+        term = input.get('term', {})
+        term_id = term.get('@id')
         region = input.get('region')
         country = input.get('country')
-        impact = find_closest_impact(cycle, date, input, region, country)
+        impact = find_closest_impact(cycle, date, term, region, country)
 
+        search_by_region_id = (region or country or {}).get('@id') or 'region-world'
         debugValues(cycle, model=model, term=term_id, key=MODEL_KEY,
-                    input_region=(region or {}).get('@id'),
-                    input_country=(country or {}).get('@id'),
-                    impact=(impact or {}).get('@id'))
+                    search_by_input_term_id=term_id,
+                    search_by_region_id=search_by_region_id,
+                    search_by_end_date=str(date),
+                    impact_assessment_id_found=(impact or {}).get('@id'))
 
         should_run = all([impact is not None])
         logShouldRun(cycle, model, term_id, should_run)
         logShouldRun(cycle, model, term_id, should_run, key=MODEL_KEY)  # show specifically under Input
 
-        return {**input, MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True} if impact else None
+        return input | {MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True} if impact else None
     return run
 
 

hestia_earth/models/utils/array_builders.py

@@ -527,12 +527,13 @@ def avg_run_in_rowwise(arr: NDArray, n: int):
     return avg_run_in_columnwise(arr.transpose(), n).transpose()
 
 
-def gen_seed(node: dict) -> int:
+def gen_seed(node: dict, *args: tuple[str]) -> int:
     """
-    Generate a seed based on a node's `@id` so that rng is the same each time the model is re-run.
+    Generate a seed based on a node's `@id` and optional args so that rng is the same each time the model is re-run.
     """
     node_id = node.get("@id", "")
-    hashed = hashlib.shake_128(node_id.encode(), usedforsecurity=False).hexdigest(4)
+    seed_str = "".join([node_id] + [str(arg) for arg in args])
+    hashed = hashlib.shake_128(seed_str.encode(), usedforsecurity=False).hexdigest(4)
     return abs(int(hashed, 16))
 
 

hestia_earth/models/utils/blank_node.py

@@ -12,7 +12,6 @@ from typing import (
     Optional,
     Union
 )
-
 from dateutil import parser
 from dateutil.relativedelta import relativedelta
 from hestia_earth.schema import TermTermType
@@ -22,6 +21,7 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import (
     flatten,
     list_sum,
+    list_average,
     safe_parse_date,
     safe_parse_float,
     non_empty_list
@@ -35,7 +35,7 @@ from .lookup import (
     is_product_id_allowed, is_product_termType_allowed,
     is_input_id_allowed, is_input_termType_allowed, _node_value
 )
-from .property import get_node_property, get_node_property_value, find_term_property
+from .property import get_node_property, get_node_property_value
 from .term import get_lookup_value
 from ..log import debugValues, log_as_table
 
@@ -44,6 +44,14 @@ MAX_DEPTH = 1000
 OLDEST_DATE = '1800'
 
 
+def group_by_term(values: list):
+    def group_by(groups: dict, value: dict):
+        key = value.get('term', {}).get('@id')
+        groups[key] = groups.get(key, []) + [value]
+        return groups
+    return reduce(group_by, values, {})
+
+
 def merge_blank_nodes(source: list, new_values: list):
     """
     Merge a list of blank nodes into an existing list of blank nodes.
@@ -1335,11 +1343,14 @@ def _node_from_group(nodes: list):
     # `nodes` contain list with consecutive dates
     return nodes[0] if len(nodes) == 1 else (
         # if all nodes have the same dates, sum up the values
-        nodes[0] | {'value': _sum_nodes_value(nodes)} if _same_dates(nodes)
-        else nodes[0] | {
-            'startDate': min(n.get('startDate') for n in nodes),
-            'endDate': max(n.get('endDate') for n in nodes)
-        }
+        nodes[0] | (
+            {
+                'value': _sum_nodes_value(nodes)
+            } if _same_dates(nodes) else {
+                'startDate': min(n.get('startDate') for n in nodes),
+                'endDate': max(n.get('endDate') for n in nodes)
+            }
+        )
     )
 
 
@@ -1349,7 +1360,7 @@ def _condense_nodes(nodes: list):
     return flatten(map(_node_from_group, grouped_nodes))
 
 
-def _group_nodes_to_condense(nodes: list) -> dict:
+def _group_nodes_by_value_and_properties(nodes: list) -> dict:
     def _group_node(group: dict, node: dict):
         value = node.get('value', [])
         value = '-'.join(map(str, value if isinstance(value, list) else [value]))
@@ -1359,7 +1370,6 @@ def _group_nodes_to_condense(nodes: list) -> dict:
                 f"{p.get('value')}"
             ])) for p in node.get('properties', [])
         ]))
-        # group by term, value, and properties
         group_key = '-'.join(non_empty_list([
             node.get('term', {}).get('@id', ''),
             value,
@@ -1371,8 +1381,51 @@ def _group_nodes_to_condense(nodes: list) -> dict:
     return reduce(_group_node, nodes, {})
 
 
+def _group_nodes_by_dates(nodes: list) -> dict:
+    def _group_node(group: dict, node: dict):
+        group_key = '-'.join(non_empty_list([
+            node.get('term', {}).get('@id', ''),
+            node.get('startDate'),
+            node.get('endDate'),
+        ]))
+        group[group_key] = group.get(group_key, []) + [node]
+        return group
+
+    return reduce(_group_node, nodes, {})
+
+
+def _average_properties(properties: list):
+    # group properties by term
+    grouped_properties = group_by_term(properties)
+    return [
+        props[0] | {
+            'value': list_average(non_empty_list([p.get('value') for p in props]), default=props[0].get('value'))
+        }
+        for props in grouped_properties.values()
+    ]
+
+
+def _merge_same_dates(nodes: list):
+    # group by term, startDate and endDate
+    grouped_nodes = _group_nodes_by_dates(nodes)
+
+    def merge_nodes(nodes: list):
+        properties = flatten([n.get('properties', []) for n in nodes])
+        return nodes[0] | (
+            {
+                'value': _sum_nodes_value(nodes)
+            } | ({
+                'properties': _average_properties(properties)
+            } if properties else {})
+        ) if len(nodes) > 1 else nodes[0]
+
+    return list(map(merge_nodes, grouped_nodes.values()))
+
+
 def condense_nodes(nodes: list) -> list:
-    grouped_nodes = _group_nodes_to_condense(nodes)
+    # merge nodes with the same term and dates as they need to be unique
+    values = _merge_same_dates(nodes)
+    grouped_nodes = _group_nodes_by_value_and_properties(values)
     return flatten(map(_condense_nodes, grouped_nodes.values()))
 
 
@@ -1446,13 +1499,13 @@ def most_relevant_blank_node_by_id(nodes: list, term_id: str, date: str):
 PROPERTY_UNITS_CONVERSIONS = {
     Units.KG.value: {
         Units.MJ.value: [
-            'energyContentHigherHeatingValue',  # "kg" to "mj"
+            'energyContentLowerHeatingValue',  # "kg" to "mj"
         ]
     },
     Units.M3.value: {
         Units.MJ.value: [
             'density',  # "m3" to "kg"
-            'energyContentHigherHeatingValue',  # "kg" to "mj"
+            'energyContentLowerHeatingValue',  # "kg" to "mj"
         ]
     }
 }
@@ -1468,16 +1521,20 @@ def _convert_via_property(node: dict, node_value: Union[int, float], property_fi
 
     Parameters
     ----------
-    node: a dict containing a term
-    node_value: value to be converted as float or int
-    property_field: str such as "density"
+    node: dict
+        Blank node containing a term
+    node_value: int | float
+        Value to be converted as float or int
+    property_field: str
+        E.g., "density"
 
-    Returns float or None
+    Returns
     -------
+        Float or None
     """
-    node_property = find_term_property(node, property_field, default={}, keep_in_memory=True)
-    node_property_value = safe_parse_float(node_property.get("value", 0))
-
+    node_property_value = get_node_property_value(
+        model=None, node=node, prop_id=property_field, default=0, handle_percents=False
+    )
     return node_value * node_property_value if node_value is not None and bool(node_property_value) else None
 
 
@@ -1486,7 +1543,7 @@ def convert_unit(node, dest_unit: Units, node_value: Union[int, float] = None) -
     Convert a number `value` inside a node or a optional `node_value` belonging to a term `node`, to unit `dest_unit`
     using the ATOMIC_WEIGHT_CONVERSIONS map or failing that, the PROPERTY_UNITS_CONVERSIONS map and lookups
     """
-    src_unit = node.get("units", "")
+    src_unit = node.get("units") or node.get('term', {}).get('units', "")
 
     node_value = _node_value(node) if node_value is None else node_value
 
@@ -1505,7 +1562,7 @@ def convert_unit_properties(node_value: Union[int, float], node: dict, dest_unit
     Uses cached calls to download_hestia() internally for speedup
     Returns None if no conversion possible.
     """
-    src_unit = node.get('units', '')
+    src_unit = node.get("units") or node.get('term', {}).get('units', "")
     conversions = PROPERTY_UNITS_CONVERSIONS.get(src_unit, {}).get(dest_unit.value, [])
     return reduce(
         lambda value, conversion_property_field: _convert_via_property(node, value, conversion_property_field),

hestia_earth/models/utils/ecoClimateZone.py

@@ -36,8 +36,8 @@ def get_eco_climate_zone_value(node: dict, as_enum: bool = False) -> Union[int,
     Parameters
     ----------
     node : dict
-        A HESTIA [Site](https://www-staging.hestia.earth/schema/Site) or
-        [Cycle](https://www-staging.hestia.earth/schema/Cycle).
+        A HESTIA [Site](https://hestia.earth/schema/Site) or
+        [Cycle](https://hestia.earth/schema/Cycle).
 
     Returns
     -------

hestia_earth/models/utils/impact_assessment.py

@@ -131,6 +131,7 @@ def impact_country_value(
     lookup: str,
     group_key: str = None,
     country_fallback: bool = False,
+    default_no_values=None
 ) -> float:
     """
     Calculate the value of the impact based on lookup factors and `site.country.@id`.
@@ -166,13 +167,13 @@ def impact_country_value(
     debugValues(impact, model=model, term=term_id,
                 values_used=log_as_table(values))
 
-    all_with_factors = all([v.get('coefficient') is not None for v in values if v.get('value') is not None])
+    has_values = len(values) > 0
+    missing_values = set([v.get('id') for v in values if v.get('value') and v.get('coefficient') is None])
+    all_with_factors = not missing_values
     values = [float((v.get('value') or 0) * (v.get('coefficient') or 0)) for v in values]
 
     # fail if some factors are missing
-    return None if not all_with_factors else (
-        list_sum(values) if len(values) > 0 else None
-    )
+    return None if not all_with_factors else (list_sum(values) if has_values else default_no_values)
 
 
 def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, group_key: str = None) -> float:

hestia_earth/models/utils/lookup.py

@@ -49,8 +49,9 @@ def all_factor_value(
 ):
     values = list(map(_factor_value(model, term_id, lookup_name, lookup_col, grouped_key), blank_nodes))
 
-    missing_values = set([v.get('id') for v in values if v.get('value') is not None and v.get('coefficient') is None])
-    all_with_factors = all([v.get('coefficient') is not None for v in values if v.get('value') is not None])
+    has_values = len(values) > 0
+    missing_values = set([v.get('id') for v in values if v.get('value') and v.get('coefficient') is None])
+    all_with_factors = not missing_values
 
     for missing_value in missing_values:
         debugMissingLookup(lookup_name, 'termid', missing_value, lookup_col, None, model=model, term=term_id)
@@ -58,14 +59,13 @@ def all_factor_value(
     debugValues(node, model=model, term=term_id,
                 all_with_factors=all_with_factors,
                 missing_lookup_factor=';'.join(missing_values),
+                has_values=has_values,
                 values_used=log_as_table(values))
 
     values = [float((v.get('value') or 0) * (v.get('coefficient') or 0)) for v in values]
 
     # fail if some factors are missing
-    return None if not all_with_factors else (
-        list_sum(values) if len(values) > 0 else default_no_values
-    )
+    return None if not all_with_factors else (list_sum(values) if has_values else default_no_values)
 
 
 def _term_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):

hestia_earth/models/utils/property.py

@@ -57,7 +57,8 @@ def find_term_property(term, property: str, default=None, keep_in_memory=False)
     return find_term_match(props, property, default)
 
 
-def get_node_property(node: dict, property: str, find_default_property: bool = True):
+def get_node_property(node: dict, property: str, find_default_property: bool = True,
+                      keep_in_memory: bool = False) -> dict:
     """
     Get the property by `@id` linked to the Blank Node in the glossary.
 
@@ -73,6 +74,8 @@ def get_node_property(node: dict, property: str, find_default_property: bool = T
         The `term.@id` of the property. Example: `nitrogenContent`.
     find_default_property : bool
         Default to fetching the property from the `defaultProperties` of the `Term`.
+    keep_in_memory:
+        If True and find_default_property is True, will cache this term_id call to api
 
     Returns
     -------
@@ -80,7 +83,7 @@ def get_node_property(node: dict, property: str, find_default_property: bool = T
         The property if found, `None` otherwise.
     """
     prop = find_term_match(node.get('properties', []), property, None)
-    return find_term_property(node.get('term', {}), property, {}) if all([
+    return find_term_property(node.get('term', {}), property, {}, keep_in_memory) if all([
         find_default_property,
         prop is None
     ]) else (prop or {})

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.66.0'
+VERSION = '0.67.1'

hestia_earth/orchestrator/log.py

@@ -1,5 +1,7 @@
 import os
 import sys
+import platform
+import resource
 import logging
 
 LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
@@ -42,6 +44,15 @@ if LOG_FILENAME is not None:
 def _join_args(**kwargs): return ', '.join([f"{key}={value}" for key, value in kwargs.items()])
 
 
+def log_memory_usage(**kwargs):
+    factor = 1024 * (
+        1024 if platform.system() in ['Darwin', 'Windows'] else 1
+    )
+    value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / factor
+    extra = (', ' + _join_args(**kwargs)) if len(kwargs.keys()) > 0 else ''
+    logger.info('memory used=%s, unit=MB' + extra, value)
+
+
 def _log_node_suffix(node: dict = {}):
     node_type = node.get('@type', node.get('type')) if node else None
     node_id = node.get('@id', node.get('id', node.get('term', {}).get('@id'))) if node else None

hestia_earth/orchestrator/models/__init__.py

@@ -7,7 +7,7 @@ from copy import deepcopy
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.version import VERSION
-from ..log import logger
+from ..log import logger, log_memory_usage
 from ..utils import get_required_model_param, _snakecase
 from ..strategies.run import should_run
 from ..strategies.merge import merge
@@ -76,10 +76,15 @@ def _run_post_checks(data: dict):
 
 
 def _run_model(data: dict, model: dict, all_models: list):
-    module = _import_model(get_required_model_param(model, 'model'))
-    # if no value is provided, use all the models but this one
+    model_id = get_required_model_param(model, 'model')
     model_value = model.get('value') or _list_except_item(all_models, model)
+    log_memory_usage(model_model=model_id, model_value=model_value, step='before')
+
+    module = _import_model(model_id.replace('-', '_'))
+    # if no value is provided, use all the models but this one
     result = module.get('run')(model_value, data)
+
+    log_memory_usage(model_model=model_id, model_value=model_value, step='after')
     return {'data': data, 'model': model, 'version': module.get('version'), 'result': result}
 
 

{hestia_earth_models-0.66.0.dist-info → hestia_earth_models-0.67.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.66.0
+Version: 0.67.1
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team