hestia-earth-models 0.67.1__py3-none-any.whl → 0.68.0__py3-none-any.whl

hestia_earth/models/utils/measurement.py

@@ -3,16 +3,15 @@ from collections.abc import Iterable
 from functools import reduce
 from statistics import mode, mean
 from typing import Any, Optional, Union
-
 from hestia_earth.schema import MeasurementMethodClassification, SchemaType
-from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import linked_node
 from hestia_earth.utils.tools import non_empty_list, flatten, safe_parse_float
 from hestia_earth.utils.date import diff_in_days
 
-from hestia_earth.models.utils.blank_node import most_relevant_blank_node_by_id
-from . import _term_id, _include_method, flatten_args
-from .term import get_lookup_value
+from . import flatten_args
+from .blank_node import most_relevant_blank_node_by_id
+from .method import include_methodModel
+from .term import download_term, get_lookup_value
 
 
 # TODO: verify those values
@@ -29,8 +28,8 @@ MEASUREMENT_METHOD_CLASSIFICATIONS = [e.value for e in MeasurementMethodClassifi
 
 def _new_measurement(term, model=None):
     node = {'@type': SchemaType.MEASUREMENT.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
-    return _include_method(node, term_id=model)
+    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
+    return include_methodModel(node, term_id=model)
 
 
 def measurement_value(measurement: dict, is_larger_unit: bool = False) -> float:

hestia_earth/models/utils/method.py

@@ -0,0 +1,20 @@
+from typing import Union
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import linked_node
+
+from .term import download_term
+
+
+def include_method(node: dict, term_id: Union[None, str, dict], key='method'):
+    term = (
+        download_term(term_id, TermTermType.MODEL) or download_term(term_id) or {}
+    ) if isinstance(term_id, str) else term_id
+    return node | ({} if term is None or term.get('@id') is None else {key: linked_node(term)})
+
+
+def include_model(node: dict, term_id: Union[None, str, dict]):
+    return include_method(node, term_id=term_id, key='model')
+
+
+def include_methodModel(node: dict, term_id: Union[None, str, dict]):
+    return include_method(node, term_id=term_id, key='methodModel')

hestia_earth/models/utils/practice.py

@@ -1,15 +1,14 @@
 from hestia_earth.schema import SchemaType
-from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import linked_node
 
-from . import _term_id, _include_model
-from .term import get_lookup_value
+from .term import get_lookup_value, download_term
+from .method import include_model
 
 
 def _new_practice(term, model=None):
     node = {'@type': SchemaType.PRACTICE.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
-    return _include_model(node, model)
+    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
+    return include_model(node, model)
 
 
 def is_model_enabled(model: str, term_id: str, practice: dict = None):

hestia_earth/models/utils/product.py

@@ -1,26 +1,25 @@
 from hestia_earth.schema import SchemaType, TermTermType, UNIQUENESS_FIELDS
-from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import filter_list_term_type, find_term_match, linked_node
 from hestia_earth.utils.tools import flatten, list_sum, non_empty_list, get_dict_key
 
-from . import _term_id, _include_model
 from .blank_node import get_total_value, get_total_value_converted
 from .constant import Units
 from .currency import DEFAULT_CURRENCY
 from .property import _get_nitrogen_content, get_node_property_value
-from .term import get_rice_paddy_terms
+from .term import get_rice_paddy_terms, download_term
+from .method import include_model
 
 
 def _new_product(term, value: float = None, model=None):
     node = {'@type': SchemaType.PRODUCT.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
+    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
     if value is not None:
         node['value'] = [value]
     elif value == 0:
         node['economicValueShare'] = 0
         node['revenue'] = 0
         node['currency'] = DEFAULT_CURRENCY
-    return _include_model(node, model)
+    return include_model(node, model)
 
 
 def _match_list_el(source: list, dest: list, key: str):

hestia_earth/models/utils/property.py

@@ -1,22 +1,17 @@
-from functools import cache
-
 from hestia_earth.schema import SchemaType, TermTermType
-from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.lookup import download_lookup, extract_grouped_data, get_table_value, column_name
 from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.tools import list_sum, safe_parse_float
 
-from . import _term_id, _include_methodModel
-from .term import get_lookup_value
-from ..log import debugMissingLookup
-
-download_hestia_cached = cache(download_hestia)
+from hestia_earth.models.log import debugMissingLookup
+from .method import include_methodModel
+from .term import download_term, get_lookup_value
 
 
 def _new_property(term, model=None):
     node = {'@type': SchemaType.PROPERTY.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
-    return _include_methodModel(node, model)
+    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term, TermTermType.PROPERTY))
+    return include_methodModel(node, model)
 
 
 def merge_properties(properties: list, new_properties: list):
@@ -30,7 +25,7 @@ def get_property_lookup_value(model: str, term_id: str, column: str):
     return get_lookup_value(term, column, model=model, term=term_id)
 
 
-def find_term_property(term, property: str, default=None, keep_in_memory=False) -> dict:
+def find_term_property(term, property: str, default=None) -> dict:
     """
     Get the property by `@id` linked to the `Term` in the glossary.
 
@@ -42,23 +37,20 @@ def find_term_property(term, property: str, default=None, keep_in_memory=False)
         The `term.@id` of the property. Example: `nitrogenContent`.
     default : Any
         The default value if the property is not found. Defaults to `None`.
-    keep_in_memory: bool
-        Should we cache results from download_hestia(). Default False
 
     Returns
     -------
     dict
         The property if found, `default` otherwise.
     """
-    download_func = download_hestia_cached if keep_in_memory else download_hestia
     props = term.get('defaultProperties', []) if isinstance(term, dict) else []
-    term_id = _term_id(term)
-    props = (download_func(term_id) or {}).get('defaultProperties', []) if len(props) == 0 and term_id else props
+    props = (
+        download_term(term, TermTermType.PROPERTY) or {}
+    ).get('defaultProperties', []) if len(props) == 0 and term else props
     return find_term_match(props, property, default)
 
 
-def get_node_property(node: dict, property: str, find_default_property: bool = True,
-                      keep_in_memory: bool = False) -> dict:
+def get_node_property(node: dict, property: str, find_default_property: bool = True) -> dict:
     """
     Get the property by `@id` linked to the Blank Node in the glossary.
 
@@ -74,8 +66,6 @@ def get_node_property(node: dict, property: str, find_default_property: bool = T
         The `term.@id` of the property. Example: `nitrogenContent`.
     find_default_property : bool
         Default to fetching the property from the `defaultProperties` of the `Term`.
-    keep_in_memory:
-        If True and find_default_property is True, will cache this term_id call to api
 
     Returns
     -------
@@ -83,7 +73,7 @@ def get_node_property(node: dict, property: str, find_default_property: bool = T
         The property if found, `None` otherwise.
     """
     prop = find_term_match(node.get('properties', []), property, None)
-    return find_term_property(node.get('term', {}), property, {}, keep_in_memory) if all([
+    return find_term_property(node.get('term', {}), property, {}) if all([
         find_default_property,
         prop is None
     ]) else (prop or {})
@@ -115,7 +105,7 @@ def node_property_lookup_value(model: str, node_term: dict, prop_id: str, defaul
 
 def get_node_property_value(model: str, node: dict, prop_id: str, default=None, handle_percents=True, **log_args):
     prop = get_node_property(node, prop_id)
-    term = (prop or {}).get('term', download_hestia(prop_id))
+    term = (prop or {}).get('term') or download_term(prop_id, TermTermType.PROPERTY)
     units = (term or {}).get('units')
     value = prop.get('value') if prop else node_property_lookup_value(model, node.get('term', {}), prop_id, **log_args)
     return default if value is None else (value / 100 if units == '%' and handle_percents else value)

hestia_earth/models/utils/site.py

@@ -1,11 +1,12 @@
+from typing import Optional
+
 from hestia_earth.schema import SchemaType, SiteSiteType, TermTermType
 from hestia_earth.utils.api import find_related
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 from hestia_earth.utils.tools import non_empty_list, flatten, safe_parse_date
 
-from hestia_earth.models.log import debugMissingLookup
 from . import cached_value, _load_calculated_node
 from .term import get_land_cover_siteTypes
+from .lookup import get_region_lookup_value
 
 CACHE_YEARS_KEY = 'years'
 WATER_TYPES = [
@@ -67,7 +68,7 @@ def years_from_cycles(cycles: list):
     ] for cycle in cycles))))
 
 
-def related_cycles(site: dict):
+def related_cycles(site: dict, cycles_mapping: Optional[dict[str, dict]] = None):
     """
     Get the list of `Cycle` related to the `Site`.
     Gets the `recalculated` data if available, else `original`.
@@ -76,15 +77,23 @@ def related_cycles(site: dict):
     ----------
     site_id : str
         The `@id` of the `Site`.
+    cycles_mapping : dict[str, dict], optional
+        An optional dict of related `Cycle`s for which the data has already been retrieved, with the format
+        `{cycle_id (str): cycle (dict), ...ids}`.
 
     Returns
     -------
     list[dict]
         The related `Cycle`s as `dict`.
     """
+    cycles_mapping = cycles_mapping or {}
+
     cached_nodes = [n for n in cached_value(site, 'related', []) if n.get('@type') == SchemaType.CYCLE.value]
     related_nodes = cached_nodes or find_related(SchemaType.SITE, site.get('@id'), SchemaType.CYCLE) or []
-    return non_empty_list(map(lambda node: _load_calculated_node(node, SchemaType.CYCLE), related_nodes))
+    return non_empty_list(map(
+        lambda node: cycles_mapping.get(node['@id']) or _load_calculated_node(node, SchemaType.CYCLE),
+        related_nodes
+    ))
 
 
 def related_years(site: dict):
@@ -117,10 +126,7 @@ def valid_site_type(site: dict, site_types=[SiteSiteType.CROPLAND.value, SiteSit
 
 
 def region_factor(model: str, region_id: str, term_id: str, termType: TermTermType):
-    lookup_name = f"region-{termType.value}.csv"
-    value = get_table_value(download_lookup(lookup_name), 'termid', region_id, column_name(term_id))
-    debugMissingLookup(lookup_name, 'termid', region_id, term_id, value, model=model, term=term_id)
-    return value
+    return get_region_lookup_value(f"region-{termType.value}.csv", region_id, term_id, model=model, term=term_id)
 
 
 def get_land_cover_term_id(site_type: str):

hestia_earth/models/utils/term.py

@@ -1,5 +1,9 @@
+from typing import Union
+from functools import lru_cache
+import json
 from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType
-from hestia_earth.utils.api import find_node, search
+from hestia_earth.utils.storage import _load_from_storage
+from hestia_earth.utils.api import find_node, search, download_hestia
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 
 from .constant import Units
@@ -20,6 +24,28 @@ def get_lookup_value(lookup_term: dict, column: str, skip_debug: bool = False, *
     return value
 
 
+@lru_cache()
+def _load_term_file(term_type: str):
+    try:
+        filepath = f"glossary/{term_type}.json"
+        nodes = json.loads(_load_from_storage(filepath, glossary=True))
+        return {node.get('@id'): node for node in nodes}
+    except Exception as e:
+        print(e)
+        return {}
+
+
+def download_term(term: Union[str, dict], termType: Union[str, TermTermType] = None):
+    term_id = term.get('@id', term.get('id')) if isinstance(term, dict) else term
+    term_type = (
+        termType if isinstance(termType, str) else termType.value
+    ) if termType else (
+        term.get('termType') if isinstance(term, dict) else None
+    )
+    cached_nodes = _load_term_file(term_type) if term_type else {}
+    return cached_nodes.get(term_id) or download_hestia(term_id)
+
+
 def get_liquid_fuel_terms():
     """
     Find all "liquid" `fuel` terms from the Glossary:

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.67.1'
+VERSION = '0.68.0'

hestia_earth/orchestrator/log.py

@@ -1,7 +1,5 @@
 import os
 import sys
-import platform
-import resource
 import logging
 
 LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
@@ -44,15 +42,6 @@ if LOG_FILENAME is not None:
 def _join_args(**kwargs): return ', '.join([f"{key}={value}" for key, value in kwargs.items()])
 
 
-def log_memory_usage(**kwargs):
-    factor = 1024 * (
-        1024 if platform.system() in ['Darwin', 'Windows'] else 1
-    )
-    value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / factor
-    extra = (', ' + _join_args(**kwargs)) if len(kwargs.keys()) > 0 else ''
-    logger.info('memory used=%s, unit=MB' + extra, value)
-
-
 def _log_node_suffix(node: dict = {}):
     node_type = node.get('@type', node.get('type')) if node else None
     node_id = node.get('@id', node.get('id', node.get('term', {}).get('@id'))) if node else None

hestia_earth/orchestrator/models/__init__.py

@@ -1,18 +1,27 @@
 import os
+import resource
+import platform
 from typing import Union, List
 import importlib
 from functools import reduce
 import concurrent.futures
 from copy import deepcopy
-from hestia_earth.utils.tools import non_empty_list
+from hestia_earth.utils.tools import non_empty_list, current_time_ms
 
 from hestia_earth.models.version import VERSION
-from ..log import logger, log_memory_usage
+from ..log import logger
 from ..utils import get_required_model_param, _snakecase
 from ..strategies.run import should_run
 from ..strategies.merge import merge
 
 
+def _memory_usage():
+    factor = 1024 * (
+        1024 if platform.system() in ['Darwin', 'Windows'] else 1
+    )
+    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / factor
+
+
 def _max_workers(type: str):
     try:
         return int(os.getenv(f"MAX_WORKERS_{type.upper()}"))
@@ -78,13 +87,17 @@ def _run_post_checks(data: dict):
 def _run_model(data: dict, model: dict, all_models: list):
     model_id = get_required_model_param(model, 'model')
     model_value = model.get('value') or _list_except_item(all_models, model)
-    log_memory_usage(model_model=model_id, model_value=model_value, step='before')
+
+    now = current_time_ms()
+    memory_usage = _memory_usage()
 
     module = _import_model(model_id.replace('-', '_'))
     # if no value is provided, use all the models but this one
     result = module.get('run')(model_value, data)
 
-    log_memory_usage(model_model=model_id, model_value=model_value, step='after')
+    logger.info('model_model=%s, model_value=%s, time=%s, memory_used=%s',
+                model_id, model.get('value'), current_time_ms() - now, _memory_usage() - memory_usage)
+
     return {'data': data, 'model': model, 'version': module.get('version'), 'result': result}
 
 

hestia_earth/orchestrator/strategies/run/add_blank_node_if_missing.py

@@ -1,29 +1,11 @@
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
-from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.lookup_utils import is_node_type_allowed
 
 from hestia_earth.orchestrator.log import debugValues, logShouldRun
 from hestia_earth.orchestrator.utils import get_required_model_param, find_term_match
 
-_ALLOW_ALL = 'all'
-
-
-def _lookup_values(term: dict, column: str):
-    term_id = term.get('@id')
-    term_type = term.get('termType')
-    lookup = download_lookup(f"{term_type}.csv")
-    values = get_table_value(lookup, 'termid', term_id, column_name(column))
-    return (values or _ALLOW_ALL).split(';')
-
-
-def _is_node_type_allowed(data: dict, term_id: str):
-    node_type = data.get('@type', data.get('type'))
-    term = download_hestia(term_id)
-    allowed_types = _lookup_values(term, 'typesAllowed') if term else [_ALLOW_ALL]
-    return True if _ALLOW_ALL in allowed_types or not node_type else node_type in allowed_types
-
 
 def _run_required(data: dict, model: str, term_id: str):
-    node_type_allowed = _is_node_type_allowed(data, term_id)
+    node_type_allowed = is_node_type_allowed(data, term_id)
 
     run_required = all([node_type_allowed])
     debugValues(data, model=model, term=term_id,

{hestia_earth_models-0.67.1.dist-info → hestia_earth_models-0.68.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.67.1
+Version: 0.68.0
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hestia-earth-schema==30.*
-Requires-Dist: hestia-earth-utils>=0.13.19
+Requires-Dist: hestia-earth-utils>=0.14.0
 Requires-Dist: python-dateutil>=2.8.1
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0