hestia-earth-models 0.74.10__py3-none-any.whl → 0.74.12__py3-none-any.whl

hestia_earth/models/utils/emission.py

@@ -1,11 +1,11 @@
 from collections.abc import Iterable
-from typing import Optional, Union
+from typing import Optional, Union, List
 from hestia_earth.schema import EmissionMethodTier, SchemaType, TermTermType
 from hestia_earth.utils.model import linked_node
 from hestia_earth.utils.emission import cycle_emissions_in_system_boundary, emissions_in_system_boundary
 from hestia_earth.utils.term import download_term
 
-from . import flatten_args
+from . import flatten_args, set_node_value, set_node_stats, set_node_term
 from .blank_node import find_terms_value
 from .method import include_methodModel
 from .constant import Units, get_atomic_conversion
@@ -13,14 +13,28 @@ from .constant import Units, get_atomic_conversion
 EMISSION_METHOD_TIERS = [e.value for e in EmissionMethodTier]
 
 
-def _new_emission(term, model=None, country_id: str = None, key_id: str = None):
-    node = {'@type': SchemaType.EMISSION.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term, TermTermType.EMISSION))
-    if country_id:
-        node['country'] = linked_node(download_term(country_id, TermTermType.REGION))
-    if key_id:
-        node['key'] = linked_node(download_term(key_id))
-    return include_methodModel(node, model)
+def _new_emission(
+    term: Union[dict, str],
+    value: List[Optional[float]] = None,
+    sd: float = None,
+    min: float = None,
+    max: float = None,
+    model: Optional[Union[dict, str]] = None,
+    country_id: str = None,
+    key_id: str = None
+):
+    return set_node_stats(
+        include_methodModel({
+            '@type': SchemaType.EMISSION.value,
+            'term': linked_node(term if isinstance(term, dict) else download_term(term))
+        }, model) |
+        set_node_value('value', value, is_list=True) |
+        (set_node_value('sd', sd, is_list=True) if sd is not None else {}) |
+        set_node_value('min', min, is_list=True) |
+        set_node_value('max', max, is_list=True) |
+        set_node_term('country', country_id, TermTermType.REGION) |
+        set_node_term('key', key_id)
+    )
 
 
 def get_nh3_no3_nox_to_n(cycle: dict, nh3_term_id: str = None, no3_term_id: str = None, nox_term_id: str = None):

hestia_earth/models/utils/indicator.py

@@ -1,22 +1,37 @@
+from typing import Union, Optional
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.model import linked_node
 from hestia_earth.utils.term import download_term
 
+from . import set_node_value, set_node_term, set_node_stats
 from .method import include_methodModel
 
 
 def _new_indicator(
-    term: dict, model=None,
-    land_cover_id: str = None, previous_land_cover_id: str = None, country_id: str = None, key_id: str = None
+    term: Union[dict, str],
+    value: float,
+    sd: float = None,
+    min: float = None,
+    max: float = None,
+    inputs: list = None,
+    model: Optional[Union[dict, str]] = None,
+    land_cover_id: str = None,
+    previous_land_cover_id: str = None,
+    country_id: str = None,
+    key_id: str = None
 ):
-    node = {'@type': SchemaType.INDICATOR.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
-    if land_cover_id:
-        node['landCover'] = linked_node(download_term(land_cover_id, TermTermType.LANDCOVER))
-    if previous_land_cover_id:
-        node['previousLandCover'] = linked_node(download_term(previous_land_cover_id, TermTermType.LANDCOVER))
-    if country_id:
-        node['country'] = linked_node(download_term(country_id, TermTermType.REGION))
-    if key_id:
-        node['key'] = linked_node(download_term(key_id))
-    return include_methodModel(node, model)
+    return set_node_stats(
+        include_methodModel({
+            '@type': SchemaType.INDICATOR.value,
+            'term': linked_node(term if isinstance(term, dict) else download_term(term))
+        }, model) |
+        set_node_value('value', value) |
+        set_node_value('sd', sd) |
+        set_node_value('min', min) |
+        set_node_value('max', max) |
+        (set_node_value('inputs', inputs) if inputs else {}) |
+        set_node_term('landCover', land_cover_id, TermTermType.LANDCOVER) |
+        set_node_term('previousLandCover', previous_land_cover_id, TermTermType.LANDCOVER) |
+        set_node_term('country', country_id, TermTermType.REGION) |
+        set_node_term('key', key_id)
+    ) if value is not None else None  # cannot return an indicator without a value

hestia_earth/models/utils/input.py

@@ -1,3 +1,4 @@
+from typing import Union, Optional, List
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.model import find_term_match, linked_node, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, non_empty_list, list_average, flatten
@@ -5,16 +6,32 @@ from hestia_earth.utils.lookup import download_lookup, get_table_value, column_n
 from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logger
-from . import _filter_list_term_unit, _load_calculated_node, group_by
+from . import (
+    _filter_list_term_unit, _load_calculated_node, group_by, set_node_value, set_node_stats
+)
 from .constant import Units
 from .blank_node import get_total_value, get_total_value_converted, get_lookup_value
 from .method import include_model
 
 
-def _new_input(term, model=None):
-    node = {'@type': SchemaType.INPUT.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
-    return include_model(node, model)
+def _new_input(
+    term: Union[dict, str],
+    value: List[Optional[float]] = None,
+    sd: float = None,
+    min: float = None,
+    max: float = None,
+    model: Optional[Union[dict, str]] = None,
+):
+    return set_node_stats(
+        include_model({
+            '@type': SchemaType.INPUT.value,
+            'term': linked_node(term if isinstance(term, dict) else download_term(term))
+        }, model) |
+        set_node_value('value', value, is_list=True) |
+        (set_node_value('sd', sd, is_list=True) if sd is not None else {}) |
+        set_node_value('min', min, is_list=True) |
+        set_node_value('max', max, is_list=True)
+    )
 
 
 def load_impacts(inputs: list):

hestia_earth/models/utils/management.py

@@ -1,11 +1,25 @@
+from typing import Union, Optional, List
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.model import linked_node
 from hestia_earth.utils.term import download_term
 
+from . import set_node_value
 from .method import include_model
 
 
-def _new_management(term, model=None):
-    node = {'@type': SchemaType.MANAGEMENT.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
-    return include_model(node, model)
+def _new_management(
+    term: Union[dict, str],
+    value: Optional[Union[float, List[float]]] = None,
+    model: Optional[Union[dict, str]] = None,
+    end_date: str = None,
+    start_date: str = None
+):
+    return (
+        include_model({
+            '@type': SchemaType.MANAGEMENT.value,
+            'term': linked_node(term if isinstance(term, dict) else download_term(term))
+        }, model) |
+        set_node_value('value', value) |
+        set_node_value('endDate', end_date) |
+        set_node_value('startDate', start_date)
+    )

hestia_earth/models/utils/measurement.py

@@ -2,14 +2,14 @@ from collections import defaultdict
 from collections.abc import Iterable
 from functools import reduce
 from statistics import mode, mean
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, List
 from hestia_earth.schema import MeasurementMethodClassification, SchemaType, TermTermType
 from hestia_earth.utils.model import linked_node, filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list, flatten, safe_parse_float, list_sum
 from hestia_earth.utils.date import diff_in_days
 from hestia_earth.utils.term import download_term
 
-from . import flatten_args
+from . import flatten_args, set_node_value
 from .blank_node import most_relevant_blank_node_by_id
 from .method import include_method
 from .term import get_lookup_value
@@ -27,10 +27,18 @@ MEASUREMENT_REDUCE = {
 MEASUREMENT_METHOD_CLASSIFICATIONS = [e.value for e in MeasurementMethodClassification]
 
 
-def _new_measurement(term, method=None):
-    node = {'@type': SchemaType.MEASUREMENT.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
-    return include_method(node, term_id=method)
+def _new_measurement(
+    term: Union[dict, str],
+    value: List[Union[float, bool]] = None,
+    model: Optional[Union[dict, str]] = None
+):
+    return (
+        include_method({
+            '@type': SchemaType.MEASUREMENT.value,
+            'term': linked_node(term if isinstance(term, dict) else download_term(term))
+        }, model) |
+        set_node_value('value', value, is_list=True)
+    )
 
 
 def measurement_value(measurement: dict, is_larger_unit: bool = False, default=None) -> float:

hestia_earth/models/utils/practice.py

@@ -1,11 +1,31 @@
+from typing import Union, Optional, List
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.model import linked_node
 from hestia_earth.utils.term import download_term
 
+from . import set_node_value, set_node_stats
 from .method import include_model
 
 
-def _new_practice(term, model=None):
-    node = {'@type': SchemaType.PRACTICE.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
-    return include_model(node, model)
+def _new_practice(
+    term: Union[dict, str],
+    value: Optional[Union[float, List[float]]] = None,
+    sd: float = None,
+    min: float = None,
+    max: float = None,
+    model: Optional[Union[dict, str]] = None,
+    end_date: str = None,
+    start_date: str = None
+):
+    return set_node_stats(
+        include_model({
+            '@type': SchemaType.PRACTICE.value,
+            'term': linked_node(term if isinstance(term, dict) else download_term(term))
+        }, model) |
+        set_node_value('value', value, is_list=True) |
+        (set_node_value('sd', sd, is_list=True) if sd is not None else {}) |
+        set_node_value('min', min, is_list=True) |
+        set_node_value('max', max, is_list=True) |
+        set_node_value('endDate', end_date) |
+        set_node_value('startDate', start_date)
+    )

hestia_earth/models/utils/product.py

@@ -1,8 +1,10 @@
+from typing import Union, Optional, List
 from hestia_earth.schema import SchemaType, TermTermType, UNIQUENESS_FIELDS
 from hestia_earth.utils.model import filter_list_term_type, find_term_match, linked_node
 from hestia_earth.utils.tools import flatten, list_sum, non_empty_list, get_dict_key
 from hestia_earth.utils.term import download_term
 
+from . import set_node_value, set_node_stats
 from .blank_node import get_total_value, get_total_value_converted
 from .constant import Units
 from .currency import DEFAULT_CURRENCY
@@ -11,16 +13,28 @@ from .term import get_rice_paddy_terms
 from .method import include_model
 
 
-def _new_product(term, value: float = None, model=None):
-    node = {'@type': SchemaType.PRODUCT.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
-    if value is not None:
-        node['value'] = [value]
-    elif value == 0:
-        node['economicValueShare'] = 0
-        node['revenue'] = 0
-        node['currency'] = DEFAULT_CURRENCY
-    return include_model(node, model)
+def _new_product(
+    term: Union[dict, str],
+    value: Optional[Union[float, List[float]]] = None,
+    sd: float = None,
+    min: float = None,
+    max: float = None,
+    model: Optional[Union[dict, str]] = None,
+):
+    return set_node_stats(
+        include_model({
+            '@type': SchemaType.PRODUCT.value,
+            'term': linked_node(term if isinstance(term, dict) else download_term(term))
+        }, model) |
+        set_node_value('value', value, is_list=True) |
+        (set_node_value('sd', sd, is_list=True) if sd is not None else {}) |
+        set_node_value('min', min, is_list=True) |
+        set_node_value('max', max, is_list=True)
+    ) | ({
+        'economicValueShare': 0,
+        'revenue': 0,
+        'currency': DEFAULT_CURRENCY
+    } if value == 0 else {})
 
 
 def _match_list_el(source: list, dest: list, key: str):
@@ -320,16 +334,20 @@ PRODUCT_UNITS_CONVERSIONS = {
 }
 
 
-def convert_product_to_unit(product: dict, dest_unit: Units):
+def convert_product_to_unit(product: dict, dest_unit: Units, **log_args):
     from_units = product.get('term', {}).get('units')
     to_units = dest_unit if isinstance(dest_unit, str) else dest_unit.value
     conversions = PRODUCT_UNITS_CONVERSIONS.get(from_units, {}).get(to_units)
     return None if len(product.get('value', [])) == 0 else 0 if conversions is None else list_sum(
         flatten([
-            get_total_value_converted([product], properties, multiply) for properties, multiply in conversions
+            get_total_value_converted([product], properties, multiply, **log_args)
+            for properties, multiply in conversions
         ]) if len(conversions) > 0 else get_total_value([product])
     )
 
 
-def liveweight_produced(products: list):
-    return list_sum([convert_product_to_unit(p, Units.KG_LIVEWEIGHT) for p in products])
+def liveweight_produced(products: list, **log_args):
+    return list_sum([
+        convert_product_to_unit(product=p, dest_unit=Units.KG_LIVEWEIGHT, **log_args)
+        for p in products
+    ])

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.74.10'
+VERSION = '0.74.12'

hestia_earth/models/webbEtAl2012AndSintermannEtAl2012/nh3ToAirOrganicFertiliser.py

@@ -59,8 +59,7 @@ NH3_TAN_FACTOR = {
 
 
 def _emission(value: float):
-    emission = _new_emission(TERM_ID, MODEL)
-    emission['value'] = [value]
+    emission = _new_emission(term=TERM_ID, model=MODEL, value=value)
     emission['methodTier'] = TIER
     return emission
 

hestia_earth/orchestrator/models/__init__.py

@@ -1,7 +1,6 @@
 import os
-import resource
-import platform
-from typing import Union, List
+import sys
+from typing import Union, List, Optional
 import importlib
 from functools import reduce
 import concurrent.futures
@@ -16,15 +15,27 @@ from hestia_earth.orchestrator.strategies.merge import merge
 
 
 def _memory_usage():
-    factor = 1024 * (
-        1024 if platform.system() in ['Darwin', 'Windows'] else 1
-    )
-    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / factor
+    try:
+        factor = 1024 * (
+            1024 if sys.platform in ['darwin', 'win32'] else 1
+        )
+        if sys.platform == "win32":
+            import psutil
+            return psutil.Process(os.getpid()).memory_info().peak_wset / factor
+        else:
+            # Fallback to resource library from psutil
+            # https://psutil.readthedocs.io/en/latest/#psutil.Process.memory_info
+            import resource
+            return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / factor
+    except Exception:
+        # skip resource cannot be computed
+        return 0
 
 
 def _max_workers(type: str):
     try:
-        return int(os.getenv(f"MAX_WORKERS_{type.upper()}"))
+        max_workers_str = os.getenv(f"MAX_WORKERS_{type.upper()}")
+        return int(max_workers_str) if max_workers_str is not None else None
     except Exception:
         return None
 
@@ -51,7 +62,7 @@ def _list_except_item(list, item):
     return list[:idx] + list[idx+1:]
 
 
-def _filter_models_stage(models: list, stage: Union[int, List[int]] = None):
+def _filter_models_stage(models: list, stage: Optional[Union[int, List[int]]] = None):
     stages = stage if isinstance(stage, list) else [stage] if stage is not None else None
     return models if stage is None else non_empty_list([
         (_filter_models_stage(m, stage) if isinstance(m, list) else m) for m in models if (
@@ -140,7 +151,7 @@ def _run(data: dict, model: dict, all_models: list):
     return _run_model(data, model, all_models) if should_run(data, model) else None
 
 
-def _run_serie(data: dict, models: list, stage: Union[int, List[int]] = None):
+def _run_serie(data: dict, models: list, stage: Optional[Union[int, List[int]]] = None):
     return reduce(
         lambda prev, m: merge(
             prev, _run_parallel(prev, m, models) if isinstance(m, list) else [_run(_node_copy(prev), m, models)]
@@ -163,7 +174,7 @@ def _run_parallel(data: dict, model: list, all_models: list):
     return results
 
 
-def run(data: dict, models: list, stage: Union[int, List[int]] = None):
+def run(data: dict, models: list, stage: Optional[Union[int, List[int]]] = None):
     # make sure we reset before recalculating the node
     reset_index()
     # run pre-checks if exist

{hestia_earth_models-0.74.10.dist-info → hestia_earth_models-0.74.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.74.10
+Version: 0.74.12
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hestia-earth-schema<34.0.0,>=33.5.0
-Requires-Dist: hestia-earth-utils>=0.15.4
+Requires-Dist: hestia-earth-utils>=0.15.12
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0
 Requires-Dist: pydash