hestia-earth-models 0.64.9__py3-none-any.whl → 0.64.10__py3-none-any.whl

hestia_earth/models/poschEtAl2008/terrestrialAcidificationPotentialAccumulatedExceedance.py

@@ -36,8 +36,8 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_country_value(MODEL, TERM_ID, impact_assessment, f"{list(LOOKUPS.keys())[0]}.csv",
-                                 country_fallback=True, default_no_emissions=0)
+                                 country_fallback=True)
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     value=value)
-    logShouldRun(impact_assessment, MODEL, TERM_ID, True)
-    return _indicator(value)
+    logShouldRun(impact_assessment, MODEL, TERM_ID, value is not None)
+    return None if value is None else _indicator(value)

hestia_earth/models/poschEtAl2008/terrestrialEutrophicationPotentialAccumulatedExceedance.py

@@ -36,8 +36,8 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_country_value(MODEL, TERM_ID, impact_assessment, f"{list(LOOKUPS.keys())[0]}.csv",
-                                 country_fallback=True, default_no_emissions=0)
+                                 country_fallback=True)
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     value=value)
-    logShouldRun(impact_assessment, MODEL, TERM_ID, True)
-    return _indicator(value)
+    logShouldRun(impact_assessment, MODEL, TERM_ID, value is not None)
+    return None if value is None else _indicator(value)

hestia_earth/models/preload_requests.py

@@ -5,7 +5,7 @@ import json
 import os
 
 from .log import logger
-from .mocking.mock_search import create_search_results
+from .mocking.build_mock_search import create_search_results
 from .mocking import RESULTS_PATH, enable_mock as _mock
 
 

hestia_earth/models/schmidt2007/utils.py

@@ -3,14 +3,23 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.utils.completeness import _is_term_type_complete
-from hestia_earth.models.utils.lookup import factor_value
+from hestia_earth.models.utils.lookup import all_factor_value
+from . import MODEL
 
 
 def get_waste_values(term_id: str, cycle: dict, lookup_col: str):
     products = filter_list_term_type(cycle.get('products', []), TermTermType.WASTE)
-    values = non_empty_list(map(factor_value(None, term_id, f"{TermTermType.WASTE.value}.csv", lookup_col), products))
+    value = all_factor_value(
+        model=MODEL,
+        term_id=term_id,
+        node=cycle,
+        lookup_name=f"{TermTermType.WASTE.value}.csv",
+        lookup_col=lookup_col,
+        blank_nodes=products,
+        default_no_values=None
+    )
     return [0] if all([
-        len(values) == 0,
+        value is None,
         _is_term_type_complete(cycle, TermTermType.WASTE),
         cycle.get('@type', cycle.get('type')) == NodeType.CYCLE.value  # ignore adding 0 value for Transformation
-    ]) else values
+    ]) else non_empty_list([value])

hestia_earth/models/utils/blank_node.py

@@ -28,14 +28,14 @@ from hestia_earth.utils.tools import (
 )
 
 from . import is_from_model, _filter_list_term_unit, is_iterable, full_date_str
-from .constant import Units
+from .constant import Units, get_atomic_conversion
 from .lookup import (
     is_model_siteType_allowed,
     is_siteType_allowed,
     is_product_id_allowed, is_product_termType_allowed,
-    is_input_id_allowed, is_input_termType_allowed
+    is_input_id_allowed, is_input_termType_allowed, _node_value
 )
-from .property import get_node_property, get_node_property_value
+from .property import get_node_property, get_node_property_value, find_term_property
 from .term import get_lookup_value
 from ..log import debugValues, log_as_table
 
@@ -1387,3 +1387,73 @@ def most_relevant_blank_node_by_id(nodes: list, term_id: str, date: str):
     return {} if len(filtered_nodes) == 0 \
         else _shallowest_node(_most_recent_nodes(filtered_nodes, date)) \
         if date and len(filtered_nodes) > 1 else filtered_nodes[0]
+
+
+PROPERTY_UNITS_CONVERSIONS = {
+    Units.KG.value: {
+        Units.MJ.value: [
+            'energyContentHigherHeatingValue',  # "kg" to "mj"
+        ]
+    },
+    Units.M3.value: {
+        Units.MJ.value: [
+            'density',  # "m3" to "kg"
+            'energyContentHigherHeatingValue',  # "kg" to "mj"
+        ]
+    }
+}
+
+
+def _convert_via_property(node: dict, node_value: Union[int, float], property_field: str) -> Optional[float]:
+    """
+    Converts a node_value number from one unit to another using a property_field  associated
+    with a term inside term node such as "density" or 'energyContentHigherHeatingValue' or listed
+    in https://www.hestia.earth/glossary?page=1&termType=property
+
+    Will return none if the property_field is not found
+
+    Parameters
+    ----------
+    node: a dict containing a term
+    node_value: value to be converted as float or int
+    property_field: str such as "density"
+
+    Returns float or None
+    -------
+    """
+    node_property = find_term_property(node, property_field, default={}, keep_in_memory=True)
+    node_property_value = safe_parse_float(node_property.get("value", 0))
+
+    return node_value * node_property_value if node_value is not None and bool(node_property_value) else None
+
+
+def convert_unit(node, dest_unit: Units, node_value: Union[int, float] = None) -> Optional[Union[int, float]]:
+    """
+    Convert a number `value` inside a node or a optional `node_value` belonging to a term `node`, to unit `dest_unit`
+    using the ATOMIC_WEIGHT_CONVERSIONS map or failing that, the PROPERTY_UNITS_CONVERSIONS map and lookups
+    """
+    src_unit = node.get("units", "")
+
+    node_value = _node_value(node) if node_value is None else node_value
+
+    return node_value if src_unit == dest_unit.value else (
+        node_value * get_atomic_conversion(src_unit, dest_unit)
+        if get_atomic_conversion(src_unit, dest_unit, default_value=None) is not None
+        else convert_unit_properties(node_value, node, dest_unit)
+    ) if node_value else None
+
+
+def convert_unit_properties(node_value: Union[int, float], node: dict, dest_unit: Units) -> Optional[Union[int, float]]:
+    """
+    Convert a number `node_value` belonging to a term `node`, to unit `to_units` by chaining multiple unit conversions
+    together.
+    Uses terms properties for the conversion.
+    Uses cached calls to download_hestia() internally for speedup
+    Returns None if no conversion possible.
+    """
+    src_unit = node.get('units', '')
+    conversions = PROPERTY_UNITS_CONVERSIONS.get(src_unit, {}).get(dest_unit.value, [])
+    return reduce(
+        lambda value, conversion_property_field: _convert_via_property(node, value, conversion_property_field),
+        conversions, node_value
+    ) if conversions else None

hestia_earth/models/utils/constant.py

@@ -1,4 +1,5 @@
 from enum import Enum
+
 from hestia_earth.utils.tools import list_sum
 
 
@@ -35,6 +36,9 @@ class Units(Enum):
     PERCENTAGE_AREA = '% area'
     TO_C = '-C'
     TO_N = '-N'
+    KW_H = 'kWh'
+    MJ = 'MJ'
+    M3 = "m3"
 
 
 C = 12.012
@@ -93,7 +97,10 @@ ATOMIC_WEIGHT_CONVERSIONS = {
     },
     Units.KG_NH4.value: {
         Units.TO_N.value: (N + H*4) / N  # Conv_Mol_NH4N_NH4
-    }
+    },
+    Units.KW_H.value: {
+        Units.MJ.value: 3.6
+    },
 }
 
 

hestia_earth/models/utils/cycle.py

@@ -3,7 +3,7 @@ from hestia_earth.utils.model import filter_list_term_type, find_term_match, fin
 from hestia_earth.utils.tools import list_sum, safe_parse_float, safe_parse_date, non_empty_list
 
 from ..log import logRequirements, debugValues
-from .lookup import factor_value, is_siteType_allowed
+from .lookup import all_factor_value, is_siteType_allowed
 from .term import get_lookup_value
 from .property import get_node_property
 from .completeness import _is_term_type_complete
@@ -386,9 +386,7 @@ def cycle_end_year(cycle: dict):
     return date.year if date else None
 
 
-def impact_lookup_value(
-    model: str, term_id: str, blank_nodes: list, lookup_col: str, allow_missing: bool = True
-) -> float:
+def impact_lookup_value(model: str, term_id: str, cycle: dict, blank_nodes: list, lookup_col: str) -> float:
     """
     Calculate the value of the impact based on lookup factors and cycle values.
 
@@ -409,15 +407,14 @@ def impact_lookup_value(
         The impact total value.
     """
     term_type = blank_nodes[0].get('term', {}).get('termType') if len(blank_nodes) > 0 else None
-    get_factor_value = factor_value(model, term_id, f"{term_type}.csv", lookup_col)
-    factors = [(node.get('term', {}).get('@id'), get_factor_value(node)) for node in blank_nodes]
-    values = [value for (_term, value) in factors if value is not None]
-    missing_values = set([term for (term, value) in factors if value is None])
-    all_nodes_have_value = len(values) == len(factors)
-    debugValues({'type': 'Cycle'}, model=model, term=term_id,
-                missing_lookup_factor=';'.join(missing_values),
-                **{f"all_{term_type}_have_lookup_value": all_nodes_have_value})
-    return list_sum(values) if len(values) > 0 and (allow_missing or all_nodes_have_value) else None
+    return all_factor_value(
+        model=model,
+        term_id=term_id,
+        node=cycle,
+        lookup_name=f"{term_type}.csv",
+        lookup_col=lookup_col,
+        blank_nodes=blank_nodes
+    )
 
 
 def get_ecoClimateZone(cycle: dict) -> int:

hestia_earth/models/utils/fuel.py

@@ -13,7 +13,7 @@ def impact_lookup_value(model: str, term_id: str, impact_assessment: dict, looku
     fuels = filter_list_term_type(cycle.get('inputs', []), TermTermType.FUEL)
     has_fuels_inputs = len(fuels) > 0
     fuels_total_value = convert_value_from_cycle(
-        product, cycle_lookup_value(model, term_id, fuels, lookup_col), model=model, term_id=term_id
+        product, cycle_lookup_value(model, term_id, cycle, fuels, lookup_col), model=model, term_id=term_id
     ) if has_fuels_inputs else None
     logRequirements(impact_assessment, model=model, term=term_id,
                     term_type_electricityFuel_complete=fuel_complete,

hestia_earth/models/utils/impact_assessment.py

@@ -4,10 +4,10 @@ from hestia_earth.utils.lookup import download_lookup
 from hestia_earth.utils.model import find_term_match
 from hestia_earth.utils.tools import list_sum, safe_parse_date
 
-from .lookup import factor_value, _term_factor_value, _aware_factor_value, fallback_country
+from hestia_earth.models.log import logRequirements, debugValues, log_as_table
+from .lookup import all_factor_value, _term_factor_value, _aware_factor_value, fallback_country
 from .product import find_by_product
 from .site import region_level_1_id
-from ..log import logRequirements
 
 
 def impact_end_year(impact_assessment: dict) -> int:
@@ -110,15 +110,19 @@ def impact_lookup_value(model: str, term_id: str, impact: dict, lookup_col: str,
     int
         The impact total value.
     """
-    nodes = impact.get('emissionsResourceUse', [])
-
-    factors = list(map(factor_value(model, term_id, 'emission.csv', lookup_col, grouped_key), nodes))
-    values = [value for value in factors if value is not None]
-    return list_sum(values) if len(values) > 0 else None
+    return all_factor_value(
+        model=model,
+        term_id=term_id,
+        node=impact,
+        lookup_name='emission.csv',
+        lookup_col=lookup_col,
+        blank_nodes=impact.get('emissionsResourceUse', []),
+        grouped_key=grouped_key
+    )
 
 
 def impact_country_value(model: str, term_id: str, impact: dict, lookup: str, group_key: str = None,
-                         country_fallback: bool = False, default_no_emissions=None) -> float:
+                         country_fallback: bool = False) -> float:
     """
     Calculate the value of the impact based on lookup factors and `site.country.@id`.
 
@@ -137,8 +141,6 @@ def impact_country_value(model: str, term_id: str, impact: dict, lookup: str, gr
     country_fallback : bool
         Optional: if True fallback to default `region-world` country_id if country_id in `ImpactAssessment` not found in
         lookup file containing factors.
-    default_no_emissions :
-        Optional: if set, will return this value if no contributing terms found in emissionsResourceUse
 
     Returns
     -------
@@ -149,9 +151,17 @@ def impact_country_value(model: str, term_id: str, impact: dict, lookup: str, gr
     country_id = get_country_id(impact)
     country_id = fallback_country(country_id, [download_lookup(lookup)]) if country_fallback else country_id
 
-    factors = list(map(_term_factor_value(model, term_id, lookup, country_id, group_key), nodes))
-    values = [value for value in factors if value is not None]
-    return list_sum(values) if len(values) > 0 else default_no_emissions
+    values = list(map(_term_factor_value(model, term_id, lookup, country_id, group_key), nodes))
+    debugValues(impact, model=model, term=term_id,
+                values_used=log_as_table(values))
+
+    all_with_factors = all([v.get('coefficient') is not None for v in values if v.get('value') is not None])
+    values = [float((v.get('value') or 0) * (v.get('coefficient') or 0)) for v in values]
+
+    # fail if some factors are missing
+    return None if not all_with_factors else (
+        list_sum(values) if len(values) > 0 else 0
+    )
 
 
 def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, group_key: str = None) -> float:
@@ -210,9 +220,14 @@ def impact_endpoint_value(model: str, term_id: str, impact: dict, lookup_col: st
         i.get('methodModel').get('@id') == model or
         not i.get('methodModel').get('@id').startswith(model[0:6])  # allow other non-related models to be accounted for
     )]
-    factors = list(map(factor_value(model, term_id, 'characterisedIndicator.csv', lookup_col), nodes))
-    values = [value for value in factors if value is not None]
-    return list_sum(values) if len(values) > 0 else None
+    return all_factor_value(
+        model=model,
+        term_id=term_id,
+        node=impact,
+        lookup_name='characterisedIndicator.csv',
+        lookup_col=lookup_col,
+        blank_nodes=nodes
+    )
 
 
 def emission_value(impact_assessment: dict, term_id: str):

hestia_earth/models/utils/lookup.py

@@ -6,7 +6,7 @@ from hestia_earth.utils.lookup import (
 )
 from hestia_earth.utils.tools import list_sum, safe_parse_float, non_empty_list
 
-from ..log import debugValues
+from ..log import debugValues, log_as_table, debugMissingLookup
 
 
 def _node_value(node):
@@ -14,7 +14,7 @@ def _node_value(node):
     return list_sum(value) if isinstance(value, list) else value
 
 
-def factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, grouped_key: Optional[str] = None):
+def _factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, grouped_key: Optional[str] = None):
     lookup = download_lookup(lookup_name)
 
     def get_value(data: dict):
@@ -33,11 +33,41 @@ def factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, gr
                             operation=data.get('operation', {}).get('@id'),
                             value=value,
                             coefficient=coefficient)
-            return value * coefficient
-        return None
+        return {'id': node_term_id, 'value': value, 'coefficient': coefficient}
     return get_value
 
 
+def all_factor_value(
+    model: str,
+    term_id: str,
+    node: dict,
+    lookup_name: str,
+    lookup_col: str,
+    blank_nodes: List[dict],
+    grouped_key: Optional[str] = None,
+    default_no_values=0
+):
+    values = list(map(_factor_value(model, term_id, lookup_name, lookup_col, grouped_key), blank_nodes))
+
+    missing_values = set([v.get('id') for v in values if v.get('value') is not None and v.get('coefficient') is None])
+    all_with_factors = all([v.get('coefficient') is not None for v in values if v.get('value') is not None])
+
+    for term_id in missing_values:
+        debugMissingLookup(lookup_name, 'termid', term_id, lookup_col, None)
+
+    debugValues(node, model=model, term=term_id,
+                all_with_factors=all_with_factors,
+                missing_lookup_factor=';'.join(missing_values),
+                values_used=log_as_table(values))
+
+    values = [float((v.get('value') or 0) * (v.get('coefficient') or 0)) for v in values]
+
+    # fail if some factors are missing
+    return None if not all_with_factors else (
+        list_sum(values) if len(values) > 0 else default_no_values
+    )
+
+
 def _term_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):
     lookup = download_lookup(lookup_name, False)  # avoid saving in memory as there could be many different files used
 
@@ -45,11 +75,10 @@ def _term_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_i
         node_term_id = data.get('term', {}).get('@id')
         value = _node_value(data)
         coefficient = get_table_value(lookup, 'termid', lookup_term_id, column_name(node_term_id))
-        coefficient = safe_parse_float(extract_grouped_data(coefficient, group_key)) if group_key else coefficient
+        coefficient = safe_parse_float(extract_grouped_data(coefficient, group_key) if group_key else coefficient)
         if value is not None and coefficient is not None:
             debugValues(data, model=model, term=term_id, node=node_term_id, value=value, coefficient=coefficient)
-            return value * coefficient
-        return None
+        return {'id': node_term_id, 'value': value, 'coefficient': coefficient}
     return get_value
 
 

hestia_earth/models/utils/pesticideAI.py

@@ -22,7 +22,7 @@ def impact_lookup_value(model: str, term_id: str, impact_assessment: dict, looku
         get_pesticides_from_inputs(cycle)
     has_pesticides_inputs = len(pesticides) > 0
     pesticides_total_value = convert_value_from_cycle(
-        product, cycle_lookup_value(model, term_id, pesticides, lookup_col, False), model=model, term_id=term_id
+        product, cycle_lookup_value(model, term_id, cycle, pesticides, lookup_col), model=model, term_id=term_id
     ) if has_pesticides_inputs else None
 
     logRequirements(impact_assessment, model=model, term=term_id,

hestia_earth/models/utils/property.py

@@ -1,12 +1,16 @@
+from functools import cache
+
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.lookup import download_lookup, extract_grouped_data, get_table_value, column_name
+from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.tools import list_sum, safe_parse_float
 
-from ..log import debugMissingLookup
 from . import _term_id, _include_methodModel
 from .term import get_lookup_value
+from ..log import debugMissingLookup
+
+download_hestia_cached = cache(download_hestia)
 
 
 def _new_property(term, model=None):
@@ -26,7 +30,7 @@ def get_property_lookup_value(model: str, term_id: str, column: str):
     return get_lookup_value(term, column, model=model, term=term_id)
 
 
-def find_term_property(term, property: str, default=None) -> dict:
+def find_term_property(term, property: str, default=None, keep_in_memory=False) -> dict:
     """
     Get the property by `@id` linked to the `Term` in the glossary.
 
@@ -38,15 +42,18 @@ def find_term_property(term, property: str, default=None) -> dict:
         The `term.@id` of the property. Example: `nitrogenContent`.
     default : Any
         The default value if the property is not found. Defaults to `None`.
+    keep_in_memory: bool
+        Should we cache results from download_hestia(). Default False
 
     Returns
     -------
     dict
         The property if found, `default` otherwise.
     """
+    download_func = download_hestia_cached if keep_in_memory else download_hestia
     props = term.get('defaultProperties', []) if isinstance(term, dict) else []
     term_id = _term_id(term)
-    props = (download_hestia(term_id) or {}).get('defaultProperties', []) if len(props) == 0 and term_id else props
+    props = (download_func(term_id) or {}).get('defaultProperties', []) if len(props) == 0 and term_id else props
     return find_term_match(props, property, default)
 
 

hestia_earth/models/utils/term.py

@@ -673,8 +673,8 @@ def get_land_cover_siteTypes():
     return search({
         "bool": {
             "must": [
-                {"match": {"@type": "Term"}},
-                {"match": {"termType": "landCover"}}
+                {"match": {"@type": SchemaType.TERM.value}},
+                {"match": {"termType": TermTermType.LANDCOVER.value}}
             ],
             "should": [{"match": {"name": siteType.value}} for siteType in SiteSiteType],
             "minimum_should_match": 1
@@ -682,13 +682,20 @@ def get_land_cover_siteTypes():
     }, limit=LIMIT)
 
 
-def download_all_land_cover_terms():
-    result = search({
+def get_land_cover_terms():
+    """
+    Find all `Land Cover` terms from the Glossary: https://hestia.earth/glossary?termType=landCover
+
+    Returns
+    -------
+        List of landCover terms with associated siteTypes.
+    """
+    terms = search({
         "bool": {
             "must": [
-                {"match": {"@type": "Term"}},
-                {"match": {"termType": "landCover"}}
+                {"match": {"@type": SchemaType.TERM.value}},
+                {"match": {"termType": TermTermType.LANDCOVER.value}}
             ]
         },
-    }, fields=['@id', 'nameNormalized', 'subClassOf'], limit=10000)
-    return result
+    }, limit=LIMIT, fields=['@id'])
+    return list(map(lambda n: n["@id"], terms))

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.64.9'
+VERSION = '0.64.10'

{hestia_earth_models-0.64.9.dist-info → hestia_earth_models-0.64.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.64.9
+Version: 0.64.10
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hestia-earth.schema==30.*
-Requires-Dist: hestia-earth.utils>=0.13.5
+Requires-Dist: hestia-earth.utils>=0.13.9
 Requires-Dist: python-dateutil>=2.8.1
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0