hestia-earth-models 0.64.9__py3-none-any.whl → 0.64.11__py3-none-any.whl

hestia_earth/models/utils/impact_assessment.py

@@ -1,13 +1,13 @@
 from typing import Optional
-
+from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import download_lookup
-from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, safe_parse_date
 
-from .lookup import factor_value, _term_factor_value, _aware_factor_value, fallback_country
+from hestia_earth.models.log import logRequirements, debugValues, log_as_table
+from .lookup import all_factor_value, _term_factor_value, _aware_factor_value, fallback_country
 from .product import find_by_product
 from .site import region_level_1_id
-from ..log import logRequirements
 
 
 def impact_end_year(impact_assessment: dict) -> int:
@@ -88,8 +88,9 @@ def get_country_id(impact_assessment: dict) -> str:
     return impact_assessment.get('country', get_site(impact_assessment).get('country', {})).get('@id')
 
 
-def impact_lookup_value(model: str, term_id: str, impact: dict, lookup_col: str,
-                        grouped_key: Optional[str] = None) -> float:
+def impact_emission_lookup_value(
+    model: str, term_id: str, impact: dict, lookup_col: str, grouped_key: Optional[str] = None
+) -> float:
     """
     Calculate the value of the impact based on lookup factors and emissions value.
 
@@ -105,20 +106,31 @@ def impact_lookup_value(model: str, term_id: str, impact: dict, lookup_col: str,
         The lookup column to fetch the factors from.
     grouped_key : str
         key of grouped data to extract in a lookup table
+
     Returns
     -------
     int
         The impact total value.
     """
-    nodes = impact.get('emissionsResourceUse', [])
-
-    factors = list(map(factor_value(model, term_id, 'emission.csv', lookup_col, grouped_key), nodes))
-    values = [value for value in factors if value is not None]
-    return list_sum(values) if len(values) > 0 else None
-
-
-def impact_country_value(model: str, term_id: str, impact: dict, lookup: str, group_key: str = None,
-                         country_fallback: bool = False, default_no_emissions=None) -> float:
+    return all_factor_value(
+        model=model,
+        term_id=term_id,
+        node=impact,
+        lookup_name='emission.csv',
+        lookup_col=lookup_col,
+        blank_nodes=filter_list_term_type(impact.get('emissionsResourceUse', []), TermTermType.EMISSION),
+        grouped_key=grouped_key
+    )
+
+
+def impact_country_value(
+    model: str,
+    term_id: str,
+    impact: dict,
+    lookup: str,
+    group_key: str = None,
+    country_fallback: bool = False,
+) -> float:
     """
     Calculate the value of the impact based on lookup factors and `site.country.@id`.
 
@@ -137,21 +149,29 @@ def impact_country_value(model: str, term_id: str, impact: dict, lookup: str, gr
     country_fallback : bool
         Optional: if True fallback to default `region-world` country_id if country_id in `ImpactAssessment` not found in
         lookup file containing factors.
-    default_no_emissions :
-        Optional: if set, will return this value if no contributing terms found in emissionsResourceUse
 
     Returns
     -------
     int
         The impact total value.
     """
-    nodes = impact.get('emissionsResourceUse', [])
+    term_type = TermTermType.RESOURCEUSE.value if 'resourceUse' in lookup else TermTermType.EMISSION.value
+    nodes = filter_list_term_type(impact.get('emissionsResourceUse', []), term_type)
+
     country_id = get_country_id(impact)
     country_id = fallback_country(country_id, [download_lookup(lookup)]) if country_fallback else country_id
 
-    factors = list(map(_term_factor_value(model, term_id, lookup, country_id, group_key), nodes))
-    values = [value for value in factors if value is not None]
-    return list_sum(values) if len(values) > 0 else default_no_emissions
+    values = list(map(_term_factor_value(model, term_id, lookup, country_id, group_key), nodes))
+    debugValues(impact, model=model, term=term_id,
+                values_used=log_as_table(values))
+
+    all_with_factors = all([v.get('coefficient') is not None for v in values if v.get('value') is not None])
+    values = [float((v.get('value') or 0) * (v.get('coefficient') or 0)) for v in values]
+
+    # fail if some factors are missing
+    return None if not all_with_factors else (
+        list_sum(values) if len(values) > 0 else 0
+    )
 
 
 def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, group_key: str = None) -> float:
@@ -210,9 +230,14 @@ def impact_endpoint_value(model: str, term_id: str, impact: dict, lookup_col: st
         i.get('methodModel').get('@id') == model or
         not i.get('methodModel').get('@id').startswith(model[0:6])  # allow other non-related models to be accounted for
     )]
-    factors = list(map(factor_value(model, term_id, 'characterisedIndicator.csv', lookup_col), nodes))
-    values = [value for value in factors if value is not None]
-    return list_sum(values) if len(values) > 0 else None
+    return all_factor_value(
+        model=model,
+        term_id=term_id,
+        node=impact,
+        lookup_name='characterisedIndicator.csv',
+        lookup_col=lookup_col,
+        blank_nodes=nodes
+    )
 
 
 def emission_value(impact_assessment: dict, term_id: str):

hestia_earth/models/utils/lookup.py

@@ -6,7 +6,7 @@ from hestia_earth.utils.lookup import (
 )
 from hestia_earth.utils.tools import list_sum, safe_parse_float, non_empty_list
 
-from ..log import debugValues
+from ..log import debugValues, log_as_table, debugMissingLookup
 
 
 def _node_value(node):
@@ -14,7 +14,7 @@ def _node_value(node):
     return list_sum(value) if isinstance(value, list) else value
 
 
-def factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, grouped_key: Optional[str] = None):
+def _factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, grouped_key: Optional[str] = None):
     lookup = download_lookup(lookup_name)
 
     def get_value(data: dict):
@@ -33,11 +33,41 @@ def factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, gr
                             operation=data.get('operation', {}).get('@id'),
                             value=value,
                             coefficient=coefficient)
-            return value * coefficient
-        return None
+        return {'id': node_term_id, 'value': value, 'coefficient': coefficient}
     return get_value
 
 
+def all_factor_value(
+    model: str,
+    term_id: str,
+    node: dict,
+    lookup_name: str,
+    lookup_col: str,
+    blank_nodes: List[dict],
+    grouped_key: Optional[str] = None,
+    default_no_values=0
+):
+    values = list(map(_factor_value(model, term_id, lookup_name, lookup_col, grouped_key), blank_nodes))
+
+    missing_values = set([v.get('id') for v in values if v.get('value') is not None and v.get('coefficient') is None])
+    all_with_factors = all([v.get('coefficient') is not None for v in values if v.get('value') is not None])
+
+    for term_id in missing_values:
+        debugMissingLookup(lookup_name, 'termid', term_id, lookup_col, None)
+
+    debugValues(node, model=model, term=term_id,
+                all_with_factors=all_with_factors,
+                missing_lookup_factor=';'.join(missing_values),
+                values_used=log_as_table(values))
+
+    values = [float((v.get('value') or 0) * (v.get('coefficient') or 0)) for v in values]
+
+    # fail if some factors are missing
+    return None if not all_with_factors else (
+        list_sum(values) if len(values) > 0 else default_no_values
+    )
+
+
 def _term_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):
     lookup = download_lookup(lookup_name, False)  # avoid saving in memory as there could be many different files used
 
@@ -45,11 +75,10 @@ def _term_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_i
         node_term_id = data.get('term', {}).get('@id')
         value = _node_value(data)
         coefficient = get_table_value(lookup, 'termid', lookup_term_id, column_name(node_term_id))
-        coefficient = safe_parse_float(extract_grouped_data(coefficient, group_key)) if group_key else coefficient
+        coefficient = safe_parse_float(extract_grouped_data(coefficient, group_key) if group_key else coefficient)
         if value is not None and coefficient is not None:
             debugValues(data, model=model, term=term_id, node=node_term_id, value=value, coefficient=coefficient)
-            return value * coefficient
-        return None
+        return {'id': node_term_id, 'value': value, 'coefficient': coefficient}
     return get_value
 
 

hestia_earth/models/utils/pesticideAI.py

@@ -22,7 +22,7 @@ def impact_lookup_value(model: str, term_id: str, impact_assessment: dict, looku
         get_pesticides_from_inputs(cycle)
     has_pesticides_inputs = len(pesticides) > 0
     pesticides_total_value = convert_value_from_cycle(
-        product, cycle_lookup_value(model, term_id, pesticides, lookup_col, False), model=model, term_id=term_id
+        product, cycle_lookup_value(model, term_id, cycle, pesticides, lookup_col), model=model, term_id=term_id
     ) if has_pesticides_inputs else None
 
     logRequirements(impact_assessment, model=model, term=term_id,

hestia_earth/models/utils/property.py

@@ -1,12 +1,16 @@
+from functools import cache
+
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.lookup import download_lookup, extract_grouped_data, get_table_value, column_name
+from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.tools import list_sum, safe_parse_float
 
-from ..log import debugMissingLookup
 from . import _term_id, _include_methodModel
 from .term import get_lookup_value
+from ..log import debugMissingLookup
+
+download_hestia_cached = cache(download_hestia)
 
 
 def _new_property(term, model=None):
@@ -26,7 +30,7 @@ def get_property_lookup_value(model: str, term_id: str, column: str):
     return get_lookup_value(term, column, model=model, term=term_id)
 
 
-def find_term_property(term, property: str, default=None) -> dict:
+def find_term_property(term, property: str, default=None, keep_in_memory=False) -> dict:
     """
     Get the property by `@id` linked to the `Term` in the glossary.
 
@@ -38,15 +42,18 @@ def find_term_property(term, property: str, default=None) -> dict:
         The `term.@id` of the property. Example: `nitrogenContent`.
     default : Any
         The default value if the property is not found. Defaults to `None`.
+    keep_in_memory: bool
+        Should we cache results from download_hestia(). Default False
 
     Returns
     -------
     dict
         The property if found, `default` otherwise.
     """
+    download_func = download_hestia_cached if keep_in_memory else download_hestia
     props = term.get('defaultProperties', []) if isinstance(term, dict) else []
     term_id = _term_id(term)
-    props = (download_hestia(term_id) or {}).get('defaultProperties', []) if len(props) == 0 and term_id else props
+    props = (download_func(term_id) or {}).get('defaultProperties', []) if len(props) == 0 and term_id else props
     return find_term_match(props, property, default)
 
 

hestia_earth/models/utils/term.py

@@ -673,8 +673,8 @@ def get_land_cover_siteTypes():
     return search({
         "bool": {
             "must": [
-                {"match": {"@type": "Term"}},
-                {"match": {"termType": "landCover"}}
+                {"match": {"@type": SchemaType.TERM.value}},
+                {"match": {"termType": TermTermType.LANDCOVER.value}}
             ],
             "should": [{"match": {"name": siteType.value}} for siteType in SiteSiteType],
             "minimum_should_match": 1
@@ -682,13 +682,20 @@ def get_land_cover_siteTypes():
     }, limit=LIMIT)
 
 
-def download_all_land_cover_terms():
-    result = search({
+def get_land_cover_terms():
+    """
+    Find all `Land Cover` terms from the Glossary: https://hestia.earth/glossary?termType=landCover
+
+    Returns
+    -------
+        List of landCover terms with associated siteTypes.
+    """
+    terms = search({
         "bool": {
             "must": [
-                {"match": {"@type": "Term"}},
-                {"match": {"termType": "landCover"}}
+                {"match": {"@type": SchemaType.TERM.value}},
+                {"match": {"termType": TermTermType.LANDCOVER.value}}
             ]
         },
-    }, fields=['@id', 'nameNormalized', 'subClassOf'], limit=10000)
-    return result
+    }, limit=LIMIT, fields=['@id'])
+    return list(map(lambda n: n["@id"], terms))

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.64.9'
+VERSION = '0.64.11'

{hestia_earth_models-0.64.9.dist-info → hestia_earth_models-0.64.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.64.9
+Version: 0.64.11
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hestia-earth.schema==30.*
-Requires-Dist: hestia-earth.utils>=0.13.5
+Requires-Dist: hestia-earth.utils>=0.13.10
 Requires-Dist: python-dateutil>=2.8.1
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0