hestia-earth-models 0.70.0__py3-none-any.whl → 0.70.2__py3-none-any.whl

hestia_earth/models/pooreNemecek2018/utils.py

@@ -31,12 +31,18 @@ def run_products_average(cycle: dict, term_id: str, get_value_func):
     return list_average(values) if should_run else None
 
 
+def _excreta_product_id(data: str, term_id):
+    value = extract_grouped_data(data, term_id) or ''
+    return value.split('|')[0] if '|' in value else value
+
+
 def get_excreta_product_with_ratio(cycle: dict, lookup: str, **log_args):
     product = find_primary_product(cycle) or {}
 
     data = get_lookup_value(product.get('term'), lookup, model=MODEL, **log_args)
 
-    default_product_id = extract_grouped_data(data, 'default')
+    # TODO: handle multiple products delimited by `|`
+    default_product_id = _excreta_product_id(data, 'default')
 
     # find matching practices and assign a ratio for each
     # if no matches, use default id with 100% ratio
@@ -44,7 +50,7 @@ def get_excreta_product_with_ratio(cycle: dict, lookup: str, **log_args):
     practices = [
         {
             'id': practice.get('term', {}).get('@id'),
-            'product-id': extract_grouped_data(data, practice.get('term', {}).get('@id')) or default_product_id,
+            'product-id': _excreta_product_id(data, practice.get('term', {}).get('@id')) or default_product_id,
             'value': list_sum(practice.get('value'))
         }
         for practice in practices

hestia_earth/models/schmidt2007/ch4ToAirWasteTreatment.py

@@ -8,10 +8,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "products": [{"@type": "Product", "value": "", "term.termType": "waste"}],
-            "completeness.waste": ""
-        }
+        "products": [{"@type": "Product", "value": ""}]
     }
 }
 RETURNS = {

hestia_earth/models/schmidt2007/h2SToAirWasteTreatment.py

@@ -8,10 +8,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "products": [{"@type": "Product", "value": "", "term.termType": "waste"}],
-            "completeness.waste": ""
-        }
+        "products": [{"@type": "Product", "value": ""}]
     }
 }
 RETURNS = {

hestia_earth/models/schmidt2007/n2OToAirWasteTreatmentDirect.py

@@ -8,10 +8,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "products": [{"@type": "Product", "value": "", "term.termType": "waste"}],
-            "completeness.waste": ""
-        }
+        "products": [{"@type": "Product", "value": ""}]
     }
 }
 RETURNS = {

hestia_earth/models/schmidt2007/nh3ToAirWasteTreatment.py

@@ -8,10 +8,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "products": [{"@type": "Product", "value": "", "term.termType": "waste"}],
-            "completeness.waste": ""
-        }
+        "products": [{"@type": "Product", "value": ""}]
     }
 }
 RETURNS = {

hestia_earth/models/site/grouped_measurement.py

@@ -0,0 +1,132 @@
+from functools import reduce
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import non_empty_list, flatten, is_number, list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import _include
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.blank_node import get_lookup_value
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "measurements": [
+            {"@type": "Measurement", "depthUpper": "", "depthLower": "", "value": ""}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "0",
+        "depthUpper": "",
+        "depthLower": "",
+        "dates": "",
+        "methodClassification": "modelled using other measurements"
+    }]
+}
+LOOKUPS = {
+    "measurement": ["sumIs100Group", "sumMax100Group"]
+}
+MODEL_KEY = 'grouped_measurement'
+
+
+def _measurement(term_id: str, measurement: dict):
+    data = _new_measurement(term_id) | _include(measurement, [
+        'depthUpper', 'depthLower', 'startDate', 'endDate', 'dates'
+    ])
+    data["value"] = [0]
+    data["methodClassification"] = MeasurementMethodClassification.MODELLED_USING_OTHER_MEASUREMENTS.value
+    return data
+
+
+def _measurement_term_ids(measurements: list):
+    return list(set([m.get('term', {}).get('@id') for m in measurements]))
+
+
+def _formatDepth(depth: str):
+    # handle float values
+    return str(int(depth)) if is_number(depth) else ''
+
+
+def _group_by_depth(measurements: list):
+    def group_by(group: dict, measurement: dict):
+        keys = non_empty_list([
+            _formatDepth(measurement.get('depthUpper')),
+            _formatDepth(measurement.get('depthLower'))
+        ])
+        key = '-'.join(keys) if len(keys) > 0 else 'default'
+        group[key] = group.get(key, []) + [measurement]
+        return group
+
+    return reduce(group_by, measurements, {})
+
+
+def _run_measurements(site: dict, all_term_ids: list, measurements: list):
+    term_ids = _measurement_term_ids(measurements)
+
+    # check the total value is 100%
+    total_value = list_sum([
+        list_sum(m.get('value')) for m in measurements
+    ])
+    is_total_100 = 99.5 <= total_value <= 100.5
+
+    should_run = all([is_total_100])
+
+    other_term_ids = [v for v in all_term_ids if v not in term_ids]
+    for term_id in other_term_ids:
+        logRequirements(site, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                        total_value=total_value)
+
+        logShouldRun(site, MODEL, term_id, should_run, model_key=MODEL_KEY)
+
+    return [
+        _measurement(term_id=term_id, measurement=measurements[0])
+        for term_id in other_term_ids
+    ] if should_run else []
+
+
+def _run(site: dict, measurements: list):
+    term_ids = _measurement_term_ids(measurements)
+    grouped_measurements = _group_by_depth(measurements)
+
+    return flatten([
+        _run_measurements(site, term_ids, values)
+        for values in grouped_measurements.values()
+    ]) if len(grouped_measurements.keys()) > 1 else []
+
+
+def _group_by_100group(measurements: list):
+    def group_by(group: dict, measurement: dict):
+        term = measurement.get('term', {})
+        sum_group_key = (
+            get_lookup_value(term, 'sumIs100Group', skip_debug=True, model=MODEL) or
+            get_lookup_value(term, 'sumMax100Group', skip_debug=True, model=MODEL)
+        )
+        keys = non_empty_list([
+            sum_group_key,
+            measurement.get('startDate'),
+            measurement.get('endDate'),
+            '-'.join(measurement.get('dates') or []),
+        ])
+        key = '-'.join(keys) if len(keys) > 0 else 'default'
+
+        if all([
+            sum_group_key,
+            measurement.get('value', []),
+            measurement.get('depthUpper') is not None,
+            measurement.get('depthLower') is not None
+        ]):
+            group[key] = group.get(key, []) + [measurement]
+
+        return group
+
+    return reduce(group_by, measurements, {})
+
+
+def run(site: dict):
+    grouped_measurements = _group_by_100group(site.get('measurements', []))
+    return flatten([
+        _run(site, values)
+        for values in grouped_measurements.values()
+        if len(values) > 1
+    ])

hestia_earth/models/utils/__init__.py

@@ -8,7 +8,7 @@ import datetime
 from functools import reduce
 import operator
 from pydash.objects import get
-from typing import Any, Callable
+from typing import Union, List, Callable
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import flatten, non_empty_list
@@ -54,8 +54,9 @@ def _load_calculated_node(node, type: SchemaType, data_state='recalculated'):
 def _unit_str(unit) -> str: return unit if isinstance(unit, str) else unit.value
 
 
-def _filter_list_term_unit(values: list, unit: Any):
-    units = list(map(_unit_str, unit)) if isinstance(unit, list) else [_unit_str(unit)]
+def _filter_list_term_unit(values: list, unit: Union[str, List[str]]):
+    units = unit if isinstance(unit, list) else[unit]
+    units = list(map(_unit_str, units))
     return list(filter(lambda i: i.get('term', {}).get('units') in units, values))
 
 

hestia_earth/models/utils/background_emissions.py

@@ -0,0 +1,52 @@
+from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.tools import flatten
+
+from . import is_from_model
+
+
+def _animal_inputs(animal: dict):
+    inputs = animal.get('inputs', [])
+    return [(input | {'animal': animal.get('term', {})}) for input in inputs]
+
+
+def _should_run_input(products: list):
+    def should_run(input: dict):
+        return all([
+            # make sure Input is not a Product as well or we might double-count emissions
+            find_term_match(products, input.get('term', {}).get('@id'), None) is None,
+            # ignore inputs which are flagged as Product of the Cycle
+            not input.get('fromCycle', False),
+            not input.get('producedInCycle', False)
+        ])
+    return should_run
+
+
+def get_background_inputs(cycle: dict, extra_inputs: list = []):
+    # add all the properties of some Term that inlcude others with the mapping
+    inputs = flatten(
+        cycle.get('inputs', []) +
+        list(map(_animal_inputs, cycle.get('animals', []))) +
+        extra_inputs
+    )
+    return list(filter(_should_run_input(cycle.get('products', [])), inputs))
+
+
+def no_gap_filled_background_emissions(cycle: dict):
+    emissions = cycle.get('emissions', [])
+
+    def check_input(input: dict):
+        input_term_id = input.get('term', {}).get('@id')
+        operation_term_id = input.get('operation', {}).get('@id')
+        animal_term_id = input.get('animal', {}).get('@id')
+
+        return not any([
+            is_from_model(emission)
+            for emission in emissions
+            if all([
+                any([i.get('@id') == input_term_id for i in emission.get('inputs', [])]),
+                emission.get('operation', {}).get('@id') == operation_term_id,
+                emission.get('animal', {}).get('@id') == animal_term_id
+            ])
+        ])
+
+    return check_input

hestia_earth/models/utils/blank_node.py

@@ -22,11 +22,11 @@ from hestia_earth.utils.tools import (
     list_sum,
     list_average,
     safe_parse_date,
-    safe_parse_float,
     non_empty_list
 )
 from hestia_earth.utils.lookup_utils import (
     is_model_siteType_allowed,
+    is_model_product_id_allowed,
     is_siteType_allowed,
     is_product_id_allowed,
     is_product_termType_allowed,
@@ -124,13 +124,16 @@ def _module_term_id(term_id: str, module):
     return getattr(module, 'TERM_ID', term_id_str).split(',')[0]
 
 
-def _run_model_required(model: str, term_id: str, data: dict):
+def _run_model_required(model: str, term_id: str, data: dict, skip_logs: bool = False):
     siteType_allowed = is_model_siteType_allowed(model, term_id, data)
+    product_id_allowed = is_model_product_id_allowed(model, term_id, data)
 
-    run_required = all([siteType_allowed])
-    debugValues(data, model=model, term=term_id,
-                run_required=run_required,
-                siteType_allowed=siteType_allowed)
+    run_required = all([siteType_allowed, product_id_allowed])
+    if not skip_logs:
+        debugValues(data, model=model, term=term_id,
+                    run_required=run_required,
+                    siteType_allowed=siteType_allowed,
+                    product_id_allowed=product_id_allowed)
     return run_required
 
 
@@ -234,16 +237,21 @@ def get_total_value(nodes: list):
 
 def _value_as(term_id: str, convert_to_property=True):
     def get_value(node: dict):
-        property = get_node_property(node, term_id)
+        factor = get_node_property_value(
+            None, node, term_id, default=0, handle_percents=False
+        ) or get_lookup_value(
+            lookup_term=node.get('term', {}),
+            column=term_id
+        ) or 0
         # ignore node value if property is not found
-        factor = safe_parse_float(property.get('value', 0))
         value = list_sum(node.get('value', []))
+        property = get_node_property(node, term_id, find_default_property=False, download_from_hestia=True)
         ratio = factor / 100 if property.get('term', {}).get('units', '') == '%' else factor
         return 0 if ratio == 0 else (value * ratio if convert_to_property else value / ratio)
     return get_value
 
 
-def get_total_value_converted(nodes: list, conversion_property, convert_to_property=True):
+def get_total_value_converted(nodes: list, conversion_property: Union[List[str], str], convert_to_property=True):
     """
     Get the total `value` of a list of Blank Nodes converted using a property of each Blank Node.
 
@@ -367,14 +375,39 @@ def get_KG_total(nodes: list) -> list:
     return get_total_value(kg_nodes) + get_total_value_converted(kg_N_nodes, 'nitrogenContent', False)
 
 
+def get_P_total(nodes: list) -> list:
+    """
+    Get the total phosphorous content of a list of Blank Node.
+
+    The result contains the values of the following nodes:
+    1. Every blank node specified in `kg P` will be used.
+    1. Every blank node specified in `kg N` will be multiplied by the `phosphateContentAsP` property.
+    2. Every blank node specified in `kg` will be multiplied by the `phosphateContentAsP` property.
+
+    Parameters
+    ----------
+    nodes : list
+        A list of Blank Node.
+
+    Returns
+    -------
+    list
+        The phosphorous values as a list of numbers.
+    """
+    kg_P_nodes = _filter_list_term_unit(nodes, Units.KG_P)
+    kg_N_nodes = _filter_list_term_unit(nodes, Units.KG_N)
+    kg_nodes = _filter_list_term_unit(nodes, Units.KG)
+    return get_total_value(kg_P_nodes) + get_total_value_converted(kg_N_nodes + kg_nodes, 'phosphateContentAsP')
+
+
 def get_P2O5_total(nodes: list) -> list:
     """
     Get the total phosphate content of a list of Blank Node.
 
     The result contains the values of the following nodes:
-    1. Every organic fertiliser specified in `kg P2O5` will be used.
-    1. Every organic fertiliser specified in `kg N` will be multiplied by the `phosphateContentAsP2O5` property.
-    2. Every organic fertiliser specified in `kg` will be multiplied by the `phosphateContentAsP2O5` property.
+    1. Every blank node specified in `kg P2O5` will be used.
+    1. Every blank node specified in `kg N` will be multiplied by the `phosphateContentAsP2O5` property.
+    2. Every blank node specified in `kg` will be multiplied by the `phosphateContentAsP2O5` property.
 
     Parameters
     ----------
@@ -386,10 +419,10 @@ def get_P2O5_total(nodes: list) -> list:
     list
         The phosphate values as a list of numbers.
     """
-    kg_P_nodes = _filter_list_term_unit(nodes, Units.KG_P2O5)
+    kg_P2O5_nodes = _filter_list_term_unit(nodes, Units.KG_P2O5)
     kg_N_nodes = _filter_list_term_unit(nodes, Units.KG_N)
     kg_nodes = _filter_list_term_unit(nodes, Units.KG)
-    return get_total_value(kg_P_nodes) + get_total_value_converted(kg_N_nodes + kg_nodes, 'phosphateContentAsP2O5')
+    return get_total_value(kg_P2O5_nodes) + get_total_value_converted(kg_N_nodes + kg_nodes, 'phosphateContentAsP2O5')
 
 
 def convert_to_nitrogen(node: dict, model: str, blank_nodes: list, **log_args):

hestia_earth/models/utils/constant.py

@@ -19,7 +19,7 @@ class Units(Enum):
     KG_CO2 = 'kg CO2'
     KG_K = 'kg K'
     KG_K2O = 'kg K2O'
-    KG_MGCO3 = 'kg MgCO3'
+    KG_CAMGCO32 = 'kg CaMg(CO3)2'
     KG_N = 'kg N'
     KG_N2 = 'kg N2'
     KG_N2O = 'kg N2O'
@@ -48,58 +48,62 @@ C = 12.012
 CA = 40.078
 H = 1.008
 K = 39.098
+MG = 24.305
 N = 14.007
 _O = 15.999
 P = 30.974
 ATOMIC_WEIGHT_CONVERSIONS = {
     Units.KG_P.value: {
-        Units.KG_P2O5.value: (P*2) / (P*2 + _O*5),  # Conv_Mol_P_P2O5
-        Units.KG_PO43.value: P / ((P + _O*4)*3)  # Conv_Mol_P_PO43-
+        Units.KG_P2O5.value: (P*2) / (P*2 + _O*5),
+        Units.KG_PO43.value: P / ((P + _O*4)*3)
     },
     Units.KG_PO43.value: {
-        Units.KG_P2O5.value: ((P + _O*4)*3) / (P*2 + _O*5)  # Conv_Mol_PO43-_P2O5
+        Units.KG_P2O5.value: ((P + _O*4)*3) / (P*2 + _O*5)
+    },
+    Units.KG_P2O5.value: {
+        Units.KG_P.value: (P*2 + _O*5) / (P*2)
     },
     Units.KG_K.value: {
-        Units.KG_K2O.value: (K*2) / (K*2 + _O)  # Conv_Mol_K_K2O
+        Units.KG_K2O.value: (K*2) / (K*2 + _O)
     },
     Units.KG_CA.value: {
-        Units.KG_CAO.value: CA / (CA + _O)  # Conv_Mol_Ca_CaO
+        Units.KG_CAO.value: CA / (CA + _O)
     },
     Units.KG_CAO.value: {
-        Units.KG_CACO3.value: (CA + _O) / (CA + C + _O*3)  # Conv_Mol_CaO_CaCO3
+        Units.KG_CACO3.value: (CA + _O) / (CA + C + _O*3)
     },
     Units.KG_CACO3.value: {
-        Units.KG_CO2.value: 0.12
+        Units.KG_CO2.value: (CA + C + _O*3) / C
     },
-    Units.KG_MGCO3.value: {
-        Units.KG_CO2.value: 0.13
+    Units.KG_CAMGCO32.value: {
+        Units.KG_CO2.value: (CA + MG + (C + _O*3)*2) / (C*2)
     },
     Units.KG_CH4.value: {
-        Units.TO_C.value: (C + H*4) / C  # Conv_Mol_CH4C_CH4
+        Units.TO_C.value: (C + H*4) / C
     },
     Units.KG_CO2.value: {
-        Units.TO_C.value: (C + _O*2) / C  # Conv_Mol_CO2C_CO2
+        Units.TO_C.value: (C + _O*2) / C
     },
     Units.KG_NOX.value: {
-        Units.TO_N.value: (N + _O) / N  # Conv_Mol_NON_NO
+        Units.TO_N.value: (N + _O) / N
     },
     Units.KG_N2.value: {
         Units.TO_N.value: 1
     },
     Units.KG_N2O.value: {
-        Units.TO_N.value: (N*2 + _O) / (N*2)  # Conv_Mol_N2ON_N2O
+        Units.TO_N.value: (N*2 + _O) / (N*2)
     },
     Units.KG_NO2.value: {
-        Units.TO_N.value: (N + _O*2) / N  # Conv_Mol_NO2N_NO2
+        Units.TO_N.value: (N + _O*2) / N
     },
     Units.KG_NO3.value: {
-        Units.TO_N.value: (N + _O*3) / N  # Conv_Mol_NO3N_NO3
+        Units.TO_N.value: (N + _O*3) / N
     },
     Units.KG_NH3.value: {
-        Units.TO_N.value: (N + H*3) / N  # Conv_Mol_NH3N_NH3
+        Units.TO_N.value: (N + H*3) / N
     },
     Units.KG_NH4.value: {
-        Units.TO_N.value: (N + H*4) / N  # Conv_Mol_NH4N_NH4
+        Units.TO_N.value: (N + H*4) / N
     },
     Units.KW_H.value: {
         Units.MJ.value: 3.6
@@ -113,8 +117,10 @@ def get_atomic_conversion(src_unit: Units, dest_unit: Units, default_value=1):
     return ATOMIC_WEIGHT_CONVERSIONS.get(src_key, {}).get(dest_key, default_value)
 
 
-def convert_to_unit(node: dict, dest_unit: Units):
-    return list_sum(node.get('value', [])) * get_atomic_conversion(node.get('term', {}).get('units'), dest_unit)
+def convert_to_unit(node: dict, dest_unit: Units, default_value=None):
+    value = list_sum(node.get('value', []), default=None)
+    conversion = get_atomic_conversion(node.get('term', {}).get('units'), dest_unit, default_value=None)
+    return value / conversion if all([value is not None, conversion is not None]) else default_value
 
 
 def convert_to_N(node: dict):

hestia_earth/models/utils/impact_assessment.py

@@ -3,7 +3,7 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, safe_parse_date
 
-from hestia_earth.models.log import debugValues, log_as_table
+from hestia_earth.models.log import debugValues
 from .lookup import all_factor_value, _term_factor_value, _aware_factor_value, fallback_country
 from .product import find_by_product
 from .site import region_level_1_id
@@ -157,22 +157,22 @@ def impact_country_value(
         The impact total value.
     """
     term_type = TermTermType.RESOURCEUSE.value if 'resourceUse' in lookup else TermTermType.EMISSION.value
-    nodes = filter_list_term_type(impact.get('emissionsResourceUse', []), term_type)
+    blank_nodes = filter_list_term_type(impact.get('emissionsResourceUse', []), term_type)
 
     country_id = get_country_id(impact)
     country_id = fallback_country(country_id, [lookup]) if country_fallback else country_id
 
-    values = list(map(_term_factor_value(model, term_id, lookup, country_id, group_key), nodes))
-    debugValues(impact, model=model, term=term_id,
-                values_used=log_as_table(values))
-
-    has_values = len(values) > 0
-    missing_values = set([v.get('id') for v in values if v.get('value') and v.get('coefficient') is None])
-    all_with_factors = not missing_values
-    values = [float((v.get('value') or 0) * (v.get('coefficient') or 0)) for v in values]
-
-    # fail if some factors are missing
-    return None if not all_with_factors else (list_sum(values) if has_values else default_no_values)
+    return all_factor_value(
+        model=model,
+        term_id=term_id,
+        node=impact,
+        lookup_name=lookup,
+        lookup_col=country_id,
+        blank_nodes=blank_nodes,
+        grouped_key=group_key,
+        default_no_values=None,
+        factor_value_func=_term_factor_value
+    )
 
 
 def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, group_key: str = None) -> float:
@@ -197,14 +197,23 @@ def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, grou
     int
         The impact total value.
     """
-    nodes = impact.get('emissionsResourceUse', [])
+    blank_nodes = impact.get('emissionsResourceUse', [])
     site = get_site(impact)
     aware_id = site.get('awareWaterBasinId')
     if aware_id is None:
         return None
-    factors = list(map(_aware_factor_value(model, term_id, lookup, aware_id, group_key), nodes))
-    values = [value for value in factors if value is not None]
-    return list_sum(values) if len(values) > 0 else None
+
+    return all_factor_value(
+        model=model,
+        term_id=term_id,
+        node=impact,
+        lookup_name=lookup,
+        lookup_col=aware_id,
+        blank_nodes=blank_nodes,
+        grouped_key=group_key,
+        default_no_values=None,
+        factor_value_func=_aware_factor_value
+    )
 
 
 def impact_endpoint_value(model: str, term_id: str, impact: dict, lookup_col: str) -> float: