hestia-earth-models 0.70.1__py3-none-any.whl → 0.70.2__py3-none-any.whl

hestia_earth/models/site/grouped_measurement.py

@@ -0,0 +1,132 @@
+from functools import reduce
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import non_empty_list, flatten, is_number, list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import _include
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.blank_node import get_lookup_value
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "measurements": [
+            {"@type": "Measurement", "depthUpper": "", "depthLower": "", "value": ""}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "0",
+        "depthUpper": "",
+        "depthLower": "",
+        "dates": "",
+        "methodClassification": "modelled using other measurements"
+    }]
+}
+LOOKUPS = {
+    "measurement": ["sumIs100Group", "sumMax100Group"]
+}
+MODEL_KEY = 'grouped_measurement'
+
+
+def _measurement(term_id: str, measurement: dict):
+    data = _new_measurement(term_id) | _include(measurement, [
+        'depthUpper', 'depthLower', 'startDate', 'endDate', 'dates'
+    ])
+    data["value"] = [0]
+    data["methodClassification"] = MeasurementMethodClassification.MODELLED_USING_OTHER_MEASUREMENTS.value
+    return data
+
+
+def _measurement_term_ids(measurements: list):
+    return list(set([m.get('term', {}).get('@id') for m in measurements]))
+
+
+def _formatDepth(depth: str):
+    # handle float values
+    return str(int(depth)) if is_number(depth) else ''
+
+
+def _group_by_depth(measurements: list):
+    def group_by(group: dict, measurement: dict):
+        keys = non_empty_list([
+            _formatDepth(measurement.get('depthUpper')),
+            _formatDepth(measurement.get('depthLower'))
+        ])
+        key = '-'.join(keys) if len(keys) > 0 else 'default'
+        group[key] = group.get(key, []) + [measurement]
+        return group
+
+    return reduce(group_by, measurements, {})
+
+
+def _run_measurements(site: dict, all_term_ids: list, measurements: list):
+    term_ids = _measurement_term_ids(measurements)
+
+    # check the total value is 100%
+    total_value = list_sum([
+        list_sum(m.get('value')) for m in measurements
+    ])
+    is_total_100 = 99.5 <= total_value <= 100.5
+
+    should_run = all([is_total_100])
+
+    other_term_ids = [v for v in all_term_ids if v not in term_ids]
+    for term_id in other_term_ids:
+        logRequirements(site, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                        total_value=total_value)
+
+        logShouldRun(site, MODEL, term_id, should_run, model_key=MODEL_KEY)
+
+    return [
+        _measurement(term_id=term_id, measurement=measurements[0])
+        for term_id in other_term_ids
+    ] if should_run else []
+
+
+def _run(site: dict, measurements: list):
+    term_ids = _measurement_term_ids(measurements)
+    grouped_measurements = _group_by_depth(measurements)
+
+    return flatten([
+        _run_measurements(site, term_ids, values)
+        for values in grouped_measurements.values()
+    ]) if len(grouped_measurements.keys()) > 1 else []
+
+
+def _group_by_100group(measurements: list):
+    def group_by(group: dict, measurement: dict):
+        term = measurement.get('term', {})
+        sum_group_key = (
+            get_lookup_value(term, 'sumIs100Group', skip_debug=True, model=MODEL) or
+            get_lookup_value(term, 'sumMax100Group', skip_debug=True, model=MODEL)
+        )
+        keys = non_empty_list([
+            sum_group_key,
+            measurement.get('startDate'),
+            measurement.get('endDate'),
+            '-'.join(measurement.get('dates') or []),
+        ])
+        key = '-'.join(keys) if len(keys) > 0 else 'default'
+
+        if all([
+            sum_group_key,
+            measurement.get('value', []),
+            measurement.get('depthUpper') is not None,
+            measurement.get('depthLower') is not None
+        ]):
+            group[key] = group.get(key, []) + [measurement]
+
+        return group
+
+    return reduce(group_by, measurements, {})
+
+
+def run(site: dict):
+    grouped_measurements = _group_by_100group(site.get('measurements', []))
+    return flatten([
+        _run(site, values)
+        for values in grouped_measurements.values()
+        if len(values) > 1
+    ])

hestia_earth/models/utils/__init__.py

@@ -8,7 +8,7 @@ import datetime
 from functools import reduce
 import operator
 from pydash.objects import get
-from typing import Any, Callable
+from typing import Union, List, Callable
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import flatten, non_empty_list
@@ -54,8 +54,9 @@ def _load_calculated_node(node, type: SchemaType, data_state='recalculated'):
 def _unit_str(unit) -> str: return unit if isinstance(unit, str) else unit.value
 
 
-def _filter_list_term_unit(values: list, unit: Any):
-    units = list(map(_unit_str, unit)) if isinstance(unit, list) else [_unit_str(unit)]
+def _filter_list_term_unit(values: list, unit: Union[str, List[str]]):
+    units = unit if isinstance(unit, list) else[unit]
+    units = list(map(_unit_str, units))
     return list(filter(lambda i: i.get('term', {}).get('units') in units, values))
 
 

hestia_earth/models/utils/blank_node.py

@@ -22,7 +22,6 @@ from hestia_earth.utils.tools import (
     list_sum,
     list_average,
     safe_parse_date,
-    safe_parse_float,
     non_empty_list
 )
 from hestia_earth.utils.lookup_utils import (
@@ -238,16 +237,21 @@ def get_total_value(nodes: list):
 
 def _value_as(term_id: str, convert_to_property=True):
     def get_value(node: dict):
-        property = get_node_property(node, term_id)
+        factor = get_node_property_value(
+            None, node, term_id, default=0, handle_percents=False
+        ) or get_lookup_value(
+            lookup_term=node.get('term', {}),
+            column=term_id
+        ) or 0
         # ignore node value if property is not found
-        factor = safe_parse_float(property.get('value', 0))
         value = list_sum(node.get('value', []))
+        property = get_node_property(node, term_id, find_default_property=False, download_from_hestia=True)
         ratio = factor / 100 if property.get('term', {}).get('units', '') == '%' else factor
         return 0 if ratio == 0 else (value * ratio if convert_to_property else value / ratio)
     return get_value
 
 
-def get_total_value_converted(nodes: list, conversion_property, convert_to_property=True):
+def get_total_value_converted(nodes: list, conversion_property: Union[List[str], str], convert_to_property=True):
     """
     Get the total `value` of a list of Blank Nodes converted using a property of each Blank Node.
 
@@ -371,14 +375,39 @@ def get_KG_total(nodes: list) -> list:
     return get_total_value(kg_nodes) + get_total_value_converted(kg_N_nodes, 'nitrogenContent', False)
 
 
+def get_P_total(nodes: list) -> list:
+    """
+    Get the total phosphorous content of a list of Blank Node.
+
+    The result contains the values of the following nodes:
+    1. Every blank node specified in `kg P` will be used.
+    1. Every blank node specified in `kg N` will be multiplied by the `phosphateContentAsP` property.
+    2. Every blank node specified in `kg` will be multiplied by the `phosphateContentAsP` property.
+
+    Parameters
+    ----------
+    nodes : list
+        A list of Blank Node.
+
+    Returns
+    -------
+    list
+        The phosphorous values as a list of numbers.
+    """
+    kg_P_nodes = _filter_list_term_unit(nodes, Units.KG_P)
+    kg_N_nodes = _filter_list_term_unit(nodes, Units.KG_N)
+    kg_nodes = _filter_list_term_unit(nodes, Units.KG)
+    return get_total_value(kg_P_nodes) + get_total_value_converted(kg_N_nodes + kg_nodes, 'phosphateContentAsP')
+
+
 def get_P2O5_total(nodes: list) -> list:
     """
     Get the total phosphate content of a list of Blank Node.
 
     The result contains the values of the following nodes:
-    1. Every organic fertiliser specified in `kg P2O5` will be used.
-    1. Every organic fertiliser specified in `kg N` will be multiplied by the `phosphateContentAsP2O5` property.
-    2. Every organic fertiliser specified in `kg` will be multiplied by the `phosphateContentAsP2O5` property.
+    1. Every blank node specified in `kg P2O5` will be used.
+    1. Every blank node specified in `kg N` will be multiplied by the `phosphateContentAsP2O5` property.
+    2. Every blank node specified in `kg` will be multiplied by the `phosphateContentAsP2O5` property.
 
     Parameters
     ----------
@@ -390,10 +419,10 @@ def get_P2O5_total(nodes: list) -> list:
     list
         The phosphate values as a list of numbers.
     """
-    kg_P_nodes = _filter_list_term_unit(nodes, Units.KG_P2O5)
+    kg_P2O5_nodes = _filter_list_term_unit(nodes, Units.KG_P2O5)
     kg_N_nodes = _filter_list_term_unit(nodes, Units.KG_N)
     kg_nodes = _filter_list_term_unit(nodes, Units.KG)
-    return get_total_value(kg_P_nodes) + get_total_value_converted(kg_N_nodes + kg_nodes, 'phosphateContentAsP2O5')
+    return get_total_value(kg_P2O5_nodes) + get_total_value_converted(kg_N_nodes + kg_nodes, 'phosphateContentAsP2O5')
 
 
 def convert_to_nitrogen(node: dict, model: str, blank_nodes: list, **log_args):

hestia_earth/models/utils/constant.py

@@ -19,7 +19,7 @@ class Units(Enum):
     KG_CO2 = 'kg CO2'
     KG_K = 'kg K'
     KG_K2O = 'kg K2O'
-    KG_MGCO3 = 'kg MgCO3'
+    KG_CAMGCO32 = 'kg CaMg(CO3)2'
     KG_N = 'kg N'
     KG_N2 = 'kg N2'
     KG_N2O = 'kg N2O'
@@ -48,58 +48,62 @@ C = 12.012
 CA = 40.078
 H = 1.008
 K = 39.098
+MG = 24.305
 N = 14.007
 _O = 15.999
 P = 30.974
 ATOMIC_WEIGHT_CONVERSIONS = {
     Units.KG_P.value: {
-        Units.KG_P2O5.value: (P*2) / (P*2 + _O*5),  # Conv_Mol_P_P2O5
-        Units.KG_PO43.value: P / ((P + _O*4)*3)  # Conv_Mol_P_PO43-
+        Units.KG_P2O5.value: (P*2) / (P*2 + _O*5),
+        Units.KG_PO43.value: P / ((P + _O*4)*3)
     },
     Units.KG_PO43.value: {
-        Units.KG_P2O5.value: ((P + _O*4)*3) / (P*2 + _O*5)  # Conv_Mol_PO43-_P2O5
+        Units.KG_P2O5.value: ((P + _O*4)*3) / (P*2 + _O*5)
+    },
+    Units.KG_P2O5.value: {
+        Units.KG_P.value: (P*2 + _O*5) / (P*2)
     },
     Units.KG_K.value: {
-        Units.KG_K2O.value: (K*2) / (K*2 + _O)  # Conv_Mol_K_K2O
+        Units.KG_K2O.value: (K*2) / (K*2 + _O)
     },
     Units.KG_CA.value: {
-        Units.KG_CAO.value: CA / (CA + _O)  # Conv_Mol_Ca_CaO
+        Units.KG_CAO.value: CA / (CA + _O)
     },
     Units.KG_CAO.value: {
-        Units.KG_CACO3.value: (CA + _O) / (CA + C + _O*3)  # Conv_Mol_CaO_CaCO3
+        Units.KG_CACO3.value: (CA + _O) / (CA + C + _O*3)
     },
     Units.KG_CACO3.value: {
-        Units.KG_CO2.value: 0.12
+        Units.KG_CO2.value: (CA + C + _O*3) / C
     },
-    Units.KG_MGCO3.value: {
-        Units.KG_CO2.value: 0.13
+    Units.KG_CAMGCO32.value: {
+        Units.KG_CO2.value: (CA + MG + (C + _O*3)*2) / (C*2)
     },
     Units.KG_CH4.value: {
-        Units.TO_C.value: (C + H*4) / C  # Conv_Mol_CH4C_CH4
+        Units.TO_C.value: (C + H*4) / C
     },
     Units.KG_CO2.value: {
-        Units.TO_C.value: (C + _O*2) / C  # Conv_Mol_CO2C_CO2
+        Units.TO_C.value: (C + _O*2) / C
     },
     Units.KG_NOX.value: {
-        Units.TO_N.value: (N + _O) / N  # Conv_Mol_NON_NO
+        Units.TO_N.value: (N + _O) / N
     },
     Units.KG_N2.value: {
         Units.TO_N.value: 1
     },
     Units.KG_N2O.value: {
-        Units.TO_N.value: (N*2 + _O) / (N*2)  # Conv_Mol_N2ON_N2O
+        Units.TO_N.value: (N*2 + _O) / (N*2)
     },
     Units.KG_NO2.value: {
-        Units.TO_N.value: (N + _O*2) / N  # Conv_Mol_NO2N_NO2
+        Units.TO_N.value: (N + _O*2) / N
     },
     Units.KG_NO3.value: {
-        Units.TO_N.value: (N + _O*3) / N  # Conv_Mol_NO3N_NO3
+        Units.TO_N.value: (N + _O*3) / N
     },
     Units.KG_NH3.value: {
-        Units.TO_N.value: (N + H*3) / N  # Conv_Mol_NH3N_NH3
+        Units.TO_N.value: (N + H*3) / N
     },
     Units.KG_NH4.value: {
-        Units.TO_N.value: (N + H*4) / N  # Conv_Mol_NH4N_NH4
+        Units.TO_N.value: (N + H*4) / N
     },
     Units.KW_H.value: {
         Units.MJ.value: 3.6
@@ -113,8 +117,10 @@ def get_atomic_conversion(src_unit: Units, dest_unit: Units, default_value=1):
     return ATOMIC_WEIGHT_CONVERSIONS.get(src_key, {}).get(dest_key, default_value)
 
 
-def convert_to_unit(node: dict, dest_unit: Units):
-    return list_sum(node.get('value', [])) * get_atomic_conversion(node.get('term', {}).get('units'), dest_unit)
+def convert_to_unit(node: dict, dest_unit: Units, default_value=None):
+    value = list_sum(node.get('value', []), default=None)
+    conversion = get_atomic_conversion(node.get('term', {}).get('units'), dest_unit, default_value=None)
+    return value / conversion if all([value is not None, conversion is not None]) else default_value
 
 
 def convert_to_N(node: dict):

hestia_earth/models/utils/product.py

@@ -166,6 +166,12 @@ PRODUCT_UNITS_CONVERSIONS = {
         ],
         Units.KG_VS.value: [
             ('volatileSolidsContent', True)
+        ],
+        Units.KG_P.value: [
+            ('phosphorusContentAsP', True)
+        ],
+        Units.KG_P2O5.value: [
+            ('phosphateContentAsP2O5', True)
         ]
     },
     Units.KG_N.value: {
@@ -175,6 +181,14 @@ PRODUCT_UNITS_CONVERSIONS = {
         Units.KG_VS.value: [
             ('nitrogenContent', False),
             ('volatileSolidsContent', True)
+        ],
+        Units.KG_P.value: [
+            ('nitrogenContent', False),
+            ('phosphorusContentAsP', True)
+        ],
+        Units.KG_P2O5.value: [
+            ('nitrogenContent', False),
+            ('phosphateContentAsP2O5', True)
         ]
     },
     Units.KG_VS.value: {
@@ -215,6 +229,12 @@ PRODUCT_UNITS_CONVERSIONS = {
                 ],
                 True
             )
+        ],
+        Units.KG_P.value: [
+            ('phosphorusContentAsP', True)
+        ],
+        Units.KG_P2O5.value: [
+            ('phosphateContentAsP2O5', True)
         ]
     },
     Units.KG_COLD_CARCASS_WEIGHT.value: {
@@ -225,6 +245,12 @@ PRODUCT_UNITS_CONVERSIONS = {
         Units.KG_COLD_CARCASS_WEIGHT.value: [],
         Units.KG_READY_TO_COOK_WEIGHT.value: [
             ('processingConversionColdCarcassWeightToReadyToCookWeight', True)
+        ],
+        Units.KG_P.value: [
+            ('phosphorusContentAsP', True)
+        ],
+        Units.KG_P2O5.value: [
+            ('phosphateContentAsP2O5', True)
         ]
     },
     Units.KG_COLD_DRESSED_CARCASS_WEIGHT.value: {
@@ -237,6 +263,12 @@ PRODUCT_UNITS_CONVERSIONS = {
         Units.KG_COLD_CARCASS_WEIGHT.value: [],
         Units.KG_READY_TO_COOK_WEIGHT.value: [
             ('processingConversionColdDressedCarcassWeightToReadyToCookWeight', True)
+        ],
+        Units.KG_P.value: [
+            ('phosphorusContentAsP', True)
+        ],
+        Units.KG_P2O5.value: [
+            ('phosphateContentAsP2O5', True)
         ]
     },
     Units.KG_READY_TO_COOK_WEIGHT.value: {
@@ -262,7 +294,13 @@ PRODUCT_UNITS_CONVERSIONS = {
         Units.KG_COLD_DRESSED_CARCASS_WEIGHT.value: [
             ('processingConversionColdDressedCarcassWeightToReadyToCookWeight', False)
         ],
-        Units.KG_READY_TO_COOK_WEIGHT.value: []
+        Units.KG_READY_TO_COOK_WEIGHT.value: [],
+        Units.KG_P.value: [
+            ('phosphorusContentAsP', True)
+        ],
+        Units.KG_P2O5.value: [
+            ('phosphateContentAsP2O5', True)
+        ]
     },
     Units.HEAD.value: {
         Units.KG_LIVEWEIGHT.value: [

hestia_earth/models/utils/property.py

@@ -50,7 +50,12 @@ def find_term_property(term, property: str, default=None) -> dict:
     return find_term_match(props, property, default)
 
 
-def get_node_property(node: dict, property: str, find_default_property: bool = True) -> dict:
+def get_node_property(
+    node: dict,
+    property: str,
+    find_default_property: bool = True,
+    download_from_hestia: bool = False
+) -> dict:
     """
     Get the property by `@id` linked to the Blank Node in the glossary.
 
@@ -66,6 +71,8 @@ def get_node_property(node: dict, property: str, find_default_property: bool = T
         The `term.@id` of the property. Example: `nitrogenContent`.
     find_default_property : bool
         Default to fetching the property from the `defaultProperties` of the `Term`.
+    download_from_hestia : bool
+        Default to downloading the Term from HESTIA.
 
     Returns
     -------
@@ -76,7 +83,9 @@ def get_node_property(node: dict, property: str, find_default_property: bool = T
     return find_term_property(node.get('term', {}), property, {}) if all([
         find_default_property,
         prop is None
-    ]) else (prop or {})
+    ]) else (
+        prop or ({'term': download_term(property, TermTermType.PROPERTY)} if download_from_hestia else {})
+    )
 
 
 def node_has_no_property(term_id: str):
@@ -91,9 +100,8 @@ def node_property_lookup_value(model: str, node_term: dict, prop_id: str, defaul
     # as the lookup table might not exist, we are making sure we return `0` in thise case
     try:
         lookup_name = f"{node_term.get('termType')}-property.csv"
-        lookup = download_lookup(lookup_name)
         term_id = node_term.get('@id')
-        lookup_value = get_table_value(lookup, 'termid', term_id, column_name(prop_id))
+        lookup_value = get_table_value(download_lookup(lookup_name), 'termid', term_id, column_name(prop_id))
         value = extract_grouped_data(lookup_value, 'Avg') if (
             isinstance(lookup_value, str) and 'Avg' in lookup_value
         ) else lookup_value
@@ -104,8 +112,8 @@ def node_property_lookup_value(model: str, node_term: dict, prop_id: str, defaul
 
 
 def get_node_property_value(model: str, node: dict, prop_id: str, default=None, handle_percents=True, **log_args):
-    prop = get_node_property(node, prop_id)
-    term = (prop or {}).get('term') or download_term(prop_id, TermTermType.PROPERTY)
+    prop = get_node_property(node, prop_id, download_from_hestia=True)
+    term = (prop or {}).get('term')
     units = (term or {}).get('units')
     value = prop.get('value') if prop else node_property_lookup_value(model, node.get('term', {}), prop_id, **log_args)
     return default if value is None else (value / 100 if units == '%' and handle_percents else value)

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.70.1'
+VERSION = '0.70.2'

{hestia_earth_models-0.70.1.dist-info → hestia_earth_models-0.70.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.70.1
+Version: 0.70.2
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team