hestia-earth-models 0.58.0__py3-none-any.whl → 0.59.1__py3-none-any.whl

hestia_earth/models/site/organicCarbonPerHa.py

@@ -1,13 +1,16 @@
 from typing import Optional
 
 from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.date import diff_in_days
 from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.tools import flatten, safe_parse_float
 
-from hestia_earth.models.log import logRequirements, logShouldRun
-from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.log import log_as_table, logRequirements, logShouldRun
+from hestia_earth.models.utils.blank_node import _get_last_date, group_nodes_by_last_date
 from hestia_earth.models.utils.measurement import (
-    _new_measurement, group_measurements_by_depth, _group_measurement_key, measurement_value
+    _new_measurement, group_measurements_by_depth, measurement_value, OLDEST_DATE
 )
+from hestia_earth.models.utils.source import get_source
 from . import MODEL
 
 REQUIREMENTS = {
@@ -43,6 +46,12 @@ BIBLIO_TITLE = 'Soil organic carbon sequestration rates in vineyard agroecosyste
 RESCALE_DEPTH_UPPER = 0
 RESCALE_DEPTH_LOWER = 30
 
+# --- UTILS ---
+
+MAX_DEPTH_LOWER = 100
+SOIL_BULK_DENSITY_TERM_ID = 'soilBulkDensity'
+ORGANIC_CARBON_PER_KG_SOIL_TERM_ID = 'organicCarbonPerKgSoil'
+
 
 def _measurement(site: dict, value: float, depthUpper: int, depthLower: int, date: Optional[str] = None):
     data = _new_measurement(TERM_ID)
@@ -55,14 +64,156 @@ def _measurement(site: dict, value: float, depthUpper: int, depthLower: int, dat
     return data | get_source(site, BIBLIO_TITLE)
 
 
-def _should_run_rescale(measurements: list):
-    return any([m for m in measurements if all([
-            m.get('depthUpper', 1) == RESCALE_DEPTH_UPPER,
-            m.get('depthLower', 101) <= 100
-    ])]) and not any([m for m in measurements if all([
-            m.get('depthUpper', 1) == RESCALE_DEPTH_UPPER,
-            m.get('depthLower', 101) == RESCALE_DEPTH_LOWER
-    ])])
+# --- CALCULATE `organicCarbonPerHa` ---
+
+
+def _calc_organic_carbon_per_ha(
+    depth_upper: float,
+    depth_lower: float,
+    soil_bulk_density: float,
+    organic_carbon_per_kg_soil: float
+) -> float:
+    """
+    Calculate `organicCarbonPerHa` from `soilBulkDensity` and `organicCarbonPerKgSoil` using method adapted from
+    [Payen et al (2021)](https://doi.org/10.1016/j.jclepro.2020.125736).
+
+    Parameters
+    ----------
+    depth_upper : float
+        Measurement depth upper in centimetres (min `0`).
+    depth_lower : float,
+        Measurement depth upper in centimetres (min `0`).
+    soil_bulk_density : float,
+        Soil bulk density between depth upper and depth lower, Mg soil m-3
+    organic_carbon_per_kg_soil : float
+        Soil organic carbon concentration between depth upper and depth lower, kg C kg soil-1
+
+    Return
+    ------
+    float
+        The SOC stock per hectare within the specified depth interval, kg C ha-1.
+    """
+    return (depth_lower - depth_upper) * soil_bulk_density * organic_carbon_per_kg_soil * 100
+
+
+def _should_run_calculation_group(nodes: list) -> bool:
+    """
+    Determines whether a depth interval group has sufficient data to calculate `organicCarbonPerHa` from
+    `soilBulkDensity` and `organicCarbonPerKgSoil`.
+    """
+    soilBulkDensity = find_term_match(nodes, SOIL_BULK_DENSITY_TERM_ID, None)
+    has_soil_bulk_density_depth_lower = (soilBulkDensity or {}).get('depthLower') is not None
+    has_soil_bulk_density_depth_upper = (soilBulkDensity or {}).get('depthUpper') is not None
+
+    organicCarbonPerKgSoil = find_term_match(nodes, ORGANIC_CARBON_PER_KG_SOIL_TERM_ID, None)
+    has_organic_carbon_per_kg_soil_depth_lower = (organicCarbonPerKgSoil or {}).get('depthLower') is not None
+    has_organic_carbon_per_kg_soil_depth_upper = (organicCarbonPerKgSoil or {}).get('depthUpper') is not None
+
+    return all([
+        has_soil_bulk_density_depth_lower,
+        has_soil_bulk_density_depth_upper,
+        has_organic_carbon_per_kg_soil_depth_lower,
+        has_organic_carbon_per_kg_soil_depth_upper
+    ])
+
+
+def _should_run_calculation(site: dict) -> tuple[bool, dict[str, list[dict]]]:
+    """
+    Pre-process site data and determine whether there is sufficient data to calculate `organicCarbonPerHa`.
+    """
+    grouped_measurements = {
+        depth_key: nodes for depth_key, nodes in group_measurements_by_depth(site.get('measurements', [])).items()
+    }
+
+    valid_grouped_measurements = {
+        depth_key: nodes for depth_key, nodes in grouped_measurements.items()
+        if _should_run_calculation_group(nodes)
+    }
+
+    should_run = bool(valid_grouped_measurements)
+
+    logs = {
+        "should_run_calculation": should_run,
+        "inventory_calculation": log_as_table(
+            {
+                "depth-key": str(depth_key).replace("_", "-"),
+                "should-run": depth_key in valid_grouped_measurements.keys(),
+                "has-soil-bulk-density": (
+                    find_term_match(nodes, SOIL_BULK_DENSITY_TERM_ID, {}).get('depthLower')
+                    and find_term_match(nodes, SOIL_BULK_DENSITY_TERM_ID, {}).get('depthUpper')
+                ),
+                "has-organic-carbon-per-kg-soil": (
+                    find_term_match(nodes, ORGANIC_CARBON_PER_KG_SOIL_TERM_ID, {}).get('depthLower')
+                    and find_term_match(nodes, ORGANIC_CARBON_PER_KG_SOIL_TERM_ID, {}).get('depthUpper')
+                )
+            } for depth_key, nodes in grouped_measurements.items()
+        ) or None
+    }
+
+    return should_run, logs, valid_grouped_measurements
+
+
+def _run_calculation(site: dict, depth_key: str, measurement_nodes: list[dict]) -> list[dict]:
+    """
+    Returns an `organicCarbonPerHa` measurement node for each `organicCarbonPerKgSoil` node in depth group using the
+    most relevant `soilBulkDensity` node available.
+
+    Parameters
+    ----------
+    site : dict
+        A [Site node](https://www.hestia.earth/schema/Site).
+    depth_key : str
+        A depth key in the format `"a_to_b"`.
+    measurement_nodes : list[dict]
+        A list of pre-validated [Measurement nodes](https://www.hestia.earth/schema/Measurement).
+
+    Return
+    ------
+    list[dict]
+        A list of `organicCarbonPerHa` [Measurement nodes](https://www.hestia.earth/schema/Measurement).
+    """
+    split = depth_key.split('_')  # "a_to_b"
+    depth_upper = safe_parse_float(split[0])
+    depth_lower = safe_parse_float(split[2])
+
+    soil_bulk_density_nodes = [
+        node for node in measurement_nodes if node.get('term', {}).get('@id') == SOIL_BULK_DENSITY_TERM_ID
+    ]
+
+    organic_carbon_per_kg_soil_nodes = [
+        node for node in measurement_nodes if node.get('term', {}).get('@id') == ORGANIC_CARBON_PER_KG_SOIL_TERM_ID
+    ]
+
+    dates = [_get_last_date(node.get('dates', [])) for node in organic_carbon_per_kg_soil_nodes]
+
+    def closest_bd(datestr: str):
+        """
+        Returns the `soilBulkDensity` node closest to target datestr. `nodes` input are pre-validated to always contain
+        at least one `soilBulkDensity` node.
+        """
+        return next(
+            iter(sorted(
+                soil_bulk_density_nodes,
+                key=lambda node: abs(diff_in_days(
+                    _get_last_date(node.get('dates', [])) or OLDEST_DATE,
+                    datestr or OLDEST_DATE
+                ))
+            ))
+        )
+
+    values = [
+        _calc_organic_carbon_per_ha(
+            depth_upper,
+            depth_lower,
+            measurement_value(closest_bd(datestr)),
+            measurement_value(organic_carbon_per_kg_soil_node)
+        ) for organic_carbon_per_kg_soil_node, datestr in zip(organic_carbon_per_kg_soil_nodes, dates)
+    ]
+
+    return [_measurement(site, value, depth_upper, depth_lower, datestr) for value, datestr in zip(values, dates)]
+
+
+# --- RESCALE `organicCarbonPerHa` ---
 
 
 def _c_to_depth(d: float) -> float:
@@ -72,7 +223,7 @@ def _c_to_depth(d: float) -> float:
     Parameters
     ----------
     d : float
-        Measurement depth in meters (min `0`, max `1`).
+        Measurement depth in metres (min `0`, max `1`).
 
     Returns
     -------
@@ -84,20 +235,19 @@ def _c_to_depth(d: float) -> float:
 
 def _cdf(depth_upper: float, depth_lower: float) -> float:
     """
-    The ratio between the carbon stock per m2 to depth `d` and the carbon
-    stock per meter to depth `1`.
+    The ratio between the carbon stock per m2 to depth `d` and the carbon stock per m2 to depth `1`.
 
     Parameters
     ----------
     depth_upper : float
-        Measurement depth upper in meters (min `0`, max `1`).
+        Measurement depth upper in metres (min `0`, max `1`).
     depth_lower : float
-        Measurement depth lower in meters (min `0`, max `1`).
+        Measurement depth lower in metres (min `0`, max `1`).
 
     Returns
     -------
     float
-        The proportion of carbon stored between `depth_upper` and `depth_lower` compared to between `0` and `1` meters.
+        The proportion of carbon stored between `depth_upper` and `depth_lower` compared to between `0` and `1` metres.
     """
     return (_c_to_depth(depth_lower) - _c_to_depth(depth_upper)) / _c_to_depth(1)
 
@@ -137,99 +287,111 @@ def _rescale_soc_value(
     return source_value * (cd_target / cd_measurement)
 
 
-def _get_last_date(dates: list[str]) -> str:
+def _should_run_rescale_node(node: list) -> bool:
     """
-    Reduces a node's dates field down to a single date. As the `arrayTreatment` for `organicCarbonPerKgSoil` and
-    `organicCarbonPerHa` is `mean` the latest date should be selected.
+    Validate that a node has `depthUpper` = `0` and a `depthLower` < `100`.
     """
-    return sorted(dates)[-1] if len(dates) > 0 else None
+    return all([
+        node.get('depthUpper', 1) == RESCALE_DEPTH_UPPER,
+        node.get('depthLower', 101) <= MAX_DEPTH_LOWER
+    ])
 
 
-def _run_rescale(site: dict, measurements: list):
-    measurements = [m for m in measurements if all([
-            m.get('depthUpper', 1) == RESCALE_DEPTH_UPPER,
-            m.get('depthLower', 101) <= 100
-    ])]
-    # order measurements by depthLower and use the biggest
-    measurement = sorted(measurements, key=lambda x: x.get('depthLower'))[-1] if len(measurements) > 0 else None
+def _should_run_rescale_group(nodes: list) -> bool:
+    """
+    Validate that a list of nodes doesn't contain a node with `depthUpper` = `0` and a `depthLower` < `30`.
+    """
+    return not any([
+        node for node in nodes if all([
+            node.get('depthUpper', 1) == RESCALE_DEPTH_UPPER,
+            node.get('depthLower', 101) == RESCALE_DEPTH_LOWER
+        ])
+    ])
 
-    value = _rescale_soc_value(
-        measurement_value(measurement),
-        RESCALE_DEPTH_UPPER, measurement.get('depthLower'),
-        RESCALE_DEPTH_UPPER, RESCALE_DEPTH_LOWER,
-    ) if measurement else None
 
-    date = _get_last_date(measurement.get("dates", [])) if measurement else None
+def _should_run_rescale(organic_carbon_per_ha_nodes: list) -> tuple[bool, dict[str, list[dict]]]:
+    """
+    Pre-process `organicCarbonPerHa` nodes and determine whether any need to be rescaled to a depth interval of 0-30cm.
+    """
+    grouped_nodes = group_nodes_by_last_date(
+        [node for node in organic_carbon_per_ha_nodes if _should_run_rescale_node(node)]
+    )
+
+    valid_grouped_nodes = {
+        datestr: nodes for datestr, nodes in grouped_nodes.items()
+        if _should_run_rescale_group(nodes)
+    }
 
-    return [_measurement(site, value, RESCALE_DEPTH_UPPER, RESCALE_DEPTH_LOWER, date)] if value is not None else []
+    should_run = bool(valid_grouped_nodes)
 
+    logs = {
+        "should_run_rescale": should_run,
+        "inventory_rescale": log_as_table(
+            {
+                "date": str(datestr),
+                "should-run": datestr in valid_grouped_nodes.keys()
+            } for datestr, nodes in grouped_nodes.items()
+        ) or None
+    }
 
-def _run(site: dict, measurements: list):
-    soilBulkDensity = measurement_value(find_term_match(measurements, 'soilBulkDensity'))
-    organicCarbonPerKgSoil = find_term_match(measurements, 'organicCarbonPerKgSoil')
-    organicCarbonPerKgSoil_value = measurement_value(organicCarbonPerKgSoil)
+    return should_run, logs, valid_grouped_nodes
 
-    value = (
-        organicCarbonPerKgSoil.get('depthLower') - organicCarbonPerKgSoil.get('depthUpper')
-    ) * soilBulkDensity * (organicCarbonPerKgSoil_value/10) * 1000
 
-    depthUpper = organicCarbonPerKgSoil.get('depthUpper')
-    depthLower = organicCarbonPerKgSoil.get('depthLower')
-    date = _get_last_date(organicCarbonPerKgSoil.get("dates", []))
+def _depth_distance(node: dict):
+    return abs(node.get('depthLower', 101) - RESCALE_DEPTH_LOWER)
 
-    return _measurement(site, value, depthUpper, depthLower, date)
 
+def _get_most_relevant_soc_node(organic_carbon_per_ha_nodes: list[dict]):
+    """
+    Find the `organic_carbon_per_ha_node` with the closest depth interval to 0 - 30cm. `depthLowers` greater than 30cm
+    are prioritised. Returns `{}` if input list is empty.
+    """
+    priority_nodes = [
+        node for node in organic_carbon_per_ha_nodes
+        if 'depthLower' in node and node.get('depthLower') >= RESCALE_DEPTH_LOWER
+    ]
+    nodes = priority_nodes or organic_carbon_per_ha_nodes  # If priority nodes are available use them.
 
-def _should_run_measurements(site: dict, measurements: list):
-    soilBulkDensity = find_term_match(measurements, 'soilBulkDensity', None)
-    has_soilBulkDensity_depthLower = (soilBulkDensity or {}).get('depthLower') is not None
-    has_soilBulkDensity_depthUpper = (soilBulkDensity or {}).get('depthUpper') is not None
-    organicCarbonPerKgSoil = find_term_match(measurements, 'organicCarbonPerKgSoil', None)
-    has_organicCarbonPerKgSoil_depthLower = (organicCarbonPerKgSoil or {}).get('depthLower') is not None
-    has_organicCarbonPerKgSoil_depthUpper = (organicCarbonPerKgSoil or {}).get('depthUpper') is not None
+    return next(node for node in nodes if _depth_distance(node) == min(_depth_distance(node) for node in nodes))
 
-    depth_logs = {
-        _group_measurement_key(measurements[0], include_dates=False): ';'.join([
-            '_'.join([
-                'id:soilBulkDensity',
-                f"hasDepthLower:{has_soilBulkDensity_depthLower}",
-                f"hasDepthUpper:{has_soilBulkDensity_depthUpper}"
-            ]),
-            '_'.join([
-                'id:organicCarbonPerKgSoil',
-                f"hasDepthLower:{has_organicCarbonPerKgSoil_depthLower}",
-                f"hasDepthUpper:{has_organicCarbonPerKgSoil_depthUpper}"
-            ])
-        ])
-    } if len(measurements) > 0 else {}
 
-    logRequirements(site, model=MODEL, term=TERM_ID,
-                    **depth_logs)
+def _run_rescale(site: dict, organic_carbon_per_ha_nodes: list[dict]) -> list[dict]:
+    """
+    For each unique measurement date, rescale the deepest `organicCarbonPerHa` node to a depth of 0 to 30cm.
+    """
+    node = _get_most_relevant_soc_node(organic_carbon_per_ha_nodes)
 
-    should_run = all([
-        has_soilBulkDensity_depthLower, has_soilBulkDensity_depthUpper,
-        has_organicCarbonPerKgSoil_depthLower, has_organicCarbonPerKgSoil_depthUpper
-    ])
-    return should_run
+    value = _rescale_soc_value(
+        measurement_value(node),
+        RESCALE_DEPTH_UPPER, node.get('depthLower'),
+        RESCALE_DEPTH_UPPER, RESCALE_DEPTH_LOWER,
+    ) if node else None
+    date = _get_last_date(node.get("dates", [])) if node else None
+
+    return _measurement(site, value, RESCALE_DEPTH_UPPER, RESCALE_DEPTH_LOWER, date) if value is not None else []
 
 
-def _should_run(site: dict):
-    grouped_measurements = list(group_measurements_by_depth(site.get('measurements', [])).values())
-    values = [(measurements, _should_run_measurements(site, measurements)) for measurements in grouped_measurements]
-    should_run = any([_should_run for measurements, _should_run in values])
-    logShouldRun(site, MODEL, TERM_ID, should_run)
-    return should_run, [measurements for measurements, _should_run in values if _should_run]
+# --- RUN MODEL ---
 
 
 def run(site: dict):
-    should_run, values = _should_run(site)
-    calculated_measurements = [_run(site, value) for value in values] if should_run else []
+    should_run_calculation, logs_calculation, grouped_measurements = _should_run_calculation(site)
+    result_calculation = (
+        flatten([_run_calculation(site, depth_key, nodes) for depth_key, nodes in grouped_measurements.items()])
+        if should_run_calculation else []
+    )
 
-    # rescale from existing and added measurements matching Term
-    all_measurements = [
-        m for m in (site.get('measurements', []) + calculated_measurements)
-        if m.get('term', {}).get('@id') == TERM_ID
-    ]
-    return calculated_measurements + (
-        _run_rescale(site, all_measurements) if _should_run_rescale(all_measurements) else []
+    oc_per_ha_nodes = (
+        result_calculation + [m for m in site.get('measurements', []) if m.get('term', {}).get('@id') == TERM_ID]
     )
+
+    should_run_rescale, logs_rescale, grouped_oc_per_ha_nodes = _should_run_rescale(oc_per_ha_nodes)
+    result_rescale = (
+        [_run_rescale(site, nodes) for nodes in grouped_oc_per_ha_nodes.values()]
+        if should_run_rescale else []
+    )
+
+    logRequirements(site, model=MODEL, term=TERM_ID, **logs_calculation, **logs_rescale)
+    logShouldRun(site, MODEL, TERM_ID, should_run=should_run_calculation or should_run_rescale)
+
+    return result_calculation + result_rescale