hestia-earth-models 0.61.7__py3-none-any.whl → 0.62.0__py3-none-any.whl

hestia_earth/models/site/organicCarbonPerHa.py

@@ -3,10 +3,10 @@ from typing import Optional, Union
 from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.utils.date import diff_in_days
 from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.tools import flatten, safe_parse_float
+from hestia_earth.utils.tools import flatten, non_empty_list, safe_parse_float
 
 from hestia_earth.models.log import log_as_table, logRequirements, logShouldRun
-from hestia_earth.models.utils.blank_node import _get_last_date, group_nodes_by_last_date
+from hestia_earth.models.utils.blank_node import _get_last_date, group_nodes_by_last_date, node_term_match
 from hestia_earth.models.utils.measurement import (
     _new_measurement, group_measurements_by_depth, measurement_value, OLDEST_DATE
 )
@@ -21,14 +21,17 @@ REQUIREMENTS = {
                 "value": "",
                 "term.@id": "soilBulkDensity",
                 "depthUpper": "",
-                "depthLower": ""
+                "depthLower": "",
+                "methodClassification": ["on-site physical measurement", "modelled using other measurements"]
             },
             {
                 "@type": "Measurement",
                 "value": "",
+                "dates": "",
                 "term.@id": "organicCarbonPerKgSoil",
                 "depthUpper": "",
-                "depthLower": ""
+                "depthLower": "",
+                "methodClassification": ["on-site physical measurement", "modelled using other measurements"]
             }
         ]
     }
@@ -36,6 +39,7 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
+        "dates": "",
         "depthUpper": "",
         "depthLower": "",
         "methodClassification": "modelled using other measurements"
@@ -51,6 +55,10 @@ RESCALE_DEPTH_LOWER = 30
 MAX_DEPTH_LOWER = 100
 SOIL_BULK_DENSITY_TERM_ID = 'soilBulkDensity'
 ORGANIC_CARBON_PER_KG_SOIL_TERM_ID = 'organicCarbonPerKgSoil'
+VALID_MEASUREMENT_METHOD_CLASSIFICATIONS = {
+    MeasurementMethodClassification.ON_SITE_PHYSICAL_MEASUREMENT.value,
+    MeasurementMethodClassification.MODELLED_USING_OTHER_MEASUREMENTS.value
+}
 
 
 def _measurement(
@@ -98,38 +106,40 @@ def _calc_organic_carbon_per_ha(
     return (depth_lower - depth_upper) * soil_bulk_density * organic_carbon_per_kg_soil * 100
 
 
-def _should_run_calculation_group(nodes: list) -> bool:
+def _should_run_calculation(site: dict) -> tuple[bool, dict[str, list[dict]]]:
     """
-    Determines whether a depth interval group has sufficient data to calculate `organicCarbonPerHa` from
-    `soilBulkDensity` and `organicCarbonPerKgSoil`.
+    Pre-process site data and determine whether there is sufficient data to calculate `organicCarbonPerHa`.
     """
-    soilBulkDensity = find_term_match(nodes, SOIL_BULK_DENSITY_TERM_ID, None)
-    has_soil_bulk_density_depth_lower = (soilBulkDensity or {}).get('depthLower') is not None
-    has_soil_bulk_density_depth_upper = (soilBulkDensity or {}).get('depthUpper') is not None
+    oc_nodes = [node for node in site.get("measurements", []) if _valid_measurement(node, TERM_ID)]
 
-    organicCarbonPerKgSoil = find_term_match(nodes, ORGANIC_CARBON_PER_KG_SOIL_TERM_ID, None)
-    has_organic_carbon_per_kg_soil_depth_lower = (organicCarbonPerKgSoil or {}).get('depthLower') is not None
-    has_organic_carbon_per_kg_soil_depth_upper = (organicCarbonPerKgSoil or {}).get('depthUpper') is not None
+    # We don't need to run the model for any dates we already have an `organicCarbonPerHa` value for.
+    oc_node_dates = set(non_empty_list(flatten(measurement.get("dates", []) for measurement in oc_nodes)))
 
-    return all([
-        has_soil_bulk_density_depth_lower,
-        has_soil_bulk_density_depth_upper,
-        has_organic_carbon_per_kg_soil_depth_lower,
-        has_organic_carbon_per_kg_soil_depth_upper
-    ])
+    occ_nodes = [
+        node for node in site.get("measurements", [])
+        if all([
+            _valid_measurement(node, ORGANIC_CARBON_PER_KG_SOIL_TERM_ID),
+            len(node.get("dates", [])) > 0,
+            _get_last_date(node.get("dates", [])) not in oc_node_dates
+        ])
+    ]
 
+    bd_nodes = [node for node in site.get("measurements", []) if _valid_measurement(node, SOIL_BULK_DENSITY_TERM_ID)]
 
-def _should_run_calculation(site: dict) -> tuple[bool, dict[str, list[dict]]]:
-    """
-    Pre-process site data and determine whether there is sufficient data to calculate `organicCarbonPerHa`.
-    """
-    grouped_measurements = {
-        depth_key: nodes for depth_key, nodes in group_measurements_by_depth(site.get('measurements', [])).items()
+    measurements = occ_nodes + bd_nodes
+    grouped_measurements = group_measurements_by_depth(measurements, include_dates=False)
+
+    inventory = {
+        depth_key: {
+            "measurements": nodes,
+            "has-soil-bulk-density": bool(find_term_match(nodes, SOIL_BULK_DENSITY_TERM_ID)),
+            "has-organic-carbon-per-kg-soil": bool(find_term_match(nodes, SOIL_BULK_DENSITY_TERM_ID))
+        } for depth_key, nodes in grouped_measurements.items()
     }
 
     valid_grouped_measurements = {
-        depth_key: nodes for depth_key, nodes in grouped_measurements.items()
-        if _should_run_calculation_group(nodes)
+        depth_key: group["measurements"] for depth_key, group in inventory.items()
+        if all([group["has-soil-bulk-density"], group["has-organic-carbon-per-kg-soil"]])
     }
 
     should_run = bool(valid_grouped_measurements)
@@ -139,20 +149,24 @@ def _should_run_calculation(site: dict) -> tuple[bool, dict[str, list[dict]]]:
         "inventory_calculation": log_as_table(
             {
                 "depth-key": str(depth_key).replace("_", "-"),
-                "should-run": depth_key in valid_grouped_measurements.keys(),
-                "has-soil-bulk-density": (
-                    find_term_match(nodes, SOIL_BULK_DENSITY_TERM_ID, {}).get('depthLower')
-                    and find_term_match(nodes, SOIL_BULK_DENSITY_TERM_ID, {}).get('depthUpper')
-                ),
-                "has-organic-carbon-per-kg-soil": (
-                    find_term_match(nodes, ORGANIC_CARBON_PER_KG_SOIL_TERM_ID, {}).get('depthLower')
-                    and find_term_match(nodes, ORGANIC_CARBON_PER_KG_SOIL_TERM_ID, {}).get('depthUpper')
-                )
-            } for depth_key, nodes in grouped_measurements.items()
-        ) or None
+                "should-run": depth_key in valid_grouped_measurements,
+                "has-soil-bulk-density": group["has-soil-bulk-density"],
+                "has-organic-carbon-per-kg-soil": group["has-organic-carbon-per-kg-soil"]
+            } for depth_key, group in inventory.items()
+        ) if inventory else "None"
     }
 
-    return should_run, logs, valid_grouped_measurements
+    return should_run, valid_grouped_measurements, logs
+
+
+def _valid_measurement(node: dict, target_term_id: str) -> bool:
+    return all([
+        node_term_match(node, target_term_id),
+        node.get("value"),
+        node.get("depthLower") is not None,
+        node.get("depthUpper") is not None,
+        node.get("methodClassification") in VALID_MEASUREMENT_METHOD_CLASSIFICATIONS
+    ])
 
 
 def _run_calculation(site: dict, depth_key: str, measurement_nodes: list[dict]) -> list[dict]:
@@ -332,11 +346,11 @@ def _should_run_rescale(organic_carbon_per_ha_nodes: list) -> tuple[bool, dict[s
             {
                 "date": str(datestr),
                 "should-run": datestr in valid_grouped_nodes.keys()
-            } for datestr, nodes in grouped_nodes.items()
-        ) or None
+            } for datestr in grouped_nodes.keys()
+        ) if grouped_nodes else "None"
     }
 
-    return should_run, logs, valid_grouped_nodes
+    return should_run, valid_grouped_nodes, logs
 
 
 def _depth_distance(node: dict):
@@ -377,7 +391,7 @@ def _run_rescale(site: dict, organic_carbon_per_ha_nodes: list[dict]) -> list[di
 
 
 def run(site: dict):
-    should_run_calculation, logs_calculation, grouped_measurements = _should_run_calculation(site)
+    should_run_calculation, grouped_measurements, logs_calculation = _should_run_calculation(site)
     result_calculation = (
         flatten([_run_calculation(site, depth_key, nodes) for depth_key, nodes in grouped_measurements.items()])
         if should_run_calculation else []
@@ -387,7 +401,7 @@ def run(site: dict):
         result_calculation + [m for m in site.get('measurements', []) if m.get('term', {}).get('@id') == TERM_ID]
     )
 
-    should_run_rescale, logs_rescale, grouped_oc_per_ha_nodes = _should_run_rescale(oc_per_ha_nodes)
+    should_run_rescale, grouped_oc_per_ha_nodes, logs_rescale = _should_run_rescale(oc_per_ha_nodes)
     result_rescale = (
         [_run_rescale(site, nodes) for nodes in grouped_oc_per_ha_nodes.values()]
         if should_run_rescale else []

hestia_earth/models/site/soilMeasurement.py

@@ -8,7 +8,7 @@ from copy import deepcopy
 from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.utils.tools import non_empty_list, flatten
 
-from hestia_earth.models.log import logRequirements, logShouldRun, logErrorRun
+from hestia_earth.models.log import logRequirements, logShouldRun, logErrorRun, log_as_table
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.term import get_lookup_value
 from . import MODEL
@@ -20,7 +20,6 @@ REQUIREMENTS = {
         ]
     }
 }
-
 RETURNS = {
     "Measurement": [{
         "value": "",
@@ -30,11 +29,9 @@ RETURNS = {
         "methodClassification": "modelled using other measurements"
     }]
 }
-
 LOOKUPS = {
-    "measurement": ["recommendAddingDepth", "depthSensitive"]
+    "measurement": "depthSensitive"
 }
-
 MODEL_KEY = 'soilMeasurement'
 STANDARD_DEPTHS = {(0, 30), (0, 50)}
 
@@ -154,17 +151,25 @@ def _get_depths_from_measurements(measurements: list) -> list:
     return needed_depths
 
 
-def _should_run(site: dict, model_key: str):
+def _should_run(site: dict):
     # we only work with measurements with depths
-    measurements = [m for m in site.get("measurements", []) if all([
-        get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][0], model=MODEL, model_key=model_key),
-        m.get('value', [])
+    measurements = site.get("measurements", [])
+    measurement_sensitivity = {
+        m.get('term', {}).get('@id'): get_lookup_value(
+            m.get('term', {}), LOOKUPS["measurement"], model=MODEL, model_key=MODEL_KEY
+        )
+        for m in measurements
+    }
+    measurements_with_depths = [m for m in measurements if all([
+        not measurement_sensitivity[m.get("term", {}).get('@id')],
+        m.get('value', []),
+        "depthUpper" in m,
+        "depthLower" in m
     ])]
-
-    measurements_with_depths = [m for m in measurements if "depthUpper" in m and "depthLower" in m]
     has_measurements_with_depths = len(measurements_with_depths) > 0
 
-    logRequirements(site, model=MODEL, model_key=model_key,
+    logRequirements(site, model=MODEL, model_key=MODEL_KEY,
+                    measurements_depth_sensitive=log_as_table(measurement_sensitivity),
                     has_measurements_with_depths=has_measurements_with_depths)
 
     should_run = has_measurements_with_depths
@@ -175,5 +180,5 @@ def _should_run(site: dict, model_key: str):
 
 
 def run(site: dict):
-    should_run, measurements_with_depths = _should_run(site=site, model_key=MODEL_KEY)
+    should_run, measurements_with_depths = _should_run(site)
     return non_empty_list(flatten(_run_harmonisation(measurements=measurements_with_depths))) if should_run else []

hestia_earth/models/utils/__init__.py

@@ -1,4 +1,6 @@
 from os.path import dirname, abspath
+from collections.abc import Generator, Iterable
+from itertools import tee
 import sys
 import datetime
 from functools import reduce
@@ -7,6 +9,7 @@ from typing import Any, Union
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import linked_node
+from hestia_earth.utils.tools import flatten, non_empty_list
 
 from .constant import Units
 
@@ -137,3 +140,28 @@ def last_day_of_month(year: int, month: int):
     return datetime.date(int(year), 12, 31) if month == 12 else (
         datetime.date(int(year) + int(int(month) / 12), (int(month) % 12) + 1, 1) - datetime.timedelta(days=1)
     )
+
+
+def flatten_args(args) -> list:
+    """
+    Flatten the input args into a single list.
+    """
+    return non_empty_list(flatten([list(arg) if is_iterable(arg) else [arg] for arg in args]))
+
+
+def is_iterable(arg) -> bool:
+    """
+    Return `True` if the input arg is an instance of an `Iterable` (excluding `str` and `bytes`) or a `Generator`, else
+    return `False`.
+    """
+    return isinstance(arg, (Iterable, Generator)) and not isinstance(arg, (str, bytes))
+
+
+def pairwise(iterable):
+    """
+    from https://docs.python.org/3.9/library/itertools.html#itertools-recipes
+    s -> (s0,s1), (s1,s2), (s2, s3), ...
+    """
+    a, b = tee(iterable)
+    next(b, None)
+    return zip(a, b)