hestia-earth-models 0.73.3__tar.gz → 0.73.5__tar.gz

{hestia_earth_models-0.73.3 → hestia_earth_models-0.73.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_models
-Version: 0.73.3
+Version: 0.73.5
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

{hestia_earth_models-0.73.3 → hestia_earth_models-0.73.5}/hestia_earth/models/akagiEtAl2011/utils.py

@@ -21,27 +21,26 @@ def _emission(term_id: str, value: float, sd: float = None):
     return emission
 
 
-def _run(term_id: str, product_value: list):
-    value = sum(product_value)
-    term = {'termType': TermTermType.EMISSION.value, '@id': term_id}
-    factor = get_lookup_value(term, LOOKUP_NAME)
-    factor_sd = get_lookup_value(term, LOOKUP_NAME + '-sd')
-    emission_value = multiply_values([value, factor])
-    return [] if emission_value is None else [_emission(term_id, emission_value, multiply_values([value, factor_sd]))]
+def _run(term_id: str, value: float, factor: float, factor_sd: float = None):
+    return [_emission(term_id, value * factor, multiply_values([value, factor_sd]))]
 
 
 def _should_run(term_id: str, cycle: dict):
     crop_residue_burnt_value = get_crop_residue_burnt_value(cycle)
-    has_crop_residue_burnt = len(crop_residue_burnt_value) > 0
+    term = {'termType': TermTermType.EMISSION.value, '@id': term_id}
+    factor = get_lookup_value(term, LOOKUP_NAME)
+    factor_sd = get_lookup_value(term, LOOKUP_NAME + '-sd')
 
     logRequirements(cycle, model=MODEL, term=term_id,
-                    has_crop_residue_burnt=has_crop_residue_burnt)
+                    crop_residue_burnt_value=crop_residue_burnt_value,
+                    burning_factor=factor,
+                    burning_factor_sd=factor_sd)
 
-    should_run = all([has_crop_residue_burnt])
+    should_run = all([crop_residue_burnt_value is not None, factor is not None])
     logShouldRun(cycle, MODEL, term_id, should_run, methodTier=TIER)
-    return should_run, crop_residue_burnt_value
+    return should_run, crop_residue_burnt_value, factor, factor_sd
 
 
 def run_emission(term_id: str, cycle: dict):
-    should_run, crop_residue_burnt_value = _should_run(term_id, cycle)
-    return _run(term_id, crop_residue_burnt_value) if should_run else []
+    should_run, value, factor, factor_sd = _should_run(term_id, cycle)
+    return _run(term_id, value, factor, factor_sd) if should_run else []

{hestia_earth_models-0.73.3 → hestia_earth_models-0.73.5}/hestia_earth/models/config/Site.json

@@ -523,6 +523,17 @@
           "replaceThreshold": ["value", 0.01]
         },
         "stage": 2
+      },
+      {
+        "key": "measurements",
+        "model": "ipcc2019",
+        "value": "biocharOrganicCarbonPerHa",
+        "runStrategy": "always",
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
       }
     ]
   ]

{hestia_earth_models-0.73.3 → hestia_earth_models-0.73.5}/hestia_earth/models/hestia/landCover.py

@@ -64,6 +64,11 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
+    "region-crop-cropGroupingFAOSTAT-landCover-annualCropland": "",
+    "region-crop-cropGroupingFAOSTAT-landCover-forest": "",
+    "region-crop-cropGroupingFAOSTAT-landCover-otherLand": "",
+    "region-crop-cropGroupingFAOSTAT-landCover-permanentCropland": "",
+    "region-crop-cropGroupingFAOSTAT-landCover-permanentPasture": "",
     "region-crop-cropGroupingFaostatProduction-areaHarvestedUpTo20YearExpansion": "",
     "region-crop-cropGroupingFaostatProduction-areaHarvested": "",
     "region-faostatArea-UpTo20YearExpansion": "",
@@ -116,6 +121,36 @@ def _get_lookup_with_cache(lookup_term, column):
     )
 
 
+def get_landCover_lookups(country_id: str, end_year: int, product_name: str):
+    """
+    Attempts to get the pre-calculated values for the landCover model calculation.
+    Returns: {"Arable land": <value>, "Forest land": <value>, "Other land": <value>,
+              "Permanent crops": <value>, "Permanent meadows and pastures": <value>}
+    Missing values are returned as None.
+    """
+    return {
+        # Divide by 100 to match site_area ratios
+        suffix: value / 100 if value is not None else value
+        for suffix, value in
+        {
+            suffix: safe_parse_float(
+                value=extract_grouped_data(
+                    data=get_region_lookup_value(
+                        lookup_name=f"region-crop-cropGroupingFAOSTAT-landCover-{suffix}.csv",
+                        term_id=country_id,
+                        column=product_name,
+                        model=MODEL,
+                        key=MODEL_KEY
+                    ),
+                    key=str(end_year)
+                ),
+                default=None
+            )
+            for suffix in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
+        }.items()
+    }
+
+
 def _management(term_id: str, value: float, start_date: str, end_date: str):
     node = _new_management(term_id, MODEL)
     node['value'] = value
@@ -821,11 +856,21 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
         target_node=relevant_nodes[0]
     ) if relevant_nodes else None
 
-    should_run_nodes, site_area = _should_run_historical_land_use_change(
-        site=site,
-        nodes=relevant_nodes,
-        land_use_type=land_use_type
-    ) if all([land_use_type, has_no_prior_land_cover_data]) else (False, {})
+    landCover_from_lookups = get_landCover_lookups(
+        country_id=site.get("country", {}).get("@id"),
+        end_year=_get_year_from_landCover(relevant_nodes[0]),
+        product_name=relevant_nodes[0].get("term", {}).get("name", "")
+    ) if relevant_nodes else {}
+
+    should_run_nodes, site_area = (
+        (False, {}) if not all([land_use_type, has_no_prior_land_cover_data])
+        else (True, landCover_from_lookups) if landCover_from_lookups and all(landCover_from_lookups.values())
+        else _should_run_historical_land_use_change(
+            site=site,
+            nodes=relevant_nodes,
+            land_use_type=land_use_type
+        )
+    )
 
     logRequirements(site, model=MODEL, model_key=MODEL_KEY,
                     has_management_nodes=bool(relevant_nodes),
@@ -833,6 +878,7 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
                     allowed_land_use_types=';'.join(ALLOWED_LAND_USE_TYPES),
                     has_no_prior_land_cover_data=has_no_prior_land_cover_data,
                     management_nodes=log_as_table([_omit(n, ['term']) for n in relevant_nodes]),
+                    landCover_from_lookups=landCover_from_lookups,
                     should_run_nodes=should_run_nodes)
 
     should_run = all([land_use_type, has_no_prior_land_cover_data, should_run_nodes])

hestia_earth_models-0.73.5/hestia_earth/models/ipcc2019/biocharOrganicCarbonPerHa.py

@@ -0,0 +1,435 @@
+from functools import reduce
+import numpy as np
+from typing import Union
+
+from hestia_earth.schema import (
+    CycleFunctionalUnit, MeasurementMethodClassification, MeasurementStatsDefinition, SiteSiteType, TermTermType
+)
+
+from hestia_earth.utils.blank_node import get_node_value
+from hestia_earth.utils.descriptive_stats import calc_descriptive_stats
+from hestia_earth.utils.stats import gen_seed, truncated_normal_1d
+from hestia_earth.utils.tools import non_empty_list
+
+from hestia_earth.models.log import log_as_table, logRequirements, logShouldRun
+from hestia_earth.models.utils.blank_node import group_nodes_by_year, filter_list_term_type
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.property import get_node_property
+from hestia_earth.models.utils.site import related_cycles
+from hestia_earth.models.utils.term import get_lookup_value
+
+from .organicCarbonPerHa_utils import IpccSoilCategory
+from .organicCarbonPerHa_tier_1 import _assign_ipcc_soil_category
+
+from . import MODEL
+
+
+REQUIREMENTS = {
+    "Site": {
+        "siteType": ["cropland", "permanent pasture"],
+        "related": {
+            "Cycle": [{
+                "endDate": "",
+                "functionalUnit": "1 ha",
+                "optional": {
+                    "startDate": "",
+                    "inputs": [{
+                        "@type": "Input",
+                        "term.termType": "biochar"
+                    }]
+                }
+            }]
+        }
+    }
+}
+LOOKUPS = {
+    "biochar": [
+        "FRAC_OC_REMAINING_100_YEARS",
+        "FRAC_OC_REMAINING_100_YEARS_SD"
+    ]
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "sd": "",
+        "min": "",
+        "max": "",
+        "statsDefinition": "simulated",
+        "observations": "",
+        "dates": "",
+        "methodClassification": "tier 1 model"
+    }]
+}
+TERM_ID = 'biocharOrganicCarbonPerHa'
+
+_ITERATIONS = 1000
+_METHOD_CLASSIFICATION = MeasurementMethodClassification.TIER_1_MODEL.value
+_STATS_DEFINITION = MeasurementStatsDefinition.SIMULATED.value
+
+_VALID_SITE_TYPES = [SiteSiteType.CROPLAND.value, SiteSiteType.PERMANENT_PASTURE.value]
+_VALID_FUNCTIONAL_UNITS = [CycleFunctionalUnit._1_HA.value]
+
+
+def run(site: dict) -> list[dict]:
+    """
+    Run the model on a Site.
+
+    Parameters
+    ----------
+    site : dict
+        A valid HESTIA [Site](https://www.hestia.earth/schema/Site).
+
+    Returns
+    -------
+    list[dict]
+        A list of HESTIA [Measurement](https://www.hestia.earth/schema/Measurement) nodes with `term.termType` =
+        `aboveGroundBiomass`
+    """
+    should_run, inventory = _should_run(site)
+    return _run(inventory) if should_run else []
+
+
+def _should_run(site: dict) -> tuple[bool, dict]:
+    """
+    Extract and organise required data from the input [Site](https://www.hestia.earth/schema/Site) node and determine
+    whether the model should run.
+
+    Parameters
+    ----------
+    site : dict
+        A valid HESTIA [Site](https://www.hestia.earth/schema/Site).
+
+    Returns
+    -------
+    tuple[bool, dict, dict]
+        should_run, inventory
+    """
+    cycles = related_cycles(site)
+    site_type = site.get("siteType")
+    ipcc_soil_category = _assign_ipcc_soil_category(site.get("measurements", []))
+
+    has_cycles = len(cycles) > 0
+    has_valid_site_type = site_type in _VALID_SITE_TYPES
+    has_functional_unit_1_ha = all(cycle.get("functionalUnit") in _VALID_FUNCTIONAL_UNITS for cycle in cycles)
+    has_mineral_soils = ipcc_soil_category not in [IpccSoilCategory.ORGANIC_SOILS]
+
+    seed = gen_seed(site, MODEL, TERM_ID)
+    rng = np.random.default_rng(seed)
+
+    should_compile_inventory = all([
+        has_cycles,
+        has_valid_site_type,
+        has_functional_unit_1_ha,
+        has_mineral_soils
+    ])
+
+    inventory, logs = _compile_inventory(cycles, rng) if should_compile_inventory else ({}, {})
+
+    logRequirements(
+        site, model=MODEL, term=TERM_ID,
+        has_cycles=has_cycles,
+        site_type=site_type,
+        has_valid_site_type=has_valid_site_type,
+        has_functional_unit_1_ha=has_functional_unit_1_ha,
+        has_mineral_soils=has_mineral_soils,
+        ipcc_soil_category=ipcc_soil_category,
+        should_compile_inventory=should_compile_inventory,
+        seed=seed,
+        inventory=_format_inventory(inventory),
+        **_format_logs(logs)
+    )
+
+    should_run = all([
+        len(inventory) > 0  # are there any cycles?
+    ])
+
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return should_run, inventory
+
+
+def _compile_inventory(
+    cycles: list[dict],
+    rng: Union[int, np.random.Generator, None] = None
+) -> dict:
+    """
+    Build an annual inventory of model input data.
+
+    Parameters
+    ----------
+    land_cover_nodes : list[dict]
+        A list of HESTIA [Cycles](https://www.hestia.earth/schema/Cycle).
+    seed : int | random.Generator | None
+        The rng/seed for the random sampling of model parameters.
+
+    Returns
+    -------
+    dict
+        Annual inventory of model data.
+    """
+    COPY_FIELDS = ("startDate", "endDate")
+
+    cycle_data = {
+        cycle.get("@id"): {
+            "biochar_nodes": filter_list_term_type(cycle.get("inputs", []), TermTermType.BIOCHAR),
+            **{field: cycle.get(field) for field in COPY_FIELDS}
+        } for cycle in cycles
+    }
+
+    biochar_term_ids = sorted(reduce(
+        lambda result, data: result.union(_get_unique_term_ids(data.get("biochar_nodes", []))),
+        cycle_data.values(),
+        set()
+    ))
+
+    factor_cache = {
+        term_id: {
+            "oc_content": _sample_oc_content(term_id, rng),
+            "frac_remaining": _sample_frac_remaining(term_id, rng),
+        } for term_id in biochar_term_ids
+    }
+
+    total_oc = {
+        id: reduce(
+            lambda result, node: result + _calc_total_oc(node, factor_cache),
+            data.get("biochar_nodes", []),
+            0
+        ) for id, data in cycle_data.items()
+    }
+
+    grouped = group_nodes_by_year(
+        [
+            {
+                "total_oc": total_oc.get(id, 0),
+                **{field: data.get(field) for field in COPY_FIELDS}
+            } for id, data in cycle_data.items()
+        ],
+        include_spillovers=True
+    )
+
+    inventory = {
+        year: reduce(
+            lambda result, item: result + item.get("total_oc", 0) * item.get("fraction_of_node_duration", 0),
+            data,
+            0
+        ) for year, data in grouped.items()
+    }
+
+    logs = {
+        "factor_cache": factor_cache
+    }
+
+    return inventory, logs
+
+
+def _get_unique_term_ids(nodes: list[dict]) -> set[str]:
+    return set(node.get("term", {}).get("@id") for node in nodes)
+
+
+def _sample_oc_content(term_id: str, rng):
+    """
+    Get an array of random samples based on the default organic carbon content of a biochar term.
+    """
+    node = {"term": {"@id": term_id, "termType": TermTermType.BIOCHAR.value}}
+
+    oc_prop = get_node_property(node, "organicCarbonContent")
+    mu = oc_prop.get("value")
+    sigma = oc_prop.get("sd")
+
+    return truncated_normal_1d((1, _ITERATIONS), mu/100, sigma/100, 0, 1, seed=rng) if (mu and sigma) else 0
+
+
+def _sample_frac_remaining(term_id: str, rng):
+    """
+    Get an array of random samples based on the `FRAC_OC_REMAINING_100_YEARS` lookups of a biochar term.
+    """
+    term = {"@id": term_id, "termType": TermTermType.BIOCHAR.value}
+
+    mu = get_lookup_value(term, LOOKUPS["biochar"][0])
+    sigma = get_lookup_value(term, LOOKUPS["biochar"][1])
+
+    return truncated_normal_1d((1, _ITERATIONS), mu, sigma, 0, 1, seed=rng) if (mu and sigma) else 0
+
+
+def _calc_total_oc(biochar_node: dict, factor_cache: dict):
+    """
+    Calculate the total amount of stable organic carbon added to the soil from an application of biochar.
+    """
+    term_id = biochar_node.get("term", {}).get("@id")
+
+    mass = get_node_value(biochar_node)
+    oc_content = factor_cache.get(term_id, {}).get("oc_content", 0)
+    frac_remaining = factor_cache.get(term_id, {}).get("frac_remaining", 0)
+
+    return mass * oc_content * frac_remaining
+
+
+def _format_inventory(inventory: dict) -> str:
+    """
+    Format the biochar inventory for logging as a table.
+    """
+    inventory_years = sorted(set(non_empty_list(years for years in inventory.keys())))
+
+    should_run = inventory and len(inventory_years) > 0
+
+    return log_as_table(
+        {
+            "year": year,
+            "stable-oc-from-biochar": _format_factor(inventory.get(year))
+        } for year in inventory_years
+    ) if should_run else "None"
+
+
+def _format_logs(logs: dict):
+    """
+    Format model logs. Format method selected based on dict key, with `_format_str` as fallback.
+    """
+    return {
+        _format_str(key): _LOG_KEY_TO_FORMAT_FUNC.get(key, _format_str)(value) for key, value in logs.items()
+    }
+
+
+def _format_factor_cache(factor_cache: dict) -> str:
+    """
+    Format the SOC inventory for logging as a table.
+    """
+    should_run = factor_cache and len(factor_cache) > 0
+
+    return log_as_table(
+        {
+            "term-id": term_id,
+            **{_format_str(key): _format_factor(value) for key, value in factor_dict.items()}
+        } for term_id, factor_dict in factor_cache.items()
+    ) if should_run else "None"
+
+
+def _format_factor(value) -> str:
+    """
+    Format a model factor. Method selected based on factor type (ndarray, int or float).
+    """
+    format_func = next(
+        (func for type, func in _TYPE_TO_FORMAT_FUNC.items() if isinstance(value, type)),
+        None
+    )
+    return format_func(value) if format_func else "None"
+
+
+def _format_nd_array(value) -> str:
+    return f"{np.mean(value):.3g} ± {np.std(value):.3g}"
+
+
+def _format_number(value) -> str:
+    return f"{value:.3g}"
+
+
+_INVALID_CHARS = {"_", ":", ",", "="}
+_REPLACEMENT_CHAR = "-"
+
+
+def _format_str(value: str, *_) -> str:
+    """Format a string for logging in a table. Remove all characters used to render the table on the front end."""
+    return reduce(lambda x, char: x.replace(char, _REPLACEMENT_CHAR), _INVALID_CHARS, str(value))
+
+
+_TYPE_TO_FORMAT_FUNC = {
+    np.ndarray: _format_nd_array,
+    (float, int): _format_number
+}
+
+
+_LOG_KEY_TO_FORMAT_FUNC = {
+    "factor_cache": _format_factor_cache
+}
+
+
+def _run(
+    inventory: dict
+) -> list[dict]:
+    """
+    Calculate the annual biochar organic carbon stock based on an inventory of biochar application data.
+
+    Parameters
+    ----------
+    inventory : dict
+        The annual inventory of biochar data.
+
+    Returns
+    -------
+    list[dict]
+        A list of HESTIA [Measurement](https://www.hestia.earth/schema/Measurement) nodes with `term.@id` =
+        `biocharOrganicCarbonPerHa`
+    """
+
+    start_year = min(inventory.keys()) - 1
+    end_year = max(inventory.keys()) + 1
+
+    def accumulate_oc(result, year):
+        value = inventory.get(year, np.zeros((1, _ITERATIONS)))
+        prev = result.get(year-1, np.zeros((1, _ITERATIONS)))
+
+        updated = result | {year: value+prev}
+        return updated
+
+    accumlated_oc = reduce(accumulate_oc, range(start_year, end_year), {})
+
+    dates = [f"{year}-12-31" for year in accumlated_oc]
+    values = np.vstack(tuple(accumlated_oc.values()))
+
+    descriptive_stats = calc_descriptive_stats(
+        values,
+        _STATS_DEFINITION,
+        axis=1,     # Calculate stats rowwise.
+        decimals=6  # Round values to the nearest milligram.
+    )
+    return [_measurement(dates, **descriptive_stats)]
+
+
+def _measurement(
+    dates: list[int],
+    value: list[float],
+    sd: list[float] = None,
+    min: list[float] = None,
+    max: list[float] = None,
+    statsDefinition: str = None,
+    observations: list[int] = None
+) -> dict:
+    """
+    Build a Hestia `Measurement` node to contain a value and descriptive statistics calculated by the models.
+
+    Parameters
+    ----------
+    timestamps : list[int]
+        A list of calendar years associated to the calculated SOC stocks.
+    value : list[float]
+        A list of values representing the mean biomass stock for each year of the inventory
+    sd : list[float]
+        A list of standard deviations representing the standard deviation of the biomass stock for each year of the
+        inventory.
+    min : list[float]
+        A list of minimum values representing the minimum modelled biomass stock for each year of the inventory.
+    max : list[float]
+        A list of maximum values representing the maximum modelled biomass stock for each year of the inventory.
+    statsDefinition : str
+        The [statsDefinition](https://hestia.earth/schema/Measurement#statsDefinition) of the measurement.
+    observations : list[int]
+        The number of model iterations used to calculate the descriptive statistics.
+
+    Returns
+    -------
+    dict
+        A valid HESTIA `Measurement` node, see: https://www.hestia.earth/schema/Measurement.
+    """
+    update_dict = {
+        "value": value,
+        "sd": sd,
+        "min": min,
+        "max": max,
+        "statsDefinition": statsDefinition,
+        "observations": observations,
+        "dates": dates,
+        "methodClassification": _METHOD_CLASSIFICATION
+    }
+    measurement = _new_measurement(TERM_ID, MODEL) | {
+        key: value for key, value in update_dict.items() if value
+    }
+    return measurement

{hestia_earth_models-0.73.3 → hestia_earth_models-0.73.5}/hestia_earth/models/ipcc2019/n2OToAirCropResidueBurningDirect.py

@@ -38,25 +38,19 @@ def _emission(value: float):
     return emission
 
 
-def _run(product_value: list, factor: float):
-    value = sum(product_value)
-    return [_emission(value * factor)]
-
-
 def _should_run(cycle: dict):
     crop_residue_burnt_value = get_crop_residue_burnt_value(cycle)
-    has_crop_residue_burnt = len(crop_residue_burnt_value) > 0
     factor = get_lookup_value({'termType': 'emission', '@id': TERM_ID}, LOOKUPS['emission'][0])
 
     logRequirements(cycle, model=MODEL, term=TERM_ID,
-                    has_crop_residue_burnt=has_crop_residue_burnt,
+                    crop_residue_burnt_value=crop_residue_burnt_value,
                     burning_factor=factor)
 
-    should_run = all([has_crop_residue_burnt, factor is not None])
+    should_run = all([crop_residue_burnt_value is not None, factor is not None])
     logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
     return should_run, crop_residue_burnt_value, factor
 
 
 def run(cycle: dict):
-    should_run, crop_residue_burnt_value, factor = _should_run(cycle)
-    return _run(crop_residue_burnt_value, factor) if should_run else []
+    should_run, value, factor = _should_run(cycle)
+    return [_emission(value * factor)] if should_run else []