hestia-earth-models 0.70.0__py3-none-any.whl → 0.70.2__py3-none-any.whl

hestia_earth/models/ipcc2019/co2ToAirCarbonStockChange_utils.py

@@ -319,7 +319,13 @@ def _add_carbon_stock_change_emissions(
     value = emission_1.value + emission_2.value
     start_date = min(emission_1.start_date, emission_2.start_date)
     end_date = max(emission_1.end_date, emission_2.end_date)
-    method = min_emission_method_tier(emission_1.method, emission_2.method)
+
+    methods = [
+        method for emission in (emission_1, emission_2)
+        if isinstance((method := emission.method), EmissionMethodTier)
+    ]
+
+    method = min_emission_method_tier(*methods) if methods else None
 
     return CarbonStockChangeEmission(value, start_date, end_date, method)
 
@@ -442,7 +448,6 @@ def create_should_run_function(
 
         inventory, inventory_logs = (
             compile_inventory_func(
-                cycle_id,
                 cycles,
                 carbon_stock_measurements,
                 land_cover_nodes
@@ -464,7 +469,8 @@ def create_should_run_function(
         logs = should_compile_logs | inventory_logs | {
             "seed": seed,
             "has_valid_inventory": has_valid_inventory,
-            "has_consecutive_years": has_consecutive_years
+            "has_consecutive_years": has_consecutive_years,
+            "has_stock_measurements": bool(carbon_stock_measurements)
         }
 
         return should_run_, kwargs, logs
@@ -554,7 +560,6 @@ def _create_compile_inventory_function(
         The `compile_inventory` function.
     """
     def compile_inventory(
-        cycle_id: str,
         cycles: list[dict],
         carbon_stock_measurements: list[dict],
         land_cover_nodes: list[dict]
@@ -796,26 +801,8 @@ def _preprocess_carbon_stocks(
     list[CarbonStock]
         A list of carbon stocks sorted by date.
     """
-    sorted_measurements = sorted(
-        flatten([split_node_by_dates(m) for m in carbon_stock_measurements]),
-        key=lambda node: _gapfill_datestr(node["dates"][0], DatestrGapfillMode.END)
-    )
-
-    values = flatten(node["value"] for node in sorted_measurements)
-
-    sds = flatten(
-        node.get("sd", []) or [_calc_nominal_sd(v, _NOMINAL_ERROR) for v in node["value"]]
-        for node in sorted_measurements
-    )
-
-    dates = flatten(
-        [_gapfill_datestr(datestr, DatestrGapfillMode.END) for datestr in node["dates"]]
-        for node in sorted_measurements
-    )
-
-    methods = flatten(
-        [MeasurementMethodClassification(node.get("methodClassification")) for _ in node["value"]]
-        for node in sorted_measurements
+    dates, values, sds, methods = _extract_node_data(
+        flatten([split_node_by_dates(m) for m in carbon_stock_measurements])
     )
 
     correlation_matrix = compute_time_series_correlation_matrix(
@@ -842,6 +829,35 @@ def _preprocess_carbon_stocks(
     ]
 
 
+def _extract_node_data(nodes: list[dict]) -> list[dict]:
+
+    def group_node(result, node) -> dict[str, dict]:
+        date = _gapfill_datestr(node["dates"][0], DatestrGapfillMode.END)
+        result[date] = result.get(date, []) + [node]
+        return result
+
+    grouped_nodes = reduce(group_node, nodes, dict())
+
+    def get_values(date):
+        return flatten(node.get("value", []) for node in grouped_nodes[date])
+
+    def get_sds(date):
+        return flatten(
+            node.get("sd", []) or [_calc_nominal_sd(v, _NOMINAL_ERROR) for v in node.get("value", [])]
+            for node in grouped_nodes[date]
+        )
+
+    def get_methods(date):
+        return flatten(node.get("methodClassification", []) for node in grouped_nodes[date])
+
+    dates = sorted(grouped_nodes.keys())
+    values = [mean(get_values(date)) for date in dates]
+    sds = [mean(get_sds(date)) for date in dates]
+    methods = [min_measurement_method_classification(get_methods(date)) for date in dates]
+
+    return dates, values, sds, methods
+
+
 def _calc_nominal_sd(value: float, error: float) -> float:
     """
     Calculate a nominal SD for a carbon stock measurement. Can be used to gap fill SD when information not present in
@@ -1166,20 +1182,19 @@ def _squash_inventory(
         return _InventoryKey.CO2_EMISSION in carbon_stock_inventory.get(method, {}).get(year, {}).keys()
 
     def squash(result: dict, year: int) -> dict:
-        update_dict = next(
-            (
-                {
-                    year: {
-                        **_get_land_use_change_data(year, land_use_inventory),
-                        **reduce(merge, [
-                            carbon_stock_inventory.get(method, {}).get(year, {}),
-                            cycle_inventory.get(year, {})
-                        ], dict())
-                    }
-                } for method in measurement_method_ranking if should_run_group(method, year)
-            ),
-            {}
+        method = next(
+            (method for method in measurement_method_ranking if should_run_group(method, year)),
+            None
         )
+        update_dict = {
+            year: {
+                **_get_land_use_change_data(year, land_use_inventory),
+                **reduce(merge, [
+                    carbon_stock_inventory.get(method, {}).get(year, {}),
+                    cycle_inventory.get(year, {})
+                ], dict())
+            }
+        }
         return result | update_dict
 
     return reduce(squash, inventory_years, dict())
@@ -1415,36 +1430,31 @@ def create_run_function(
         Assign emissions to either the land use or management change term ids and sum together.
         """
         data = inventory[year]
+
         years_since_luc_event = data[_InventoryKey.YEARS_SINCE_LUC_EVENT]
         years_since_inventory_start = data[_InventoryKey.YEARS_SINCE_INVENTORY_START]
+        share_of_emission = data[_InventoryKey.SHARE_OF_EMISSION][cycle_id]
 
+        co2_emission = data.get(_InventoryKey.CO2_EMISSION)
+
+        has_co2_emission = bool(co2_emission)
         is_luc_emission = bool(years_since_luc_event) and years_since_luc_event <= _TRANSITION_PERIOD_YEARS
         is_data_complete = bool(years_since_inventory_start) and years_since_inventory_start >= _TRANSITION_PERIOD_YEARS
 
-        if is_luc_emission:
-            # If LUC emission allocate emissions to land use change AND add corresponding zero emission to management
-            emission_term_id = land_use_change_emission_term_id
-            zero_emission_term_id = management_change_emission_term_id
-        elif is_data_complete:
-            # If management emission && data complete allocate emissions to management change AND add corresponding
-            # zero emission to management
-            emission_term_id = management_change_emission_term_id
-            zero_emission_term_id = land_use_change_emission_term_id
-        else:
-            # If management emission, but not data complete allocate emissions to management change only
-            emission_term_id = management_change_emission_term_id
-            zero_emission_term_id = None
+        emission_term_id = (
+            land_use_change_emission_term_id if is_luc_emission else management_change_emission_term_id
+        ) if has_co2_emission else None
 
-        rescaled_emission = _rescale_carbon_stock_change_emission(
-            data[_InventoryKey.CO2_EMISSION], data[_InventoryKey.SHARE_OF_EMISSION][cycle_id]
+        zero_emission_term_id = (
+            management_change_emission_term_id if is_luc_emission else
+            (land_use_change_emission_term_id if is_data_complete else None)
         )
 
-        zero_emission = CarbonStockChangeEmission(
-            value=array(0),
-            start_date=rescaled_emission.start_date,
-            end_date=rescaled_emission.end_date,
-            method=rescaled_emission.method
-        ) if zero_emission_term_id else None
+        rescaled_emission = _rescale_carbon_stock_change_emission(
+            co2_emission, share_of_emission
+        ) if emission_term_id else None
+
+        zero_emission = get_zero_emission(year) if zero_emission_term_id else None
 
         previous_emission = result.get(emission_term_id)
         previous_zero_emission = result.get(zero_emission_term_id)
@@ -1454,7 +1464,7 @@ def create_run_function(
                 _add_carbon_stock_change_emissions(previous_emission, rescaled_emission) if previous_emission
                 else rescaled_emission
             )
-        }
+        } if emission_term_id else {}
 
         zero_emission_dict = {
             zero_emission_term_id: (
@@ -1497,7 +1507,7 @@ def create_run_function(
         return [
             new_emission_func(
                 term_id=emission_term_id,
-                method_tier=total_emission.method,
+                method_tier=_get_emission_method(total_emission),
                 start_date=_get_emission_start_date(total_emission, cycle_start_date),
                 end_date=_get_emission_end_date(total_emission, cycle_end_date),
                 **calc_descriptive_stats(
@@ -1506,18 +1516,33 @@ def create_run_function(
                     decimals=6
                 )
             ) for emission_term_id, total_emission in assigned_emissions.items()
+            if isinstance(total_emission, CarbonStockChangeEmission)
         ]
 
     return run
 
 
+def get_zero_emission(year):
+    return CarbonStockChangeEmission(
+        value=array(0),
+        start_date=_gapfill_datestr(year),
+        end_date=_gapfill_datestr(year, DatestrGapfillMode.END),
+        method=None
+    )
+
+
+def _get_emission_method(emission: CarbonStockChangeEmission):
+    method = emission.method
+    return method if isinstance(method, EmissionMethodTier) else EmissionMethodTier.TIER_1
+
+
 def _get_emission_start_date(emission: CarbonStockChangeEmission, cycle_start_date: str) -> str:
     cycle_datetime = safe_parse_date(_gapfill_datestr(cycle_start_date))
     emission_datetime = safe_parse_date(emission.start_date)
 
     should_run = (
         cycle_datetime and emission_datetime
-        and cycle_datetime <= emission_datetime  # If the cycle starts before the emission, add a `startDate`
+        and cycle_datetime < emission_datetime  # If the cycle starts before the emission, add a `startDate`
     )
 
     return (
@@ -1527,12 +1552,12 @@ def _get_emission_start_date(emission: CarbonStockChangeEmission, cycle_start_da
 
 
 def _get_emission_end_date(emission: CarbonStockChangeEmission, cycle_end_date: str) -> str:
-    cycle_datetime = safe_parse_date(_gapfill_datestr(cycle_end_date))
+    cycle_datetime = safe_parse_date(_gapfill_datestr(cycle_end_date, DatestrGapfillMode.END))
     emission_datetime = safe_parse_date(emission.end_date)
 
     should_run = (
         cycle_datetime and emission_datetime
-        and cycle_datetime >= emission_datetime  # If the cycle ends after the emission, add an `endDate`
+        and cycle_datetime > emission_datetime  # If the cycle ends after the emission, add an `endDate`
     )
 
     return (

hestia_earth/models/ipcc2019/co2ToAirLimeHydrolysis.py

@@ -1,5 +1,5 @@
 from hestia_earth.schema import EmissionMethodTier, TermTermType
-from hestia_earth.utils.tools import list_sum
+from hestia_earth.utils.tools import list_sum, non_empty_list
 from hestia_earth.utils.model import filter_list_term_type
 
 from hestia_earth.models.log import logRequirements, logShouldRun
@@ -16,7 +16,7 @@ REQUIREMENTS = {
             "@type": "Input",
             "value": "",
             "term.termType": "soilAmendment",
-            "term.units": ["kg CaCO3", "kg MgCO3"]
+            "term.units": ["kg CaCO3", "kg CaMg(CO3)2"]
         }]
     }
 }
@@ -38,8 +38,10 @@ def _emission(value: float):
 
 
 def _get_lime_values(cycle: dict, inputs: list):
-    # TODO: use lookup table
-    values = [convert_to_unit(i, Units.KG_CO2) for i in inputs if len(i.get('value', [])) > 0]
+    values = non_empty_list([
+        convert_to_unit(i, Units.KG_CO2)
+        for i in inputs if len(i.get('value', [])) > 0
+    ])
     return [0] if (
         len(values) == 0 and _is_term_type_complete(cycle, TermTermType.SOILAMENDMENT)
     ) else values
@@ -56,7 +58,7 @@ def _run(CaCO3_values: list, MgCO3_values: list):
 def _should_run(cycle: dict):
     inputs = filter_list_term_type(cycle.get('inputs', []), TermTermType.SOILAMENDMENT)
     CaCO3_values = _get_lime_values(cycle, _filter_list_term_unit(inputs, Units.KG_CACO3))
-    MgCO3_values = _get_lime_values(cycle, _filter_list_term_unit(inputs, Units.KG_MGCO3))
+    MgCO3_values = _get_lime_values(cycle, _filter_list_term_unit(inputs, Units.KG_CAMGCO32))
 
     logRequirements(cycle, model=MODEL, term=TERM_ID,
                     CaCO3_values=len(CaCO3_values),

hestia_earth/models/ipcc2019/co2ToAirOrganicSoilCultivation.py

@@ -0,0 +1,215 @@
+import numpy as np
+import numpy.typing as npt
+from typing import Callable, Union
+
+from hestia_earth.schema import EmissionMethodTier, EmissionStatsDefinition
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.array_builders import gen_seed, repeat_single, truncated_normal_1d
+from hestia_earth.models.utils.cycle import land_occupation_per_ha
+from hestia_earth.models.utils.descriptive_stats import calc_descriptive_stats
+from hestia_earth.models.utils.ecoClimateZone import EcoClimateZone, get_eco_climate_zone_value
+from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.measurement import most_relevant_measurement_value
+from hestia_earth.models.utils.site import valid_site_type
+
+from .organicSoilCultivation_utils import (
+    assign_organic_soil_category, calc_emission, get_emission_factor, OrganicSoilCategory, valid_eco_climate_zone
+)
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": [
+            {
+                "cycleDuration": "",
+                "practices": [{"@type": "Practice", "value": "", "term.@id": "longFallowRatio"}]
+            },
+            {
+                "@doc": "for plantations, additional properties are required",
+                "practices": [
+                    {"@type": "Practice", "value": "", "term.@id": "nurseryDensity"},
+                    {"@type": "Practice", "value": "", "term.@id": "nurseryDuration"},
+                    {"@type": "Practice", "value": "", "term.@id": "plantationProductiveLifespan"},
+                    {"@type": "Practice", "value": "", "term.@id": "plantationDensity"},
+                    {"@type": "Practice", "value": "", "term.@id": "plantationLifespan"},
+                    {"@type": "Practice", "value": "", "term.@id": "rotationDuration"}
+                ]
+            }
+        ],
+        "site": {
+            "@type": "Site",
+            "measurements": [
+                {"@type": "Measurement", "value": "", "term.@id": "histosol"},
+                {"@type": "Measurement", "value": "", "term.@id": "ecoClimateZone"}
+            ]
+        },
+        "optional": {
+            "cycleDuration": ""
+        }
+    }
+}
+LOOKUPS = {
+    "crop": [
+        "isPlantation",
+        "IPCC_2013_ORGANIC_SOIL_CULTIVATION_CATEGORY"
+    ],
+    "forage": [
+        "isPlantation",
+        "IPCC_2013_ORGANIC_SOIL_CULTIVATION_CATEGORY"
+    ],
+    "ecoClimateZone": [
+        "IPCC_2013_ORGANIC_SOILS_TONNES_CO2-C_HECTARE_ANNUAL_CROPS",
+        "IPCC_2013_ORGANIC_SOILS_TONNES_CO2-C_HECTARE_PERENNIAL_CROPS",
+        "IPCC_2013_ORGANIC_SOILS_TONNES_CO2-C_HECTARE_ACACIA",
+        "IPCC_2013_ORGANIC_SOILS_TONNES_CO2-C_HECTARE_OIL_PALM",
+        "IPCC_2013_ORGANIC_SOILS_TONNES_CO2-C_HECTARE_SAGO_PALM",
+        "IPCC_2013_ORGANIC_SOILS_TONNES_CO2-C_HECTARE_PADDY_RICE_CULTIVATION",
+        "IPCC_2013_ORGANIC_SOILS_TONNES_CO2-C_HECTARE_GRASSLAND",
+        "IPCC_2013_ORGANIC_SOILS_TONNES_CO2-C_HECTARE_OTHER"
+    ]
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "sd": "",
+        "min": "",
+        "max": "",
+        "observations": "",
+        "statsDefinition": "simulated",
+        "methodTier": "tier 1"
+    }]
+}
+TERM_ID = 'co2ToAirOrganicSoilCultivation'
+TIER = EmissionMethodTier.TIER_1.value
+
+_STATS_DEFINITION = EmissionStatsDefinition.SIMULATED.value
+_ITERATIONS = 100000
+
+
+def _emission(descriptive_stats: dict):
+    emission = _new_emission(TERM_ID, MODEL) | descriptive_stats
+    emission['methodTier'] = TIER
+    return emission
+
+
+def sample_emission_factor(
+    eco_climate_zone: EcoClimateZone,
+    organic_soil_category: OrganicSoilCategory,
+    seed: Union[int, np.random.Generator, None] = None,
+) -> npt.NDArray:
+    factor_data = get_emission_factor(TERM_ID, eco_climate_zone, organic_soil_category)
+    sample_func = _get_sample_func(factor_data)
+    return sample_func(iterations=_ITERATIONS, seed=seed, **factor_data)
+
+
+def _sample_truncated_normal(
+    *, iterations: int, value: float, sd: float, seed: Union[int, np.random.Generator, None] = None, **_
+) -> npt.NDArray:
+    """
+    Randomly sample a model parameter with a truncated normal distribution. Emission factors annot be below 0, so
+    truncated normal sampling used.
+    """
+    return truncated_normal_1d(shape=(1, iterations), mu=value, sigma=sd, low=0, high=np.inf, seed=seed)
+
+
+def _sample_constant(*, value: float, **_) -> npt.NDArray:
+    """Sample a constant model parameter."""
+    return repeat_single(shape=(1, 1), value=value)
+
+
+_KWARGS_TO_SAMPLE_FUNC = {
+    ("value", "sd"): _sample_truncated_normal,
+    ("value",): _sample_constant
+}
+"""
+Mapping from available distribution data to sample function.
+"""
+
+
+def _get_sample_func(kwargs: dict) -> Callable:
+    """
+    Select the correct sample function for a parameter based on the distribution data available. All possible
+    parameters for the model should have, at a minimum, a `value`, meaning that no default function needs to be
+    specified.
+
+    This function has been extracted into it's own method to allow for mocking of sample function.
+
+    Keyword Args
+    ------------
+    value : float
+        The distribution mean.
+    sd : float
+        The standard deviation of the distribution.
+    se : float
+        The standard error of the distribution.
+    n : float
+        Sample size.
+
+    Returns
+    -------
+    Callable
+        The sample function for the distribution.
+    """
+    return next(
+        sample_func for required_kwargs, sample_func in _KWARGS_TO_SAMPLE_FUNC.items()
+        if all(kwarg in kwargs.keys() for kwarg in required_kwargs)
+    )
+
+
+def _should_run(cycle: dict):
+    end_date = cycle.get('endDate')
+    site = cycle.get('site', {})
+    measurements = site.get('measurements', [])
+
+    seed = gen_seed(cycle, MODEL, TERM_ID)
+    rng = np.random.default_rng(seed)
+
+    def _get_measurement_content(term_id: str):
+        return most_relevant_measurement_value(measurements, term_id, end_date)
+
+    histosol = _get_measurement_content('histosol')
+    eco_climate_zone = get_eco_climate_zone_value(cycle, as_enum=True)
+    organic_soil_category = assign_organic_soil_category(cycle, log_id=TERM_ID)
+
+    emission_factor = (
+        sample_emission_factor(eco_climate_zone, organic_soil_category, seed=rng) if eco_climate_zone
+        else None
+    )
+    land_occupation = land_occupation_per_ha(MODEL, TERM_ID, cycle)
+
+    logRequirements(
+        cycle, model=MODEL, term=TERM_ID,
+        eco_climate_zone=eco_climate_zone,
+        organic_soil_category=organic_soil_category,
+        emission_factor=f"{np.mean(emission_factor):.3f}",
+        land_occupation=land_occupation,
+        histosol=histosol
+    )
+
+    should_run = all([
+        valid_site_type(site),
+        valid_eco_climate_zone(eco_climate_zone),
+        all(
+            var is not None for var in [
+                emission_factor,
+                land_occupation,
+                histosol
+            ]
+        )
+    ])
+
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+
+    return should_run, emission_factor, histosol, land_occupation
+
+
+def _run(emission_factor: npt.NDArray, histosol: float, land_occupation: float):
+    result = calc_emission(TERM_ID, emission_factor, histosol, land_occupation)
+    descriptive_stats = calc_descriptive_stats(result, _STATS_DEFINITION)
+    return [_emission(descriptive_stats)]
+
+
+def run(cycle: dict):
+    should_run, emission_factor, histosol, land_occupation = _should_run(cycle)
+    return _run(emission_factor, histosol, land_occupation) if should_run else []

hestia_earth/models/ipcc2019/co2ToAirSoilOrganicCarbonStockChange.py

@@ -199,13 +199,11 @@ def _should_compile_inventory_func(
         ) for cycle in cycles
     )
 
-    has_stock_measurements = len(carbon_stock_measurements) > 0
     has_cycles = len(cycles) > 0
     has_functional_unit_1_ha = all(cycle.get('functionalUnit') == CycleFunctionalUnit._1_HA.value for cycle in cycles)
 
     should_run = all([
         has_soil,
-        has_stock_measurements,
         has_cycles,
         has_functional_unit_1_ha
     ])
@@ -214,7 +212,6 @@ def _should_compile_inventory_func(
         "site_type": site_type,
         "has_soil": has_soil,
         "carbon_stock_term": _CARBON_STOCK_TERM_ID,
-        "has_stock_measurements": has_stock_measurements,
         "has_cycles": has_cycles,
         "has_functional_unit_1_ha": has_functional_unit_1_ha,
     }

hestia_earth/models/ipcc2019/co2ToAirUreaHydrolysis.py

@@ -1,5 +1,5 @@
 from hestia_earth.schema import EmissionMethodTier
-from hestia_earth.utils.tools import list_sum, safe_parse_float
+from hestia_earth.utils.tools import list_sum, safe_parse_float, non_empty_list
 from hestia_earth.utils.model import find_term_match
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
@@ -68,14 +68,21 @@ def _should_run(cycle: dict):
     uan_share = get_country_breakdown(MODEL, TERM_ID, country_id, LOOKUPS['inorganicFertiliser'][1])
     urea_unspecified_as_n = list_sum(find_term_match(inputs, UNSPECIFIED_TERM_ID).get('value', []))
 
-    urea_values = [{'id': id, 'values': _get_urea_values(cycle, inputs, id)} for id in term_ids] + ([
-        {'id': 'ureaKgN', 'values': [urea_unspecified_as_n * urea_share]},
-        {'id': 'ureaAmmoniumNitrateKgN', 'values': [urea_unspecified_as_n * uan_share]}
-    ] if all([
-        urea_share,
-        uan_share,
-        urea_unspecified_as_n > 0
-    ]) else [])
+    urea_values = [
+        {
+            'id': id,
+            'values': _get_urea_values(cycle, inputs, id)
+        } for id in term_ids
+    ] + non_empty_list([
+        {
+            'id': 'ureaKgN',
+            'values': [urea_unspecified_as_n * urea_share]
+        } if urea_share is not None else None,
+        {
+            'id': 'ureaAmmoniumNitrateKgN',
+            'values': [urea_unspecified_as_n * uan_share]
+        } if urea_share is not None else None
+    ] if urea_unspecified_as_n > 0 else [])
     has_urea_value = any([len(data.get('values')) > 0 for data in urea_values])
 
     logRequirements(cycle, model=MODEL, term=TERM_ID,