hestia-earth-models 0.62.0__py3-none-any.whl → 0.62.2__py3-none-any.whl

hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_2_utils.py

@@ -19,13 +19,14 @@ from hestia_earth.schema import (
 )
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import flatten, list_sum, non_empty_list
+from hestia_earth.utils.blank_node import get_node_value
 
 from hestia_earth.models.utils.array_builders import (
     avg_run_in_columnwise, gen_seed, grouped_avg, repeat_1d_array_as_columns
 )
 from hestia_earth.models.utils.blank_node import (
-    cumulative_nodes_lookup_match, cumulative_nodes_term_match, get_node_value, group_nodes_by_year,
-    group_nodes_by_year_and_month, GroupNodesByYearMode, node_term_match
+    cumulative_nodes_lookup_match, cumulative_nodes_term_match, group_nodes_by_year,
+    group_nodes_by_year_and_month, GroupNodesByYearMode, node_lookup_match, node_term_match
 )
 from hestia_earth.models.utils.cycle import check_cycle_site_ids_identical
 from hestia_earth.models.utils.descriptive_stats import calc_descriptive_stats
@@ -35,16 +36,16 @@ from hestia_earth.models.utils.site import related_cycles
 
 from .organicCarbonPerHa_utils import (
     CarbonSource, check_consecutive, DEPTH_LOWER, DEPTH_UPPER, check_irrigation,
-    get_crop_residue_inc_or_left_terms_with_cache,
-    get_upland_rice_crop_terms_with_cache,
-    get_upland_rice_land_cover_terms_with_cache,
-    IPCC_LAND_USE_CATEGORY_TO_LAND_COVER_LOOKUP_VALUE, IPCC_MANAGEMENT_CATEGORY_TO_TILLAGE_MANAGEMENT_LOOKUP_VALUE,
-    IpccLandUseCategory, IpccManagementCategory, MIN_AREA_THRESHOLD, MIN_YIELD_THRESHOLD, sample_constant,
-    sample_plus_minus_uncertainty, sample_truncated_normal, STATS_DEFINITION
+    get_cover_crop_property_terms_with_cache, get_upland_rice_crop_terms_with_cache,
+    get_upland_rice_land_cover_terms_with_cache, IPCC_LAND_USE_CATEGORY_TO_LAND_COVER_LOOKUP_VALUE,
+    IPCC_MANAGEMENT_CATEGORY_TO_TILLAGE_MANAGEMENT_LOOKUP_VALUE, IpccLandUseCategory, IpccManagementCategory,
+    MIN_AREA_THRESHOLD, MIN_YIELD_THRESHOLD, sample_constant, sample_plus_minus_uncertainty, sample_truncated_normal,
+    STATS_DEFINITION
 )
 
 _LOOKUPS = {
     "crop": "IPCC_LAND_USE_CATEGORY",
+    "landCover": "IPCC_LAND_USE_CATEGORY",
     "tillage": "IPCC_TILLAGE_MANAGEMENT_CATEGORY"
 }
 
@@ -58,10 +59,27 @@ _NUMBER_OF_TILLAGES_TERM_ID = "numberOfTillages"
 _TEMPERATURE_MONTHLY_TERM_ID = "temperatureMonthly"
 _PRECIPITATION_MONTHLY_TERM_ID = "precipitationMonthly"
 _PET_MONTHLY_TERM_ID = "potentialEvapotranspirationMonthly"
+_ABOVE_GROUND_CROP_RESIDUE_TOTAL_TERM_ID = "aboveGroundCropResidueTotal"
 _CARBON_CONTENT_TERM_ID = "carbonContent"
 _NITROGEN_CONTENT_TERM_ID = "nitrogenContent"
 _LIGNIN_CONTENT_TERM_ID = "ligninContent"
 
+_CROP_RESIDUE_MANAGEMENT_TERM_IDS = [
+    "residueIncorporated",
+    "residueIncorporatedLessThan30DaysBeforeCultivation",
+    "residueIncorporatedMoreThan30DaysBeforeCultivation",
+    "residueLeftOnField"
+]
+
+_CARBON_SOURCE_TERM_IDS = [
+    "discardedCropIncorporated",
+    "discardedCropLeftOnField",
+    "belowGroundCropResidue",
+]
+
+_MIN_RESIDUE_LEFT_ON_FIELD = 20  # TODO: Confirm assumption
+_DEFAULT_COVER_CROP_BIOMASS = 4000  # TODO: Confirm assumption, Source PAS 2050-1:2012
+
 _CARBON_INPUT_PROPERTY_TERM_IDS = [
     _CARBON_CONTENT_TERM_ID,
     _NITROGEN_CONTENT_TERM_ID,
@@ -1283,8 +1301,8 @@ def _get_grouped_sand_content_measurements(grouped_measurements: dict) -> dict:
 
 def _get_grouped_carbon_input_data(grouped_cycles: dict) -> dict:
     grouped_carbon_sources = {
-        year: _get_carbon_sources_from_cycles(cycle)
-        for year, cycle in grouped_cycles.items()
+        year: flatten([_get_carbon_sources(cycle) for cycle in cycles])
+        for year, cycles in grouped_cycles.items()
     }
 
     return {
@@ -1296,67 +1314,219 @@ def _get_grouped_carbon_input_data(grouped_cycles: dict) -> dict:
     }
 
 
-def _get_carbon_sources_from_cycles(cycles: dict) -> list[CarbonSource]:
+def _get_carbon_sources(cycle: dict) -> list[CarbonSource]:
     """
-    Retrieves and formats all of the valid carbon sources from a list of cycles.
+    Extract and format the carbon source data from a cycle's inputs and products.
 
     Carbon sources can be either a Hestia `Product` node (e.g. crop residue) or `Input` node (e.g. organic amendment).
 
     Parameters
     ----------
-    cycles : list[dict]
-        A list of Hestia `Cycle` nodes, see: https://www.hestia.earth/schema/Cycle.
+    cycle : list[dict]
+        A Hestia `Cycle` node, see: https://www.hestia.earth/schema/Cycle.
 
     Returns
     -------
     list[CarbonSource]
-        A formatted list of `CarbonSource`s for the inputted `Cycle`s.
+        A formatted list of `CarbonSource`s.
     """
-    inputs_and_products = non_empty_list(flatten(
-        [cycle.get("inputs", []) + cycle.get("products", []) for cycle in cycles]
-    ))
-
+    inputs_and_products = cycle.get("inputs", []) + cycle.get("products", [])
     return non_empty_list([
-        _iterate_carbon_source(node) for node in inputs_and_products
-        if any([
-            node.get("term", {}).get("@id") in get_crop_residue_inc_or_left_terms_with_cache(),
-            node.get("term", {}).get("termType") in _CARBON_SOURCE_TERM_TYPES
-        ])
+        next(
+            (_func(node, cycle) for validator, _func in _CARBON_SOURCE_DECISION_TREE.items() if validator(node)),
+            None
+        ) for node in inputs_and_products
+    ])
+
+
+def _should_run_carbon_source_ag_residue(node: dict) -> bool:
+    """
+    Determine whether an input or product is a valid above-ground biomass carbon source.
+
+    Parameters
+    ----------
+    node : dict
+        A Hestia [Input](https://www.hestia.earth/schema/Input) or [Product](https://www.hestia.earth/schema/Product)
+        node.
+
+    Returns
+    -------
+    bool
+        Whether the node satisfies the critera.
+    """
+    return node.get("term", {}).get("@id") == _ABOVE_GROUND_CROP_RESIDUE_TOTAL_TERM_ID
+
+
+def _calc_carbon_source_ag_crop_residue(node: dict, cycle: dict) -> Union[CarbonSource, None]:
+    """
+    Extract and format the carbon source data for above-ground crop residues.
+
+    n.b., We assume that even if a cycle's residue management states that 100% of above-ground crop residues are
+    removed or burnt, a minimal amount of crop residues do remain on site to become a carbon source (see
+    `_MIN_RESIDUE_LEFT_ON_FIELD` variable).
+
+    Parameters
+    ----------
+    node : dict
+        A Hestia [Product](https://www.hestia.earth/schema/Product) node with `term.termType` == `landCover`.
+
+    Returns
+    -------
+    CarbonSource | None
+        The carbon source data of the above-ground residues, or `None` if carbon source data incomplete.
+    """
+    value = get_node_value(node)
+    residue_left_on_field = list_sum([
+        get_node_value(practice) for practice in cycle.get("practices", [])
+        if node_term_match(practice, _CROP_RESIDUE_MANAGEMENT_TERM_IDS)
+    ])
+    mass = value * max(residue_left_on_field, _MIN_RESIDUE_LEFT_ON_FIELD) / 100
+
+    carbon_source = CarbonSource(
+        mass,
+        *_retrieve_carbon_source_properties(node)
+    )
+
+    return carbon_source if _validate_carbon_source(carbon_source) else None
+
+
+def _should_run_carbon_source_cover_crop(node: dict) -> bool:
+    """
+    Determine whether a product is a valid above cover crop carbon source.
+
+    Parameters
+    ----------
+    node : dict
+        A Hestia [Input](https://www.hestia.earth/schema/Input) or [Product](https://www.hestia.earth/schema/Product)
+        node.
+
+    Returns
+    -------
+    bool
+        Whether the node satisfies the critera.
+    """
+    LOOKUP = _LOOKUPS["landCover"]
+    return all([
+        node.get("term", {}).get("termType") in [TermTermType.LANDCOVER.value],
+        node_lookup_match(
+            node, LOOKUP, IPCC_LAND_USE_CATEGORY_TO_LAND_COVER_LOOKUP_VALUE[IpccLandUseCategory.ANNUAL_CROPS]
+        ),
+        any(
+            get_node_property(node, term_id, False).get("value", False)
+            for term_id in get_cover_crop_property_terms_with_cache()
+        )
     ])
 
 
-def _iterate_carbon_source(node: dict) -> Union[CarbonSource, None]:
+def _calc_carbon_source_cover_crop(node: dict, *_) -> Union[CarbonSource, None]:
     """
-    Validates whether a node is a valid carbon source and returns a `CarbonSource` named tuple if yes.
+    Extract and format the carbon source data for an annual cover crop.
+
+    n.b., We make the assumption that the entirety of the cover crop's biomass remains on site.
 
     Parameters
     ----------
     node : dict
-        A Hestia `Product` or `Input` node, see: https://www.hestia.earth/schema/Product
-        or https://www.hestia.earth/schema/Input.
+        A Hestia [Product](https://www.hestia.earth/schema/Product) node with `term.termType` == `landCover`.
 
     Returns
     -------
     CarbonSource | None
-        A `CarbonSource` named tuple if the node is a carbon source with the required properties, else `None`.
+        The carbon source data of the cover crop, or `None` if carbon source data incomplete.
+    """
+    value = get_node_value(node)
+    carbon_source = CarbonSource(
+        _DEFAULT_COVER_CROP_BIOMASS * value / 100,
+        _Parameter.DEFAULT_CARBON_CONTENT.value.get("value"),
+        _Parameter.DEFAULT_NITROGEN_CONTENT.value.get("value"),
+        _Parameter.DEFAULT_NITROGEN_CONTENT.value.get("value")
+    )
+    return carbon_source
+
+
+def _should_run_carbon_source(node: dict) -> bool:
+    """
+    Determine whether an input or product is a valid carbon source.
+
+    Parameters
+    ----------
+    node : dict
+        A Hestia [Input](https://www.hestia.earth/schema/Input) or [Product](https://www.hestia.earth/schema/Product)
+        node.
+
+    Returns
+    -------
+    bool
+        Whether the node satisfies the critera.
+    """
+    return any([
+        node.get("term", {}).get("@id") in _CARBON_SOURCE_TERM_IDS,
+        node.get("term", {}).get("termType") in _CARBON_SOURCE_TERM_TYPES
+    ])
+
+
+def _calc_carbon_source(node: dict, *_) -> Union[CarbonSource, None]:
+    """
+    Extract and format the carbon source data for an input or product.
+
+    Parameters
+    ----------
+    node : dict
+        A Hestia [Input](https://www.hestia.earth/schema/Input) or [Product](https://www.hestia.earth/schema/Product)
+        node.
+
+    Returns
+    -------
+    CarbonSource | None
+        The carbon source data of the cover crop, or `None` if carbon source data incomplete.
+    """
+    carbon_source = CarbonSource(
+        get_node_value(node),
+        *_retrieve_carbon_source_properties(node)
+    )
+
+    return carbon_source if _validate_carbon_source(carbon_source) else None
+
+
+def _retrieve_carbon_source_properties(node: dict) -> tuple[float, float, float]:
+    """
+    Extract the carbon source properties from an input or product node or, if required, retrieve them from default
+    properties.
+
+    Parameters
+    ----------
+    node : dict
+        A Hestia [Input](https://www.hestia.earth/schema/Input) or [Product](https://www.hestia.earth/schema/Product)
+        node.
+
+    Returns
+    -------
+    tuple[float, float, float]
+        `(carbon_content, nitrogen_content, lignin_content)`
     """
-    mass = list_sum(node.get("value", []))
     carbon_content, nitrogen_content, lignin_content = (
         get_node_property(node, term_id).get("value", 0)/100 for term_id in _CARBON_INPUT_PROPERTY_TERM_IDS
     )
+    return carbon_content, nitrogen_content, lignin_content
 
-    should_run_ = all([
-        mass > 0,
-        0 < carbon_content <= 1,
-        0 < nitrogen_content <= 1,
-        0 < lignin_content <= 1
+
+def _validate_carbon_source(carbon_source: CarbonSource) -> bool:
+    """
+    Validate that a `CarbonSource` named tuple is data complete.
+    """
+    return all([
+        carbon_source.mass > 0,
+        0 < carbon_source.carbon_content <= 1,
+        0 < carbon_source.nitrogen_content <= 1,
+        0 < carbon_source.lignin_content <= 1
     ])
 
-    return (
-        CarbonSource(
-            mass, carbon_content, nitrogen_content, lignin_content
-        ) if should_run_ else None
-    )
+
+_CARBON_SOURCE_DECISION_TREE = {
+    _should_run_carbon_source_ag_residue: _calc_carbon_source_ag_crop_residue,
+    _should_run_carbon_source_cover_crop: _calc_carbon_source_cover_crop,
+    _should_run_carbon_source: _calc_carbon_source,
+}
 
 
 def _calc_total_organic_carbon_input(

hestia_earth/models/ipcc2019/organicCarbonPerHa_utils.py

@@ -10,8 +10,8 @@ from hestia_earth.models.utils.array_builders import (
 )
 from hestia_earth.models.utils.blank_node import cumulative_nodes_term_match
 from hestia_earth.models.utils.term import (
-    get_cover_crop_property_terms, get_crop_residue_incorporated_or_left_on_field_terms, get_irrigated_terms,
-    get_residue_removed_or_burnt_terms, get_upland_rice_crop_terms, get_upland_rice_land_cover_terms
+    get_cover_crop_property_terms, get_irrigated_terms, get_residue_removed_or_burnt_terms, get_upland_rice_crop_terms,
+    get_upland_rice_land_cover_terms
 )
 
 STATS_DEFINITION = MeasurementStatsDefinition.SIMULATED.value
@@ -27,10 +27,6 @@ def get_cover_crop_property_terms_with_cache():
     return lru_cache()(get_cover_crop_property_terms)()
 
 
-def get_crop_residue_inc_or_left_terms_with_cache():
-    return lru_cache()(get_crop_residue_incorporated_or_left_on_field_terms)()
-
-
 def get_irrigated_terms_with_cache():
     return lru_cache()(get_irrigated_terms)()
 

hestia_earth/models/ipcc2019/pastureGrass.py

@@ -12,7 +12,8 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum
 
-from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils.blank_node import lookups_logs, properties_logs
 from hestia_earth.models.utils.input import _new_input
 from hestia_earth.models.utils.term import get_wool_terms
 from hestia_earth.models.utils.completeness import _is_term_type_complete, _is_term_type_incomplete
@@ -25,6 +26,7 @@ from .pastureGrass_utils import (
     calculate_REM,
     calculate_REG,
     calculate_NEfeed,
+    calculate_GE,
     product_wool_energy,
     get_animals,
     get_animal_values
@@ -119,7 +121,8 @@ REQUIREMENTS = {
 }
 LOOKUPS = {
     "animalManagement": [
-        "mjKgEvMilkIpcc2019"
+        "mjKgEvMilkIpcc2019",
+        "defaultFatContentEvMilkIpcc2019"
     ],
     "animalProduct": ["mjKgEvWoolNetEnergyWoolIpcc2019"],
     "liveAnimal": [
@@ -127,7 +130,8 @@ LOOKUPS = {
         "mjDayKgCfiNetEnergyMaintenanceIpcc2019",
         "ratioCPregnancyNetEnergyPregnancyIpcc2019",
         "ratioCNetEnergyGrowthCattleBuffaloIpcc2019",
-        "mjKgABNetEnergyGrowthSheepGoatsIpcc2019"
+        "mjKgABNetEnergyGrowthSheepGoatsIpcc2019",
+        "isWoolProducingAnimal"
     ],
     "system-liveAnimal-activityCoefficient-ipcc2019": "using animal term @id",
     "crop-property": ["energyDigestibilityRuminants", "energyContentHigherHeatingValue"],
@@ -150,9 +154,6 @@ def _input(term_id: str, value: float):
     return node
 
 
-def _sum_values(values: list, index=0): return list_sum([v[index] for v in values])
-
-
 def calculate_NEwool(cycle: dict) -> float:
     term_ids = get_wool_terms()
     products = [p for p in cycle.get('products', []) if p.get('term', {}).get('@id') in term_ids]
@@ -162,39 +163,20 @@ def calculate_NEwool(cycle: dict) -> float:
     return sum([value * lookup_value for (value, lookup_value) in product_values])
 
 
-def _calculate_GE(
-    cycle: dict, animals: list, REM: float, REG: float, NEwool: float, NEm_feed: float, NEg_feed: float, system: dict
-) -> float:
-    values = [get_animal_values(cycle, animal, system) for animal in animals]
-    NEm = _sum_values(values, 0)
-    NEa = _sum_values(values, 1)
-    NEl = _sum_values(values, 2)
-    NEwork = _sum_values(values, 3)
-    NEp = _sum_values(values, 4)
-    NEg = _sum_values(values, 5)
-
-    debugValues(cycle, model=MODEL, term=MODEL_KEY, model_key=MODEL_KEY,
-                NEm=NEm,
-                NEa=NEa,
-                NEl=NEl,
-                NEwork=NEwork,
-                NEp=NEp,
-                NEg=NEg,
-                NEm_feed=NEm_feed,
-                NEg_feed=NEg_feed)
-
-    return (NEm + NEa + NEl + NEwork + NEp - NEm_feed)/REM + (NEg + NEwool - NEg_feed)/REG
-
-
 def _run_practice(cycle: dict, meanDE: float, meanECHHV: float, system: dict):
     animals = get_animals(cycle)
     REM = calculate_REM(meanDE)
     REG = calculate_REG(meanDE)
     NEwool = calculate_NEwool(cycle)
     NEm_feed, NEg_feed = calculate_NEfeed(cycle)
+
+    animal_values = [{
+        'animalId': animal.get('term', {}).get('@id')
+    } | get_animal_values(cycle, animal, system) for animal in animals]
+
     GE = (
-        _calculate_GE(cycle, animals, REM, REG, NEwool, NEm_feed, NEg_feed, system) / (meanDE/100)
-    ) if all([REM, REG]) else 0
+        calculate_GE(animal_values, REM, REG, NEwool, NEm_feed, NEg_feed) / (meanDE/100)
+    ) if meanDE else 0
 
     def run(practice: dict):
         key = practice.get('key', {})
@@ -202,14 +184,34 @@ def _run_practice(cycle: dict, meanDE: float, meanECHHV: float, system: dict):
         input_term_id = practice_input_id(practice)
         value = (GE / meanECHHV) * (list_sum(practice.get('value', [0])) / 100)
 
+        logs = log_as_table([v | {
+            'practiceKeyId': key_id,
+            'REM': REM,
+            'REG': REG,
+            'NEwool': NEwool,
+            'NEmFeed': NEm_feed,
+            'NEgFeed': NEg_feed,
+            'GE': GE,
+            'meanECHHV': meanECHHV,
+            'meanDE': meanDE
+        } for v in animal_values])
+        animal_lookups = lookups_logs(MODEL, animals, LOOKUPS, model_key=MODEL_KEY, term=input_term_id)
+        animal_properties = properties_logs(animals, properties=[
+            'liveweightPerHead',
+            'hoursWorkedPerDay',
+            'animalsPerBirth',
+            'pregnancyRateTotal',
+            'weightAtMaturity',
+            'liveweightGain',
+            'weightAtWeaning',
+            'weightAtOneYear',
+            'weightAtSlaughter'
+        ])
+
         logRequirements(cycle, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
-                        REM=REM,
-                        REG=REG,
-                        NEwool=NEwool,
-                        NEm_feed=NEm_feed,
-                        NEg_feed=NEg_feed,
-                        GE=GE,
-                        key_id=key_id)
+                        animal_logs=logs,
+                        animal_lookups=animal_lookups,
+                        animal_properties=animal_properties)
 
         logShouldRun(cycle, MODEL, input_term_id, True, model_key=MODEL_KEY)
 
@@ -242,15 +244,15 @@ def _should_run(cycle: dict, practices: dict):
         meanECHHV > 0
     ])
 
-    for term_id in [MODEL_KEY] + [practice_input_id(p) for p in practices]:
+    for term_id in [practice_input_id(p) for p in practices] or [MODEL_KEY]:
         logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
                         term_type_animalFeed_complete=animalFeed_complete,
                         term_type_animalPopulation_complete=animalPopulation_complete,
                         term_type_freshForage_incomplete=freshForage_incomplete,
                         has_cycle_inputs_feed=has_cycle_inputs_feed,
                         all_animals_have_value=all_animals_have_value,
-                        meanDE=meanDE,
-                        meanECHHV=meanECHHV)
+                        meanDE=calculate_meanDE(practices, term=term_id),
+                        meanECHHV=calculate_meanECHHV(practices, term=term_id))
 
         logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)