hestia-earth-models 0.59.6__tar.gz → 0.60.0__tar.gz

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2023 Harmonized Environmental Storage and Tracking of the Impacts of Agriculture (Hestia) Project
+Copyright (c) 2019-2024 Harmonised Environmental Storage and Tracking of the Impacts of Agriculture (HESTIA) Project
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{hestia-earth-models-0.59.6/hestia_earth_models.egg-info → hestia-earth-models-0.60.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.59.6
+Version: 0.60.0
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/hestia_earth/models/cache_sites.py

@@ -34,7 +34,7 @@ def _cache_results(results: list, collections: list, index: int):
     return cache_site_results(results[start:end], collections)
 
 
-def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors: list = [], years: list = []):
+def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors: list = [], years: list = None):
     param_values = list(map(_VALUE_AS_PARAM.get(param_type), sites))
     # unique list
     param_values = list(set(param_values)) if param_type == ParamType.GADM_IDS else list({
@@ -66,11 +66,9 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
         } | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
         return {
             **site,
-            CACHE_KEY: {
-                **cached_value(site),
-                CACHE_YEARS_KEY: years,
-                CACHE_GEOSPATIAL_KEY: cached_data
-            }
+            CACHE_KEY: (
+                cached_value(site) | {CACHE_GEOSPATIAL_KEY: cached_data}
+            ) | ({CACHE_YEARS_KEY: years} if years else {})
         }
 
     return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
@@ -114,7 +112,7 @@ def _group_sites(sites: dict):
     }
 
 
-def run(sites: list, years: list, include_region: bool = False):
+def run(sites: list, years: list = None, include_region: bool = False, years_only: bool = False):
     """
     Run all queries at once for the list of provided Sites.
     Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
@@ -128,8 +126,10 @@ def run(sites: list, years: list, include_region: bool = False):
     include_region : bool
         Prefecth region IDs.
         This will cache region-level data and will make the request slower. Only use if needed.
+    years_only : bool
+        Run only the collections that depend on the years (if provided).
     """
-    rasters, vectors = list_collections(years, include_region=include_region)
+    rasters, vectors = list_collections(years, include_region, years_only)
 
     filtered_data = _group_sites(sites)
 

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/hestia_earth/models/cycle/siteDuration.py

@@ -9,7 +9,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "cycleDuration": "365",
+        "cycleDuration": "> 0",
         "none": {
             "otherSites": ""
         }

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/hestia_earth/models/emepEea2019/nh3ToAirInorganicFertiliser.py

@@ -17,14 +17,17 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "inputs": [{
+        "completeness.fertiliser": "True",
+        "inputs": [
+            {
                 "@type": "Input",
                 "value": "",
-                "term.termType": ["inorganicFertiliser", "inorganicNitrogenFertiliserUnspecifiedKgN"]
-            }],
-            "completeness.fertiliser": "True"
-        },
+                "term.termType": "inorganicFertiliser",
+                "optional": {
+                    "properties": [{"@type": "Property", "value": "", "term.@id": "nitrogenContent"}]
+                }
+            }
+        ],
         "site": {
             "@type": "Site",
             "country": {"@type": "Term", "termType": "region"},

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/hestia_earth/models/faostat2018/product/price.py

@@ -41,7 +41,8 @@ LOOKUPS = {
     "liveAnimal": ["primaryMeatProductFaoPriceTermId"],
     "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO"],
     "region-animalProduct-animalProductGroupingFAO-price": "use value from above",
-    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above"
+    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above",
+    "region-animalProduct-animalProductGroupingFAO-weightPerItem": "use value from above"
 }
 MODEL_KEY = 'price'
 LOOKUP_NAME = {
@@ -52,6 +53,9 @@ LOOKUP_GROUPING = {
     TermTermType.CROP.value: get_crop_grouping_faostat_production,
     TermTermType.ANIMALPRODUCT.value: get_animalProduct_grouping_fao
 }
+LOOKUP_UNITS_NUMBER = {
+    TermTermType.ANIMALPRODUCT.value: f"region-{TermTermType.ANIMALPRODUCT.value}-{FAO_LOOKUP_COLUMN}-weightPerItem.csv"
+}
 
 
 def _term_grouping(term: dict): return LOOKUP_GROUPING.get(term.get('termType'), lambda *_: None)(MODEL, term)
@@ -123,6 +127,7 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     product_term = product.get('term', {})
     term_id = product_term.get('@id')
     term_type = product_term.get('termType')
+    term_units = product_term.get('units')
 
     has_yield = len(product.get('value', [])) > 0
     not_already_set = MODEL_KEY not in product.keys()
@@ -133,17 +138,23 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     should_run = all([not_already_set, has_yield, grouping])
     value = _lookup_data(term_id, grouping, country_id, year, term_type=term_type) if should_run else None
 
+    # if units is number instead of kg, need to convert to number first
+    conversion_to_number = safe_parse_float(get_table_value(
+        download_lookup(LOOKUP_UNITS_NUMBER.get(term_type)), 'termid', country_id, column_name(grouping)
+    ), 1) if term_units == Units.NUMBER.value else 1
+
     logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='country',
                     has_yield=has_yield,
                     not_already_set=not_already_set,
                     year=year,
                     price_per_ton=value,
-                    groupingFAO=f"'{grouping}'")
+                    groupingFAO=f"'{grouping}'",
+                    conversion_to_number=conversion_to_number)
 
     logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY, by='country')
 
     # divide by 1000 to convert price per tonne to kg
-    return _product(product, value / 1000) if value is not None else None
+    return _product(product, value / 1000 * conversion_to_number) if value is not None else None
 
 
 def _should_run_product(product: dict):

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/hestia_earth/models/geospatialDatabase/utils.py

@@ -175,7 +175,6 @@ def download(term: str, site: dict, data: dict, only_coordinates=False) -> dict:
                 }
             ]
         }
-        print(query)
         value = _parse_run_query(term, query)
 
     if value is None:

hestia-earth-models-0.60.0/hestia_earth/models/ipcc2019/animal/pastureGrass.py

@@ -0,0 +1,298 @@
+"""
+Full Grass Consumption
+
+This model estimates the energetic requirements of ruminants and can be used to estimate the amount of grass they graze.
+Source:
+[IPCC 2019, Vol.4, Chapter 10](https://www.ipcc-nggip.iges.or.jp/public/2019rf/pdf/4_Volume4/19R_V4_Ch10_Livestock.pdf).
+"""
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
+from hestia_earth.models.utils.input import _new_input
+from hestia_earth.models.utils.term import get_wool_terms, get_lookup_value
+from hestia_earth.models.utils.completeness import _is_term_type_complete, _is_term_type_incomplete
+from hestia_earth.models.utils.property import get_node_property
+from .. import MODEL
+from ..pastureGrass_utils import (
+    practice_input_id,
+    should_run_practice,
+    calculate_meanDE,
+    calculate_meanECHHV,
+    calculate_REM,
+    calculate_REG,
+    calculate_NEfeed,
+    product_wool_energy,
+    get_animals,
+    get_animal_values
+)
+
+REQUIREMENTS = {
+    "Cycle": {
+        "completeness.animalFeed": "True",
+        "completeness.animalPopulation": "True",
+        "completeness.freshForage": "False",
+        "site": {
+            "@type": "Site",
+            "siteType": "permanent pasture"
+        },
+        "practices": [{
+            "@type": "Practice",
+            "value": "",
+            "term.@id": "pastureGrass",
+            "key": {
+                "@type": "Term",
+                "term.termType": "landCover"
+            }
+        }],
+        "animals": [{
+            "@type": "Animal",
+            "value": "> 0",
+            "term.termType": "liveAnimal",
+            "referencePeriod": "average",
+            "properties": [{
+                "@type": "Property",
+                "value": "",
+                "term.@id": [
+                    "liveweightPerHead",
+                    "weightAtMaturity"
+                ]
+            }],
+            "optional": {
+                "properties": [{
+                    "@type": "Property",
+                    "value": "",
+                    "term.@id": [
+                        "hoursWorkedPerDay",
+                        "pregnancyRateTotal",
+                        "animalsPerBirth"
+                    ]
+                }],
+                "inputs": [{
+                    "@type": "Input",
+                    "term.units": "kg",
+                    "value": "> 0",
+                    "optional": {
+                        "properties": [{
+                            "@type": "Property",
+                            "value": "",
+                            "term.@id": ["neutralDetergentFibreContent", "energyContentHigherHeatingValue"]
+                        }]
+                    }
+                }],
+                "practices": [{
+                    "@type": "Practice",
+                    "value": "",
+                    "term.termType": "animalManagement",
+                    "properties": [{
+                        "@type": "Property",
+                        "value": "",
+                        "term.@id": "fatContent"
+                    }]
+                }]
+            }
+        }],
+        "none": {
+            "inputs": [{
+                "@type": "Input",
+                "term.units": "kg",
+                "value": "> 0",
+                "isAnimalFeed": "True"
+            }]
+        },
+        "optional": {
+            "products": [{
+                "@type": "Product",
+                "value": "",
+                "term.@id": "animalProduct"
+            }]
+        }
+    }
+}
+LOOKUPS = {
+    "animalManagement": [
+        "mjKgEvMilkIpcc2019"
+    ],
+    "animalProduct": ["mjKgEvWoolNetEnergyWoolIpcc2019", "allowedLiveAnimalTermIds"],
+    "liveAnimal": [
+        "ipcc2019AnimalTypeGrouping",
+        "mjDayKgCfiNetEnergyMaintenanceIpcc2019",
+        "ratioCPregnancyNetEnergyPregnancyIpcc2019",
+        "ratioCNetEnergyGrowthCattleBuffaloIpcc2019",
+        "mjKgABNetEnergyGrowthSheepGoatsIpcc2019",
+        "isWoolProducingAnimal"
+    ],
+    "system-liveAnimal-activityCoefficient-ipcc2019": "using animal term @id",
+    "crop-property": ["energyDigestibilityRuminants", "energyContentHigherHeatingValue"],
+    "crop": "grazedPastureGrassInputId",
+    "forage-property": ["energyDigestibilityRuminants", "energyContentHigherHeatingValue"],
+    "landCover": "grazedPastureGrassInputId"
+}
+RETURNS = {
+    "Animal": [{
+        "Input": [{
+            "@type": "Input",
+            "term.termType": ["crop", "forage"],
+            "value": ""
+        }]
+    }]
+}
+MODEL_KEY = 'animal/pastureGrass'
+
+
+def _input(term_id: str, value: float):
+    node = _new_input(term_id, MODEL)
+    node['value'] = [value]
+    return node
+
+
+def _sum_liveWeightPerHead(animals: list):
+    return list_sum([
+        animal.get('value', 0) * get_node_property(animal, 'liveweightPerHead', False).get('value', 0)
+        for animal in animals
+    ])
+
+
+def _isNEwool_animal(animal: dict):
+    value = get_lookup_value(animal.get('term', {}), 'isWoolProducingAnimal', model=MODEL, model_key=MODEL_KEY)
+    return not (not value)
+
+
+def _is_NEwool_product(product: dict, animal: dict):
+    animal_term_ids = get_lookup_value(product, 'allowedLiveAnimalTermIds', model=MODEL, model_key=MODEL_KEY).split(';')
+    return animal.get('term', {}).get('@id') in animal_term_ids
+
+
+def calculate_NEwool(cycle: dict, animal: dict, products: list, total_weight: float) -> float:
+    term_id = animal.get('term', {}).get('@id')
+
+    wool_products = [p for p in products if _is_NEwool_product(p.get('term', {}), animal)]
+    total_energy = list_sum([
+        list_sum(product.get('value', [])) * product_wool_energy(product) for product in wool_products
+    ])
+    animal_weight = _sum_liveWeightPerHead([animal])
+
+    debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                total_energy=total_energy,
+                animal_liveWeightPerHead=animal_weight,
+                total_liveWeightPerHead=total_weight)
+
+    return total_energy * animal_weight/total_weight
+
+
+def _calculate_GE(
+    cycle: dict, animal: dict, REM: float, REG: float, NEwool: float, system: dict
+) -> float:
+    term_id = animal.get('term', {}).get('@id')
+
+    NEm, NEa, NEl, NEwork, NEp, NEg = get_animal_values(cycle, animal, system)
+
+    NEm_feed, NEg_feed = calculate_NEfeed(animal)
+    debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                NEm_feed=NEm_feed,
+                NEg_feed=NEg_feed)
+
+    return (NEm + NEa + NEl + NEwork + NEp - NEm_feed)/REM + (NEg + NEwool - NEg_feed)/REG
+
+
+def _run_practice(cycle: dict, GE: float, meanECHHV: float):
+    def run(practice: dict):
+        key = practice.get('key', {})
+        key_id = key.get('@id')
+        input_term_id = practice_input_id(practice)
+        value = (GE / meanECHHV) * (list_sum(practice.get('value', [0])) / 100)
+
+        logRequirements(cycle, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
+                        GE=GE,
+                        meanECHHV=meanECHHV,
+                        key_id=key_id)
+
+        logShouldRun(cycle, MODEL, input_term_id, True, model_key=MODEL_KEY)
+
+        return _input(input_term_id, value)
+
+    return run
+
+
+def _run_animal(cycle: dict, meanDE: float, meanECHHV: float, system: dict, practices: list):
+    REM = calculate_REM(meanDE)
+    REG = calculate_REG(meanDE)
+
+    wool_term_ids = get_wool_terms()
+    # list of animal product
+    wool_products = [p for p in cycle.get('products', []) if p.get('term', {}).get('@id') in wool_term_ids]
+    animals = list(filter(_isNEwool_animal, cycle.get('animals', [])))
+    total_liveWeightPerHead = _sum_liveWeightPerHead(animals)
+
+    def run(animal: dict):
+        term_id = animal.get('term', {}).get('@id')
+
+        NEwool = calculate_NEwool(cycle, animal, wool_products, total_liveWeightPerHead) if (
+            total_liveWeightPerHead > 0
+        ) else 0
+        GE = (_calculate_GE(cycle, animal, REM, REG, NEwool, system) / (meanDE/100)) if all([REM, REG]) else 0
+
+        debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                    REM=REM,
+                    REG=REG,
+                    NEwool=NEwool,
+                    GE=GE,
+                    meanDE=meanDE)
+
+        inputs = list(map(_run_practice(cycle, GE, meanECHHV), practices))
+        return animal | {
+            'inputs': animal.get('inputs', []) + inputs
+        }
+
+    return run
+
+
+def _run(cycle: dict, meanDE: float, meanECHHV: float, system: dict, practices: list):
+    animals = get_animals(cycle)
+    return list(map(_run_animal(cycle, meanDE, meanECHHV, system, practices), animals))
+
+
+def _should_run(cycle: dict, practices: dict):
+    systems = filter_list_term_type(cycle.get('practices', []), TermTermType.SYSTEM)
+    animalFeed_complete = _is_term_type_complete(cycle, 'animalFeed')
+    animalPopulation_complete = _is_term_type_complete(cycle, 'animalPopulation')
+    freshForage_incomplete = _is_term_type_incomplete(cycle, 'freshForage')
+    all_animals_have_value = all([a.get('value', 0) > 0 for a in cycle.get('animals', [])])
+
+    no_cycle_inputs_feed = all([not input.get('isAnimalFeed', False) for input in cycle.get('inputs', [])])
+
+    meanDE = calculate_meanDE(practices)
+    meanECHHV = calculate_meanECHHV(practices)
+
+    should_run = all([
+        animalFeed_complete,
+        animalPopulation_complete,
+        freshForage_incomplete,
+        no_cycle_inputs_feed,
+        all_animals_have_value,
+        len(systems) > 0,
+        len(practices) > 0,
+        meanDE > 0,
+        meanECHHV > 0
+    ])
+
+    for term_id in [MODEL_KEY] + [practice_input_id(p) for p in practices]:
+        logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                        term_type_animalFeed_complete=animalFeed_complete,
+                        term_type_animalPopulation_complete=animalPopulation_complete,
+                        term_type_freshForage_incomplete=freshForage_incomplete,
+                        no_cycle_inputs_feed=no_cycle_inputs_feed,
+                        all_animals_have_value=all_animals_have_value,
+                        meanDE=meanDE,
+                        meanECHHV=meanECHHV)
+
+        logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)
+
+    return should_run, meanDE, meanECHHV, systems[0] if systems else None
+
+
+def run(cycle: dict):
+    practices = list(filter(should_run_practice(cycle), cycle.get('practices', [])))
+    should_run, meanDE, meanECHHV, system = _should_run(cycle, practices)
+    return _run(cycle, meanDE, meanECHHV, system, practices) if should_run else []

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/hestia_earth/models/ipcc2019/co2ToAirLimeHydrolysis.py

@@ -11,7 +11,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "completeness.soilAmendments": "",
+        "completeness.soilAmendment": "",
         "inputs": [{
             "@type": "Input",
             "value": "",

hestia-earth-models-0.59.6/hestia_earth/models/ipcc2019/co2ToAirSoilCarbonStockChangeManagementChange.py → hestia-earth-models-0.60.0/hestia_earth/models/ipcc2019/co2ToAirSoilOrganicCarbonStockChangeManagementChange.py

@@ -50,7 +50,7 @@ RETURNS = {
         "depth": "30"
     }]
 }
-TERM_ID = 'co2ToAirSoilCarbonStockChangeManagementChange'
+TERM_ID = 'co2ToAirSoilOrganicCarbonStockChangeManagementChange'
 
 DEPTH_UPPER = 0
 DEPTH_LOWER = 30
@@ -101,7 +101,7 @@ MEASUREMENT_METHOD_RANKING = [
 ]
 """
 A ranking of `MeasurementMethodClassification`s from weakest to strongest used to determine the `EmissionMethodTier` of
-the `co2ToAirSoilCarbonStockChangeManagementChange` output.
+the `co2ToAirSoilOrganicCarbonStockChangeManagementChange` output.
 
 The `EmissionMethodTier` should be based on the weakest `MeasurementMethodClassification` between the current SOC and
 previous SOC.

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/hestia_earth/models/ipcc2019/co2ToAirUreaHydrolysis.py

@@ -11,13 +11,8 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "completeness.fertiliser": "",
-            "inputs": [{"@type": "Input", "value": "", "term.termType": "inorganicFertiliser"}]
-        },
-        "optional": {
-            "inputs": [{"@type": "Input", "value": "", "term.@id": "inorganicNitrogenFertiliserUnspecifiedKgN"}]
-        }
+        "completeness.fertiliser": "",
+        "inputs": [{"@type": "Input", "value": "", "term.termType": "inorganicFertiliser"}]
     }
 }
 RETURNS = {

{hestia-earth-models-0.59.6 → hestia-earth-models-0.60.0}/hestia_earth/models/ipcc2019/organicCarbonPerHa.py

@@ -259,6 +259,11 @@ The number of years required for soil organic carbon to reach equilibrium after
 a change in land use, management regime or carbon input regime.
 """
 
+EXCLUDED_ECO_CLIMATE_ZONES_TIER_1 = {
+    5,  # Polar Moist
+    6  # Polar Dry
+}
+
 VALID_SITE_TYPES_TIER_1 = [
     SiteSiteType.CROPLAND.value,
     SiteSiteType.FOREST.value,
@@ -287,7 +292,7 @@ def _measurement(year: int, value: float, method_classification: str) -> dict:
     dict
         A valid Hestia `Measurement` node, see: https://www.hestia.earth/schema/Measurement.
     """
-    measurement = _new_measurement(TERM_ID, MODEL)
+    measurement = _new_measurement(TERM_ID)
     measurement["value"] = [value]
     measurement["dates"] = [f"{year}-12-31"]
     measurement["depthUpper"] = DEPTH_UPPER
@@ -3578,7 +3583,7 @@ def _should_run_tier_1(
     Determines whether there is sufficient data in the inventory and keyword args to run the tier 1 model.
     """
     return all([
-        eco_climate_zone and eco_climate_zone > 0,
+        eco_climate_zone and eco_climate_zone not in EXCLUDED_ECO_CLIMATE_ZONES_TIER_1,
         soc_ref and soc_ref > 0,
         any(year for year, group in inventory.items() if group.get(_InventoryKey.SHOULD_RUN_TIER_1))
     ])