hestia-earth-models 0.59.6__py3-none-any.whl → 0.60.0__py3-none-any.whl

hestia_earth/models/cache_sites.py

@@ -34,7 +34,7 @@ def _cache_results(results: list, collections: list, index: int):
     return cache_site_results(results[start:end], collections)
 
 
-def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors: list = [], years: list = []):
+def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors: list = [], years: list = None):
     param_values = list(map(_VALUE_AS_PARAM.get(param_type), sites))
     # unique list
     param_values = list(set(param_values)) if param_type == ParamType.GADM_IDS else list({
@@ -66,11 +66,9 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
         } | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
         return {
             **site,
-            CACHE_KEY: {
-                **cached_value(site),
-                CACHE_YEARS_KEY: years,
-                CACHE_GEOSPATIAL_KEY: cached_data
-            }
+            CACHE_KEY: (
+                cached_value(site) | {CACHE_GEOSPATIAL_KEY: cached_data}
+            ) | ({CACHE_YEARS_KEY: years} if years else {})
         }
 
     return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
@@ -114,7 +112,7 @@ def _group_sites(sites: dict):
     }
 
 
-def run(sites: list, years: list, include_region: bool = False):
+def run(sites: list, years: list = None, include_region: bool = False, years_only: bool = False):
     """
     Run all queries at once for the list of provided Sites.
     Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
@@ -128,8 +126,10 @@ def run(sites: list, years: list, include_region: bool = False):
     include_region : bool
         Prefecth region IDs.
         This will cache region-level data and will make the request slower. Only use if needed.
+    years_only : bool
+        Run only the collections that depend on the years (if provided).
     """
-    rasters, vectors = list_collections(years, include_region=include_region)
+    rasters, vectors = list_collections(years, include_region, years_only)
 
     filtered_data = _group_sites(sites)
 

hestia_earth/models/cycle/siteDuration.py

@@ -9,7 +9,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "cycleDuration": "365",
+        "cycleDuration": "> 0",
         "none": {
             "otherSites": ""
         }

hestia_earth/models/emepEea2019/nh3ToAirInorganicFertiliser.py

@@ -17,14 +17,17 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "inputs": [{
+        "completeness.fertiliser": "True",
+        "inputs": [
+            {
                 "@type": "Input",
                 "value": "",
-                "term.termType": ["inorganicFertiliser", "inorganicNitrogenFertiliserUnspecifiedKgN"]
-            }],
-            "completeness.fertiliser": "True"
-        },
+                "term.termType": "inorganicFertiliser",
+                "optional": {
+                    "properties": [{"@type": "Property", "value": "", "term.@id": "nitrogenContent"}]
+                }
+            }
+        ],
         "site": {
             "@type": "Site",
             "country": {"@type": "Term", "termType": "region"},

hestia_earth/models/faostat2018/product/price.py

@@ -41,7 +41,8 @@ LOOKUPS = {
     "liveAnimal": ["primaryMeatProductFaoPriceTermId"],
     "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO"],
     "region-animalProduct-animalProductGroupingFAO-price": "use value from above",
-    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above"
+    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above",
+    "region-animalProduct-animalProductGroupingFAO-weightPerItem": "use value from above"
 }
 MODEL_KEY = 'price'
 LOOKUP_NAME = {
@@ -52,6 +53,9 @@ LOOKUP_GROUPING = {
     TermTermType.CROP.value: get_crop_grouping_faostat_production,
     TermTermType.ANIMALPRODUCT.value: get_animalProduct_grouping_fao
 }
+LOOKUP_UNITS_NUMBER = {
+    TermTermType.ANIMALPRODUCT.value: f"region-{TermTermType.ANIMALPRODUCT.value}-{FAO_LOOKUP_COLUMN}-weightPerItem.csv"
+}
 
 
 def _term_grouping(term: dict): return LOOKUP_GROUPING.get(term.get('termType'), lambda *_: None)(MODEL, term)
@@ -123,6 +127,7 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     product_term = product.get('term', {})
     term_id = product_term.get('@id')
     term_type = product_term.get('termType')
+    term_units = product_term.get('units')
 
     has_yield = len(product.get('value', [])) > 0
     not_already_set = MODEL_KEY not in product.keys()
@@ -133,17 +138,23 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     should_run = all([not_already_set, has_yield, grouping])
     value = _lookup_data(term_id, grouping, country_id, year, term_type=term_type) if should_run else None
 
+    # if units is number instead of kg, need to convert to number first
+    conversion_to_number = safe_parse_float(get_table_value(
+        download_lookup(LOOKUP_UNITS_NUMBER.get(term_type)), 'termid', country_id, column_name(grouping)
+    ), 1) if term_units == Units.NUMBER.value else 1
+
     logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='country',
                     has_yield=has_yield,
                     not_already_set=not_already_set,
                     year=year,
                     price_per_ton=value,
-                    groupingFAO=f"'{grouping}'")
+                    groupingFAO=f"'{grouping}'",
+                    conversion_to_number=conversion_to_number)
 
     logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY, by='country')
 
     # divide by 1000 to convert price per tonne to kg
-    return _product(product, value / 1000) if value is not None else None
+    return _product(product, value / 1000 * conversion_to_number) if value is not None else None
 
 
 def _should_run_product(product: dict):

hestia_earth/models/geospatialDatabase/utils.py

@@ -175,7 +175,6 @@ def download(term: str, site: dict, data: dict, only_coordinates=False) -> dict:
                 }
             ]
         }
-        print(query)
         value = _parse_run_query(term, query)
 
     if value is None:

hestia_earth/models/ipcc2019/animal/pastureGrass.py

@@ -0,0 +1,298 @@
+"""
+Full Grass Consumption
+
+This model estimates the energetic requirements of ruminants and can be used to estimate the amount of grass they graze.
+Source:
+[IPCC 2019, Vol.4, Chapter 10](https://www.ipcc-nggip.iges.or.jp/public/2019rf/pdf/4_Volume4/19R_V4_Ch10_Livestock.pdf).
+"""
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
+from hestia_earth.models.utils.input import _new_input
+from hestia_earth.models.utils.term import get_wool_terms, get_lookup_value
+from hestia_earth.models.utils.completeness import _is_term_type_complete, _is_term_type_incomplete
+from hestia_earth.models.utils.property import get_node_property
+from .. import MODEL
+from ..pastureGrass_utils import (
+    practice_input_id,
+    should_run_practice,
+    calculate_meanDE,
+    calculate_meanECHHV,
+    calculate_REM,
+    calculate_REG,
+    calculate_NEfeed,
+    product_wool_energy,
+    get_animals,
+    get_animal_values
+)
+
+REQUIREMENTS = {
+    "Cycle": {
+        "completeness.animalFeed": "True",
+        "completeness.animalPopulation": "True",
+        "completeness.freshForage": "False",
+        "site": {
+            "@type": "Site",
+            "siteType": "permanent pasture"
+        },
+        "practices": [{
+            "@type": "Practice",
+            "value": "",
+            "term.@id": "pastureGrass",
+            "key": {
+                "@type": "Term",
+                "term.termType": "landCover"
+            }
+        }],
+        "animals": [{
+            "@type": "Animal",
+            "value": "> 0",
+            "term.termType": "liveAnimal",
+            "referencePeriod": "average",
+            "properties": [{
+                "@type": "Property",
+                "value": "",
+                "term.@id": [
+                    "liveweightPerHead",
+                    "weightAtMaturity"
+                ]
+            }],
+            "optional": {
+                "properties": [{
+                    "@type": "Property",
+                    "value": "",
+                    "term.@id": [
+                        "hoursWorkedPerDay",
+                        "pregnancyRateTotal",
+                        "animalsPerBirth"
+                    ]
+                }],
+                "inputs": [{
+                    "@type": "Input",
+                    "term.units": "kg",
+                    "value": "> 0",
+                    "optional": {
+                        "properties": [{
+                            "@type": "Property",
+                            "value": "",
+                            "term.@id": ["neutralDetergentFibreContent", "energyContentHigherHeatingValue"]
+                        }]
+                    }
+                }],
+                "practices": [{
+                    "@type": "Practice",
+                    "value": "",
+                    "term.termType": "animalManagement",
+                    "properties": [{
+                        "@type": "Property",
+                        "value": "",
+                        "term.@id": "fatContent"
+                    }]
+                }]
+            }
+        }],
+        "none": {
+            "inputs": [{
+                "@type": "Input",
+                "term.units": "kg",
+                "value": "> 0",
+                "isAnimalFeed": "True"
+            }]
+        },
+        "optional": {
+            "products": [{
+                "@type": "Product",
+                "value": "",
+                "term.@id": "animalProduct"
+            }]
+        }
+    }
+}
+LOOKUPS = {
+    "animalManagement": [
+        "mjKgEvMilkIpcc2019"
+    ],
+    "animalProduct": ["mjKgEvWoolNetEnergyWoolIpcc2019", "allowedLiveAnimalTermIds"],
+    "liveAnimal": [
+        "ipcc2019AnimalTypeGrouping",
+        "mjDayKgCfiNetEnergyMaintenanceIpcc2019",
+        "ratioCPregnancyNetEnergyPregnancyIpcc2019",
+        "ratioCNetEnergyGrowthCattleBuffaloIpcc2019",
+        "mjKgABNetEnergyGrowthSheepGoatsIpcc2019",
+        "isWoolProducingAnimal"
+    ],
+    "system-liveAnimal-activityCoefficient-ipcc2019": "using animal term @id",
+    "crop-property": ["energyDigestibilityRuminants", "energyContentHigherHeatingValue"],
+    "crop": "grazedPastureGrassInputId",
+    "forage-property": ["energyDigestibilityRuminants", "energyContentHigherHeatingValue"],
+    "landCover": "grazedPastureGrassInputId"
+}
+RETURNS = {
+    "Animal": [{
+        "Input": [{
+            "@type": "Input",
+            "term.termType": ["crop", "forage"],
+            "value": ""
+        }]
+    }]
+}
+MODEL_KEY = 'animal/pastureGrass'
+
+
+def _input(term_id: str, value: float):
+    node = _new_input(term_id, MODEL)
+    node['value'] = [value]
+    return node
+
+
+def _sum_liveWeightPerHead(animals: list):
+    return list_sum([
+        animal.get('value', 0) * get_node_property(animal, 'liveweightPerHead', False).get('value', 0)
+        for animal in animals
+    ])
+
+
+def _isNEwool_animal(animal: dict):
+    value = get_lookup_value(animal.get('term', {}), 'isWoolProducingAnimal', model=MODEL, model_key=MODEL_KEY)
+    return not (not value)
+
+
+def _is_NEwool_product(product: dict, animal: dict):
+    animal_term_ids = get_lookup_value(product, 'allowedLiveAnimalTermIds', model=MODEL, model_key=MODEL_KEY).split(';')
+    return animal.get('term', {}).get('@id') in animal_term_ids
+
+
+def calculate_NEwool(cycle: dict, animal: dict, products: list, total_weight: float) -> float:
+    term_id = animal.get('term', {}).get('@id')
+
+    wool_products = [p for p in products if _is_NEwool_product(p.get('term', {}), animal)]
+    total_energy = list_sum([
+        list_sum(product.get('value', [])) * product_wool_energy(product) for product in wool_products
+    ])
+    animal_weight = _sum_liveWeightPerHead([animal])
+
+    debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                total_energy=total_energy,
+                animal_liveWeightPerHead=animal_weight,
+                total_liveWeightPerHead=total_weight)
+
+    return total_energy * animal_weight/total_weight
+
+
+def _calculate_GE(
+    cycle: dict, animal: dict, REM: float, REG: float, NEwool: float, system: dict
+) -> float:
+    term_id = animal.get('term', {}).get('@id')
+
+    NEm, NEa, NEl, NEwork, NEp, NEg = get_animal_values(cycle, animal, system)
+
+    NEm_feed, NEg_feed = calculate_NEfeed(animal)
+    debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                NEm_feed=NEm_feed,
+                NEg_feed=NEg_feed)
+
+    return (NEm + NEa + NEl + NEwork + NEp - NEm_feed)/REM + (NEg + NEwool - NEg_feed)/REG
+
+
+def _run_practice(cycle: dict, GE: float, meanECHHV: float):
+    def run(practice: dict):
+        key = practice.get('key', {})
+        key_id = key.get('@id')
+        input_term_id = practice_input_id(practice)
+        value = (GE / meanECHHV) * (list_sum(practice.get('value', [0])) / 100)
+
+        logRequirements(cycle, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
+                        GE=GE,
+                        meanECHHV=meanECHHV,
+                        key_id=key_id)
+
+        logShouldRun(cycle, MODEL, input_term_id, True, model_key=MODEL_KEY)
+
+        return _input(input_term_id, value)
+
+    return run
+
+
+def _run_animal(cycle: dict, meanDE: float, meanECHHV: float, system: dict, practices: list):
+    REM = calculate_REM(meanDE)
+    REG = calculate_REG(meanDE)
+
+    wool_term_ids = get_wool_terms()
+    # list of animal product
+    wool_products = [p for p in cycle.get('products', []) if p.get('term', {}).get('@id') in wool_term_ids]
+    animals = list(filter(_isNEwool_animal, cycle.get('animals', [])))
+    total_liveWeightPerHead = _sum_liveWeightPerHead(animals)
+
+    def run(animal: dict):
+        term_id = animal.get('term', {}).get('@id')
+
+        NEwool = calculate_NEwool(cycle, animal, wool_products, total_liveWeightPerHead) if (
+            total_liveWeightPerHead > 0
+        ) else 0
+        GE = (_calculate_GE(cycle, animal, REM, REG, NEwool, system) / (meanDE/100)) if all([REM, REG]) else 0
+
+        debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                    REM=REM,
+                    REG=REG,
+                    NEwool=NEwool,
+                    GE=GE,
+                    meanDE=meanDE)
+
+        inputs = list(map(_run_practice(cycle, GE, meanECHHV), practices))
+        return animal | {
+            'inputs': animal.get('inputs', []) + inputs
+        }
+
+    return run
+
+
+def _run(cycle: dict, meanDE: float, meanECHHV: float, system: dict, practices: list):
+    animals = get_animals(cycle)
+    return list(map(_run_animal(cycle, meanDE, meanECHHV, system, practices), animals))
+
+
+def _should_run(cycle: dict, practices: dict):
+    systems = filter_list_term_type(cycle.get('practices', []), TermTermType.SYSTEM)
+    animalFeed_complete = _is_term_type_complete(cycle, 'animalFeed')
+    animalPopulation_complete = _is_term_type_complete(cycle, 'animalPopulation')
+    freshForage_incomplete = _is_term_type_incomplete(cycle, 'freshForage')
+    all_animals_have_value = all([a.get('value', 0) > 0 for a in cycle.get('animals', [])])
+
+    no_cycle_inputs_feed = all([not input.get('isAnimalFeed', False) for input in cycle.get('inputs', [])])
+
+    meanDE = calculate_meanDE(practices)
+    meanECHHV = calculate_meanECHHV(practices)
+
+    should_run = all([
+        animalFeed_complete,
+        animalPopulation_complete,
+        freshForage_incomplete,
+        no_cycle_inputs_feed,
+        all_animals_have_value,
+        len(systems) > 0,
+        len(practices) > 0,
+        meanDE > 0,
+        meanECHHV > 0
+    ])
+
+    for term_id in [MODEL_KEY] + [practice_input_id(p) for p in practices]:
+        logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                        term_type_animalFeed_complete=animalFeed_complete,
+                        term_type_animalPopulation_complete=animalPopulation_complete,
+                        term_type_freshForage_incomplete=freshForage_incomplete,
+                        no_cycle_inputs_feed=no_cycle_inputs_feed,
+                        all_animals_have_value=all_animals_have_value,
+                        meanDE=meanDE,
+                        meanECHHV=meanECHHV)
+
+        logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)
+
+    return should_run, meanDE, meanECHHV, systems[0] if systems else None
+
+
+def run(cycle: dict):
+    practices = list(filter(should_run_practice(cycle), cycle.get('practices', [])))
+    should_run, meanDE, meanECHHV, system = _should_run(cycle, practices)
+    return _run(cycle, meanDE, meanECHHV, system, practices) if should_run else []

hestia_earth/models/ipcc2019/co2ToAirLimeHydrolysis.py

@@ -11,7 +11,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "completeness.soilAmendments": "",
+        "completeness.soilAmendment": "",
         "inputs": [{
             "@type": "Input",
             "value": "",

hestia_earth/models/ipcc2019/{co2ToAirSoilCarbonStockChangeManagementChange.py → co2ToAirSoilOrganicCarbonStockChangeManagementChange.py}

@@ -50,7 +50,7 @@ RETURNS = {
         "depth": "30"
     }]
 }
-TERM_ID = 'co2ToAirSoilCarbonStockChangeManagementChange'
+TERM_ID = 'co2ToAirSoilOrganicCarbonStockChangeManagementChange'
 
 DEPTH_UPPER = 0
 DEPTH_LOWER = 30
@@ -101,7 +101,7 @@ MEASUREMENT_METHOD_RANKING = [
 ]
 """
 A ranking of `MeasurementMethodClassification`s from weakest to strongest used to determine the `EmissionMethodTier` of
-the `co2ToAirSoilCarbonStockChangeManagementChange` output.
+the `co2ToAirSoilOrganicCarbonStockChangeManagementChange` output.
 
 The `EmissionMethodTier` should be based on the weakest `MeasurementMethodClassification` between the current SOC and
 previous SOC.

hestia_earth/models/ipcc2019/co2ToAirUreaHydrolysis.py

@@ -11,13 +11,8 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "completeness.fertiliser": "",
-            "inputs": [{"@type": "Input", "value": "", "term.termType": "inorganicFertiliser"}]
-        },
-        "optional": {
-            "inputs": [{"@type": "Input", "value": "", "term.@id": "inorganicNitrogenFertiliserUnspecifiedKgN"}]
-        }
+        "completeness.fertiliser": "",
+        "inputs": [{"@type": "Input", "value": "", "term.termType": "inorganicFertiliser"}]
     }
 }
 RETURNS = {

hestia_earth/models/ipcc2019/organicCarbonPerHa.py

@@ -259,6 +259,11 @@ The number of years required for soil organic carbon to reach equilibrium after
 a change in land use, management regime or carbon input regime.
 """
 
+EXCLUDED_ECO_CLIMATE_ZONES_TIER_1 = {
+    5,  # Polar Moist
+    6  # Polar Dry
+}
+
 VALID_SITE_TYPES_TIER_1 = [
     SiteSiteType.CROPLAND.value,
     SiteSiteType.FOREST.value,
@@ -287,7 +292,7 @@ def _measurement(year: int, value: float, method_classification: str) -> dict:
     dict
         A valid Hestia `Measurement` node, see: https://www.hestia.earth/schema/Measurement.
     """
-    measurement = _new_measurement(TERM_ID, MODEL)
+    measurement = _new_measurement(TERM_ID)
     measurement["value"] = [value]
     measurement["dates"] = [f"{year}-12-31"]
     measurement["depthUpper"] = DEPTH_UPPER
@@ -3578,7 +3583,7 @@ def _should_run_tier_1(
     Determines whether there is sufficient data in the inventory and keyword args to run the tier 1 model.
     """
     return all([
-        eco_climate_zone and eco_climate_zone > 0,
+        eco_climate_zone and eco_climate_zone not in EXCLUDED_ECO_CLIMATE_ZONES_TIER_1,
         soc_ref and soc_ref > 0,
         any(year for year, group in inventory.items() if group.get(_InventoryKey.SHOULD_RUN_TIER_1))
     ])