hestia-earth-models 0.59.7__py3-none-any.whl → 0.60.1__py3-none-any.whl

hestia_earth/models/cache_sites.py

@@ -1,5 +1,6 @@
 from functools import reduce
 from enum import Enum
+from pydash.objects import merge
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import flatten
 
@@ -64,12 +65,13 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
             **_cache_results(raster_results, rasters, index),
             **_cache_results(vector_results, vectors, index)
         } | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
-        return {
-            **site,
-            CACHE_KEY: (
-                cached_value(site) | {CACHE_GEOSPATIAL_KEY: cached_data}
-            ) | ({CACHE_YEARS_KEY: years} if years else {})
-        }
+        cached_data = merge(cached_value(site, CACHE_GEOSPATIAL_KEY, {}), cached_data)
+        site_cache = merge(
+            site.get(CACHE_KEY, {}),
+            {CACHE_GEOSPATIAL_KEY: cached_data},
+            ({CACHE_YEARS_KEY: cached_value(site, CACHE_YEARS_KEY, []) + years} if years else {})
+        )
+        return merge(site, {CACHE_KEY: site_cache})
 
     return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
 
@@ -82,7 +84,7 @@ def _preload_regions_area_size(sites: dict):
     return {term_id: download_hestia(term_id).get('area') for term_id in region_ids}
 
 
-def _group_sites(sites: dict):
+def _group_sites(sites: dict, check_has_cache: bool = True):
     # preload area size for all regions
     regions_area_size = _preload_regions_area_size(sites)
 
@@ -90,7 +92,7 @@ def _group_sites(sites: dict):
         return regions_area_size.get(_site_gadm_id(site)) if _should_preload_region_area_size(site) else None
 
     sites = [
-        (n, ) + (_should_run(n, get_region_area_size(n))) for n in sites
+        (n, ) + (_should_run(n, area_size=get_region_area_size(n), check_has_cache=check_has_cache)) for n in sites
     ]
     # restrict sites based on should_cache result
     sites = [(site, area_size) for site, should_cache, area_size in sites if should_cache]
@@ -112,6 +114,24 @@ def _group_sites(sites: dict):
     }
 
 
+def _run(sites: list, years: list, include_region: bool, years_only: bool = False):
+    rasters, vectors = list_collections(years, include_region, years_only)
+    filtered_data = _group_sites(sites, not years_only)
+    return flatten([
+        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
+        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
+    ])
+
+
+def _group_years(years: list, years_range: int):
+    batches = sorted(list(set(list(range(years[0], years[-1] + 1, years_range)) + [years[0], years[-1]])))
+    grouped_batches = [batches[i:i+2] for i in range(0, len(batches))]
+    return [
+        # make sure we don't overlap
+        [v[0] + (0 if v[0] == years[0] else 1), v[1]] for v in grouped_batches if len(v) == 2
+    ]
+
+
 def run(sites: list, years: list = None, include_region: bool = False):
     """
     Run all queries at once for the list of provided Sites.
@@ -127,11 +147,15 @@ def run(sites: list, years: list = None, include_region: bool = False):
         Prefecth region IDs.
         This will cache region-level data and will make the request slower. Only use if needed.
     """
-    rasters, vectors = list_collections(years, include_region)
-
-    filtered_data = _group_sites(sites)
-
-    return flatten([
-        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
-        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
-    ])
+    try:
+        return _run(sites, years, include_region)
+    except Exception as e:
+        # when querying with multiple years, we can reach a compute memory limit, so run the years separately
+        if str(e) == 'User memory limit exceeded.' and years:
+            sites = _run(sites, [], include_region)
+            # query for subranges
+            for sub_years in _group_years(years, years_range=5):
+                sites = _run(sites, sub_years, include_region, years_only=True)
+            return sites
+
+    return []

hestia_earth/models/ipcc2019/animal/pastureGrass.py

@@ -0,0 +1,298 @@
+"""
+Full Grass Consumption
+
+This model estimates the energetic requirements of ruminants and can be used to estimate the amount of grass they graze.
+Source:
+[IPCC 2019, Vol.4, Chapter 10](https://www.ipcc-nggip.iges.or.jp/public/2019rf/pdf/4_Volume4/19R_V4_Ch10_Livestock.pdf).
+"""
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
+from hestia_earth.models.utils.input import _new_input
+from hestia_earth.models.utils.term import get_wool_terms, get_lookup_value
+from hestia_earth.models.utils.completeness import _is_term_type_complete, _is_term_type_incomplete
+from hestia_earth.models.utils.property import get_node_property
+from .. import MODEL
+from ..pastureGrass_utils import (
+    practice_input_id,
+    should_run_practice,
+    calculate_meanDE,
+    calculate_meanECHHV,
+    calculate_REM,
+    calculate_REG,
+    calculate_NEfeed,
+    product_wool_energy,
+    get_animals,
+    get_animal_values
+)
+
+REQUIREMENTS = {
+    "Cycle": {
+        "completeness.animalFeed": "True",
+        "completeness.animalPopulation": "True",
+        "completeness.freshForage": "False",
+        "site": {
+            "@type": "Site",
+            "siteType": "permanent pasture"
+        },
+        "practices": [{
+            "@type": "Practice",
+            "value": "",
+            "term.@id": "pastureGrass",
+            "key": {
+                "@type": "Term",
+                "term.termType": "landCover"
+            }
+        }],
+        "animals": [{
+            "@type": "Animal",
+            "value": "> 0",
+            "term.termType": "liveAnimal",
+            "referencePeriod": "average",
+            "properties": [{
+                "@type": "Property",
+                "value": "",
+                "term.@id": [
+                    "liveweightPerHead",
+                    "weightAtMaturity"
+                ]
+            }],
+            "optional": {
+                "properties": [{
+                    "@type": "Property",
+                    "value": "",
+                    "term.@id": [
+                        "hoursWorkedPerDay",
+                        "pregnancyRateTotal",
+                        "animalsPerBirth"
+                    ]
+                }],
+                "inputs": [{
+                    "@type": "Input",
+                    "term.units": "kg",
+                    "value": "> 0",
+                    "optional": {
+                        "properties": [{
+                            "@type": "Property",
+                            "value": "",
+                            "term.@id": ["neutralDetergentFibreContent", "energyContentHigherHeatingValue"]
+                        }]
+                    }
+                }],
+                "practices": [{
+                    "@type": "Practice",
+                    "value": "",
+                    "term.termType": "animalManagement",
+                    "properties": [{
+                        "@type": "Property",
+                        "value": "",
+                        "term.@id": "fatContent"
+                    }]
+                }]
+            }
+        }],
+        "none": {
+            "inputs": [{
+                "@type": "Input",
+                "term.units": "kg",
+                "value": "> 0",
+                "isAnimalFeed": "True"
+            }]
+        },
+        "optional": {
+            "products": [{
+                "@type": "Product",
+                "value": "",
+                "term.@id": "animalProduct"
+            }]
+        }
+    }
+}
+LOOKUPS = {
+    "animalManagement": [
+        "mjKgEvMilkIpcc2019"
+    ],
+    "animalProduct": ["mjKgEvWoolNetEnergyWoolIpcc2019", "allowedLiveAnimalTermIds"],
+    "liveAnimal": [
+        "ipcc2019AnimalTypeGrouping",
+        "mjDayKgCfiNetEnergyMaintenanceIpcc2019",
+        "ratioCPregnancyNetEnergyPregnancyIpcc2019",
+        "ratioCNetEnergyGrowthCattleBuffaloIpcc2019",
+        "mjKgABNetEnergyGrowthSheepGoatsIpcc2019",
+        "isWoolProducingAnimal"
+    ],
+    "system-liveAnimal-activityCoefficient-ipcc2019": "using animal term @id",
+    "crop-property": ["energyDigestibilityRuminants", "energyContentHigherHeatingValue"],
+    "crop": "grazedPastureGrassInputId",
+    "forage-property": ["energyDigestibilityRuminants", "energyContentHigherHeatingValue"],
+    "landCover": "grazedPastureGrassInputId"
+}
+RETURNS = {
+    "Animal": [{
+        "Input": [{
+            "@type": "Input",
+            "term.termType": ["crop", "forage"],
+            "value": ""
+        }]
+    }]
+}
+MODEL_KEY = 'animal/pastureGrass'
+
+
+def _input(term_id: str, value: float):
+    node = _new_input(term_id, MODEL)
+    node['value'] = [value]
+    return node
+
+
+def _sum_liveWeightPerHead(animals: list):
+    return list_sum([
+        animal.get('value', 0) * get_node_property(animal, 'liveweightPerHead', False).get('value', 0)
+        for animal in animals
+    ])
+
+
+def _isNEwool_animal(animal: dict):
+    value = get_lookup_value(animal.get('term', {}), 'isWoolProducingAnimal', model=MODEL, model_key=MODEL_KEY)
+    return not (not value)
+
+
+def _is_NEwool_product(product: dict, animal: dict):
+    animal_term_ids = get_lookup_value(product, 'allowedLiveAnimalTermIds', model=MODEL, model_key=MODEL_KEY).split(';')
+    return animal.get('term', {}).get('@id') in animal_term_ids
+
+
+def calculate_NEwool(cycle: dict, animal: dict, products: list, total_weight: float) -> float:
+    term_id = animal.get('term', {}).get('@id')
+
+    wool_products = [p for p in products if _is_NEwool_product(p.get('term', {}), animal)]
+    total_energy = list_sum([
+        list_sum(product.get('value', [])) * product_wool_energy(product) for product in wool_products
+    ])
+    animal_weight = _sum_liveWeightPerHead([animal])
+
+    debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                total_energy=total_energy,
+                animal_liveWeightPerHead=animal_weight,
+                total_liveWeightPerHead=total_weight)
+
+    return total_energy * animal_weight/total_weight
+
+
+def _calculate_GE(
+    cycle: dict, animal: dict, REM: float, REG: float, NEwool: float, system: dict
+) -> float:
+    term_id = animal.get('term', {}).get('@id')
+
+    NEm, NEa, NEl, NEwork, NEp, NEg = get_animal_values(cycle, animal, system)
+
+    NEm_feed, NEg_feed = calculate_NEfeed(animal)
+    debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                NEm_feed=NEm_feed,
+                NEg_feed=NEg_feed)
+
+    return (NEm + NEa + NEl + NEwork + NEp - NEm_feed)/REM + (NEg + NEwool - NEg_feed)/REG
+
+
+def _run_practice(cycle: dict, GE: float, meanECHHV: float):
+    def run(practice: dict):
+        key = practice.get('key', {})
+        key_id = key.get('@id')
+        input_term_id = practice_input_id(practice)
+        value = (GE / meanECHHV) * (list_sum(practice.get('value', [0])) / 100)
+
+        logRequirements(cycle, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
+                        GE=GE,
+                        meanECHHV=meanECHHV,
+                        key_id=key_id)
+
+        logShouldRun(cycle, MODEL, input_term_id, True, model_key=MODEL_KEY)
+
+        return _input(input_term_id, value)
+
+    return run
+
+
+def _run_animal(cycle: dict, meanDE: float, meanECHHV: float, system: dict, practices: list):
+    REM = calculate_REM(meanDE)
+    REG = calculate_REG(meanDE)
+
+    wool_term_ids = get_wool_terms()
+    # list of animal product
+    wool_products = [p for p in cycle.get('products', []) if p.get('term', {}).get('@id') in wool_term_ids]
+    animals = list(filter(_isNEwool_animal, cycle.get('animals', [])))
+    total_liveWeightPerHead = _sum_liveWeightPerHead(animals)
+
+    def run(animal: dict):
+        term_id = animal.get('term', {}).get('@id')
+
+        NEwool = calculate_NEwool(cycle, animal, wool_products, total_liveWeightPerHead) if (
+            total_liveWeightPerHead > 0
+        ) else 0
+        GE = (_calculate_GE(cycle, animal, REM, REG, NEwool, system) / (meanDE/100)) if all([REM, REG]) else 0
+
+        debugValues(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                    REM=REM,
+                    REG=REG,
+                    NEwool=NEwool,
+                    GE=GE,
+                    meanDE=meanDE)
+
+        inputs = list(map(_run_practice(cycle, GE, meanECHHV), practices))
+        return animal | {
+            'inputs': animal.get('inputs', []) + inputs
+        }
+
+    return run
+
+
+def _run(cycle: dict, meanDE: float, meanECHHV: float, system: dict, practices: list):
+    animals = get_animals(cycle)
+    return list(map(_run_animal(cycle, meanDE, meanECHHV, system, practices), animals))
+
+
+def _should_run(cycle: dict, practices: dict):
+    systems = filter_list_term_type(cycle.get('practices', []), TermTermType.SYSTEM)
+    animalFeed_complete = _is_term_type_complete(cycle, 'animalFeed')
+    animalPopulation_complete = _is_term_type_complete(cycle, 'animalPopulation')
+    freshForage_incomplete = _is_term_type_incomplete(cycle, 'freshForage')
+    all_animals_have_value = all([a.get('value', 0) > 0 for a in cycle.get('animals', [])])
+
+    no_cycle_inputs_feed = all([not input.get('isAnimalFeed', False) for input in cycle.get('inputs', [])])
+
+    meanDE = calculate_meanDE(practices)
+    meanECHHV = calculate_meanECHHV(practices)
+
+    should_run = all([
+        animalFeed_complete,
+        animalPopulation_complete,
+        freshForage_incomplete,
+        no_cycle_inputs_feed,
+        all_animals_have_value,
+        len(systems) > 0,
+        len(practices) > 0,
+        meanDE > 0,
+        meanECHHV > 0
+    ])
+
+    for term_id in [MODEL_KEY] + [practice_input_id(p) for p in practices]:
+        logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                        term_type_animalFeed_complete=animalFeed_complete,
+                        term_type_animalPopulation_complete=animalPopulation_complete,
+                        term_type_freshForage_incomplete=freshForage_incomplete,
+                        no_cycle_inputs_feed=no_cycle_inputs_feed,
+                        all_animals_have_value=all_animals_have_value,
+                        meanDE=meanDE,
+                        meanECHHV=meanECHHV)
+
+        logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)
+
+    return should_run, meanDE, meanECHHV, systems[0] if systems else None
+
+
+def run(cycle: dict):
+    practices = list(filter(should_run_practice(cycle), cycle.get('practices', [])))
+    should_run, meanDE, meanECHHV, system = _should_run(cycle, practices)
+    return _run(cycle, meanDE, meanECHHV, system, practices) if should_run else []

hestia_earth/models/ipcc2019/{co2ToAirSoilCarbonStockChangeManagementChange.py → co2ToAirSoilOrganicCarbonStockChangeManagementChange.py}

@@ -50,7 +50,7 @@ RETURNS = {
         "depth": "30"
     }]
 }
-TERM_ID = 'co2ToAirSoilCarbonStockChangeManagementChange'
+TERM_ID = 'co2ToAirSoilOrganicCarbonStockChangeManagementChange'
 
 DEPTH_UPPER = 0
 DEPTH_LOWER = 30
@@ -101,7 +101,7 @@ MEASUREMENT_METHOD_RANKING = [
 ]
 """
 A ranking of `MeasurementMethodClassification`s from weakest to strongest used to determine the `EmissionMethodTier` of
-the `co2ToAirSoilCarbonStockChangeManagementChange` output.
+the `co2ToAirSoilOrganicCarbonStockChangeManagementChange` output.
 
 The `EmissionMethodTier` should be based on the weakest `MeasurementMethodClassification` between the current SOC and
 previous SOC.