hestia-earth-models 0.59.5__py3-none-any.whl → 0.59.7__py3-none-any.whl

hestia_earth/models/cache_sites.py

@@ -34,7 +34,7 @@ def _cache_results(results: list, collections: list, index: int):
     return cache_site_results(results[start:end], collections)
 
 
-def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors: list = [], years: list = []):
+def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors: list = [], years: list = None):
     param_values = list(map(_VALUE_AS_PARAM.get(param_type), sites))
     # unique list
     param_values = list(set(param_values)) if param_type == ParamType.GADM_IDS else list({
@@ -66,11 +66,9 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
         } | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
         return {
             **site,
-            CACHE_KEY: {
-                **cached_value(site),
-                CACHE_YEARS_KEY: years,
-                CACHE_GEOSPATIAL_KEY: cached_data
-            }
+            CACHE_KEY: (
+                cached_value(site) | {CACHE_GEOSPATIAL_KEY: cached_data}
+            ) | ({CACHE_YEARS_KEY: years} if years else {})
         }
 
     return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
@@ -114,7 +112,7 @@ def _group_sites(sites: dict):
     }
 
 
-def run(sites: list, years: list, include_region: bool = False):
+def run(sites: list, years: list = None, include_region: bool = False):
     """
     Run all queries at once for the list of provided Sites.
     Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
@@ -129,7 +127,7 @@ def run(sites: list, years: list, include_region: bool = False):
         Prefecth region IDs.
         This will cache region-level data and will make the request slower. Only use if needed.
     """
-    rasters, vectors = list_collections(years, include_region=include_region)
+    rasters, vectors = list_collections(years, include_region)
 
     filtered_data = _group_sites(sites)
 

hestia_earth/models/cycle/animal/milkYield.py

@@ -0,0 +1,86 @@
+"""
+Milk Yield
+
+This model gap-fills the practice "Milk yield per animal X (raw/FPCM)" (e.g. `Milk yield per cow (raw)`) when:
+- the practice is added to the Cycle itself, and we can match it with the `term` of an Animal blank node;
+- the Animal blank node Term has a lookup value of `kgDayMilkForFeedingOffspring`.
+"""
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import non_empty_list, safe_parse_float
+
+from hestia_earth.models.log import logShouldRun, logRequirements, log_blank_nodes_id
+from hestia_earth.models.utils.term import get_lookup_value
+from hestia_earth.models.utils.practice import _new_practice
+from .. import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "animals": [{
+            "@type": "Animal",
+            "term.termType": "liveAnimal"
+        }],
+        "optional": {
+            "practices": [{
+                "@type": "Practice",
+                "term.termType": "animalManagement"
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Animal": [{
+        "practices": [{
+            "@type": "Practice",
+            "term.termType": "animalManagement"
+        }]
+    }]
+}
+LOOKUPS = {
+    "liveAnimal": ["milkYieldPracticeTermIds", "kgDayMilkForFeedingOffspring"]
+}
+
+MODEL_KEY = 'milkYield'
+
+
+def _default_practice(animal: dict, practice_term_id: str):
+    term = animal.get('term', {})
+    value = get_lookup_value(term, LOOKUPS['liveAnimal'][1], model=MODEL, model_key=MODEL_KEY)
+    return (_new_practice(practice_term_id) | {'value': [safe_parse_float(value)]}) if value else None
+
+
+def _run(cycle: dict, animal: dict):
+    term = animal.get('term', {})
+    term_id = term.get('@id')
+    value = get_lookup_value(term, LOOKUPS['liveAnimal'][0], model=MODEL, model_key=MODEL_KEY)
+    practice_ids = non_empty_list((value or '').split(';'))
+    practices = non_empty_list(
+        [p for p in cycle.get('practices', []) if p.get('term', {}).get('@id') in practice_ids] or (
+            [_default_practice(animal, practice_ids[0])] if practice_ids else []
+        )
+    )
+
+    logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                    practice_ids=log_blank_nodes_id(practices))
+
+    for practice in practices:
+        logShouldRun(cycle, MODEL, practice.get('term', {}).get('@id'), True, model_key=MODEL_KEY)
+
+    return {
+        **animal,
+        'practices': animal.get('practices', []) + practices
+    } if practices else None
+
+
+def _should_run(cycle: dict):
+    animals = filter_list_term_type(cycle.get('animals', []), TermTermType.LIVEANIMAL)
+    has_animals = len(animals) > 0
+
+    should_run = all([has_animals])
+    logShouldRun(cycle, MODEL, None, should_run, model_key=MODEL_KEY)
+    return should_run, animals
+
+
+def run(cycle: dict):
+    should_run, animals = _should_run(cycle)
+    return non_empty_list([_run(cycle, a) for a in animals]) if should_run else []

hestia_earth/models/cycle/endDate.py

@@ -0,0 +1,50 @@
+"""
+Start Date
+
+This model updates the [Cycle endDate](https://hestia.earth/schema/Cycle#endDate) to be in the following format:
+`YYYY-MM-DD`.
+"""
+from hestia_earth.utils.date import is_in_days
+from hestia_earth.utils.tools import non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import last_day_of_month
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "endDate": ""
+    }
+}
+RETURNS = {
+    "The endDate as a string": ""
+}
+MODEL_KEY = 'endDate'
+
+
+def _last_day(date: str):
+    last_day = last_day_of_month(date[0:4], date[5:7])
+    return str(last_day.day)
+
+
+def _run(cycle: dict):
+    value = cycle.get('endDate')
+    month = '12' if len(value) == 4 else ''
+    day = '31' if len(value) == 4 else _last_day(value)
+    return '-'.join(non_empty_list([value, month, day]))
+
+
+def _should_run(cycle: dict):
+    has_endDate = cycle.get('endDate') is not None
+    has_incorrect_format = has_endDate and not is_in_days(cycle.get('endDate'))
+
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
+                    has_endDate=has_endDate,
+                    has_incorrect_format=has_incorrect_format)
+
+    should_run = all([has_endDate, has_incorrect_format])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else None

hestia_earth/models/cycle/inorganicFertiliser.py

@@ -34,7 +34,7 @@ RETURNS = {
 }
 LOOKUPS = {
     "inorganicFertiliser": [
-        "mustIncludeId",
+        "complementaryTermIds",
         "nitrogenContent", "nitrogenContent-min", "nitrogenContent-max",
         "phosphateContentAsP2O5", "phosphateContentAsP2O5-min", "phosphateContentAsP2O5-max",
         "potassiumContentAsK2O", "potassiumContentAsK2O-min", "potassiumContentAsK2O-max"
@@ -75,7 +75,8 @@ def _input(term_id: str, value: float, min: float = None, max: float = None):
     return input
 
 
-def _include_term_ids(term_id: str): return non_empty_list((get_term_lookup(term_id, 'mustIncludeId') or '').split(';'))
+def _include_term_ids(term_id: str):
+    return non_empty_list((get_term_lookup(term_id, LOOKUPS['inorganicFertiliser'][0]) or '').split(';'))
 
 
 def _run_input(cycle: dict, input: dict):

hestia_earth/models/cycle/milkYield.py

@@ -48,7 +48,7 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
-    "animalProduct": ["liveAnimalTermId", "milkYieldPracticeId"]
+    "animalProduct": ["liveAnimalTermId", "milkYieldPracticeTermId"]
 }
 
 MODEL_KEY = 'milkYield'
@@ -74,11 +74,16 @@ def practice(term_id: str, value: float, properties: list, sd: float = None, min
     return data
 
 
+def _practice_id(term: dict):
+    value = get_lookup_value(term, LOOKUPS['animalProduct'][1], model=MODEL, model_key=MODEL_KEY)
+    return value.split(';')[0] if value else None
+
+
 def _run(cycle: dict, product: dict):
     cycleDuration = cycle.get('cycleDuration')
 
     term = product.get('term', {})
-    practice_id = get_lookup_value(term, 'milkYieldPracticeId')
+    practice_id = _practice_id(term)
 
     live_animal_term_id = _get_liveAnimal_term_id(product, model_key=MODEL_KEY)
     live_animal_node = find_term_match(cycle.get('animals', []), live_animal_term_id)
@@ -114,7 +119,7 @@ def _should_run_product(cycle: dict, product: dict):
     live_animal_node = find_term_match(cycle.get('animals', []), live_animal_term_id)
     has_live_animal_node_value = live_animal_node.get('value', 0) > 0
 
-    practice_id = get_lookup_value(term, 'milkYieldPracticeId', model=MODEL, model_key=MODEL_KEY)
+    practice_id = _practice_id(term)
     has_practice_id = any([
         find_term_match(cycle.get('practices', []), practice_id),
         find_term_match((live_animal_node or {}).get('practices', []), practice_id)

hestia_earth/models/cycle/pre_checks/__init__.py

@@ -2,14 +2,13 @@ from os.path import dirname, abspath
 import sys
 
 from hestia_earth.models.utils import _run_in_serie
-from . import site, startDate, cache_sources
+from . import site, cache_sources
 
 CURRENT_DIR = dirname(abspath(__file__)) + '/'
 sys.path.append(CURRENT_DIR)
 
 MODELS = [
     site.run,
-    startDate.run,
     cache_sources.run
 ]
 

hestia_earth/models/cycle/siteDuration.py

@@ -9,7 +9,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "cycleDuration": "365",
+        "cycleDuration": "> 0",
         "none": {
             "otherSites": ""
         }

hestia_earth/models/cycle/startDate.py

@@ -0,0 +1,42 @@
+"""
+Start Date
+
+This model updates the [Cycle startDate](https://hestia.earth/schema/Cycle#startDate) to be in the following format:
+`YYYY-MM-DD`.
+"""
+from hestia_earth.utils.date import is_in_days
+from hestia_earth.utils.tools import non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "startDate": ""
+    }
+}
+RETURNS = {
+    "The startDate as a string": ""
+}
+MODEL_KEY = 'startDate'
+
+
+def _run(cycle: dict):
+    value = cycle.get('startDate')
+    return '-'.join(non_empty_list([value, '01' if len(value) == 4 else '', '01']))
+
+
+def _should_run(cycle: dict):
+    has_startDate = cycle.get('startDate') is not None
+    has_incorrect_format = has_startDate and not is_in_days(cycle.get('startDate'))
+
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
+                    has_startDate=has_startDate,
+                    has_incorrect_format=has_incorrect_format)
+
+    should_run = all([has_startDate, has_incorrect_format])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else None

hestia_earth/models/emepEea2019/nh3ToAirInorganicFertiliser.py

@@ -17,14 +17,17 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "inputs": [{
+        "completeness.fertiliser": "True",
+        "inputs": [
+            {
                 "@type": "Input",
                 "value": "",
-                "term.termType": ["inorganicFertiliser", "inorganicNitrogenFertiliserUnspecifiedKgN"]
-            }],
-            "completeness.fertiliser": "True"
-        },
+                "term.termType": "inorganicFertiliser",
+                "optional": {
+                    "properties": [{"@type": "Property", "value": "", "term.@id": "nitrogenContent"}]
+                }
+            }
+        ],
         "site": {
             "@type": "Site",
             "country": {"@type": "Term", "termType": "region"},

hestia_earth/models/faostat2018/liveweightPerHead.py

@@ -1,20 +1,25 @@
 from hestia_earth.schema import TermTermType
+from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.lookup import get_table_value, column_name, download_lookup, extract_grouped_data_closest_date
 from hestia_earth.utils.model import filter_list_term_type
-from hestia_earth.utils.tools import non_empty_list, safe_parse_date
+from hestia_earth.utils.tools import non_empty_list, safe_parse_date, safe_parse_float
 
-from hestia_earth.models.log import logRequirements, logShouldRun
-from hestia_earth.models.utils import _filter_list_term_unit
+from hestia_earth.models.log import logRequirements, logShouldRun, debugMissingLookup
 from hestia_earth.models.utils.constant import Units
 from hestia_earth.models.utils.property import _new_property, node_has_no_property
+from hestia_earth.models.utils.product import convert_product_to_unit
+from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_lookup_value
+from .utils import get_liveAnimal_to_animalProduct_id
 from . import MODEL
-from .utils import product_equivalent_value
 
 REQUIREMENTS = {
     "Cycle": {
         "endDate": "",
-        "products": [
-            {"@type": "Product", "value": "", "term.termType": "animalProduct", "term.units": "kg liveweight"}
-        ],
+        "products": [{
+            "@type": "Product",
+            "value": "",
+            "term.termType": ["animalProduct", "liveAnimal"]
+        }],
         "site": {
             "@type": "Site",
             "country": {"@type": "Term", "termType": "region"}
@@ -22,8 +27,9 @@ REQUIREMENTS = {
     }
 }
 LOOKUPS = {
-    "region-animalProduct-animalProductGroupingFAO-productionQuantity": "production quantity",
-    "region-animalProduct-animalProductGroupingFAO-head": "number of heads"
+    "liveAnimal": ["primaryMeatProductFaoProductionTermId"],
+    "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO"],
+    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above"
 }
 RETURNS = {
     "Product": [{
@@ -34,6 +40,7 @@ RETURNS = {
     }]
 }
 TERM_ID = 'liveweightPerHead'
+LOOKUP_WEIGHT = 'region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight.csv'
 
 
 def _property(value: float):
@@ -42,47 +49,76 @@ def _property(value: float):
     return prop
 
 
-def _should_run_product(cycle: dict, year: int, country: str):
-    def exec(product: dict):
-        product_id = product.get('term', {}).get('@id')
-        value = product_equivalent_value(product, year, country)
-        should_run = all([value])
-        logShouldRun(cycle, MODEL, product_id, should_run, property=TERM_ID)
-        return should_run, product, value
-    return exec
+def _product_value(product: dict, year: int, country_id: str):
+    product_id = product.get('term', {}).get('@id')
+    groupingFAO = get_animalProduct_lookup_value(MODEL, product_id, FAO_LOOKUP_COLUMN)
 
+    lookup = download_lookup(LOOKUP_WEIGHT)
+    data = get_table_value(lookup, 'termid', country_id, column_name(groupingFAO))
+    debugMissingLookup(LOOKUP_WEIGHT, 'termid', country_id, groupingFAO, data, model=MODEL, term=TERM_ID)
+    average_carcass_weight = safe_parse_float(extract_grouped_data_closest_date(data, year), None)
+    # average_carcass_weight is in hg, divide by 10 to go back to kg
+    kg_carcass_weight = average_carcass_weight / 10 if average_carcass_weight else None
 
-def _run(products: list):
-    def run_product(values: tuple):
-        product, value = values
-        prop = _property(value)
-        return {**product, 'properties': product.get('properties', []) + [prop]} if prop else None
+    kg_liveweight = convert_product_to_unit({
+        **product,
+        'value': [kg_carcass_weight]
+    }, Units.KG_LIVEWEIGHT) if kg_carcass_weight else None
 
-    return non_empty_list(map(run_product, products))
+    return kg_liveweight, groupingFAO
 
 
-def _should_run(cycle: dict):
-    products = filter_list_term_type(cycle.get('products', []), TermTermType.ANIMALPRODUCT)
-    products = _filter_list_term_unit(products, Units.KG_LIVEWEIGHT)
-    products = list(filter(node_has_no_property(TERM_ID), products))
-    has_kg_liveweight_products = len(products) > 0
+def _should_run_liveAnimal(product: dict):
+    return product.get('term', {}).get('termType') == TermTermType.LIVEANIMAL.value
 
-    end_date = safe_parse_date(cycle.get('endDate'))
-    year = end_date.year if end_date else None
-    country = cycle.get('site', {}).get('country', {}).get('@id')
 
-    logRequirements(cycle, model=MODEL, term=TERM_ID,
-                    has_kg_liveweight_products=has_kg_liveweight_products,
+def _run_liveAnimal(cycle: dict, product: dict, year: int, country_id: str):
+    # find the animalProduct to get the average carcass weight
+    product_id = product.get('term', {}).get('@id')
+    animal_product_id = get_liveAnimal_to_animalProduct_id(product_id, LOOKUPS['liveAnimal'][0], term=TERM_ID)
+
+    animal_product_term = download_hestia(animal_product_id)
+    kg_liveweight, groupingFAO = _product_value({**product, 'term': animal_product_term}, year, country_id)
+
+    logRequirements(cycle, model=MODEL, term=product_id, property=TERM_ID,
+                    animal_product_id=animal_product_id,
+                    country_id=country_id,
                     year=year,
-                    country=country)
+                    kg_liveweight=kg_liveweight,
+                    groupingFAO=groupingFAO)
 
-    should_run = all([has_kg_liveweight_products, year, country])
-    logShouldRun(cycle, MODEL, TERM_ID, should_run)
-    return should_run, products, year, country
+    should_run = all([kg_liveweight])
+    logShouldRun(cycle, MODEL, product_id, should_run, property=TERM_ID)
+
+    return {**product, 'properties': product.get('properties', []) + [_property(kg_liveweight)]} if should_run else None
+
+
+def _run_animalProduct(cycle: dict, product: dict, year: int, country_id: str):
+    product_id = product.get('term', {}).get('@id')
+    kg_liveweight, groupingFAO = _product_value(product, year, country_id)
+
+    logRequirements(cycle, model=MODEL, term=product_id, property=TERM_ID,
+                    country_id=country_id,
+                    year=year,
+                    kg_liveweight=kg_liveweight,
+                    groupingFAO=groupingFAO)
+
+    should_run = all([kg_liveweight])
+    logShouldRun(cycle, MODEL, product_id, should_run, property=TERM_ID)
+
+    return {**product, 'properties': product.get('properties', []) + [_property(kg_liveweight)]} if should_run else None
 
 
 def run(cycle: dict):
-    should_run, products, year, country = _should_run(cycle)
-    products = list(map(_should_run_product(cycle, year, country), products)) if should_run else []
-    products = [(product, value) for (should_run, product, value) in products if should_run]
-    return _run(products) if should_run else []
+    country_id = cycle.get('site', {}).get('country', {}).get('@id')
+    end_date = safe_parse_date(cycle.get('endDate'))
+    year = end_date.year if end_date else None
+
+    products = filter_list_term_type(cycle.get('products', []), [TermTermType.ANIMALPRODUCT, TermTermType.LIVEANIMAL])
+    products = list(filter(node_has_no_property(TERM_ID), products))
+
+    return non_empty_list([
+        (
+            (_run_liveAnimal if _should_run_liveAnimal(p) else _run_animalProduct)(cycle, p, year, country_id)
+        ) for p in products
+    ]) if country_id else []