hestia-earth-models 0.59.4__tar.gz → 0.59.6__tar.gz

{hestia-earth-models-0.59.4/hestia_earth_models.egg-info → hestia-earth-models-0.59.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.59.4
+Version: 0.59.6
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team

hestia-earth-models-0.59.6/hestia_earth/models/cycle/animal/milkYield.py

@@ -0,0 +1,86 @@
+"""
+Milk Yield
+
+This model gap-fills the practice "Milk yield per animal X (raw/FPCM)" (e.g. `Milk yield per cow (raw)`) when:
+- the practice is added to the Cycle itself, and we can match it with the `term` of an Animal blank node;
+- the Animal blank node Term has a lookup value of `kgDayMilkForFeedingOffspring`.
+"""
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import non_empty_list, safe_parse_float
+
+from hestia_earth.models.log import logShouldRun, logRequirements, log_blank_nodes_id
+from hestia_earth.models.utils.term import get_lookup_value
+from hestia_earth.models.utils.practice import _new_practice
+from .. import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "animals": [{
+            "@type": "Animal",
+            "term.termType": "liveAnimal"
+        }],
+        "optional": {
+            "practices": [{
+                "@type": "Practice",
+                "term.termType": "animalManagement"
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Animal": [{
+        "practices": [{
+            "@type": "Practice",
+            "term.termType": "animalManagement"
+        }]
+    }]
+}
+LOOKUPS = {
+    "liveAnimal": ["milkYieldPracticeTermIds", "kgDayMilkForFeedingOffspring"]
+}
+
+MODEL_KEY = 'milkYield'
+
+
+def _default_practice(animal: dict, practice_term_id: str):
+    term = animal.get('term', {})
+    value = get_lookup_value(term, LOOKUPS['liveAnimal'][1], model=MODEL, model_key=MODEL_KEY)
+    return (_new_practice(practice_term_id) | {'value': [safe_parse_float(value)]}) if value else None
+
+
+def _run(cycle: dict, animal: dict):
+    term = animal.get('term', {})
+    term_id = term.get('@id')
+    value = get_lookup_value(term, LOOKUPS['liveAnimal'][0], model=MODEL, model_key=MODEL_KEY)
+    practice_ids = non_empty_list((value or '').split(';'))
+    practices = non_empty_list(
+        [p for p in cycle.get('practices', []) if p.get('term', {}).get('@id') in practice_ids] or (
+            [_default_practice(animal, practice_ids[0])] if practice_ids else []
+        )
+    )
+
+    logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                    practice_ids=log_blank_nodes_id(practices))
+
+    for practice in practices:
+        logShouldRun(cycle, MODEL, practice.get('term', {}).get('@id'), True, model_key=MODEL_KEY)
+
+    return {
+        **animal,
+        'practices': animal.get('practices', []) + practices
+    } if practices else None
+
+
+def _should_run(cycle: dict):
+    animals = filter_list_term_type(cycle.get('animals', []), TermTermType.LIVEANIMAL)
+    has_animals = len(animals) > 0
+
+    should_run = all([has_animals])
+    logShouldRun(cycle, MODEL, None, should_run, model_key=MODEL_KEY)
+    return should_run, animals
+
+
+def run(cycle: dict):
+    should_run, animals = _should_run(cycle)
+    return non_empty_list([_run(cycle, a) for a in animals]) if should_run else []

hestia-earth-models-0.59.6/hestia_earth/models/cycle/endDate.py

@@ -0,0 +1,50 @@
+"""
+Start Date
+
+This model updates the [Cycle endDate](https://hestia.earth/schema/Cycle#endDate) to be in the following format:
+`YYYY-MM-DD`.
+"""
+from hestia_earth.utils.date import is_in_days
+from hestia_earth.utils.tools import non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import last_day_of_month
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "endDate": ""
+    }
+}
+RETURNS = {
+    "The endDate as a string": ""
+}
+MODEL_KEY = 'endDate'
+
+
+def _last_day(date: str):
+    last_day = last_day_of_month(date[0:4], date[5:7])
+    return str(last_day.day)
+
+
+def _run(cycle: dict):
+    value = cycle.get('endDate')
+    month = '12' if len(value) == 4 else ''
+    day = '31' if len(value) == 4 else _last_day(value)
+    return '-'.join(non_empty_list([value, month, day]))
+
+
+def _should_run(cycle: dict):
+    has_endDate = cycle.get('endDate') is not None
+    has_incorrect_format = has_endDate and not is_in_days(cycle.get('endDate'))
+
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
+                    has_endDate=has_endDate,
+                    has_incorrect_format=has_incorrect_format)
+
+    should_run = all([has_endDate, has_incorrect_format])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else None

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/cycle/inorganicFertiliser.py

@@ -34,7 +34,7 @@ RETURNS = {
 }
 LOOKUPS = {
     "inorganicFertiliser": [
-        "mustIncludeId",
+        "complementaryTermIds",
         "nitrogenContent", "nitrogenContent-min", "nitrogenContent-max",
         "phosphateContentAsP2O5", "phosphateContentAsP2O5-min", "phosphateContentAsP2O5-max",
         "potassiumContentAsK2O", "potassiumContentAsK2O-min", "potassiumContentAsK2O-max"
@@ -75,7 +75,8 @@ def _input(term_id: str, value: float, min: float = None, max: float = None):
     return input
 
 
-def _include_term_ids(term_id: str): return non_empty_list((get_term_lookup(term_id, 'mustIncludeId') or '').split(';'))
+def _include_term_ids(term_id: str):
+    return non_empty_list((get_term_lookup(term_id, LOOKUPS['inorganicFertiliser'][0]) or '').split(';'))
 
 
 def _run_input(cycle: dict, input: dict):

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/cycle/liveAnimal.py

@@ -43,6 +43,9 @@ RETURNS = {
         "value": ""
     }]
 }
+LOOKUPS = {
+    "animalProduct": "liveAnimalTermId"
+}
 MODEL_KEY = 'liveAnimal'
 VALID_SITE_TYPES = [SiteSiteType.ANIMAL_HOUSING.value, SiteSiteType.PERMANENT_PASTURE.value]
 

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/cycle/milkYield.py

@@ -48,7 +48,7 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
-    "animalProduct": ["liveAnimal", "milkYieldPracticeId"]
+    "animalProduct": ["liveAnimalTermId", "milkYieldPracticeTermId"]
 }
 
 MODEL_KEY = 'milkYield'
@@ -74,11 +74,16 @@ def practice(term_id: str, value: float, properties: list, sd: float = None, min
     return data
 
 
+def _practice_id(term: dict):
+    value = get_lookup_value(term, LOOKUPS['animalProduct'][1], model=MODEL, model_key=MODEL_KEY)
+    return value.split(';')[0] if value else None
+
+
 def _run(cycle: dict, product: dict):
     cycleDuration = cycle.get('cycleDuration')
 
     term = product.get('term', {})
-    practice_id = get_lookup_value(term, 'milkYieldPracticeId')
+    practice_id = _practice_id(term)
 
     live_animal_term_id = _get_liveAnimal_term_id(product, model_key=MODEL_KEY)
     live_animal_node = find_term_match(cycle.get('animals', []), live_animal_term_id)
@@ -114,7 +119,7 @@ def _should_run_product(cycle: dict, product: dict):
     live_animal_node = find_term_match(cycle.get('animals', []), live_animal_term_id)
     has_live_animal_node_value = live_animal_node.get('value', 0) > 0
 
-    practice_id = get_lookup_value(term, 'milkYieldPracticeId', model=MODEL, model_key=MODEL_KEY)
+    practice_id = _practice_id(term)
     has_practice_id = any([
         find_term_match(cycle.get('practices', []), practice_id),
         find_term_match((live_animal_node or {}).get('practices', []), practice_id)

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/cycle/pre_checks/__init__.py

@@ -2,14 +2,13 @@ from os.path import dirname, abspath
 import sys
 
 from hestia_earth.models.utils import _run_in_serie
-from . import site, startDate, cache_sources
+from . import site, cache_sources
 
 CURRENT_DIR = dirname(abspath(__file__)) + '/'
 sys.path.append(CURRENT_DIR)
 
 MODELS = [
     site.run,
-    startDate.run,
     cache_sources.run
 ]
 

hestia-earth-models-0.59.6/hestia_earth/models/cycle/startDate.py

@@ -0,0 +1,42 @@
+"""
+Start Date
+
+This model updates the [Cycle startDate](https://hestia.earth/schema/Cycle#startDate) to be in the following format:
+`YYYY-MM-DD`.
+"""
+from hestia_earth.utils.date import is_in_days
+from hestia_earth.utils.tools import non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "startDate": ""
+    }
+}
+RETURNS = {
+    "The startDate as a string": ""
+}
+MODEL_KEY = 'startDate'
+
+
+def _run(cycle: dict):
+    value = cycle.get('startDate')
+    return '-'.join(non_empty_list([value, '01' if len(value) == 4 else '', '01']))
+
+
+def _should_run(cycle: dict):
+    has_startDate = cycle.get('startDate') is not None
+    has_incorrect_format = has_startDate and not is_in_days(cycle.get('startDate'))
+
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
+                    has_startDate=has_startDate,
+                    has_incorrect_format=has_incorrect_format)
+
+    should_run = all([has_startDate, has_incorrect_format])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else None

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/cycle/utils.py

@@ -6,7 +6,7 @@ from . import MODEL
 
 
 def _get_liveAnimal_term_id(product: dict, **log_ars):
-    term_id = get_lookup_value(product.get('term', {}), 'liveAnimal', model=MODEL, **log_ars)
+    term_id = get_lookup_value(product.get('term', {}), 'liveAnimalTermId', model=MODEL, **log_ars)
     return term_id.split(';')[0] if term_id else None
 
 

hestia-earth-models-0.59.6/hestia_earth/models/faostat2018/liveweightPerHead.py

@@ -0,0 +1,124 @@
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.lookup import get_table_value, column_name, download_lookup, extract_grouped_data_closest_date
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import non_empty_list, safe_parse_date, safe_parse_float
+
+from hestia_earth.models.log import logRequirements, logShouldRun, debugMissingLookup
+from hestia_earth.models.utils.constant import Units
+from hestia_earth.models.utils.property import _new_property, node_has_no_property
+from hestia_earth.models.utils.product import convert_product_to_unit
+from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_lookup_value
+from .utils import get_liveAnimal_to_animalProduct_id
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "endDate": "",
+        "products": [{
+            "@type": "Product",
+            "value": "",
+            "term.termType": ["animalProduct", "liveAnimal"]
+        }],
+        "site": {
+            "@type": "Site",
+            "country": {"@type": "Term", "termType": "region"}
+        }
+    }
+}
+LOOKUPS = {
+    "liveAnimal": ["primaryMeatProductFaoProductionTermId"],
+    "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO"],
+    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above"
+}
+RETURNS = {
+    "Product": [{
+        "properties": [{
+            "@type": "Property",
+            "value": ""
+        }]
+    }]
+}
+TERM_ID = 'liveweightPerHead'
+LOOKUP_WEIGHT = 'region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight.csv'
+
+
+def _property(value: float):
+    prop = _new_property(TERM_ID, MODEL)
+    prop['value'] = value
+    return prop
+
+
+def _product_value(product: dict, year: int, country_id: str):
+    product_id = product.get('term', {}).get('@id')
+    groupingFAO = get_animalProduct_lookup_value(MODEL, product_id, FAO_LOOKUP_COLUMN)
+
+    lookup = download_lookup(LOOKUP_WEIGHT)
+    data = get_table_value(lookup, 'termid', country_id, column_name(groupingFAO))
+    debugMissingLookup(LOOKUP_WEIGHT, 'termid', country_id, groupingFAO, data, model=MODEL, term=TERM_ID)
+    average_carcass_weight = safe_parse_float(extract_grouped_data_closest_date(data, year), None)
+    # average_carcass_weight is in hg, divide by 10 to go back to kg
+    kg_carcass_weight = average_carcass_weight / 10 if average_carcass_weight else None
+
+    kg_liveweight = convert_product_to_unit({
+        **product,
+        'value': [kg_carcass_weight]
+    }, Units.KG_LIVEWEIGHT) if kg_carcass_weight else None
+
+    return kg_liveweight, groupingFAO
+
+
+def _should_run_liveAnimal(product: dict):
+    return product.get('term', {}).get('termType') == TermTermType.LIVEANIMAL.value
+
+
+def _run_liveAnimal(cycle: dict, product: dict, year: int, country_id: str):
+    # find the animalProduct to get the average carcass weight
+    product_id = product.get('term', {}).get('@id')
+    animal_product_id = get_liveAnimal_to_animalProduct_id(product_id, LOOKUPS['liveAnimal'][0], term=TERM_ID)
+
+    animal_product_term = download_hestia(animal_product_id)
+    kg_liveweight, groupingFAO = _product_value({**product, 'term': animal_product_term}, year, country_id)
+
+    logRequirements(cycle, model=MODEL, term=product_id, property=TERM_ID,
+                    animal_product_id=animal_product_id,
+                    country_id=country_id,
+                    year=year,
+                    kg_liveweight=kg_liveweight,
+                    groupingFAO=groupingFAO)
+
+    should_run = all([kg_liveweight])
+    logShouldRun(cycle, MODEL, product_id, should_run, property=TERM_ID)
+
+    return {**product, 'properties': product.get('properties', []) + [_property(kg_liveweight)]} if should_run else None
+
+
+def _run_animalProduct(cycle: dict, product: dict, year: int, country_id: str):
+    product_id = product.get('term', {}).get('@id')
+    kg_liveweight, groupingFAO = _product_value(product, year, country_id)
+
+    logRequirements(cycle, model=MODEL, term=product_id, property=TERM_ID,
+                    country_id=country_id,
+                    year=year,
+                    kg_liveweight=kg_liveweight,
+                    groupingFAO=groupingFAO)
+
+    should_run = all([kg_liveweight])
+    logShouldRun(cycle, MODEL, product_id, should_run, property=TERM_ID)
+
+    return {**product, 'properties': product.get('properties', []) + [_property(kg_liveweight)]} if should_run else None
+
+
+def run(cycle: dict):
+    country_id = cycle.get('site', {}).get('country', {}).get('@id')
+    end_date = safe_parse_date(cycle.get('endDate'))
+    year = end_date.year if end_date else None
+
+    products = filter_list_term_type(cycle.get('products', []), [TermTermType.ANIMALPRODUCT, TermTermType.LIVEANIMAL])
+    products = list(filter(node_has_no_property(TERM_ID), products))
+
+    return non_empty_list([
+        (
+            (_run_liveAnimal if _should_run_liveAnimal(p) else _run_animalProduct)(cycle, p, year, country_id)
+        ) for p in products
+    ]) if country_id else []

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/faostat2018/product/price.py

@@ -12,10 +12,9 @@ from hestia_earth.models.log import debugMissingLookup, debugValues, logRequirem
 from hestia_earth.models.utils.constant import Units
 from hestia_earth.models.utils.currency import DEFAULT_CURRENCY
 from hestia_earth.models.utils.crop import FAOSTAT_PRODUCTION_LOOKUP_COLUMN, get_crop_grouping_faostat_production
-from hestia_earth.models.utils.animalProduct import (
-    FAO_LOOKUP_COLUMN, get_animalProduct_grouping_fao, get_animalProduct_lookup_value
-)
+from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_grouping_fao
 from hestia_earth.models.utils.product import convert_product_to_unit
+from ..utils import get_liveAnimal_to_animalProduct_id
 from .. import MODEL
 
 REQUIREMENTS = {
@@ -39,8 +38,8 @@ LOOKUPS = {
     "@doc": "Depending on the primary product [termType](https://hestia.earth/schema/Product#term)",
     "crop": "cropGroupingFaostatProduction",
     "region-crop-cropGroupingFaostatProduction-price": "use value from above",
-    "liveAnimal": "primaryMeatProductFAO",
-    "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO", "liveAnimal"],
+    "liveAnimal": ["primaryMeatProductFaoPriceTermId"],
+    "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO"],
     "region-animalProduct-animalProductGroupingFAO-price": "use value from above",
     "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above"
 }
@@ -53,7 +52,6 @@ LOOKUP_GROUPING = {
     TermTermType.CROP.value: get_crop_grouping_faostat_production,
     TermTermType.ANIMALPRODUCT.value: get_animalProduct_grouping_fao
 }
-LOOKUP_WEIGHT = 'region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight.csv'
 
 
 def _term_grouping(term: dict): return LOOKUP_GROUPING.get(term.get('termType'), lambda *_: None)(MODEL, term)
@@ -65,8 +63,7 @@ def _lookup_data(
     lookup_name = lookup_name or LOOKUP_NAME.get(term_type)
     lookup = download_lookup(lookup_name)
     data = get_table_value(lookup, 'termid', country_id, column_name(grouping))
-    debugMissingLookup(lookup_name, 'termid', country_id, grouping, data,
-                       model=MODEL, term=term_id, key=MODEL_KEY)
+    debugMissingLookup(lookup_name, 'termid', country_id, grouping, data, model=MODEL, term=term_id, key=MODEL_KEY)
     price = extract_grouped_data(data, str(year)) or extract_grouped_data(data, 'Average_price_per_tonne')
     return safe_parse_float(price, None)
 
@@ -76,58 +73,46 @@ def _product(product: dict, value: float):
     return product | {'currency': DEFAULT_CURRENCY, MODEL_KEY: round(value, 4)}
 
 
-def _get_animalProductId(term_id: str):
-    lookup_name = 'liveAnimal.csv'
-    lookup = download_lookup(lookup_name)
-    value = get_table_value(lookup, 'termid', term_id, column_name(LOOKUPS.get('liveAnimal')))
-    debugMissingLookup(lookup_name, 'termid', term_id, LOOKUPS.get('liveAnimal'), value,
-                       model=MODEL, term=term_id, key=MODEL_KEY)
-    return value
-
-
 def _get_liveAnimal_lookup_values(cycle: dict, product: dict, country_id: str, year: int = None):
     term_id = product.get('term', {}).get('@id')
-    animal_product = _get_animalProductId(term_id)
-    groupingFAO = get_animalProduct_lookup_value(MODEL, animal_product, FAO_LOOKUP_COLUMN) if animal_product else None
+    animal_product = get_liveAnimal_to_animalProduct_id(term_id, LOOKUPS['liveAnimal'][0], key=MODEL_KEY)
+    groupingFAO = _term_grouping({'termType': TermTermType.ANIMALPRODUCT.value, '@id': animal_product})
 
     # one live animal can be linked to many animal product, hence go one by one until we have a match
     if groupingFAO:
-        average_carcass_weight = _lookup_data(animal_product, groupingFAO, country_id, year, lookup_name=LOOKUP_WEIGHT)
-        price = _lookup_data(term_id, groupingFAO, country_id, year, term_type=TermTermType.ANIMALPRODUCT.value)
+        price_per_ton_liveweight = _lookup_data(
+            term_id, groupingFAO, country_id, year, term_type=TermTermType.ANIMALPRODUCT.value
+        )
         debugValues(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='liveAnimal',
                     animal_product=animal_product,
-                    groupingFAO=f"'{groupingFAO}'",
-                    average_carcass_weight_hg=average_carcass_weight,
-                    price_per_ton=price)
-        if price and average_carcass_weight:
+                    price_per_ton_liveweight=price_per_ton_liveweight,
+                    groupingFAO=f"'{groupingFAO}'")
+        if price_per_ton_liveweight:
             # price is per 1000kg, divide by 1000 to go back to USD/kg
-            # average_carcass_weight is in hg, divide by 10 to go back to kg
-            return (animal_product, price / 1000, average_carcass_weight / 10)
-    return (None, None, None)
+            return (animal_product, price_per_ton_liveweight / 1000)
+    return (None, None)
 
 
 def _run_by_liveAnimal(cycle: dict, product: dict, country_id: str, year: int = None):
     term_id = product.get('term', {}).get('@id')
-    animal_product, price, carcass_weight = _get_liveAnimal_lookup_values(cycle, product, country_id, year)
+    animal_product_id, price_per_kg_liveweight = _get_liveAnimal_lookup_values(cycle, product, country_id, year)
 
-    animal_product = download_hestia(animal_product)
-    price_per_carcass_weight = convert_product_to_unit({
+    animal_product = download_hestia(animal_product_id)
+    price_per_head = convert_product_to_unit({
+        **product,
         'term': animal_product,
-        'value': [price]
-    }, Units.KG_COLD_CARCASS_WEIGHT) if price else None
-    should_run = all([price_per_carcass_weight, carcass_weight])
-    value = price_per_carcass_weight * carcass_weight if should_run else None
+        'value': [price_per_kg_liveweight]
+    }, Units.HEAD) if price_per_kg_liveweight else None
 
     logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='liveAnimal',
-                    price_from_lookup=price,
-                    carcass_weight_kg=carcass_weight,
-                    price_per_carcass_weight=price_per_carcass_weight,
                     year=year,
-                    price=value)
+                    price_per_kg_liveweight=price_per_kg_liveweight,
+                    price_per_head=price_per_head)
 
+    should_run = all([animal_product, price_per_head])
     logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY, by='liveAnimal')
 
-    return _product(product, value) if value is not None else None
+    return _product(product, price_per_head) if price_per_head is not None else None
 
 
 def _should_run_liveAnimal(product: dict):
@@ -151,9 +136,9 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='country',
                     has_yield=has_yield,
                     not_already_set=not_already_set,
-                    groupingFAO=f"'{grouping}'",
                     year=year,
-                    price_per_ton=value)
+                    price_per_ton=value,
+                    groupingFAO=f"'{grouping}'")
 
     logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY, by='country')
 
@@ -165,27 +150,17 @@ def _should_run_product(product: dict):
     return product.get(MODEL_KEY) is None
 
 
-def _should_run(cycle: dict):
-    country_id = cycle.get('site', {}).get('country', {}).get('@id')
-
-    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
-                    country_id=country_id)
-
-    should_run = all([country_id])
-    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
-    return should_run, country_id
-
-
 def run(cycle: dict):
-    should_run_by_country, country_id = _should_run(cycle)
+    country_id = cycle.get('site', {}).get('country', {}).get('@id')
     end_date = safe_parse_date(cycle.get('endDate'))
     year = end_date.year if end_date else None
+
     products = list(filter(_should_run_product, cycle.get('products', [])))
     return non_empty_list([
         (
             (
                 (_run_by_liveAnimal(cycle, p, country_id, year) if _should_run_liveAnimal(p) else None)
                 or _run_by_country(cycle, p, country_id, year)
-            ) if should_run_by_country else None
+            ) if country_id else None
         ) for p in products
     ])

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/faostat2018/utils.py

@@ -3,7 +3,7 @@ from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data_closest_date
 from hestia_earth.utils.tools import safe_parse_float
 
-from hestia_earth.models.log import logger
+from hestia_earth.models.log import logger, debugMissingLookup
 from hestia_earth.models.utils.animalProduct import (
     FAO_LOOKUP_COLUMN, FAO_EQUIVALENT_LOOKUP_COLUMN, get_animalProduct_lookup_value
 )
@@ -13,9 +13,17 @@ from . import MODEL
 LOOKUP_PREFIX = f"{TermTermType.REGION.value}-{TermTermType.ANIMALPRODUCT.value}-{FAO_LOOKUP_COLUMN}"
 
 
+def get_liveAnimal_to_animalProduct_id(product_term_id: str, column: str, **log_args):
+    lookup_name = 'liveAnimal.csv'
+    lookup = download_lookup(lookup_name)
+    value = get_table_value(lookup, 'termid', product_term_id, column_name(column))
+    debugMissingLookup(lookup_name, 'termid', product_term_id, column, value, model=MODEL, **log_args)
+    return value
+
+
 def product_equivalent_value(product: dict, year: int, country: str):
     term_id = product.get('term', {}).get('@id')
-    fao_product_id = get_animalProduct_lookup_value(MODEL, term_id, FAO_EQUIVALENT_LOOKUP_COLUMN)
+    fao_product_id = get_animalProduct_lookup_value(MODEL, term_id, FAO_EQUIVALENT_LOOKUP_COLUMN) or term_id
     grouping = get_animalProduct_lookup_value(MODEL, fao_product_id, FAO_LOOKUP_COLUMN)
 
     if not grouping or not fao_product_id:

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/geospatialDatabase/potentialEvapotranspirationLongTermAnnualMean.py

@@ -47,9 +47,9 @@ def _measurement(value: float):
 
 
 def _download(site: dict):
-    scale = 10
+    factor = 0.1
     value = download(TERM_ID, site, EE_PARAMS)
-    return value / scale if value else None
+    return value * factor if value else None
 
 
 def _run(site: dict):

{hestia-earth-models-0.59.4 → hestia-earth-models-0.59.6}/hestia_earth/models/geospatialDatabase/potentialEvapotranspirationMonthly.py

@@ -10,7 +10,7 @@ from hestia_earth.models.utils import first_day_of_month, last_day_of_month
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from hestia_earth.models.utils.site import related_years
-from .utils import KELVIN_0, download, has_geospatial_data, should_download
+from .utils import download, has_geospatial_data, should_download
 from . import MODEL
 
 REQUIREMENTS = {
@@ -48,9 +48,6 @@ def _measurement(site: dict, value: list, dates: list):
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _to_celcius(kelvin_value: int): return kelvin_value - KELVIN_0 if kelvin_value else None
-
-
 def _download(site: dict,  start_date: str, end_date: str):
     return download(
         TERM_ID,
@@ -63,6 +60,12 @@ def _download(site: dict,  start_date: str, end_date: str):
     )
 
 
+def _value_at(site: dict, start_date: str, end_date: str):
+    factor = 0.1
+    value = _download(site, start_date, end_date)
+    return (value * factor, start_date[0:7]) if value is not None else None
+
+
 def _run(site: dict, years: list):
     # fetch from first year to last
     years = range(years[0], years[-1] + 1) if len(years) > 1 else years
@@ -73,10 +76,8 @@ def _run(site: dict, years: list):
             for month in range(1, 13)
         ] for year in years
     ])
-    values = non_empty_list([
-        (_to_celcius(_download(site, start_date, end_date)), start_date[0:7]) for start_date, end_date in dates
-    ])
-    return _measurement(site, [v for v, d in values], [d for v, d in values])
+    values = non_empty_list([_value_at(site, start_date, end_date) for start_date, end_date in dates])
+    return [_measurement(site, [v for v, d in values], [d for v, d in values])] if values else []
 
 
 def run(site: dict):