hestia-earth-models 0.59.5__py3-none-any.whl → 0.59.7__py3-none-any.whl

hestia_earth/models/faostat2018/product/price.py

@@ -12,10 +12,9 @@ from hestia_earth.models.log import debugMissingLookup, debugValues, logRequirem
 from hestia_earth.models.utils.constant import Units
 from hestia_earth.models.utils.currency import DEFAULT_CURRENCY
 from hestia_earth.models.utils.crop import FAOSTAT_PRODUCTION_LOOKUP_COLUMN, get_crop_grouping_faostat_production
-from hestia_earth.models.utils.animalProduct import (
-    FAO_LOOKUP_COLUMN, get_animalProduct_grouping_fao, get_animalProduct_lookup_value
-)
+from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_grouping_fao
 from hestia_earth.models.utils.product import convert_product_to_unit
+from ..utils import get_liveAnimal_to_animalProduct_id
 from .. import MODEL
 
 REQUIREMENTS = {
@@ -39,10 +38,11 @@ LOOKUPS = {
     "@doc": "Depending on the primary product [termType](https://hestia.earth/schema/Product#term)",
     "crop": "cropGroupingFaostatProduction",
     "region-crop-cropGroupingFaostatProduction-price": "use value from above",
-    "liveAnimal": "primaryMeatProductFAO",
-    "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO", "liveAnimal"],
+    "liveAnimal": ["primaryMeatProductFaoPriceTermId"],
+    "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO"],
     "region-animalProduct-animalProductGroupingFAO-price": "use value from above",
-    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above"
+    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above",
+    "region-animalProduct-animalProductGroupingFAO-weightPerItem": "use value from above"
 }
 MODEL_KEY = 'price'
 LOOKUP_NAME = {
@@ -53,7 +53,9 @@ LOOKUP_GROUPING = {
     TermTermType.CROP.value: get_crop_grouping_faostat_production,
     TermTermType.ANIMALPRODUCT.value: get_animalProduct_grouping_fao
 }
-LOOKUP_WEIGHT = 'region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight.csv'
+LOOKUP_UNITS_NUMBER = {
+    TermTermType.ANIMALPRODUCT.value: f"region-{TermTermType.ANIMALPRODUCT.value}-{FAO_LOOKUP_COLUMN}-weightPerItem.csv"
+}
 
 
 def _term_grouping(term: dict): return LOOKUP_GROUPING.get(term.get('termType'), lambda *_: None)(MODEL, term)
@@ -65,8 +67,7 @@ def _lookup_data(
     lookup_name = lookup_name or LOOKUP_NAME.get(term_type)
     lookup = download_lookup(lookup_name)
     data = get_table_value(lookup, 'termid', country_id, column_name(grouping))
-    debugMissingLookup(lookup_name, 'termid', country_id, grouping, data,
-                       model=MODEL, term=term_id, key=MODEL_KEY)
+    debugMissingLookup(lookup_name, 'termid', country_id, grouping, data, model=MODEL, term=term_id, key=MODEL_KEY)
     price = extract_grouped_data(data, str(year)) or extract_grouped_data(data, 'Average_price_per_tonne')
     return safe_parse_float(price, None)
 
@@ -76,58 +77,46 @@ def _product(product: dict, value: float):
     return product | {'currency': DEFAULT_CURRENCY, MODEL_KEY: round(value, 4)}
 
 
-def _get_animalProductId(term_id: str):
-    lookup_name = 'liveAnimal.csv'
-    lookup = download_lookup(lookup_name)
-    value = get_table_value(lookup, 'termid', term_id, column_name(LOOKUPS.get('liveAnimal')))
-    debugMissingLookup(lookup_name, 'termid', term_id, LOOKUPS.get('liveAnimal'), value,
-                       model=MODEL, term=term_id, key=MODEL_KEY)
-    return value
-
-
 def _get_liveAnimal_lookup_values(cycle: dict, product: dict, country_id: str, year: int = None):
     term_id = product.get('term', {}).get('@id')
-    animal_product = _get_animalProductId(term_id)
-    groupingFAO = get_animalProduct_lookup_value(MODEL, animal_product, FAO_LOOKUP_COLUMN) if animal_product else None
+    animal_product = get_liveAnimal_to_animalProduct_id(term_id, LOOKUPS['liveAnimal'][0], key=MODEL_KEY)
+    groupingFAO = _term_grouping({'termType': TermTermType.ANIMALPRODUCT.value, '@id': animal_product})
 
     # one live animal can be linked to many animal product, hence go one by one until we have a match
     if groupingFAO:
-        average_carcass_weight = _lookup_data(animal_product, groupingFAO, country_id, year, lookup_name=LOOKUP_WEIGHT)
-        price = _lookup_data(term_id, groupingFAO, country_id, year, term_type=TermTermType.ANIMALPRODUCT.value)
+        price_per_ton_liveweight = _lookup_data(
+            term_id, groupingFAO, country_id, year, term_type=TermTermType.ANIMALPRODUCT.value
+        )
         debugValues(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='liveAnimal',
                     animal_product=animal_product,
-                    groupingFAO=f"'{groupingFAO}'",
-                    average_carcass_weight_hg=average_carcass_weight,
-                    price_per_ton=price)
-        if price and average_carcass_weight:
+                    price_per_ton_liveweight=price_per_ton_liveweight,
+                    groupingFAO=f"'{groupingFAO}'")
+        if price_per_ton_liveweight:
             # price is per 1000kg, divide by 1000 to go back to USD/kg
-            # average_carcass_weight is in hg, divide by 10 to go back to kg
-            return (animal_product, price / 1000, average_carcass_weight / 10)
-    return (None, None, None)
+            return (animal_product, price_per_ton_liveweight / 1000)
+    return (None, None)
 
 
 def _run_by_liveAnimal(cycle: dict, product: dict, country_id: str, year: int = None):
     term_id = product.get('term', {}).get('@id')
-    animal_product, price, carcass_weight = _get_liveAnimal_lookup_values(cycle, product, country_id, year)
+    animal_product_id, price_per_kg_liveweight = _get_liveAnimal_lookup_values(cycle, product, country_id, year)
 
-    animal_product = download_hestia(animal_product)
-    price_per_carcass_weight = convert_product_to_unit({
+    animal_product = download_hestia(animal_product_id)
+    price_per_head = convert_product_to_unit({
+        **product,
         'term': animal_product,
-        'value': [price]
-    }, Units.KG_COLD_CARCASS_WEIGHT) if price else None
-    should_run = all([price_per_carcass_weight, carcass_weight])
-    value = price_per_carcass_weight * carcass_weight if should_run else None
+        'value': [price_per_kg_liveweight]
+    }, Units.HEAD) if price_per_kg_liveweight else None
 
     logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='liveAnimal',
-                    price_from_lookup=price,
-                    carcass_weight_kg=carcass_weight,
-                    price_per_carcass_weight=price_per_carcass_weight,
                     year=year,
-                    price=value)
+                    price_per_kg_liveweight=price_per_kg_liveweight,
+                    price_per_head=price_per_head)
 
+    should_run = all([animal_product, price_per_head])
     logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY, by='liveAnimal')
 
-    return _product(product, value) if value is not None else None
+    return _product(product, price_per_head) if price_per_head is not None else None
 
 
 def _should_run_liveAnimal(product: dict):
@@ -138,6 +127,7 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     product_term = product.get('term', {})
     term_id = product_term.get('@id')
     term_type = product_term.get('termType')
+    term_units = product_term.get('units')
 
     has_yield = len(product.get('value', [])) > 0
     not_already_set = MODEL_KEY not in product.keys()
@@ -148,44 +138,40 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     should_run = all([not_already_set, has_yield, grouping])
     value = _lookup_data(term_id, grouping, country_id, year, term_type=term_type) if should_run else None
 
+    # if units is number instead of kg, need to convert to number first
+    conversion_to_number = safe_parse_float(get_table_value(
+        download_lookup(LOOKUP_UNITS_NUMBER.get(term_type)), 'termid', country_id, column_name(grouping)
+    ), 1) if term_units == Units.NUMBER.value else 1
+
     logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='country',
                     has_yield=has_yield,
                     not_already_set=not_already_set,
-                    groupingFAO=f"'{grouping}'",
                     year=year,
-                    price_per_ton=value)
+                    price_per_ton=value,
+                    groupingFAO=f"'{grouping}'",
+                    conversion_to_number=conversion_to_number)
 
     logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY, by='country')
 
     # divide by 1000 to convert price per tonne to kg
-    return _product(product, value / 1000) if value is not None else None
+    return _product(product, value / 1000 * conversion_to_number) if value is not None else None
 
 
 def _should_run_product(product: dict):
     return product.get(MODEL_KEY) is None
 
 
-def _should_run(cycle: dict):
-    country_id = cycle.get('site', {}).get('country', {}).get('@id')
-
-    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
-                    country_id=country_id)
-
-    should_run = all([country_id])
-    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
-    return should_run, country_id
-
-
 def run(cycle: dict):
-    should_run_by_country, country_id = _should_run(cycle)
+    country_id = cycle.get('site', {}).get('country', {}).get('@id')
     end_date = safe_parse_date(cycle.get('endDate'))
     year = end_date.year if end_date else None
+
     products = list(filter(_should_run_product, cycle.get('products', [])))
     return non_empty_list([
         (
             (
                 (_run_by_liveAnimal(cycle, p, country_id, year) if _should_run_liveAnimal(p) else None)
                 or _run_by_country(cycle, p, country_id, year)
-            ) if should_run_by_country else None
+            ) if country_id else None
         ) for p in products
     ])

hestia_earth/models/faostat2018/utils.py

@@ -3,7 +3,7 @@ from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data_closest_date
 from hestia_earth.utils.tools import safe_parse_float
 
-from hestia_earth.models.log import logger
+from hestia_earth.models.log import logger, debugMissingLookup
 from hestia_earth.models.utils.animalProduct import (
     FAO_LOOKUP_COLUMN, FAO_EQUIVALENT_LOOKUP_COLUMN, get_animalProduct_lookup_value
 )
@@ -13,9 +13,17 @@ from . import MODEL
 LOOKUP_PREFIX = f"{TermTermType.REGION.value}-{TermTermType.ANIMALPRODUCT.value}-{FAO_LOOKUP_COLUMN}"
 
 
+def get_liveAnimal_to_animalProduct_id(product_term_id: str, column: str, **log_args):
+    lookup_name = 'liveAnimal.csv'
+    lookup = download_lookup(lookup_name)
+    value = get_table_value(lookup, 'termid', product_term_id, column_name(column))
+    debugMissingLookup(lookup_name, 'termid', product_term_id, column, value, model=MODEL, **log_args)
+    return value
+
+
 def product_equivalent_value(product: dict, year: int, country: str):
     term_id = product.get('term', {}).get('@id')
-    fao_product_id = get_animalProduct_lookup_value(MODEL, term_id, FAO_EQUIVALENT_LOOKUP_COLUMN)
+    fao_product_id = get_animalProduct_lookup_value(MODEL, term_id, FAO_EQUIVALENT_LOOKUP_COLUMN) or term_id
     grouping = get_animalProduct_lookup_value(MODEL, fao_product_id, FAO_LOOKUP_COLUMN)
 
     if not grouping or not fao_product_id:

hestia_earth/models/geospatialDatabase/utils.py

@@ -175,7 +175,6 @@ def download(term: str, site: dict, data: dict, only_coordinates=False) -> dict:
                 }
             ]
         }
-        print(query)
         value = _parse_run_query(term, query)
 
     if value is None:

hestia_earth/models/haversineFormula/transport/distance.py

@@ -27,8 +27,11 @@ REQUIREMENTS = {
     }
 }
 RETURNS = {
-    "Transport": [{
-        "distance": ""
+    "Input": [{
+        "transport": [{
+            "@type": "Transport",
+            "distance": ""
+        }]
     }]
 }
 MODEL_KEY = 'distance'
@@ -98,7 +101,7 @@ def _should_run(cycle: dict):
     # can only run if the site country has centroid coordinates
     logRequirements(cycle, model=MODEL, term=None, key=MODEL_KEY,
                     latitude=country.get('latitude'),
-                    longitude=country.get('latitude'),
+                    longitude=country.get('longitude'),
                     has_inputs_transport=len(inputs) > 0)
     should_run = all([country.get('latitude'), country.get('latitude'), len(inputs) > 0])
     logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)

hestia_earth/models/ipcc2006/n2OToAirInorganicFertiliserIndirect.py

@@ -55,7 +55,7 @@ def _run(cycle: dict, N_total: float):
                 nh3_n=nh3_n,
                 nox_n=nox_n)
     value = COEFF_NH3NOX_N2O * (
-        sum_values([nh3_n, nox_n]) or N_total * 0.2
+        sum_values([nh3_n, nox_n]) or N_total * 0.1
     ) + COEFF_NO3_N2O * (
         no3_n or N_total * 0.3
     )

hestia_earth/models/ipcc2019/co2ToAirLimeHydrolysis.py

@@ -11,7 +11,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "completeness.soilAmendments": "",
+        "completeness.soilAmendment": "",
         "inputs": [{
             "@type": "Input",
             "value": "",

hestia_earth/models/ipcc2019/co2ToAirUreaHydrolysis.py

@@ -11,13 +11,8 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "or": {
-            "completeness.fertiliser": "",
-            "inputs": [{"@type": "Input", "value": "", "term.termType": "inorganicFertiliser"}]
-        },
-        "optional": {
-            "inputs": [{"@type": "Input", "value": "", "term.@id": "inorganicNitrogenFertiliserUnspecifiedKgN"}]
-        }
+        "completeness.fertiliser": "",
+        "inputs": [{"@type": "Input", "value": "", "term.termType": "inorganicFertiliser"}]
     }
 }
 RETURNS = {

hestia_earth/models/ipcc2019/organicCarbonPerHa.py

@@ -2,7 +2,13 @@
 The IPCC model for estimating soil organic carbon stock changes in the 0 - 30cm depth interval due to management
 changes. This model combines the Tier 1 & Tier 2 methodologies. It first tries to run Tier 2 (only on croplands
 remaining croplands). If Tier 2 cannot run, it will try to run Tier 1 (for croplands remaining croplands and for
-grasslands remaining grasslands). Source:
+grasslands remaining grasslands).
+
+More information on this model, including data requirements **and** recommendations, tier methodologies, and examples,
+can be found in the
+[Hestia SOC wiki](https://gitlab.com/hestia-earth/hestia-engine-models/-/wikis/Soil-organic-carbon-modelling).
+
+Source:
 [IPCC 2019, Vol. 4, Chapter 10](https://www.ipcc-nggip.iges.or.jp/public/2019rf/pdf/4_Volume4/19R_V4_Ch05_Cropland.pdf).
 """
 from enum import Enum
@@ -253,6 +259,11 @@ The number of years required for soil organic carbon to reach equilibrium after
 a change in land use, management regime or carbon input regime.
 """
 
+EXCLUDED_ECO_CLIMATE_ZONES_TIER_1 = {
+    5,  # Polar Moist
+    6  # Polar Dry
+}
+
 VALID_SITE_TYPES_TIER_1 = [
     SiteSiteType.CROPLAND.value,
     SiteSiteType.FOREST.value,
@@ -281,7 +292,7 @@ def _measurement(year: int, value: float, method_classification: str) -> dict:
     dict
         A valid Hestia `Measurement` node, see: https://www.hestia.earth/schema/Measurement.
     """
-    measurement = _new_measurement(TERM_ID, MODEL)
+    measurement = _new_measurement(TERM_ID)
     measurement["value"] = [value]
     measurement["dates"] = [f"{year}-12-31"]
     measurement["depthUpper"] = DEPTH_UPPER
@@ -846,7 +857,7 @@ def _calc_water_factor(
     float
         The water effect on decomposition for a given month, dimensionless, between `0.2129` and `1.5`.
     """
-    mappet = min(1.25, precipitation / pet)
+    mappet = min(1.25, precipitation / pet) if pet else 1.25
     return 0.775 if is_irrigated else 0.2129 + (water_factor_slope * (mappet)) - (0.2413 * pow(mappet, 2))
 
 
@@ -937,7 +948,8 @@ def _calc_average_nitrogen_content_of_organic_carbon_sources(
     weighted_values = [
         c.mass * (c.nitrogen_content if c.nitrogen_content else default_nitrogen_content) for c in carbon_sources
     ]
-    return sum(weighted_values) / total_weight if total_weight > 0 else default_nitrogen_content
+    should_run = total_weight > 0
+    return sum(weighted_values) / total_weight if should_run else 0
 
 
 def _calc_average_lignin_content_of_organic_carbon_sources(
@@ -963,7 +975,8 @@ def _calc_average_lignin_content_of_organic_carbon_sources(
     weighted_values = [
         c.mass * (c.lignin_content if c.lignin_content else default_lignin_content) for c in carbon_sources
     ]
-    return sum(weighted_values) / total_weight if total_weight > 0 else default_lignin_content
+    should_run = total_weight > 0
+    return sum(weighted_values) / total_weight if should_run else 0
 
 
 # --- TIER 2 FUNCTIONS: ACTIVE SUB-POOL SOC STOCK ---
@@ -3570,7 +3583,7 @@ def _should_run_tier_1(
     Determines whether there is sufficient data in the inventory and keyword args to run the tier 1 model.
     """
     return all([
-        eco_climate_zone and eco_climate_zone > 0,
+        eco_climate_zone and eco_climate_zone not in EXCLUDED_ECO_CLIMATE_ZONES_TIER_1,
         soc_ref and soc_ref > 0,
         any(year for year, group in inventory.items() if group.get(_InventoryKey.SHOULD_RUN_TIER_1))
     ])
@@ -3724,9 +3737,16 @@ def _should_run_inventory_year_tier_2(group: dict) -> bool:
         }
     )
 
+    carbon_input_data_complete = all([
+        group.get(_InventoryKey.CARBON_INPUT, 0) > 0,
+        group.get(_InventoryKey.N_CONTENT, 0) > 0,
+        group.get(_InventoryKey.LIGNIN_CONTENT, 0) > 0,
+    ])
+
     return all([
         not group.get(_InventoryKey.IS_PADDY_RICE),
         monthly_data_complete,
+        carbon_input_data_complete,
         all(key in group.keys() for key in REQUIRED_KEYS_TIER_2),
     ])
 

hestia_earth/models/ipcc2019/pastureGrass.py

@@ -497,7 +497,8 @@ def _sum_values(values): return sum([value for term_id, value in values])
 def _calculate_GE(cycle: dict, meanDE: float, system: dict):
     animals = [
         a for a in cycle.get('animals', []) if all([
-            a.get('value') and a.get('referencePeriod') == AnimalReferencePeriod.AVERAGE.value
+            a.get('value'),
+            a.get('referencePeriod') == AnimalReferencePeriod.AVERAGE.value
         ])
     ]
 

hestia_earth/models/linkedImpactAssessment/__init__.py

@@ -75,7 +75,7 @@ def _run_emission(cycle: dict, term_id: str, data: dict):
 
         details = values.get('details', {})
         logRequirements(cycle, model=model, term=term_id,
-                        values=log_as_table([{'impact_assessment_id': key} | value for key, value in details.items()]))
+                        values=log_as_table([{'impact-assessment-id': key} | value for key, value in details.items()]))
 
         logShouldRun(cycle, model, term_id, True, methodTier=TIER,
                      input=term.get('@id'),
@@ -138,8 +138,8 @@ def _group_inputs(group: dict, values: tuple):
             grouped_inputs['value'] = grouped_inputs['value'] + (emission_value * input_value)
             # for logging
             grouped_inputs['details'][input.get('impactAssessment', {}).get('@id')] = {
-                'emission_value': emission_value,
-                'input_value': input_value
+                'emission-value': emission_value,
+                'input-value': input_value
             }
             group[emission_term_id][input_group_key] = grouped_inputs
     return group