hestia-earth-models 0.67.0__py3-none-any.whl → 0.68.0__py3-none-any.whl

hestia_earth/models/faostat2018/product/price.py

@@ -4,16 +4,17 @@ Product Price
 Calculates the price of `crop` and `liveAnimal` using FAOSTAT data.
 """
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.lookup import get_table_value, column_name, download_lookup, extract_grouped_data
+from hestia_earth.utils.lookup import extract_grouped_data
 from hestia_earth.utils.tools import non_empty_list, safe_parse_float, safe_parse_date
 
-from hestia_earth.models.log import debugMissingLookup, debugValues, logRequirements, logShouldRun
+from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
 from hestia_earth.models.utils.constant import Units
+from hestia_earth.models.utils.term import download_term
 from hestia_earth.models.utils.currency import DEFAULT_CURRENCY
 from hestia_earth.models.utils.crop import FAOSTAT_PRODUCTION_LOOKUP_COLUMN, get_crop_grouping_faostat_production
 from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_grouping_fao
 from hestia_earth.models.utils.product import convert_product_to_unit
+from hestia_earth.models.utils.lookup import get_region_lookup_value
 from ..utils import get_liveAnimal_to_animalProduct_id
 from .. import MODEL
 
@@ -37,12 +38,12 @@ RETURNS = {
 LOOKUPS = {
     "@doc": "Depending on the primary product [termType](https://hestia.earth/schema/Product#term)",
     "crop": "cropGroupingFaostatProduction",
-    "region-crop-cropGroupingFaostatProduction-price": "use value from above",
+    "region-crop-cropGroupingFaostatProduction-price": "",
     "liveAnimal": ["primaryMeatProductFaoPriceTermId"],
     "animalProduct": ["animalProductGroupingFAOEquivalent", "animalProductGroupingFAO"],
-    "region-animalProduct-animalProductGroupingFAO-price": "use value from above",
-    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "use value from above",
-    "region-animalProduct-animalProductGroupingFAO-weightPerItem": "use value from above"
+    "region-animalProduct-animalProductGroupingFAO-price": "",
+    "region-animalProduct-animalProductGroupingFAO-averageColdCarcassWeight": "",
+    "region-animalProduct-animalProductGroupingFAO-weightPerItem": ""
 }
 MODEL_KEY = 'price'
 LOOKUP_NAME = {
@@ -62,14 +63,19 @@ def _term_grouping(term: dict): return LOOKUP_GROUPING.get(term.get('termType'),
 
 
 def _lookup_data(
-    term_id: str, grouping: str, country_id: str, year: int, term_type: str = None, lookup_name: str = None
+    term_id: str, grouping: str, country: dict, year: int, term_type: str = None, lookup_name: str = None
 ):
-    lookup_name = lookup_name or LOOKUP_NAME.get(term_type)
-    lookup = download_lookup(lookup_name)
-    data = get_table_value(lookup, 'termid', country_id, column_name(grouping))
-    debugMissingLookup(lookup_name, 'termid', country_id, grouping, data, model=MODEL, term=term_id, key=MODEL_KEY)
-    price = extract_grouped_data(data, str(year)) or extract_grouped_data(data, 'Average_price_per_tonne')
-    return safe_parse_float(price, None)
+    lookup_name = lookup_name or LOOKUP_NAME.get(term_type, '')
+
+    def get_data(country_id):
+        data = get_region_lookup_value(lookup_name, country_id, grouping, model=MODEL, term=term_id, key=MODEL_KEY)
+        price = extract_grouped_data(data, str(year)) or extract_grouped_data(data, 'Average_price_per_tonne')
+        return safe_parse_float(price, None)
+
+    # try get country data first, falls back to region data
+    country_id = country.get('@id')
+    region_id = (country.get('subClassOf') or [{}])[0].get('@id')
+    return get_data(country_id) or (get_data(region_id) if region_id else None)
 
 
 def _product(product: dict, value: float):
@@ -77,7 +83,7 @@ def _product(product: dict, value: float):
     return product | {'currency': DEFAULT_CURRENCY, MODEL_KEY: round(value, 4)}
 
 
-def _get_liveAnimal_lookup_values(cycle: dict, product: dict, country_id: str, year: int = None):
+def _get_liveAnimal_lookup_values(cycle: dict, product: dict, country: dict, year: int = None):
     term_id = product.get('term', {}).get('@id')
     animal_product = get_liveAnimal_to_animalProduct_id(term_id, LOOKUPS['liveAnimal'][0], key=MODEL_KEY)
     groupingFAO = _term_grouping({'termType': TermTermType.ANIMALPRODUCT.value, '@id': animal_product})
@@ -85,7 +91,7 @@ def _get_liveAnimal_lookup_values(cycle: dict, product: dict, country_id: str, y
     # one live animal can be linked to many animal product, hence go one by one until we have a match
     if groupingFAO:
         price_per_ton_liveweight = _lookup_data(
-            term_id, groupingFAO, country_id, year, term_type=TermTermType.ANIMALPRODUCT.value
+            term_id, groupingFAO, country, year, term_type=TermTermType.ANIMALPRODUCT.value
         )
         debugValues(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='liveAnimal',
                     animal_product=animal_product,
@@ -97,11 +103,11 @@ def _get_liveAnimal_lookup_values(cycle: dict, product: dict, country_id: str, y
     return (None, None)
 
 
-def _run_by_liveAnimal(cycle: dict, product: dict, country_id: str, year: int = None):
+def _run_by_liveAnimal(cycle: dict, product: dict, country: dict, year: int = None):
     term_id = product.get('term', {}).get('@id')
-    animal_product_id, price_per_kg_liveweight = _get_liveAnimal_lookup_values(cycle, product, country_id, year)
+    animal_product_id, price_per_kg_liveweight = _get_liveAnimal_lookup_values(cycle, product, country, year)
 
-    animal_product = download_hestia(animal_product_id)
+    animal_product = download_term(animal_product_id, TermTermType.ANIMALPRODUCT)
     price_per_head = convert_product_to_unit({
         **product,
         'term': animal_product,
@@ -123,7 +129,7 @@ def _should_run_liveAnimal(product: dict):
     return product.get('term', {}).get('termType') == TermTermType.LIVEANIMAL.value
 
 
-def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = None):
+def _run_by_country(cycle: dict, product: dict, country: dict, year: int = None):
     product_term = product.get('term', {})
     term_id = product_term.get('@id')
     term_type = product_term.get('termType')
@@ -136,12 +142,12 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     grouping = _term_grouping(product_term) or None
 
     should_run = all([not_already_set, has_yield, grouping])
-    value = _lookup_data(term_id, grouping, country_id, year, term_type=term_type) if should_run else None
+    value = _lookup_data(term_id, grouping, country, year, term_type=term_type) if should_run else None
 
     # if units is number instead of kg, need to convert to number first
-    conversion_to_number = safe_parse_float(get_table_value(
-        download_lookup(LOOKUP_UNITS_NUMBER.get(term_type)), 'termid', country_id, column_name(grouping)
-    ), 1) if term_units == Units.NUMBER.value else 1
+    conversion_to_number = safe_parse_float(
+        get_region_lookup_value(LOOKUP_UNITS_NUMBER.get(term_type), country.get('@id'), grouping), 1
+    ) if term_units == Units.NUMBER.value else 1
 
     logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by='country',
                     has_yield=has_yield,
@@ -161,7 +167,7 @@ def _should_run_product(product: dict): return product.get(MODEL_KEY) is None
 
 
 def run(cycle: dict):
-    country_id = cycle.get('site', {}).get('country', {}).get('@id')
+    country = cycle.get('site', {}).get('country', {})
     end_date = safe_parse_date(cycle.get('endDate'))
     year = end_date.year if end_date else None
 
@@ -169,8 +175,8 @@ def run(cycle: dict):
     return non_empty_list([
         (
             (
-                (_run_by_liveAnimal(cycle, p, country_id, year) if _should_run_liveAnimal(p) else None)
-                or _run_by_country(cycle, p, country_id, year)
-            ) if country_id else None
+                (_run_by_liveAnimal(cycle, p, country, year) if _should_run_liveAnimal(p) else None)
+                or _run_by_country(cycle, p, country, year)
+            ) if country else None
         ) for p in products
     ])

hestia_earth/models/faostat2018/readyToCookWeightPerHead.py

@@ -22,8 +22,8 @@ REQUIREMENTS = {
     }
 }
 LOOKUPS = {
-    "region-animalProduct-animalProductGroupingFAO-productionQuantity": "production quantity",
-    "region-animalProduct-animalProductGroupingFAO-head": "number of heads"
+    "region-animalProduct-animalProductGroupingFAO-productionQuantity": "",
+    "region-animalProduct-animalProductGroupingFAO-head": ""
 }
 RETURNS = {
     "Product": [{

hestia_earth/models/faostat2018/utils.py

@@ -1,4 +1,3 @@
-from numpy import recarray
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data_closest_date
@@ -11,9 +10,11 @@ from hestia_earth.models.utils.animalProduct import (
 from hestia_earth.models.utils.product import convert_product_to_unit
 from hestia_earth.models.utils.impact_assessment import get_country_id, impact_end_year
 from hestia_earth.models.utils.indicator import _new_indicator
+from hestia_earth.models.utils.lookup import get_region_lookup_value
 from . import MODEL
 
 LOOKUP_PREFIX = f"{TermTermType.REGION.value}-{TermTermType.ANIMALPRODUCT.value}-{FAO_LOOKUP_COLUMN}"
+FAOSTAT_AREA_LOOKUP = 'region-faostatArea.csv'
 
 
 def get_liveAnimal_to_animalProduct_id(product_term_id: str, column: str, **log_args):
@@ -32,12 +33,10 @@ def product_equivalent_value(product: dict, year: int, country: str):
     if not grouping or not fao_product_id:
         return None
 
-    lookup = download_lookup(f"{LOOKUP_PREFIX}-productionQuantity.csv")
-    quantity_values = get_table_value(lookup, 'termid', country, column_name(grouping))
+    quantity_values = get_region_lookup_value(f"{LOOKUP_PREFIX}-productionQuantity.csv", country, grouping)
     quantity = safe_parse_float(extract_grouped_data_closest_date(quantity_values, year))
 
-    lookup = download_lookup(f"{LOOKUP_PREFIX}-head.csv")
-    head_values = get_table_value(lookup, 'termid', country, column_name(grouping))
+    head_values = get_region_lookup_value(f"{LOOKUP_PREFIX}-head.csv", country, grouping)
     head = safe_parse_float(extract_grouped_data_closest_date(head_values, year))
 
     # quantity is in Tonnes
@@ -64,11 +63,11 @@ def _split_delta(table_value: str, start_year: int, end_year: int):
     ]) else None
 
 
-def get_sum_of_columns(lookup: recarray, country: str, year: int, columns_list: list) -> float:
+def get_sum_of_columns(country: str, year: int, columns_list: list) -> float:
     return sum(
         [safe_parse_float(
             extract_grouped_data_closest_date(
-                data=get_table_value(lookup, 'termid', country, column_name(col)),
+                data=get_region_lookup_value(FAOSTAT_AREA_LOOKUP, country, col, model=MODEL),
                 year=year
             )
         ) for col in columns_list]
@@ -76,9 +75,8 @@ def get_sum_of_columns(lookup: recarray, country: str, year: int, columns_list:
 
 
 def get_single_delta(country: str, start_year: int, end_year: int, column: str):
-    lookup = download_lookup('region-faostatArea.csv')
     return _split_delta(
-        get_table_value(lookup, 'termid', country, column_name(column)), start_year, end_year
+        get_region_lookup_value(FAOSTAT_AREA_LOOKUP, country, column, model=MODEL), start_year, end_year
     )
 
 
@@ -88,27 +86,24 @@ def get_land_ratio(
     """
     total_column is optional. Assumes that, if missing, total is the sum of values from first and second.
     """
-    lookup = download_lookup('region-faostatArea.csv')
     first_delta = _split_delta(
-        get_table_value(lookup, 'termid', country, column_name(first_column)), start_year, end_year
+        get_region_lookup_value(FAOSTAT_AREA_LOOKUP, country, first_column, model=MODEL), start_year, end_year
     )
     second_delta = _split_delta(
-        get_table_value(lookup, 'termid', country, column_name(second_column)), start_year, end_year
+        get_region_lookup_value(FAOSTAT_AREA_LOOKUP, country, second_column, model=MODEL), start_year, end_year
     )
     total_delta = (
         get_sum_of_columns(
-            lookup=lookup,
             country=country,
             year=end_year,
             columns_list=[first_column, second_column]
         ) - get_sum_of_columns(
-            lookup=lookup,
             country=country,
             year=start_year,
             columns_list=[first_column, second_column]
         )
     ) if total_column is None else _split_delta(
-        get_table_value(lookup, 'termid', country, column_name(total_column)), start_year, end_year
+        get_region_lookup_value(FAOSTAT_AREA_LOOKUP, country, total_column, model=MODEL), start_year, end_year
     )
 
     return (None, None, None) if any([
@@ -130,18 +125,11 @@ def get_cropland_ratio(country: str, start_year: int, end_year: int):
 
 
 def get_change_in_harvested_area_for_crop(country_id: str, crop_name: str, start_year: int, end_year: int = 0):
-    lookup = download_lookup('region-crop-cropGroupingFaostatProduction-areaHarvested.csv')
-    if end_year == 0 or end_year == start_year:
-        return safe_parse_float(
-            extract_grouped_data_closest_date(
-                data=get_table_value(lookup, 'termid', country_id, column_name(crop_name)),
-                year=start_year
-            )
-        )
-    else:
-        return _split_delta(
-            get_table_value(lookup, 'termid', country_id, column_name(crop_name)), start_year, end_year
-        )
+    lookup_name = 'region-crop-cropGroupingFaostatProduction-areaHarvested.csv'
+    value = get_region_lookup_value(lookup_name, country_id, crop_name)
+    return safe_parse_float(
+        extract_grouped_data_closest_date(value, start_year)
+    ) if end_year == 0 or end_year == start_year else _split_delta(value, start_year, end_year)
 
 
 def should_run_landTransformationFromCropland(term_id: str, impact: dict):

hestia_earth/models/frischknechtEtAl2000/ionisingRadiationKbqU235Eq.py

@@ -63,9 +63,11 @@ def _run(emissions: list):
 
 
 def _should_run(impact_assessment: dict):
-    emissions = [emission for emission in filter_list_term_type(impact_assessment.get('emissionsResourceUse', []),
-                                                                TermTermType.EMISSION)
-                 if emission.get('term', {}).get('@id', '') in LOOKUPS[TermTermType.WASTE.value]]
+    emissions = [
+        emission
+        for emission in filter_list_term_type(impact_assessment.get('emissionsResourceUse', []), TermTermType.EMISSION)
+        if emission.get('term', {}).get('@id', '') in LOOKUPS[TermTermType.WASTE.value]
+    ]
 
     has_emissions = bool(emissions)
 
@@ -80,7 +82,7 @@ def _should_run(impact_assessment: dict):
                     "input": input,
                     "indicator-input-is-valid": _valid_waste(input),
                     "value": _node_value(emission),
-                    "coefficient": get_table_value(array=download_lookup(filename="waste.csv"),
+                    "coefficient": get_table_value(lookup=download_lookup(filename="waste.csv"),
                                                    col_match='termid',
                                                    col_match_with=input.get('@id'),
                                                    col_val=column_name(emission['term']['@id'])) if input else None
@@ -89,9 +91,13 @@ def _should_run(impact_assessment: dict):
         ]
     )
 
-    valid_emission_with_cf = [em for em in emissions_unpacked if em['coefficient'] is not None
-                              and em['indicator-is-valid'] is True
-                              and em['indicator-input-is-valid'] is True]
+    valid_emission_with_cf = [
+        em for em in emissions_unpacked if all([
+            em['coefficient'] is not None,
+            em['indicator-is-valid'] is True,
+            em['indicator-input-is-valid'] is True
+        ])
+    ]
 
     valid_input_requirements = all([
         all([
@@ -101,8 +107,9 @@ def _should_run(impact_assessment: dict):
         for em in emissions_unpacked
     ])
 
-    all_emissions_have_known_cf = all([em['coefficient'] is not None for em in emissions_unpacked]) and bool(
-        emissions_unpacked)
+    all_emissions_have_known_cf = all([
+        em['coefficient'] is not None for em in emissions_unpacked
+    ]) and bool(emissions_unpacked)
 
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     has_emissions=has_emissions,

hestia_earth/models/geospatialDatabase/altitude.py

@@ -0,0 +1,60 @@
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import safe_parse_float
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from .utils import download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'altitude'
+EE_PARAMS = {
+    'collection': 'USGS/GMTED2010_FULL',
+    'ee_type': 'raster',
+    'band_name': 'med',
+    'is_image': True
+}
+BIBLIO_TITLE = 'An Enhanced Global Elevation Model Generalized From Multiple Higher Resolution Source Datasets'
+
+
+def _measurement(site: dict, value: float):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = [value]
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _run(site: dict):
+    value = download(TERM_ID, site, EE_PARAMS)
+    return [_measurement(site, safe_parse_float(value))] if value is not None else []
+
+
+def _should_run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size)
+
+    should_run = all([contains_geospatial_data, below_max_area_size])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+    return should_run
+
+
+def run(site: dict): return _run(site) if _should_run(site) else []

hestia_earth/models/geospatialDatabase/croppingIntensity.py

@@ -65,7 +65,7 @@ def _should_run(cycle: dict):
 
 
 def run(cycle: dict):
-    sites = get_allowed_sites(MODEL, TERM_ID, TermTermType.LANDUSEMANAGEMENT, cycle) if _should_run(cycle) else []
+    sites = get_allowed_sites(MODEL, TERM_ID, cycle) if _should_run(cycle) else []
     sites = [
         (site, get_region_factor(TERM_ID, site, TermTermType.LANDUSEMANAGEMENT)) for site in sites
     ]

hestia_earth/models/geospatialDatabase/ecoClimateZone.py

@@ -79,7 +79,7 @@ def _run_default(site: dict):
                     region_factor=region_factor)
 
     should_run = all([region_factor])
-    logShouldRun(site, MODEL, TERM_ID, should_run)
+    logShouldRun(site, MODEL, TERM_ID, should_run, run_by='Lookup')
     return [_measurement(site, round(region_factor))] if should_run else []
 
 
@@ -92,7 +92,7 @@ def _should_run(site: dict):
                     below_max_area_size=below_max_area_size)
 
     should_run = all([contains_geospatial_data, below_max_area_size])
-    logShouldRun(site, MODEL, TERM_ID, should_run)
+    logShouldRun(site, MODEL, TERM_ID, should_run, run_by='Earth Engine')
     return should_run
 
 

hestia_earth/models/geospatialDatabase/longFallowRatio.py

@@ -65,7 +65,7 @@ def _should_run(cycle: dict):
 
 
 def run(cycle: dict):
-    sites = get_allowed_sites(MODEL, TERM_ID, TermTermType.LANDUSEMANAGEMENT, cycle) if _should_run(cycle) else []
+    sites = get_allowed_sites(MODEL, TERM_ID, cycle) if _should_run(cycle) else []
     sites = [
         (site, get_region_factor(TERM_ID, site, TermTermType.LANDUSEMANAGEMENT)) for site in sites
     ]

hestia_earth/models/geospatialDatabase/utils.py

@@ -196,4 +196,7 @@ def download(term: str, site: dict, data: dict, only_coordinates=False) -> dict:
 def get_region_factor(term_id: str, site: dict, termType: TermTermType = TermTermType.MEASUREMENT):
     region_id = region_level_1_id(site.get('region', {}).get('@id'))
     country_id = site.get('country', {}).get('@id')
-    return region_factor(MODEL, region_id, term_id, termType) or region_factor(MODEL, country_id, term_id, termType)
+    return (
+        # `region-measurement` only exists for countries
+        region_factor(MODEL, region_id, term_id, termType) if termType != TermTermType.MEASUREMENT else None
+    ) or region_factor(MODEL, country_id, term_id, termType)

hestia_earth/models/globalCropWaterModel2008/rootingDepth.py

@@ -1,11 +1,10 @@
 from hestia_earth.schema import CycleFunctionalUnit, TermTermType
-from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, non_empty_list, safe_parse_float
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.property import _new_property, node_has_no_property
-from hestia_earth.models.utils.term import get_irrigation_terms
+from hestia_earth.models.utils.term import get_irrigation_terms, download_term
 from hestia_earth.models.utils.crop import get_crop_lookup_value
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from . import MODEL
@@ -81,7 +80,7 @@ def _should_run_product(cycle: dict):
 
 
 def _run_cycle(products: list):
-    term = download_hestia(TERM_ID)
+    term = download_term(TERM_ID, TermTermType.PROPERTY)
 
     def run_product(values: tuple):
         product, value = values

hestia_earth/models/haversineFormula/transport/distance.py

@@ -10,7 +10,7 @@ from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
-from hestia_earth.models.utils import _include_methodModel
+from hestia_earth.models.utils.method import include_methodModel
 from .. import MODEL
 
 REQUIREMENTS = {
@@ -39,8 +39,8 @@ MODEL_KEY = 'distance'
 
 def _run_transport(cycle: dict, distance_kms: float):
     def exec(transport: dict):
-        return _include_methodModel({
-            **transport, MODEL_KEY: distance_kms
+        return include_methodModel(transport | {
+            MODEL_KEY: distance_kms
         }, MODEL) if _should_run_transport(cycle, transport) else transport
     return exec