hestia-earth-models 0.75.0__tar.gz → 0.75.2__tar.gz

{hestia_earth_models-0.75.0/hestia_earth_models.egg-info → hestia_earth_models-0.75.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_models
-Version: 0.75.0
+Version: 0.75.2
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -11,7 +11,7 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hestia-earth-schema<36.0.0,>=35.0.1
-Requires-Dist: hestia-earth-utils>=0.16.1
+Requires-Dist: hestia-earth-utils>=0.16.7
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0
 Requires-Dist: pydash

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/aware/scarcityWeightedWaterUse.py

@@ -1,6 +1,6 @@
 from hestia_earth.schema import SiteSiteType
 from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.lookup import download_lookup, _get_single_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.utils.tools import safe_parse_float
 
 from hestia_earth.models.log import logRequirements, debugMissingLookup, logShouldRun
@@ -57,9 +57,7 @@ def _indicator(value: float): return _new_indicator(term=TERM_ID, model=MODEL, v
 
 def _get_factor_from_basinId(site: dict, aware_id: str):
     lookup_col = 'YR_IRRI' if site.get('siteType') in IRRIGATED_SITE_TYPES else 'YR_NONIRRI'
-    value = _get_single_table_value(
-        download_lookup(_AWARE_LOOKUP), column_name(AWARE_KEY), int(aware_id), column_name(lookup_col)
-    )
+    value = get_table_value(download_lookup(_AWARE_LOOKUP), AWARE_KEY, int(aware_id), lookup_col)
     debugMissingLookup(_AWARE_LOOKUP, AWARE_KEY, aware_id, lookup_col, value, model=MODEL, term=TERM_ID)
     return safe_parse_float(value, default=None)
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/aware2_0/scarcityWeightedWaterUse.py

@@ -1,6 +1,6 @@
 from hestia_earth.schema import SiteSiteType
 from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.lookup import download_lookup, _get_single_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.utils.tools import safe_parse_float
 
 from hestia_earth.models.log import logRequirements, debugMissingLookup, logShouldRun
@@ -67,10 +67,7 @@ def _lookup_column(site: dict):
 def _get_factor_from_basinId(site: dict, aware_id: str):
     lookup_col = _lookup_column(site)
     lookup = download_lookup(_AWARE_LOOKUP)
-    try:
-        value = _get_single_table_value(lookup, column_name(AWARE_KEY), int(aware_id), column_name(lookup_col))
-    except Exception:
-        value = None
+    value = get_table_value(lookup, AWARE_KEY, int(aware_id), lookup_col)
     debugMissingLookup(_AWARE_LOOKUP, AWARE_KEY, aware_id, lookup_col, value, model=MODEL, term=TERM_ID)
     return safe_parse_float(value, default=None)
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/chaudharyBrooks2018/utils.py

@@ -1,4 +1,4 @@
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data
+from hestia_earth.utils.lookup import download_lookup, get_table_value, extract_grouped_data
 from hestia_earth.utils.tools import safe_parse_float
 
 from hestia_earth.models.log import debugMissingLookup, logRequirements
@@ -8,7 +8,7 @@ from . import MODEL
 
 
 def _lookup_value(term_id: str, lookup_name: str, col_match: str, col_val: str, column: str):
-    value = get_table_value(download_lookup(lookup_name), col_match, col_val, column_name(column))
+    value = get_table_value(download_lookup(lookup_name), col_match, col_val, column)
     debugMissingLookup(lookup_name, col_match, col_val, column, value, model=MODEL, term=term_id)
     return value
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/cml2001Baseline/abioticResourceDepletionFossilFuels.py

@@ -1,5 +1,5 @@
 from itertools import chain
-from hestia_earth.utils.lookup import download_lookup, column_name, find_term_ids_by
+from hestia_earth.utils.lookup import download_lookup, find_term_ids_by
 from hestia_earth.utils.tools import list_sum, flatten
 
 from hestia_earth.models.log import logShouldRun, logRequirements, log_as_table
@@ -49,7 +49,8 @@ def get_all_non_renewable_terms(lookup_file_name: str, column: str) -> list:
     returns all non renewable term ids in lookup files like `electricity.csv` or `fuel.csv`
     """
     lookup = download_lookup(lookup_file_name)
-    return find_term_ids_by(lookup, column_name(column), True)
+
+    return find_term_ids_by(lookup, column, True)
 
 
 def _valid_resource_indicator(resource: dict) -> bool:

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/cml2001Baseline/abioticResourceDepletionMineralsAndMetals.py

@@ -1,7 +1,8 @@
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import get_table_value, download_lookup, column_name
+from hestia_earth.utils.lookup import get_table_value, download_lookup
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum, flatten
+from hestia_earth.utils.lookup import is_missing_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
 from . import MODEL
@@ -81,16 +82,22 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
                     "value": _node_value(resource_indicator),
                     "coefficient": get_table_value(
                         lookup=download_lookup(filename=f"{input.get('termType')}.csv"),
-                        col_match='termid',
+                        col_match='term.id',
                         col_match_with=input.get('@id'),
-                        col_val=column_name(LOOKUPS.get(input.get('termType'), ''))) if input else None
+                        col_val=LOOKUPS.get(input.get('termType', '')),
+                        default_value=None
+                    ) if input else None
                 } for input in resource_indicator['inputs'] or [{}]]
             for resource_indicator in emissions_resource_use
         ]
     )
-    valid_resources_with_cf = [em for em in resource_uses_unpacked if em['coefficient'] is not None
-                               and em['indicator-is-valid'] is True
-                               and em['indicator-input-is-valid'] is True]
+    valid_resources_with_cf = [
+        em for em in resource_uses_unpacked if all([
+            not is_missing_value(em['coefficient']),
+            em['indicator-is-valid'] is True,
+            em['indicator-input-is-valid'] is True,
+        ])
+    ]
 
     has_valid_input_requirements = all([
         all([

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/config/Cycle.json

@@ -1294,6 +1294,21 @@
         },
         "stage": 2
       },
+      {
+        "key": "emissions",
+        "model": "ipcc2019",
+        "value": "emissionsToAirOrganicSoilBurning",
+        "runStrategy": "always",
+        "runArgs": {
+          "runNonMeasured": true,
+          "runNonAddedTerm": true
+        },
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
+      },
       {
         "key": "emissions",
         "model": "schmidt2007",

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/cycle/product/economicValueShare.py

@@ -1,5 +1,5 @@
 from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.tools import list_sum
+from hestia_earth.utils.tools import list_sum, to_precision
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.term import get_lookup_value
@@ -43,7 +43,7 @@ MIN_COMPLETE_VALUE = 80  # when the products are complete lower the min threshol
 
 
 def _product(product: dict, value: float):
-    return product | {MODEL_KEY: value}
+    return product | {MODEL_KEY: to_precision(value, 2 if value < 1 else 3 if value < 10 else 4)}
 
 
 def _is_complete(cycle: dict): return cycle.get('completeness', {}).get('product', False)
@@ -57,11 +57,11 @@ def _total_evs(products: list): return sum([p.get(MODEL_KEY, 0) for p in product
 
 def _product_with_value(product: dict):
     value = product.get(MODEL_KEY, lookup_share(MODEL_KEY, product))
-    return product | {MODEL_KEY: value} if value is not None else product
+    return _product(product, value) if value is not None else product
 
 
 def _rescale_value(products: list, total: float):
-    return list(map(lambda p: {**p, MODEL_KEY: p.get(MODEL_KEY) * 100 / total}, products))
+    return list(map(lambda p: _product(p, p.get(MODEL_KEY) * 100 / total), products))
 
 
 def _should_run_by_default(cycle: dict, products: list):

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/ecoalimV9/utils.py

@@ -1,13 +1,13 @@
 import os
 from functools import lru_cache
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.utils.tools import non_empty_list, safe_parse_float, omit
 
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.background_emissions import convert_background_lookup
 
-_LOOKUP_INDEX_KEY = column_name('ecoalimMappingName')
+_LOOKUP_INDEX_KEY = 'ecoalimMappingName'
 
 
 def get_input_mappings(model: str, input: dict):
@@ -58,7 +58,7 @@ def _cutoff_id(term_id: str, country_id: str = None, key_id: str = None):
 def cutoff_value(cutoff_lookup, term_id: str, country_id: str = None, key_id: str = None):
     cutoff_id = _cutoff_id(term_id=term_id, country_id=country_id, key_id=key_id)
     return None if cutoff_lookup is None else safe_parse_float(
-        get_table_value(cutoff_lookup, 'termid', cutoff_id, 'percentage'),
+        get_table_value(cutoff_lookup, 'term.id', cutoff_id, 'percentage'),
         default=None
     )
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/ecoinventV3/utils.py

@@ -1,13 +1,13 @@
 from functools import lru_cache
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import column_name, load_lookup
+from hestia_earth.utils.lookup import load_lookup
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.data.ecoinventV3 import get_filepath
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.background_emissions import convert_background_lookup
 
-_LOOKUP_INDEX_KEY = column_name('ecoinventName')
+_LOOKUP_INDEX_KEY = 'ecoinventName'
 
 
 def get_input_mappings(model: str, input: dict):

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/ecoinventV3AndEmberClimate/utils.py

@@ -1,4 +1,4 @@
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data
+from hestia_earth.utils.lookup import download_lookup, get_table_value, extract_grouped_data
 from hestia_earth.utils.tools import safe_parse_float
 
 from hestia_earth.models.utils.cycle import cycle_end_year
@@ -14,7 +14,7 @@ def get_input_coefficient(model: str, cycle: dict, country_id: str, ecoinventNam
     # find the matching ember source with the ecoinventName.
     # example: "electricity, high voltage, electricity production, hard coal" > "Coal"
     ember_ecoinvent_lookup = download_lookup(EMBER_ECOINVENT_LOOKUP_NAME)
-    source_name = get_table_value(ember_ecoinvent_lookup, column_name('ecoinventName'), ecoinventName, 'ember')
+    source_name = get_table_value(ember_ecoinvent_lookup, 'ecoinventName', ecoinventName, 'ember')
 
     # find the ratio for the country / year
     data = get_region_lookup_value(REGION_EMBER_SOURCES_LOOKUP_NAME, country_id, source_name, model=model)

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/emissionNotRelevant/__init__.py

@@ -1,5 +1,5 @@
 from hestia_earth.schema import NodeType, EmissionMethodTier, TermTermType
-from hestia_earth.utils.lookup import download_lookup, lookup_term_ids, lookup_columns, column_name
+from hestia_earth.utils.lookup import download_lookup, lookup_term_ids, lookup_columns
 from hestia_earth.utils.lookup_utils import is_in_system_boundary
 from hestia_earth.utils.tools import flatten
 
@@ -49,7 +49,7 @@ def _emission_ids():
 def _model_ids(lookup_suffix: str):
     return [
         col for col in lookup_columns(download_lookup(f"emission-model-{lookup_suffix}.csv"))
-        if col != column_name('term.id')
+        if col != 'term.id'
     ]
 
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/environmentalFootprintV3_1/environmentalFootprintSingleOverallScore.py

@@ -53,8 +53,7 @@ def _is_a_PEF_indicator(indicator: dict) -> bool:
 
 
 def _get_factor(term: dict, column: str) -> Optional[float]:
-    factor = get_lookup_value(term, column, model=MODEL, term=TERM_ID)
-    return float(factor) if factor is not None else None
+    return get_lookup_value(term, column, model=MODEL, term=TERM_ID)
 
 
 def _get_pef_method_model(term: dict) -> List[str]:

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/environmentalFootprintV3_1/soilQualityIndexLandOccupation.py

@@ -1,8 +1,8 @@
 from typing import Tuple
-
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum
+from hestia_earth.utils.lookup import is_missing_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.indicator import _new_indicator
@@ -86,10 +86,13 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
             for indicator in found_land_occupation_indicators
         ]) if found_land_occupation_indicators else False
 
-    valid_indicator_with_coef = [indicator for indicator in found_indicators_with_coefficient if
-                                 indicator['coefficient'] is not None and
-                                 indicator['area-by-year-is-valid'] and
-                                 indicator['area-unit-is-valid']]
+    valid_indicator_with_coef = [
+        indicator for indicator in found_indicators_with_coefficient if all([
+            not is_missing_value(indicator['coefficient']),
+            indicator['area-by-year-is-valid'],
+            indicator['area-unit-is-valid']
+        ])
+    ]
 
     has_land_occupation_indicators = bool(land_occupation_indicators)
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/faostat2018/utils.py

@@ -1,6 +1,6 @@
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data_closest_date
+from hestia_earth.utils.lookup import download_lookup, get_table_value, extract_grouped_data_closest_date
 from hestia_earth.utils.tools import safe_parse_float, flatten, non_empty_list
 
 from hestia_earth.models.log import logger, debugMissingLookup, logRequirements, logShouldRun, debugValues, log_as_table
@@ -20,8 +20,8 @@ FAOSTAT_AREA_LOOKUP = 'region-faostatArea.csv'
 def get_liveAnimal_to_animalProduct_id(product_term_id: str, column: str, **log_args):
     lookup_name = 'liveAnimal.csv'
     lookup = download_lookup(lookup_name)
-    value = get_table_value(lookup, 'termid', product_term_id, column_name(column))
-    debugMissingLookup(lookup_name, 'termid', product_term_id, column, value, model=MODEL, **log_args)
+    value = get_table_value(lookup, 'term.id', product_term_id, column)
+    debugMissingLookup(lookup_name, 'term.id', product_term_id, column, value, model=MODEL, **log_args)
     return value
 
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/frischknechtEtAl2000/ionisingRadiationKbqU235Eq.py

@@ -1,5 +1,5 @@
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import get_table_value, download_lookup, column_name
+from hestia_earth.utils.lookup import get_table_value, download_lookup, is_missing_value
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import flatten, list_sum, omit, pick, unique_values
 from hestia_earth.utils.blank_node import group_by_keys
@@ -82,9 +82,10 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
             "value": _node_value(emission),
             "coefficient": get_table_value(
                 lookup=download_lookup(filename="waste.csv"),
-                col_match='termid',
+                col_match='term.id',
                 col_match_with=emission['key'].get('@id'),
-                col_val=column_name(emission['term']['@id'])
+                col_val=emission['term']['@id'],
+                default_value=None
             )
         } | pick(emission, ['key', 'inputs'])
         for emission in emissions
@@ -93,7 +94,7 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
 
     valid_emission_with_cf = [
         em for em in emissions_unpacked if all([
-            em['coefficient'] is not None,
+            not is_missing_value(em['coefficient']),
             em['key-is-valid'] is True
         ])
     ]

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/geospatialDatabase/ecoClimateZone.py

@@ -1,5 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
-from hestia_earth.utils.lookup import download_lookup, _get_single_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
@@ -44,7 +44,7 @@ def _measurement(site: dict, value: int):
 
 def _name(value: int):
     lookup = download_lookup(f"{TERM_ID}.csv")
-    return _get_single_table_value(lookup, column_name(TERM_ID), value, 'name')
+    return get_table_value(lookup, TERM_ID, value, 'name')
 
 
 def _run(site: dict):

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/geospatialDatabase/histosol.py

@@ -1,7 +1,10 @@
-from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.schema import MeasurementMethodClassification, TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.blank_node import get_node_value
+from hestia_earth.utils.tools import pick
 
-from hestia_earth.models.log import logRequirements, logShouldRun
-from hestia_earth.models.utils.measurement import _new_measurement, total_other_soilType_value
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from .utils import download, has_geospatial_data, should_download
 from . import MODEL
@@ -16,10 +19,10 @@ REQUIREMENTS = {
         "none": {
             "measurements": [{
                 "@type": "Measurement",
-                "value": "100",
+                "value": "> 0",
                 "depthUpper": "0",
                 "depthLower": "30",
-                "term.termType": "soilType"
+                "term.termType": ["soilType", "usdaSoilType"]
             }]
         }
     }
@@ -39,12 +42,14 @@ EE_PARAMS = {
     'reducer': 'mean'
 }
 BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+_DEPTH_UPPER = 0
+_DEPTH_LOWER = 30
 
 
 def _measurement(site: dict, value: float):
     measurement = _new_measurement(term=TERM_ID, value=value)
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = _DEPTH_UPPER
+    measurement['depthLower'] = _DEPTH_LOWER
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
@@ -58,15 +63,30 @@ def _should_run(site: dict):
     contains_geospatial_data = has_geospatial_data(site)
     below_max_area_size = should_download(TERM_ID, site)
 
-    total_measurements_value = total_other_soilType_value(site.get('measurements', []), TERM_ID)
+    measurements = filter_list_term_type(site.get('measurements', []), [
+        TermTermType.SOILTYPE,
+        TermTermType.USDASOILTYPE
+    ])
+    measurements = [m for m in measurements if all([
+        m.get('depthUpper', -1) == _DEPTH_UPPER,
+        m.get('depthLower', 0) == _DEPTH_LOWER,
+        get_node_value(m) > 0
+    ])]
+    has_soil_type_measurements = len(measurements) > 0
 
     logRequirements(site, model=MODEL, term=TERM_ID,
                     contains_geospatial_data=contains_geospatial_data,
                     below_max_area_size=below_max_area_size,
-                    total_soilType_measurements_value=total_measurements_value,
-                    total_soilType_measurements_value_is_0=total_measurements_value == 0)
+                    has_soil_type_measurements=has_soil_type_measurements,
+                    soil_type_measurements=log_as_table([
+                        {
+                            'id': m.get('term', {}).get('@id'),
+                            'termType': m.get('term', {}).get('termType'),
+                            'value': get_node_value(m)
+                        } | pick(m, ['depthUpper', 'depthLower']) for m in measurements
+                    ]))
 
-    should_run = all([contains_geospatial_data, below_max_area_size, total_measurements_value == 0])
+    should_run = all([contains_geospatial_data, below_max_area_size, not has_soil_type_measurements])
     logShouldRun(site, MODEL, TERM_ID, should_run)
     return should_run
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/hestia/aboveGroundCropResidueTotal.py

@@ -9,8 +9,8 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "practices": [{"@type": "Practice", "value": "", "term.termType": "cropResidue"}],
-        "products": [{"@type": "Product", "value": "", "term.termType": "cropResidueManagement"}]
+        "practices": [{"@type": "Practice", "value": "", "term.termType": "cropResidueManagement"}],
+        "products": [{"@type": "Product", "value": "", "term.termType": "cropResidue"}]
     }
 }
 RETURNS = {

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/hestia/landCover_utils.py

@@ -3,7 +3,7 @@ from functools import lru_cache
 from collections import defaultdict
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import (
-    download_lookup, get_table_value, column_name, _is_missing_value, extract_grouped_data, lookup_columns
+    download_lookup, get_table_value, is_missing_value, extract_grouped_data, lookup_columns, lookup_term_ids
 )
 from hestia_earth.utils.tools import safe_parse_float
 
@@ -57,10 +57,6 @@ def _get_lookup_with_cache(lookup_term: dict, column: str):
 def _get_faostat_name(term: dict) -> str: return _get_lookup_with_cache(term, "cropGroupingFaostatArea")
 
 
-def _is_missing_or_none(value) -> bool:
-    return value is None or _is_missing_value(value)
-
-
 def _safe_divide(numerator, denominator, default=0) -> float:
     return default if not denominator else numerator / denominator
 
@@ -354,15 +350,15 @@ def _get_most_common_or_alphabetically_first(crop_terms: list) -> str:
     return sorted([term for term, freq in histogram.items() if freq == max_freq])[0]
 
 
+@lru_cache()
 def _get_complete_faostat_to_crop_mapping() -> dict:
     """Returns mapping in the format: {faostat_name: IPCC_LAND_USE_CATEGORY, ...}"""
     term_type = TermTermType.CROP.value
     lookup = download_lookup(f"{term_type}.csv")
+    term_ids = lookup_term_ids(lookup)
     mappings = defaultdict(list)
-    for crop_term_id in [row[0] for row in lookup]:
-        key = column_name(
-            get_table_value(lookup, 'termid', crop_term_id, column_name("cropGroupingFaostatArea"))
-        )
+    for crop_term_id in term_ids:
+        key = get_table_value(lookup, 'term.id', crop_term_id, "cropGroupingFaostatArea")
         if key:
             mappings[key].append(crop_ipcc_land_use_category(crop_term_id=crop_term_id, lookup_term_type="crop"))
     return {
@@ -409,7 +405,7 @@ def _get_sum_for_land_category(
     return sum([
         safe_parse_float(value=extract_grouped_data(table_value, str(year)), default=None)
         for fao_name, table_value in values.items()
-        if not _is_missing_or_none(extract_grouped_data(table_value, str(year))) and
+        if not is_missing_value(extract_grouped_data(table_value, str(year))) and
         fao_stat_to_ipcc_type[fao_name] == ipcc_land_use_category and
         (
             include_negatives or
@@ -426,8 +422,8 @@ def _get_sums_of_crop_expansion(country_id: str, year: int, include_negatives: b
     lookup = get_region_lookup(lookup_name=_LOOKUP_EXPANSION, term_id=country_id)
     columns = lookup_columns(lookup)
     values = {
-        name: get_table_value(lookup, 'termid', country_id, column_name(name))
-        for name in columns if name != "termid"
+        name: get_table_value(lookup, 'term.id', country_id, name)
+        for name in columns if name != 'term.id'
     }
 
     fao_stat_to_ipcc_type = _get_complete_faostat_to_crop_mapping()

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/hestia/management.py

@@ -1,7 +1,7 @@
 from typing import List
 from datetime import timedelta, datetime
 from hestia_earth.schema import SchemaType, TermTermType, COMPLETENESS_MAPPING
-from hestia_earth.utils.lookup import column_name, get_table_value, download_lookup
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import safe_parse_float, flatten, is_number, is_boolean, pick
 from hestia_earth.utils.blank_node import get_node_value
@@ -117,9 +117,9 @@ def _get_cycle_duration(cycle: dict, land_cover_id: str = None):
     cycle_duration = cycle.get('cycleDuration')
     lookup_value = None if cycle_duration or not land_cover_id else safe_parse_float(get_table_value(
         download_lookup("crop.csv"),
-        column_name('landCoverTermId'),
+        'landCoverTermId',
         land_cover_id,
-        column_name('maximumCycleDuration')
+        'maximumCycleDuration'
     ), default=None)
     return cycle_duration or lookup_value
 

{hestia_earth_models-0.75.0 → hestia_earth_models-0.75.2}/hestia_earth/models/hestia/seed_emissions.py

@@ -1,7 +1,7 @@
 from functools import reduce
 from hestia_earth.schema import TermTermType, EmissionMethodTier, SiteSiteType
 from hestia_earth.utils.lookup import (
-    download_lookup, column_name, extract_grouped_data_closest_date, find_term_ids_by
+    download_lookup, extract_grouped_data_closest_date, find_term_ids_by
 )
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list, flatten, list_sum, safe_parse_float, omit
@@ -103,7 +103,7 @@ def _map_group_emissions(group_id: str, required_emission_term_ids: list, emissi
     lookup = download_lookup('emission.csv')
     emissions = list(filter(
         lambda id: id in required_emission_term_ids,
-        find_term_ids_by(lookup, column_name('inputProductionGroupId'), group_id)
+        find_term_ids_by(lookup, 'inputProductionGroupId', group_id)
     ))
     included_emissions = list(filter(lambda v: v in emission_ids, emissions))
     missing_emissions = list(filter(lambda v: v not in emission_ids, emissions))