hestia-earth-models 0.73.7__py3-none-any.whl → 0.74.0__py3-none-any.whl

hestia_earth/models/config/__init__.py

@@ -1,10 +1,16 @@
 import os
 import json
+from enum import Enum
 from hestia_earth.utils.tools import flatten
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+class AWARE_VERSION(Enum):
+    V1 = '1.2'
+    V2 = '2.0'
+
+
 def _is_aggregated_model(model: dict):
     return isinstance(model, dict) and 'aggregated' in model.get('value', '').lower()
 
@@ -17,12 +23,26 @@ def _remove_aggregated(models: list):
     return list(filter(lambda v: v is not None, values))
 
 
+def _use_aware_1(models: list):
+    return [
+        _use_aware_1(m) if isinstance(m, list) else
+        m | {'model': 'aware'} if m.get('model') == 'aware2-0' else
+        m | {'value': 'awareWaterBasinId_v1'} if m.get('value') == 'awareWaterBasinId' else
+        m
+        for m in models
+    ]
+
+
 def _load_config(filename: str) -> dict:
     with open(os.path.join(CURRENT_DIR, f"{filename}.json"), 'r') as f:
         return json.load(f)
 
 
-def load_config(node_type: str, skip_aggregated_models: bool = False) -> dict:
+def load_config(
+    node_type: str,
+    skip_aggregated_models: bool = False,
+    use_aware_version: AWARE_VERSION = AWARE_VERSION.V2
+) -> dict:
     """
     Load the configuration associated with the Node Type.
 
@@ -32,11 +52,15 @@ def load_config(node_type: str, skip_aggregated_models: bool = False) -> dict:
         The Node Type to load configuration. Can be: `Cycle`, `Site`, `ImpactAssessment`.
     skip_aggregated_models : bool
         Include models using aggregated data. Included by default.
+    use_aware_version : AWARE_VERSION
+        Choose which AWARE version to use. Defaults to using version `2.0`.
     """
     try:
         config = _load_config(node_type)
         models = config.get('models')
-        return config | {'models': _remove_aggregated(models) if skip_aggregated_models else models}
+        models = _remove_aggregated(models) if skip_aggregated_models else models
+        models = _use_aware_1(models) if use_aware_version == AWARE_VERSION.V1 else models
+        return config | {'models': models}
     except FileNotFoundError:
         raise Exception(f"Invalid type {node_type}.")
 

hestia_earth/models/cycle/animal/input/hestiaAggregatedData.py

@@ -56,12 +56,12 @@ def _should_run_animal(cycle: dict, animal: dict):
     inputs = list(filter(should_link_input_to_impact(cycle), inputs))
     nb_inputs = len(inputs)
 
-    logRequirements(cycle, model=MODEL_ID, term=term_id, key=MODEL_KEY,
+    logRequirements(cycle, model=MODEL_ID, term=term_id, key=MODEL_KEY, animalId=animal.get('animalId'),
                     end_date=end_date,
                     nb_inputs=nb_inputs)
 
     should_run = all([end_date, nb_inputs > 0])
-    logShouldRun(cycle, MODEL_ID, term_id, should_run, key=MODEL_KEY)
+    logShouldRun(cycle, MODEL_ID, term_id, should_run, key=MODEL_KEY, animalId=animal.get('animalId'))
     return should_run, inputs
 
 

hestia_earth/models/cycle/animal/input/properties.py

@@ -50,7 +50,7 @@ def _find_related_product(input: dict):
     return find_term_match(products, input.get('term', {}).get('@id'))
 
 
-def _run_input_by_impactAssessment(cycle: dict):
+def _run_input_by_impactAssessment(cycle: dict, **log_args):
     def exec(input: dict):
         term_id = input.get('term', {}).get('@id')
         product = _find_related_product(input)
@@ -58,7 +58,7 @@ def _run_input_by_impactAssessment(cycle: dict):
         all_properties = input.get('properties', [])
         new_properties = [p for p in properties if not find_term_match(all_properties, p.get('term', {}).get('@id'))]
         for prop in new_properties:
-            logShouldRun(cycle, MODEL, term_id, True, property=prop.get('term', {}).get('@id'))
+            logShouldRun(cycle, MODEL, term_id, True, property=prop.get('term', {}).get('@id'), **log_args)
         return {**input, 'properties': merge_blank_nodes(all_properties, new_properties)} if new_properties else input
     return exec
 
@@ -80,9 +80,10 @@ def _run_animal(cycle: dict, animal: dict):
             should_run_properties_value(i)
         ])
     ]
-    inputs = list(map(_run_input_by_impactAssessment(cycle), inputs))
-    inputs = rescale_properties_from_dryMatter(MODEL, cycle, inputs)
-    inputs = average_blank_node_properties_value(cycle, inputs)
+    log_args = {'animalId': animal.get('animalId')}
+    inputs = list(map(_run_input_by_impactAssessment(cycle, **log_args), inputs))
+    inputs = rescale_properties_from_dryMatter(MODEL, cycle, inputs, **log_args)
+    inputs = average_blank_node_properties_value(cycle, inputs, **log_args)
     return animal | {'inputs': inputs}
 
 

hestia_earth/models/cycle/animal/milkYield.py

@@ -42,12 +42,17 @@ def _run(cycle: dict, animal: dict):
     practices = non_empty_list(
         [p for p in cycle.get('practices', []) if p.get('term', {}).get('@id') in practice_ids]
     )
+    log_args = {
+        'model_key': MODEL_KEY,
+        'animalId': animal.get('animalId')
+    }
 
-    logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                    practice_ids=log_blank_nodes_id(practices))
+    logRequirements(cycle, model=MODEL, term=term_id,
+                    practice_ids=log_blank_nodes_id(practices),
+                    **log_args)
 
     for practice in practices:
-        logShouldRun(cycle, MODEL, practice.get('term', {}).get('@id'), True, model_key=MODEL_KEY)
+        logShouldRun(cycle, MODEL, practice.get('term', {}).get('@id'), True, **log_args)
 
     return {
         **animal,

hestia_earth/models/cycle/utils.py

@@ -11,21 +11,21 @@ def _should_run_property_by_min_max(property: dict):
     ])
 
 
-def _run_property(cycle: dict, property: dict):
+def _run_property(cycle: dict, property: dict, **log_args):
     term_id = property.get('term', {}).get('@id')
 
     should_run = _should_run_property_by_min_max(property)
-    logShouldRun(cycle, MODEL, term_id, should_run, key='value')
+    logShouldRun(cycle, MODEL, term_id, should_run, key='value', **log_args)
 
     return property | ({
         'value': list_average([property.get('min'), property.get('max')])
     } if should_run else {})
 
 
-def _run_properties(cycle: dict, blank_node: dict):
+def _run_properties(cycle: dict, blank_node: dict, **log_args):
     properties = blank_node.get('properties', [])
     return blank_node | ({
-        'properties': [_run_property(cycle, p) for p in properties]
+        'properties': [_run_property(cycle, p, **log_args) for p in properties]
     } if properties else {})
 
 
@@ -33,5 +33,5 @@ def should_run_properties_value(blank_node: dict):
     return any(map(_should_run_property_by_min_max, blank_node.get('properties', [])))
 
 
-def average_blank_node_properties_value(cycle: dict, blank_nodes: list):
-    return [_run_properties(cycle, v) for v in blank_nodes]
+def average_blank_node_properties_value(cycle: dict, blank_nodes: list, **log_args):
+    return [_run_properties(cycle, v, **log_args) for v in blank_nodes]

hestia_earth/models/dammgen2009/noxToAirExcreta.py

@@ -22,7 +22,7 @@ RETURNS = {
 }
 TERM_ID = 'noxToAirExcreta'
 TIER = EmissionMethodTier.TIER_1.value
-N2O_TERM_ID = 'n2OToAirExcretaDirect'
+_N2O_TERM_ID = 'n2OToAirExcretaDirect'
 
 
 def _emission(value: float):
@@ -32,22 +32,24 @@ def _emission(value: float):
     return emission
 
 
-def _run(n2o: dict):
-    value = 0.1 * list_sum(n2o.get("value", [])) / get_atomic_conversion(Units.KG_N2O, Units.TO_N)
+def _run(n2o_value: float):
+    value = 0.1 * n2o_value / get_atomic_conversion(Units.KG_N2O, Units.TO_N)
     value = value * get_atomic_conversion(Units.KG_NOX, Units.TO_N)
     return [_emission(value)]
 
 
 def _should_run(cycle: dict):
-    n2o = find_term_match(cycle.get('emissions', []), N2O_TERM_ID)
+    n2o = find_term_match(cycle.get('emissions', []), _N2O_TERM_ID)
+    n2o_value = list_sum(n2o.get("value", []), default=None)
 
-    logRequirements(cycle, model=MODEL, term=TERM_ID, has_n2o=n2o is not None)
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    **{_N2O_TERM_ID: n2o_value})
 
-    should_run = all([n2o])
+    should_run = all([n2o_value is not None])
     logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
-    return should_run, n2o
+    return should_run, n2o_value
 
 
 def run(cycle: dict):
-    should_run, n2o = _should_run(cycle)
-    return _run(n2o) if should_run else []
+    should_run, n2o_value = _should_run(cycle)
+    return _run(n2o_value) if should_run else []

hestia_earth/models/data/ecoinventV3/__init__.py

@@ -1,36 +1,18 @@
 import os
-from functools import lru_cache
-from hestia_earth.utils.lookup import column_name, get_table_value, load_lookup, lookup_columns
-from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logger
 
 _CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-_ENV_NAME = 'ECOINVENT_V3_FILEPATH'
+_ENV_FOLDER = 'ECOINVENT_V3_FOLDER'
+_ECOINVENT_FOLDER = os.getenv(_ENV_FOLDER) or _CURRENT_DIR
+_ECOINVENT_VERSION = os.getenv('ECOINVENT_VERSION', '3.9')
 
 
-@lru_cache()
-def _get_file():
-    filepath = os.getenv(_ENV_NAME, f"{os.path.join(_CURRENT_DIR, 'ecoinventV3_excerpt')}.csv")
-
+def get_filepath(term_type: str):
+    filename = f"ecoinventV{_ECOINVENT_VERSION.replace('.', '_')}-{term_type}.csv"
+    filepath = os.path.join(_ECOINVENT_FOLDER, filename)
     if not os.path.exists(filepath):
-        logger.warning('Ecoinvent file not found. Please make sure to set env variable "%s".', _ENV_NAME)
+        logger.warning('Ecoinvent file not found. Please make sure to set env variable "%s".', _ENV_FOLDER)
         return None
 
-    return load_lookup(filepath=filepath, keep_in_memory=True)
-
-
-def ecoinventV3_emissions(ecoinventName: str):
-    lookup = _get_file()
-    col_name = column_name('ecoinventName')
-
-    def emission(column: str):
-        id = get_table_value(lookup, col_name, ecoinventName, column_name(column))
-        value = get_table_value(lookup, col_name, ecoinventName, column_name(column.replace('termid', 'value')))
-        return (id, value) if id else None
-
-    columns = [
-        col for col in lookup_columns(lookup)
-        if col.endswith(column_name('termid'))
-    ]
-    return non_empty_list(map(emission, columns))
+    return filepath

hestia_earth/models/ecoalimV9/cycle.py

@@ -1,13 +1,18 @@
-from functools import reduce
 from statistics import mean
-from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.schema import EmissionMethodTier, TermTermType
 from hestia_earth.utils.tools import flatten, list_sum
+from hestia_earth.utils.blank_node import group_by_keys
 
-from hestia_earth.models.log import debugValues, logShouldRun, logRequirements
+from hestia_earth.models.log import logShouldRun, logRequirements
 from hestia_earth.models.utils.emission import _new_emission
-from hestia_earth.models.utils.background_emissions import get_background_inputs, no_gap_filled_background_emissions
-from hestia_earth.models.utils.blank_node import group_by_keys
-from .utils import get_input_mappings, ecoalim_values
+from hestia_earth.models.utils.background_emissions import (
+    get_background_inputs,
+    no_gap_filled_background_emissions,
+    log_missing_emissions,
+    parse_term_id,
+    process_input_mappings
+)
+from .utils import get_input_mappings, extract_input_mapping
 from . import MODEL
 
 REQUIREMENTS = {
@@ -46,19 +51,20 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
-    "ecoalim-emissionsResourceUse": "emission-",
-    "crop": "ecoalimMapping",
-    "processedFood": "ecoalimMapping",
+    "ecoalim-emission": "emission-",
+    "emission": "inputProductionGroupId",
     "animalProduct": "ecoalimMapping",
+    "crop": "ecoalimMapping",
+    "feedFoodAdditive": "ecoalimMapping",
     "forage": "ecoalimMapping",
-    "feedFoodAdditive": "ecoalimMapping"
+    "processedFood": "ecoalimMapping"
 }
 MODEL_KEY = 'cycle'
 TIER = EmissionMethodTier.BACKGROUND.value
 
 
-def _emission(term_id: str, value: float, input: dict):
-    emission = _new_emission(term_id, MODEL)
+def _emission(term_id: str, value: float, input: dict, country_id: str = None, key_id: str = None):
+    emission = _new_emission(term_id, MODEL, country_id, key_id)
     emission['value'] = [value]
     emission['methodTier'] = TIER
     emission['inputs'] = [input.get('term')]
@@ -69,32 +75,9 @@ def _emission(term_id: str, value: float, input: dict):
     return emission
 
 
-def _add_emission(cycle: dict, input: dict):
-    input_term_id = input.get('term', {}).get('@id')
-    operation_term_id = input.get('operation', {}).get('@id')
-    animal_term_id = input.get('animal', {}).get('@id')
-
-    def add(prev: dict, mapping: tuple):
-        gadm_id, ecoalim_key = mapping
-        # all countries have the same coefficient
-        coefficient = 1
-        emissions = ecoalim_values(ecoalim_key, 'emission')
-        for emission_term_id, value in emissions:
-            # log run on each emission so we know it did run
-            logShouldRun(cycle, MODEL, input_term_id, True, methodTier=TIER, emission_id=emission_term_id)
-            debugValues(cycle, model=MODEL, term=emission_term_id, model_key=MODEL_KEY,
-                        value=value,
-                        coefficient=coefficient,
-                        input=input_term_id,
-                        operation=operation_term_id,
-                        animal=animal_term_id)
-            prev[emission_term_id] = prev.get(emission_term_id, []) + [value * coefficient]
-        return prev
-    return add
-
-
 def _run_input(cycle: dict):
     no_gap_filled_background_emissions_func = no_gap_filled_background_emissions(cycle)
+    log_missing_emissions_func = log_missing_emissions(cycle, model=MODEL, methodTier=TIER)
 
     def run(inputs: list):
         input = inputs[0]
@@ -103,27 +86,50 @@ def _run_input(cycle: dict):
         mappings = get_input_mappings(MODEL, input)
         has_mappings = len(mappings) > 0
 
+        # grouping the inputs together in the logs
+        input_parent_term_id = (input.get('parent', {})).get('@id') or input.get('animalId', {})
+        extra_logs = {
+            **({'input_group_id': input_parent_term_id} if input_parent_term_id else {}),
+            **({'animalId': input.get('animalId')} if input.get('animalId') else {})
+        }
+
         # skip input that has background emissions we have already gap-filled (model run before)
         has_no_gap_filled_background_emissions = no_gap_filled_background_emissions_func(input)
 
         logRequirements(cycle, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
-                        has_ecoalim_mappings=has_mappings,
-                        ecoalim_mappings=';'.join([v[1] for v in mappings]),
+                        has_mappings=has_mappings,
+                        mappings=';'.join([v[1] for v in mappings]),
                         has_no_gap_filled_background_emissions=has_no_gap_filled_background_emissions,
-                        input_value=input_value)
+                        input_value=input_value,
+                        **extra_logs)
 
         should_run = all([has_mappings, has_no_gap_filled_background_emissions, input_value])
-        logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER, model_key=MODEL_KEY)
-
-        grouped_emissions = reduce(_add_emission(cycle, input), mappings, {}) if should_run else {}
+        logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER, model_key=MODEL_KEY, **extra_logs)
+
+        results = process_input_mappings(
+            cycle, input, mappings, TermTermType.EMISSION,
+            extract_mapping=extract_input_mapping,
+            **(
+                extra_logs | {'model': MODEL, 'model_key': MODEL_KEY}
+            )
+        ) if should_run else {}
+        log_missing_emissions_func(input_term_id, list(map(parse_term_id, results.keys())), **(
+            extra_logs | {'has_mappings': has_mappings}
+        ))
         return [
-            _emission(term_id, mean(value) * input_value, input)
-            for term_id, value in grouped_emissions.items()
+            _emission(
+                term_id=parse_term_id(term_id),
+                value=mean([v['value'] * v['coefficient'] for v in values]) * input_value,
+                input=input,
+                country_id=values[0].get('country'),
+                key_id=values[0].get('key'),
+            )
+            for term_id, values in results.items()
         ]
     return run
 
 
 def run(cycle: dict):
     inputs = get_background_inputs(cycle)
-    grouped_inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
+    grouped_inputs = group_by_keys(inputs, ['term', 'operation', 'animal'])
     return flatten(map(_run_input(cycle), grouped_inputs.values()))

hestia_earth/models/ecoalimV9/impact_assessment.py

@@ -1,13 +1,17 @@
-from functools import reduce
 from statistics import mean
-from hestia_earth.schema import IndicatorMethodTier
+from hestia_earth.schema import IndicatorMethodTier, TermTermType
 from hestia_earth.utils.tools import flatten, list_sum
+from hestia_earth.utils.blank_node import group_by_keys
 
-from hestia_earth.models.log import debugValues, logShouldRun, logRequirements
+from hestia_earth.models.log import logShouldRun, logRequirements
 from hestia_earth.models.utils.indicator import _new_indicator
-from hestia_earth.models.utils.background_emissions import get_background_inputs
-from hestia_earth.models.utils.blank_node import group_by_keys
-from .utils import get_input_mappings, ecoalim_values
+from hestia_earth.models.utils.background_emissions import (
+    get_background_inputs,
+    log_missing_emissions,
+    parse_term_id,
+    process_input_mappings
+)
+from .utils import get_input_mappings, extract_input_mapping
 from . import MODEL
 
 REQUIREMENTS = {
@@ -48,53 +52,42 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
-    "ecoalim-emissionsResourceUse": "resourceUse-",
-    "crop": "ecoalimMapping",
-    "processedFood": "ecoalimMapping",
+    "ecoalim-resourceUse": "resourceUse-",
     "animalProduct": "ecoalimMapping",
+    "crop": "ecoalimMapping",
+    "feedFoodAdditive": "ecoalimMapping",
     "forage": "ecoalimMapping",
-    "feedFoodAdditive": "ecoalimMapping"
+    "processedFood": "ecoalimMapping"
 }
 MODEL_KEY = 'impact_assessment'
 TIER = IndicatorMethodTier.BACKGROUND.value
 
 
-def _indicator(term_id: str, value: float, input: dict):
-    indicator = _new_indicator(term_id, MODEL)
+def _indicator(
+    term_id: str,
+    value: float,
+    input: dict,
+    country_id: str = None,
+    key_id: str = None,
+    land_cover_id: str = None,
+    previous_land_cover_id: str = None,
+):
+    indicator = _new_indicator(term_id, MODEL, land_cover_id, previous_land_cover_id, country_id, key_id)
     indicator['value'] = value
     indicator['methodTier'] = TIER
     indicator['inputs'] = [input.get('term')]
     if input.get('operation'):
         indicator['operation'] = input.get('operation')
+    if input.get('animal'):
+        indicator['animals'] = [input.get('animal')]
     return indicator
 
 
-def _add_indicator(cycle: dict, input: dict):
-    input_term_id = input.get('term', {}).get('@id')
-    operation_term_id = input.get('operation', {}).get('@id')
-    animal_term_id = input.get('animal', {}).get('@id')
-
-    def add(prev: dict, mapping: tuple):
-        gadm_id, ecoalim_key = mapping
-        # all countries have the same coefficient
-        coefficient = 1
-        indicators = ecoalim_values(ecoalim_key, 'resourceUse')
-        for indicator_term_id, value in indicators:
-            # log run on each indicator so we know it did run
-            logShouldRun(cycle, MODEL, input_term_id, True, methodTier=TIER, emission_id=indicator_term_id)
-            debugValues(cycle, model=MODEL, term=indicator_term_id, model_key=MODEL_KEY,
-                        value=value,
-                        coefficient=coefficient,
-                        input=input_term_id,
-                        operation=operation_term_id,
-                        animal=animal_term_id)
-            if value is not None:
-                prev[indicator_term_id] = prev.get(indicator_term_id, []) + [value * coefficient]
-        return prev
-    return add
-
-
 def _run_input(impact_assessment: dict):
+    log_missing_emissions_func = log_missing_emissions(
+        impact_assessment, TermTermType.RESOURCEUSE, model=MODEL, methodTier=TIER
+    )
+
     def run(inputs: list):
         input = inputs[0]
         input_term_id = input.get('term', {}).get('@id')
@@ -102,25 +95,50 @@ def _run_input(impact_assessment: dict):
         mappings = get_input_mappings(MODEL, input)
         has_mappings = len(mappings) > 0
 
+        # grouping the inputs together in the logs
+        input_parent_term_id = (input.get('parent', {})).get('@id') or input.get('animalId', {})
+        extra_logs = {
+            **({'input_group_id': input_parent_term_id} if input_parent_term_id else {}),
+            **({'animalId': input.get('animalId')} if input.get('animalId') else {})
+        }
+
         logRequirements(impact_assessment, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
-                        has_ecoalim_mappings=has_mappings,
-                        ecoalim_mappings=';'.join([v[1] for v in mappings]),
-                        input_value=input_value)
+                        has_mappings=has_mappings,
+                        mappings=';'.join([v[1] for v in mappings]),
+                        input_value=input_value,
+                        **extra_logs)
 
         should_run = all([has_mappings, input_value])
         logShouldRun(
-            impact_assessment, MODEL, input_term_id, should_run, methodTier=TIER, model_key=MODEL_KEY
+            impact_assessment, MODEL, input_term_id, should_run, methodTier=TIER, model_key=MODEL_KEY, **extra_logs
         )
 
-        grouped_indicators = reduce(_add_indicator(impact_assessment, input), mappings, {}) if should_run else {}
+        results = process_input_mappings(
+            impact_assessment, input, mappings, TermTermType.RESOURCEUSE,
+            extract_mapping=extract_input_mapping,
+            **(
+                extra_logs | {'model': MODEL, 'model_key': MODEL_KEY}
+            )
+        ) if should_run else {}
+        log_missing_emissions_func(input_term_id, list(map(parse_term_id, results.keys())), **(
+            extra_logs | {'has_mappings': has_mappings}
+        ))
         return [
-            _indicator(term_id, mean(value) * input_value, input)
-            for term_id, value in grouped_indicators.items()
+            _indicator(
+                term_id=parse_term_id(term_id),
+                value=mean([v['value'] * v['coefficient'] for v in values]) * input_value,
+                input=input,
+                country_id=values[0].get('country'),
+                key_id=values[0].get('key'),
+                land_cover_id=values[0].get('landCover'),
+                previous_land_cover_id=values[0].get('previousLandCover'),
+            )
+            for term_id, values in results.items()
         ]
     return run
 
 
 def run(impact_assessment: dict):
     inputs = get_background_inputs(impact_assessment.get('cycle', {}))
-    grouped_inputs = reduce(group_by_keys(['term', 'operation']), inputs, {})
+    grouped_inputs = group_by_keys(inputs, ['term', 'operation', 'animal'])
     return flatten(map(_run_input(impact_assessment), grouped_inputs.values()))

hestia_earth/models/ecoalimV9/utils.py

@@ -1,10 +1,12 @@
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, lookup_columns
+from functools import lru_cache
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.lookup import download_lookup, column_name
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.utils.term import get_lookup_value
+from hestia_earth.models.utils.background_emissions import convert_background_lookup
 
-
-_LOOKUP = "ecoalim-emissionsResourceUse.csv"
+_LOOKUP_INDEX_KEY = column_name('ecoalimMappingName')
 
 
 def get_input_mappings(model: str, input: dict):
@@ -15,17 +17,21 @@ def get_input_mappings(model: str, input: dict):
     return [(m.split(':')[0], m.split(':')[1]) for m in mappings]
 
 
-def ecoalim_values(mapping: str, column_prefix: str):
-    lookup = download_lookup(_LOOKUP)
-    col_name = column_name('ecoalimMappingName')
+def extract_input_mapping(mapping: tuple, term_type: TermTermType):
+    gadm_id, mapping_name = mapping
+    # # all countries have the same coefficient
+    coefficient = 1
+    values = ecoalim_values(mapping_name, term_type)
+    return values, coefficient
+
+
+@lru_cache()
+def _build_lookup(term_type: str):
+    lookup = download_lookup(f"ecoalim-{term_type}.csv", keep_in_memory=False)
+    return convert_background_lookup(lookup=lookup, index_column=_LOOKUP_INDEX_KEY)
 
-    def emission(column: str):
-        id = get_table_value(lookup, col_name, mapping, column)
-        value = get_table_value(lookup, col_name, mapping, column.replace('term', 'value'))
-        return (id, value) if id else None
 
-    columns = [
-        col for col in lookup_columns(lookup)
-        if col.startswith(column_name(column_prefix)) and col.endswith(column_name('term'))
-    ]
-    return non_empty_list(map(emission, columns))
+@lru_cache()
+def ecoalim_values(mapping: str, term_type: TermTermType):
+    data = _build_lookup(term_type.value)
+    return list(data[mapping].items())