hestia-earth-models 0.73.8__py3-none-any.whl → 0.74.0__py3-none-any.whl

hestia_earth/models/ecoalimV9/utils.py

@@ -1,15 +1,12 @@
-from functools import lru_cache, reduce
-from hestia_earth.schema import EmissionMethodTier, TermTermType
-from hestia_earth.utils.lookup import download_lookup, _is_missing_value, column_name, lookup_columns
-from hestia_earth.utils.tools import non_empty_list, safe_parse_float
+from functools import lru_cache
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.lookup import download_lookup, column_name
+from hestia_earth.utils.tools import non_empty_list
 
-from hestia_earth.models.log import debugValues, logShouldRun
-from hestia_earth.models.utils import _omit
 from hestia_earth.models.utils.term import get_lookup_value
-from . import MODEL
+from hestia_earth.models.utils.background_emissions import convert_background_lookup
 
 _LOOKUP_INDEX_KEY = column_name('ecoalimMappingName')
-_TIER = EmissionMethodTier.BACKGROUND.value
 
 
 def get_input_mappings(model: str, input: dict):
@@ -20,75 +17,18 @@ def get_input_mappings(model: str, input: dict):
     return [(m.split(':')[0], m.split(':')[1]) for m in mappings]
 
 
-def parse_term_id(term_id: str): return term_id.split('-')[0]
-
-
-def _process_mapping(node: dict, input: dict, term_type: TermTermType, **log_args):
-    input_term_id = input.get('term', {}).get('@id')
-    operation_term_id = input.get('operation', {}).get('@id')
-    animal_term_id = input.get('animal', {}).get('@id')
-
-    def add(prev: dict, mapping: tuple):
-        gadm_id, ecoalim_key = mapping
-        # all countries have the same coefficient
-        coefficient = 1
-        values = ecoalim_values(ecoalim_key, term_type)
-        for term_id, data in values:
-            # log run on each node so we know it did run
-            logShouldRun(node, MODEL, input_term_id, True, methodTier=_TIER, emission_id=term_id)
-            debugValues(node, model=MODEL, term=term_id,
-                        value=data.get('value'),
-                        coefficient=coefficient,
-                        input=input_term_id,
-                        operation=operation_term_id,
-                        animal=animal_term_id,
-                        **log_args)
-            group_id = '-'.join(non_empty_list([term_id] + list(_omit(data, ['value']).values())))
-            prev[group_id] = prev.get(group_id, []) + [data | {'coefficient': coefficient}]
-        return prev
-    return add
-
-
-def process_input(node: dict, input: dict, mappings: list, term_type: TermTermType, **log_args):
-    return reduce(_process_mapping(node, input, term_type, **log_args), mappings, {})
-
-
-_KEY_TO_FIELD = {
-    'inputs': 'key'
-}
-
-
-def _key_to_field(key: str): return _KEY_TO_FIELD.get(key) or key
-
-
-def _values_from_column(column: str, value: str):
-    values = column.split('+')
-    term_id = values[0]
-    value = safe_parse_float(value, default=None)
-    return {
-        term_id: {
-            'value': value
-        } | {
-            _key_to_field(v.split('[')[0]): v.split('[')[1][:-1] for v in values[1:]
-        }
-    } if all([
-        column != _LOOKUP_INDEX_KEY,
-        not _is_missing_value(value)
-    ]) else {}
+def extract_input_mapping(mapping: tuple, term_type: TermTermType):
+    gadm_id, mapping_name = mapping
+    # # all countries have the same coefficient
+    coefficient = 1
+    values = ecoalim_values(mapping_name, term_type)
+    return values, coefficient
 
 
 @lru_cache()
 def _build_lookup(term_type: str):
     lookup = download_lookup(f"ecoalim-{term_type}.csv", keep_in_memory=False)
-    columns = lookup_columns(lookup)
-    return {
-        row[_LOOKUP_INDEX_KEY]: reduce(
-            lambda prev, curr: prev | _values_from_column(curr, row[curr]),
-            columns,
-            {}
-        )
-        for row in lookup
-    }
+    return convert_background_lookup(lookup=lookup, index_column=_LOOKUP_INDEX_KEY)
 
 
 @lru_cache()

hestia_earth/models/ecoinventV3/__init__.py

@@ -1,145 +1,13 @@
-from functools import reduce
-from hestia_earth.schema import EmissionMethodTier
-from hestia_earth.utils.tools import flatten, list_sum
+from os.path import dirname, abspath
+import sys
+from importlib import import_module
 
-from hestia_earth.models.log import debugValues, logShouldRun, logRequirements
-from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
-from hestia_earth.models.utils.emission import _new_emission
-from hestia_earth.models.utils.background_emissions import (
-    get_background_inputs,
-    no_gap_filled_background_emissions,
-    log_missing_emissions
-)
-from hestia_earth.models.utils.blank_node import group_by_keys
-from hestia_earth.models.utils.pesticideAI import get_pesticides_from_inputs
-from hestia_earth.models.utils.fertiliser import get_fertilisers_from_inputs
-from .utils import get_input_mappings
+from hestia_earth.models.utils.blank_node import run_if_required
 
-REQUIREMENTS = {
-    "Cycle": {
-        "inputs": [{
-            "@type": "Input",
-            "value": "> 0",
-            "none": {
-                "fromCycle": "True",
-                "producedInCycle": "True"
-            }
-        }],
-        "optional": {
-            "animals": [{
-                "@type": "Animal",
-                "inputs": [{
-                    "@type": "Input",
-                    "value": "> 0",
-                    "none": {
-                        "fromCycle": "True",
-                        "producedInCycle": "True"
-                    }
-                }]
-            }]
-        }
-    }
-}
-RETURNS = {
-    "Emission": [{
-        "term": "",
-        "value": "",
-        "methodTier": "background",
-        "inputs": "",
-        "operation": "",
-        "animals": ""
-    }]
-}
-LOOKUPS = {
-    "emission": "inputProductionGroupId",
-    "electricity": "ecoinventMapping",
-    "fuel": "ecoinventMapping",
-    "inorganicFertiliser": "ecoinventMapping",
-    "material": "ecoinventMapping",
-    "pesticideAI": "ecoinventMapping",
-    "soilAmendment": "ecoinventMapping",
-    "transport": "ecoinventMapping",
-    "veterinaryDrugs": "ecoinventMapping",
-    "feedFoodAdditive": "ecoinventMapping"
-}
+CURRENT_DIR = dirname(abspath(__file__)) + '/'
+sys.path.append(CURRENT_DIR)
 MODEL = 'ecoinventV3'
-MODEL_KEY = 'impactAssessment'  # keep to generate entry in "model-links.json"
-TIER = EmissionMethodTier.BACKGROUND.value
+PKG = '.'.join(['hestia_earth', 'models', MODEL])
 
 
-def _emission(term_id: str, value: float, input: dict):
-    emission = _new_emission(term_id, MODEL)
-    emission['value'] = [value]
-    emission['methodTier'] = TIER
-    emission['inputs'] = [input.get('term')]
-    if input.get('operation'):
-        emission['operation'] = input.get('operation')
-    if input.get('animal'):
-        emission['animals'] = [input.get('animal')]
-    return emission
-
-
-def _add_emission(cycle: dict, input: dict, **extra_logs):
-    input_term_id = input.get('term', {}).get('@id')
-    operation_term_id = input.get('operation', {}).get('@id')
-    animal_term_id = input.get('animal', {}).get('@id')
-
-    def add(prev: dict, mapping: tuple):
-        ecoinventName, coefficient = mapping
-        emissions = ecoinventV3_emissions(ecoinventName)
-        for emission_term_id, value in emissions:
-            # log run on each emission so we know it did run
-            logShouldRun(cycle, MODEL, input_term_id, True, methodTier=TIER, emission_id=emission_term_id, **extra_logs)
-            debugValues(cycle, model=MODEL, term=emission_term_id,
-                        value=value,
-                        coefficient=coefficient,
-                        input=input_term_id,
-                        operation=operation_term_id,
-                        animal=animal_term_id)
-            prev[emission_term_id] = prev.get(emission_term_id, 0) + (value * coefficient)
-        return prev
-    return add
-
-
-def _run_input(cycle: dict):
-    no_gap_filled_background_emissions_func = no_gap_filled_background_emissions(cycle)
-    log_missing_emissions_func = log_missing_emissions(cycle, model=MODEL, methodTier=TIER)
-
-    def run(inputs: list):
-        input = inputs[0]
-        input_term_id = input.get('term', {}).get('@id')
-        input_value = list_sum(flatten(input.get('value', []) for input in inputs))
-        mappings = get_input_mappings(MODEL, cycle, input)
-        has_mappings = len(mappings) > 0
-
-        # grouping the inputs together in the logs
-        input_parent_term_id = input.get('parent', {}).get('@id')
-        extra_logs = {'input_group_id': input_parent_term_id} if input_parent_term_id else {}
-
-        # skip input that has background emissions we have already gap-filled (model run before)
-        has_no_gap_filled_background_emissions = no_gap_filled_background_emissions_func(input)
-
-        logRequirements(cycle, model=MODEL, term=input_term_id,
-                        has_ecoinvent_mappings=has_mappings,
-                        ecoinvent_mappings=';'.join([v[0] for v in mappings]),
-                        has_no_gap_filled_background_emissions=has_no_gap_filled_background_emissions,
-                        input_value=input_value,
-                        **extra_logs)
-
-        should_run = all([has_mappings, has_no_gap_filled_background_emissions, input_value])
-        logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER, **extra_logs)
-
-        results = reduce(_add_emission(cycle, input, **extra_logs), mappings, {}) if should_run else {}
-        log_missing_emissions_func(input_term_id, list(results.keys()), **extra_logs)
-        return [
-            _emission(term_id, value * input_value, input)
-            for term_id, value in results.items()
-        ]
-    return run
-
-
-def run(_, cycle: dict):
-    extra_inputs = get_pesticides_from_inputs(cycle) + get_fertilisers_from_inputs(cycle)
-    inputs = get_background_inputs(cycle, extra_inputs=extra_inputs)
-    grouped_inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
-    return flatten(map(_run_input(cycle), grouped_inputs.values()))
+def run(model: str, data): return run_if_required(MODEL, model, data, import_module(f".{model}", package=PKG))

hestia_earth/models/ecoinventV3/cycle.py

@@ -0,0 +1,140 @@
+from hestia_earth.schema import EmissionMethodTier, TermTermType
+from hestia_earth.utils.tools import flatten, list_sum
+from hestia_earth.utils.blank_node import group_by_keys
+
+from hestia_earth.models.log import logShouldRun, logRequirements
+from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.background_emissions import (
+    get_background_inputs,
+    no_gap_filled_background_emissions,
+    log_missing_emissions,
+    parse_term_id,
+    process_input_mappings
+)
+from hestia_earth.models.utils.pesticideAI import get_pesticides_from_inputs
+from hestia_earth.models.utils.fertiliser import get_fertilisers_from_inputs
+from .utils import get_input_mappings, extract_input_mapping
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "inputs": [{
+            "@type": "Input",
+            "value": "> 0",
+            "none": {
+                "fromCycle": "True",
+                "producedInCycle": "True"
+            }
+        }],
+        "optional": {
+            "animals": [{
+                "@type": "Animal",
+                "inputs": [{
+                    "@type": "Input",
+                    "value": "> 0",
+                    "none": {
+                        "fromCycle": "True",
+                        "producedInCycle": "True"
+                    }
+                }]
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "term": "",
+        "value": "",
+        "methodTier": "background",
+        "inputs": "",
+        "operation": "",
+        "animals": ""
+    }]
+}
+LOOKUPS = {
+    "emission": "inputProductionGroupId",
+    "electricity": "ecoinventMapping",
+    "fuel": "ecoinventMapping",
+    "inorganicFertiliser": "ecoinventMapping",
+    "material": "ecoinventMapping",
+    "pesticideAI": "ecoinventMapping",
+    "soilAmendment": "ecoinventMapping",
+    "transport": "ecoinventMapping",
+    "veterinaryDrugs": "ecoinventMapping",
+    "feedFoodAdditive": "ecoinventMapping"
+}
+MODEL_KEY = 'cycle'
+TIER = EmissionMethodTier.BACKGROUND.value
+
+
+def _emission(term_id: str, value: float, input: dict, country_id: str = None, key_id: str = None):
+    emission = _new_emission(term_id, MODEL, country_id, key_id)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    emission['inputs'] = [input.get('term')]
+    if input.get('operation'):
+        emission['operation'] = input.get('operation')
+    if input.get('animal'):
+        emission['animals'] = [input.get('animal')]
+    return emission
+
+
+def _run_input(cycle: dict):
+    no_gap_filled_background_emissions_func = no_gap_filled_background_emissions(cycle)
+    log_missing_emissions_func = log_missing_emissions(cycle, model=MODEL, methodTier=TIER)
+
+    def run(inputs: list):
+        input = inputs[0]
+        input_term_id = input.get('term', {}).get('@id')
+        input_value = list_sum(flatten(input.get('value', []) for input in inputs))
+        mappings = get_input_mappings(MODEL, input)
+        has_mappings = len(mappings) > 0
+
+        # grouping the inputs together in the logs
+        input_parent_term_id = (input.get('parent', {})).get('@id') or input.get('animalId', {})
+        extra_logs = {
+            **({'input_group_id': input_parent_term_id} if input_parent_term_id else {}),
+            **({'animalId': input.get('animalId')} if input.get('animalId') else {})
+        }
+
+        # skip input that has background emissions we have already gap-filled (model run before)
+        has_no_gap_filled_background_emissions = no_gap_filled_background_emissions_func(input)
+
+        logRequirements(cycle, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
+                        has_mappings=has_mappings,
+                        mappings=';'.join([v[0] for v in mappings]),
+                        has_no_gap_filled_background_emissions=has_no_gap_filled_background_emissions,
+                        input_value=input_value,
+                        **extra_logs)
+
+        should_run = all([has_mappings, has_no_gap_filled_background_emissions, input_value])
+        logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER, model_key=MODEL_KEY, **extra_logs)
+
+        results = process_input_mappings(
+            cycle, input, mappings, TermTermType.EMISSION,
+            extract_mapping=extract_input_mapping,
+            **(
+                extra_logs | {'model': MODEL, 'model_key': MODEL_KEY}
+            )
+        ) if should_run else {}
+        log_missing_emissions_func(input_term_id, list(map(parse_term_id, results.keys())), **(
+            extra_logs | {'has_mappings': has_mappings}
+        ))
+        return [
+            _emission(
+                term_id=parse_term_id(term_id),
+                value=sum([v['value'] * v['coefficient'] for v in values]) * input_value,
+                input=input,
+                country_id=values[0].get('country'),
+                key_id=values[0].get('key'),
+            )
+            for term_id, values in results.items()
+        ]
+    return run
+
+
+def run(cycle: dict):
+    extra_inputs = get_pesticides_from_inputs(cycle) + get_fertilisers_from_inputs(cycle)
+    inputs = get_background_inputs(cycle, extra_inputs=extra_inputs)
+    grouped_inputs = group_by_keys(inputs, ['term', 'operation', 'animal'])
+    return flatten(map(_run_input(cycle), grouped_inputs.values()))

hestia_earth/models/ecoinventV3/utils.py

@@ -1,11 +1,38 @@
+from functools import lru_cache
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.lookup import column_name, load_lookup
 from hestia_earth.utils.tools import non_empty_list
 
+from hestia_earth.models.data.ecoinventV3 import get_filepath
 from hestia_earth.models.utils.term import get_lookup_value
+from hestia_earth.models.utils.background_emissions import convert_background_lookup
 
+_LOOKUP_INDEX_KEY = column_name('ecoinventName')
 
-def get_input_mappings(model: str, cycle: dict, input: dict):
+
+def get_input_mappings(model: str, input: dict):
     term = input.get('term', {})
     term_id = term.get('@id')
     value = get_lookup_value(term, 'ecoinventMapping', model=model, term=term_id)
     mappings = non_empty_list(value.split(';')) if value else []
     return [(m.split(':')[0], float(m.split(':')[1])) for m in mappings]
+
+
+def extract_input_mapping(mapping: tuple, term_type: TermTermType):
+    mapping_name, coefficient = mapping
+    values = ecoinvent_values(mapping_name, term_type)
+    return values, coefficient
+
+
+@lru_cache()
+def _build_lookup(term_type: str):
+    filepath = get_filepath(term_type)
+
+    lookup = load_lookup(filepath=filepath, keep_in_memory=False)
+    return convert_background_lookup(lookup=lookup, index_column=_LOOKUP_INDEX_KEY)
+
+
+@lru_cache()
+def ecoinvent_values(mapping: str, term_type: TermTermType):
+    data = _build_lookup(term_type.value)
+    return list(data.get(mapping, {}).items())

hestia_earth/models/ecoinventV3AndEmberClimate/__init__.py

@@ -1,142 +1,13 @@
-from functools import reduce
-from hestia_earth.utils.tools import list_sum, flatten
-from hestia_earth.schema import EmissionMethodTier
+from os.path import dirname, abspath
+import sys
+from importlib import import_module
 
-from hestia_earth.models.log import logShouldRun, logRequirements, debugValues
-from hestia_earth.models.utils.emission import _new_emission
-from hestia_earth.models.utils.background_emissions import get_background_inputs, no_gap_filled_background_emissions
-from hestia_earth.models.utils.blank_node import group_by_keys
-from hestia_earth.models.utils.completeness import _is_term_type_complete
-from hestia_earth.models.utils.term import get_electricity_grid_mix_terms
-from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
-from .utils import get_input_coefficient
-from ..ecoinventV3.utils import get_input_mappings
-
-REQUIREMENTS = {
-    "Cycle": {
-        "completeness.electricityFuel": "True",
-        "site": {
-            "@type": "Site",
-            "country": {"@type": "Term", "termType": "region"}
-        },
-        "inputs": [{
-            "@type": "Input",
-            "term.@id": ["electricityGridMarketMix", "electricityGridRenewableMix"],
-            "value": "> 0",
-            "none": {
-                "fromCycle": "True",
-                "producedInCycle": "True"
-            }
-        }],
-        "optional": {
-            "animals": [{
-                "@type": "Animal",
-                "inputs": [{
-                    "@type": "Input",
-                    "term.@id": ["electricityGridMarketMix", "electricityGridRenewableMix"],
-                    "value": "> 0",
-                    "none": {
-                        "fromCycle": "True",
-                        "producedInCycle": "True"
-                    }
-                }]
-            }]
-        }
-    }
-}
-RETURNS = {
-    "Emission": [{
-        "value": "",
-        "methodTier": "background",
-        "@type": "Emission",
-        "inputs": ""
-    }]
-}
-LOOKUPS = {
-    "electricity": "ecoinventMapping",
-    "region-ember-energySources": "",
-    "ember-ecoinvent-mapping": ["ember", "ecoinventId", "ecoinventName"]
-}
+from hestia_earth.models.utils.blank_node import run_if_required
 
+CURRENT_DIR = dirname(abspath(__file__)) + '/'
+sys.path.append(CURRENT_DIR)
 MODEL = 'ecoinventV3AndEmberClimate'
-MODEL_KEY = 'impactAssessment'  # keep to generate entry in "model-links.json"
-TIER = EmissionMethodTier.BACKGROUND.value
-
-
-def _emission(term_id: str, value: float, input: dict):
-    emission = _new_emission(term_id, MODEL)
-    emission['value'] = [value]
-    emission['methodTier'] = TIER
-    emission['inputs'] = [input.get('term')]
-    if input.get('operation'):
-        emission['operation'] = input.get('operation')
-    if input.get('animal'):
-        emission['animals'] = [input.get('animal')]
-    return emission
-
-
-def _add_emission(cycle: dict, input: dict):
-    input_term_id = input.get('term', {}).get('@id')
-    operation_term_id = input.get('operation', {}).get('@id')
-    animal_term_id = input.get('animal', {}).get('@id')
-    # TODO: as inputs are grouped, there could be more than 1 country
-    country_id = input.get('country', cycle.get('site', {}).get('country', {})).get('@id')
-
-    def add(prev: dict, mapping: tuple):
-        ecoinventName, _coefficient = mapping
-        # recalculate the coefficient using the country and year if it should be included
-        coefficient = get_input_coefficient(MODEL, cycle, country_id, ecoinventName) if _coefficient > 0 else 0
-        emissions = ecoinventV3_emissions(ecoinventName)
-        for emission_term_id, value in emissions:
-            # log run on each emission so we know it did run
-            logShouldRun(cycle, MODEL, input_term_id, True, methodTier=TIER, emission_id=emission_term_id)
-            debugValues(cycle, model=MODEL, term=emission_term_id,
-                        value=value,
-                        coefficient=coefficient,
-                        input=input_term_id,
-                        operation=operation_term_id,
-                        animal=animal_term_id)
-            prev[emission_term_id] = prev.get(emission_term_id, 0) + (value * coefficient)
-        return prev
-    return add
-
-
-def _run_input(cycle: dict):
-    electricity_complete = _is_term_type_complete(cycle, 'electricityFuel')
-    no_gap_filled_background_emissions_func = no_gap_filled_background_emissions(cycle)
-
-    def run(inputs: list):
-        input = inputs[0]
-        input_value = list_sum(flatten(input.get('value', []) for input in inputs))
-        input_term_id = input.get('term', {}).get('@id')
-        mappings = get_input_mappings(MODEL, cycle, input)
-        has_mappings = len(mappings) > 0
-
-        # skip input that has background emissions we have already gap-filled (model run before)
-        has_no_gap_filled_background_emissions = no_gap_filled_background_emissions_func(input)
-
-        logRequirements(cycle, model=MODEL, term=input_term_id,
-                        has_ecoinvent_mappings=has_mappings,
-                        ecoinvent_mappings=';'.join([v[0] for v in mappings]),
-                        has_no_gap_filled_background_emissions=has_no_gap_filled_background_emissions,
-                        termType_electricityFuel_complete=electricity_complete,
-                        input_value=input_value)
-
-        should_run = all([electricity_complete, has_mappings, has_no_gap_filled_background_emissions, input_value])
-        logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER)
-
-        grouped_emissions = reduce(_add_emission(cycle, input), mappings, {}) if should_run else {}
-        return [
-            _emission(term_id, value * input_value, input)
-            for term_id, value in grouped_emissions.items()
-        ]
-    return run
+PKG = '.'.join(['hestia_earth', 'models', MODEL])
 
 
-def run(_, cycle: dict):
-    terms = get_electricity_grid_mix_terms()
-    inputs = get_background_inputs(cycle)
-    # only keep the inputs matching the grid terms
-    inputs = [i for i in inputs if i.get('term', {}).get('@id') in terms]
-    grouped_inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
-    return flatten(map(_run_input(cycle), grouped_inputs.values()))
+def run(model: str, data): return run_if_required(MODEL, model, data, import_module(f".{model}", package=PKG))