hestia-earth-models 0.62.2__py3-none-any.whl → 0.62.3__py3-none-any.whl

hestia_earth/models/agribalyse2016/fuelElectricity.py

@@ -93,7 +93,7 @@ def _should_run_operation(cycle: dict):
 
         logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
                         has_value=has_value,
-                        has_lookup_value=has_lookup_value)
+                        has_fuelUse_lookup_value=has_lookup_value)
 
         should_run = all([has_value, has_lookup_value])
         logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)

hestia_earth/models/cycle/animal/input/hestiaAggregatedData.py

@@ -1,5 +1,5 @@
 """
-Animal Input Hestia Aggregated Data
+Animal Input HESTIA Aggregated Data
 
 This model adds `impactAssessment` to Animal inputs based on data which has been aggregated into country level averages.
 Note: to get more accurate impacts, we recommend setting the

hestia_earth/models/cycle/cycleDuration.py

@@ -10,7 +10,7 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import find_term_match, find_primary_product
 from hestia_earth.utils.date import diff_in_days
 
-from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
 from hestia_earth.models.utils.crop import get_crop_grouping_fao
 from . import MODEL
 
@@ -66,7 +66,12 @@ def _run_by_crop(cycle: dict):
     grouping = get_crop_grouping_fao(MODEL, MODEL_KEY, product.get('term', {}))
     is_permanent_crop = grouping == 'Permanent crops'
     croppingIntensity = find_term_match(cycle.get('practices', []), 'croppingIntensity').get('value', [1])[0]
-    return DEFAULT_DURATION * (1 if is_permanent_crop else croppingIntensity)
+    ratio = 1 if is_permanent_crop else croppingIntensity
+    debugValues(cycle, model=MODEL, key=MODEL_KEY, by='product',
+                crop_grouping=grouping,
+                croppingIntensity=croppingIntensity,
+                ratio=ratio)
+    return round(DEFAULT_DURATION * ratio, 3)
 
 
 def _should_run_by_crop(cycle: dict):

hestia_earth/models/cycle/input/hestiaAggregatedData.py

@@ -1,5 +1,5 @@
 """
-Input Hestia Aggregated Data
+Input HESTIA Aggregated Data
 
 This model adds `impactAssessment` to inputs based on data which has been aggregated into country level averages.
 Note: to get more accurate impacts, we recommend setting the

hestia_earth/models/cycle/product/price.py

@@ -42,9 +42,13 @@ def _should_run_product_by_share_0(cycle: dict, product: dict):
     return should_run
 
 
+def _should_run_product(product: dict): return product.get(MODEL_KEY) is None
+
+
 def run(cycle: dict):
+    products = list(filter(_should_run_product, cycle.get('products', [])))
     return non_empty_list([
         (
             _product(p, 0) if _should_run_product_by_share_0(cycle, p) else None
-        ) for p in cycle.get('products', [])
+        ) for p in products
     ])

hestia_earth/models/cycle/product/revenue.py

@@ -53,23 +53,22 @@ def _should_run(cycle: dict):
         has_yield_and_price = has_yield and has_price
         is_yield_0 = list_sum(product.get('value', []), -1) == 0
         is_price_0 = product.get('price', -1) == 0
-        not_already_set = MODEL_KEY not in product.keys()
 
         logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY,
-                        not_already_set=not_already_set,
                         has_yield_and_price=has_yield_and_price,
                         is_yield_0=is_yield_0,
                         is_price_0=is_price_0)
 
-        should_run = all([
-            not_already_set,
-            any([has_yield_and_price, is_yield_0, is_price_0])
-        ])
+        should_run = any([has_yield_and_price, is_yield_0, is_price_0])
         logShouldRun(cycle, MODEL, term_id, should_run, key=MODEL_KEY)
         return should_run
     return should_run_product
 
 
+def _should_run_product(product: dict): return product.get(MODEL_KEY) is None
+
+
 def run(cycle: dict):
-    products = list(filter(_should_run(cycle), cycle.get('products', [])))
+    products = list(filter(_should_run_product, cycle.get('products', [])))
+    products = list(filter(_should_run(cycle), products))
     return non_empty_list(map(_run(cycle), products))

hestia_earth/models/ecoinventV3/__init__.py

@@ -4,7 +4,7 @@ ecoinvent v3
 This model calculates background emissions related to the production of Inputs from the ecoinvent database, version 3.
 
 Note: to use the `ecoinventV3` model locally or in the
-[Hestia Community Edition](https://gitlab.com/hestia-earth/hestia-community-edition) you need a valid ecoinvent license.
+[HESTIA Community Edition](https://gitlab.com/hestia-earth/hestia-community-edition) you need a valid ecoinvent license.
 Please contact us at community@hestia.earth for instructions to download the required file to run the model.
 
 **Pesticide Brand Name**
@@ -15,16 +15,15 @@ default list of [Pesticide Active Ingredient](https://hestia.earth/glossary?pest
 """
 from functools import reduce
 from hestia_earth.schema import EmissionMethodTier
-from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.tools import flatten, list_sum, non_empty_list
+from hestia_earth.utils.tools import flatten, list_sum
 
-from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
+from hestia_earth.models.log import debugValues, logShouldRun, logRequirements
 from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
-from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.blank_node import group_by_keys
 from hestia_earth.models.utils.pesticideAI import get_pesticides_from_inputs
 from hestia_earth.models.utils.fertiliser import get_fertilisers_from_inputs
+from .utils import get_background_inputs, get_input_mappings
 
 REQUIREMENTS = {
     "Cycle": {
@@ -96,69 +95,43 @@ def _add_emission(cycle: dict, input: dict):
     def add(prev: dict, mapping: tuple):
         ecoinventName, coefficient = mapping
         emissions = ecoinventV3_emissions(ecoinventName)
-        for id, value in emissions:
+        for emission_term_id, value in emissions:
             # log run on each emission so we know it did run
-            debugValues(cycle, model=MODEL, term=id,
+            debugValues(cycle, model=MODEL, term=emission_term_id,
                         value=value,
                         coefficient=coefficient,
                         input=input_term_id,
                         operation=operation_term_id,
                         animal=animal_term_id)
-            prev[id] = prev.get(id, 0) + (value * coefficient)
+            prev[emission_term_id] = prev.get(emission_term_id, 0) + (value * coefficient)
         return prev
     return add
 
 
-def _get_input_mappings(cycle: dict, input: dict):
-    term = input.get('term', {})
-    term_id = term.get('@id')
-    value = get_lookup_value(term, 'ecoinventMapping', model=MODEL, term=term_id)
-    mappings = non_empty_list(value.split(';')) if value else []
-    logRequirements(cycle, model=MODEL, term=term_id,
-                    mappings=';'.join(mappings))
-    return [(m.split(':')[0], float(m.split(':')[1])) for m in mappings]
-
-
 def _run_input(cycle: dict):
     def run(inputs: list):
         input = inputs[0]
+        input_term_id = input.get('term', {}).get('@id')
         input_value = list_sum(flatten(input.get('value', []) for input in inputs))
-        mappings = _get_input_mappings(cycle, input)
-        term_id = input.get('term', {}).get('@id')
-        should_run = len(mappings) > 0
-        logShouldRun(cycle, MODEL, term_id, should_run, methodTier=TIER)
-        grouped_emissions = reduce(_add_emission(cycle, input), mappings, {}) if should_run else {}
-        return [_emission(term_id, value * input_value, input) for term_id, value in grouped_emissions.items()]
-    return run
+        mappings = get_input_mappings(MODEL, cycle, input)
+        has_mappings = len(mappings) > 0
 
+        logRequirements(cycle, model=MODEL, term=input_term_id,
+                        has_ecoinvent_mappings=has_mappings,
+                        ecoinvent_mappings=';'.join([v[0] for v in mappings]))
 
-def _should_run_input(products: list):
-    def should_run(input: dict):
-        return all([
-            list_sum(input.get('value', [])) > 0,
-            # make sure Input is not a Product as well or we might double-count emissions
-            find_term_match(products, input.get('term', {}).get('@id'), None) is None,
-            # ignore inputs which are flagged as Product of the Cycle
-            not input.get('fromCycle', False),
-            not input.get('producedInCycle', False)
-        ])
-    return should_run
-
-
-def _animal_inputs(animal: dict):
-    inputs = animal.get('inputs', [])
-    return [(input | {'animal': animal.get('term', {})}) for input in inputs]
+        should_run = all([has_mappings])
+        logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER)
+        grouped_emissions = reduce(_add_emission(cycle, input), mappings, {}) if should_run else {}
+        return [
+            _emission(term_id, value * input_value, input)
+            for term_id, value in grouped_emissions.items()
+        ]
+    return run
 
 
 def run(_, cycle: dict):
-    # add all the properties of some Term that inlcude others with the mapping
-    inputs = flatten(
-        cycle.get('inputs', []) +
-        list(map(_animal_inputs, cycle.get('animals', []))) +
-        get_pesticides_from_inputs(cycle) +
-        get_fertilisers_from_inputs(cycle)
-    )
-    inputs = list(filter(_should_run_input(cycle.get('products', [])), inputs))
-    # group inputs with same id/operation to avoid adding emissions twice
-    inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
-    return flatten(map(_run_input(cycle), inputs.values()))
+    extra_inputs = get_pesticides_from_inputs(cycle) + get_fertilisers_from_inputs(cycle)
+    inputs = get_background_inputs(cycle, extra_inputs=extra_inputs)
+    grouped_inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
+    return flatten(map(_run_input(cycle), grouped_inputs.values()))

hestia_earth/models/ecoinventV3/utils.py

@@ -0,0 +1,40 @@
+from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.tools import list_sum, flatten, non_empty_list
+
+from hestia_earth.models.utils.term import get_lookup_value
+
+
+def _animal_inputs(animal: dict):
+    inputs = animal.get('inputs', [])
+    return [(input | {'animal': animal.get('term', {})}) for input in inputs]
+
+
+def _should_run_input(products: list):
+    def should_run(input: dict):
+        return all([
+            list_sum(input.get('value', [])) > 0,
+            # make sure Input is not a Product as well or we might double-count emissions
+            find_term_match(products, input.get('term', {}).get('@id'), None) is None,
+            # ignore inputs which are flagged as Product of the Cycle
+            not input.get('fromCycle', False),
+            not input.get('producedInCycle', False)
+        ])
+    return should_run
+
+
+def get_background_inputs(cycle: dict, extra_inputs: list = []):
+    # add all the properties of some Term that inlcude others with the mapping
+    inputs = flatten(
+        cycle.get('inputs', []) +
+        list(map(_animal_inputs, cycle.get('animals', []))) +
+        extra_inputs
+    )
+    return list(filter(_should_run_input(cycle.get('products', [])), inputs))
+
+
+def get_input_mappings(model: str, cycle: dict, input: dict):
+    term = input.get('term', {})
+    term_id = term.get('@id')
+    value = get_lookup_value(term, 'ecoinventMapping', model=model, term=term_id)
+    mappings = non_empty_list(value.split(';')) if value else []
+    return [(m.split(':')[0], float(m.split(':')[1])) for m in mappings]

hestia_earth/models/ecoinventV3AndEmberClimate/__init__.py

@@ -1,25 +1,30 @@
 """
 ecoinvent v3 and Ember Climate
 
-All emissions to air for the cycle.
+This model calculates background emissions related to the production of Inputs from the ecoinvent database, version 3,
+and using the percentage of production for each energy source from
+[Ember](https://ember-climate.org/data-catalogue/yearly-electricity-data/).
+
+Note: to use the `ecoinventV3AndEmberClimate` model locally or in the
+[Hestia Community Edition](https://gitlab.com/hestia-earth/hestia-community-edition) you need a valid ecoinvent license.
+Please contact us at community@hestia.earth for instructions to download the required file to run the model.
 """
 from functools import reduce
-from typing import Tuple
-
 from hestia_earth.utils.tools import list_sum, flatten
-from hestia_earth.utils.model import linked_node
 from hestia_earth.schema import EmissionMethodTier
 
-from hestia_earth.models.log import logShouldRun, logRequirements, log_blank_nodes_id
+from hestia_earth.models.log import logShouldRun, logRequirements, debugValues
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.blank_node import group_by_keys
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.term import get_electricity_grid_mix_terms
-from hestia_earth.models.utils.cycle import cycle_end_year
-from .utils import get_emission, get_all_emission_terms
+from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
+from .utils import get_input_coefficient
+from ..ecoinventV3.utils import get_background_inputs, get_input_mappings
 
 REQUIREMENTS = {
     "Cycle": {
+        "completeness.electricityFuel": "True",
         "site": {
             "@type": "Site",
             "country": {"@type": "Term", "termType": "region"}
@@ -27,9 +32,26 @@ REQUIREMENTS = {
         "inputs": [{
             "@type": "Input",
             "term.@id": ["electricityGridMarketMix", "electricityGridRenewableMix"],
-            "value": ""
+            "value": "> 0",
+            "none": {
+                "fromCycle": "True",
+                "producedInCycle": "True"
+            }
         }],
-        "completeness.electricityFuel": "True"
+        "optional": {
+            "animals": [{
+                "@type": "Animal",
+                "inputs": [{
+                    "@type": "Input",
+                    "term.@id": ["electricityGridMarketMix", "electricityGridRenewableMix"],
+                    "value": "> 0",
+                    "none": {
+                        "fromCycle": "True",
+                        "producedInCycle": "True"
+                    }
+                }]
+            }]
+        }
     }
 }
 RETURNS = {
@@ -41,6 +63,7 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
+    "electricity": "ecoinventMapping",
     "region-ember-energySources": "using `country`",
     "ember-ecoinvent-mapping": ["ember", "ecoinventId", "ecoinventName"]
 }
@@ -50,94 +73,72 @@ MODEL_KEY = 'impactAssessment'  # keep to generate entry in "model-links.json"
 TIER = EmissionMethodTier.BACKGROUND.value
 
 
-def _emission(value: float, term_id: str, inputs: list, operation: dict) -> dict:
+def _emission(term_id: str, value: float, input: dict):
     emission = _new_emission(term_id, MODEL)
     emission['value'] = [value]
     emission['methodTier'] = TIER
-    emission['inputs'] = list(map(linked_node, inputs))
-    if operation:
-        emission["operation"] = operation
+    emission['inputs'] = [input.get('term')]
+    if input.get('operation'):
+        emission['operation'] = input.get('operation')
+    if input.get('animal'):
+        emission['animals'] = [input.get('animal')]
     return emission
 
 
-def _grid_inputs(inputs: list, electricity_grid_terms: list):
-    electricity_grid_term_ids = [v.get('@id') for v in electricity_grid_terms]
-    return [
-        i for i in inputs if i.get("term", {}).get("@id") in electricity_grid_term_ids
-    ]
-
-
-def _run_input(cycle: dict, inputs: list, emission_term_id: str, electricity_grid_term: dict):
-    electricity_grid_inputs = [electricity_grid_term]
-    inputs = _grid_inputs(inputs, electricity_grid_inputs)
-
-    def run(grouped_inputs: list):
-        input = grouped_inputs[0]
-        term_id = input.get('term', {}).get('@id')
-        input_value = list_sum(flatten(input.get('value', []) for input in grouped_inputs))
-        country_id = cycle.get("site", {}).get("country", {}).get("@id")
-        year = cycle_end_year(cycle)
-        value = get_emission(
-            term_id=emission_term_id,
-            country=country_id,
-            energy=input_value,
-            year=year,
-            model=MODEL
-        )
-
-        logRequirements(cycle, model=MODEL, term=term_id,
-                        input_value=input_value,
-                        emission_value=value)
-
-        should_run = all([input_value > 0, value])
-        logShouldRun(cycle, MODEL, term_id, should_run, methodTier=TIER)
-
-        return _emission(value, emission_term_id, electricity_grid_inputs, input.get("operation"))
-
-    return list(map(run, _group_by_operation(inputs).values() if inputs else []))
-
-
-def _group_by_operation(inputs: list) -> dict:
-    return reduce(group_by_keys(['operation']), inputs, {})
-
-
-def _run_emission(cycle: dict, electricity_grid_terms: list, inputs: list, emission_term_id: str) -> list:
-    return flatten([
-        _run_input(
-            cycle=cycle,
-            inputs=inputs,
-            emission_term_id=emission_term_id,
-            electricity_grid_term=electricity_grid_term
-        ) for electricity_grid_term in electricity_grid_terms
-    ])
-
-
-def _should_run_emission(cycle: dict, electricity_grid_terms: list, term_id: str) -> Tuple[bool, list]:
-    term_type_complete = _is_term_type_complete(cycle, 'electricityFuel')
-    inputs = _grid_inputs(cycle.get('inputs', []), electricity_grid_terms)
-    has_relevant_inputs = bool(inputs)
-    has_country = bool(cycle.get("site", {}).get("country", {}))
-    has_end_date = bool(cycle.get("endDate"))
-
-    logRequirements(cycle, model=MODEL, term=term_id,
-                    term_type_electricityFuel_complete=term_type_complete,
-                    input_ids=log_blank_nodes_id(inputs),
-                    has_relevant_inputs=has_relevant_inputs,
-                    has_country=has_country,
-                    has_end_date=has_end_date)
-
-    should_run = all([term_type_complete, has_relevant_inputs, has_country, has_end_date])
-    logShouldRun(cycle, MODEL, term_id, should_run, methodTier=TIER)
-    return should_run, inputs
-
-
-def _run_emissions(cycle: dict, electricity_grid_terms: list):
-    def run_emissions_for_term(term_id: str) -> list:
-        should_run, inputs = _should_run_emission(cycle, electricity_grid_terms, term_id)
-        return _run_emission(cycle, electricity_grid_terms, inputs, term_id) if should_run else []
-    return run_emissions_for_term
+def _add_emission(cycle: dict, input: dict):
+    input_term_id = input.get('term', {}).get('@id')
+    operation_term_id = input.get('operation', {}).get('@id')
+    animal_term_id = input.get('animal', {}).get('@id')
+    # TODO: as inputs are grouped, there could be more than 1 country
+    country_id = input.get('country', cycle.get('site', {}).get('country', {})).get('@id')
+
+    def add(prev: dict, mapping: tuple):
+        ecoinventName, _coefficient = mapping
+        # recalculate the coefficient using the country and year if it should be included
+        coefficient = get_input_coefficient(MODEL, cycle, country_id, ecoinventName) if _coefficient > 0 else 0
+        emissions = ecoinventV3_emissions(ecoinventName)
+        for emission_term_id, value in emissions:
+            # log run on each emission so we know it did run
+            debugValues(cycle, model=MODEL, term=emission_term_id,
+                        value=value,
+                        coefficient=coefficient,
+                        input=input_term_id,
+                        operation=operation_term_id,
+                        animal=animal_term_id)
+            prev[emission_term_id] = prev.get(emission_term_id, 0) + (value * coefficient)
+        return prev
+    return add
+
+
+def _run_input(cycle: dict):
+    electricity_complete = _is_term_type_complete(cycle, 'electricityFuel')
+
+    def run(inputs: list):
+        input = inputs[0]
+        input_value = list_sum(flatten(input.get('value', []) for input in inputs))
+        input_term_id = input.get('term', {}).get('@id')
+        mappings = get_input_mappings(MODEL, cycle, input)
+        has_mappings = len(mappings) > 0
+
+        logRequirements(cycle, model=MODEL, term=input_term_id,
+                        has_ecoinvent_mappings=has_mappings,
+                        ecoinvent_mappings=';'.join([v[0] for v in mappings]))
+
+        should_run = all([electricity_complete, has_mappings])
+        logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER)
+
+        grouped_emissions = reduce(_add_emission(cycle, input), mappings, {}) if should_run else {}
+        return [
+            _emission(term_id, value * input_value, input)
+            for term_id, value in grouped_emissions.items()
+        ]
+    return run
 
 
 def run(_, cycle: dict):
-    electricity_grid_terms = get_electricity_grid_mix_terms()
-    return flatten(list(map(_run_emissions(cycle, electricity_grid_terms), get_all_emission_terms())))
+    terms = get_electricity_grid_mix_terms()
+    inputs = get_background_inputs(cycle)
+    # only keep the inputs matching the grid terms
+    inputs = [i for i in inputs if i.get('term', {}).get('@id') in terms]
+    grouped_inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
+    return flatten(map(_run_input(cycle), grouped_inputs.values()))

hestia_earth/models/ecoinventV3AndEmberClimate/utils.py

@@ -1,116 +1,26 @@
-from typing import Dict, Any, Union, List
-
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data
-from hestia_earth.utils.tools import safe_parse_float, non_empty_list
+from hestia_earth.utils.tools import safe_parse_float
 
 from hestia_earth.models.log import debugMissingLookup
-from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
-
+from hestia_earth.models.utils.cycle import cycle_end_year
 
 EMBER_ECOINVENT_LOOKUP_NAME = "ember-ecoinvent-mapping.csv"
+REGION_EMBER_SOURCES_LOOKUP_NAME = "region-ember-energySources.csv"
 
 
-def _lookup_data(term_id: str, grouping: str, country_id: str, year: str, lookup_name: str, **log_args):
-    lookup = download_lookup(lookup_name)
-    column = column_name(grouping)
-    data = get_table_value(lookup, 'termid', country_id, column)
-    percentage = extract_grouped_data(data, year)
-    debugMissingLookup(lookup_name, 'termid', country_id, column, percentage, year=year, term=term_id, **log_args)
-    return safe_parse_float(percentage, None)
-
-
-def _convert_name(name: str) -> str: return name.replace(";", ",")
-
-
-def _zero_from_non_numeric(value: Any) -> float:
-    try:
-        return float(value)
-    except (ValueError, TypeError):
-        return 0.0
-
-
-def _extract_emission_value(value_iter: Any) -> Union[float, None]:
-    value_list = list(value_iter)
-    try:
-        if len(list(value_list)) > 0 and len(list(value_list)[0]) > 1:
-            return safe_parse_float(list(value_list)[0][1], None)
-    except ValueError:
-        return None
-
-
-def _get_emission_rate_per_source(ember_ecoinvent_mapping: Dict, emission_term_id: str) -> Dict:
-    """
-    Returns the emissions rate in kg/kWh as a mapping indexed by ember energy source name.
-    eg: {"bioenergy": 0.8947372128046288, "coal": 0.000124581084822, ... }
-    """
+def get_input_coefficient(model: str, cycle: dict, country_id: str, ecoinventName: str):
+    year = cycle_end_year(cycle)
 
-    return {
-        ember_source: _extract_emission_value(filter(
-            lambda x: x[0] == emission_term_id,
-            ecoinventV3_emissions(ecoinvent_lookup["ecoinventname"])
-        ))
-        for ember_source, ecoinvent_lookup in ember_ecoinvent_mapping.items()
-    }
-
-
-def get_emission(term_id: str, country: str, year: int, energy: float, model: str) -> float:
-    """
-    Get the <term_id> emissions in kg for the energy consumed.
-        a: ecoInventId of each source        - from "ember-ecoinvent-mapping.csv"
-        b: Ember sources list                - from b
-        c: Percentages per source            - from region-ember-energySources.csv (country, ember-source, year:value)
-        d: Energy per source (kWh)           - from energy * each of c
-        e: Emissions/kWh per source          - from ecoinvent (using id from b)
-        f: Emissions per source (kg)         - from e * d for each source
-        g: Total emissions                   - from sum(f)
-    """
-    # a: ecoInventId of each source
+    # find the matching ember source with the ecoinventName.
+    # example: "electricity, high voltage, electricity production, hard coal" > "Coal"
     ember_ecoinvent_lookup = download_lookup(EMBER_ECOINVENT_LOOKUP_NAME)
-    ember_ecoinvent_mapping = {
-        row["ember"].lower(): {"ecoinventid": row["ecoinventid"], "ecoinventname": _convert_name(row["ecoinventname"])}
-        for row in ember_ecoinvent_lookup
-    }
-
-    # b: Ember sources
-    ember_sources = list(ember_ecoinvent_mapping.keys())
-
-    # c: Percentages per source
-    percentages_per_source = {
-        source: _lookup_data(
-            term_id=source,
-            grouping=source,
-            country_id=country,
-            year=str(year),
-            lookup_name="region-ember-energySources.csv",
-            model=model
-        )
-        for source in ember_sources
-    }
-
-    # d: Energy per source (kWh)
-    energy_per_source = {
-        source: energy * _zero_from_non_numeric(percentage) / 100
-        for source, percentage in percentages_per_source.items()
-    }
+    source_name = get_table_value(ember_ecoinvent_lookup, column_name('ecoinventName'), ecoinventName, 'ember')
 
-    # e: Emissions/kWh per source
-    emission_rate_per_source = _get_emission_rate_per_source(
-        ember_ecoinvent_mapping=ember_ecoinvent_mapping,
-        emission_term_id=term_id
-    )
+    # find the ratio for the country / year
+    region_ember_sources_lookup = download_lookup(REGION_EMBER_SOURCES_LOOKUP_NAME)
+    data = get_table_value(region_ember_sources_lookup, 'termid', country_id, column_name(source_name))
+    percentage = extract_grouped_data(data, str(year))
+    debugMissingLookup(REGION_EMBER_SOURCES_LOOKUP_NAME,
+                       'termid', country_id, source_name, percentage, year=year, model=model)
 
-    # f: Emissions per source (kg)
-    emissions_per_source = {
-        source: energy_per_source[source] * emission_rate_per_source[source]
-        for source in ember_sources
-        if energy_per_source[source] is not None and emission_rate_per_source[source] is not None
-    }
-
-    # g: Total emissions (kg of <term_id>)
-    return sum(emissions_per_source.values())
-
-
-def get_all_emission_terms() -> List[str]:
-    ember_ecoinvent_lookup = download_lookup(EMBER_ECOINVENT_LOOKUP_NAME)
-    eco_invent_name = _convert_name(ember_ecoinvent_lookup[0]["ecoinventname"])
-    return [e[0] for e in non_empty_list(ecoinventV3_emissions(eco_invent_name))]
+    return safe_parse_float(percentage, 0) / 100

hestia_earth/models/faostat2018/product/price.py

@@ -157,8 +157,7 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     return _product(product, value / 1000 * conversion_to_number) if value is not None else None
 
 
-def _should_run_product(product: dict):
-    return product.get(MODEL_KEY) is None
+def _should_run_product(product: dict): return product.get(MODEL_KEY) is None
 
 
 def run(cycle: dict):

hestia_earth/models/geospatialDatabase/croppingIntensity.py

@@ -37,7 +37,8 @@ TERM_ID = 'croppingIntensity'
 
 def _practice(value: float):
     practice = _new_practice(TERM_ID)
-    practice['value'] = [round(value, 7)]
+    # force conversion to float from numpy, avoiding errors when reading it in other models
+    practice['value'] = [float(round(value, 7))]
     return practice
 
 

hestia_earth/models/geospatialDatabase/utils.py

@@ -152,7 +152,7 @@ def _get_cached_data(term: str, site: dict, data: dict):
 
 def download(term: str, site: dict, data: dict, only_coordinates=False) -> dict:
     """
-    Downloads data from Hestia Earth Engine API.
+    Downloads data from HESTIA Earth Engine API.
 
     Returns
     -------