hestia-earth-models 0.62.1__py3-none-any.whl → 0.62.3__py3-none-any.whl

hestia_earth/models/ecoinventV3AndEmberClimate/utils.py

@@ -1,116 +1,26 @@
-from typing import Dict, Any, Union, List
-
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data
-from hestia_earth.utils.tools import safe_parse_float, non_empty_list
+from hestia_earth.utils.tools import safe_parse_float
 
 from hestia_earth.models.log import debugMissingLookup
-from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
-
+from hestia_earth.models.utils.cycle import cycle_end_year
 
 EMBER_ECOINVENT_LOOKUP_NAME = "ember-ecoinvent-mapping.csv"
+REGION_EMBER_SOURCES_LOOKUP_NAME = "region-ember-energySources.csv"
 
 
-def _lookup_data(term_id: str, grouping: str, country_id: str, year: str, lookup_name: str, **log_args):
-    lookup = download_lookup(lookup_name)
-    column = column_name(grouping)
-    data = get_table_value(lookup, 'termid', country_id, column)
-    percentage = extract_grouped_data(data, year)
-    debugMissingLookup(lookup_name, 'termid', country_id, column, percentage, year=year, term=term_id, **log_args)
-    return safe_parse_float(percentage, None)
-
-
-def _convert_name(name: str) -> str: return name.replace(";", ",")
-
-
-def _zero_from_non_numeric(value: Any) -> float:
-    try:
-        return float(value)
-    except (ValueError, TypeError):
-        return 0.0
-
-
-def _extract_emission_value(value_iter: Any) -> Union[float, None]:
-    value_list = list(value_iter)
-    try:
-        if len(list(value_list)) > 0 and len(list(value_list)[0]) > 1:
-            return safe_parse_float(list(value_list)[0][1], None)
-    except ValueError:
-        return None
-
-
-def _get_emission_rate_per_source(ember_ecoinvent_mapping: Dict, emission_term_id: str) -> Dict:
-    """
-    Returns the emissions rate in kg/kWh as a mapping indexed by ember energy source name.
-    eg: {"bioenergy": 0.8947372128046288, "coal": 0.000124581084822, ... }
-    """
+def get_input_coefficient(model: str, cycle: dict, country_id: str, ecoinventName: str):
+    year = cycle_end_year(cycle)
 
-    return {
-        ember_source: _extract_emission_value(filter(
-            lambda x: x[0] == emission_term_id,
-            ecoinventV3_emissions(ecoinvent_lookup["ecoinventname"])
-        ))
-        for ember_source, ecoinvent_lookup in ember_ecoinvent_mapping.items()
-    }
-
-
-def get_emission(term_id: str, country: str, year: int, energy: float, model: str) -> float:
-    """
-    Get the <term_id> emissions in kg for the energy consumed.
-        a: ecoInventId of each source        - from "ember-ecoinvent-mapping.csv"
-        b: Ember sources list                - from b
-        c: Percentages per source            - from region-ember-energySources.csv (country, ember-source, year:value)
-        d: Energy per source (kWh)           - from energy * each of c
-        e: Emissions/kWh per source          - from ecoinvent (using id from b)
-        f: Emissions per source (kg)         - from e * d for each source
-        g: Total emissions                   - from sum(f)
-    """
-    # a: ecoInventId of each source
+    # find the matching ember source with the ecoinventName.
+    # example: "electricity, high voltage, electricity production, hard coal" > "Coal"
     ember_ecoinvent_lookup = download_lookup(EMBER_ECOINVENT_LOOKUP_NAME)
-    ember_ecoinvent_mapping = {
-        row["ember"].lower(): {"ecoinventid": row["ecoinventid"], "ecoinventname": _convert_name(row["ecoinventname"])}
-        for row in ember_ecoinvent_lookup
-    }
-
-    # b: Ember sources
-    ember_sources = list(ember_ecoinvent_mapping.keys())
-
-    # c: Percentages per source
-    percentages_per_source = {
-        source: _lookup_data(
-            term_id=source,
-            grouping=source,
-            country_id=country,
-            year=str(year),
-            lookup_name="region-ember-energySources.csv",
-            model=model
-        )
-        for source in ember_sources
-    }
-
-    # d: Energy per source (kWh)
-    energy_per_source = {
-        source: energy * _zero_from_non_numeric(percentage) / 100
-        for source, percentage in percentages_per_source.items()
-    }
+    source_name = get_table_value(ember_ecoinvent_lookup, column_name('ecoinventName'), ecoinventName, 'ember')
 
-    # e: Emissions/kWh per source
-    emission_rate_per_source = _get_emission_rate_per_source(
-        ember_ecoinvent_mapping=ember_ecoinvent_mapping,
-        emission_term_id=term_id
-    )
+    # find the ratio for the country / year
+    region_ember_sources_lookup = download_lookup(REGION_EMBER_SOURCES_LOOKUP_NAME)
+    data = get_table_value(region_ember_sources_lookup, 'termid', country_id, column_name(source_name))
+    percentage = extract_grouped_data(data, str(year))
+    debugMissingLookup(REGION_EMBER_SOURCES_LOOKUP_NAME,
+                       'termid', country_id, source_name, percentage, year=year, model=model)
 
-    # f: Emissions per source (kg)
-    emissions_per_source = {
-        source: energy_per_source[source] * emission_rate_per_source[source]
-        for source in ember_sources
-        if energy_per_source[source] is not None and emission_rate_per_source[source] is not None
-    }
-
-    # g: Total emissions (kg of <term_id>)
-    return sum(emissions_per_source.values())
-
-
-def get_all_emission_terms() -> List[str]:
-    ember_ecoinvent_lookup = download_lookup(EMBER_ECOINVENT_LOOKUP_NAME)
-    eco_invent_name = _convert_name(ember_ecoinvent_lookup[0]["ecoinventname"])
-    return [e[0] for e in non_empty_list(ecoinventV3_emissions(eco_invent_name))]
+    return safe_parse_float(percentage, 0) / 100

hestia_earth/models/faostat2018/product/price.py

@@ -157,8 +157,7 @@ def _run_by_country(cycle: dict, product: dict, country_id: str, year: int = Non
     return _product(product, value / 1000 * conversion_to_number) if value is not None else None
 
 
-def _should_run_product(product: dict):
-    return product.get(MODEL_KEY) is None
+def _should_run_product(product: dict): return product.get(MODEL_KEY) is None
 
 
 def run(cycle: dict):

hestia_earth/models/geospatialDatabase/croppingIntensity.py

@@ -37,7 +37,8 @@ TERM_ID = 'croppingIntensity'
 
 def _practice(value: float):
     practice = _new_practice(TERM_ID)
-    practice['value'] = [round(value, 7)]
+    # force conversion to float from numpy, avoiding errors when reading it in other models
+    practice['value'] = [float(round(value, 7))]
     return practice
 
 

hestia_earth/models/geospatialDatabase/utils.py

@@ -152,7 +152,7 @@ def _get_cached_data(term: str, site: dict, data: dict):
 
 def download(term: str, site: dict, data: dict, only_coordinates=False) -> dict:
     """
-    Downloads data from Hestia Earth Engine API.
+    Downloads data from HESTIA Earth Engine API.
 
     Returns
     -------

hestia_earth/models/ipcc2019/aboveGroundCropResidueTotal.py

@@ -2,7 +2,7 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum
 
-from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
+from hestia_earth.models.log import debugValues, logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.completeness import _is_term_type_incomplete
 from hestia_earth.models.utils.product import _new_product
 from hestia_earth.models.utils.property import get_node_property
@@ -46,16 +46,21 @@ def _product_value(product: dict):
     value = list_sum(product.get('value'))
     dm = get_node_property(product, PROPERTY_KEY).get('value', 0)
     yield_dm = get_yield_dm(TERM_ID, term) or 0
-    debugValues(product, model=MODEL, term=TERM_ID,
-                product=term_id,
-                value=value,
-                dryMatter=dm,
-                ratio_yield_dm=yield_dm)
-    return value * dm / 100 * yield_dm
+    total = value * dm / 100 * yield_dm
+    return {
+        'id': term_id,
+        'value': value,
+        'dryMatter': dm,
+        'RatioYieldDM': yield_dm,
+        'total': total
+    }
 
 
-def _run(products: list):
-    value = sum(map(_product_value, products))
+def _run(cycle: dict, products: list):
+    values = list(map(_product_value, products))
+    debugValues(cycle, model=MODEL, term=TERM_ID,
+                details=log_as_table(values))
+    value = sum([value.get('total', 0) for value in values])
     return [_product(value)]
 
 
@@ -85,4 +90,4 @@ def _should_run(cycle: dict):
 
 def run(cycle: dict):
     should_run, products = _should_run(cycle)
-    return _run(products) if should_run else []
+    return _run(cycle, products) if should_run else []

hestia_earth/models/ipcc2019/animal/pastureGrass.py

@@ -12,7 +12,8 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum
 
-from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues, log_as_table
+from hestia_earth.models.utils.blank_node import lookups_logs, properties_logs
 from hestia_earth.models.utils.input import _new_input
 from hestia_earth.models.utils.term import get_wool_terms, get_lookup_value
 from hestia_earth.models.utils.completeness import _is_term_type_complete, _is_term_type_incomplete
@@ -26,6 +27,7 @@ from ..pastureGrass_utils import (
     calculate_REM,
     calculate_REG,
     calculate_NEfeed,
+    calculate_GE,
     product_wool_energy,
     get_animals,
     get_animal_values
@@ -115,7 +117,8 @@ REQUIREMENTS = {
 }
 LOOKUPS = {
     "animalManagement": [
-        "mjKgEvMilkIpcc2019"
+        "mjKgEvMilkIpcc2019",
+        "defaultFatContentEvMilkIpcc2019"
     ],
     "animalProduct": ["mjKgEvWoolNetEnergyWoolIpcc2019", "allowedLiveAnimalTermIds"],
     "liveAnimal": [
@@ -184,38 +187,50 @@ def calculate_NEwool(cycle: dict, animal: dict, products: list, total_weight: fl
     return total_energy * animal_weight/total_weight
 
 
-def _calculate_GE(
-    cycle: dict, animal: dict, REM: float, REG: float, NEwool: float, system: dict
-) -> float:
-    term_id = animal.get('term', {}).get('@id')
-
-    NEm, NEa, NEl, NEwork, NEp, NEg = get_animal_values(cycle, animal, system, log_node=animal)
-
-    NEm_feed, NEg_feed = calculate_NEfeed(animal)
-    debugValues(animal, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                NEm=NEm,
-                NEa=NEa,
-                NEl=NEl,
-                NEwork=NEwork,
-                NEp=NEp,
-                NEg=NEg,
-                NEm_feed=NEm_feed,
-                NEg_feed=NEg_feed)
-
-    return (NEm + NEa + NEl + NEwork + NEp - NEm_feed)/REM + (NEg + NEwool - NEg_feed)/REG
-
-
-def _run_practice(animal: dict, GE: float, meanECHHV: float):
+def _run_practice(
+    animal: dict, values: dict, meanDE: float, meanECHHV: float, REM: float, REG: float,
+    NEwool: float, NEm_feed: float, NEg_feed: float
+):
     def run(practice: dict):
         key = practice.get('key', {})
         key_id = key.get('@id')
         input_term_id = practice_input_id(practice)
+
+        GE = (
+            calculate_GE([values], REM, REG, NEwool, NEm_feed, NEg_feed) / (meanDE/100)
+        ) if meanDE else 0
+
         value = (GE / meanECHHV) * (list_sum(practice.get('value', [0])) / 100)
 
+        logs = log_as_table(values | {
+            'animalId': animal.get('term', {}).get('@id'),
+            'practiceKeyId': key_id,
+            'GE': GE,
+            'NEmFeed': NEm_feed,
+            'NEgFeed': NEg_feed,
+            'REM': REM,
+            'REG': REG,
+            'NEwool': NEwool,
+            'meanECHHV': meanECHHV,
+            'meanDE': meanDE
+        })
+        animal_lookups = lookups_logs(MODEL, [animal], LOOKUPS, model_key=MODEL_KEY, term=input_term_id)
+        animal_properties = properties_logs([animal], properties=[
+            'liveweightPerHead',
+            'hoursWorkedPerDay',
+            'animalsPerBirth',
+            'pregnancyRateTotal',
+            'weightAtMaturity',
+            'liveweightGain',
+            'weightAtWeaning',
+            'weightAtOneYear',
+            'weightAtSlaughter'
+        ])
+
         logRequirements(animal, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
-                        GE=GE,
-                        meanECHHV=meanECHHV,
-                        practice_key_id=key_id)
+                        animal_logs=logs,
+                        animal_lookups=animal_lookups,
+                        animal_properties=animal_properties)
 
         logShouldRun(animal, MODEL, input_term_id, True, model_key=MODEL_KEY)
 
@@ -235,21 +250,16 @@ def _run_animal(cycle: dict, meanDE: float, meanECHHV: float, system: dict, prac
     total_liveWeightPerHead = _sum_liveWeightPerHead(animals)
 
     def run(animal: dict):
-        term_id = animal.get('term', {}).get('@id')
-
         NEwool = calculate_NEwool(cycle, animal, wool_products, total_liveWeightPerHead) if (
             total_liveWeightPerHead > 0
         ) else 0
-        GE = (_calculate_GE(cycle, animal, REM, REG, NEwool, system) / (meanDE/100)) if all([REM, REG]) else 0
-
-        debugValues(animal, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                    REM=REM,
-                    REG=REG,
-                    NEwool=NEwool,
-                    GE=GE,
-                    meanDE=meanDE)
+        NEm_feed, NEg_feed = calculate_NEfeed(animal)
+        animal_values = get_animal_values(cycle, animal, system)
 
-        inputs = list(map(_run_practice(animal, GE, meanECHHV), practices))
+        inputs = list(map(
+            _run_practice(animal, animal_values, meanDE, meanECHHV, REM, REG, NEwool, NEm_feed, NEg_feed),
+            practices
+        ))
         return animal | {
             'inputs': animal.get('inputs', []) + inputs
         }
@@ -289,8 +299,8 @@ def _should_run(cycle: dict, animals: list, practices: dict):
                             term_type_freshForage_incomplete=freshForage_incomplete,
                             no_cycle_inputs_feed=no_cycle_inputs_feed,
                             all_animals_have_value=all_animals_have_value,
-                            meanDE=meanDE,
-                            meanECHHV=meanECHHV)
+                            meanDE=calculate_meanDE(practices, term=term_id),
+                            meanECHHV=calculate_meanECHHV(practices, term=term_id))
 
             logShouldRun(animal, MODEL, term_id, should_run, model_key=MODEL_KEY)
 

hestia_earth/models/ipcc2019/belowGroundCropResidue.py

@@ -2,7 +2,7 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum, safe_parse_float
 
-from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
+from hestia_earth.models.log import debugValues, logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.completeness import _is_term_type_incomplete
 from hestia_earth.models.utils.product import _new_product
 from hestia_earth.models.utils.property import get_node_property
@@ -52,17 +52,22 @@ def _product_value(product: dict):
     dm = get_node_property(product, PROPERTY_KEY).get('value', 0)
     yield_dm = get_yield_dm(TERM_ID, term) or 0
     ratio = _get_lookup_value(term, 'IPCC_2019_Ratio_BGRes_AGRes') or 0
-    debugValues(product, model=MODEL, term=TERM_ID,
-                product=term_id,
-                value=value,
-                dryMatter=dm,
-                ratio_yield_dm=yield_dm,
-                ratio=ratio)
-    return value * dm / 100 * yield_dm * ratio
+    total = value * dm / 100 * yield_dm * ratio
+    return {
+        'id': term_id,
+        'value': value,
+        'dryMatter': dm,
+        'RatioYieldDM': yield_dm,
+        'RatioAboveGroundToBelowGround': ratio,
+        'total': total
+    }
 
 
-def _run(products: list):
-    value = sum(map(_product_value, products))
+def _run(cycle: dict, products: list):
+    values = list(map(_product_value, products))
+    debugValues(cycle, model=MODEL, term=TERM_ID,
+                details=log_as_table(values))
+    value = sum([value.get('total', 0) for value in values])
     return [_product(value)]
 
 
@@ -92,4 +97,4 @@ def _should_run(cycle: dict):
 
 def run(cycle: dict):
     should_run, products = _should_run(cycle)
-    return _run(products) if should_run else []
+    return _run(cycle, products) if should_run else []

hestia_earth/models/ipcc2019/carbonContent.py

@@ -114,7 +114,7 @@ def _multiple_product_values(crop: dict, residue_id: str):
     # LIGNIN_CONTENT_AG_CROP_RESIDUE or LIGNIN_CONTENT_BG_CROP_RESIDUE
     c_content = _get_lookup_value(term, _crop_residue_lookup_col(residue_id))
     ratio = _get_lookup_value(term, 'IPCC_2019_Ratio_BGRes_AGRes') if residue_id == 'belowGroundCropResidue' else 1
-    debugValues(crop, model=MODEL, term=residue_id,
+    debugValues(crop, model=MODEL, term=residue_id, property=TERM_ID,
                 crop=term_id,
                 dryMatter=dm,
                 ratio_yield_dm=yield_dm,

hestia_earth/models/ipcc2019/croppingDuration.py

@@ -80,11 +80,11 @@ def _should_run(cycle: dict):
     logRequirements(cycle, model=MODEL, term=TERM_ID,
                     country=country,
                     has_flooded_rice=flooded_rice,
-                    cycleDuration=cycleDuration,
+                    cycleDuration=cycle.get('cycleDuration'),
                     croppingDuration=croppingDuration,
                     croppingDuration_below_cycleDuration=croppingDuration_below_cycleDuration)
 
-    should_run = all([country, cycleDuration, flooded_rice])
+    should_run = all([country, cycleDuration > 0, croppingDuration_below_cycleDuration, flooded_rice])
     logShouldRun(cycle, MODEL, TERM_ID, should_run)
     return should_run, country
 

hestia_earth/models/ipcc2019/ligninContent.py

@@ -114,7 +114,7 @@ def _multiple_product_values(crop: dict, residue_id: str):
     # LIGNIN_CONTENT_AG_CROP_RESIDUE or LIGNIN_CONTENT_BG_CROP_RESIDUE
     l_content = _get_lookup_value(term, _crop_residue_lookup_col(residue_id))
     ratio = _get_lookup_value(term, 'IPCC_2019_Ratio_BGRes_AGRes') if residue_id == 'belowGroundCropResidue' else 1
-    debugValues(crop, model=MODEL, term=residue_id,
+    debugValues(crop, model=MODEL, term=residue_id, property=TERM_ID,
                 crop=term_id,
                 dryMatter=dm,
                 ratio_yield_dm=yield_dm,

hestia_earth/models/ipcc2019/nitrogenContent.py

@@ -114,7 +114,7 @@ def _multiple_product_value(crop: dict, residue_id: str):
     # N_Content_AG_Residue or N_Content_BG_Residue
     n_content = _get_lookup_value(term, _crop_residue_lookup_col(residue_id))
     ratio = _get_lookup_value(term, 'IPCC_2019_Ratio_BGRes_AGRes') if residue_id == 'belowGroundCropResidue' else 1
-    debugValues(crop, model=MODEL, term=residue_id,
+    debugValues(crop, model=MODEL, term=residue_id, property=TERM_ID,
                 crop=term_id,
                 dryMatter=dm,
                 ratio_yield_dm=yield_dm,

hestia_earth/models/ipcc2019/organicCarbonPerHa.py

@@ -8,7 +8,7 @@ Both tier methodologies are run as Monte Carlo simulations with 10000 iterations
 
 The requirements in this file are for the Tier 1 methodology only, as it has simpler requirements. The requirements for
 the Tier 2 methodology can be found in the
-[Hestia SOC wiki](https://gitlab.com/hestia-earth/hestia-engine-models/-/wikis/Soil-organic-carbon-modelling)
+[HESTIA SOC wiki](https://gitlab.com/hestia-earth/hestia-engine-models/-/wikis/Soil-organic-carbon-modelling)
 alongside data recommendations, examples and explanations for both tiers.
 """
 from functools import reduce
@@ -130,12 +130,12 @@ def run(site: dict) -> list[dict]:
     Parameters
     ----------
     site : dict
-        A Hestia `Site` node, see: https://www.hestia.earth/schema/Site.
+        A HESTIA `Site` node, see: https://www.hestia.earth/schema/Site.
 
     Returns
     -------
     list[dict]
-        A list of Hestia `Measurement` nodes containing the calculated SOC stocks and additional relevant data.
+        A list of HESTIA `Measurement` nodes containing the calculated SOC stocks and additional relevant data.
     """
     should_run, run_data = _should_run(site)
     _log_data(site, should_run, run_data)

hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_1_utils.py

@@ -5,7 +5,7 @@ management changes.
 The model cannot not run on Sites with polar moist (ecoClimateZone 5) or polar dry (ecoClimateZone 6).
 
 More information on this model, including data requirements **and** recommendations, and examples can be found in the
-[Hestia SOC wiki](https://gitlab.com/hestia-earth/hestia-engine-models/-/wikis/Soil-organic-carbon-modelling).
+[HESTIA SOC wiki](https://gitlab.com/hestia-earth/hestia-engine-models/-/wikis/Soil-organic-carbon-modelling).
 
 Source: [IPCC 2019, Vol. 4, Chapter 2](https://www.ipcc-nggip.iges.or.jp/public/2019rf/vol4.html).
 """
@@ -92,7 +92,7 @@ def _measurement(
     descriptive_stats_dict: dict
 ) -> dict:
     """
-    Build a Hestia `Measurement` node to contain a value and descriptive statistics calculated by the models.
+    Build a HESTIA `Measurement` node to contain a value and descriptive statistics calculated by the models.
 
     The `descriptive_stats_dict` parameter should include the following keys and values from the
     [Measurement](https://www-staging.hestia.earth/schema/Measurement) schema:
@@ -117,7 +117,7 @@ def _measurement(
     Returns
     -------
     dict
-        A valid Hestia `Measurement` node, see: https://www.hestia.earth/schema/Measurement.
+        A valid HESTIA `Measurement` node, see: https://www.hestia.earth/schema/Measurement.
     """
     measurement = _new_measurement(_TERM_ID) | descriptive_stats_dict
     measurement["dates"] = [f"{year}-12-31" for year in timestamps]
@@ -640,7 +640,7 @@ def should_run(site: dict) -> tuple[bool, dict, dict]:
     Parameters
     ----------
     site : dict
-        A Hestia `Site` node, see: https://www.hestia.earth/schema/Site.
+        A HESTIA `Site` node, see: https://www.hestia.earth/schema/Site.
 
     Returns
     -------
@@ -694,7 +694,7 @@ def run(
 ) -> list[dict]:
     """
     Run the IPCC (2019) Tier 1 methodology for calculating SOC stocks (in kg C ha-1) for each year in the inventory
-    and wrap each of the calculated values in Hestia measurement nodes. To avoid any errors, the `inventory` parameter
+    and wrap each of the calculated values in HESTIA measurement nodes. To avoid any errors, the `inventory` parameter
     must be pre-validated by the `should_run` function.
 
     See [IPCC (2019) Vol. 4, Ch. 2](https://www.ipcc-nggip.iges.or.jp/public/2019rf/vol4.html) for more information.