hestia-earth-models 0.65.11__tar.gz → 0.67.0__tar.gz

{hestia_earth_models-0.65.11 → hestia_earth_models-0.67.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_models
-Version: 0.65.11
+Version: 0.67.0
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

{hestia_earth_models-0.65.11 → hestia_earth_models-0.67.0}/hestia_earth/models/cache_sites.py

@@ -8,7 +8,7 @@ from .log import logger
 from .utils import CACHE_KEY, cached_value
 from .utils.site import CACHE_YEARS_KEY
 from .site.pre_checks.cache_geospatialDatabase import (
-    list_collections, cache_site_results, _should_run
+    list_vectors, list_rasters, cache_site_results, _should_run
 )
 from .geospatialDatabase.utils import (
     CACHE_VALUE as CACHE_GEOSPATIAL_KEY, CACHE_AREA_SIZE,
@@ -115,8 +115,9 @@ def _group_sites(sites: dict, check_has_cache: bool = True):
     }
 
 
-def _run(sites: list, years: list = [], include_region: bool = False, years_only: bool = False):
-    rasters, vectors = list_collections(years, include_region, years_only)
+def _run(sites: list, years: list = [], years_only: bool = False):
+    rasters = list_rasters(years=years, years_only=years_only)
+    vectors = [] if years_only else list_vectors(sites)
     filtered_data = _group_sites(sites, not years_only)
     return flatten([
         _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
@@ -124,7 +125,7 @@ def _run(sites: list, years: list = [], include_region: bool = False, years_only
     ])
 
 
-def run(sites: list, years: list = None, include_region: bool = False):
+def run(sites: list, years: list = None):
     """
     Run all queries at once for the list of provided Sites.
     Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
@@ -135,11 +136,8 @@ def run(sites: list, years: list = None, include_region: bool = False):
         List of Site as dict.
     years : list[int]
         List of related years to fetch annual data.
-    include_region : bool
-        Prefecth region IDs.
-        This will cache region-level data and will make the request slower. Only use if needed.
     """
-    sites = _run(sites, include_region=include_region)
+    sites = _run(sites)
 
     # avoid memory limit errors by running only a few years at a time
     unique_years = sorted(list(set(years)))
@@ -149,6 +147,6 @@ def run(sites: list, years: list = None, include_region: bool = False):
     for batch_index in batches:
         logger.info(f"Processing sites in batch {batch_index + 1} of {len(batches)}...")
         sub_years = unique_years[batch_index:batch_index + batch_size]
-        sites = _run(sites, sub_years, include_region, years_only=True)
+        sites = _run(sites, sub_years, years_only=True)
 
     return sites

{hestia_earth_models-0.65.11 → hestia_earth_models-0.67.0}/hestia_earth/models/cml2001Baseline/abioticResourceDepletionFossilFuels.py

@@ -10,23 +10,18 @@ are converted into MJ.
 
 Source : [Life Cycle Assessment & the EF methods - Comprehensive coverage of impacts](https://green-business.ec.europa.eu/environmental-footprint-methods/life-cycle-assessment-ef-methods_en)
 
-The model accepts any non-renewable energy term that can be expressed in, or converted to, MegaJoules.
+The model accepts any non-renewable energy term in MegaJoules, or that has been converted to MegaJoules.
 
 Source: [JRC Technical reports Suggestions for updating the Product Environmental Footprint (PEF) method](https://eplca.jrc.ec.europa.eu/permalink/PEF_method.pdf#%5B%7B%22num%22%3A80%2C%22gen%22%3A0%7D%2C%7B%22name%22%3A%22XYZ%22%7D%2C68%2C363%2C0%5D)
 
 Source: [Differences between EF model versions](https://eplca.jrc.ec.europa.eu/EFVersioning.html)
 """  # noqa: E501
 from itertools import chain
-from typing import Optional
-
-from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import download_lookup, column_name
-from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum, flatten
 
 from hestia_earth.models.log import logShouldRun, logRequirements, log_as_table
 from . import MODEL
-from ..utils.blank_node import convert_unit
 from ..utils.constant import Units
 from ..utils.indicator import _new_indicator
 from ..utils.lookup import _node_value
@@ -38,29 +33,13 @@ REQUIREMENTS = {
                 "@type": "Indicator",
                 "term.termType": "resourceUse",
                 "term.@id": ["resourceUseEnergyDepletionInputsProduction", "resourceUseEnergyDepletionDuringCycle"],
+                "term.units": "MJ",
                 "value": "> 0",
                 "inputs":
                     {
                         "@type": "Input",
                         "term.name": "non-renewable\" energy terms only,\"",
-                        "term.units": ["kg", "m3", "kWh", "MJ"],
-                        "term.termType": ["fuel", "electricity"],
-                        "optional": {
-                            "defaultProperties": [
-                                {
-                                    "@type": "Property",
-                                    "value": "",
-                                    "term.@id": "energyContentHigherHeatingValue",
-                                    "term.units": "MJ / kg"
-                                },
-                                {
-                                    "@type": "Property",
-                                    "value": "",
-                                    "term.@id": "density",
-                                    "term.units": "kg / m3"
-                                }
-                            ]
-                        }
+                        "term.termType": ["fuel", "electricity"]
                     }
             }
         ]
@@ -68,8 +47,8 @@ REQUIREMENTS = {
 }
 
 LOOKUPS = {
-    "fuel": ["energyContentHigherHeatingValue", "density", "abioticResourceDepletionFossilFuelsCml2001Baseline"],
-    "electricity": ["abioticResourceDepletionFossilFuelsCml2001Baseline"]
+    "fuel": ["consideredFossilFuelUnderCml2001Baseline"],
+    "electricity": ["consideredFossilFuelUnderCml2001Baseline"]
 }
 
 RETURNS = {
@@ -79,39 +58,31 @@ RETURNS = {
 }
 TERM_ID = 'abioticResourceDepletionFossilFuels'
 
-ENERGY_CARRIERS_TERMIDS = REQUIREMENTS['ImpactAssessment']['emissionsResourceUse'][0]['term.@id']
-
-INPUTS_TYPES_UNITS = {
-    TermTermType.FUEL.value: [Units.KG.value, Units.M3.value, Units.MJ.value],
-    TermTermType.ELECTRICITY.value: [Units.KW_H.value, Units.MJ.value]
-}
+_ENERGY_CARRIERS_TERMIDS = REQUIREMENTS['ImpactAssessment']['emissionsResourceUse'][0]['term.@id']
 
 
-def download_all_non_renewable_terms(lookup_file_name: str) -> list:
+def get_all_non_renewable_terms(lookup_file_name: str, column: str) -> list:
     """
     returns all non renewable term ids in lookup files like `electricity.csv` or `fuel.csv`
     """
     lookup = download_lookup(lookup_file_name)
-    results = lookup[
-        lookup[column_name("abioticResourceDepletionFossilFuelsCml2001Baseline")] == True  # noqa: E712
-    ]["termid"]
+    results = lookup[lookup[column_name(column)] == True]["termid"]  # noqa: E712
     return list(map(str, results))
 
 
 def _valid_resource_indicator(resource: dict) -> bool:
-    return len(resource.get('inputs', [])) == 1 and \
-        isinstance(_node_value(resource), (int, float)) and \
-        _node_value(resource) > 0
+    return (
+        len(resource.get('inputs', [])) == 1 and
+        isinstance(_node_value(resource), (int, float)) and
+        _node_value(resource) > 0 and
+        resource.get("term", {}).get("units", "") == Units.MJ.value
+    )
 
 
 def _valid_input(input: dict) -> bool:
     return input.get("@id") in list(chain.from_iterable([
-        download_all_non_renewable_terms(f"{termType}.csv") for termType in LOOKUPS.keys()
-    ])) and input.get("units", "") in INPUTS_TYPES_UNITS.get(input.get("termType", ""))
-
-
-def _get_value_in_mj(input: dict, indicator: dict) -> Optional[float]:
-    return convert_unit(input, dest_unit=Units.MJ, node_value=_node_value(indicator)) if _valid_input(input) else None
+        get_all_non_renewable_terms(f"{termType}.csv", columns[0]) for termType, columns in LOOKUPS.items()
+    ]))
 
 
 def _indicator(value: float):
@@ -126,9 +97,9 @@ def _run(energy_resources_in_mj: list):
 
 def _should_run(impact_assessment: dict) -> tuple[bool, list]:
     emissions_resource_use = [
-        resource for resource in
-        filter_list_term_type(impact_assessment.get('emissionsResourceUse', []), TermTermType.RESOURCEUSE) if
-        resource.get('term', {}).get('@id', '') in ENERGY_CARRIERS_TERMIDS]
+        resource for resource in impact_assessment.get('emissionsResourceUse', [])
+        if resource.get('term', {}).get('@id') in _ENERGY_CARRIERS_TERMIDS
+    ]
 
     has_resource_use_entries = bool(emissions_resource_use)
 
@@ -140,11 +111,8 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
                     "input-term-type": input.get('termType'),
                     "indicator-term-id": resource_indicator['term']['@id'],
                     "indicator-is-valid": _valid_resource_indicator(resource_indicator),
-                    "input": input,
                     "indicator-input-is-valid": _valid_input(input),
-                    "value": _node_value(resource_indicator),
-                    "input-unit": input.get('units'),
-                    "value-in-MJ": _get_value_in_mj(input, resource_indicator)
+                    "value-in-MJ": _node_value(resource_indicator) if _valid_input(input) else None,
                 } for input in resource_indicator['inputs'] or [{}]]
             for resource_indicator in emissions_resource_use
         ]
@@ -157,8 +125,9 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
 
     energy_resources_in_mj = [energy_input['value-in-MJ'] for energy_input in resource_uses_unpacked]
     valid_energy_resources_in_mj = [value for value in energy_resources_in_mj if value is not None]
-    all_inputs_have_valid_mj_value = all([energy is not None for energy in energy_resources_in_mj]
-                                         ) and bool(energy_resources_in_mj)
+    all_inputs_have_valid_mj_value = all(
+        [energy is not None for energy in energy_resources_in_mj] + [bool(energy_resources_in_mj)]
+    )
 
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     has_resource_use_entries=has_resource_use_entries,

hestia_earth_models-0.67.0/hestia_earth/models/cml2001Baseline/resourceUseEnergyDepletionDuringCycle.py

@@ -0,0 +1,152 @@
+"""
+This model converts all "energy" terms found in a `Cycle > Inputs` to `MJ` using optional
+`energyContentHigherHeatingValue` and `density` properties or the term's "defaultProperties",
+ aggregates them, and places them inside a 'resourceUseEnergyDepletionDuringCycle' indicator per aggregated input id.
+"""
+from collections import defaultdict
+from itertools import groupby
+from typing import Tuple, Optional
+
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils import Units
+from hestia_earth.models.utils.blank_node import convert_unit
+from hestia_earth.models.utils.indicator import _new_indicator
+from . import MODEL
+from ..utils.lookup import _node_value
+
+REQUIREMENTS = {
+    "Cycle": {
+        "inputs": [
+            {
+                "@type": "Input",
+                "term.units": ["kg", "m3", "kWh", "MJ"],
+                "term.termType": ["fuel", "electricity"],
+                "value": ">0",
+                "optional": {
+                    "properties": [
+                        {
+                            "@type": "Property",
+                            "value": "",
+                            "term.@id": "energyContentHigherHeatingValue",
+                            "term.units": "MJ / kg"
+                        },
+                        {
+                            "@type": "Property",
+                            "value": "",
+                            "term.@id": "density",
+                            "term.units": "kg / m3"
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+}
+
+RETURNS = {
+    "Indicator": [
+        {
+            "value": "",
+            "inputs": ""
+        }
+    ]
+}
+
+LOOKUPS = {
+    "fuel": ["energyContentHigherHeatingValue", "density"]
+}
+
+TERM_ID = 'resourceUseEnergyDepletionDuringCycle'
+
+INPUTS_TYPES_UNITS = {
+    TermTermType.FUEL.value: [Units.KG.value, Units.M3.value, Units.MJ.value],
+    TermTermType.ELECTRICITY.value: [Units.KW_H.value, Units.MJ.value]
+}
+
+
+def _indicator(values: list[float], cycle_input: dict):
+    indicator = _new_indicator(TERM_ID, MODEL)
+    indicator['value'] = list_sum(values)
+    indicator['inputs'] = [cycle_input | {"units": "MJ"}]
+    return indicator
+
+
+def _run(grouped_energy_terms: dict):
+    indicators = [_indicator(values=[input['value-in-MJ'] for input in energy_term_group_vals],
+                             cycle_input=energy_term_group_vals[0]['input']['term'])
+                  for energy_term_group_vals in grouped_energy_terms.values()]
+    return indicators
+
+
+def _valid_input(input: dict) -> bool:
+    return (isinstance(_node_value(input), (int, float))
+            and _node_value(input) > 0
+            and input.get("term", {}).get("units", "") in INPUTS_TYPES_UNITS.get(input.get("term", {}).get("termType")))
+
+
+def _get_value_in_mj(input: dict) -> Optional[float]:
+    return convert_unit(input, dest_unit=Units.MJ, node_value=_node_value(input)) if _valid_input(input) else None
+
+
+def _should_run(cycle: dict) -> Tuple[bool, dict]:
+    energy_input_terms = filter_list_term_type(cycle.get('inputs', []), [TermTermType.FUEL, TermTermType.ELECTRICITY])
+
+    has_energy_terms = bool(energy_input_terms)
+
+    energy_input_terms_unpacked = [
+        {
+            "id": input.get('term', {}).get('@id'),
+            "termType": input.get('term', {}).get('termType'),
+            "input-is-valid": _valid_input(input),
+            "value": _node_value(input),
+            "input": input,
+            "units": input.get('term', {}).get('units'),
+            "value-in-MJ": _get_value_in_mj(input)
+        }
+        for input in energy_input_terms
+    ]
+
+    has_valid_input_requirements = all([energy_input['input-is-valid'] for energy_input in energy_input_terms_unpacked])
+
+    energy_input_terms_valid = [e for e in energy_input_terms_unpacked if e["value-in-MJ"] is not None]
+
+    energy_input_terms_in_mj = [energy_input['value-in-MJ'] for energy_input in energy_input_terms_unpacked]
+    all_inputs_have_valid_mj_value = all([mj_value is not None for mj_value in energy_input_terms_in_mj]
+                                         ) and bool(energy_input_terms_in_mj)
+
+    grouped_energy_terms = defaultdict(list)
+    for k, v in groupby(energy_input_terms_valid, key=lambda x: x.get('id')):
+        grouped_energy_terms[k].extend(list(v))
+
+    logs = [
+        {
+            'id': input.get('@id'),
+            'units': input.get('units'),
+            'termType': input.get('termType'),
+            'value': input.get('value'),
+            'value-in-MJ': input.get('value-in-MJ'),
+            'properties': "    ".join([
+                f"{p.get('term', {}).get('@id')}= {p.get('value')} ({p.get('term', {}).get('units')})"
+                for p in input.get('properties', [])
+            ])
+        } for input in energy_input_terms_unpacked
+    ]
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_energy_terms=has_energy_terms,
+                    has_valid_input_requirements=has_valid_input_requirements,
+                    all_inputs_have_valid_mj_value=all_inputs_have_valid_mj_value,
+                    energy_resources_used=log_as_table(logs))
+
+    should_run = all([has_valid_input_requirements, all_inputs_have_valid_mj_value])
+
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run, grouped_energy_terms
+
+
+def run(cycle: dict):
+    should_run, grouped_energy_terms = _should_run(cycle)
+    return _run(grouped_energy_terms) if should_run else []

hestia_earth_models-0.67.0/hestia_earth/models/cml2001Baseline/resourceUseEnergyDepletionInputsProduction.py

@@ -0,0 +1,40 @@
+from hestia_earth.models.linkedImpactAssessment.utils import run_inputs_production
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "product": {"@type": "Term"},
+        "cycle": {
+            "@type": "Cycle",
+            "products": [{
+                "@type": "Product",
+                "primary": "True",
+                "value": "> 0",
+                "economicValueShare": "> 0"
+            }],
+            "inputs": [{
+                "@type": "Input",
+                "impactAssessment": {
+                    "@type": "ImpactAssessment",
+                    "emissionsResourceUse": [{
+                        "@type": "Indicator",
+                        "term.@id": [
+                            "resourceUseEnergyDepletionDuringCycle",
+                            "resourceUseEnergyDepletionInputsProduction"
+                        ],
+                        "value": "> 0"
+                    }]
+                }
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Indicator": [{
+        "value": "",
+        "inputs": "units in MJ"
+    }]
+}
+TERM_ID = 'resourceUseEnergyDepletionInputsProduction'
+
+
+def run(impact_assessment: dict): return run_inputs_production(impact_assessment, TERM_ID)

hestia_earth_models-0.67.0/hestia_earth/models/cml2001Baseline/resourceUseMineralsAndMetalsDuringCycle.py

@@ -0,0 +1,80 @@
+"""
+This model converts most abiotic resources terms found in a `Cycle > Inputs`, aggregates them,
+and converts them to one indicator per input.
+"""
+from collections import defaultdict
+from itertools import groupby
+from typing import Tuple
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import _filter_list_term_unit, Units
+from hestia_earth.models.utils.blank_node import _sum_nodes_value
+from hestia_earth.models.utils.indicator import _new_indicator
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "inputs": [
+            {
+                "@type": "Input",
+                "term.units": "kg",
+                "term.termType": ["material", "soilAmendment", "otherInorganicChemical"]}
+        ]
+    }
+}
+
+RETURNS = {
+    "Indicator": [{
+        "value": "",
+        "inputs": ""
+    }]
+}
+TERM_ID = 'resourceUseMineralsAndMetalsDuringCycle'
+
+authorised_term_types = [TermTermType.MATERIAL, TermTermType.SOILAMENDMENT, TermTermType.OTHERINORGANICCHEMICAL]
+
+
+def _indicator(value: list[float], cycle_input: dict):
+    indicator = _new_indicator(TERM_ID, MODEL)
+    indicator['value'] = list_sum(value)
+    indicator['inputs'] = [cycle_input]
+    return indicator
+
+
+def _run(grouped_abiotic_terms: dict):
+    indicators = [
+        _indicator(value=_sum_nodes_value(abiotic_term_group_vals),
+                   cycle_input=abiotic_term_group_vals[0]['term'])
+        for abiotic_term_group_vals in grouped_abiotic_terms.values()
+    ]
+    return indicators
+
+
+def _should_run(cycle: dict) -> Tuple[bool, dict]:
+    abiotic_terms = filter_list_term_type(cycle.get('inputs', []), authorised_term_types)
+    abiotic_terms_valid_units = _filter_list_term_unit(abiotic_terms, Units.KG)
+
+    has_abiotic_terms = bool(abiotic_terms)
+    has_valid_terms = bool(abiotic_terms_valid_units)
+
+    grouped_abiotic_terms = defaultdict(list)
+    for k, v in groupby(abiotic_terms_valid_units,
+                        key=lambda x: (x['term']['@id'], x['term']['units'], x['term']['termType'])):
+        grouped_abiotic_terms[k].extend(list(v))
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_abiotic_terms=has_abiotic_terms,
+                    has_valid_terms=has_valid_terms,
+                    )
+
+    should_run = all([has_valid_terms])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run, grouped_abiotic_terms
+
+
+def run(cycle: dict):
+    should_run, grouped_abiotic_terms = _should_run(cycle)
+    return _run(grouped_abiotic_terms) if should_run else []

hestia_earth_models-0.67.0/hestia_earth/models/cml2001Baseline/resourceUseMineralsAndMetalsInputsProduction.py

@@ -0,0 +1,40 @@
+from hestia_earth.models.linkedImpactAssessment.utils import run_inputs_production
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "product": {"@type": "Term"},
+        "cycle": {
+            "@type": "Cycle",
+            "products": [{
+                "@type": "Product",
+                "primary": "True",
+                "value": "> 0",
+                "economicValueShare": "> 0"
+            }],
+            "inputs": [{
+                "@type": "Input",
+                "impactAssessment": {
+                    "@type": "ImpactAssessment",
+                    "emissionsResourceUse": [{
+                        "@type": "Indicator",
+                        "term.@id": [
+                            "resourceUseMineralsAndMetalsDuringCycle",
+                            "resourceUseMineralsAndMetalsInputsProduction"
+                        ],
+                        "value": "> 0"
+                    }]
+                }
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Indicator": [{
+        "value": "",
+        "inputs": ""
+    }]
+}
+TERM_ID = 'resourceUseMineralsAndMetalsInputsProduction'
+
+
+def run(impact_assessment: dict): return run_inputs_production(impact_assessment, TERM_ID)

{hestia_earth_models-0.65.11 → hestia_earth_models-0.67.0}/hestia_earth/models/config/Cycle.json

@@ -166,6 +166,14 @@
       "mergeStrategy": "list",
       "stage": 1
     },
+    {
+      "key": "practices",
+      "model": "hestia",
+      "value": "residueRemoved",
+      "runStrategy": "add_blank_node_if_missing",
+      "mergeStrategy": "list",
+      "stage": 1
+    },
     {
       "key": "practices",
       "model": "cycle",
@@ -287,22 +295,6 @@
         "mergeStrategy": "list",
         "stage": 1
       },
-      {
-        "key": "inputs",
-        "model": "faostat2018",
-        "value": "seed",
-        "runStrategy": "add_blank_node_if_missing",
-        "mergeStrategy": "list",
-        "stage": 1
-      },
-      {
-        "key": "inputs",
-        "model": "pooreNemecek2018",
-        "value": "saplingsDepreciatedAmountPerCycle",
-        "runStrategy": "add_blank_node_if_missing",
-        "mergeStrategy": "list",
-        "stage": 1
-      },
       {
         "key": "practices",
         "model": "cycle",
@@ -441,6 +433,32 @@
         "stage": 1
       }
     ],
+    [
+      {
+        "key": "inputs",
+        "model": "faostat2018",
+        "value": "seed",
+        "runStrategy": "add_blank_node_if_missing",
+        "mergeStrategy": "list",
+        "stage": 1
+      },
+      {
+        "key": "inputs",
+        "model": "pooreNemecek2018",
+        "value": "saplingsDepreciatedAmountPerCycle",
+        "runStrategy": "add_blank_node_if_missing",
+        "mergeStrategy": "list",
+        "stage": 1
+      }
+    ],
+    {
+      "key": "completeness",
+      "model": "cycle",
+      "value": "completeness",
+      "runStrategy": "always",
+      "mergeStrategy": "node",
+      "stage": 1
+    },
     [
       {
         "key": "animals",