hestia-earth-models 0.65.11__py3-none-any.whl → 0.67.0__py3-none-any.whl

hestia_earth/models/cache_sites.py

@@ -8,7 +8,7 @@ from .log import logger
 from .utils import CACHE_KEY, cached_value
 from .utils.site import CACHE_YEARS_KEY
 from .site.pre_checks.cache_geospatialDatabase import (
-    list_collections, cache_site_results, _should_run
+    list_vectors, list_rasters, cache_site_results, _should_run
 )
 from .geospatialDatabase.utils import (
     CACHE_VALUE as CACHE_GEOSPATIAL_KEY, CACHE_AREA_SIZE,
@@ -115,8 +115,9 @@ def _group_sites(sites: dict, check_has_cache: bool = True):
     }
 
 
-def _run(sites: list, years: list = [], include_region: bool = False, years_only: bool = False):
-    rasters, vectors = list_collections(years, include_region, years_only)
+def _run(sites: list, years: list = [], years_only: bool = False):
+    rasters = list_rasters(years=years, years_only=years_only)
+    vectors = [] if years_only else list_vectors(sites)
     filtered_data = _group_sites(sites, not years_only)
     return flatten([
         _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
@@ -124,7 +125,7 @@ def _run(sites: list, years: list = [], include_region: bool = False, years_only
     ])
 
 
-def run(sites: list, years: list = None, include_region: bool = False):
+def run(sites: list, years: list = None):
     """
     Run all queries at once for the list of provided Sites.
     Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
@@ -135,11 +136,8 @@ def run(sites: list, years: list = None, include_region: bool = False):
         List of Site as dict.
     years : list[int]
         List of related years to fetch annual data.
-    include_region : bool
-        Prefecth region IDs.
-        This will cache region-level data and will make the request slower. Only use if needed.
     """
-    sites = _run(sites, include_region=include_region)
+    sites = _run(sites)
 
     # avoid memory limit errors by running only a few years at a time
     unique_years = sorted(list(set(years)))
@@ -149,6 +147,6 @@ def run(sites: list, years: list = None, include_region: bool = False):
     for batch_index in batches:
         logger.info(f"Processing sites in batch {batch_index + 1} of {len(batches)}...")
         sub_years = unique_years[batch_index:batch_index + batch_size]
-        sites = _run(sites, sub_years, include_region, years_only=True)
+        sites = _run(sites, sub_years, years_only=True)
 
     return sites

hestia_earth/models/cml2001Baseline/abioticResourceDepletionFossilFuels.py

@@ -10,23 +10,18 @@ are converted into MJ.
 
 Source : [Life Cycle Assessment & the EF methods - Comprehensive coverage of impacts](https://green-business.ec.europa.eu/environmental-footprint-methods/life-cycle-assessment-ef-methods_en)
 
-The model accepts any non-renewable energy term that can be expressed in, or converted to, MegaJoules.
+The model accepts any non-renewable energy term in MegaJoules, or that has been converted to MegaJoules.
 
 Source: [JRC Technical reports Suggestions for updating the Product Environmental Footprint (PEF) method](https://eplca.jrc.ec.europa.eu/permalink/PEF_method.pdf#%5B%7B%22num%22%3A80%2C%22gen%22%3A0%7D%2C%7B%22name%22%3A%22XYZ%22%7D%2C68%2C363%2C0%5D)
 
 Source: [Differences between EF model versions](https://eplca.jrc.ec.europa.eu/EFVersioning.html)
 """  # noqa: E501
 from itertools import chain
-from typing import Optional
-
-from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import download_lookup, column_name
-from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum, flatten
 
 from hestia_earth.models.log import logShouldRun, logRequirements, log_as_table
 from . import MODEL
-from ..utils.blank_node import convert_unit
 from ..utils.constant import Units
 from ..utils.indicator import _new_indicator
 from ..utils.lookup import _node_value
@@ -38,29 +33,13 @@ REQUIREMENTS = {
                 "@type": "Indicator",
                 "term.termType": "resourceUse",
                 "term.@id": ["resourceUseEnergyDepletionInputsProduction", "resourceUseEnergyDepletionDuringCycle"],
+                "term.units": "MJ",
                 "value": "> 0",
                 "inputs":
                     {
                         "@type": "Input",
                         "term.name": "non-renewable\" energy terms only,\"",
-                        "term.units": ["kg", "m3", "kWh", "MJ"],
-                        "term.termType": ["fuel", "electricity"],
-                        "optional": {
-                            "defaultProperties": [
-                                {
-                                    "@type": "Property",
-                                    "value": "",
-                                    "term.@id": "energyContentHigherHeatingValue",
-                                    "term.units": "MJ / kg"
-                                },
-                                {
-                                    "@type": "Property",
-                                    "value": "",
-                                    "term.@id": "density",
-                                    "term.units": "kg / m3"
-                                }
-                            ]
-                        }
+                        "term.termType": ["fuel", "electricity"]
                     }
             }
         ]
@@ -68,8 +47,8 @@ REQUIREMENTS = {
 }
 
 LOOKUPS = {
-    "fuel": ["energyContentHigherHeatingValue", "density", "abioticResourceDepletionFossilFuelsCml2001Baseline"],
-    "electricity": ["abioticResourceDepletionFossilFuelsCml2001Baseline"]
+    "fuel": ["consideredFossilFuelUnderCml2001Baseline"],
+    "electricity": ["consideredFossilFuelUnderCml2001Baseline"]
 }
 
 RETURNS = {
@@ -79,39 +58,31 @@ RETURNS = {
 }
 TERM_ID = 'abioticResourceDepletionFossilFuels'
 
-ENERGY_CARRIERS_TERMIDS = REQUIREMENTS['ImpactAssessment']['emissionsResourceUse'][0]['term.@id']
-
-INPUTS_TYPES_UNITS = {
-    TermTermType.FUEL.value: [Units.KG.value, Units.M3.value, Units.MJ.value],
-    TermTermType.ELECTRICITY.value: [Units.KW_H.value, Units.MJ.value]
-}
+_ENERGY_CARRIERS_TERMIDS = REQUIREMENTS['ImpactAssessment']['emissionsResourceUse'][0]['term.@id']
 
 
-def download_all_non_renewable_terms(lookup_file_name: str) -> list:
+def get_all_non_renewable_terms(lookup_file_name: str, column: str) -> list:
     """
     returns all non renewable term ids in lookup files like `electricity.csv` or `fuel.csv`
     """
     lookup = download_lookup(lookup_file_name)
-    results = lookup[
-        lookup[column_name("abioticResourceDepletionFossilFuelsCml2001Baseline")] == True  # noqa: E712
-    ]["termid"]
+    results = lookup[lookup[column_name(column)] == True]["termid"]  # noqa: E712
     return list(map(str, results))
 
 
 def _valid_resource_indicator(resource: dict) -> bool:
-    return len(resource.get('inputs', [])) == 1 and \
-        isinstance(_node_value(resource), (int, float)) and \
-        _node_value(resource) > 0
+    return (
+        len(resource.get('inputs', [])) == 1 and
+        isinstance(_node_value(resource), (int, float)) and
+        _node_value(resource) > 0 and
+        resource.get("term", {}).get("units", "") == Units.MJ.value
+    )
 
 
 def _valid_input(input: dict) -> bool:
     return input.get("@id") in list(chain.from_iterable([
-        download_all_non_renewable_terms(f"{termType}.csv") for termType in LOOKUPS.keys()
-    ])) and input.get("units", "") in INPUTS_TYPES_UNITS.get(input.get("termType", ""))
-
-
-def _get_value_in_mj(input: dict, indicator: dict) -> Optional[float]:
-    return convert_unit(input, dest_unit=Units.MJ, node_value=_node_value(indicator)) if _valid_input(input) else None
+        get_all_non_renewable_terms(f"{termType}.csv", columns[0]) for termType, columns in LOOKUPS.items()
+    ]))
 
 
 def _indicator(value: float):
@@ -126,9 +97,9 @@ def _run(energy_resources_in_mj: list):
 
 def _should_run(impact_assessment: dict) -> tuple[bool, list]:
     emissions_resource_use = [
-        resource for resource in
-        filter_list_term_type(impact_assessment.get('emissionsResourceUse', []), TermTermType.RESOURCEUSE) if
-        resource.get('term', {}).get('@id', '') in ENERGY_CARRIERS_TERMIDS]
+        resource for resource in impact_assessment.get('emissionsResourceUse', [])
+        if resource.get('term', {}).get('@id') in _ENERGY_CARRIERS_TERMIDS
+    ]
 
     has_resource_use_entries = bool(emissions_resource_use)
 
@@ -140,11 +111,8 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
                     "input-term-type": input.get('termType'),
                     "indicator-term-id": resource_indicator['term']['@id'],
                     "indicator-is-valid": _valid_resource_indicator(resource_indicator),
-                    "input": input,
                     "indicator-input-is-valid": _valid_input(input),
-                    "value": _node_value(resource_indicator),
-                    "input-unit": input.get('units'),
-                    "value-in-MJ": _get_value_in_mj(input, resource_indicator)
+                    "value-in-MJ": _node_value(resource_indicator) if _valid_input(input) else None,
                 } for input in resource_indicator['inputs'] or [{}]]
             for resource_indicator in emissions_resource_use
         ]
@@ -157,8 +125,9 @@ def _should_run(impact_assessment: dict) -> tuple[bool, list]:
 
     energy_resources_in_mj = [energy_input['value-in-MJ'] for energy_input in resource_uses_unpacked]
     valid_energy_resources_in_mj = [value for value in energy_resources_in_mj if value is not None]
-    all_inputs_have_valid_mj_value = all([energy is not None for energy in energy_resources_in_mj]
-                                         ) and bool(energy_resources_in_mj)
+    all_inputs_have_valid_mj_value = all(
+        [energy is not None for energy in energy_resources_in_mj] + [bool(energy_resources_in_mj)]
+    )
 
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     has_resource_use_entries=has_resource_use_entries,

hestia_earth/models/cml2001Baseline/resourceUseEnergyDepletionDuringCycle.py

@@ -0,0 +1,152 @@
+"""
+This model converts all "energy" terms found in a `Cycle > Inputs` to `MJ` using optional
+`energyContentHigherHeatingValue` and `density` properties or the term's "defaultProperties",
+ aggregates them, and places them inside a 'resourceUseEnergyDepletionDuringCycle' indicator per aggregated input id.
+"""
+from collections import defaultdict
+from itertools import groupby
+from typing import Tuple, Optional
+
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils import Units
+from hestia_earth.models.utils.blank_node import convert_unit
+from hestia_earth.models.utils.indicator import _new_indicator
+from . import MODEL
+from ..utils.lookup import _node_value
+
+REQUIREMENTS = {
+    "Cycle": {
+        "inputs": [
+            {
+                "@type": "Input",
+                "term.units": ["kg", "m3", "kWh", "MJ"],
+                "term.termType": ["fuel", "electricity"],
+                "value": ">0",
+                "optional": {
+                    "properties": [
+                        {
+                            "@type": "Property",
+                            "value": "",
+                            "term.@id": "energyContentHigherHeatingValue",
+                            "term.units": "MJ / kg"
+                        },
+                        {
+                            "@type": "Property",
+                            "value": "",
+                            "term.@id": "density",
+                            "term.units": "kg / m3"
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+}
+
+RETURNS = {
+    "Indicator": [
+        {
+            "value": "",
+            "inputs": ""
+        }
+    ]
+}
+
+LOOKUPS = {
+    "fuel": ["energyContentHigherHeatingValue", "density"]
+}
+
+TERM_ID = 'resourceUseEnergyDepletionDuringCycle'
+
+INPUTS_TYPES_UNITS = {
+    TermTermType.FUEL.value: [Units.KG.value, Units.M3.value, Units.MJ.value],
+    TermTermType.ELECTRICITY.value: [Units.KW_H.value, Units.MJ.value]
+}
+
+
+def _indicator(values: list[float], cycle_input: dict):
+    indicator = _new_indicator(TERM_ID, MODEL)
+    indicator['value'] = list_sum(values)
+    indicator['inputs'] = [cycle_input | {"units": "MJ"}]
+    return indicator
+
+
+def _run(grouped_energy_terms: dict):
+    indicators = [_indicator(values=[input['value-in-MJ'] for input in energy_term_group_vals],
+                             cycle_input=energy_term_group_vals[0]['input']['term'])
+                  for energy_term_group_vals in grouped_energy_terms.values()]
+    return indicators
+
+
+def _valid_input(input: dict) -> bool:
+    return (isinstance(_node_value(input), (int, float))
+            and _node_value(input) > 0
+            and input.get("term", {}).get("units", "") in INPUTS_TYPES_UNITS.get(input.get("term", {}).get("termType")))
+
+
+def _get_value_in_mj(input: dict) -> Optional[float]:
+    return convert_unit(input, dest_unit=Units.MJ, node_value=_node_value(input)) if _valid_input(input) else None
+
+
+def _should_run(cycle: dict) -> Tuple[bool, dict]:
+    energy_input_terms = filter_list_term_type(cycle.get('inputs', []), [TermTermType.FUEL, TermTermType.ELECTRICITY])
+
+    has_energy_terms = bool(energy_input_terms)
+
+    energy_input_terms_unpacked = [
+        {
+            "id": input.get('term', {}).get('@id'),
+            "termType": input.get('term', {}).get('termType'),
+            "input-is-valid": _valid_input(input),
+            "value": _node_value(input),
+            "input": input,
+            "units": input.get('term', {}).get('units'),
+            "value-in-MJ": _get_value_in_mj(input)
+        }
+        for input in energy_input_terms
+    ]
+
+    has_valid_input_requirements = all([energy_input['input-is-valid'] for energy_input in energy_input_terms_unpacked])
+
+    energy_input_terms_valid = [e for e in energy_input_terms_unpacked if e["value-in-MJ"] is not None]
+
+    energy_input_terms_in_mj = [energy_input['value-in-MJ'] for energy_input in energy_input_terms_unpacked]
+    all_inputs_have_valid_mj_value = all([mj_value is not None for mj_value in energy_input_terms_in_mj]
+                                         ) and bool(energy_input_terms_in_mj)
+
+    grouped_energy_terms = defaultdict(list)
+    for k, v in groupby(energy_input_terms_valid, key=lambda x: x.get('id')):
+        grouped_energy_terms[k].extend(list(v))
+
+    logs = [
+        {
+            'id': input.get('@id'),
+            'units': input.get('units'),
+            'termType': input.get('termType'),
+            'value': input.get('value'),
+            'value-in-MJ': input.get('value-in-MJ'),
+            'properties': "    ".join([
+                f"{p.get('term', {}).get('@id')}= {p.get('value')} ({p.get('term', {}).get('units')})"
+                for p in input.get('properties', [])
+            ])
+        } for input in energy_input_terms_unpacked
+    ]
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_energy_terms=has_energy_terms,
+                    has_valid_input_requirements=has_valid_input_requirements,
+                    all_inputs_have_valid_mj_value=all_inputs_have_valid_mj_value,
+                    energy_resources_used=log_as_table(logs))
+
+    should_run = all([has_valid_input_requirements, all_inputs_have_valid_mj_value])
+
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run, grouped_energy_terms
+
+
+def run(cycle: dict):
+    should_run, grouped_energy_terms = _should_run(cycle)
+    return _run(grouped_energy_terms) if should_run else []

hestia_earth/models/cml2001Baseline/resourceUseEnergyDepletionInputsProduction.py

@@ -0,0 +1,40 @@
+from hestia_earth.models.linkedImpactAssessment.utils import run_inputs_production
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "product": {"@type": "Term"},
+        "cycle": {
+            "@type": "Cycle",
+            "products": [{
+                "@type": "Product",
+                "primary": "True",
+                "value": "> 0",
+                "economicValueShare": "> 0"
+            }],
+            "inputs": [{
+                "@type": "Input",
+                "impactAssessment": {
+                    "@type": "ImpactAssessment",
+                    "emissionsResourceUse": [{
+                        "@type": "Indicator",
+                        "term.@id": [
+                            "resourceUseEnergyDepletionDuringCycle",
+                            "resourceUseEnergyDepletionInputsProduction"
+                        ],
+                        "value": "> 0"
+                    }]
+                }
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Indicator": [{
+        "value": "",
+        "inputs": "units in MJ"
+    }]
+}
+TERM_ID = 'resourceUseEnergyDepletionInputsProduction'
+
+
+def run(impact_assessment: dict): return run_inputs_production(impact_assessment, TERM_ID)

hestia_earth/models/cml2001Baseline/resourceUseMineralsAndMetalsDuringCycle.py

@@ -0,0 +1,80 @@
+"""
+This model converts most abiotic resources terms found in a `Cycle > Inputs`, aggregates them,
+and converts them to one indicator per input.
+"""
+from collections import defaultdict
+from itertools import groupby
+from typing import Tuple
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import _filter_list_term_unit, Units
+from hestia_earth.models.utils.blank_node import _sum_nodes_value
+from hestia_earth.models.utils.indicator import _new_indicator
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "inputs": [
+            {
+                "@type": "Input",
+                "term.units": "kg",
+                "term.termType": ["material", "soilAmendment", "otherInorganicChemical"]}
+        ]
+    }
+}
+
+RETURNS = {
+    "Indicator": [{
+        "value": "",
+        "inputs": ""
+    }]
+}
+TERM_ID = 'resourceUseMineralsAndMetalsDuringCycle'
+
+authorised_term_types = [TermTermType.MATERIAL, TermTermType.SOILAMENDMENT, TermTermType.OTHERINORGANICCHEMICAL]
+
+
+def _indicator(value: list[float], cycle_input: dict):
+    indicator = _new_indicator(TERM_ID, MODEL)
+    indicator['value'] = list_sum(value)
+    indicator['inputs'] = [cycle_input]
+    return indicator
+
+
+def _run(grouped_abiotic_terms: dict):
+    indicators = [
+        _indicator(value=_sum_nodes_value(abiotic_term_group_vals),
+                   cycle_input=abiotic_term_group_vals[0]['term'])
+        for abiotic_term_group_vals in grouped_abiotic_terms.values()
+    ]
+    return indicators
+
+
+def _should_run(cycle: dict) -> Tuple[bool, dict]:
+    abiotic_terms = filter_list_term_type(cycle.get('inputs', []), authorised_term_types)
+    abiotic_terms_valid_units = _filter_list_term_unit(abiotic_terms, Units.KG)
+
+    has_abiotic_terms = bool(abiotic_terms)
+    has_valid_terms = bool(abiotic_terms_valid_units)
+
+    grouped_abiotic_terms = defaultdict(list)
+    for k, v in groupby(abiotic_terms_valid_units,
+                        key=lambda x: (x['term']['@id'], x['term']['units'], x['term']['termType'])):
+        grouped_abiotic_terms[k].extend(list(v))
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_abiotic_terms=has_abiotic_terms,
+                    has_valid_terms=has_valid_terms,
+                    )
+
+    should_run = all([has_valid_terms])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run, grouped_abiotic_terms
+
+
+def run(cycle: dict):
+    should_run, grouped_abiotic_terms = _should_run(cycle)
+    return _run(grouped_abiotic_terms) if should_run else []

hestia_earth/models/cml2001Baseline/resourceUseMineralsAndMetalsInputsProduction.py

@@ -0,0 +1,40 @@
+from hestia_earth.models.linkedImpactAssessment.utils import run_inputs_production
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "product": {"@type": "Term"},
+        "cycle": {
+            "@type": "Cycle",
+            "products": [{
+                "@type": "Product",
+                "primary": "True",
+                "value": "> 0",
+                "economicValueShare": "> 0"
+            }],
+            "inputs": [{
+                "@type": "Input",
+                "impactAssessment": {
+                    "@type": "ImpactAssessment",
+                    "emissionsResourceUse": [{
+                        "@type": "Indicator",
+                        "term.@id": [
+                            "resourceUseMineralsAndMetalsDuringCycle",
+                            "resourceUseMineralsAndMetalsInputsProduction"
+                        ],
+                        "value": "> 0"
+                    }]
+                }
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Indicator": [{
+        "value": "",
+        "inputs": ""
+    }]
+}
+TERM_ID = 'resourceUseMineralsAndMetalsInputsProduction'
+
+
+def run(impact_assessment: dict): return run_inputs_production(impact_assessment, TERM_ID)

hestia_earth/models/config/Cycle.json

@@ -166,6 +166,14 @@
       "mergeStrategy": "list",
       "stage": 1
     },
+    {
+      "key": "practices",
+      "model": "hestia",
+      "value": "residueRemoved",
+      "runStrategy": "add_blank_node_if_missing",
+      "mergeStrategy": "list",
+      "stage": 1
+    },
     {
       "key": "practices",
       "model": "cycle",
@@ -287,22 +295,6 @@
         "mergeStrategy": "list",
         "stage": 1
       },
-      {
-        "key": "inputs",
-        "model": "faostat2018",
-        "value": "seed",
-        "runStrategy": "add_blank_node_if_missing",
-        "mergeStrategy": "list",
-        "stage": 1
-      },
-      {
-        "key": "inputs",
-        "model": "pooreNemecek2018",
-        "value": "saplingsDepreciatedAmountPerCycle",
-        "runStrategy": "add_blank_node_if_missing",
-        "mergeStrategy": "list",
-        "stage": 1
-      },
       {
         "key": "practices",
         "model": "cycle",
@@ -441,6 +433,32 @@
         "stage": 1
       }
     ],
+    [
+      {
+        "key": "inputs",
+        "model": "faostat2018",
+        "value": "seed",
+        "runStrategy": "add_blank_node_if_missing",
+        "mergeStrategy": "list",
+        "stage": 1
+      },
+      {
+        "key": "inputs",
+        "model": "pooreNemecek2018",
+        "value": "saplingsDepreciatedAmountPerCycle",
+        "runStrategy": "add_blank_node_if_missing",
+        "mergeStrategy": "list",
+        "stage": 1
+      }
+    ],
+    {
+      "key": "completeness",
+      "model": "cycle",
+      "value": "completeness",
+      "runStrategy": "always",
+      "mergeStrategy": "node",
+      "stage": 1
+    },
     [
       {
         "key": "animals",