hestia-earth-models 0.65.3__py3-none-any.whl → 0.65.5__py3-none-any.whl

hestia_earth/models/agribalyse2016/fuelElectricity.py

@@ -1,14 +1,14 @@
 """
 Fuel and Electricity
 
-This model calculates fuel and electricity data from the number of hours each machine is operated for using.
+This model calculates fuel and electricity data from the number of hours each machine is used for.
 """
-from functools import reduce
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import flatten, list_sum, non_empty_list
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils import group_by
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.input import _new_input
 from . import MODEL
@@ -68,57 +68,63 @@ def _run_operation(cycle: dict):
     return exec
 
 
-def _group_operations(operations: list):
-    def grouper(group: dict, operation: dict):
-        input_term_id = operation.get('input').get('id')
-        group[input_term_id] = group.get(input_term_id, [])
-        group[input_term_id].append(operation)
-        return group
+def _operation_data(practice: dict):
+    term = practice.get('term', {})
+    values = practice.get('value', [])
+    value = list_sum(values) if all([not isinstance(v, str) for v in values]) else None  # str allowed for Practice
 
-    return reduce(grouper, operations, {})
+    coeffs = get_lookup_value(term, LOOKUPS['operation'], model=MODEL, model_key=MODEL_KEY)
+    values = non_empty_list(coeffs.split(';')) if coeffs else []
+    inputs = [{'id': c.split(':')[0], 'value': float(c.split(':')[1])} for c in values]
 
+    return [{
+        'term': term,
+        'value': value,
+        'input': input,
+        'dates': ';'.join(practice.get('dates', []))
+    } for input in inputs]
 
-def _should_run_operation(cycle: dict):
-    def exec(practice: dict):
-        term = practice.get('term', {})
-        term_id = term.get('@id')
-        values = practice.get('value', [])
-        value = list_sum(values) if all([not isinstance(v, str) for v in values]) else 0  # str allowed for Practice
-        has_value = value > 0
 
-        coeffs = get_lookup_value(term, LOOKUPS['operation'], model=MODEL, model_key=MODEL_KEY)
-        values = non_empty_list(coeffs.split(';')) if coeffs else []
-        inputs = [{'id': c.split(':')[0], 'value': float(c.split(':')[1])} for c in values]
-        has_lookup_value = len(inputs) > 0
+def _should_run(cycle: dict):
+    is_incomplete = not cycle.get('completeness', {}).get('electricityFuel', False)
+    operations = filter_list_term_type(cycle.get('practices', []), TermTermType.OPERATION)
 
-        logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                        has_value=has_value,
-                        has_fuelUse_lookup_value=has_lookup_value)
+    operations = flatten(map(_operation_data, operations))
+    term_ids = list(set(non_empty_list(map(lambda v: v.get('term', {}).get('@id'), operations))))
 
-        should_run = all([has_value, has_lookup_value])
-        logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)
-        return [{'term': term, 'value': value, 'input': input} for input in inputs] if should_run else []
-    return exec
+    valid_operations = [v for v in operations if (v.get('value') or 0) > 0]
+    has_operations = len(valid_operations) > 0
 
+    # group operations by term to show in logs
+    grouped_operations = group_by(operations, ['term.@id'])
 
-def _should_run(cycle: dict):
-    is_incomplete = not cycle.get('completeness', {}).get('electricityFuel', False)
-    operations = filter_list_term_type(cycle.get('practices', []), TermTermType.OPERATION)
-    operations = flatten(map(_should_run_operation(cycle), operations))
-    has_operations = len(operations) > 0
+    for term_id, operations in grouped_operations.items():
+        logs = [
+            {
+                'value': operation.get('value'),
+                'dates': operation.get('dates'),
+                'input-id': operation.get('input', {}).get('@id'),
+            }
+            for operation in operations
+        ]
+        logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                        details=log_as_table(logs))
+
+        should_run = any([(v.get('value') or 0) > 0 for v in operations])
+        logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)
 
     logRequirements(cycle, model=MODEL, model_key=MODEL_KEY,
                     is_term_type_electricityFuel_incomplete=is_incomplete,
                     has_operations=has_operations,
-                    operations=';'.join(non_empty_list(map(lambda v: v.get('term', {}).get('@id'), operations))))
+                    operations=';'.join(term_ids))
 
     should_run = all([is_incomplete, has_operations])
     logShouldRun(cycle, MODEL, None, should_run, model_key=MODEL_KEY)
-    return should_run, operations
+    return should_run, valid_operations
 
 
 def run(cycle: dict):
     should_run, operations = _should_run(cycle)
     # group operations by input to show logs as table
-    grouped_operations = _group_operations(operations)
+    grouped_operations = group_by(operations, ['input.id'])
     return flatten(map(_run_operation(cycle), grouped_operations.values())) if should_run else []

hestia_earth/models/aware/scarcityWeightedWaterUse.py

@@ -89,7 +89,7 @@ def _run(impact_assessment: dict):
         _get_factor_from_basinId(site, aware_id) if aware_id else None
     ) or _get_factor_from_region(impact_assessment, site)
     inputs_value = convert_value_from_cycle(
-        product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
+        impact_assessment, product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
     )
 
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,

hestia_earth/models/cache_sites.py

@@ -4,6 +4,7 @@ from pydash.objects import merge
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import flatten
 
+from .log import logger
 from .utils import CACHE_KEY, cached_value
 from .utils.site import CACHE_YEARS_KEY
 from .site.pre_checks.cache_geospatialDatabase import (
@@ -146,6 +147,7 @@ def run(sites: list, years: list = None, include_region: bool = False):
     batches = range(0, len(unique_years), batch_size)
 
     for batch_index in batches:
+        logger.info(f"Processing sites in batch {batch_index + 1} of {len(batches)}...")
         sub_years = unique_years[batch_index:batch_index + batch_size]
         sites = _run(sites, sub_years, include_region, years_only=True)
 

hestia_earth/models/chaudharyBrooks2018/damageToTerrestrialEcosystemsLandOccupation.py

@@ -73,7 +73,7 @@ def _run(impact_assessment: dict):
     land_occupation_m2_kg = land_occupation_per_kg(MODEL, TERM_ID, cycle, site, product)
     factor = get_region_factor(TERM_ID, impact_assessment, LOOKUP_SUFFIX, 'medium_intensity')
     inputs_value = convert_value_from_cycle(
-        product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
+        impact_assessment, product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
     )
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     landOccupation=land_occupation_m2_kg,

hestia_earth/models/chaudharyBrooks2018/damageToTerrestrialEcosystemsLandTransformation.py

@@ -68,7 +68,7 @@ def _run(impact_assessment: dict):
     landTransformation = _value(impact_assessment, _TRANSFORMATION_TERM_ID)
     region_factor = get_region_factor(TERM_ID, impact_assessment, _LOOKUP_SUFFIX, 'medium_intensity')
     inputs_value = convert_value_from_cycle(
-        product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
+        impact_assessment, product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
     )
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     landTransformation=landTransformation,

hestia_earth/models/chaudharyBrooks2018/damageToTerrestrialEcosystemsTotalLandUseEffects.py

@@ -57,7 +57,7 @@ def run(impact_assessment: dict):
     cycle = impact_assessment.get('cycle', {})
     product = get_product(impact_assessment)
     inputs_value = convert_value_from_cycle(
-        product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
+        impact_assessment, product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
     )
     value = sum_values([landUseEffects, inputs_value])
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,

hestia_earth/models/cycle/completeness/__init__.py

@@ -31,7 +31,7 @@ def _run_model(model, cycle: dict):
 
 def _run(cycle: dict):
     values = non_empty_list([_run_model(model, cycle) for model in MODELS])
-    value = reduce(lambda prev, curr: {**prev, **curr}, values, cycle.get('completeness', {}))
+    value = reduce(lambda prev, curr: prev | curr, values, cycle.get('completeness', {}))
     keys = ','.join([next(iter(val)) for val in values])
     debugValues(cycle, model=MODEL, keys=keys)
     return value if len(values) > 0 else None

hestia_earth/models/cycle/completeness/electricityFuel.py

@@ -40,7 +40,7 @@ def _lookup_value(practice: dict, lookup_name: str):
 def _practice_value(practice: dict):
     term = practice.get('term', {})
     fuel_use = _lookup_value(practice, LOOKUPS['operation'][0])
-    return {'id': term.get('@id'), 'fuel_use': fuel_use}
+    return {'id': term.get('@id'), 'value': practice.get('value'), 'fuel_use': fuel_use}
 
 
 def run(cycle: dict):
@@ -55,6 +55,8 @@ def run(cycle: dict):
                     term_type_operation_complete=operation_complete,
                     values=log_as_table(practices_values))
 
-    is_complete = all([operation_complete] + [p.get('fuel_use') for p in practices_values])
+    is_complete = all([operation_complete] + [
+        all([p.get('fuel_use'), p.get('value') is not None]) for p in practices_values
+    ])
 
     return is_complete

hestia_earth/models/emepEea2019/nh3ToAirInorganicFertiliser.py

@@ -78,7 +78,7 @@ def _get_groupings():
 
     def get_grouping(groupings: dict, term_id: str):
         grouping = get_term_lookup(term_id, 'fertGroupingNitrogen')
-        return {**groupings, **({grouping: term_id} if len(grouping) > 0 else {})}
+        return groupings | ({grouping: term_id} if len(grouping) > 0 else {})
 
     return reduce(get_grouping, term_ids, {})
 

hestia_earth/models/geospatialDatabase/precipitationAnnual.py

@@ -31,8 +31,8 @@ RETURNS = {
 }
 TERM_ID = 'precipitationAnnual'
 EE_PARAMS = {
-    'collection': 'ECMWF/ERA5/MONTHLY',
-    'band_name': 'total_precipitation',
+    'collection': 'ECMWF/ERA5_LAND/MONTHLY_AGGR',
+    'band_name': 'total_precipitation_sum',
     'ee_type': 'raster',
     'reducer': 'mean',
     'reducer_annual': 'sum'

hestia_earth/models/geospatialDatabase/precipitationLongTermAnnualMean.py

@@ -27,8 +27,8 @@ TERM_ID = 'precipitationLongTermAnnualMean'
 START_DATE = '1979-01-01'
 END_DATE = '2020-12-31'
 EE_PARAMS = {
-    'collection': 'ECMWF/ERA5/MONTHLY',
-    'band_name': 'total_precipitation',
+    'collection': 'ECMWF/ERA5_LAND/MONTHLY_AGGR',
+    'band_name': 'total_precipitation_sum',
     'ee_type': 'raster',
     'reducer': 'mean',
     'reducer_annual': 'sum',

hestia_earth/models/geospatialDatabase/precipitationMonthly.py

@@ -31,8 +31,8 @@ RETURNS = {
 }
 TERM_ID = 'precipitationMonthly'
 EE_PARAMS = {
-    'collection': 'ECMWF/ERA5/MONTHLY',
-    'band_name': 'total_precipitation',
+    'collection': 'ECMWF/ERA5_LAND/MONTHLY_AGGR',
+    'band_name': 'total_precipitation_sum',
     'ee_type': 'raster',
     'reducer': 'mean',
     'reducer_annual': 'sum'

hestia_earth/models/geospatialDatabase/temperatureAnnual.py

@@ -31,8 +31,8 @@ RETURNS = {
 }
 TERM_ID = 'temperatureAnnual'
 EE_PARAMS = {
-    'collection': 'ECMWF/ERA5/MONTHLY',
-    'band_name': 'mean_2m_air_temperature',
+    'collection': 'ECMWF/ERA5_LAND/MONTHLY_AGGR',
+    'band_name': 'temperature_2m',
     'ee_type': 'raster',
     'reducer': 'mean',
     'reducer_annual': 'mean'

hestia_earth/models/geospatialDatabase/temperatureLongTermAnnualMean.py

@@ -27,8 +27,8 @@ TERM_ID = 'temperatureLongTermAnnualMean'
 START_DATE = '1979-01-01'
 END_DATE = '2020-12-31'
 EE_PARAMS = {
-    'collection': 'ECMWF/ERA5/MONTHLY',
-    'band_name': 'mean_2m_air_temperature',
+    'collection': 'ECMWF/ERA5_LAND/MONTHLY_AGGR',
+    'band_name': 'temperature_2m',
     'ee_type': 'raster',
     'reducer': 'mean',
     'reducer_annual': 'mean',

hestia_earth/models/geospatialDatabase/temperatureMonthly.py

@@ -31,8 +31,8 @@ RETURNS = {
 }
 TERM_ID = 'temperatureMonthly'
 EE_PARAMS = {
-    'collection': 'ECMWF/ERA5/MONTHLY',
-    'band_name': 'mean_2m_air_temperature',
+    'collection': 'ECMWF/ERA5_LAND/MONTHLY_AGGR',
+    'band_name': 'temperature_2m',
     'ee_type': 'raster',
     'reducer': 'mean',
     'reducer_annual': 'mean'

hestia_earth/models/hestia/landCover.py

@@ -17,6 +17,20 @@ from hestia_earth.utils.tools import safe_parse_float, to_precision, non_empty_v
 from hestia_earth.models.log import logRequirements, log_as_table, logShouldRun
 from hestia_earth.models.utils.management import _new_management
 from hestia_earth.models.utils.term import get_lookup_value
+from .utils import (
+    IPCC_LAND_USE_CATEGORY_ANNUAL,
+    IPCC_LAND_USE_CATEGORY_PERENNIAL,
+    LAND_USE_TERMS_FOR_TRANSFORMATION,
+    ANNUAL_CROPLAND,
+    PERMANENT_CROPLAND,
+    FOREST_LAND,
+    OTHER_LAND,
+    PERMANENT_PASTURE,
+    TOTAL_CROPLAND,
+    TOTAL_AGRICULTURAL_CHANGE,
+    ALL_LAND_USE_TERMS,
+    crop_ipcc_land_use_category,
+)
 from . import MODEL
 
 REQUIREMENTS = {
@@ -69,30 +83,6 @@ LOOKUPS = {
 MODEL_KEY = 'landCover'
 
 LAND_AREA = LOOKUPS["region-faostatArea"][3]
-TOTAL_CROPLAND = "Cropland"
-ANNUAL_CROPLAND = "Arable land"
-FOREST_LAND = "Forest land"
-OTHER_LAND = "Other land"
-PERMANENT_CROPLAND = "Permanent crops"
-PERMANENT_PASTURE = "Permanent meadows and pastures"
-TOTAL_AGRICULTURAL_CHANGE = "Total agricultural change"
-ALL_LAND_USE_TERMS = [
-    FOREST_LAND,
-    TOTAL_CROPLAND,
-    ANNUAL_CROPLAND,
-    PERMANENT_CROPLAND,
-    PERMANENT_PASTURE,
-    OTHER_LAND
-]
-# Mapping from Land use terms to Management node terms.
-# land use term: (@id, name)
-LAND_USE_TERMS_FOR_TRANSFORMATION = {
-    FOREST_LAND: ("forest", "Forest"),
-    ANNUAL_CROPLAND: ("annualCropland", "Annual cropland"),
-    PERMANENT_CROPLAND: ("permanentCropland", "Permanent cropland"),
-    PERMANENT_PASTURE: ("permanentPasture", "Permanent pasture"),
-    OTHER_LAND: ("otherLand", OTHER_LAND)  # Not used yet
-}
 SITE_TYPES = {
     SiteSiteType.CROPLAND.value,
     SiteSiteType.FOREST.value,
@@ -100,8 +90,6 @@ SITE_TYPES = {
     SiteSiteType.PERMANENT_PASTURE.value
 }
 DEFAULT_WINDOW_IN_YEARS = 20
-IPCC_LAND_USE_CATEGORY_ANNUAL = "Annual crops"
-IPCC_LAND_USE_CATEGORY_PERENNIAL = "Perennial crops"
 OUTPUT_SIGNIFICANT_DIGITS = 3
 
 
@@ -137,22 +125,6 @@ def _lookup_land_use_type(nodes: list) -> str:
     )
 
 
-def _crop_ipcc_land_use_category(
-    crop_term_id: str,
-    lookup_term_type: str = TermTermType.LANDCOVER.value
-) -> str:
-    """
-    Looks up the crop in the lookup.
-    Returns the IPCC_LAND_USE_CATEGORY.
-    """
-    return get_lookup_value(
-        lookup_term={"@id": crop_term_id, "type": "Term", "termType": lookup_term_type},
-        column=LOOKUPS.get("crop")[1],
-        model=MODEL,
-        term={"@id": crop_term_id, "type": "Term", "termType": lookup_term_type}
-    )
-
-
 def get_changes(country_id: str, end_year: int) -> dict:
     """
     For each entry in ALL_LAND_USE_TERMS, creates a key: value in output dictionary, also TOTAL
@@ -399,7 +371,7 @@ def _get_complete_faostat_to_crop_mapping() -> dict:
             get_table_value(lookup, 'termid', crop_term_id, column_name("cropGroupingFaostatArea"))
         )
         if key:
-            mappings[key].append(_crop_ipcc_land_use_category(crop_term_id=crop_term_id, lookup_term_type="crop"))
+            mappings[key].append(crop_ipcc_land_use_category(crop_term_id=crop_term_id, lookup_term_type="crop"))
     return {
         fao_name: max(set(crop_terms), key=crop_terms.count)
         for fao_name, crop_terms in mappings.items()
@@ -420,6 +392,18 @@ def _get_harvested_area(country_id: str, year: int, faostat_name: str) -> float:
     )
 
 
+def _get_term_id_for_crop(nodes: set, land_type: str) -> str:
+    """Use the original crop term id for permanent/perennial crops and the land use id for other types."""
+    result = next(
+        (node for node in nodes if crop_ipcc_land_use_category(node[0]) == IPCC_LAND_USE_CATEGORY_PERENNIAL), None
+    )
+    return (
+        # Take first perennial crop - not multi-cropping
+        result[0] if land_type == PERMANENT_CROPLAND and result else
+        LAND_USE_TERMS_FOR_TRANSFORMATION[land_type][0]
+    )
+
+
 def _run_make_management_nodes(existing_nodes: list, percentage_transformed_from: dict, start_year: int) -> list:
     """Creates a list of new management nodes, excluding any dates matching existing ones."""
     existing_nodes_set = {
@@ -435,7 +419,8 @@ def _run_make_management_nodes(existing_nodes: list, percentage_transformed_from
             "percentage": 0 if ratio == -0.0 else to_precision(
                 number=ratio * 100,
                 digits=OUTPUT_SIGNIFICANT_DIGITS
-            )
+            ),
+            "term_id": _get_term_id_for_crop(existing_nodes_set, land_type=land_type)
         }
         for land_type, ratio in percentage_transformed_from.items()
     ]
@@ -443,7 +428,7 @@ def _run_make_management_nodes(existing_nodes: list, percentage_transformed_from
 
     return [
         _management(
-            term_id=LAND_USE_TERMS_FOR_TRANSFORMATION[value.get("land_type")][0],
+            term_id=value.get("term_id"),
             value=value.get("percentage"),
             start_date=value.get("land_management_key")[1],
             end_date=value.get("land_management_key")[2]
@@ -698,7 +683,7 @@ def _should_run_historical_land_use_change_single_crop(
             sum_of_site_areas_is_100
         ]
     )
-    logShouldRun(site, MODEL, term=term.get("@id"), should_run=should_run, key=MODEL_KEY)
+    logShouldRun(site, MODEL, term.get("@id"), should_run, model_key=MODEL_KEY)
 
     return should_run, site_area
 

hestia_earth/models/hestia/landTransformation100YearAverageDuringCycle.py

@@ -0,0 +1,49 @@
+"""
+Creates an [emissionsResourceUse](https://hestia.earth/schema/Emission) for every landCover land transformation.
+contained within the [ImpactAssesment.cycle](https://hestia.earth/schema/ImpactAssessment#cycle), averaged over the last
+100 years.
+
+It does this by multiplying the land occupation during the cycle by the
+[Site](https://www-staging.hestia.earth/schema/Site) area 100 years ago and dividing by 100.
+
+Land transformation from [land type] 100 years =
+(Land occupation, during Cycle * Site Percentage Area 100 years ago [land type] / 100) / 100
+"""
+from .resourceUse_utils import run_resource_use
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "Site": {
+            "management": [{"@type": "Management", "value": ">=0", "term.termType": "landCover", "endDate": ""}]
+        },
+        "emissionsResourceUse": [
+            {
+                "@type": "Indicator",
+                "term.@id": "landOccupationDuringCycle",
+                "landCover": {
+                    "@type": "Term",
+                    "termType": "landCover"
+                },
+                "value": ">=0"
+            }
+        ],
+        "endDate": ""
+    }
+}
+RETURNS = {
+    "Indicator": [{
+        "value": "",
+        "landCover": "",
+        "previousLandCover": ""
+    }]
+}
+TERM_ID = 'landTransformation100YearAverageDuringCycle'
+_HISTORIC_DATE_OFFSET = 100
+
+
+def run(impact_assessment: dict):
+    return run_resource_use(
+        impact_assessment=impact_assessment,
+        historic_date_offset=_HISTORIC_DATE_OFFSET,
+        term_id=TERM_ID
+    )

hestia_earth/models/hestia/landTransformation20YearAverageDuringCycle.py

@@ -0,0 +1,49 @@
+"""
+Creates an [emissionsResourceUse](https://hestia.earth/schema/Emission) for every landCover land transformation.
+contained within the [ImpactAssesment.cycle](https://hestia.earth/schema/ImpactAssessment#cycle), averaged over the last
+20 years.
+
+It does this by multiplying the land occupation during the cycle by the
+[Site](https://www-staging.hestia.earth/schema/Site) area 20 years ago and dividing by 20.
+
+Land transformation from [land type] 20 years =
+(Land occupation, during Cycle * Site Percentage Area 20 years ago [land type] / 100) / 20
+"""
+from .resourceUse_utils import run_resource_use
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "Site": {
+            "management": [{"@type": "Management", "value": ">=0", "term.termType": "landCover", "endDate": ""}]
+        },
+        "emissionsResourceUse": [
+            {
+                "@type": "Indicator",
+                "term.@id": "landOccupationDuringCycle",
+                "landCover": {
+                    "@type": "Term",
+                    "termType": "landCover"
+                },
+                "value": ">=0"
+            }
+        ],
+        "endDate": ""
+    }
+}
+RETURNS = {
+    "Indicator": [{
+        "value": "",
+        "landCover": "",
+        "previousLandCover": ""
+    }]
+}
+TERM_ID = 'landTransformation20YearAverageDuringCycle'
+_HISTORIC_DATE_OFFSET = 20
+
+
+def run(impact_assessment: dict):
+    return run_resource_use(
+        impact_assessment=impact_assessment,
+        historic_date_offset=_HISTORIC_DATE_OFFSET,
+        term_id=TERM_ID
+    )