hestia-earth-models 0.73.4__py3-none-any.whl → 0.73.6__py3-none-any.whl

hestia_earth/models/config/Cycle.json

@@ -2093,51 +2093,6 @@
         },
         "stage": 2
       },
-      {
-        "key": "emissions",
-        "model": "ipcc2006",
-        "value": "n2OToAirCropResidueDecompositionIndirect",
-        "runStrategy": "add_blank_node_if_missing",
-        "runArgs": {
-          "runNonMeasured": true,
-          "runNonAddedTerm": true
-        },
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 2
-      },
-      {
-        "key": "emissions",
-        "model": "ipcc2006",
-        "value": "n2OToAirCropResidueDecompositionDirect",
-        "runStrategy": "add_blank_node_if_missing",
-        "runArgs": {
-          "runNonMeasured": true,
-          "runNonAddedTerm": true
-        },
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 2
-      },
-      {
-        "key": "emissions",
-        "model": "ipcc2006",
-        "value": "n2OToAirExcretaIndirect",
-        "runStrategy": "add_blank_node_if_missing",
-        "runArgs": {
-          "runNonMeasured": true,
-          "runNonAddedTerm": true
-        },
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 2
-      },
       {
         "key": "emissions",
         "model": "jarvisAndPain1994",

hestia_earth/models/config/ImpactAssessment.json

@@ -203,30 +203,6 @@
         "stage": 1
       }
     ],
-    [
-      {
-        "key": "emissionsResourceUse",
-        "model": "faostat2018",
-        "value": "landTransformation20YearAverageDuringCycle",
-        "runStrategy": "always",
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 1
-      },
-      {
-        "key": "emissionsResourceUse",
-        "model": "faostat2018",
-        "value": "landTransformation100YearAverageDuringCycle",
-        "runStrategy": "always",
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 1
-      }
-    ],
     {
       "key": "emissionsResourceUse",
       "model": "resourceUseNotRelevant",

hestia_earth/models/config/Site.json

@@ -523,6 +523,17 @@
           "replaceThreshold": ["value", 0.01]
         },
         "stage": 2
+      },
+      {
+        "key": "measurements",
+        "model": "ipcc2019",
+        "value": "biocharOrganicCarbonPerHa",
+        "runStrategy": "always",
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
       }
     ]
   ]

hestia_earth/models/geospatialDatabase/potentialEvapotranspirationAnnual.py

@@ -2,6 +2,7 @@ from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import max_date
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.site import related_years
 from .utils import download, has_geospatial_data, should_download
@@ -39,7 +40,7 @@ def _measurement(value: float, year: int):
     measurement['value'] = [value]
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     measurement['startDate'] = f"{year}-01-01"
-    measurement['endDate'] = f"{year}-12-31"
+    measurement['endDate'] = max_date(f"{year}-12-31")
     return measurement
 
 

hestia_earth/models/geospatialDatabase/precipitationAnnual.py

@@ -2,6 +2,7 @@ from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import max_date
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from hestia_earth.models.utils.site import related_years
@@ -41,7 +42,7 @@ def _measurement(site: dict, value: float, year: int):
     measurement['value'] = [value]
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     measurement['startDate'] = f"{year}-01-01"
-    measurement['endDate'] = f"{year}-12-31"
+    measurement['endDate'] = max_date(f"{year}-12-31")
     return measurement | get_source(site, BIBLIO_TITLE)
 
 

hestia_earth/models/geospatialDatabase/temperatureAnnual.py

@@ -2,6 +2,7 @@ from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import max_date
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from hestia_earth.models.utils.site import related_years
@@ -40,7 +41,7 @@ def _measurement(site: dict, value: float, year: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
     measurement['startDate'] = f"{year}-01-01"
-    measurement['endDate'] = f"{year}-12-31"
+    measurement['endDate'] = max_date(f"{year}-12-31")
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 

hestia_earth/models/hestia/landCover.py

@@ -2,7 +2,6 @@ import functools
 import math
 from collections import defaultdict
 from datetime import datetime, timedelta
-
 from hestia_earth.schema import SiteSiteType, TermTermType
 from hestia_earth.utils.lookup import (
     download_lookup, get_table_value, column_name, _is_missing_value, extract_grouped_data, lookup_columns
@@ -64,6 +63,11 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
+    "region-crop-cropGroupingFAOSTAT-landCover-annualCropland": "",
+    "region-crop-cropGroupingFAOSTAT-landCover-forest": "",
+    "region-crop-cropGroupingFAOSTAT-landCover-otherLand": "",
+    "region-crop-cropGroupingFAOSTAT-landCover-permanentCropland": "",
+    "region-crop-cropGroupingFAOSTAT-landCover-permanentPasture": "",
     "region-crop-cropGroupingFaostatProduction-areaHarvestedUpTo20YearExpansion": "",
     "region-crop-cropGroupingFaostatProduction-areaHarvested": "",
     "region-faostatArea-UpTo20YearExpansion": "",
@@ -82,18 +86,30 @@ LOOKUPS = {
 }
 MODEL_KEY = 'landCover'
 
-LAND_AREA = LOOKUPS["region-faostatArea"][3]
-SITE_TYPES = {
+_LAND_AREA = LOOKUPS["region-faostatArea"][3]
+_ALLOWED_SITE_TYPES = {
     SiteSiteType.CROPLAND.value,
     SiteSiteType.FOREST.value,
     SiteSiteType.OTHER_NATURAL_VEGETATION.value,
     SiteSiteType.PERMANENT_PASTURE.value
 }
-DEFAULT_WINDOW_IN_YEARS = 20
-DATE_TOLERANCE_IN_YEARS = 2
-OUTPUT_SIGNIFICANT_DIGITS = 3
-ALLOWED_LAND_USE_TYPES = [ANNUAL_CROPLAND, PERMANENT_CROPLAND, PERMANENT_PASTURE]
+_BUILDING_SITE_TYPES = [
+    SiteSiteType.AGRI_FOOD_PROCESSOR.value,
+    SiteSiteType.ANIMAL_HOUSING.value,
+    SiteSiteType.FOOD_RETAILER.value
+]
+_DEFAULT_WINDOW_IN_YEARS = 20
+_DATE_TOLERANCE_IN_YEARS = 2
+_OUTPUT_SIGNIFICANT_DIGITS = 3
+_ALLOWED_LAND_USE_TYPES = [ANNUAL_CROPLAND, PERMANENT_CROPLAND, PERMANENT_PASTURE]
 _LOOKUP_EXPANSION = "region-crop-cropGroupingFaostatProduction-areaHarvestedUpTo20YearExpansion.csv"
+_COMPLETE_CHANGES_OTHER_LAND = {
+    OTHER_LAND: 1,
+    FOREST_LAND: 0,
+    PERMANENT_PASTURE: 0,
+    ANNUAL_CROPLAND: 0,
+    PERMANENT_CROPLAND: 0
+}
 
 
 def _get_lookup_with_cache(lookup_term, column):
@@ -116,6 +132,41 @@ def _get_lookup_with_cache(lookup_term, column):
     )
 
 
+def _get_land_cover_lookup_suffix(land_type: str) -> str:
+    return LAND_USE_TERMS_FOR_TRANSFORMATION[land_type][0]
+
+
+def get_landCover_lookups(country_id: str, end_year: int, product_name: str):
+    """
+    Attempts to get the pre-calculated values for the landCover model calculation.
+    Returns: {"Arable land": <value>, "Forest land": <value>, "Other land": <value>,
+              "Permanent crops": <value>, "Permanent meadows and pastures": <value>}
+    Missing values are returned as None.
+    """
+    lookup_prefix = 'region-crop-cropGroupingFAOSTAT-landCover'
+    return {
+        # Divide by 100 to match site_area ratios
+        land_type: value / 100 if value is not None else value
+        for land_type, value in
+        {
+            land_type: safe_parse_float(
+                value=extract_grouped_data(
+                    data=get_region_lookup_value(
+                        lookup_name=f"{lookup_prefix}-{_get_land_cover_lookup_suffix(land_type)}.csv",
+                        term_id=country_id,
+                        column=product_name,
+                        model=MODEL,
+                        key=MODEL_KEY
+                    ),
+                    key=str(end_year)
+                ),
+                default=None
+            )
+            for land_type in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
+        }.items()
+    }
+
+
 def _management(term_id: str, value: float, start_date: str, end_date: str):
     node = _new_management(term_id, MODEL)
     node['value'] = value
@@ -176,7 +227,7 @@ def _get_changes(country_id: str, end_year: int) -> tuple[dict, bool]:
             ),
             default=None
         )
-        for land_use_term in ALL_LAND_USE_TERMS + [LAND_AREA]
+        for land_use_term in ALL_LAND_USE_TERMS + [_LAND_AREA]
     }
     missing_changes = [k for k, v in changes_dict.items() if v is None]
     changes_dict = {k: v if v is not None else 0 for k, v in changes_dict.items()}
@@ -267,7 +318,7 @@ def _allocate_pasture_loss_to_cropland(changes: dict, land_required_for_cropland
 
 
 def _allocate_other_land(
-        changes: dict, max_forest_loss_to: dict, pasture_loss_to_cropland: float, cropland_loss_to_pasture: float
+    changes: dict, max_forest_loss_to: dict, pasture_loss_to_cropland: float, cropland_loss_to_pasture: float
 ) -> dict:
     """Allocate changes between Other land and cropland"""
     other_land_loss_to_cropland = (
@@ -461,7 +512,7 @@ def _get_term_id_for_crop(nodes: set, land_type: str) -> str:
 
 
 def _run(site: dict, existing_nodes: list, percentage_transformed_from: dict):
-    start_year = _get_year_from_landCover(existing_nodes[0]) - DEFAULT_WINDOW_IN_YEARS
+    start_year = _get_year_from_landCover(existing_nodes[0]) - _DEFAULT_WINDOW_IN_YEARS
 
     """Creates a list of new management nodes, excluding any dates matching existing ones."""
     existing_nodes_set = {
@@ -476,7 +527,7 @@ def _run(site: dict, existing_nodes: list, percentage_transformed_from: dict):
             "land_type": land_type,
             "percentage": 0 if ratio == -0.0 else to_precision(
                 number=ratio * 100,
-                digits=OUTPUT_SIGNIFICANT_DIGITS
+                digits=_OUTPUT_SIGNIFICANT_DIGITS
             ),
             "term_id": _get_term_id_for_crop(existing_nodes_set, land_type=land_type)
         }
@@ -565,15 +616,17 @@ def _get_sums_of_crop_expansion(country_id: str, year: int, include_negatives: b
 def _get_net_expansion_cultivated_vs_harvested(
     annual_crops_net_expansion, changes, land_use_type, permanent_crops_net_expansion
 ):
-    if land_use_type == ANNUAL_CROPLAND:
-        net_expansion_cultivated_vs_harvested = _safe_divide(numerator=max(0, changes[ANNUAL_CROPLAND]),
-                                                             denominator=(annual_crops_net_expansion / 1000))
-    elif land_use_type == PERMANENT_CROPLAND:
-        net_expansion_cultivated_vs_harvested = _safe_divide(numerator=max(0, changes[PERMANENT_CROPLAND]),
-                                                             denominator=(permanent_crops_net_expansion / 1000))
-    else:
-        net_expansion_cultivated_vs_harvested = 1
-    return net_expansion_cultivated_vs_harvested
+    return (
+        _safe_divide(
+            numerator=max(0, changes[ANNUAL_CROPLAND]),
+            denominator=(annual_crops_net_expansion / 1000)
+        ) if land_use_type == ANNUAL_CROPLAND else
+        _safe_divide(
+            numerator=max(0, changes[PERMANENT_CROPLAND]),
+            denominator=(permanent_crops_net_expansion / 1000)
+        ) if land_use_type == PERMANENT_CROPLAND else
+        1
+    )
 
 
 def _get_year_from_landCover(node: dict):
@@ -739,7 +792,7 @@ def _should_run_historical_land_use_change_single_crop(
     capped_site_area = cap_values(dictionary=_scale_site_area_errors(site_area))
 
     sum_of_site_areas_is_100 = site_area_sum_to_100(capped_site_area)
-    site_type_allowed = site.get("siteType") in SITE_TYPES
+    site_type_allowed = site.get("siteType") in _ALLOWED_SITE_TYPES
 
     logRequirements(
         log_node=site,
@@ -787,9 +840,9 @@ def _no_prior_land_cover_data(nodes: list, target_node: dict) -> bool:
     target_date = (
         datetime.strptime(target_node.get('startDate') or target_node.get('endDate'),
                           DatestrFormat.YEAR_MONTH_DAY.value)
-        - timedelta(days=DEFAULT_WINDOW_IN_YEARS * DAYS_IN_YEAR)
+        - timedelta(days=_DEFAULT_WINDOW_IN_YEARS * DAYS_IN_YEAR)
     )
-    tolerance = timedelta(days=DATE_TOLERANCE_IN_YEARS * DAYS_IN_YEAR)
+    tolerance = timedelta(days=_DATE_TOLERANCE_IN_YEARS * DAYS_IN_YEAR)
     previous_nodes = [
         node for node in nodes
         if datetime.strptime(node.get("startDate"), DatestrFormat.YEAR_MONTH_DAY.value) - tolerance
@@ -800,16 +853,18 @@ def _no_prior_land_cover_data(nodes: list, target_node: dict) -> bool:
 
 
 def _should_run(site: dict) -> tuple[bool, list, dict]:
-    management_nodes = _collect_land_use_types(
-        [
-            node for node in filter_list_term_type(site.get("management", []), TermTermType.LANDCOVER)
-            if not _should_group_landCover(node)
-        ]
-    )
+    is_site_building = site.get('siteType') in _BUILDING_SITE_TYPES
+
+    allowed_land_use_types = _ALLOWED_LAND_USE_TYPES + ([OTHER_LAND] if is_site_building else [])
+
+    management_nodes = _collect_land_use_types([
+        node for node in filter_list_term_type(site.get("management", []), TermTermType.LANDCOVER)
+        if not _should_group_landCover(node)
+    ])
     relevant_nodes = sorted(
         [
             node for node in management_nodes
-            if node["land-use-type"] in ALLOWED_LAND_USE_TYPES
+            if node["land-use-type"] in allowed_land_use_types
         ],
         key=lambda n: n.get("startDate") or n.get("endDate")
     )
@@ -821,18 +876,32 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
         target_node=relevant_nodes[0]
     ) if relevant_nodes else None
 
-    should_run_nodes, site_area = _should_run_historical_land_use_change(
-        site=site,
-        nodes=relevant_nodes,
-        land_use_type=land_use_type
-    ) if all([land_use_type, has_no_prior_land_cover_data]) else (False, {})
+    landCover_from_lookups = get_landCover_lookups(
+        country_id=site.get("country", {}).get("@id"),
+        end_year=_get_year_from_landCover(relevant_nodes[0]),
+        product_name=relevant_nodes[0].get("term", {}).get("name", "")
+    ) if relevant_nodes else {}
+
+    should_run_nodes, site_area = (
+        (False, {}) if not all([land_use_type, has_no_prior_land_cover_data])
+        else (True, landCover_from_lookups) if landCover_from_lookups and all(landCover_from_lookups.values())
+        else (True, _COMPLETE_CHANGES_OTHER_LAND) if is_site_building
+        else _should_run_historical_land_use_change(
+            site=site,
+            nodes=relevant_nodes,
+            land_use_type=land_use_type
+        )
+    )
 
     logRequirements(site, model=MODEL, model_key=MODEL_KEY,
                     has_management_nodes=bool(relevant_nodes),
                     land_use_type=land_use_type,
-                    allowed_land_use_types=';'.join(ALLOWED_LAND_USE_TYPES),
+                    allowed_land_use_types=';'.join(allowed_land_use_types),
                     has_no_prior_land_cover_data=has_no_prior_land_cover_data,
                     management_nodes=log_as_table([_omit(n, ['term']) for n in relevant_nodes]),
+                    landCover_from_lookups=log_as_table([
+                        {land_type: value} for land_type, value in landCover_from_lookups.items()
+                    ]),
                     should_run_nodes=should_run_nodes)
 
     should_run = all([land_use_type, has_no_prior_land_cover_data, should_run_nodes])