hestia-earth-models 0.73.5__tar.gz → 0.73.7__tar.gz

{hestia_earth_models-0.73.5 → hestia_earth_models-0.73.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_models
-Version: 0.73.5
+Version: 0.73.7
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

{hestia_earth_models-0.73.5 → hestia_earth_models-0.73.7}/hestia_earth/models/config/Cycle.json

@@ -2093,51 +2093,6 @@
         },
         "stage": 2
       },
-      {
-        "key": "emissions",
-        "model": "ipcc2006",
-        "value": "n2OToAirCropResidueDecompositionIndirect",
-        "runStrategy": "add_blank_node_if_missing",
-        "runArgs": {
-          "runNonMeasured": true,
-          "runNonAddedTerm": true
-        },
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 2
-      },
-      {
-        "key": "emissions",
-        "model": "ipcc2006",
-        "value": "n2OToAirCropResidueDecompositionDirect",
-        "runStrategy": "add_blank_node_if_missing",
-        "runArgs": {
-          "runNonMeasured": true,
-          "runNonAddedTerm": true
-        },
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 2
-      },
-      {
-        "key": "emissions",
-        "model": "ipcc2006",
-        "value": "n2OToAirExcretaIndirect",
-        "runStrategy": "add_blank_node_if_missing",
-        "runArgs": {
-          "runNonMeasured": true,
-          "runNonAddedTerm": true
-        },
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 2
-      },
       {
         "key": "emissions",
         "model": "jarvisAndPain1994",

{hestia_earth_models-0.73.5 → hestia_earth_models-0.73.7}/hestia_earth/models/config/ImpactAssessment.json

@@ -203,30 +203,6 @@
         "stage": 1
       }
     ],
-    [
-      {
-        "key": "emissionsResourceUse",
-        "model": "faostat2018",
-        "value": "landTransformation20YearAverageDuringCycle",
-        "runStrategy": "always",
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 1
-      },
-      {
-        "key": "emissionsResourceUse",
-        "model": "faostat2018",
-        "value": "landTransformation100YearAverageDuringCycle",
-        "runStrategy": "always",
-        "mergeStrategy": "list",
-        "mergeArgs": {
-          "replaceThreshold": ["value", 0.01]
-        },
-        "stage": 1
-      }
-    ],
     {
       "key": "emissionsResourceUse",
       "model": "resourceUseNotRelevant",

{hestia_earth_models-0.73.5 → hestia_earth_models-0.73.7}/hestia_earth/models/geospatialDatabase/potentialEvapotranspirationAnnual.py

@@ -2,6 +2,7 @@ from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import max_date
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.site import related_years
 from .utils import download, has_geospatial_data, should_download
@@ -39,7 +40,7 @@ def _measurement(value: float, year: int):
     measurement['value'] = [value]
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     measurement['startDate'] = f"{year}-01-01"
-    measurement['endDate'] = f"{year}-12-31"
+    measurement['endDate'] = max_date(f"{year}-12-31")
     return measurement
 
 

{hestia_earth_models-0.73.5 → hestia_earth_models-0.73.7}/hestia_earth/models/geospatialDatabase/precipitationAnnual.py

@@ -2,6 +2,7 @@ from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import max_date
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from hestia_earth.models.utils.site import related_years
@@ -41,7 +42,7 @@ def _measurement(site: dict, value: float, year: int):
     measurement['value'] = [value]
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     measurement['startDate'] = f"{year}-01-01"
-    measurement['endDate'] = f"{year}-12-31"
+    measurement['endDate'] = max_date(f"{year}-12-31")
     return measurement | get_source(site, BIBLIO_TITLE)
 
 

{hestia_earth_models-0.73.5 → hestia_earth_models-0.73.7}/hestia_earth/models/geospatialDatabase/temperatureAnnual.py

@@ -2,6 +2,7 @@ from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import max_date
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from hestia_earth.models.utils.site import related_years
@@ -40,7 +41,7 @@ def _measurement(site: dict, value: float, year: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
     measurement['startDate'] = f"{year}-01-01"
-    measurement['endDate'] = f"{year}-12-31"
+    measurement['endDate'] = max_date(f"{year}-12-31")
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 

{hestia_earth_models-0.73.5 → hestia_earth_models-0.73.7}/hestia_earth/models/hestia/landCover.py

@@ -2,7 +2,6 @@ import functools
 import math
 from collections import defaultdict
 from datetime import datetime, timedelta
-
 from hestia_earth.schema import SiteSiteType, TermTermType
 from hestia_earth.utils.lookup import (
     download_lookup, get_table_value, column_name, _is_missing_value, extract_grouped_data, lookup_columns
@@ -15,7 +14,7 @@ from hestia_earth.models.utils import _omit
 from hestia_earth.models.utils.constant import DAYS_IN_YEAR
 from hestia_earth.models.utils.management import _new_management
 from hestia_earth.models.utils.term import get_lookup_value
-from hestia_earth.models.utils.lookup import get_region_lookup_value
+from hestia_earth.models.utils.lookup import get_region_lookup, get_region_lookup_value
 from hestia_earth.models.utils.blank_node import DatestrFormat, _gapfill_datestr, DatestrGapfillMode
 from .utils import (
     IPCC_LAND_USE_CATEGORY_ANNUAL,
@@ -87,18 +86,30 @@ LOOKUPS = {
 }
 MODEL_KEY = 'landCover'
 
-LAND_AREA = LOOKUPS["region-faostatArea"][3]
-SITE_TYPES = {
+_LAND_AREA = LOOKUPS["region-faostatArea"][3]
+_ALLOWED_SITE_TYPES = {
     SiteSiteType.CROPLAND.value,
     SiteSiteType.FOREST.value,
     SiteSiteType.OTHER_NATURAL_VEGETATION.value,
     SiteSiteType.PERMANENT_PASTURE.value
 }
-DEFAULT_WINDOW_IN_YEARS = 20
-DATE_TOLERANCE_IN_YEARS = 2
-OUTPUT_SIGNIFICANT_DIGITS = 3
-ALLOWED_LAND_USE_TYPES = [ANNUAL_CROPLAND, PERMANENT_CROPLAND, PERMANENT_PASTURE]
+_BUILDING_SITE_TYPES = [
+    SiteSiteType.AGRI_FOOD_PROCESSOR.value,
+    SiteSiteType.ANIMAL_HOUSING.value,
+    SiteSiteType.FOOD_RETAILER.value
+]
+_DEFAULT_WINDOW_IN_YEARS = 20
+_DATE_TOLERANCE_IN_YEARS = 2
+_OUTPUT_SIGNIFICANT_DIGITS = 3
+_ALLOWED_LAND_USE_TYPES = [ANNUAL_CROPLAND, PERMANENT_CROPLAND, PERMANENT_PASTURE]
 _LOOKUP_EXPANSION = "region-crop-cropGroupingFaostatProduction-areaHarvestedUpTo20YearExpansion.csv"
+_COMPLETE_CHANGES_OTHER_LAND = {
+    OTHER_LAND: 1,
+    FOREST_LAND: 0,
+    PERMANENT_PASTURE: 0,
+    ANNUAL_CROPLAND: 0,
+    PERMANENT_CROPLAND: 0
+}
 
 
 def _get_lookup_with_cache(lookup_term, column):
@@ -121,6 +132,10 @@ def _get_lookup_with_cache(lookup_term, column):
     )
 
 
+def _get_land_cover_lookup_suffix(land_type: str) -> str:
+    return LAND_USE_TERMS_FOR_TRANSFORMATION[land_type][0]
+
+
 def get_landCover_lookups(country_id: str, end_year: int, product_name: str):
     """
     Attempts to get the pre-calculated values for the landCover model calculation.
@@ -128,15 +143,16 @@ def get_landCover_lookups(country_id: str, end_year: int, product_name: str):
               "Permanent crops": <value>, "Permanent meadows and pastures": <value>}
     Missing values are returned as None.
     """
+    lookup_prefix = 'region-crop-cropGroupingFAOSTAT-landCover'
     return {
         # Divide by 100 to match site_area ratios
-        suffix: value / 100 if value is not None else value
-        for suffix, value in
+        land_type: value / 100 if value is not None else value
+        for land_type, value in
         {
-            suffix: safe_parse_float(
+            land_type: safe_parse_float(
                 value=extract_grouped_data(
                     data=get_region_lookup_value(
-                        lookup_name=f"region-crop-cropGroupingFAOSTAT-landCover-{suffix}.csv",
+                        lookup_name=f"{lookup_prefix}-{_get_land_cover_lookup_suffix(land_type)}.csv",
                         term_id=country_id,
                         column=product_name,
                         model=MODEL,
@@ -146,7 +162,7 @@ def get_landCover_lookups(country_id: str, end_year: int, product_name: str):
                 ),
                 default=None
             )
-            for suffix in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
+            for land_type in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
         }.items()
     }
 
@@ -211,7 +227,7 @@ def _get_changes(country_id: str, end_year: int) -> tuple[dict, bool]:
             ),
             default=None
         )
-        for land_use_term in ALL_LAND_USE_TERMS + [LAND_AREA]
+        for land_use_term in ALL_LAND_USE_TERMS + [_LAND_AREA]
     }
     missing_changes = [k for k, v in changes_dict.items() if v is None]
     changes_dict = {k: v if v is not None else 0 for k, v in changes_dict.items()}
@@ -302,7 +318,7 @@ def _allocate_pasture_loss_to_cropland(changes: dict, land_required_for_cropland
 
 
 def _allocate_other_land(
-        changes: dict, max_forest_loss_to: dict, pasture_loss_to_cropland: float, cropland_loss_to_pasture: float
+    changes: dict, max_forest_loss_to: dict, pasture_loss_to_cropland: float, cropland_loss_to_pasture: float
 ) -> dict:
     """Allocate changes between Other land and cropland"""
     other_land_loss_to_cropland = (
@@ -496,7 +512,7 @@ def _get_term_id_for_crop(nodes: set, land_type: str) -> str:
 
 
 def _run(site: dict, existing_nodes: list, percentage_transformed_from: dict):
-    start_year = _get_year_from_landCover(existing_nodes[0]) - DEFAULT_WINDOW_IN_YEARS
+    start_year = _get_year_from_landCover(existing_nodes[0]) - _DEFAULT_WINDOW_IN_YEARS
 
     """Creates a list of new management nodes, excluding any dates matching existing ones."""
     existing_nodes_set = {
@@ -511,7 +527,7 @@ def _run(site: dict, existing_nodes: list, percentage_transformed_from: dict):
             "land_type": land_type,
             "percentage": 0 if ratio == -0.0 else to_precision(
                 number=ratio * 100,
-                digits=OUTPUT_SIGNIFICANT_DIGITS
+                digits=_OUTPUT_SIGNIFICANT_DIGITS
             ),
             "term_id": _get_term_id_for_crop(existing_nodes_set, land_type=land_type)
         }
@@ -570,10 +586,10 @@ def _get_sums_of_crop_expansion(country_id: str, year: int, include_negatives: b
     Sum net expansion for all annual and permanent crops, returned as two values.
     Returns a tuple of (expansion of annual crops, expansion of permanent crops)
     """
-    lookup = download_lookup(_LOOKUP_EXPANSION.replace('region', f"{country_id}")) or download_lookup(_LOOKUP_EXPANSION)
+    lookup = get_region_lookup(lookup_name=_LOOKUP_EXPANSION, term_id=country_id)
     columns = lookup_columns(lookup)
     values = {
-        name: get_region_lookup_value(lookup_name=_LOOKUP_EXPANSION, term_id=country_id, column=name, model=MODEL)
+        name: get_table_value(lookup, 'termid', country_id, column_name(name))
         for name in columns if name != "termid"
     }
 
@@ -600,15 +616,17 @@ def _get_sums_of_crop_expansion(country_id: str, year: int, include_negatives: b
 def _get_net_expansion_cultivated_vs_harvested(
     annual_crops_net_expansion, changes, land_use_type, permanent_crops_net_expansion
 ):
-    if land_use_type == ANNUAL_CROPLAND:
-        net_expansion_cultivated_vs_harvested = _safe_divide(numerator=max(0, changes[ANNUAL_CROPLAND]),
-                                                             denominator=(annual_crops_net_expansion / 1000))
-    elif land_use_type == PERMANENT_CROPLAND:
-        net_expansion_cultivated_vs_harvested = _safe_divide(numerator=max(0, changes[PERMANENT_CROPLAND]),
-                                                             denominator=(permanent_crops_net_expansion / 1000))
-    else:
-        net_expansion_cultivated_vs_harvested = 1
-    return net_expansion_cultivated_vs_harvested
+    return (
+        _safe_divide(
+            numerator=max(0, changes[ANNUAL_CROPLAND]),
+            denominator=(annual_crops_net_expansion / 1000)
+        ) if land_use_type == ANNUAL_CROPLAND else
+        _safe_divide(
+            numerator=max(0, changes[PERMANENT_CROPLAND]),
+            denominator=(permanent_crops_net_expansion / 1000)
+        ) if land_use_type == PERMANENT_CROPLAND else
+        1
+    )
 
 
 def _get_year_from_landCover(node: dict):
@@ -774,7 +792,7 @@ def _should_run_historical_land_use_change_single_crop(
     capped_site_area = cap_values(dictionary=_scale_site_area_errors(site_area))
 
     sum_of_site_areas_is_100 = site_area_sum_to_100(capped_site_area)
-    site_type_allowed = site.get("siteType") in SITE_TYPES
+    site_type_allowed = site.get("siteType") in _ALLOWED_SITE_TYPES
 
     logRequirements(
         log_node=site,
@@ -822,9 +840,9 @@ def _no_prior_land_cover_data(nodes: list, target_node: dict) -> bool:
     target_date = (
         datetime.strptime(target_node.get('startDate') or target_node.get('endDate'),
                           DatestrFormat.YEAR_MONTH_DAY.value)
-        - timedelta(days=DEFAULT_WINDOW_IN_YEARS * DAYS_IN_YEAR)
+        - timedelta(days=_DEFAULT_WINDOW_IN_YEARS * DAYS_IN_YEAR)
     )
-    tolerance = timedelta(days=DATE_TOLERANCE_IN_YEARS * DAYS_IN_YEAR)
+    tolerance = timedelta(days=_DATE_TOLERANCE_IN_YEARS * DAYS_IN_YEAR)
     previous_nodes = [
         node for node in nodes
         if datetime.strptime(node.get("startDate"), DatestrFormat.YEAR_MONTH_DAY.value) - tolerance
@@ -835,16 +853,18 @@ def _no_prior_land_cover_data(nodes: list, target_node: dict) -> bool:
 
 
 def _should_run(site: dict) -> tuple[bool, list, dict]:
-    management_nodes = _collect_land_use_types(
-        [
-            node for node in filter_list_term_type(site.get("management", []), TermTermType.LANDCOVER)
-            if not _should_group_landCover(node)
-        ]
-    )
+    is_site_building = site.get('siteType') in _BUILDING_SITE_TYPES
+
+    allowed_land_use_types = _ALLOWED_LAND_USE_TYPES + ([OTHER_LAND] if is_site_building else [])
+
+    management_nodes = _collect_land_use_types([
+        node for node in filter_list_term_type(site.get("management", []), TermTermType.LANDCOVER)
+        if not _should_group_landCover(node)
+    ])
     relevant_nodes = sorted(
         [
             node for node in management_nodes
-            if node["land-use-type"] in ALLOWED_LAND_USE_TYPES
+            if node["land-use-type"] in allowed_land_use_types
         ],
         key=lambda n: n.get("startDate") or n.get("endDate")
     )
@@ -865,6 +885,7 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
     should_run_nodes, site_area = (
         (False, {}) if not all([land_use_type, has_no_prior_land_cover_data])
         else (True, landCover_from_lookups) if landCover_from_lookups and all(landCover_from_lookups.values())
+        else (True, _COMPLETE_CHANGES_OTHER_LAND) if is_site_building
         else _should_run_historical_land_use_change(
             site=site,
             nodes=relevant_nodes,
@@ -875,10 +896,12 @@ def _should_run(site: dict) -> tuple[bool, list, dict]:
     logRequirements(site, model=MODEL, model_key=MODEL_KEY,
                     has_management_nodes=bool(relevant_nodes),
                     land_use_type=land_use_type,
-                    allowed_land_use_types=';'.join(ALLOWED_LAND_USE_TYPES),
+                    allowed_land_use_types=';'.join(allowed_land_use_types),
                     has_no_prior_land_cover_data=has_no_prior_land_cover_data,
                     management_nodes=log_as_table([_omit(n, ['term']) for n in relevant_nodes]),
-                    landCover_from_lookups=landCover_from_lookups,
+                    landCover_from_lookups=log_as_table([
+                        {land_type: value} for land_type, value in landCover_from_lookups.items()
+                    ]),
                     should_run_nodes=should_run_nodes)
 
     should_run = all([land_use_type, has_no_prior_land_cover_data, should_run_nodes])

{hestia_earth_models-0.73.5 → hestia_earth_models-0.73.7}/hestia_earth/models/hestia/pastureSystem.py

@@ -41,7 +41,7 @@ def _practice(term_id: str):
 def _run(cycle: dict):
     end_date = cycle.get('endDate')
     measurements = cycle.get('site', {}).get('measurements', [])
-    slope = measurement_value(most_relevant_blank_node_by_id(measurements, 'slope', end_date))
+    slope = measurement_value(most_relevant_blank_node_by_id(measurements, 'slope', end_date), default=0)
     term_id = 'confinedPastureSystem' if slope <= 2.5 else 'hillyPastureSystem'
 
     debugValues(cycle, model=MODEL, term=term_id,