hestia-earth-models 0.74.8__tar.gz → 0.74.10__tar.gz

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_models
-Version: 0.74.8
+Version: 0.74.10
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/cache_sites.py

@@ -3,11 +3,11 @@ from enum import Enum
 from pydash.objects import merge
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import flatten, non_empty_list
+from hestia_earth.utils.term import download_term
 
 from .log import logger
 from .utils import CACHE_KEY, cached_value
 from .utils.site import CACHE_YEARS_KEY
-from .utils.term import download_term
 from .site.pre_checks.cache_geospatialDatabase import (
     list_vectors, list_rasters, cache_site_results, _should_run
 )

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/faostat2018/liveweightPerHead.py

@@ -2,10 +2,10 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import extract_grouped_data_closest_date
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list, safe_parse_date, safe_parse_float
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.constant import Units
-from hestia_earth.models.utils.term import download_term
 from hestia_earth.models.utils.property import _new_property, node_has_no_property
 from hestia_earth.models.utils.product import convert_product_to_unit
 from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_lookup_value

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/faostat2018/product/price.py

@@ -1,10 +1,10 @@
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import extract_grouped_data
 from hestia_earth.utils.tools import non_empty_list, safe_parse_float, safe_parse_date
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
 from hestia_earth.models.utils.constant import Units
-from hestia_earth.models.utils.term import download_term
 from hestia_earth.models.utils.currency import DEFAULT_CURRENCY
 from hestia_earth.models.utils.crop import FAOSTAT_PRODUCTION_LOOKUP_COLUMN, get_crop_grouping_faostat_production
 from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_grouping_fao

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/geospatialDatabase/ecoClimateZone.py

@@ -77,4 +77,4 @@ def _should_run(site: dict):
     return should_run
 
 
-def run(site: dict): return _run(site) if _should_run(site) else _run_default(site)
+def run(site: dict): return (_run(site) if _should_run(site) else []) or _run_default(site)

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/geospatialDatabase/region.py

@@ -1,9 +1,9 @@
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import linked_node
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
 from .utils import download, has_coordinates
-from hestia_earth.models.utils.term import download_term
 from . import MODEL
 
 REQUIREMENTS = {

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/geospatialDatabase/utils.py

@@ -4,10 +4,10 @@ from area import area
 from functools import reduce, lru_cache
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import non_empty_list
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import debugValues, logErrorRun, logRequirements
 from hestia_earth.models.utils.site import cached_value, region_factor, region_level_1_id
-from hestia_earth.models.utils.term import download_term
 from . import MODEL
 
 MAX_AREA_SIZE = int(os.getenv('MAX_AREA_SIZE', '5000'))

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/globalCropWaterModel2008/rootingDepth.py

@@ -1,10 +1,11 @@
 from hestia_earth.schema import CycleFunctionalUnit, TermTermType
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, non_empty_list, safe_parse_float
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.property import _new_property, node_has_no_property
-from hestia_earth.models.utils.term import get_irrigation_terms, download_term
+from hestia_earth.models.utils.term import get_irrigation_terms
 from hestia_earth.models.utils.crop import get_crop_lookup_value
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from . import MODEL

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/haversineFormula/transport/distance.py

@@ -1,10 +1,10 @@
 from haversine import haversine
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import non_empty_list
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
 from hestia_earth.models.utils.method import include_methodModel
-from hestia_earth.models.utils.term import download_term
 from .. import MODEL
 
 REQUIREMENTS = {

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/hestia/aboveGroundCropResidue.py

@@ -112,12 +112,10 @@ def _run(cycle: dict, total_values: list):
 
         if value == 0:
             values.extend([_product(term_id, value)])
-        elif remaining_value > 0 and value is not None and value >= 0:
+        elif remaining_value >= 0 and value is not None and value >= 0:
             value = value if value < remaining_value else remaining_value
             values.extend([_product(term_id, value)])
             remaining_value = remaining_value - value
-            if remaining_value == 0:
-                break
 
     return values + [
         # whatever remains is "left on field"

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/hestia/cropResidueManagement.py

@@ -21,6 +21,7 @@ RETURNS = {
     }]
 }
 MODEL_KEY = 'cropResidueManagement'
+TERM_ID = 'residueBurnt,residueIncorporated,residueLeftOnField,residueRemoved,residueIncorporatedLessThan30DaysBeforeCultivation,residueIncorporatedMoreThan30DaysBeforeCultivation'  # noqa: E501
 PRACTICE_IDS = [
     residueBurnt.TERM_ID,
     residueIncorporated.TERM_ID,

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/hestia/excretaKgMass.py

@@ -1,10 +1,10 @@
 from hestia_earth.schema import NodeType, TermTermType
 from hestia_earth.utils.model import filter_list_term_type, find_term_match
 from hestia_earth.utils.tools import non_empty_list, list_sum
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
 from hestia_earth.models.utils import get_kg_term_id, get_kg_N_term_id, get_kg_VS_term_id, _filter_list_term_unit
-from hestia_earth.models.utils.term import download_term
 from hestia_earth.models.utils.constant import Units
 from hestia_earth.models.utils.product import _new_product, convert_product_to_unit
 from . import MODEL

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/hestia/landCover.py

@@ -10,7 +10,7 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import safe_parse_float, to_precision
 
 from hestia_earth.models.log import logRequirements, log_as_table, logShouldRun
-from hestia_earth.models.utils import _omit
+from hestia_earth.models.utils import _omit, clamp
 from hestia_earth.models.utils.constant import DAYS_IN_YEAR
 from hestia_earth.models.utils.management import _new_management
 from hestia_earth.models.utils.term import get_lookup_value
@@ -896,11 +896,15 @@ def _should_run_historical_land_use_change_single_crop(
         land_use_type=land_use_type,
         permanent_crops_net_expansion=permanent_crops_net_expansion
     )
+    capped_expansion_factor = clamp(
+        value=expansion_factor * e9_net_expansion * net_expansion_cultivated_vs_harvested,
+        min_value=0,
+        max_value=1
+    )
 
     site_area = {
-        land_type: (
-           shares_of_expansion[land_type] * expansion_factor * e9_net_expansion * net_expansion_cultivated_vs_harvested
-        ) for land_type in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
+        land_type: (shares_of_expansion[land_type] * capped_expansion_factor)
+        for land_type in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
         if land_type != land_use_type
     }
     site_area[land_use_type] = 1 - sum(site_area.values())
@@ -941,6 +945,9 @@ def _get_land_use_term_from_node(node: dict):
     return _get_lookup_with_cache(lookup_term=node.get("term", {}), column=LOOKUPS.get("landCover")[1])
 
 
+def _date_strip(date: str): return date[:10] if date else None
+
+
 def _collect_land_use_types(nodes: list) -> list:
     """Look up the land use type from management nodes."""
     return [
@@ -949,8 +956,8 @@ def _collect_land_use_types(nodes: list) -> list:
             "term": node.get("term", {}),
             "id": node.get("term", {}).get("@id"),
             "land-use-type": _get_land_use_term_from_node(node),
-            "endDate": _gapfill_datestr(datestr=node.get("endDate"), mode=DatestrGapfillMode.END)[:10],
-            "startDate": _gapfill_datestr(datestr=node.get("startDate"), mode=DatestrGapfillMode.START)[:10]
+            "endDate": _date_strip(_gapfill_datestr(datestr=node.get("endDate"), mode=DatestrGapfillMode.END)),
+            "startDate": _date_strip(_gapfill_datestr(datestr=node.get("startDate"), mode=DatestrGapfillMode.START))
         } for node in nodes
     ]
 

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/hestia/landOccupationDuringCycle.py

@@ -212,7 +212,7 @@ def _format_inventory(inventory: list[SiteData], default: str = "None") -> str:
 
 def _should_run(impact_assessment: dict):
 
-    cycle = impact_assessment.get("cycle")
+    cycle = impact_assessment.get("cycle", {})
     functional_unit = cycle.get("functionalUnit")
 
     product = get_product(impact_assessment)

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/hestia/management.py

@@ -286,7 +286,7 @@ def _dates_overlap(target_practice: dict, node: dict, cycle: dict, site_type_id:
     ])
 
 
-def _should_run_practice(management_nodes: list, cycle: dict, site_type_id: str):
+def _should_run_practice(site: dict, management_nodes: list, cycle: dict, site_type_id: str):
     """
     Include only landUseManagement practices where GAP_FILL_TO_MANAGEMENT = True
     """
@@ -297,21 +297,34 @@ def _should_run_practice(management_nodes: list, cycle: dict, site_type_id: str)
         for node in filter_list_term_type(management_nodes, TermTermType.LANDCOVER)
     ]
 
-    def run(practice: dict):
+    def exec(practice: dict):
         term = practice.get('term', {})
-        target_group = get_lookup_value(practice.get("term", {}), 'sumIs100Group', skip_debug=True, model=MODEL)
-        has_other_land_cover_in_same_group = next((
+        term_id = term['@id']
+        should_gap_fill = term.get('termType') != TermTermType.LANDUSEMANAGEMENT.value or _should_gap_fill(term)
+        target_group = get_lookup_value(term, 'sumIs100Group', skip_debug=True, model=MODEL)
+        no_other_land_cover_in_same_group = next((
             True for node in landCover_management_nodes
             if (
                 node['sumIs100Group'] == target_group and
                 _dates_overlap(target_practice=practice, node=node, cycle=cycle, site_type_id=site_type_id)
             )
-        ), None) is not None
-        return (
-            (term.get('termType') != TermTermType.LANDUSEMANAGEMENT.value or _should_gap_fill(term)) and
-            not has_other_land_cover_in_same_group
-        )
-    return run
+        ), None) is None
+        # cannot gap-fill landCover without a `startDate`
+        has_required_startDate = term.get('termType') != TermTermType.LANDCOVER.value or practice.get('startDate')
+
+        should_run = all([
+            should_gap_fill,
+            has_required_startDate,
+            no_other_land_cover_in_same_group
+        ])
+        if not should_run:
+            logRequirements(site, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                            should_gap_fill=should_gap_fill,
+                            has_required_startDate=has_required_startDate,
+                            no_other_land_cover_in_same_group=no_other_land_cover_in_same_group)
+            logShouldRun(site, MODEL, term_id, False, model_key=MODEL_KEY)
+        return should_run
+    return exec
 
 
 def _run_from_practices(site: dict, cycle: dict, site_type_id: str):
@@ -330,7 +343,8 @@ def _run_from_practices(site: dict, cycle: dict, site_type_id: str):
     ]
     management_nodes = site.get("management", [])
     return list(map(_map_to_value, filter(
-        _should_run_practice(management_nodes=management_nodes, cycle=cycle, site_type_id=site_type_id), practices
+        _should_run_practice(site, management_nodes, cycle, site_type_id),
+        practices
     )))
 
 

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/hestia/pastureGrass.py

@@ -1,9 +1,9 @@
 from hestia_earth.schema import SiteSiteType, TermTermType
 from hestia_earth.utils.model import linked_node
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.practice import _new_practice
-from hestia_earth.models.utils.term import download_term
 from . import MODEL
 
 REQUIREMENTS = {

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/impact_assessment/post_checks/__init__.py

@@ -2,7 +2,7 @@ from os.path import dirname, abspath
 import sys
 
 from hestia_earth.models.utils import _run_in_serie
-from . import cycle, site, remove_cache_fields
+from . import cycle, site, remove_cache_fields, remove_no_value
 
 CURRENT_DIR = dirname(abspath(__file__)) + '/'
 sys.path.append(CURRENT_DIR)
@@ -10,7 +10,8 @@ sys.path.append(CURRENT_DIR)
 MODELS = [
     cycle.run,
     site.run,
-    remove_cache_fields.run
+    remove_cache_fields.run,
+    remove_no_value.run
 ]
 
 

hestia_earth_models-0.74.10/hestia_earth/models/impact_assessment/post_checks/remove_no_value.py

@@ -0,0 +1,13 @@
+_KEYS = ['impacts', 'endpoints']
+
+
+def _has_value(blank_node: dict):
+    return blank_node.get('value') is not None
+
+
+def _filter_has_value(impact: dict, key: str):
+    return list(filter(_has_value, impact[key]))
+
+
+def run(impact: dict):
+    return impact | {key: _filter_has_value(impact, key) for key in _KEYS if impact.get(key)}

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/ipcc2019/biocharOrganicCarbonPerHa.py

@@ -110,7 +110,7 @@ def _should_run(site: dict) -> tuple[bool, dict]:
     """
     cycles = related_cycles(site)
     site_type = site.get("siteType")
-    ipcc_soil_category = _assign_ipcc_soil_category(site.get("measurements", []))
+    ipcc_soil_category, soil_logs = _assign_ipcc_soil_category(site.get("measurements", []))
 
     has_cycles = len(cycles) > 0
     has_valid_site_type = site_type in _VALID_SITE_TYPES
@@ -140,6 +140,7 @@ def _should_run(site: dict) -> tuple[bool, dict]:
         should_compile_inventory=should_compile_inventory,
         seed=seed,
         inventory=_format_inventory(inventory),
+        **soil_logs,
         **_format_logs(logs)
     )
 

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/ipcc2019/ch4ToAirExcreta.py

@@ -7,7 +7,6 @@ from hestia_earth.utils.tools import safe_parse_float, list_sum
 from hestia_earth.models.log import debugValues, logRequirements, debugMissingLookup, logShouldRun, log_as_table
 from hestia_earth.models.utils import _filter_list_term_unit
 from hestia_earth.models.utils.constant import Units, DAYS_PER_MONTH
-from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.productivity import PRODUCTIVITY, get_productivity
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.measurement import most_relevant_measurement_value
@@ -17,7 +16,6 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "completeness.excreta": "",
         "cycleDuration": "",
         "endDate": "",
         "practices": [{"@type": "Practice", "value": "", "term.termType": "excretaManagement"}],
@@ -75,7 +73,7 @@ def _emission(value: float):
 
 
 def _run(excreta_b0_product: float, ch4_conv_factor: float):
-    value = excreta_b0_product * 0.67 * ch4_conv_factor / 100
+    value = (excreta_b0_product or 0) * 0.67 * (ch4_conv_factor or 0) / 100
     return [_emission(value)]
 
 
@@ -120,12 +118,15 @@ def _get_ch4_conv_factor(cycle: dict):
                 ecoClimateZone=ecoClimateZone,
                 practice_id=practice_id)
 
-    return _get_excretaManagement_MCF_from_lookup(practice_id, ecoClimateZone, duration_key) if practice_id else 0
+    return _get_excretaManagement_MCF_from_lookup(practice_id, ecoClimateZone, duration_key) if all([
+        practice_id,
+        ecoClimateZone is not None
+    ]) else None
 
 
 def _should_run(cycle: dict):
     country = cycle.get('site', {}).get('country', {})
-    excreta_complete = _is_term_type_complete(cycle, TermTermType.EXCRETA)
+
     # total of excreta including the CH4 factor
     excreta = filter_list_term_type(cycle.get('inputs', []), TermTermType.EXCRETA)
     excreta = _filter_list_term_unit(excreta, Units.KG_VS)
@@ -135,21 +136,23 @@ def _should_run(cycle: dict):
     excreta_logs = log_as_table([
         {'id': id, 'value': v, 'b0': b0} for id, v, b0 in excreta_values
     ])
-    excreta_b0_product = list_sum([v * f for id, v, f in excreta_values if v is not None and f is not None])
+    excreta_total = list_sum([
+        v * f for id, v, f in excreta_values if v is not None and f is not None
+    ], default=None)
 
     ch4_conv_factor = _get_ch4_conv_factor(cycle)
 
     logRequirements(cycle, model=MODEL, term=TERM_ID,
-                    term_type_excreta_complete=excreta_complete,
-                    excreta=excreta_logs,
-                    ch4_conv_factor=ch4_conv_factor,
+                    excreta_details=excreta_logs,
+                    excreta_total=excreta_total,
+                    CH4_conv_factor=ch4_conv_factor,
                     country=country.get('@id'))
 
-    should_run = excreta_complete or all([excreta_b0_product, ch4_conv_factor])
+    should_run = all([excreta_total is not None, ch4_conv_factor is not None])
     logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
-    return should_run, excreta_b0_product, ch4_conv_factor
+    return should_run, excreta_total, ch4_conv_factor
 
 
 def run(cycle: dict):
-    should_run, excreta_b0_product, ch4_conv_factor = _should_run(cycle)
-    return _run(excreta_b0_product, ch4_conv_factor) if should_run else []
+    should_run, excreta_total, ch4_conv_factor = _should_run(cycle)
+    return _run(excreta_total, ch4_conv_factor) if should_run else []

{hestia_earth_models-0.74.8 → hestia_earth_models-0.74.10}/hestia_earth/models/ipcc2019/organicCarbonPerHa.py

@@ -23,7 +23,11 @@ REQUIREMENTS = {
         ],
         "optional": {
             "measurements": [
-                {"@type": "Measurement", "value": "", "term.termType": ["soilType", "usdaSoilType"]}
+                {
+                    "@doc": "The model cannot run on sites with more than 30 percent organic soils (`histols`, `histosol` and their subclasses).",  # noqa: E501
+                    "@type": "Measurement", "value": "",
+                    "term.termType": ["soilType", "usdaSoilType"]
+                }
             ],
             "management": [
                 {