hestia-earth-models 0.74.7__py3-none-any.whl → 0.74.9__py3-none-any.whl

hestia_earth/models/cache_sites.py

@@ -3,11 +3,11 @@ from enum import Enum
 from pydash.objects import merge
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import flatten, non_empty_list
+from hestia_earth.utils.term import download_term
 
 from .log import logger
 from .utils import CACHE_KEY, cached_value
 from .utils.site import CACHE_YEARS_KEY
-from .utils.term import download_term
 from .site.pre_checks.cache_geospatialDatabase import (
     list_vectors, list_rasters, cache_site_results, _should_run
 )

hestia_earth/models/data/ecoinventV3/__init__.py

@@ -4,15 +4,15 @@ from hestia_earth.models.log import logger
 
 _CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 _ENV_FOLDER = 'ECOINVENT_V3_FOLDER'
-_ECOINVENT_FOLDER = os.getenv(_ENV_FOLDER) or _CURRENT_DIR
+_DATA_FOLDER = os.getenv(_ENV_FOLDER) or _CURRENT_DIR
 _ECOINVENT_VERSION = os.getenv('ECOINVENT_VERSION', '3.9')
 
 
 def get_filepath(term_type: str):
     filename = f"ecoinventV{_ECOINVENT_VERSION.replace('.', '_')}-{term_type}.csv"
-    filepath = os.path.join(_ECOINVENT_FOLDER, filename)
+    filepath = os.path.join(_DATA_FOLDER, filename)
     if not os.path.exists(filepath):
-        logger.warning('Ecoinvent file not found. Please make sure to set env variable "%s".', _ENV_FOLDER)
+        logger.warning('%s file not found. Please make sure to set env variable "%s".', filename, _ENV_FOLDER)
         return None
 
     return filepath

hestia_earth/models/data/hestiaAggregatedData/__init__.py

@@ -5,8 +5,9 @@ from hestia_earth.utils.storage._s3_client import _load_from_bucket
 from hestia_earth.utils.api import _safe_get_request
 
 _CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+_ENV_FOLDER = 'HESTIA_AGGREGATED_DATA_FOLDER'
+_DATA_FOLDER = os.getenv(_ENV_FOLDER) or _CURRENT_DIR
 _FILENAME = 'hestiaAggregatedData.json'
-FILEPATH = os.path.join(_CURRENT_DIR, _FILENAME)
 _CACHED_DATA = {}
 DEFAULT_COUNTRY_ID = 'region-world'
 
@@ -23,16 +24,17 @@ def _download_data():
 
 def _load_data():
     data = None
+    filepath = os.path.join(_DATA_FOLDER, _FILENAME)
 
-    if os.path.exists(FILEPATH):
-        with open(FILEPATH, 'r') as f:
+    if os.path.exists(filepath):
+        with open(filepath, 'r') as f:
             data = json.load(f)
 
     is_data_valid = data and data['date'] == _today()
 
     if not is_data_valid:
         data = _download_data()
-        with open(FILEPATH, 'w') as f:
+        with open(filepath, 'w') as f:
             f.write(json.dumps(data))
 
     return data

hestia_earth/models/faostat2018/liveweightPerHead.py

@@ -2,10 +2,10 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import extract_grouped_data_closest_date
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import non_empty_list, safe_parse_date, safe_parse_float
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.constant import Units
-from hestia_earth.models.utils.term import download_term
 from hestia_earth.models.utils.property import _new_property, node_has_no_property
 from hestia_earth.models.utils.product import convert_product_to_unit
 from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_lookup_value

hestia_earth/models/faostat2018/product/price.py

@@ -1,10 +1,10 @@
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import extract_grouped_data
 from hestia_earth.utils.tools import non_empty_list, safe_parse_float, safe_parse_date
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
 from hestia_earth.models.utils.constant import Units
-from hestia_earth.models.utils.term import download_term
 from hestia_earth.models.utils.currency import DEFAULT_CURRENCY
 from hestia_earth.models.utils.crop import FAOSTAT_PRODUCTION_LOOKUP_COLUMN, get_crop_grouping_faostat_production
 from hestia_earth.models.utils.animalProduct import FAO_LOOKUP_COLUMN, get_animalProduct_grouping_fao

hestia_earth/models/geospatialDatabase/ecoClimateZone.py

@@ -77,4 +77,4 @@ def _should_run(site: dict):
     return should_run
 
 
-def run(site: dict): return _run(site) if _should_run(site) else _run_default(site)
+def run(site: dict): return (_run(site) if _should_run(site) else []) or _run_default(site)

hestia_earth/models/geospatialDatabase/region.py

@@ -1,9 +1,9 @@
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import linked_node
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
 from .utils import download, has_coordinates
-from hestia_earth.models.utils.term import download_term
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/geospatialDatabase/utils.py

@@ -4,10 +4,10 @@ from area import area
 from functools import reduce, lru_cache
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import non_empty_list
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import debugValues, logErrorRun, logRequirements
 from hestia_earth.models.utils.site import cached_value, region_factor, region_level_1_id
-from hestia_earth.models.utils.term import download_term
 from . import MODEL
 
 MAX_AREA_SIZE = int(os.getenv('MAX_AREA_SIZE', '5000'))

hestia_earth/models/globalCropWaterModel2008/rootingDepth.py

@@ -1,10 +1,11 @@
 from hestia_earth.schema import CycleFunctionalUnit, TermTermType
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, non_empty_list, safe_parse_float
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.property import _new_property, node_has_no_property
-from hestia_earth.models.utils.term import get_irrigation_terms, download_term
+from hestia_earth.models.utils.term import get_irrigation_terms
 from hestia_earth.models.utils.crop import get_crop_lookup_value
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from . import MODEL

hestia_earth/models/haversineFormula/transport/distance.py

@@ -1,10 +1,10 @@
 from haversine import haversine
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import non_empty_list
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
 from hestia_earth.models.utils.method import include_methodModel
-from hestia_earth.models.utils.term import download_term
 from .. import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/hestia/aboveGroundCropResidue.py

@@ -112,12 +112,10 @@ def _run(cycle: dict, total_values: list):
 
         if value == 0:
             values.extend([_product(term_id, value)])
-        elif remaining_value > 0 and value is not None and value >= 0:
+        elif remaining_value >= 0 and value is not None and value >= 0:
             value = value if value < remaining_value else remaining_value
             values.extend([_product(term_id, value)])
             remaining_value = remaining_value - value
-            if remaining_value == 0:
-                break
 
     return values + [
         # whatever remains is "left on field"

hestia_earth/models/hestia/cropResidueManagement.py

@@ -21,6 +21,7 @@ RETURNS = {
     }]
 }
 MODEL_KEY = 'cropResidueManagement'
+TERM_ID = 'residueBurnt,residueIncorporated,residueLeftOnField,residueRemoved,residueIncorporatedLessThan30DaysBeforeCultivation,residueIncorporatedMoreThan30DaysBeforeCultivation'  # noqa: E501
 PRACTICE_IDS = [
     residueBurnt.TERM_ID,
     residueIncorporated.TERM_ID,

hestia_earth/models/hestia/excretaKgMass.py

@@ -1,10 +1,10 @@
 from hestia_earth.schema import NodeType, TermTermType
 from hestia_earth.utils.model import filter_list_term_type, find_term_match
 from hestia_earth.utils.tools import non_empty_list, list_sum
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
 from hestia_earth.models.utils import get_kg_term_id, get_kg_N_term_id, get_kg_VS_term_id, _filter_list_term_unit
-from hestia_earth.models.utils.term import download_term
 from hestia_earth.models.utils.constant import Units
 from hestia_earth.models.utils.product import _new_product, convert_product_to_unit
 from . import MODEL

hestia_earth/models/hestia/landCover.py

@@ -10,7 +10,7 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import safe_parse_float, to_precision
 
 from hestia_earth.models.log import logRequirements, log_as_table, logShouldRun
-from hestia_earth.models.utils import _omit
+from hestia_earth.models.utils import _omit, clamp
 from hestia_earth.models.utils.constant import DAYS_IN_YEAR
 from hestia_earth.models.utils.management import _new_management
 from hestia_earth.models.utils.term import get_lookup_value
@@ -896,11 +896,15 @@ def _should_run_historical_land_use_change_single_crop(
         land_use_type=land_use_type,
         permanent_crops_net_expansion=permanent_crops_net_expansion
     )
+    capped_expansion_factor = clamp(
+        value=expansion_factor * e9_net_expansion * net_expansion_cultivated_vs_harvested,
+        min_value=0,
+        max_value=1
+    )
 
     site_area = {
-        land_type: (
-           shares_of_expansion[land_type] * expansion_factor * e9_net_expansion * net_expansion_cultivated_vs_harvested
-        ) for land_type in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
+        land_type: (shares_of_expansion[land_type] * capped_expansion_factor)
+        for land_type in LAND_USE_TERMS_FOR_TRANSFORMATION.keys()
         if land_type != land_use_type
     }
     site_area[land_use_type] = 1 - sum(site_area.values())
@@ -941,6 +945,9 @@ def _get_land_use_term_from_node(node: dict):
     return _get_lookup_with_cache(lookup_term=node.get("term", {}), column=LOOKUPS.get("landCover")[1])
 
 
+def _date_strip(date: str): return date[:10] if date else None
+
+
 def _collect_land_use_types(nodes: list) -> list:
     """Look up the land use type from management nodes."""
     return [
@@ -949,8 +956,8 @@ def _collect_land_use_types(nodes: list) -> list:
             "term": node.get("term", {}),
             "id": node.get("term", {}).get("@id"),
             "land-use-type": _get_land_use_term_from_node(node),
-            "endDate": _gapfill_datestr(datestr=node.get("endDate"), mode=DatestrGapfillMode.END)[:10],
-            "startDate": _gapfill_datestr(datestr=node.get("startDate"), mode=DatestrGapfillMode.START)[:10]
+            "endDate": _date_strip(_gapfill_datestr(datestr=node.get("endDate"), mode=DatestrGapfillMode.END)),
+            "startDate": _date_strip(_gapfill_datestr(datestr=node.get("startDate"), mode=DatestrGapfillMode.START))
         } for node in nodes
     ]
 

hestia_earth/models/hestia/landOccupationDuringCycle.py

@@ -212,7 +212,7 @@ def _format_inventory(inventory: list[SiteData], default: str = "None") -> str:
 
 def _should_run(impact_assessment: dict):
 
-    cycle = impact_assessment.get("cycle")
+    cycle = impact_assessment.get("cycle", {})
     functional_unit = cycle.get("functionalUnit")
 
     product = get_product(impact_assessment)

hestia_earth/models/hestia/management.py

@@ -286,7 +286,7 @@ def _dates_overlap(target_practice: dict, node: dict, cycle: dict, site_type_id:
     ])
 
 
-def _should_run_practice(management_nodes: list, cycle: dict, site_type_id: str):
+def _should_run_practice(site: dict, management_nodes: list, cycle: dict, site_type_id: str):
     """
     Include only landUseManagement practices where GAP_FILL_TO_MANAGEMENT = True
     """
@@ -297,21 +297,34 @@ def _should_run_practice(management_nodes: list, cycle: dict, site_type_id: str)
         for node in filter_list_term_type(management_nodes, TermTermType.LANDCOVER)
     ]
 
-    def run(practice: dict):
+    def exec(practice: dict):
         term = practice.get('term', {})
-        target_group = get_lookup_value(practice.get("term", {}), 'sumIs100Group', skip_debug=True, model=MODEL)
-        has_other_land_cover_in_same_group = next((
+        term_id = term['@id']
+        should_gap_fill = term.get('termType') != TermTermType.LANDUSEMANAGEMENT.value or _should_gap_fill(term)
+        target_group = get_lookup_value(term, 'sumIs100Group', skip_debug=True, model=MODEL)
+        no_other_land_cover_in_same_group = next((
             True for node in landCover_management_nodes
             if (
                 node['sumIs100Group'] == target_group and
                 _dates_overlap(target_practice=practice, node=node, cycle=cycle, site_type_id=site_type_id)
             )
-        ), None) is not None
-        return (
-            (term.get('termType') != TermTermType.LANDUSEMANAGEMENT.value or _should_gap_fill(term)) and
-            not has_other_land_cover_in_same_group
-        )
-    return run
+        ), None) is None
+        # cannot gap-fill landCover without a `startDate`
+        has_required_startDate = term.get('termType') != TermTermType.LANDCOVER.value or practice.get('startDate')
+
+        should_run = all([
+            should_gap_fill,
+            has_required_startDate,
+            no_other_land_cover_in_same_group
+        ])
+        if not should_run:
+            logRequirements(site, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                            should_gap_fill=should_gap_fill,
+                            has_required_startDate=has_required_startDate,
+                            no_other_land_cover_in_same_group=no_other_land_cover_in_same_group)
+            logShouldRun(site, MODEL, term_id, False, model_key=MODEL_KEY)
+        return should_run
+    return exec
 
 
 def _run_from_practices(site: dict, cycle: dict, site_type_id: str):
@@ -330,7 +343,8 @@ def _run_from_practices(site: dict, cycle: dict, site_type_id: str):
     ]
     management_nodes = site.get("management", [])
     return list(map(_map_to_value, filter(
-        _should_run_practice(management_nodes=management_nodes, cycle=cycle, site_type_id=site_type_id), practices
+        _should_run_practice(site, management_nodes, cycle, site_type_id),
+        practices
     )))
 
 

hestia_earth/models/hestia/pastureGrass.py

@@ -1,9 +1,9 @@
 from hestia_earth.schema import SiteSiteType, TermTermType
 from hestia_earth.utils.model import linked_node
+from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.practice import _new_practice
-from hestia_earth.models.utils.term import download_term
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/impact_assessment/post_checks/__init__.py

@@ -2,7 +2,7 @@ from os.path import dirname, abspath
 import sys
 
 from hestia_earth.models.utils import _run_in_serie
-from . import cycle, site, remove_cache_fields
+from . import cycle, site, remove_cache_fields, remove_no_value
 
 CURRENT_DIR = dirname(abspath(__file__)) + '/'
 sys.path.append(CURRENT_DIR)
@@ -10,7 +10,8 @@ sys.path.append(CURRENT_DIR)
 MODELS = [
     cycle.run,
     site.run,
-    remove_cache_fields.run
+    remove_cache_fields.run,
+    remove_no_value.run
 ]
 
 

hestia_earth/models/impact_assessment/post_checks/remove_no_value.py

@@ -0,0 +1,13 @@
+_KEYS = ['impacts', 'endpoints']
+
+
+def _has_value(blank_node: dict):
+    return blank_node.get('value') is not None
+
+
+def _filter_has_value(impact: dict, key: str):
+    return list(filter(_has_value, impact[key]))
+
+
+def run(impact: dict):
+    return impact | {key: _filter_has_value(impact, key) for key in _KEYS if impact.get(key)}

hestia_earth/models/ipcc2019/biocharOrganicCarbonPerHa.py

@@ -110,7 +110,7 @@ def _should_run(site: dict) -> tuple[bool, dict]:
     """
     cycles = related_cycles(site)
     site_type = site.get("siteType")
-    ipcc_soil_category = _assign_ipcc_soil_category(site.get("measurements", []))
+    ipcc_soil_category, soil_logs = _assign_ipcc_soil_category(site.get("measurements", []))
 
     has_cycles = len(cycles) > 0
     has_valid_site_type = site_type in _VALID_SITE_TYPES
@@ -140,6 +140,7 @@ def _should_run(site: dict) -> tuple[bool, dict]:
         should_compile_inventory=should_compile_inventory,
         seed=seed,
         inventory=_format_inventory(inventory),
+        **soil_logs,
         **_format_logs(logs)
     )
 

hestia_earth/models/ipcc2019/ch4ToAirExcreta.py

@@ -120,7 +120,10 @@ def _get_ch4_conv_factor(cycle: dict):
                 ecoClimateZone=ecoClimateZone,
                 practice_id=practice_id)
 
-    return _get_excretaManagement_MCF_from_lookup(practice_id, ecoClimateZone, duration_key) if practice_id else 0
+    return _get_excretaManagement_MCF_from_lookup(practice_id, ecoClimateZone, duration_key) if all([
+        practice_id,
+        ecoClimateZone is not None
+    ]) else 0
 
 
 def _should_run(cycle: dict):

hestia_earth/models/ipcc2019/organicCarbonPerHa.py

@@ -23,7 +23,11 @@ REQUIREMENTS = {
         ],
         "optional": {
             "measurements": [
-                {"@type": "Measurement", "value": "", "term.termType": ["soilType", "usdaSoilType"]}
+                {
+                    "@doc": "The model cannot run on sites with more than 30 percent organic soils (`histols`, `histosol` and their subclasses).",  # noqa: E501
+                    "@type": "Measurement", "value": "",
+                    "term.termType": ["soilType", "usdaSoilType"]
+                }
             ],
             "management": [
                 {