hestia-earth-models 0.63.0__py3-none-any.whl → 0.64.0__py3-none-any.whl

hestia_earth/models/geospatialDatabase/totalNitrogenPerKgSoil.py

@@ -1,5 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
-from hestia_earth.utils.tools import safe_parse_float
+from hestia_earth.utils.tools import safe_parse_float, non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
@@ -19,33 +19,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "50",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'totalNitrogenPerKgSoil'
-EE_PARAMS = {
-    'collection': 'T_N',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized soil property values for broad-scale modelling (WISE30sec) '
-'with estimates of global soil carbon stocks'
+EE_PARAMS = [
+    {
+        'collection': 'T_N_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_N_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_N_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_N_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: float):
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 50
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, safe_parse_float(value), params.get('depthUpper'), params.get('depthLower'))
+    )
+
+
 def _run(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return [_measurement(site, safe_parse_float(value))] if value is not None else []
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):

hestia_earth/models/geospatialDatabase/utils.py

@@ -134,13 +134,19 @@ def _parse_run_query(term: str, query: dict):
         return None
 
 
+def _cache_sub_key(collection: dict):
+    return '-'.join(non_empty_list([
+        str(collection.get('year', '')),
+        str(collection.get('start_date', '')),
+        str(collection.get('end_date', '')),
+        str(collection.get('depthUpper', '')),
+        str(collection.get('depthLower', ''))
+    ]))
+
+
 def _get_cached_data(term: str, site: dict, data: dict):
     cache = _cached_value(site, term)
-    cache_sub_key = '-'.join(non_empty_list([
-        data.get('year'),
-        data.get('start_date'),
-        data.get('end_date')
-    ]))
+    cache_sub_key = _cache_sub_key(data)
     # data can be grouped by year when required
     value = cache.get(cache_sub_key) if (
         isinstance(cache, dict) and cache_sub_key in cache and cache is not None

hestia_earth/models/hyde32/landTransformationFromCropland100YearAverageDuringCycle.py

@@ -50,7 +50,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromCropland100YearAverageDuringCycle'

hestia_earth/models/hyde32/landTransformationFromCropland20YearAverageDuringCycle.py

@@ -50,7 +50,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromCropland20YearAverageDuringCycle'

hestia_earth/models/hyde32/landTransformationFromForest100YearAverageDuringCycle.py

@@ -50,7 +50,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromForest100YearAverageDuringCycle'

hestia_earth/models/hyde32/landTransformationFromForest20YearAverageDuringCycle.py

@@ -50,7 +50,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromForest20YearAverageDuringCycle'

hestia_earth/models/hyde32/landTransformationFromOtherNaturalVegetation100YearAverageDuringCycle.py

@@ -50,7 +50,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromOtherNaturalVegetation100YearAverageDuringCycle'

hestia_earth/models/hyde32/landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle.py

@@ -50,7 +50,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle'

hestia_earth/models/hyde32/landTransformationFromPermanentPasture100YearAverageDuringCycle.py

@@ -50,7 +50,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromPermanentPasture100YearAverageDuringCycle'

hestia_earth/models/hyde32/landTransformationFromPermanentPasture20YearAverageDuringCycle.py

@@ -50,7 +50,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromPermanentPasture20YearAverageDuringCycle'

hestia_earth/models/hyde32/utils.py

@@ -6,6 +6,7 @@ from hestia_earth.models.log import debugMissingLookup, debugValues, logRequirem
 from hestia_earth.models.utils.indicator import _new_indicator
 from hestia_earth.models.utils.impact_assessment import get_product, get_site, get_region_id
 from hestia_earth.models.utils.cycle import land_occupation_per_kg
+from hestia_earth.models.utils.site import get_land_cover_term_id
 from . import MODEL
 
 
@@ -28,17 +29,19 @@ def _get_emission_factor(term_id: str, impact_assessment: dict, average_years: s
     return safe_parse_float(extract_grouped_data_closest_date(value, end_date.year), None) if end_date else None
 
 
-def _indicator(term_id: str, value: float):
-    indicator = _new_indicator(term_id, MODEL)
+def _indicator(term_id: str, value: float, land_covert_term_id: str):
+    indicator = _new_indicator(term_id, MODEL, land_covert_term_id)
     indicator['value'] = value
     return indicator
 
 
 def _run(impact_assessment: dict, term_id: str, land_occupation_m2: float, factor: float):
+    land_covert_term_id = get_land_cover_term_id(get_site(impact_assessment).get('siteType'))
     value = land_occupation_m2 * (factor or 0)
     debugValues(impact_assessment, model=MODEL, term=term_id,
-                value=value)
-    return _indicator(term_id, value)
+                value=value,
+                land_covert_term_id=land_covert_term_id)
+    return _indicator(term_id, value, land_covert_term_id)
 
 
 def _should_run(impact_assessment: dict, term_id: str, from_site_type: SiteSiteType, years: int):

hestia_earth/models/impact_assessment/allocationMethod.py

@@ -0,0 +1,39 @@
+"""
+Allocation Method
+
+If any of the impact `emissionsResourceUse`, `impacts`, or `endpoints`, have been modified by the calculations,
+the `allocationMethod` will be set to `economic`.
+"""
+from hestia_earth.schema import ImpactAssessmentAllocationMethod
+
+from hestia_earth.models.log import logShouldRun, logRequirements
+from . import MODEL
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "updated": ["emissionsResourceUse", "impacts", "endpoints"],
+        "added": ["emissionsResourceUse", "impacts", "endpoints"]
+    }
+}
+RETURNS = {
+    "`economic` if any indicators were updated or added by the HESTIA models": ""
+}
+
+MODEL_KEY = 'allocationMethod'
+_MODIFIED_KEYS = ["emissionsResourceUse", "impacts", "endpoints"]
+
+
+def _should_run(impact: dict):
+    modified = impact.get('updated', []) + impact.get('added', [])
+    is_modified_by_hestia = any([key for key in _MODIFIED_KEYS if key in modified])
+
+    logRequirements(impact, model=MODEL, key=MODEL_KEY,
+                    is_modified_by_hestia=is_modified_by_hestia)
+
+    should_run = all([is_modified_by_hestia])
+    logShouldRun(impact, MODEL, None, should_run, key=MODEL_KEY)
+    return should_run
+
+
+def run(impact: dict):
+    return ImpactAssessmentAllocationMethod.ECONOMIC.value if _should_run(impact) else None

hestia_earth/models/impact_assessment/emissions.py

@@ -31,7 +31,8 @@ REQUIREMENTS = {
 RETURNS = {
     "Indicator": [{
         "term": "",
-        "value": ""
+        "value": "",
+        "methodTier": ""
     }]
 }
 MODEL_KEY = 'emissions'
@@ -44,6 +45,7 @@ def _indicator(product: dict):
 
         indicator = _new_indicator(emission.get('term', {}), emission.get('methodModel'))
         indicator['value'] = value
+        indicator['methodTier'] = emission.get('methodTier')
 
         if len(emission.get('inputs', [])):
             indicator['inputs'] = emission['inputs']

hestia_earth/models/mocking/search-results.json

@@ -979,11 +979,11 @@
       },
       {
         "@type": "Term",
-        "@id": "shortFallowCrop"
+        "@id": "longFallowCrop"
       },
       {
         "@type": "Term",
-        "@id": "longFallowCrop"
+        "@id": "shortFallowCrop"
       }
     ]
   },
@@ -1058,11 +1058,11 @@
       },
       {
         "@type": "Term",
-        "@id": "residueIncorporatedMoreThan30DaysBeforeCultivation"
+        "@id": "residueRemoved"
       },
       {
         "@type": "Term",
-        "@id": "residueRemoved"
+        "@id": "residueIncorporatedMoreThan30DaysBeforeCultivation"
       },
       {
         "@type": "Term",
@@ -1475,6 +1475,48 @@
       }
     ]
   },
+  {
+    "name": "get_flooded_pre_season_terms",
+    "query": {
+      "bool": {
+        "must": [
+          {
+            "match": {
+              "@type": "Term"
+            }
+          },
+          {
+            "match": {
+              "termType": "landUseManagement"
+            }
+          },
+          {
+            "match_phrase": {
+              "name": "flooded pre-season"
+            }
+          }
+        ]
+      }
+    },
+    "results": [
+      {
+        "@type": "Term",
+        "@id": "floodedPreSeasonMoreThan30Days"
+      },
+      {
+        "@type": "Term",
+        "@id": "nonFloodedPreSeasonMoreThan180Days"
+      },
+      {
+        "@type": "Term",
+        "@id": "nonFloodedPreSeasonLessThan180Days"
+      },
+      {
+        "@type": "Term",
+        "@id": "nonFloodedPreSeasonMoreThan365Days"
+      }
+    ]
+  },
   {
     "name": "get_generic_crop",
     "query": {
@@ -1486,7 +1528,7 @@
         "@type": "Term",
         "name": "Generic crop, seed",
         "@id": "genericCropSeed",
-        "_score": 26.36456
+        "_score": 26.448057
       }
     ]
   },
@@ -1722,61 +1764,61 @@
         "@type": "Term",
         "name": "Glass or high accessible cover",
         "@id": "glassOrHighAccessibleCover",
-        "_score": 59.78251
+        "_score": 59.747684
       },
       {
         "@type": "Term",
         "name": "River or stream",
         "@id": "riverOrStream",
-        "_score": 51.118263
+        "_score": 51.048874
       },
       {
         "@type": "Term",
         "name": "Other natural vegetation",
         "@id": "otherNaturalVegetation",
-        "_score": 41.25843
+        "_score": 41.325626
       },
       {
         "@type": "Term",
         "name": "Natural forest",
         "@id": "naturalForest",
-        "_score": 31.747753
+        "_score": 31.698704
       },
       {
         "@type": "Term",
         "name": "Permanent pasture",
         "@id": "permanentPasture",
-        "_score": 28.454437
+        "_score": 28.502478
       },
       {
         "@type": "Term",
         "name": "Animal housing",
         "@id": "animalHousing",
-        "_score": 27.424242
+        "_score": 27.39837
       },
       {
         "@type": "Term",
         "name": "Root or tuber crop plant",
         "@id": "rootOrTuberCropPlant",
-        "_score": 25.260658
+        "_score": 25.22627
       },
       {
         "@type": "Term",
         "name": "High intensity grazing pasture",
         "@id": "highIntensityGrazingPasture",
-        "_score": 24.038225
+        "_score": 24.007807
       },
       {
         "@type": "Term",
         "name": "Permanent cropland",
         "@id": "permanentCropland",
-        "_score": 20.689457
+        "_score": 20.544846
       },
       {
         "@type": "Term",
         "name": "Forest",
         "@id": "forest",
-        "_score": 20.366226
+        "_score": 20.397137
       }
     ]
   },
@@ -2024,15 +2066,15 @@
     "results": [
       {
         "@type": "Term",
-        "@id": "noTillage"
+        "@id": "verticalTillage"
       },
       {
         "@type": "Term",
-        "@id": "ridgeTillage"
+        "@id": "noTillage"
       },
       {
         "@type": "Term",
-        "@id": "verticalTillage"
+        "@id": "ridgeTillage"
       },
       {
         "@type": "Term",

hestia_earth/models/site/management.py

@@ -85,7 +85,8 @@ LOOKUPS = {
     "forage": ["landCoverTermId"],
     "inorganicFertiliser": "nitrogenContent",
     "organicFertiliser": "ANIMAL_MANURE",
-    "soilAmendment": "PRACTICE_INCREASING_C_INPUT"
+    "soilAmendment": "PRACTICE_INCREASING_C_INPUT",
+    "landUseManagement": "GAP_FILL_TO_MANAGEMENT"
 }
 MODEL_KEY = 'management'
 LAND_COVER_KEY = LOOKUPS['crop'][0]
@@ -216,6 +217,18 @@ def _get_relevant_inputs(cycles: list[dict]) -> list:
     return relevant_inputs
 
 
+def _has_gap_fill_to_management_set(practices: list) -> list:
+    """
+    Include only landUseManagement practices where GAP_FILL_TO_MANAGEMENT = True
+    """
+    result = [
+        p for p in practices
+        if p.get("term", {}).get("termType", {}) != TermTermType.LANDUSEMANAGEMENT.value
+        or get_lookup_value(lookup_term=p.get("term", {}), column=LOOKUPS["landUseManagement"])
+    ]
+    return result
+
+
 def _should_run_all_products(cycles: list, site_type: str):
     products_land_cover = [
         _extract_node_value(
@@ -292,6 +305,7 @@ def _should_run(site: dict):
             ]
         )
     ]
+    practices = _has_gap_fill_to_management_set(practices)
     practices = condense_nodes(practices)
 
     relevant_inputs = _get_relevant_inputs(cycles)

hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py

@@ -3,15 +3,16 @@ Pre Checks Cache Geospatial Database
 
 This model caches results from Geospatial Database.
 """
+from typing import Union, List
 from functools import reduce
-from hestia_earth.utils.tools import flatten, non_empty_list
+from hestia_earth.utils.tools import flatten
 
 from hestia_earth.models.log import debugValues
 from hestia_earth.models.utils import CACHE_KEY, cached_value, first_day_of_month, last_day_of_month
 from hestia_earth.models.utils.site import CACHE_YEARS_KEY
 from hestia_earth.models.geospatialDatabase.utils import (
     MAX_AREA_SIZE, CACHE_VALUE, CACHE_AREA_SIZE,
-    has_geospatial_data, has_coordinates, get_area_size, geospatial_data, _run_query, _collection_name
+    has_geospatial_data, has_coordinates, get_area_size, geospatial_data, _run_query, _collection_name, _cache_sub_key
 )
 from hestia_earth.models.geospatialDatabase import list_ee_params
 
@@ -34,11 +35,7 @@ def cache_site_results(results: list, collections: list, area_size: int = None):
         collection = collections[index]
         name = collection.get('name')
         value = results[index]
-        cache_sub_key = '-'.join(non_empty_list([
-            collection.get('year'),
-            collection.get('start_date'),
-            collection.get('end_date')
-        ]))
+        cache_sub_key = _cache_sub_key(collection)
         data = (group.get(name, {}) | {cache_sub_key: value} if cache_sub_key else value)
         return group | {name: data}
 
@@ -62,20 +59,24 @@ def _extend_collection_by_month(year: int):
     } for month in range(1, 13)]
 
 
-def _extend_collection(name: str, collection: dict, years: list = []):
-    data = collection | {'name': name, 'collection': _collection_name(collection.get('collection'))}
+def _extend_collection_data(name: str, collection: dict):
+    return collection | {'name': name, 'collection': _collection_name(collection.get('collection'))}
 
+
+def _extend_collection(name: str, collection: Union[List[dict], dict], years: list = []):
     year_params = [{'year': str(year)} for year in years]
     # fetch from first year to last
     month_years = range(years[0], years[-1] + 1) if len(years) > 1 else years
     month_params = flatten(map(_extend_collection_by_month, month_years))
 
     return [
-        (data | params) for params in year_params
+        (_extend_collection_data(name, collection) | params) for params in year_params
     ] if name.endswith('Annual') else [
-        (data | params) for params in month_params
+        (_extend_collection_data(name, collection) | params) for params in month_params
     ] if name.endswith('Monthly') else [
-        data
+        _extend_collection_data(name, col) for col in collection
+    ] if isinstance(collection, list) else [
+        _extend_collection_data(name, collection)
     ]
 
 
@@ -85,18 +86,24 @@ def _extend_collections(values: list, years: list = []):
     ])
 
 
+def _is_type(value: dict, ee_type: str):
+    params = value.get('params')
+    return any([
+        p.get('ee_type') == ee_type for p in params
+    ]) if isinstance(params, list) else params.get('ee_type') == ee_type
+
+
 def list_collections(years: list = [], include_region: bool = False, years_only: bool = False):
     ee_params = list_ee_params()
     # only cache `raster` results as can be combined in a single query
-    rasters = [value for value in ee_params if value.get('params').get('ee_type') == 'raster']
+    rasters = [value for value in ee_params if _is_type(value, 'raster')]
     rasters = _extend_collections(rasters, years or [])
     rasters = [raster for raster in rasters if not years_only or _is_collection_by_year(raster)]
 
     vectors = [
-        value for value in ee_params if all([
-            value.get('params').get('ee_type') == 'vector',
+        value for value in ee_params if _is_type(value, 'vector') and (
             include_region or not value.get('params').get('collection', '').startswith('gadm36')
-        ])
+        )
     ]
     # no vectors are running with specific years
     vectors = [] if years_only else _extend_collections(vectors)

hestia_earth/models/utils/indicator.py

@@ -5,7 +5,9 @@ from hestia_earth.utils.model import linked_node
 from . import _term_id, _include_methodModel
 
 
-def _new_indicator(term, model=None):
+def _new_indicator(term, model=None, land_cover_id=None):
     node = {'@type': SchemaType.INDICATOR.value}
     node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
+    if land_cover_id:
+        node['landCover'] = linked_node(download_hestia(land_cover_id))
     return _include_methodModel(node, model)

hestia_earth/models/utils/term.py

@@ -252,6 +252,40 @@ def get_rice_paddy_terms():
     return [n['@id'] for n in terms if 'depth' not in n['@id'].lower()]
 
 
+def get_flooded_pre_season_terms():
+    """
+    Find all `landUseManagement` terms of "flooded pre season" from the Glossary:
+    https://hestia.earth/glossary?termType=landUseManagement&query=flooded%20pre-season
+
+    Returns
+    -------
+    list
+        List of matching term `@id` as `str`.
+    """
+    terms = search({
+        "bool": {
+            "must": [
+                {
+                    "match": {
+                        "@type": SchemaType.TERM.value
+                    }
+                },
+                {
+                    "match": {
+                        "termType": TermTermType.LANDUSEMANAGEMENT.value
+                    }
+                },
+                {
+                    "match_phrase": {
+                        "name": "flooded pre-season"
+                    }
+                }
+            ]
+        }
+    }, limit=LIMIT)
+    return list(map(lambda n: n['@id'], terms))
+
+
 def get_crop_residue_terms():
     """
     Find all `cropResidue` terms from the Glossary:

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.63.0'
+VERSION = '0.64.0'

{hestia_earth_models-0.63.0.dist-info → hestia_earth_models-0.64.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.63.0
+Version: 0.64.0
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
@@ -11,7 +11,7 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hestia-earth.schema==29.*
+Requires-Dist: hestia-earth.schema==30.*
 Requires-Dist: hestia-earth.utils>=0.13.3
 Requires-Dist: python-dateutil>=2.8.1
 Requires-Dist: CurrencyConverter==0.16.8