hestia-earth-models 0.60.0__tar.gz → 0.60.1__tar.gz

{hestia-earth-models-0.60.0/hestia_earth_models.egg-info → hestia-earth-models-0.60.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.60.0
+Version: 0.60.1
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team

{hestia-earth-models-0.60.0 → hestia-earth-models-0.60.1}/hestia_earth/models/cache_sites.py

@@ -1,5 +1,6 @@
 from functools import reduce
 from enum import Enum
+from pydash.objects import merge
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import flatten
 
@@ -64,12 +65,13 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
             **_cache_results(raster_results, rasters, index),
             **_cache_results(vector_results, vectors, index)
         } | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
-        return {
-            **site,
-            CACHE_KEY: (
-                cached_value(site) | {CACHE_GEOSPATIAL_KEY: cached_data}
-            ) | ({CACHE_YEARS_KEY: years} if years else {})
-        }
+        cached_data = merge(cached_value(site, CACHE_GEOSPATIAL_KEY, {}), cached_data)
+        site_cache = merge(
+            site.get(CACHE_KEY, {}),
+            {CACHE_GEOSPATIAL_KEY: cached_data},
+            ({CACHE_YEARS_KEY: cached_value(site, CACHE_YEARS_KEY, []) + years} if years else {})
+        )
+        return merge(site, {CACHE_KEY: site_cache})
 
     return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
 
@@ -82,7 +84,7 @@ def _preload_regions_area_size(sites: dict):
     return {term_id: download_hestia(term_id).get('area') for term_id in region_ids}
 
 
-def _group_sites(sites: dict):
+def _group_sites(sites: dict, check_has_cache: bool = True):
     # preload area size for all regions
     regions_area_size = _preload_regions_area_size(sites)
 
@@ -90,7 +92,7 @@ def _group_sites(sites: dict):
         return regions_area_size.get(_site_gadm_id(site)) if _should_preload_region_area_size(site) else None
 
     sites = [
-        (n, ) + (_should_run(n, get_region_area_size(n))) for n in sites
+        (n, ) + (_should_run(n, area_size=get_region_area_size(n), check_has_cache=check_has_cache)) for n in sites
     ]
     # restrict sites based on should_cache result
     sites = [(site, area_size) for site, should_cache, area_size in sites if should_cache]
@@ -112,7 +114,25 @@ def _group_sites(sites: dict):
     }
 
 
-def run(sites: list, years: list = None, include_region: bool = False, years_only: bool = False):
+def _run(sites: list, years: list, include_region: bool, years_only: bool = False):
+    rasters, vectors = list_collections(years, include_region, years_only)
+    filtered_data = _group_sites(sites, not years_only)
+    return flatten([
+        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
+        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
+    ])
+
+
+def _group_years(years: list, years_range: int):
+    batches = sorted(list(set(list(range(years[0], years[-1] + 1, years_range)) + [years[0], years[-1]])))
+    grouped_batches = [batches[i:i+2] for i in range(0, len(batches))]
+    return [
+        # make sure we don't overlap
+        [v[0] + (0 if v[0] == years[0] else 1), v[1]] for v in grouped_batches if len(v) == 2
+    ]
+
+
+def run(sites: list, years: list = None, include_region: bool = False):
     """
     Run all queries at once for the list of provided Sites.
     Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
@@ -126,14 +146,16 @@ def run(sites: list, years: list = None, include_region: bool = False, years_onl
     include_region : bool
         Prefecth region IDs.
         This will cache region-level data and will make the request slower. Only use if needed.
-    years_only : bool
-        Run only the collections that depend on the years (if provided).
     """
-    rasters, vectors = list_collections(years, include_region, years_only)
-
-    filtered_data = _group_sites(sites)
-
-    return flatten([
-        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
-        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
-    ])
+    try:
+        return _run(sites, years, include_region)
+    except Exception as e:
+        # when querying with multiple years, we can reach a compute memory limit, so run the years separately
+        if str(e) == 'User memory limit exceeded.' and years:
+            sites = _run(sites, [], include_region)
+            # query for subranges
+            for sub_years in _group_years(years, years_range=5):
+                sites = _run(sites, sub_years, include_region, years_only=True)
+            return sites
+
+    return []

{hestia-earth-models-0.60.0 → hestia-earth-models-0.60.1}/hestia_earth/models/mocking/search-results.json

@@ -1319,7 +1319,7 @@
         "@type": "Term",
         "name": "Generic crop, seed",
         "@id": "genericCropSeed",
-        "_score": 23.075872
+        "_score": 23.075811
       }
     ]
   },

{hestia-earth-models-0.60.0 → hestia-earth-models-0.60.1}/hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py

@@ -125,11 +125,11 @@ def _cache_results(site: dict, area_size: float):
     return cache_site_results(raster_results + vector_results, rasters + vectors, area_size)
 
 
-def _should_run(site: dict, area_size: float = None):
+def _should_run(site: dict, area_size: float = None, check_has_cache: bool = True):
     area_size = area_size or get_area_size(site)
     contains_geospatial_data = has_geospatial_data(site)
     contains_coordinates = has_coordinates(site)
-    has_cache = cached_value(site, CACHE_VALUE) is not None
+    has_cache = check_has_cache and cached_value(site, CACHE_VALUE) is not None
 
     debugValues(site,
                 area_size=area_size,

hestia-earth-models-0.60.1/hestia_earth/models/version.py

@@ -0,0 +1 @@
+VERSION = '0.60.1'

{hestia-earth-models-0.60.0 → hestia-earth-models-0.60.1/hestia_earth_models.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.60.0
+Version: 0.60.1
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team

{hestia-earth-models-0.60.0 → hestia-earth-models-0.60.1}/tests/models/test_cache_sites.py

@@ -57,15 +57,3 @@ def test_run_include_region(mock_run_query, *args):
             "coordinates": coordinates
         })
     ])
-
-
-@patch(f"{class_path}._run_query", return_value=[10] * 100)
-def test_run_years_only(*args):
-    with open(f"{fixtures_folder}/data.json", encoding='utf-8') as f:
-        data = json.load(f)
-    with open(f"{fixtures_folder}/years-only/cache.json", encoding='utf-8') as f:
-        cache = json.load(f)
-
-    sites = run(data.get('nodes', []), [2019, 2020], years_only=True)
-    expected = [site | {'_cache': cache} for site in data.get('nodes', [])]
-    assert sites == expected

hestia-earth-models-0.60.0/hestia_earth/models/version.py

@@ -1 +0,0 @@
-VERSION = '0.60.0'