hestia-earth-models 0.60.0__py3-none-any.whl → 0.60.1__py3-none-any.whl

hestia_earth/models/cache_sites.py

@@ -1,5 +1,6 @@
 from functools import reduce
 from enum import Enum
+from pydash.objects import merge
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import flatten
 
@@ -64,12 +65,13 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
             **_cache_results(raster_results, rasters, index),
             **_cache_results(vector_results, vectors, index)
         } | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
-        return {
-            **site,
-            CACHE_KEY: (
-                cached_value(site) | {CACHE_GEOSPATIAL_KEY: cached_data}
-            ) | ({CACHE_YEARS_KEY: years} if years else {})
-        }
+        cached_data = merge(cached_value(site, CACHE_GEOSPATIAL_KEY, {}), cached_data)
+        site_cache = merge(
+            site.get(CACHE_KEY, {}),
+            {CACHE_GEOSPATIAL_KEY: cached_data},
+            ({CACHE_YEARS_KEY: cached_value(site, CACHE_YEARS_KEY, []) + years} if years else {})
+        )
+        return merge(site, {CACHE_KEY: site_cache})
 
     return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
 
@@ -82,7 +84,7 @@ def _preload_regions_area_size(sites: dict):
     return {term_id: download_hestia(term_id).get('area') for term_id in region_ids}
 
 
-def _group_sites(sites: dict):
+def _group_sites(sites: dict, check_has_cache: bool = True):
     # preload area size for all regions
     regions_area_size = _preload_regions_area_size(sites)
 
@@ -90,7 +92,7 @@ def _group_sites(sites: dict):
         return regions_area_size.get(_site_gadm_id(site)) if _should_preload_region_area_size(site) else None
 
     sites = [
-        (n, ) + (_should_run(n, get_region_area_size(n))) for n in sites
+        (n, ) + (_should_run(n, area_size=get_region_area_size(n), check_has_cache=check_has_cache)) for n in sites
     ]
     # restrict sites based on should_cache result
     sites = [(site, area_size) for site, should_cache, area_size in sites if should_cache]
@@ -112,7 +114,25 @@ def _group_sites(sites: dict):
     }
 
 
-def run(sites: list, years: list = None, include_region: bool = False, years_only: bool = False):
+def _run(sites: list, years: list, include_region: bool, years_only: bool = False):
+    rasters, vectors = list_collections(years, include_region, years_only)
+    filtered_data = _group_sites(sites, not years_only)
+    return flatten([
+        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
+        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
+    ])
+
+
+def _group_years(years: list, years_range: int):
+    batches = sorted(list(set(list(range(years[0], years[-1] + 1, years_range)) + [years[0], years[-1]])))
+    grouped_batches = [batches[i:i+2] for i in range(0, len(batches))]
+    return [
+        # make sure we don't overlap
+        [v[0] + (0 if v[0] == years[0] else 1), v[1]] for v in grouped_batches if len(v) == 2
+    ]
+
+
+def run(sites: list, years: list = None, include_region: bool = False):
     """
     Run all queries at once for the list of provided Sites.
     Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
@@ -126,14 +146,16 @@ def run(sites: list, years: list = None, include_region: bool = False, years_onl
     include_region : bool
         Prefecth region IDs.
         This will cache region-level data and will make the request slower. Only use if needed.
-    years_only : bool
-        Run only the collections that depend on the years (if provided).
     """
-    rasters, vectors = list_collections(years, include_region, years_only)
-
-    filtered_data = _group_sites(sites)
-
-    return flatten([
-        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
-        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
-    ])
+    try:
+        return _run(sites, years, include_region)
+    except Exception as e:
+        # when querying with multiple years, we can reach a compute memory limit, so run the years separately
+        if str(e) == 'User memory limit exceeded.' and years:
+            sites = _run(sites, [], include_region)
+            # query for subranges
+            for sub_years in _group_years(years, years_range=5):
+                sites = _run(sites, sub_years, include_region, years_only=True)
+            return sites
+
+    return []

hestia_earth/models/mocking/search-results.json

@@ -1319,7 +1319,7 @@
         "@type": "Term",
         "name": "Generic crop, seed",
         "@id": "genericCropSeed",
-        "_score": 23.075872
+        "_score": 23.075811
       }
     ]
   },

hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py

@@ -125,11 +125,11 @@ def _cache_results(site: dict, area_size: float):
     return cache_site_results(raster_results + vector_results, rasters + vectors, area_size)
 
 
-def _should_run(site: dict, area_size: float = None):
+def _should_run(site: dict, area_size: float = None, check_has_cache: bool = True):
     area_size = area_size or get_area_size(site)
     contains_geospatial_data = has_geospatial_data(site)
     contains_coordinates = has_coordinates(site)
-    has_cache = cached_value(site, CACHE_VALUE) is not None
+    has_cache = check_has_cache and cached_value(site, CACHE_VALUE) is not None
 
     debugValues(site,
                 area_size=area_size,

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.60.0'
+VERSION = '0.60.1'

{hestia_earth_models-0.60.0.dist-info → hestia_earth_models-0.60.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.60.0
+Version: 0.60.1
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team

{hestia_earth_models-0.60.0.dist-info → hestia_earth_models-0.60.1.dist-info}/RECORD

@@ -1,9 +1,9 @@
 hestia_earth/__init__.py,sha256=G-d438vPx7m_ks5e9XTtM3u7LDRO5dSSukibukWmyPM,56
 hestia_earth/models/__init__.py,sha256=qEFeq3yuf3lQKVseALmL8aPM8fpCS54B_5pry00M3hk,76
-hestia_earth/models/cache_sites.py,sha256=wsRjO7B14dd0XQtCqkjK0_W8P-45t_VhxWfaAFYYzKg,4916
+hestia_earth/models/cache_sites.py,sha256=KQp9cUKE-aIcYJoMWEtKFYS8gBFfsx5LKQhqoWpUSoM,6065
 hestia_earth/models/log.py,sha256=b63I3qyTtQs17xxbq8RI0Fv2lvZ1oDZ9k0njhxqiFFk,3459
 hestia_earth/models/requirements.py,sha256=znNZJAhwX2iYiKcAQXPftY7z_1MsNa0QxCXkXyHm_U0,17363
-hestia_earth/models/version.py,sha256=HxmxadwsbabLEgeV2_2efW6fNlYUuCRBJVCOpiPm7qg,19
+hestia_earth/models/version.py,sha256=jkZGW0WUYUex4V7qv5qCaUMLtL80ETv20-KbHpaM3KA,19
 hestia_earth/models/agribalyse2016/__init__.py,sha256=WvK0qCQbnYtg9oZxrACd1wGormZyXibPtpCnIQeDqbw,415
 hestia_earth/models/agribalyse2016/fuelElectricity.py,sha256=mrh8seYSYdTgcMDCETLiknuPeJehg071YoG4UiyW0yU,4404
 hestia_earth/models/agribalyse2016/machineryInfrastructureDepreciatedAmountPerCycle.py,sha256=_Rbngu0DzHKa62JwBl58ZC_ui1zLF2que_nB7ukhOQc,3392
@@ -356,7 +356,7 @@ hestia_earth/models/lcImpactCertainEffectsInfinite/damageToTerrestrialEcosystems
 hestia_earth/models/linkedImpactAssessment/__init__.py,sha256=d19wzr4GnWXpMWRhchqwp3S8txrUkpIWrGPwnWdXhjI,6236
 hestia_earth/models/mocking/__init__.py,sha256=kmSeOTSvurMUxw7Ajhf3G-SVPQ1NgmirMTk4TSOEicY,765
 hestia_earth/models/mocking/mock_search.py,sha256=V-ycVBTkJu7PP37Ivy_16hpKBV4aEtJb5S9DfChPNSU,2038
-hestia_earth/models/mocking/search-results.json,sha256=XZzKZ_aXsJJ2XgzlDbIYwmLozqdNHitUnDMDKNKWOG8,39547
+hestia_earth/models/mocking/search-results.json,sha256=d8oBES7NajlmrvfYZUeyGhDJsOQIOXQlY4ky6lsmoZ8,39547
 hestia_earth/models/pooreNemecek2018/__init__.py,sha256=nPboL7ULJzL5nJD5q7q9VOZt_fxbKVm8fmn1Az5YkVY,417
 hestia_earth/models/pooreNemecek2018/aboveGroundCropResidueTotal.py,sha256=Qt-mel4dkhK6N5uUOutNOinCTFjbjtGzITaaI0LvYc4,2396
 hestia_earth/models/pooreNemecek2018/belowGroundCropResidue.py,sha256=JT0RybbvWVlo01FO8K0Yj41HrEaJT3Kj1xfayr2X-xw,2315
@@ -476,7 +476,7 @@ hestia_earth/models/site/measurement/value.py,sha256=7IhUbIj7n5vB7yXoNxXsWbliEJj
 hestia_earth/models/site/post_checks/__init__.py,sha256=CkExxesk1GuG8NjrbKfix1iDuVUgU-9i1ccM_X7MZn4,284
 hestia_earth/models/site/post_checks/cache.py,sha256=_MZsNsclecUdHDT2MsYx4cEsVUXydIasddgZNA6SU4k,284
 hestia_earth/models/site/pre_checks/__init__.py,sha256=fjv6nU5fiL-CLyaa-cBpiLB-xujgPzeK7i7ZJuTOjCI,394
-hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py,sha256=89ZbSyJjP9adH09JhVlaFfLeXq3ND3w8Nxmd4HVB-5k,5240
+hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py,sha256=MoQ76e0S63uvoFhqBbhG0GuoJ9bZgX_vrXz2_Id44Jw,5290
 hestia_earth/models/site/pre_checks/cache_sources.py,sha256=RzvSgHJTpVkAB3mEvRju_irDQmdJRK7GUdU6PhS2Gaw,904
 hestia_earth/models/site/pre_checks/cache_years.py,sha256=qGwTaHlWxnVT7iVxXVcpJ-oG6M-VH4ZpCDTdTixUHR4,883
 hestia_earth/models/stehfestBouwman2006/__init__.py,sha256=EhvD4NK6oEPevusLb1WdYV3GT_fCtQx4gvdMhK_dEIQ,420
@@ -546,7 +546,7 @@ hestia_earth/models/utils/transformation.py,sha256=nyT5Mz4_VgFwhkL8JoNX9kxxow0zu
 hestia_earth/models/webbEtAl2012AndSintermannEtAl2012/__init__.py,sha256=Niv7ZFMBCwThlbCKGOwA17QdkpOUDFrqrFItGNqnZAA,434
 hestia_earth/models/webbEtAl2012AndSintermannEtAl2012/nh3ToAirOrganicFertiliser.py,sha256=G2eq00XGdr1ZEZ5Ru1nBIR-sD7iM9hGYhvk8r5wL3dA,3910
 tests/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/models/test_cache_sites.py,sha256=6Dp5SpshEFRavtDixA7ymxPv9j-ei_bqdbowYVMPYMQ,2465
+tests/models/test_cache_sites.py,sha256=AsZSGb4ruFqag74wRaQxpoZxbMxexPY8q8sMb2ergzg,1986
 tests/models/test_ecoinventV3.py,sha256=zyIWeeS7VndjXAHcROhDi3E0I52U8uQu8u-t1EXwqW8,1492
 tests/models/test_emissionNotRelevant.py,sha256=YXTdRfcdR_JepHuj2P3Y3r0aFMKNOmsXQHY48tmLTQo,1316
 tests/models/test_linkedImpactAssessment.py,sha256=RiBNTF34GMA5YsuJ0MAsEPhMuHNC2W0Qq9YnWYDpXM4,1172
@@ -1054,8 +1054,8 @@ tests/models/utils/test_source.py,sha256=mv3vHZV5cjpoLA2I1109-YUkuzAiuhbRSnv_76_
 tests/models/utils/test_term.py,sha256=M5Sa26v2gzQYbZ4H_fo7DspnaCx__-WtL-MULGapCWk,3509
 tests/models/webbEtAl2012AndSintermannEtAl2012/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/models/webbEtAl2012AndSintermannEtAl2012/test_nh3ToAirOrganicFertiliser.py,sha256=qi2FNXS5Af2WDtm7nq_FsprH3BfCF0XxnE0XHmC4aIY,2244
-hestia_earth_models-0.60.0.dist-info/LICENSE,sha256=AC7h7GAgCZGJK_Tzh6LUCrML9gQEfowWwecEw2w54QM,1154
-hestia_earth_models-0.60.0.dist-info/METADATA,sha256=9LXKP4GRx3ee43zO2ew5DLQYvFL8XkDUlAXWfeFr6yQ,3134
-hestia_earth_models-0.60.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-hestia_earth_models-0.60.0.dist-info/top_level.txt,sha256=1dqA9TqpOLTEgpqa-YBsmbCmmNU1y56AtfFGEceZ2A0,19
-hestia_earth_models-0.60.0.dist-info/RECORD,,
+hestia_earth_models-0.60.1.dist-info/LICENSE,sha256=AC7h7GAgCZGJK_Tzh6LUCrML9gQEfowWwecEw2w54QM,1154
+hestia_earth_models-0.60.1.dist-info/METADATA,sha256=tJULz_gVDoPAkvfkSD8ooAES548NXNxa4hQkQ_sS2WM,3134
+hestia_earth_models-0.60.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+hestia_earth_models-0.60.1.dist-info/top_level.txt,sha256=1dqA9TqpOLTEgpqa-YBsmbCmmNU1y56AtfFGEceZ2A0,19
+hestia_earth_models-0.60.1.dist-info/RECORD,,

tests/models/test_cache_sites.py

@@ -57,15 +57,3 @@ def test_run_include_region(mock_run_query, *args):
             "coordinates": coordinates
         })
     ])
-
-
-@patch(f"{class_path}._run_query", return_value=[10] * 100)
-def test_run_years_only(*args):
-    with open(f"{fixtures_folder}/data.json", encoding='utf-8') as f:
-        data = json.load(f)
-    with open(f"{fixtures_folder}/years-only/cache.json", encoding='utf-8') as f:
-        cache = json.load(f)
-
-    sites = run(data.get('nodes', []), [2019, 2020], years_only=True)
-    expected = [site | {'_cache': cache} for site in data.get('nodes', [])]
-    assert sites == expected