hestia-earth-models 0.68.0__py3-none-any.whl → 0.68.1__py3-none-any.whl

hestia_earth/models/cache_sites.py

@@ -80,7 +80,7 @@ def _run_values(
         site_cache = merge(
             site.get(CACHE_KEY, {}),
             {CACHE_GEOSPATIAL_KEY: cached_data},
-            ({CACHE_YEARS_KEY: cached_value(site, CACHE_YEARS_KEY, []) + years} if years else {})
+            ({CACHE_YEARS_KEY: list(set(cached_value(site, CACHE_YEARS_KEY, []) + years))} if years else {})
         )
         return merge(site, {CACHE_KEY: site_cache})
 
@@ -137,6 +137,17 @@ def _run(sites: list, years: list = [], years_only: bool = False):
     ] + _run_values(sites_no_run, years=years))
 
 
+def _run_by_years(sites: list, years: list, batch_size: int):
+    batches = range(0, len(years), batch_size)
+
+    for batch_index in batches:
+        logger.info(f"Processing sites in batch {batch_index + 1} of {len(batches)}...")
+        sub_years = years[batch_index:batch_index + batch_size]
+        sites = _run(sites, sub_years, years_only=True)
+
+    return sites
+
+
 def run(sites: list, years: list = None):
     """
     Run all queries at once for the list of provided Sites.
@@ -153,12 +164,10 @@ def run(sites: list, years: list = None):
 
     # avoid memory limit errors by running only a few years at a time
     unique_years = sorted(list(set(years)))
-    batch_size = 5
-    batches = range(0, len(unique_years), batch_size)
-
-    for batch_index in batches:
-        logger.info(f"Processing sites in batch {batch_index + 1} of {len(batches)}...")
-        sub_years = unique_years[batch_index:batch_index + batch_size]
-        sites = _run(sites, sub_years, years_only=True)
 
-    return sites
+    try:
+        return _run_by_years(sites, unique_years, batch_size=5)
+    except Exception as e:
+        if 'exceeded' in str(e):
+            return _run_by_years(sites, unique_years, batch_size=1)
+        raise e