ee-client 2.4.0__tar.gz → 2.5.0__tar.gz

{ee_client-2.4.0 → ee_client-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ee-client
-Version: 2.4.0
+Version: 2.5.0
 Summary: extends the capabilities of the earthengine-api by providing custom session management and client interactions
 Author-email: Daniel Guerrero <dfgm2006@gmail.com>
 Project-URL: Homepage, https://github.com/dfguerrerom/ee-client

{ee_client-2.4.0 → ee_client-2.5.0}/ee_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ee-client
-Version: 2.4.0
+Version: 2.5.0
 Summary: extends the capabilities of the earthengine-api by providing custom session management and client interactions
 Author-email: Daniel Guerrero <dfgm2006@gmail.com>
 Project-URL: Homepage, https://github.com/dfguerrerom/ee-client

{ee_client-2.4.0 → ee_client-2.5.0}/ee_client.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ ee_client.egg-info/dependency_links.txt
 ee_client.egg-info/requires.txt
 ee_client.egg-info/top_level.txt
 eeclient/__init__.py
+eeclient/cache.py
 eeclient/client.py
 eeclient/data.py
 eeclient/exceptions.py
@@ -22,6 +23,8 @@ eeclient/interfaces/__init__.py
 eeclient/interfaces/export.py
 eeclient/interfaces/operations.py
 eeclient/interfaces/tasks.py
+tests/test_cache.py
 tests/test_client.py
 tests/test_data.py
+tests/test_integration_get_assets.py
 tests/test_models.py

{ee_client-2.4.0 → ee_client-2.5.0}/eeclient/__init__.py

@@ -1,6 +1,6 @@
 __title__ = "eeclient"
 __summary__ = "A client for Google Earth Engine"
-__version__ = "2.4.0"
+__version__ = "2.5.0"
 
 __author__ = "Daniel Guerrero"
 __email__ = "dfgm2006@gmail.com"

ee_client-2.5.0/eeclient/cache.py

@@ -0,0 +1,114 @@
+import asyncio
+import hashlib
+import json
+import time
+from collections import OrderedDict
+from typing import Any, Optional
+
+
+class CacheEntry:
+    """Represents a cached response with TTL and task tracking."""
+
+    def __init__(
+        self,
+        value: Any = None,
+        error: Optional[Exception] = None,
+        task: Optional[asyncio.Task] = None,
+    ):
+        self.value = value
+        self.error = error
+        self.task = task
+        self.timestamp = time.time()
+
+    def is_expired(self, ttl: float) -> bool:
+        """Check if the cache entry has exceeded its TTL."""
+        return (time.time() - self.timestamp) > ttl
+
+    def get_result(self):
+        """Get the cached result or raise the cached error."""
+        if self.error:
+            raise self.error
+        return self.value
+
+
+class ResponseCache:
+    """Simple LRU cache with TTL for async function responses.
+
+    Features:
+    - Fixed TTL (10 seconds by default)
+    - LRU eviction at max size (100 entries by default)
+    - Concurrent request deduplication (singleflight)
+    - Error caching to prevent retry storms
+    """
+
+    def __init__(self, ttl: float = 10.0, max_size: int = 100):
+        self.ttl = ttl
+        self.max_size = max_size
+        self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
+        self._lock = asyncio.Lock()
+
+    def make_cache_key(self, *args, **kwargs) -> str:
+        """Generate a stable cache key from function arguments."""
+        sorted_kwargs = sorted(kwargs.items())
+        key_data = {"args": args, "kwargs": sorted_kwargs}
+        key_json = json.dumps(key_data, sort_keys=True, default=str)
+        return hashlib.sha256(key_json.encode()).hexdigest()
+
+    async def get_or_fetch(self, key: str, fetch_func, *args, **kwargs) -> Any:
+        """Get cached result or fetch new one, with concurrent request deduplication.
+
+        Args:
+            key: Cache key
+            fetch_func: Async function to call if cache miss
+            *args, **kwargs: Arguments to pass to fetch_func
+
+        Returns:
+            The cached or freshly fetched result
+        """
+        # Check cache and get task reference if needed
+        async with self._lock:
+            if key in self._cache:
+                entry = self._cache[key]
+
+                # If there's an in-flight task, we'll wait for it
+                if entry.task and not entry.task.done():
+                    in_flight_task = entry.task
+                elif not entry.is_expired(self.ttl):
+                    # Valid cached result - return it
+                    self._cache.move_to_end(key)
+                    return entry.get_result()
+                else:
+                    # Expired entry - remove it and fetch new
+                    del self._cache[key]
+                    in_flight_task = None
+            else:
+                in_flight_task = None
+
+            # No in-flight task - create a new one
+            if in_flight_task is None:
+                task = asyncio.create_task(fetch_func(*args, **kwargs))
+                self._cache[key] = CacheEntry(task=task)
+
+                # LRU eviction
+                while len(self._cache) > self.max_size:
+                    self._cache.popitem(last=False)
+            else:
+                # Use the existing in-flight task
+                task = in_flight_task
+
+        # Wait for the task outside the lock to avoid deadlock
+        try:
+            result = await task
+            async with self._lock:
+                if key in self._cache:
+                    self._cache[key] = CacheEntry(value=result)
+            return result
+        except Exception as e:
+            async with self._lock:
+                if key in self._cache:
+                    self._cache[key] = CacheEntry(error=e)
+            raise
+
+    def clear(self):
+        """Clear all cached entries."""
+        self._cache.clear()

{ee_client-2.4.0 → ee_client-2.5.0}/eeclient/client.py

@@ -9,6 +9,7 @@ from contextlib import asynccontextmanager
 from eeclient.exceptions import EEClientError, EERestException
 from eeclient.models import GEEHeaders, SepalHeaders
 from eeclient.sepal_credential_mixin import SepalCredentialMixin
+from eeclient.cache import ResponseCache
 
 import eeclient.export as _export_module
 import eeclient.data as _operations_module
@@ -77,6 +78,8 @@ class EESession(SepalCredentialMixin):
         self._client: httpx.AsyncClient | None = None
         self._client_lock = asyncio.Lock()
 
+        self._assets_cache = ResponseCache(ttl=10.0, max_size=100)
+
         self.enforce_project_id = enforce_project_id
         super().__init__(sepal_headers)
 
@@ -181,6 +184,7 @@ class EESession(SepalCredentialMixin):
         if self._client is not None:
             await self._client.aclose()
             self._client = None
+        self._assets_cache.clear()
 
     async def rest_call(
         self,

{ee_client-2.4.0 → ee_client-2.5.0}/eeclient/data.py

@@ -167,32 +167,45 @@ async def _list_assets_concurrently(client: "EESession", folders):
 async def get_assets_async(client: "EESession", folder: str = "") -> List[dict]:
     """Get all assets in a folder recursively (async version).
 
+    Responses are cached for 10 seconds to prevent redundant API calls.
+    Concurrent requests with identical parameters are deduplicated.
+
     Args:
         client: The asynchronous session object.
         folder: The starting folder name or id.
 
     Returns:
-
+        List of assets with their type, name, and id.
     """
-    folder_queue = asyncio.Queue()
-    await folder_queue.put(folder)
-    asset_list = []
-
-    while not folder_queue.empty():
-        current_folders = [
-            await folder_queue.get() for _ in range(folder_queue.qsize())
-        ]
-        assets_groups = await _list_assets_concurrently(client, current_folders)
-
-        for assets in assets_groups:
-            for asset in assets:
-                asset_list.append(
-                    {"type": asset["type"], "name": asset["name"], "id": asset["id"]}
-                )
-                if asset["type"] == "FOLDER":
-                    await folder_queue.put(asset["name"])
-
-    return asset_list
+    cache_key = client._assets_cache.make_cache_key(folder)
+
+    async def _fetch_assets():
+        folder_queue = asyncio.Queue()
+        await folder_queue.put(folder)
+        asset_list = []
+
+        while not folder_queue.empty():
+            current_folders = [
+                await folder_queue.get() for _ in range(folder_queue.qsize())
+            ]
+            assets_groups = await _list_assets_concurrently(client, current_folders)
+
+            for assets in assets_groups:
+                for asset in assets:
+                    asset_list.append(
+                        {
+                            "type": asset["type"],
+                            "name": asset["name"],
+                            "id": asset["id"],
+                        }
+                    )
+                    if asset["type"] == "FOLDER":
+                        await folder_queue.put(asset["name"])
+
+        return asset_list
+
+    # Don't pass any arguments since _fetch_assets is a closure
+    return await client._assets_cache.get_or_fetch(cache_key, _fetch_assets)
 
 
 async def create_folder_async(client: "EESession", folder: Union[Path, str]) -> str:

{ee_client-2.4.0 → ee_client-2.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ee-client"
-version = "2.4.0"
+version = "2.5.0"
 description = "extends the capabilities of the earthengine-api by providing custom session management and client interactions"
 readme = { file = "README.rst",  content-type = "text/x-rst" }
 authors = [
@@ -66,7 +66,7 @@ branch = true
 [tool.commitizen]
 tag_format = "v$major.$minor.$patch$prerelease"
 update_changelog_on_bump = false
-version = "2.4.0"
+version = "2.5.0"
 version_files = [
     "pyproject.toml:version",
     "eeclient/__init__.py:__version__",

ee_client-2.5.0/tests/test_cache.py

@@ -0,0 +1,253 @@
+import asyncio
+import pytest
+import time
+from unittest.mock import MagicMock, patch
+
+from eeclient.cache import ResponseCache, CacheEntry
+from eeclient.data import get_assets_async
+
+
+# CacheEntry Tests
+
+
+def test_cache_entry_with_value():
+    entry = CacheEntry(value="test_value")
+    assert entry.value == "test_value"
+    assert entry.error is None
+    assert entry.task is None
+    assert entry.get_result() == "test_value"
+
+
+def test_cache_entry_with_error():
+    error = ValueError("test error")
+    entry = CacheEntry(error=error)
+    assert entry.value is None
+    assert entry.error == error
+
+    with pytest.raises(ValueError, match="test error"):
+        entry.get_result()
+
+
+def test_cache_entry_ttl_expiry():
+    entry = CacheEntry(value="test")
+    assert not entry.is_expired(ttl=10.0)
+
+    entry.timestamp = time.time() - 15.0
+    assert entry.is_expired(ttl=10.0)
+
+
+# ResponseCache Tests
+
+
+@pytest.mark.asyncio
+async def test_cache_basic_caching():
+    cache = ResponseCache(ttl=10.0, max_size=100)
+    call_count = 0
+
+    async def fetch_data(value):
+        nonlocal call_count
+        call_count += 1
+        await asyncio.sleep(0.01)
+        return f"result_{value}"
+
+    key = cache.make_cache_key("test_arg")
+
+    result1 = await cache.get_or_fetch(key, fetch_data, "test_arg")
+    result2 = await cache.get_or_fetch(key, fetch_data, "test_arg")
+
+    assert result1 == "result_test_arg"
+    assert result2 == "result_test_arg"
+    assert call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_cache_ttl_expiry():
+    cache = ResponseCache(ttl=0.1, max_size=100)
+    call_count = 0
+
+    async def fetch_data(value):
+        nonlocal call_count
+        call_count += 1
+        return f"result_{value}"
+
+    key = cache.make_cache_key("test_arg")
+
+    result1 = await cache.get_or_fetch(key, fetch_data, "test_arg")
+    assert result1 == "result_test_arg"
+    assert call_count == 1
+
+    await asyncio.sleep(0.15)
+
+    result2 = await cache.get_or_fetch(key, fetch_data, "test_arg")
+    assert result2 == "result_test_arg"
+    assert call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_cache_concurrent_request_deduplication():
+    cache = ResponseCache(ttl=10.0, max_size=100)
+    call_count = 0
+
+    async def fetch_data(value):
+        nonlocal call_count
+        call_count += 1
+        await asyncio.sleep(0.05)
+        return f"result_{value}"
+
+    key = cache.make_cache_key("test_arg")
+
+    tasks = [
+        asyncio.create_task(cache.get_or_fetch(key, fetch_data, "test_arg"))
+        for _ in range(10)
+    ]
+
+    results = await asyncio.wait_for(asyncio.gather(*tasks), timeout=5.0)
+
+    assert all(r == "result_test_arg" for r in results)
+    assert call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_cache_different_params_different_cache():
+    cache = ResponseCache(ttl=10.0, max_size=100)
+    call_count = 0
+
+    async def fetch_data(value):
+        nonlocal call_count
+        call_count += 1
+        return f"result_{value}"
+
+    key1 = cache.make_cache_key("arg1")
+    key2 = cache.make_cache_key("arg2")
+
+    result1 = await cache.get_or_fetch(key1, fetch_data, "arg1")
+    result2 = await cache.get_or_fetch(key2, fetch_data, "arg2")
+
+    assert result1 == "result_arg1"
+    assert result2 == "result_arg2"
+    assert call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_cache_error_caching():
+    cache = ResponseCache(ttl=10.0, max_size=100)
+    call_count = 0
+
+    async def fetch_data_with_error():
+        nonlocal call_count
+        call_count += 1
+        await asyncio.sleep(0.01)
+        raise ValueError("API error")
+
+    key = cache.make_cache_key("error_test")
+
+    with pytest.raises(ValueError, match="API error"):
+        await cache.get_or_fetch(key, fetch_data_with_error)
+
+    assert call_count == 1
+
+    with pytest.raises(ValueError, match="API error"):
+        await cache.get_or_fetch(key, fetch_data_with_error)
+
+    assert call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_cache_lru_eviction():
+    cache = ResponseCache(ttl=10.0, max_size=3)
+
+    async def fetch_data(value):
+        return f"result_{value}"
+
+    for i in range(5):
+        key = cache.make_cache_key(f"arg_{i}")
+        await cache.get_or_fetch(key, fetch_data, f"arg_{i}")
+
+    assert len(cache._cache) == 3
+
+
+@pytest.mark.asyncio
+async def test_cache_clear():
+    cache = ResponseCache(ttl=10.0, max_size=100)
+
+    async def fetch_data(value):
+        return f"result_{value}"
+
+    key = cache.make_cache_key("test")
+    await cache.get_or_fetch(key, fetch_data, "test")
+
+    assert len(cache._cache) == 1
+
+    cache.clear()
+
+    assert len(cache._cache) == 0
+
+
+def test_cache_key_consistency():
+    cache = ResponseCache()
+
+    key1 = cache.make_cache_key("folder1", param1="value1", param2="value2")
+    key2 = cache.make_cache_key("folder1", param2="value2", param1="value1")
+
+    assert key1 == key2
+
+
+# get_assets_async Caching Tests
+
+
+@pytest.mark.asyncio
+async def test_get_assets_caching():
+    """Test that identical get_assets calls use cache."""
+    mock_client = MagicMock()
+    mock_client._assets_cache = ResponseCache(ttl=10.0, max_size=100)
+
+    call_count = 0
+
+    async def mock_list_assets(client, folders):
+        nonlocal call_count
+        call_count += 1
+        await asyncio.sleep(0.01)
+        return [
+            [
+                {"type": "IMAGE", "name": "test_image", "id": "test_id_1"},
+                {"type": "IMAGE", "name": "test_image2", "id": "test_id_2"},
+            ]
+        ]
+
+    with patch("eeclient.data._list_assets_concurrently", new=mock_list_assets):
+        result1 = await get_assets_async(mock_client, "projects/test/assets/folder")
+        assert len(result1) == 2
+        assert call_count == 1
+
+        result2 = await get_assets_async(mock_client, "projects/test/assets/folder")
+        assert result1 == result2
+        assert call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_get_assets_concurrent_deduplication():
+    """Test that concurrent identical requests are deduplicated."""
+    mock_client = MagicMock()
+    mock_client._assets_cache = ResponseCache(ttl=10.0, max_size=100)
+
+    call_count = 0
+
+    async def mock_list_assets(client, folders):
+        nonlocal call_count
+        call_count += 1
+        await asyncio.sleep(0.05)
+        return [[{"type": "IMAGE", "name": "test_image", "id": "test_id"}]]
+
+    with patch("eeclient.data._list_assets_concurrently", new=mock_list_assets):
+        tasks = [
+            asyncio.create_task(
+                get_assets_async(mock_client, "projects/test/assets/folder")
+            )
+            for _ in range(10)
+        ]
+
+        results = await asyncio.wait_for(asyncio.gather(*tasks), timeout=5.0)
+
+        assert all(len(r) == 1 for r in results)
+        assert all(r[0]["id"] == "test_id" for r in results)
+        assert call_count == 1

{ee_client-2.4.0 → ee_client-2.5.0}/tests/test_data.py

@@ -158,9 +158,7 @@ async def test_list_assets_concurrently(sepal_headers):
     assets_lists = await _list_assets_concurrently(session, folders)
     assert isinstance(assets_lists, list)
 
-    # Depending on your project, the list might be empty or not.
     print(f"Assets listed: {assets_lists}")
-    # Clean up
     await delete_folder(session, folder_id=folder_id, recursive=True)
 
 

ee_client-2.5.0/tests/test_integration_get_assets.py

@@ -0,0 +1,84 @@
+import os
+import pytest
+import time
+import logging
+from unittest.mock import patch
+
+from eeclient.helpers import get_sepal_headers_from_auth
+from eeclient.client import EESession
+from eeclient.data import get_assets_async
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.asyncio
+async def test_real_get_assets_cache():
+    """Integration test against a real SEPAL server.
+
+    Requires the following environment variables to be set:
+      - SEPAL_HOST
+      - LOCAL_SEPAL_USER
+      - LOCAL_SEPAL_PASSWORD
+
+    The test will be skipped if those are not available.
+    """
+    sepal_host = os.getenv("SEPAL_HOST")
+    sepal_user = os.getenv("LOCAL_SEPAL_USER")
+    sepal_password = os.getenv("LOCAL_SEPAL_PASSWORD")
+
+    if not (sepal_host and sepal_user and sepal_password):
+        pytest.skip("SEPAL_HOST, LOCAL_SEPAL_USER or LOCAL_SEPAL_PASSWORD not set")
+
+    sepal_headers = get_sepal_headers_from_auth(
+        sepal_user=sepal_user, sepal_password=sepal_password, sepal_host=sepal_host
+    )
+
+    session = await EESession.create(sepal_headers=sepal_headers)
+
+    try:
+        folder = await session.get_assets_folder()
+
+        assert len(session._assets_cache._cache) == 0
+
+        logger.info(f"Making first call to get_assets_async({folder})")
+        start_time = time.time()
+        assets = await get_assets_async(session, folder)
+        first_call_duration = time.time() - start_time
+        logger.info(
+            f"First call completed in {first_call_duration:.3f}s, "
+            f"retrieved {len(assets)} assets"
+        )
+
+        assert isinstance(assets, list)
+
+        key = session._assets_cache.make_cache_key(folder)
+        assert key in session._assets_cache._cache
+        logger.debug(f"Cache entry created with key: {key[:16]}...")
+
+        cache_entry = session._assets_cache._cache[key]
+        assert cache_entry.value is not None
+        assert cache_entry.task is None
+        logger.debug("Cache entry contains value (no pending task)")
+
+        logger.info("Making second call to get_assets_async (should use cache)")
+
+        with patch("eeclient.data._list_assets_concurrently") as mock_list:
+            start_time = time.time()
+            assets2 = await get_assets_async(session, folder)
+            second_call_duration = time.time() - start_time
+            logger.info(f"Second call completed in {second_call_duration:.3f}s")
+
+            assert (
+                mock_list.call_count == 0
+            ), "Network call should NOT happen for cached result"
+            logger.info("Confirmed: NO network call made (mock not called)")
+
+        assert assets2 == assets
+        logger.debug("Results are identical")
+
+        speedup = first_call_duration / second_call_duration
+        logger.info(f"Cache speedup: {speedup:.1f}x faster")
+        assert speedup > 10, f"Cache should be much faster! Got only {speedup:.1f}x"
+
+    finally:
+        await session.aclose()