arraylake 1.0.2__tar.gz → 1.0.3__tar.gz

{arraylake-1.0.2 → arraylake-1.0.3}/.gitignore

@@ -149,3 +149,4 @@ client/arraylake/_version.py
 !/.claude/
 /.claude/*
 !/.claude/skills/
+.local/

{arraylake-1.0.2 → arraylake-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arraylake
-Version: 1.0.2
+Version: 1.0.3
 Summary: Python client for ArrayLake
 Project-URL: Documentation, https://docs.earthmover.io/
 Project-URL: Changelog, https://docs.earthmover.io/changelog

arraylake-1.0.3/arraylake/_credential_cache.py

@@ -0,0 +1,118 @@
+"""Process-wide cache for delegated bucket credentials.
+
+When users pickle an icechunk Session and ship it to many workers, every
+unpickled task can independently call the arraylake server for fresh
+credentials when the embedded ones expire. This module gives the client a
+single per-process cache so all consumers in a worker process share one
+in-flight refresh.
+
+Validity follows ``Cache-Control: max-age`` semantics: cached creds are
+served while ``expires_at - CLOCK_SKEW_SAFETY > now``. The 30s safety only
+protects against worker/server NTP-level clock drift; it is not a
+pre-refresh window.
+
+The cache is loop-agnostic: callers may live on any asyncio event loop in
+the process. In-flight refreshes are coalesced via the singleflight
+pattern using a single ``concurrent.futures.Future`` per key as the shared
+result channel; each waiter bridges into its own loop via
+``asyncio.wrap_future``. The race for who runs the fetcher is resolved by
+``dict.setdefault``, which is atomic under the CPython GIL.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Literal
+
+from arraylake.log_util import get_logger
+from arraylake.types import OrgName, Platform, TempCredentials
+
+logger = get_logger(__name__)
+
+
+CLOCK_SKEW_SAFETY = timedelta(seconds=30)
+
+
+@dataclass(eq=True, frozen=True)
+class CredentialCacheKey:
+    api_url: str
+    auth_key: int
+    scope: Literal["repo", "bucket"]
+    org: OrgName
+    identifier: str
+    platform: Platform
+    access: Literal["read", "write"] = "read"
+
+
+@dataclass
+class _Entry:
+    creds: TempCredentials
+    expires_at: datetime | None
+
+
+_CACHE: dict[CredentialCacheKey, _Entry] = {}
+_INFLIGHT: dict[CredentialCacheKey, concurrent.futures.Future[TempCredentials]] = {}
+
+
+def _fresh(entry: _Entry | None, now: datetime) -> TempCredentials | None:
+    if entry is None or entry.expires_at is None:
+        return None
+    if entry.expires_at - CLOCK_SKEW_SAFETY > now:
+        return entry.creds
+    return None
+
+
+async def get_or_refresh(
+    key: CredentialCacheKey,
+    fetcher: Callable[[], Awaitable[TempCredentials]],
+    *,
+    use_cache: bool = True,
+) -> TempCredentials:
+    if not use_cache:
+        return await fetcher()
+
+    cached = _fresh(_CACHE.get(key), datetime.now(UTC))
+    if cached is not None:
+        return cached
+
+    my_future: concurrent.futures.Future[TempCredentials] = concurrent.futures.Future()
+    existing = _INFLIGHT.setdefault(key, my_future)
+    if existing is not my_future:
+        return await asyncio.wrap_future(existing)
+
+    try:
+        cached = _fresh(_CACHE.get(key), datetime.now(UTC))
+        if cached is not None:
+            my_future.set_result(cached)
+            return cached
+
+        creds = await fetcher()
+        exp = creds.expiration
+        if exp is not None and exp.tzinfo is None:
+            exp = exp.replace(tzinfo=UTC)
+        now = datetime.now(UTC)
+        if exp is not None and exp - CLOCK_SKEW_SAFETY <= now:
+            logger.warning(
+                "delegated credentials returned with lifetime <= clock-skew safety; server contract violation",
+                remaining_seconds=(exp - now).total_seconds(),
+                api_url=key.api_url,
+                org=key.org,
+                identifier=key.identifier,
+            )
+        _CACHE[key] = _Entry(creds=creds, expires_at=exp)
+        my_future.set_result(creds)
+        return creds
+    except BaseException as e:
+        my_future.set_exception(e)
+        raise
+    finally:
+        _INFLIGHT.pop(key, None)
+
+
+def clear() -> None:
+    _CACHE.clear()
+    _INFLIGHT.clear()

{arraylake-1.0.2 → arraylake-1.0.3}/arraylake/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.2'
-__version_tuple__ = version_tuple = (1, 0, 2)
+__version__ = version = '1.0.3'
+__version_tuple__ = version_tuple = (1, 0, 3)
 
 __commit_id__ = commit_id = None

{arraylake-1.0.2 → arraylake-1.0.3}/arraylake/client.py

@@ -26,6 +26,7 @@ import icechunk
 from icechunk import IcechunkError, RepositoryConfig
 from icechunk import Repository as IcechunkRepository
 
+from arraylake._credential_cache import CredentialCacheKey, get_or_refresh
 from arraylake.asyn import async_gather_tasks, asyncio_run, sync
 from arraylake.compute.services import AsyncComputeClient, ComputeClient
 from arraylake.config import config as arraylake_config
@@ -114,12 +115,19 @@ class AsyncClient:
             [Optional] The service URI to target.
         token:
             [Optional] API token for service account authentication.
+        cache_credentials:
+            [Optional] When True (default), delegated bucket credentials are
+            served from a process-wide cache so all consumers in a worker
+            share one in-flight refresh per ``(auth, target, access)``. Set
+            to False for server-side contexts that should not retain
+            credentials in memory — every call hits the server.
     """
 
     _service_uri: str | None
     token: str | None
+    _cache_credentials: bool
 
-    def __init__(self, service_uri: str | None = None, token: str | None = None) -> None:
+    def __init__(self, service_uri: str | None = None, token: str | None = None, cache_credentials: bool = True) -> None:
         if service_uri is not None:
             _validate_service_uri(service_uri)
 
@@ -133,6 +141,7 @@ class AsyncClient:
             raise ValueError("Invalid token provided. Tokens must start with ema_ or be a JWT token.")
 
         self.token = token
+        self._cache_credentials = cache_credentials
 
     @property
     def service_uri(self) -> str:
@@ -214,9 +223,22 @@ class AsyncClient:
         Returns:
             S3Credentials: Temporary credentials for the S3 bucket.
         """
-        mstore = self._metastore_for_org(org)
-        s3_creds = await mstore.get_s3_bucket_credentials_from_repo(repo_name)
-        return s3_creds
+        key = CredentialCacheKey(
+            api_url=self.service_uri,
+            auth_key=hash(self.token),
+            scope="repo",
+            org=org,
+            identifier=repo_name,
+            platform="s3",
+        )
+
+        async def fetch() -> S3Credentials:
+            mstore = self._metastore_for_org(org)
+            return await mstore.get_s3_bucket_credentials_from_repo(repo_name)
+
+        creds = await get_or_refresh(key, fetch, use_cache=self._cache_credentials)
+        assert isinstance(creds, S3Credentials)
+        return creds
 
     async def _get_gcs_delegated_credentials_from_repo(self, org: OrgName, repo_name: RepoName) -> GSCredentials:
         """Get delegated credentials for a repo's GCS bucket.
@@ -228,9 +250,22 @@ class AsyncClient:
         Returns:
             GSCredentials: Temporary credentials for the GCS bucket.
         """
-        mstore = self._metastore_for_org(org)
-        gcs_creds = await mstore.get_gs_bucket_credentials_from_repo(repo_name)
-        return gcs_creds
+        key = CredentialCacheKey(
+            api_url=self.service_uri,
+            auth_key=hash(self.token),
+            scope="repo",
+            org=org,
+            identifier=repo_name,
+            platform="gs",
+        )
+
+        async def fetch() -> GSCredentials:
+            mstore = self._metastore_for_org(org)
+            return await mstore.get_gs_bucket_credentials_from_repo(repo_name)
+
+        creds = await get_or_refresh(key, fetch, use_cache=self._cache_credentials)
+        assert isinstance(creds, GSCredentials)
+        return creds
 
     async def _get_azure_delegated_credentials_from_repo(self, org: OrgName, repo_name: RepoName) -> AzureCredentials:
         """Get delegated credentials for a repo's Azure Blob Storage container.
@@ -242,9 +277,22 @@ class AsyncClient:
         Returns:
             AzureCredentials: Temporary credentials for the Azure Blob Storage container.
         """
-        mstore = self._metastore_for_org(org)
-        azure_creds = await mstore.get_azure_container_credentials_from_repo(repo_name)
-        return azure_creds
+        key = CredentialCacheKey(
+            api_url=self.service_uri,
+            auth_key=hash(self.token),
+            scope="repo",
+            org=org,
+            identifier=repo_name,
+            platform="azure",
+        )
+
+        async def fetch() -> AzureCredentials:
+            mstore = self._metastore_for_org(org)
+            return await mstore.get_azure_container_credentials_from_repo(repo_name)
+
+        creds = await get_or_refresh(key, fetch, use_cache=self._cache_credentials)
+        assert isinstance(creds, AzureCredentials)
+        return creds
 
     async def _get_s3_delegated_credentials_from_bucket(
         self, org: OrgName, nickname: BucketNickname, access: Literal["read", "write"] = "read"
@@ -262,10 +310,24 @@ class AsyncClient:
         Returns:
             S3Credentials: Temporary credentials for the S3 bucket.
         """
-        mstore = self._metastore_for_org(org)
-        bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
-        s3_creds = await mstore.get_s3_bucket_credentials_from_bucket(bucket_id, access=access)
-        return s3_creds
+        key = CredentialCacheKey(
+            api_url=self.service_uri,
+            auth_key=hash(self.token),
+            scope="bucket",
+            org=org,
+            identifier=nickname,
+            platform="s3",
+            access=access,
+        )
+
+        async def fetch() -> S3Credentials:
+            mstore = self._metastore_for_org(org)
+            bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
+            return await mstore.get_s3_bucket_credentials_from_bucket(bucket_id, access=access)
+
+        creds = await get_or_refresh(key, fetch, use_cache=self._cache_credentials)
+        assert isinstance(creds, S3Credentials)
+        return creds
 
     async def _get_gcs_delegated_credentials_from_bucket(
         self, org: RepoName, nickname: BucketNickname, access: Literal["read", "write"] = "read"
@@ -283,10 +345,24 @@ class AsyncClient:
         Returns:
             GSCredentials: Temporary credentials for the GCS bucket.
         """
-        mstore = self._metastore_for_org(org)
-        bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
-        gcs_creds = await mstore.get_gs_bucket_credentials_from_bucket(bucket_id, access=access)
-        return gcs_creds
+        key = CredentialCacheKey(
+            api_url=self.service_uri,
+            auth_key=hash(self.token),
+            scope="bucket",
+            org=org,
+            identifier=nickname,
+            platform="gs",
+            access=access,
+        )
+
+        async def fetch() -> GSCredentials:
+            mstore = self._metastore_for_org(org)
+            bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
+            return await mstore.get_gs_bucket_credentials_from_bucket(bucket_id, access=access)
+
+        creds = await get_or_refresh(key, fetch, use_cache=self._cache_credentials)
+        assert isinstance(creds, GSCredentials)
+        return creds
 
     async def _get_azure_delegated_credentials_from_bucket(
         self, org: RepoName, nickname: BucketNickname, access: Literal["read", "write"] = "read"
@@ -304,10 +380,24 @@ class AsyncClient:
         Returns:
             AzureCredentials: Temporary credentials for the Azure Blob Storage container.
         """
-        mstore = self._metastore_for_org(org)
-        bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
-        azure_creds = await mstore.get_azure_container_credentials_from_bucket(bucket_id, access=access)
-        return azure_creds
+        key = CredentialCacheKey(
+            api_url=self.service_uri,
+            auth_key=hash(self.token),
+            scope="bucket",
+            org=org,
+            identifier=nickname,
+            platform="azure",
+            access=access,
+        )
+
+        async def fetch() -> AzureCredentials:
+            mstore = self._metastore_for_org(org)
+            bucket_id = await self._bucket_id_for_nickname(mstore, nickname)
+            return await mstore.get_azure_container_credentials_from_bucket(bucket_id, access=access)
+
+        creds = await get_or_refresh(key, fetch, use_cache=self._cache_credentials)
+        assert isinstance(creds, AzureCredentials)
+        return creds
 
     def _get_icechunk_s3_credentials_refresh_function_for_repo(self, org: OrgName, repo_name: RepoName) -> icechunk.S3StaticCredentials:
         """Get a function that returns S3 credentials for the given org and repo
@@ -1626,12 +1716,16 @@ class Client:
             [Optional] The service URI to target.
         token (str):
             [Optional] API token for service account authentication.
+        cache_credentials (bool):
+            [Optional] When True (default), delegated bucket credentials are
+            served from a process-wide cache. Set to False for server-side
+            contexts that should not retain credentials in memory.
     """
 
     aclient: AsyncClient
 
-    def __init__(self, service_uri: str | None = None, token: str | None = None) -> None:
-        self.aclient = AsyncClient(service_uri=service_uri, token=token)
+    def __init__(self, service_uri: str | None = None, token: str | None = None, cache_credentials: bool = True) -> None:
+        self.aclient = AsyncClient(service_uri=service_uri, token=token, cache_credentials=cache_credentials)
 
     @property
     def service_uri(self) -> str:

{arraylake-1.0.2 → arraylake-1.0.3}/arraylake/types.py

@@ -554,6 +554,7 @@ class BucketModel(BaseModel):
         self,
         credentials: TempCredentials | None,
         prefix: str | None = None,
+        retry_config: Any | None = None,
     ) -> obs.store.ObjectStore:
         """Build an obstore ObjectStore for this bucket using the given credentials.
 
@@ -609,6 +610,7 @@ class BucketModel(BaseModel):
                 prefix=effective_prefix,
                 config=s3_config,
                 client_options=s3_client_options or None,
+                retry_config=retry_config,
             )
         elif self.platform == "gs":
             if credentials is None:
@@ -616,6 +618,7 @@ class BucketModel(BaseModel):
                     bucket=self.name,
                     prefix=effective_prefix,
                     config={"skip_signature": True},
+                    retry_config=retry_config,
                 )
             elif isinstance(credentials, GSCredentials):
                 # google-auth returns naive datetimes (utcnow with tzinfo stripped) but
@@ -632,6 +635,7 @@ class BucketModel(BaseModel):
                     bucket=self.name,
                     prefix=effective_prefix,
                     credential_provider=gcs_credential_provider,
+                    retry_config=retry_config,
                 )
             else:
                 raise ValueError(f"Expected GSCredentials or None for platform 'gs', got {type(credentials).__name__}")
@@ -649,6 +653,7 @@ class BucketModel(BaseModel):
                 container_name=self.name,
                 prefix=effective_prefix,
                 config=azure_config,
+                retry_config=retry_config,
             )
         else:
             raise ValueError(f"Unsupported platform: {self.platform}")

arraylake-1.0.3/tests/test_credential_cache.py

@@ -0,0 +1,254 @@
+from __future__ import annotations
+
+import asyncio
+import threading
+from datetime import UTC, datetime, timedelta
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from arraylake import AsyncClient, _credential_cache
+from arraylake._credential_cache import (
+    CLOCK_SKEW_SAFETY,
+    CredentialCacheKey,
+    clear,
+    get_or_refresh,
+)
+from arraylake.asyn import asyncio_run
+from arraylake.types import GSCredentials, S3Credentials
+
+
+def _key(identifier: str = "repo-a", auth_key: int = 1) -> CredentialCacheKey:
+    return CredentialCacheKey(
+        api_url="https://api.test",
+        auth_key=auth_key,
+        scope="repo",
+        org="org-a",
+        identifier=identifier,
+        platform="s3",
+    )
+
+
+def _creds(remaining: timedelta | None = timedelta(hours=1)) -> S3Credentials:
+    return S3Credentials(
+        aws_access_key_id="AKIA",
+        aws_secret_access_key="secret",
+        aws_session_token="token",
+        expiration=datetime.now(UTC) + remaining if remaining is not None else None,
+    )
+
+
+@pytest.fixture(autouse=True)
+def _reset_cache():
+    clear()
+    yield
+    clear()
+
+
+def test_validity_boundary():
+    """Cache freshness follows expires_at - CLOCK_SKEW_SAFETY. Each case runs
+    two back-to-back lookups; count is 1 (cache hit) or 2 (refetched)."""
+    cases = [
+        ("hour-lifetime", timedelta(hours=1), 1),
+        ("hard-expired", timedelta(seconds=-5), 2),
+        ("inside-clock-skew", CLOCK_SKEW_SAFETY // 2, 2),
+        ("just-past-clock-skew", CLOCK_SKEW_SAFETY + timedelta(seconds=10), 1),
+        ("none-never-cached", None, 2),
+    ]
+    for name, remaining, expected in cases:
+        clear()
+        calls = 0
+
+        async def fetch(_remaining=remaining) -> S3Credentials:
+            nonlocal calls
+            calls += 1
+            return _creds(remaining=_remaining)
+
+        async def body() -> None:
+            await get_or_refresh(_key(), fetch)
+            await get_or_refresh(_key(), fetch)
+
+        asyncio_run(body())
+        assert calls == expected, f"case {name}: expected {expected} fetches, got {calls}"
+
+
+def test_keys_isolate_entries():
+    """Two distinct cache keys hold independent entries."""
+    cases = [
+        ("different-identifier", _key("repo-a"), _key("repo-b")),
+        ("different-auth-key", _key(auth_key=1), _key(auth_key=2)),
+    ]
+    for name, key_a, key_b in cases:
+        clear()
+        calls = 0
+
+        async def fetch() -> S3Credentials:
+            nonlocal calls
+            calls += 1
+            return _creds()
+
+        async def body() -> None:
+            await get_or_refresh(key_a, fetch)
+            await get_or_refresh(key_b, fetch)
+
+        asyncio_run(body())
+        assert calls == 2, f"case {name}: expected 2 fetches, got {calls}"
+
+
+def test_async_single_flight():
+    started = asyncio.Event()
+    release = asyncio.Event()
+    calls = 0
+
+    async def fetch() -> S3Credentials:
+        nonlocal calls
+        calls += 1
+        started.set()
+        await release.wait()
+        return _creds()
+
+    async def body() -> None:
+        tasks = [asyncio.create_task(get_or_refresh(_key(), fetch)) for _ in range(50)]
+        await started.wait()
+        release.set()
+        results = await asyncio.gather(*tasks)
+        for r in results:
+            assert results[0] is r
+
+    asyncio_run(body())
+    assert calls == 1
+
+
+def test_cross_thread_single_flight():
+    release = threading.Event()
+    fetch_thread_arrived = threading.Event()
+    calls = 0
+
+    async def fetch() -> S3Credentials:
+        nonlocal calls
+        calls += 1
+        fetch_thread_arrived.set()
+        await asyncio.get_running_loop().run_in_executor(None, release.wait)
+        return _creds()
+
+    def worker() -> None:
+        asyncio_run(get_or_refresh(_key(), fetch))
+
+    threads = [threading.Thread(target=worker) for _ in range(30)]
+    for t in threads:
+        t.start()
+    assert fetch_thread_arrived.wait(timeout=5), "fetcher never started"
+    release.set()
+    for t in threads:
+        t.join(timeout=10)
+    assert calls == 1
+
+
+def test_works_from_foreign_loop():
+    """Regression: previously get_or_refresh asserted that the caller was
+    running on the arraylake background loop, breaking any user who awaited
+    AsyncClient methods from their own asyncio loop. Calling from a fresh
+    foreign loop must complete normally."""
+
+    async def fetch() -> S3Credentials:
+        return _creds()
+
+    foreign_loop = asyncio.new_event_loop()
+    try:
+        creds = foreign_loop.run_until_complete(get_or_refresh(_key(), fetch))
+    finally:
+        foreign_loop.close()
+    assert creds.aws_access_key_id == "AKIA"
+
+
+def test_fetcher_exception_does_not_poison_cache():
+    attempts = 0
+
+    async def flaky() -> S3Credentials:
+        nonlocal attempts
+        attempts += 1
+        if attempts == 1:
+            raise RuntimeError("transient")
+        return _creds()
+
+    async def body() -> None:
+        with pytest.raises(RuntimeError):
+            await get_or_refresh(_key(), flaky)
+        creds = await get_or_refresh(_key(), flaky)
+        assert creds.aws_access_key_id == "AKIA"
+
+    asyncio_run(body())
+    assert attempts == 2
+
+
+def test_clear_drops_entries():
+    calls = 0
+
+    async def fetch() -> S3Credentials:
+        nonlocal calls
+        calls += 1
+        return _creds()
+
+    async def body() -> None:
+        await get_or_refresh(_key(), fetch)
+        assert _credential_cache._CACHE != {}
+        clear()
+        assert _credential_cache._CACHE == {}
+        await get_or_refresh(_key(), fetch)
+
+    asyncio_run(body())
+    assert calls == 2
+
+
+def test_delegated_credentials_are_process_cached():
+    """Two back-to-back calls to the same delegated-credential fetcher must
+    only hit the metastore once. This is the core guarantee for the
+    pickled-icechunk-session-across-dask-workers scenario."""
+    expiry = datetime.now(UTC) + timedelta(hours=1)
+    s3_creds = S3Credentials(
+        aws_access_key_id="AKIA",
+        aws_secret_access_key="secret",
+        aws_session_token="token",
+        expiration=expiry,
+    )
+    gs_creds = GSCredentials(access_token="bearer", principal="sa@x", expiration=expiry)
+    aclient = AsyncClient("https://test.example", token="ema_fake_token")
+    aclient_no_cache = AsyncClient("https://test.example", token="ema_fake_token", cache_credentials=False)
+
+    cases = [
+        ("s3-repo", "s3", "repo", aclient, "_get_s3_delegated_credentials_from_repo", ("org-a", "repo-a"), 1),
+        ("gs-repo", "gs", "repo", aclient, "_get_gcs_delegated_credentials_from_repo", ("org-a", "repo-a"), 1),
+        ("s3-bucket", "s3", "bucket", aclient, "_get_s3_delegated_credentials_from_bucket", ("org-a", "bucket-a"), 1),
+        ("gs-bucket", "gs", "bucket", aclient, "_get_gcs_delegated_credentials_from_bucket", ("org-a", "bucket-a"), 1),
+        ("s3-repo-nocache", "s3", "repo", aclient_no_cache, "_get_s3_delegated_credentials_from_repo", ("org-a", "repo-a"), 2),
+        ("gs-bucket-nocache", "gs", "bucket", aclient_no_cache, "_get_gcs_delegated_credentials_from_bucket", ("org-a", "bucket-a"), 2),
+    ]
+    for name, platform, scope, client, method_name, args, expected_calls in cases:
+        clear()
+        creds = s3_creds if platform == "s3" else gs_creds
+        mstore = MagicMock()
+        mstore.get_s3_bucket_credentials_from_repo = AsyncMock(return_value=creds)
+        mstore.get_gs_bucket_credentials_from_repo = AsyncMock(return_value=creds)
+        mstore.get_s3_bucket_credentials_from_bucket = AsyncMock(return_value=creds)
+        mstore.get_gs_bucket_credentials_from_bucket = AsyncMock(return_value=creds)
+
+        method = getattr(client, method_name)
+
+        async def invoke(_method=method, _args=args) -> None:
+            await _method(*_args)
+            await _method(*_args)
+
+        with (
+            patch.object(AsyncClient, "_metastore_for_org", return_value=mstore),
+            patch.object(AsyncClient, "_bucket_id_for_nickname", new=AsyncMock(return_value="bucket-uuid")),
+        ):
+            asyncio_run(invoke())
+
+        mstore_method = f"get_{'gs' if platform == 'gs' else 's3'}_bucket_credentials_from_{scope}"
+        assert getattr(mstore, mstore_method).await_count == expected_calls, (
+            f"case {name}: expected {expected_calls} metastore call(s), got {getattr(mstore, mstore_method).await_count}"
+        )
+        if expected_calls == 1:
+            assert _credential_cache._CACHE != {}, f"case {name}: expected cache to be populated"
+        else:
+            assert _credential_cache._CACHE == {}, f"case {name}: expected cache to remain empty"