cachellm-proxy 0.1.0__py3-none-any.whl

cachellm/api/routes_chat.py

@@ -0,0 +1,385 @@
+"""The drop-in endpoint: POST /v1/chat/completions.
+
+Same request shape as OpenAI, same response shape, same error envelope, plus a
+set of ``X-Cache-*`` headers describing what happened. An application adopts
+CacheLLM by changing one base URL and nothing else.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import AsyncIterator
+from typing import Any
+
+import structlog
+from fastapi import APIRouter, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from cachellm.api import sse
+from cachellm.api.auth import verify
+from cachellm.api.deps import AppState, get_state
+from cachellm.cache.entry import CacheEntry
+from cachellm.cache.keys import prompt_fingerprint
+from cachellm.cache.policy import PolicyDecision
+from cachellm.cache.service import LookupResult
+from cachellm.errors import CacheMissError, UpstreamError
+from cachellm.models import (
+    ChatCompletionRequest,
+    ChatCompletionResponse,
+    ModelCard,
+    ModelList,
+)
+from cachellm.observability import span
+from cachellm.pricing import estimate_cost
+from cachellm.providers.base import Provider
+
+log = structlog.get_logger(__name__)
+router = APIRouter()
+
+BYPASS_LOOKUP = LookupResult(
+    status="bypass",
+    decision=PolicyDecision(False, "default", 1.0, 0, "cache_unavailable"),
+)
+
+
+def cache_headers(
+    lookup: LookupResult, *, total_ms: float, saved_usd: float = 0.0, coalesced: bool = False
+) -> dict[str, str]:
+    status = {
+        "hit": "HIT",
+        "miss": "MISS",
+        "bypass": "BYPASS",
+        "shadow_hit": "SHADOW",
+    }[lookup.status]
+    headers = {
+        "X-Cache": status,
+        "X-Cache-Category": lookup.category,
+        "X-Cache-Lookup-Ms": f"{lookup.lookup_ms:.2f}",
+        "X-Cache-Latency-Ms": f"{total_ms:.2f}",
+        "X-Cache-Threshold": f"{lookup.decision.threshold:.3f}",
+    }
+    if lookup.namespace:
+        headers["X-Cache-Namespace"] = lookup.namespace
+    if lookup.tier:
+        headers["X-Cache-Tier"] = lookup.tier
+    if lookup.status in ("hit", "shadow_hit", "miss"):
+        headers["X-Cache-Similarity"] = f"{lookup.similarity:.4f}"
+    if lookup.entry is not None:
+        headers["X-Cache-Entry-Id"] = lookup.entry.entry_id
+        headers["X-Cache-Age-Seconds"] = f"{lookup.entry.age_seconds():.0f}"
+    if lookup.decision.bypass_reason:
+        headers["X-Cache-Bypass-Reason"] = lookup.decision.bypass_reason
+    if saved_usd:
+        headers["X-Cache-Saved-USD"] = f"{saved_usd:.8f}"
+    if coalesced:
+        headers["X-Cache-Coalesced"] = "true"
+    return headers
+
+
+def _debug_block(lookup: LookupResult, saved_usd: float, coalesced: bool) -> dict[str, Any]:
+    """Non-standard extra field. SDKs ignore unknown keys; humans find it useful."""
+    return {
+        "status": lookup.status,
+        "tier": lookup.tier or None,
+        "similarity": round(lookup.similarity, 4)
+        if lookup.tier or lookup.status == "miss"
+        else None,
+        "threshold": lookup.decision.threshold,
+        "category": lookup.category,
+        "namespace": lookup.namespace or None,
+        "bypass_reason": lookup.decision.bypass_reason or None,
+        "saved_usd": round(saved_usd, 8) or None,
+        "coalesced": coalesced or None,
+        "lookup_ms": round(lookup.lookup_ms, 2),
+    }
+
+
+async def _record_outcome(
+    state: AppState, lookup: LookupResult, model: str, provider: str, total_ms: float
+) -> None:
+    metrics = state.metrics
+    result = lookup.status
+    metrics.requests.labels(
+        result=result, category=lookup.category, provider=provider, model=model
+    ).inc()
+    metrics.request_duration.labels(result=result).observe(total_ms / 1000.0)
+    if lookup.tier or result == "miss":
+        metrics.lookup_duration.labels(tier=lookup.tier or "semantic").observe(
+            lookup.lookup_ms / 1000.0
+        )
+        metrics.similarity.labels(result=result).observe(max(0.0, min(1.0, lookup.similarity)))
+    if state.analytics is not None:
+        counters = {"requests": 1}
+        if lookup.decision.cacheable:
+            counters["cacheable_requests"] = 1
+        if result == "miss":
+            counters["misses"] = 1
+        elif result == "bypass":
+            counters["bypass"] = 1
+        elif result == "shadow_hit":
+            counters["shadow_hits"] = 1
+        await state.analytics.bulk(counters)
+        await state.analytics.record_latency("hit" if result == "hit" else "miss", total_ms)
+
+
+@router.get("/models", response_model=ModelList)
+async def list_models(request: Request) -> ModelList:
+    state = get_state(request)
+    verify(request, state.settings.client_keys)
+    return ModelList(data=[ModelCard(id=m) for m in state.providers.models()])
+
+
+@router.post("/chat/completions")
+async def chat_completions(request: Request, body: ChatCompletionRequest) -> Any:
+    state = get_state(request)
+    verify(request, state.settings.client_keys)
+    started = time.perf_counter()
+
+    cache_control = request.headers.get("x-cache-control", "")
+    only_if_cached = "only-if-cached" in cache_control.lower()
+    provider, provider_name = state.providers.resolve(body.model)
+
+    with span(
+        "cachellm.request",
+        **{
+            "gen_ai.system": provider_name,
+            "gen_ai.request.model": body.model,
+            "gen_ai.request.temperature": body.effective_temperature,
+            "cachellm.prompt_fingerprint": prompt_fingerprint(body.last_user_text()),
+        },
+    ) as current:
+        # ---------------------------------------------------------- lookup
+        if state.caching_on and state.cache is not None:
+            lookup = await state.cache.lookup(body, provider_name, cache_control=cache_control)
+        else:
+            lookup = BYPASS_LOOKUP
+
+        if current is not None:
+            current.set_attribute("cachellm.status", lookup.status)
+            current.set_attribute("cachellm.category", lookup.category)
+            current.set_attribute("cachellm.similarity", round(lookup.similarity, 4))
+
+        if lookup.status == "shadow_hit" and state.analytics is not None:
+            state.metrics.shadow_hits.labels(category=lookup.category).inc()
+            log.info(
+                "shadow_hit",
+                similarity=round(lookup.similarity, 4),
+                tier=lookup.tier,
+                category=lookup.category,
+                fingerprint=prompt_fingerprint(lookup.cache_text),
+            )
+
+        # ------------------------------------------------------- cache hit
+        if lookup.served_from_cache and state.cache is not None and lookup.entry is not None:
+            entry = lookup.entry
+            saved = await state.cache.register_hit(lookup, body.model)
+            state.metrics.observe_cost(body.model, saved, "saved")
+            state.metrics.observe_tokens(
+                body.model, entry.prompt_tokens, entry.completion_tokens, "saved"
+            )
+            total_ms = (time.perf_counter() - started) * 1000
+            await _record_outcome(state, lookup, body.model, provider_name, total_ms)
+            headers = cache_headers(lookup, total_ms=total_ms, saved_usd=saved)
+            if body.stream:
+                return _replay_stream(body, entry, headers)
+            payload = ChatCompletionResponse.from_text(
+                model=body.model,
+                text=entry.response_text,
+                prompt_tokens=entry.prompt_tokens,
+                completion_tokens=entry.completion_tokens,
+                finish_reason=entry.finish_reason,
+            ).model_dump()
+            payload["cachellm"] = _debug_block(lookup, saved, False)
+            return JSONResponse(payload, headers=headers)
+
+        if only_if_cached:
+            raise CacheMissError()
+
+        # ----------------------------------------------------------- miss
+        if body.stream:
+            return await _proxy_stream(state, body, lookup, provider, provider_name, started)
+        return await _proxy_once(state, body, lookup, provider, provider_name, started)
+
+
+async def _proxy_once(
+    state: AppState,
+    body: ChatCompletionRequest,
+    lookup: LookupResult,
+    provider: Provider,
+    provider_name: str,
+    started: float,
+) -> JSONResponse:
+    """Non-streaming miss: one upstream call, shared by identical concurrent misses."""
+    coalesce_key = f"{lookup.namespace}:{lookup.exact}" if lookup.namespace else ""
+
+    async def call() -> Any:
+        with span(
+            "cachellm.provider",
+            **{"gen_ai.system": provider_name, "gen_ai.request.model": body.model},
+        ):
+            return await provider.complete(body)
+
+    try:
+        if coalesce_key and state.caching_on:
+            result, coalesced = await state.singleflight.do(coalesce_key, call)
+        else:
+            result, coalesced = await call(), False
+    except UpstreamError:
+        state.metrics.provider_errors.labels(provider=provider_name).inc()
+        if state.analytics is not None:
+            await state.analytics.incr("provider_errors")
+        raise
+
+    if coalesced:
+        state.metrics.coalesced.inc()
+        if state.analytics is not None:
+            await state.analytics.incr("coalesced")
+
+    if state.caching_on and state.cache is not None and not coalesced:
+        await state.cache.store(
+            lookup=lookup,
+            request=body,
+            provider_name=provider_name,
+            response_text=result.text,
+            prompt_tokens=result.prompt_tokens,
+            completion_tokens=result.completion_tokens,
+            finish_reason=result.finish_reason,
+            tool_calls=result.tool_calls,
+        )
+
+    spent = estimate_cost(body.model, result.prompt_tokens, result.completion_tokens)
+    state.metrics.observe_cost(body.model, spent, "spent")
+    state.metrics.observe_tokens(
+        body.model, result.prompt_tokens, result.completion_tokens, "spent"
+    )
+    if state.analytics is not None:
+        await state.analytics.incr_float("usd_spent", spent)
+
+    total_ms = (time.perf_counter() - started) * 1000
+    await _record_outcome(state, lookup, body.model, provider_name, total_ms)
+
+    payload = ChatCompletionResponse.from_text(
+        model=body.model,
+        text=result.text,
+        prompt_tokens=result.prompt_tokens,
+        completion_tokens=result.completion_tokens,
+        finish_reason=result.finish_reason,
+    ).model_dump()
+    payload["cachellm"] = _debug_block(lookup, 0.0, coalesced)
+    return JSONResponse(
+        payload, headers=cache_headers(lookup, total_ms=total_ms, coalesced=coalesced)
+    )
+
+
+def _replay_stream(
+    body: ChatCompletionRequest, entry: CacheEntry, headers: dict[str, str]
+) -> StreamingResponse:
+    """Serve a cache hit to a client that asked for a stream."""
+    stream_id = sse.new_stream_id()
+    created = int(time.time())
+
+    async def generate() -> AsyncIterator[str]:
+        yield sse.role_chunk(stream_id, body.model, created)
+        for piece in sse.split_for_replay(entry.response_text):
+            yield sse.text_chunk(stream_id, body.model, created, piece)
+        yield sse.final_chunk(
+            stream_id,
+            body.model,
+            created,
+            entry.finish_reason,
+            {
+                "prompt_tokens": entry.prompt_tokens,
+                "completion_tokens": entry.completion_tokens,
+                "total_tokens": entry.prompt_tokens + entry.completion_tokens,
+            },
+        )
+        yield sse.DONE
+
+    return StreamingResponse(generate(), media_type="text/event-stream", headers=headers)
+
+
+async def _proxy_stream(
+    state: AppState,
+    body: ChatCompletionRequest,
+    lookup: LookupResult,
+    provider: Provider,
+    provider_name: str,
+    started: float,
+) -> StreamingResponse:
+    """Streaming miss: forward chunks live while buffering for the cache.
+
+    The buffer is only committed once the upstream reports a clean finish. A
+    disconnect halfway through leaves nothing behind, which is exactly what you
+    want: a truncated answer served from cache forever would be a silent bug.
+    """
+    stream_id = sse.new_stream_id()
+    created = int(time.time())
+
+    async def generate() -> AsyncIterator[str]:
+        buffer: list[str] = []
+        finish_reason: str | None = None
+        prompt_tokens = 0
+        completion_tokens = 0
+        yield sse.role_chunk(stream_id, body.model, created)
+        try:
+            async for event in provider.stream(body):
+                if event.delta:
+                    buffer.append(event.delta)
+                    yield sse.text_chunk(stream_id, body.model, created, event.delta)
+                if event.finish_reason:
+                    finish_reason = event.finish_reason
+                if event.prompt_tokens or event.completion_tokens:
+                    prompt_tokens = event.prompt_tokens or prompt_tokens
+                    completion_tokens = event.completion_tokens or completion_tokens
+        except UpstreamError as exc:
+            state.metrics.provider_errors.labels(provider=provider_name).inc()
+            if state.analytics is not None:
+                await state.analytics.incr("provider_errors")
+            log.warning("stream_failed", error=str(exc)[:200])
+            yield sse.final_chunk(stream_id, body.model, created, "error", None)
+            yield sse.DONE
+            return
+
+        text = "".join(buffer)
+        if not completion_tokens and text:
+            completion_tokens = provider.approx_tokens(text)
+        if not prompt_tokens:
+            prompt_tokens = provider.approx_tokens(body.last_user_text())
+
+        yield sse.final_chunk(
+            stream_id,
+            body.model,
+            created,
+            finish_reason or "stop",
+            {
+                "prompt_tokens": prompt_tokens,
+                "completion_tokens": completion_tokens,
+                "total_tokens": prompt_tokens + completion_tokens,
+            },
+        )
+        yield sse.DONE
+
+        if state.caching_on and state.cache is not None and state.settings.cache_streaming:
+            await state.cache.store(
+                lookup=lookup,
+                request=body,
+                provider_name=provider_name,
+                response_text=text,
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                finish_reason=finish_reason or "stop",
+            )
+        spent = estimate_cost(body.model, prompt_tokens, completion_tokens)
+        state.metrics.observe_cost(body.model, spent, "spent")
+        state.metrics.observe_tokens(body.model, prompt_tokens, completion_tokens, "spent")
+        if state.analytics is not None:
+            await state.analytics.incr_float("usd_spent", spent)
+        total_ms = (time.perf_counter() - started) * 1000
+        await _record_outcome(state, lookup, body.model, provider_name, total_ms)
+
+    total_ms = (time.perf_counter() - started) * 1000
+    return StreamingResponse(
+        generate(),
+        media_type="text/event-stream",
+        headers=cache_headers(lookup, total_ms=total_ms),
+    )

cachellm/api/sse.py

@@ -0,0 +1,98 @@
+"""Server-sent event helpers for streaming chat completions."""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+import uuid
+from typing import Any
+
+DONE = "data: [DONE]\n\n"
+
+# Non-space run plus whatever whitespace follows it, so no separator is lost.
+_TOKEN_WITH_TRAILING_SPACE = re.compile(r"\S+\s*|\s+")
+
+
+def new_stream_id() -> str:
+    return f"chatcmpl-{uuid.uuid4().hex[:24]}"
+
+
+def chunk(
+    *,
+    stream_id: str,
+    model: str,
+    delta: dict[str, Any] | None = None,
+    finish_reason: str | None = None,
+    usage: dict[str, int] | None = None,
+    created: int | None = None,
+) -> str:
+    payload: dict[str, Any] = {
+        "id": stream_id,
+        "object": "chat.completion.chunk",
+        "created": created or int(time.time()),
+        "model": model,
+        "choices": [
+            {
+                "index": 0,
+                "delta": delta if delta is not None else {},
+                "finish_reason": finish_reason,
+            }
+        ],
+    }
+    if usage is not None:
+        payload["usage"] = usage
+    return f"data: {json.dumps(payload, separators=(',', ':'))}\n\n"
+
+
+def role_chunk(stream_id: str, model: str, created: int) -> str:
+    return chunk(
+        stream_id=stream_id,
+        model=model,
+        delta={"role": "assistant", "content": ""},
+        created=created,
+    )
+
+
+def text_chunk(stream_id: str, model: str, created: int, text: str) -> str:
+    return chunk(stream_id=stream_id, model=model, delta={"content": text}, created=created)
+
+
+def final_chunk(
+    stream_id: str, model: str, created: int, finish_reason: str, usage: dict[str, int] | None
+) -> str:
+    return chunk(
+        stream_id=stream_id,
+        model=model,
+        delta={},
+        finish_reason=finish_reason,
+        usage=usage,
+        created=created,
+    )
+
+
+def split_for_replay(text: str, max_chars: int = 24) -> list[str]:
+    """Chop a cached answer into believable stream chunks.
+
+    A cached hit has the whole answer already, but a client that asked for a
+    stream still expects a stream. Replaying in small pieces keeps SDKs and UIs
+    working unchanged; it arrives in a couple of milliseconds either way.
+
+    The invariant that matters is ``"".join(pieces) == text``: an SSE client
+    concatenates deltas directly, it does not re-insert separators. Splitting
+    on spaces and dropping them silently eats a space at every chunk boundary,
+    which is invisible in unit tests that join with a space and obvious the
+    moment a real SDK reads the stream.
+    """
+    if not text:
+        return []
+    pieces: list[str] = []
+    buffer = ""
+    for token in _TOKEN_WITH_TRAILING_SPACE.findall(text):
+        buffer += token
+        if len(buffer) >= max_chars:
+            pieces.append(buffer)
+            buffer = ""
+    if buffer:
+        pieces.append(buffer)
+    return pieces

cachellm/cache/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+__all__: list[str] = []

cachellm/cache/analytics.py

@@ -0,0 +1,150 @@
+"""Durable counters and the near-miss log.
+
+Prometheus holds the time series for dashboards, but it resets when the process
+does and it cannot answer "what did the cache do last week". These Redis-backed
+counters survive restarts and are shared across workers, which is what the
+admin endpoints and the README numbers read from.
+
+The near-miss log is the tuning instrument: every lookup that landed just below
+the threshold is kept with its score, so you can see exactly what a slightly
+looser threshold would have bought you, on your own traffic.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass
+from typing import Any
+
+import redis.asyncio as aioredis
+
+from cachellm.settings import Settings
+
+
+@dataclass
+class NearMiss:
+    prompt: str
+    matched_prompt: str
+    similarity: float
+    threshold: float
+    category: str
+    namespace: str
+    model: str
+    at: float
+
+
+class Analytics:
+    def __init__(self, redis: aioredis.Redis, settings: Settings) -> None:
+        self._redis = redis
+        self._settings = settings
+        self._counters = f"{settings.stats_prefix}counters"
+        self._near = f"{settings.stats_prefix}near_misses"
+        self._latency = f"{settings.stats_prefix}latency"
+
+    # ------------------------------------------------------------------ counters
+    async def incr(self, field: str, amount: int = 1) -> None:
+        await self._redis.hincrby(self._counters, field, amount)
+
+    async def incr_float(self, field: str, amount: float) -> None:
+        if amount:
+            await self._redis.hincrbyfloat(self._counters, field, amount)
+
+    async def bulk(self, ints: dict[str, int], floats: dict[str, float] | None = None) -> None:
+        async with self._redis.pipeline(transaction=False) as pipe:
+            for field, count in ints.items():
+                if count:
+                    pipe.hincrby(self._counters, field, count)
+            for field, value in (floats or {}).items():
+                if value:
+                    pipe.hincrbyfloat(self._counters, field, value)
+            await pipe.execute()
+
+    async def counters(self) -> dict[str, float]:
+        raw = await self._redis.hgetall(self._counters)
+        out: dict[str, float] = {}
+        for key, value in raw.items():
+            name = key.decode() if isinstance(key, bytes) else str(key)
+            text = value.decode() if isinstance(value, bytes) else str(value)
+            try:
+                out[name] = float(text)
+            except ValueError:
+                continue
+        return out
+
+    async def reset(self) -> None:
+        await self._redis.delete(self._counters, self._near, self._latency)
+
+    # ---------------------------------------------------------------- latency
+    async def record_latency(self, result: str, ms: float) -> None:
+        """Keep a rolling sample per outcome so admin/stats can show percentiles."""
+        await self._redis.lpush(f"{self._latency}:{result}", f"{ms:.3f}")
+        await self._redis.ltrim(f"{self._latency}:{result}", 0, 4_999)
+
+    async def latency_percentiles(self, result: str) -> dict[str, float]:
+        raw = await self._redis.lrange(f"{self._latency}:{result}", 0, -1)
+        values = sorted(float(v) for v in raw)
+        if not values:
+            return {}
+
+        def pct(p: float) -> float:
+            idx = min(len(values) - 1, max(0, round((p / 100.0) * (len(values) - 1))))
+            return round(values[idx], 2)
+
+        return {
+            "count": len(values),
+            "p50": pct(50),
+            "p95": pct(95),
+            "p99": pct(99),
+            "min": round(values[0], 2),
+            "max": round(values[-1], 2),
+        }
+
+    # -------------------------------------------------------------- near misses
+    async def record_near_miss(self, miss: NearMiss) -> None:
+        payload = asdict(miss)
+        if not self._settings.log_prompts:
+            payload["prompt"] = payload["prompt"][:120]
+            payload["matched_prompt"] = payload["matched_prompt"][:120]
+        await self._redis.lpush(self._near, json.dumps(payload))
+        await self._redis.ltrim(self._near, 0, self._settings.near_miss_log_size - 1)
+
+    async def near_misses(self, limit: int = 50) -> list[dict[str, Any]]:
+        raw = await self._redis.lrange(self._near, 0, limit - 1)
+        out: list[dict[str, Any]] = []
+        for item in raw:
+            try:
+                out.append(json.loads(item))
+            except json.JSONDecodeError:
+                continue
+        return out
+
+    async def near_miss_histogram(self, buckets: int = 20) -> list[dict[str, Any]]:
+        """How many near misses sit in each similarity band.
+
+        Read this as: "if I dropped the threshold to X, this many more requests
+        would have been served from cache."
+        """
+        misses = await self.near_misses(limit=self._settings.near_miss_log_size)
+        hist = [0] * buckets
+        for miss in misses:
+            sim = float(miss.get("similarity", 0.0))
+            idx = min(buckets - 1, max(0, int(sim * buckets)))
+            hist[idx] += 1
+        width = 1.0 / buckets
+        cumulative = 0
+        rows: list[dict[str, Any]] = []
+        for i in range(buckets - 1, -1, -1):
+            cumulative += hist[i]
+            rows.append(
+                {
+                    "similarity_at_least": round(i * width, 3),
+                    "in_band": hist[i],
+                    "would_hit_if_threshold_here": cumulative,
+                }
+            )
+        return rows
+
+    @staticmethod
+    def now() -> float:
+        return time.time()

cachellm/cache/coalesce.py

@@ -0,0 +1,63 @@
+"""Single-flight: collapse identical in-flight misses into one upstream call.
+
+Ten users asking the same brand-new question at the same moment is a cache
+stampede: every one of them misses, and every one of them pays for a full
+generation. The first caller here does the work and the rest await its result.
+On a cold cache under real concurrency this is worth more than a few points of
+similarity threshold.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+
+class SingleFlight:
+    def __init__(self) -> None:
+        self._inflight: dict[str, asyncio.Future[Any]] = {}
+        self._lock = asyncio.Lock()
+        self.coalesced = 0
+
+    async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> tuple[T, bool]:
+        """Run ``factory`` for ``key``, or await the run already in progress.
+
+        Returns ``(result, was_coalesced)``.
+        """
+        async with self._lock:
+            existing = self._inflight.get(key)
+            if existing is not None:
+                self.coalesced += 1
+                waiter = existing
+                joined = True
+            else:
+                waiter = asyncio.get_running_loop().create_future()
+                self._inflight[key] = waiter
+                joined = False
+
+        if joined:
+            return await asyncio.shield(waiter), True
+
+        try:
+            result = await factory()
+        except BaseException as exc:  # propagate to every waiter, then re-raise
+            async with self._lock:
+                self._inflight.pop(key, None)
+            if not waiter.done():
+                waiter.set_exception(exc)
+            # Keep the future's exception from being reported as unretrieved.
+            waiter.exception()
+            raise
+        else:
+            async with self._lock:
+                self._inflight.pop(key, None)
+            if not waiter.done():
+                waiter.set_result(result)
+            return result, False
+
+    @property
+    def in_flight(self) -> int:
+        return len(self._inflight)