language-model-common 2.1.0__tar.gz → 2.1.2__tar.gz

{language_model_common-2.1.0/language_model_common.egg-info → language_model_common-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: language-model-common
-Version: 2.1.0
+Version: 2.1.2
 Summary: Provides the underlying framework to enhance langchain and add loading configurations
 Author-email: Imran Qureshi <imran.qureshi@bwell.com>
 License: Apache-2.0

language_model_common-2.1.2/VERSION

@@ -0,0 +1 @@
+2.1.2

{language_model_common-2.1.0 → language_model_common-2.1.2/language_model_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: language-model-common
-Version: 2.1.0
+Version: 2.1.2
 Summary: Provides the underlying framework to enhance langchain and add loading configurations
 Author-email: Imran Qureshi <imran.qureshi@bwell.com>
 License: Apache-2.0

{language_model_common-2.1.0 → language_model_common-2.1.2}/language_model_common.egg-info/SOURCES.txt

@@ -48,6 +48,7 @@ languagemodelcommon/container/container_factory.py
 languagemodelcommon/context/__init__.py
 languagemodelcommon/context/request_context.py
 languagemodelcommon/converters/__init__.py
+languagemodelcommon/converters/history_cache_middleware.py
 languagemodelcommon/converters/langgraph_to_openai_converter.py
 languagemodelcommon/converters/stream_buffer.py
 languagemodelcommon/converters/stream_context_mixin.py

{language_model_common-2.1.0 → language_model_common-2.1.2}/languagemodelcommon/configs/config_reader/config_reader.py

@@ -1,4 +1,5 @@
 import asyncio
+import hashlib
 import logging
 
 from pathlib import Path
@@ -108,7 +109,7 @@ class ConfigReader:
         models_testing_path: Optional[str],
     ) -> List[ChatModelConfig]:
         # Read from MongoDB model config cache
-        models = await self._read_from_model_config_cache()
+        models = await self._read_from_model_config_cache(config_path=config_path)
         if models:
             return models
 
@@ -119,7 +120,7 @@ class ConfigReader:
 
         async with self._lock:
             # Double-check after acquiring lock
-            models = await self._read_from_model_config_cache()
+            models = await self._read_from_model_config_cache(config_path=config_path)
             if models:
                 return models
 
@@ -145,17 +146,33 @@ class ConfigReader:
                 )
                 return models
 
-            await self._write_to_model_config_cache(models)
+            await self._write_to_model_config_cache(
+                models=models, config_path=config_path
+            )
             return models
 
-    def _model_config_cache_key(self, *, model_name: str) -> str:
-        return f"v{self.SCHEMA_VERSION}:{model_name}"
+    @staticmethod
+    def _config_ref_hash(*, config_path: str) -> str:
+        """Short hash identifying the config source (including its ref/tag).
+
+        Scopes cache keys by source so that pods on different config refs
+        (e.g. mid-rollout, where an old pod's MODELS_OFFICIAL_PATH still
+        points at a stale ref) never read or write each other's cached
+        content under the same key -- see BAI-720.
+        """
+        return hashlib.sha256(config_path.encode("utf-8")).hexdigest()[:12]
+
+    def _model_config_cache_key(self, *, config_path: str, model_name: str) -> str:
+        ref_hash = self._config_ref_hash(config_path=config_path)
+        return f"v{self.SCHEMA_VERSION}:{ref_hash}:{model_name}"
 
-    async def _read_from_model_config_cache(self) -> List[ChatModelConfig] | None:
+    async def _read_from_model_config_cache(
+        self, *, config_path: str
+    ) -> List[ChatModelConfig] | None:
         """Load model configs from the model config cache (one row per model).
 
         Scans the collection for all entries matching the current schema
-        version prefix.
+        version and config-source prefix.
 
         Returns ``None`` when the cache has no entries.
         Raises on store or deserialization errors so that a misconfigured
@@ -163,7 +180,8 @@ class ConfigReader:
         """
         if not self._model_config_cache_store:
             return None
-        prefix = f"v{self.SCHEMA_VERSION}:"
+        ref_hash = self._config_ref_hash(config_path=config_path)
+        prefix = f"v{self.SCHEMA_VERSION}:{ref_hash}:"
         model_keys = await self._get_cache_keys_by_prefix(prefix=prefix)
         if not model_keys:
             return None
@@ -205,7 +223,9 @@ class ConfigReader:
 
         return []
 
-    async def _write_to_model_config_cache(self, models: List[ChatModelConfig]) -> None:
+    async def _write_to_model_config_cache(
+        self, *, models: List[ChatModelConfig], config_path: str
+    ) -> None:
         """Store each model config as a separate row in the cache.
 
         Raises on write errors so that a misconfigured cache backend
@@ -215,7 +235,9 @@ class ConfigReader:
             return
         ttl = self._environment_variables.model_config_cache_ttl_seconds
         for model in models:
-            key = self._model_config_cache_key(model_name=model.name)
+            key = self._model_config_cache_key(
+                config_path=config_path, model_name=model.name
+            )
             await self._model_config_cache_store.put(
                 key,
                 model.model_dump(),
@@ -471,7 +493,7 @@ class ConfigReader:
         """
         logger.info("Retrying MCP server resolution for models with unresolved refs")
         await self._resolve_mcp_servers_async(models=models, config_path=config_path)
-        await self._write_to_model_config_cache(models)
+        await self._write_to_model_config_cache(models=models, config_path=config_path)
         unresolved = self._get_unresolved_mcp_servers(models)
         if unresolved:
             fetcher_url = (

language_model_common-2.1.2/languagemodelcommon/converters/history_cache_middleware.py

@@ -0,0 +1,66 @@
+from typing import Any, Awaitable, Callable
+
+from langchain.agents.middleware import (
+    AgentMiddleware,
+    ExtendedModelResponse,
+    ModelRequest,
+    ModelResponse,
+)
+from langchain_aws import ChatAnthropicBedrock
+from langchain_core.messages import AIMessage, ToolMessage
+
+
+class HistoryCacheMiddleware(AgentMiddleware):
+    """Caches conversation/tool-call history within a multi-round tool-calling turn.
+
+    Anchors a `cache_control` breakpoint at the end of the message list before
+    each model call after the first in a turn, so repeat calls read prior
+    history from cache instead of resending it at full price. `ChatAnthropicBedrock`
+    (via `langchain_anthropic`'s `_apply_cache_control_to_last_eligible_block`)
+    recomputes the breakpoint's position from the live message list on every
+    call, so it always re-anchors to the current last message rather than a
+    fixed offset -- including after any history trimming -- with no
+    bookkeeping needed here.
+
+    Scoped to within-turn only (BAI-706 ADR, Phase 2 / Option C). Cross-turn
+    history caching is a separate, larger follow-up requiring its own PHI
+    review and is explicitly out of scope.
+    """
+
+    async def awrap_model_call(
+        self,
+        request: ModelRequest[Any],
+        handler: Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]],
+    ) -> ModelResponse[Any] | AIMessage | ExtendedModelResponse[Any]:
+        if not self._should_cache(request):
+            return await handler(request)
+        return await handler(self._with_cache_control(request))
+
+    def wrap_model_call(
+        self,
+        request: ModelRequest[Any],
+        handler: Callable[[ModelRequest[Any]], ModelResponse[Any]],
+    ) -> ModelResponse[Any] | AIMessage | ExtendedModelResponse[Any]:
+        if not self._should_cache(request):
+            return handler(request)
+        return handler(self._with_cache_control(request))
+
+    @staticmethod
+    def _should_cache(request: ModelRequest[Any]) -> bool:
+        """Only cache from the second model call in a turn onward.
+
+        A `ToolMessage` in the message list means at least one full
+        model -> tool -> result round-trip already happened, i.e. this is a
+        subsequent call. The first call has no prior history worth reading
+        from cache, and tagging it would only add a write premium with no
+        matching read if the turn never calls a tool.
+        """
+        if not isinstance(request.model, ChatAnthropicBedrock):
+            return False
+        return any(isinstance(message, ToolMessage) for message in request.messages)
+
+    @staticmethod
+    def _with_cache_control(request: ModelRequest[Any]) -> ModelRequest[Any]:
+        model_settings = dict(request.model_settings)
+        model_settings["cache_control"] = {"type": "ephemeral"}
+        return request.override(model_settings=model_settings)

{language_model_common-2.1.0 → language_model_common-2.1.2}/languagemodelcommon/converters/langgraph_to_openai_converter.py

@@ -55,6 +55,9 @@ from languagemodelcommon.exceptions.bailey_exception import BaileyException
 from languagemodelcommon.exceptions.rate_limit_exception import RateLimitException
 from languagemodelcommon.mcp.tool_catalog import ToolCatalog
 from languagemodelcommon.mcp.tool_discovery_middleware import ToolDiscoveryMiddleware
+from languagemodelcommon.converters.history_cache_middleware import (
+    HistoryCacheMiddleware,
+)
 from languagemodelcommon.state.messages_state import MyMessagesState
 from languagemodelcommon.structures.openai.message.chat_message_wrapper import (
     ChatMessageWrapper,
@@ -1260,6 +1263,8 @@ class LangGraphToOpenAIConverter(StreamContextMixin):
         middleware: list[AgentMiddleware] = []
         if tool_catalog is not None:
             middleware.append(ToolDiscoveryMiddleware(catalog=tool_catalog))
+        if self.environment_variables.enable_history_prompt_caching:
+            middleware.append(HistoryCacheMiddleware())
 
         react_agent_runnable = create_agent(
             model=llm,

{language_model_common-2.1.0 → language_model_common-2.1.2}/languagemodelcommon/converters/tool_event_handlers.py

@@ -378,10 +378,10 @@ class ToolEventHandler(StreamContextMixin):
 
         if self._environment_variables.write_tool_output_to_file:
             error_content: str = (
-                f"Tool: {tool_name}\nError: {error_message}\nRuntime: {runtime_str}"
+                f"Tool: {tool_name}\nError: {display_message}\nRuntime: {runtime_str}"
             )
             self._stream_debug_output_manager.append_fragment(
-                text=f"\n--- Tool Error: {tool_name} ({runtime_str}) ---\n{error_message}\n",
+                text=f"\n--- Tool Error: {tool_name} ({runtime_str}) ---\n{display_message}\n",
             )
             tool_display_name: str = self._tool_display_name_mapper.get_name_for_tool(
                 tool_name=tool_name or "unknown",

{language_model_common-2.1.0 → language_model_common-2.1.2}/languagemodelcommon/utilities/environment/language_model_common_environment_variables.py

@@ -236,6 +236,19 @@ class LanguageModelCommonEnvironmentVariables(
         """When True, automatically compact conversation context on input-too-long errors."""
         return self.str2bool(os.environ.get("CONTEXT_COMPACTION_ENABLED", "true"))
 
+    @property
+    def enable_history_prompt_caching(self) -> bool:
+        """When True, cache conversation/tool-call history within a tool-calling turn.
+
+        Defaults to False. Conversation messages (patient questions, clinical
+        note contents, tool results containing FHIR resources) are far more
+        likely to contain PHI than the system prompt or tool schemas that are
+        already cached today. This must stay off until security/EA confirm
+        Bedrock prompt-cache storage is covered by existing PHI/BAA
+        commitments (BAI-706 ADR Open Question 4) — do not flip this default.
+        """
+        return self.str2bool(os.environ.get("ENABLE_HISTORY_PROMPT_CACHING", "false"))
+
     @property
     def rate_limit_retry_enabled(self) -> bool:
         """When True, retry upstream model rate limits (HTTP 429) before surfacing them."""

language_model_common-2.1.0/VERSION

@@ -1 +0,0 @@
-2.1.0