forecasting-tools 0.3.1__tar.gz → 0.3.2__tar.gz

{forecasting_tools-0.3.1 → forecasting_tools-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forecasting-tools
-Version: 0.3.1
+Version: 0.3.2
 Summary: AI forecasting and research tools to help humans reason about and forecast the future
 License: MIT
 License-File: LICENSE

{forecasting_tools-0.3.1 → forecasting_tools-0.3.2}/forecasting_tools/ai_models/ai_utils/response_types.py

@@ -17,3 +17,4 @@ class TextTokenResponse(ModelResponse):
 
 class TextTokenCostResponse(TextTokenResponse):
     cost: float
+    serving_provider: str | None = None

{forecasting_tools-0.3.1 → forecasting_tools-0.3.2}/forecasting_tools/ai_models/general_llm.py

@@ -4,6 +4,7 @@ import asyncio
 import inspect
 import logging
 import os
+import re
 from typing import Any, Literal
 
 import litellm
@@ -57,6 +58,25 @@ class GeneralLlm(
     Litellm support every model, most every parameter, and acts as one interface for every provider.
     """
 
+    _logged_model_provider_pairs: set[tuple[str, str]] = set()
+
+    _CREDENTIAL_KWARGS: frozenset[str] = frozenset(
+        {
+            "api_key",
+            "extra_headers",
+            "headers",
+            "model_list",
+            "aws_access_key_id",
+            "aws_secret_access_key",
+            "aws_session_token",
+            "aws_web_identity_token",
+            "vertex_credentials",
+            "azure_ad_token",
+            "azure_password",
+            "client_secret",
+        }
+    )
+
     _defaults: dict[str, Any] = {
         # The lowest matching model substring is used as default (default 60s timeout)
         "gpt-4o": {
@@ -318,6 +338,7 @@ class GeneralLlm(
         ModelTracker.give_cost_tracking_warning_if_needed(
             self._litellm_model, observed_no_cost=observed_no_cost
         )
+        serving_provider = self._get_checked_serving_provider(response)
 
         if self.populate_citations:
             citations = self._extract_citations(response, choices)
@@ -341,6 +362,7 @@ class GeneralLlm(
             total_tokens_used=total_tokens,
             model=self.model,
             cost=direct_cost,
+            serving_provider=serving_provider,
         )
 
         return response
@@ -390,6 +412,58 @@ class GeneralLlm(
         )
         return mentions_temperature and is_unsupported
 
+    def _get_checked_serving_provider(self, response: ModelResponse) -> str | None:
+        serving_provider = typeguard.check_type(
+            (response.model_extra or {}).get("provider"), str | None
+        )
+        self._error_if_not_served_by_pinned_provider(serving_provider)
+        if serving_provider is None:
+            return None
+        model_and_provider = (self.model, serving_provider)
+        if model_and_provider not in GeneralLlm._logged_model_provider_pairs:
+            GeneralLlm._logged_model_provider_pairs.add(model_and_provider)
+            logger.info(
+                f"{self.model} is being served by provider '{serving_provider}'"
+            )
+        return serving_provider
+
+    def _get_pinned_provider_slugs(self) -> list[str]:
+        provider_routing = (self.litellm_kwargs.get("extra_body") or {}).get(
+            "provider"
+        ) or {}
+        if provider_routing.get("allow_fallbacks", True) is not False:
+            return []
+        return provider_routing.get("order", [])
+
+    def _error_if_not_served_by_pinned_provider(
+        self, serving_provider: str | None
+    ) -> None:
+        pinned_slugs = self._get_pinned_provider_slugs()
+        if not pinned_slugs:
+            return
+        if serving_provider is None or not any(
+            self._provider_name_matches_slug(serving_provider, slug)
+            for slug in pinned_slugs
+        ):
+            raise RuntimeError(
+                f"{self.model} is pinned to {pinned_slugs} with fallbacks disabled, "
+                f"but was served by provider '{serving_provider}'"
+            )
+
+    @staticmethod
+    def _provider_name_matches_slug(provider_name: str, endpoint_slug: str) -> bool:
+        """
+        OpenRouter reports display names that can be shorter or longer than the slug
+        (e.g. "Google" for "google-vertex", "Mancer 2" for "mancer"), so match on prefix.
+        """
+        normalized_name = re.sub(r"[^a-z0-9]", "", provider_name.lower())
+        normalized_slug = re.sub(r"[^a-z0-9]", "", endpoint_slug.split("/")[0].lower())
+        if not normalized_name or not normalized_slug:
+            return False
+        return normalized_slug.startswith(
+            normalized_name
+        ) or normalized_name.startswith(normalized_slug)
+
     def _answer_from_reasoning_content(self, message: Message) -> str | None:
         reasoning_content = getattr(message, "reasoning_content", None)
         if isinstance(reasoning_content, str) and reasoning_content:
@@ -494,6 +568,7 @@ class GeneralLlm(
         setattr(model_response, "choices", choices)
 
         model_response.model = self.model
+        model_response._hidden_params = raw_response._hidden_params
 
         setattr(
             model_response,
@@ -620,7 +695,11 @@ class GeneralLlm(
         return {
             "original_model": self.model,
             "allowed_tries": self.allowed_tries,
-            **{k: v for k, v in self.litellm_kwargs.items()},
+            **{
+                key: value
+                for key, value in self.litellm_kwargs.items()
+                if key not in self._CREDENTIAL_KWARGS
+            },
         }
 
     @classmethod

{forecasting_tools-0.3.1 → forecasting_tools-0.3.2}/forecasting_tools/data_models/questions.py

@@ -340,6 +340,10 @@ class BinaryQuestion(MetaculusQuestion):
                 q2_center_community_prediction = aggregations["unweighted"]["latest"]["centers"]  # type: ignore
             assert len(q2_center_community_prediction) == 1
             community_prediction_at_access_time = q2_center_community_prediction[0]
+        except (KeyError, TypeError):
+            community_prediction_at_access_time = None
+
+        try:
             history = api_json["question"]["my_forecasts"]["history"]
             previous_forecasts = None
             if history:
@@ -360,7 +364,6 @@ class BinaryQuestion(MetaculusQuestion):
                     for forecast in history
                 ]
         except (KeyError, TypeError):
-            community_prediction_at_access_time = None
             previous_forecasts = None
         return BinaryQuestion(
             community_prediction_at_access_time=community_prediction_at_access_time,

{forecasting_tools-0.3.1 → forecasting_tools-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "forecasting-tools"
-version = "0.3.1"
+version = "0.3.2"
 description = "AI forecasting and research tools to help humans reason about and forecast the future"
 authors = ["Benjamin Wilson <ben@metaculus.com>"]
 license = "MIT"