forecasting-tools 0.3.0__tar.gz → 0.3.2__tar.gz

{forecasting_tools-0.3.0 → forecasting_tools-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forecasting-tools
-Version: 0.3.0
+Version: 0.3.2
 Summary: AI forecasting and research tools to help humans reason about and forecast the future
 License: MIT
 License-File: LICENSE
@@ -80,7 +80,7 @@ Demo repo (get a Metaculus bot running in 30min): https://github.com/Metaculus/m
 This repository contains forecasting and research tools built with Python and Streamlit. The project aims to assist users in making predictions, conducting research, and analyzing data related to hard to answer questions (especially those from Metaculus).
 
 Here are the tools most likely to be useful to you:
-- 🎯 **Forecasting Bot:** General forecaster that integrates with the Metaculus AI benchmarking competition and provides a number of utilities. You can forecast with a pre-existing bot or override the class to customize your own (without redoing all the aggregation/API code, etc)
+- 🎯 **Forecasting Bot:** General forecaster that integrates with the Metaculus FutureEval bot tournament and provides a number of utilities. You can forecast with a pre-existing bot or override the class to customize your own (without redoing all the aggregation/API code, etc)
 - 🔌 **Metaculus API Wrapper:** for interacting with questions and tournaments
 - 🤖 **In-House Metaculus Bots**: You can see all the bots that Metaculus is running on their site in `run_bots.py`
 
@@ -91,7 +91,7 @@ Here are some other features of the project (not all are documented yet):
 
 All the examples below are in a Jupyter Notebook called `README.ipynb` which you can run locally to test the package (make sure to run the first cell though).
 
-If you decide you want to join the Metaculus AI Benchmarking Tournament, it is recommended that you start [here](https://github.com/Metaculus/metac-bot-template). This repo is easy to start with and has a 30min tutorial for how to set it up.
+If you decide you want to join the Metaculus FutureEval Bot Tournament, it is recommended that you start [here](https://github.com/Metaculus/metac-bot-template). This repo is easy to start with and has a 30min tutorial for how to set it up.
 
 Join the [discord](https://discord.gg/Dtq4JNdXnw) for updates and to give feedback (btw feedback is very appreciated, even just a quick "I did/didn't decide to use tool X for reason Y, though am busy and don't have time to elaborate" is helpful to know)
 

{forecasting_tools-0.3.0 → forecasting_tools-0.3.2}/README.md

@@ -18,7 +18,7 @@ Demo repo (get a Metaculus bot running in 30min): https://github.com/Metaculus/m
 This repository contains forecasting and research tools built with Python and Streamlit. The project aims to assist users in making predictions, conducting research, and analyzing data related to hard to answer questions (especially those from Metaculus).
 
 Here are the tools most likely to be useful to you:
-- 🎯 **Forecasting Bot:** General forecaster that integrates with the Metaculus AI benchmarking competition and provides a number of utilities. You can forecast with a pre-existing bot or override the class to customize your own (without redoing all the aggregation/API code, etc)
+- 🎯 **Forecasting Bot:** General forecaster that integrates with the Metaculus FutureEval bot tournament and provides a number of utilities. You can forecast with a pre-existing bot or override the class to customize your own (without redoing all the aggregation/API code, etc)
 - 🔌 **Metaculus API Wrapper:** for interacting with questions and tournaments
 - 🤖 **In-House Metaculus Bots**: You can see all the bots that Metaculus is running on their site in `run_bots.py`
 
@@ -29,7 +29,7 @@ Here are some other features of the project (not all are documented yet):
 
 All the examples below are in a Jupyter Notebook called `README.ipynb` which you can run locally to test the package (make sure to run the first cell though).
 
-If you decide you want to join the Metaculus AI Benchmarking Tournament, it is recommended that you start [here](https://github.com/Metaculus/metac-bot-template). This repo is easy to start with and has a 30min tutorial for how to set it up.
+If you decide you want to join the Metaculus FutureEval Bot Tournament, it is recommended that you start [here](https://github.com/Metaculus/metac-bot-template). This repo is easy to start with and has a 30min tutorial for how to set it up.
 
 Join the [discord](https://discord.gg/Dtq4JNdXnw) for updates and to give feedback (btw feedback is very appreciated, even just a quick "I did/didn't decide to use tool X for reason Y, though am busy and don't have time to elaborate" is helpful to know)
 

{forecasting_tools-0.3.0 → forecasting_tools-0.3.2}/forecasting_tools/__init__.py

@@ -150,6 +150,9 @@ from forecasting_tools.forecast_bots.official_bots.research_only_bot_2025_fall i
 from forecasting_tools.forecast_bots.official_bots.template_bot_2025_fall import (
     FallTemplateBot2025 as FallTemplateBot2025,
 )
+from forecasting_tools.forecast_bots.official_bots.template_bot_2026_fall import (
+    FallTemplateBot2026 as FallTemplateBot2026,
+)
 from forecasting_tools.forecast_bots.official_bots.template_bot_2026_spring import (
     SpringTemplateBot2026 as SpringTemplateBot2026,
 )

{forecasting_tools-0.3.0 → forecasting_tools-0.3.2}/forecasting_tools/ai_models/ai_utils/response_types.py

@@ -17,3 +17,4 @@ class TextTokenResponse(ModelResponse):
 
 class TextTokenCostResponse(TextTokenResponse):
     cost: float
+    serving_provider: str | None = None

{forecasting_tools-0.3.0 → forecasting_tools-0.3.2}/forecasting_tools/ai_models/general_llm.py

@@ -4,6 +4,7 @@ import asyncio
 import inspect
 import logging
 import os
+import re
 from typing import Any, Literal
 
 import litellm
@@ -57,6 +58,25 @@ class GeneralLlm(
     Litellm support every model, most every parameter, and acts as one interface for every provider.
     """
 
+    _logged_model_provider_pairs: set[tuple[str, str]] = set()
+
+    _CREDENTIAL_KWARGS: frozenset[str] = frozenset(
+        {
+            "api_key",
+            "extra_headers",
+            "headers",
+            "model_list",
+            "aws_access_key_id",
+            "aws_secret_access_key",
+            "aws_session_token",
+            "aws_web_identity_token",
+            "vertex_credentials",
+            "azure_ad_token",
+            "azure_password",
+            "client_secret",
+        }
+    )
+
     _defaults: dict[str, Any] = {
         # The lowest matching model substring is used as default (default 60s timeout)
         "gpt-4o": {
@@ -318,6 +338,7 @@ class GeneralLlm(
         ModelTracker.give_cost_tracking_warning_if_needed(
             self._litellm_model, observed_no_cost=observed_no_cost
         )
+        serving_provider = self._get_checked_serving_provider(response)
 
         if self.populate_citations:
             citations = self._extract_citations(response, choices)
@@ -341,6 +362,7 @@ class GeneralLlm(
             total_tokens_used=total_tokens,
             model=self.model,
             cost=direct_cost,
+            serving_provider=serving_provider,
         )
 
         return response
@@ -390,6 +412,58 @@ class GeneralLlm(
         )
         return mentions_temperature and is_unsupported
 
+    def _get_checked_serving_provider(self, response: ModelResponse) -> str | None:
+        serving_provider = typeguard.check_type(
+            (response.model_extra or {}).get("provider"), str | None
+        )
+        self._error_if_not_served_by_pinned_provider(serving_provider)
+        if serving_provider is None:
+            return None
+        model_and_provider = (self.model, serving_provider)
+        if model_and_provider not in GeneralLlm._logged_model_provider_pairs:
+            GeneralLlm._logged_model_provider_pairs.add(model_and_provider)
+            logger.info(
+                f"{self.model} is being served by provider '{serving_provider}'"
+            )
+        return serving_provider
+
+    def _get_pinned_provider_slugs(self) -> list[str]:
+        provider_routing = (self.litellm_kwargs.get("extra_body") or {}).get(
+            "provider"
+        ) or {}
+        if provider_routing.get("allow_fallbacks", True) is not False:
+            return []
+        return provider_routing.get("order", [])
+
+    def _error_if_not_served_by_pinned_provider(
+        self, serving_provider: str | None
+    ) -> None:
+        pinned_slugs = self._get_pinned_provider_slugs()
+        if not pinned_slugs:
+            return
+        if serving_provider is None or not any(
+            self._provider_name_matches_slug(serving_provider, slug)
+            for slug in pinned_slugs
+        ):
+            raise RuntimeError(
+                f"{self.model} is pinned to {pinned_slugs} with fallbacks disabled, "
+                f"but was served by provider '{serving_provider}'"
+            )
+
+    @staticmethod
+    def _provider_name_matches_slug(provider_name: str, endpoint_slug: str) -> bool:
+        """
+        OpenRouter reports display names that can be shorter or longer than the slug
+        (e.g. "Google" for "google-vertex", "Mancer 2" for "mancer"), so match on prefix.
+        """
+        normalized_name = re.sub(r"[^a-z0-9]", "", provider_name.lower())
+        normalized_slug = re.sub(r"[^a-z0-9]", "", endpoint_slug.split("/")[0].lower())
+        if not normalized_name or not normalized_slug:
+            return False
+        return normalized_slug.startswith(
+            normalized_name
+        ) or normalized_name.startswith(normalized_slug)
+
     def _answer_from_reasoning_content(self, message: Message) -> str | None:
         reasoning_content = getattr(message, "reasoning_content", None)
         if isinstance(reasoning_content, str) and reasoning_content:
@@ -494,6 +568,7 @@ class GeneralLlm(
         setattr(model_response, "choices", choices)
 
         model_response.model = self.model
+        model_response._hidden_params = raw_response._hidden_params
 
         setattr(
             model_response,
@@ -620,7 +695,11 @@ class GeneralLlm(
         return {
             "original_model": self.model,
             "allowed_tries": self.allowed_tries,
-            **{k: v for k, v in self.litellm_kwargs.items()},
+            **{
+                key: value
+                for key, value in self.litellm_kwargs.items()
+                if key not in self._CREDENTIAL_KWARGS
+            },
         }
 
     @classmethod

{forecasting_tools-0.3.0 → forecasting_tools-0.3.2}/forecasting_tools/data_models/questions.py

@@ -340,6 +340,10 @@ class BinaryQuestion(MetaculusQuestion):
                 q2_center_community_prediction = aggregations["unweighted"]["latest"]["centers"]  # type: ignore
             assert len(q2_center_community_prediction) == 1
             community_prediction_at_access_time = q2_center_community_prediction[0]
+        except (KeyError, TypeError):
+            community_prediction_at_access_time = None
+
+        try:
             history = api_json["question"]["my_forecasts"]["history"]
             previous_forecasts = None
             if history:
@@ -360,7 +364,6 @@ class BinaryQuestion(MetaculusQuestion):
                     for forecast in history
                 ]
         except (KeyError, TypeError):
-            community_prediction_at_access_time = None
             previous_forecasts = None
         return BinaryQuestion(
             community_prediction_at_access_time=community_prediction_at_access_time,

{forecasting_tools-0.3.0 → forecasting_tools-0.3.2}/forecasting_tools/forecast_bots/bot_lists.py

@@ -32,6 +32,9 @@ from forecasting_tools.forecast_bots.official_bots.research_only_bot_2025_fall i
 from forecasting_tools.forecast_bots.official_bots.template_bot_2025_fall import (
     FallTemplateBot2025,
 )
+from forecasting_tools.forecast_bots.official_bots.template_bot_2026_fall import (
+    FallTemplateBot2026,
+)
 from forecasting_tools.forecast_bots.official_bots.template_bot_2026_spring import (
     SpringTemplateBot2026,
 )
@@ -62,6 +65,7 @@ def get_all_important_bot_classes() -> list[type[ForecastBot]]:
         FallResearchOnlyBot2025,
         SpringTemplateBot2026,
         SummerTemplateBot2026,
+        FallTemplateBot2026,
         GPT41OptimizedBot,
         NoResearchOneShotBot,
     ]

{forecasting_tools-0.3.0 → forecasting_tools-0.3.2}/forecasting_tools/forecast_bots/main_bot.py

@@ -42,10 +42,12 @@ class MainBot(Q2TemplateBot2025):
     def _llm_config_defaults(cls) -> dict[str, str | GeneralLlm]:
         return {
             "default": GeneralLlm(
-                model="openai/gpt-5",
-                reasoning_effort="high",
-                temperature=0.3,
-                timeout=15 * 60,
+                model="anthropic/claude-opus-5",
+                thinking={"type": "adaptive"},
+                output_config={"effort": "high"},
+                allowed_openai_params=["output_config"],
+                max_tokens=64000,
+                timeout=160,
             ),
             "summarizer": GeneralLlm(model="openai/gpt-4o", temperature=0),
         }