forecasting-tools 0.3.1__tar.gz → 0.3.3__tar.gz

{forecasting_tools-0.3.1 → forecasting_tools-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forecasting-tools
-Version: 0.3.1
+Version: 0.3.3
 Summary: AI forecasting and research tools to help humans reason about and forecast the future
 License: MIT
 License-File: LICENSE
@@ -207,7 +207,7 @@ Note: You'll need to have your environment variables set up (see the section bel
 
 ## Customizing the Bot
 ### General Customization
-Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, creating benchmarking interfaces, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
+Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
 
 
 ```python
@@ -380,11 +380,6 @@ file_path = "temp/questions.json"
 DataOrganizer.save_questions_to_file_path(questions, file_path) # Overwrites the file if it already exists
 questions = DataOrganizer.load_questions_from_file_path(file_path)
 
-benchmark_questions = metaculus_client.get_benchmark_questions(
-    num_of_questions_to_return=20
-)
-print(f"Num benchmark questions: {len(benchmark_questions)}")
-
 metaculus_client.post_binary_question_prediction(
     question_id=578, # Note that the question ID is not always the same as the post ID
     prediction_in_decimal=0.012,  # Must be between 0.001 and 0.999
@@ -402,7 +397,6 @@ print("Posted comment")
     Question found with url: https://www.metaculus.com/questions/578
     Num tournament questions: 11
     Num filtered questions: 50
-    Num benchmark questions: 20
     Posted prediction
     Posted comment
 
@@ -413,7 +407,7 @@ Several of the methods above accept a `group_question_mode` parameter that contr
 - `"exclude"` — drop group questions from the result.
 - `"unpack_subquestions"` — turn each subquestion into a separate normal question.
 
-For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, `ApiFilter` (used by `get_questions_matching_filter`), and `get_benchmark_questions` — so group questions don't get overweighted in benchmarks. The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
+For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, and `ApiFilter` (used by `get_questions_matching_filter`). The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
 
 ```python
 from forecasting_tools import MetaculusClient, ApiFilter
@@ -572,8 +566,8 @@ Some features need heavier dependencies that a bot does not, so they are opt-in.
 
 | Extra | You need it for |
 | --- | --- |
-| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `Benchmarker`, `BenchmarkForBot`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
-| `front-end` | The Streamlit app in `forecasting_tools/front_end/` and `run_benchmark_streamlit_page` |
+| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
+| `front-end` | The Streamlit app in `forecasting_tools/front_end/` |
 | `stats` | `forecasting_tools.util.stats` and the calibration adjusters in `forecasting_tools/calibration_adjustment/` |
 | `all` | Everything above |
 
@@ -624,7 +618,7 @@ If you choose not to run Docker, you can use poetry to set up a local virtual en
 
 
 ## Installing dependencies
-Local development uses the optional extras (the tests cover the front end, the agent tools, and benchmarking), so install with:
+Local development uses the optional extras (the tests cover the front end and the agent tools), so install with:
 
 ```bash
 poetry install --all-extras

{forecasting_tools-0.3.1 → forecasting_tools-0.3.3}/README.md

@@ -145,7 +145,7 @@ Note: You'll need to have your environment variables set up (see the section bel
 
 ## Customizing the Bot
 ### General Customization
-Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, creating benchmarking interfaces, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
+Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
 
 
 ```python
@@ -318,11 +318,6 @@ file_path = "temp/questions.json"
 DataOrganizer.save_questions_to_file_path(questions, file_path) # Overwrites the file if it already exists
 questions = DataOrganizer.load_questions_from_file_path(file_path)
 
-benchmark_questions = metaculus_client.get_benchmark_questions(
-    num_of_questions_to_return=20
-)
-print(f"Num benchmark questions: {len(benchmark_questions)}")
-
 metaculus_client.post_binary_question_prediction(
     question_id=578, # Note that the question ID is not always the same as the post ID
     prediction_in_decimal=0.012,  # Must be between 0.001 and 0.999
@@ -340,7 +335,6 @@ print("Posted comment")
     Question found with url: https://www.metaculus.com/questions/578
     Num tournament questions: 11
     Num filtered questions: 50
-    Num benchmark questions: 20
     Posted prediction
     Posted comment
 
@@ -351,7 +345,7 @@ Several of the methods above accept a `group_question_mode` parameter that contr
 - `"exclude"` — drop group questions from the result.
 - `"unpack_subquestions"` — turn each subquestion into a separate normal question.
 
-For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, `ApiFilter` (used by `get_questions_matching_filter`), and `get_benchmark_questions` — so group questions don't get overweighted in benchmarks. The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
+For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, and `ApiFilter` (used by `get_questions_matching_filter`). The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
 
 ```python
 from forecasting_tools import MetaculusClient, ApiFilter
@@ -510,8 +504,8 @@ Some features need heavier dependencies that a bot does not, so they are opt-in.
 
 | Extra | You need it for |
 | --- | --- |
-| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `Benchmarker`, `BenchmarkForBot`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
-| `front-end` | The Streamlit app in `forecasting_tools/front_end/` and `run_benchmark_streamlit_page` |
+| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
+| `front-end` | The Streamlit app in `forecasting_tools/front_end/` |
 | `stats` | `forecasting_tools.util.stats` and the calibration adjusters in `forecasting_tools/calibration_adjustment/` |
 | `all` | Everything above |
 
@@ -562,7 +556,7 @@ If you choose not to run Docker, you can use poetry to set up a local virtual en
 
 
 ## Installing dependencies
-Local development uses the optional extras (the tests cover the front end, the agent tools, and benchmarking), so install with:
+Local development uses the optional extras (the tests cover the front end and the agent tools), so install with:
 
 ```bash
 poetry install --all-extras

{forecasting_tools-0.3.1 → forecasting_tools-0.3.3}/forecasting_tools/__init__.py

@@ -172,6 +172,9 @@ from forecasting_tools.helpers.asknews_searcher import (
 from forecasting_tools.helpers.metaculus_api import MetaculusApi as MetaculusApi
 from forecasting_tools.helpers.metaculus_api import MetaculusClient as MetaculusClient
 from forecasting_tools.helpers.metaculus_client import ApiFilter as ApiFilter
+from forecasting_tools.helpers.metaculus_client import (
+    AuthorMismatchError as AuthorMismatchError,
+)
 from forecasting_tools.helpers.prediction_extractor import (
     PredictionExtractor as PredictionExtractor,
 )

{forecasting_tools-0.3.1 → forecasting_tools-0.3.3}/forecasting_tools/ai_models/ai_utils/response_types.py

@@ -17,3 +17,4 @@ class TextTokenResponse(ModelResponse):
 
 class TextTokenCostResponse(TextTokenResponse):
     cost: float
+    serving_provider: str | None = None

{forecasting_tools-0.3.1 → forecasting_tools-0.3.3}/forecasting_tools/ai_models/general_llm.py

@@ -4,6 +4,7 @@ import asyncio
 import inspect
 import logging
 import os
+import re
 from typing import Any, Literal
 
 import litellm
@@ -57,6 +58,25 @@ class GeneralLlm(
     Litellm support every model, most every parameter, and acts as one interface for every provider.
     """
 
+    _logged_model_provider_pairs: set[tuple[str, str]] = set()
+
+    _CREDENTIAL_KWARGS: frozenset[str] = frozenset(
+        {
+            "api_key",
+            "extra_headers",
+            "headers",
+            "model_list",
+            "aws_access_key_id",
+            "aws_secret_access_key",
+            "aws_session_token",
+            "aws_web_identity_token",
+            "vertex_credentials",
+            "azure_ad_token",
+            "azure_password",
+            "client_secret",
+        }
+    )
+
     _defaults: dict[str, Any] = {
         # The lowest matching model substring is used as default (default 60s timeout)
         "gpt-4o": {
@@ -318,6 +338,7 @@ class GeneralLlm(
         ModelTracker.give_cost_tracking_warning_if_needed(
             self._litellm_model, observed_no_cost=observed_no_cost
         )
+        serving_provider = self._get_checked_serving_provider(response)
 
         if self.populate_citations:
             citations = self._extract_citations(response, choices)
@@ -341,6 +362,7 @@ class GeneralLlm(
             total_tokens_used=total_tokens,
             model=self.model,
             cost=direct_cost,
+            serving_provider=serving_provider,
         )
 
         return response
@@ -390,6 +412,58 @@ class GeneralLlm(
         )
         return mentions_temperature and is_unsupported
 
+    def _get_checked_serving_provider(self, response: ModelResponse) -> str | None:
+        serving_provider = typeguard.check_type(
+            (response.model_extra or {}).get("provider"), str | None
+        )
+        self._error_if_not_served_by_pinned_provider(serving_provider)
+        if serving_provider is None:
+            return None
+        model_and_provider = (self.model, serving_provider)
+        if model_and_provider not in GeneralLlm._logged_model_provider_pairs:
+            GeneralLlm._logged_model_provider_pairs.add(model_and_provider)
+            logger.info(
+                f"{self.model} is being served by provider '{serving_provider}'"
+            )
+        return serving_provider
+
+    def _get_pinned_provider_slugs(self) -> list[str]:
+        provider_routing = (self.litellm_kwargs.get("extra_body") or {}).get(
+            "provider"
+        ) or {}
+        if provider_routing.get("allow_fallbacks", True) is not False:
+            return []
+        return provider_routing.get("order", [])
+
+    def _error_if_not_served_by_pinned_provider(
+        self, serving_provider: str | None
+    ) -> None:
+        pinned_slugs = self._get_pinned_provider_slugs()
+        if not pinned_slugs:
+            return
+        if serving_provider is None or not any(
+            self._provider_name_matches_slug(serving_provider, slug)
+            for slug in pinned_slugs
+        ):
+            raise RuntimeError(
+                f"{self.model} is pinned to {pinned_slugs} with fallbacks disabled, "
+                f"but was served by provider '{serving_provider}'"
+            )
+
+    @staticmethod
+    def _provider_name_matches_slug(provider_name: str, endpoint_slug: str) -> bool:
+        """
+        OpenRouter reports display names that can be shorter or longer than the slug
+        (e.g. "Google" for "google-vertex", "Mancer 2" for "mancer"), so match on prefix.
+        """
+        normalized_name = re.sub(r"[^a-z0-9]", "", provider_name.lower())
+        normalized_slug = re.sub(r"[^a-z0-9]", "", endpoint_slug.split("/")[0].lower())
+        if not normalized_name or not normalized_slug:
+            return False
+        return normalized_slug.startswith(
+            normalized_name
+        ) or normalized_name.startswith(normalized_slug)
+
     def _answer_from_reasoning_content(self, message: Message) -> str | None:
         reasoning_content = getattr(message, "reasoning_content", None)
         if isinstance(reasoning_content, str) and reasoning_content:
@@ -494,6 +568,7 @@ class GeneralLlm(
         setattr(model_response, "choices", choices)
 
         model_response.model = self.model
+        model_response._hidden_params = raw_response._hidden_params
 
         setattr(
             model_response,
@@ -620,7 +695,11 @@ class GeneralLlm(
         return {
             "original_model": self.model,
             "allowed_tries": self.allowed_tries,
-            **{k: v for k, v in self.litellm_kwargs.items()},
+            **{
+                key: value
+                for key, value in self.litellm_kwargs.items()
+                if key not in self._CREDENTIAL_KWARGS
+            },
         }
 
     @classmethod

{forecasting_tools-0.3.1 → forecasting_tools-0.3.3}/forecasting_tools/data_models/questions.py

@@ -340,6 +340,10 @@ class BinaryQuestion(MetaculusQuestion):
                 q2_center_community_prediction = aggregations["unweighted"]["latest"]["centers"]  # type: ignore
             assert len(q2_center_community_prediction) == 1
             community_prediction_at_access_time = q2_center_community_prediction[0]
+        except (KeyError, TypeError):
+            community_prediction_at_access_time = None
+
+        try:
             history = api_json["question"]["my_forecasts"]["history"]
             previous_forecasts = None
             if history:
@@ -360,7 +364,6 @@ class BinaryQuestion(MetaculusQuestion):
                     for forecast in history
                 ]
         except (KeyError, TypeError):
-            community_prediction_at_access_time = None
             previous_forecasts = None
         return BinaryQuestion(
             community_prediction_at_access_time=community_prediction_at_access_time,

{forecasting_tools-0.3.1 → forecasting_tools-0.3.3}/forecasting_tools/helpers/metaculus_client.py

@@ -101,6 +101,10 @@ class ApiFilter(BaseModel):
         return add_timezone_to_dates_in_base_model(self)
 
 
+class AuthorMismatchError(ValueError):
+    pass
+
+
 class MetaculusClient:
     """
     Documentation for the API can be found at https://www.metaculus.com/api/
@@ -1212,8 +1216,14 @@ class MetaculusClient:
         logger.info(f"Unresolved question ID {question_id}")
 
     @retry_with_exponential_backoff()
-    def create_question(self, question: MetaculusQuestion) -> MetaculusQuestion:
-        question_data = self._get_post_create_data(question)
+    def create_question(
+        self, question: MetaculusQuestion, author_username: str | None = None
+    ) -> MetaculusQuestion:
+        """
+        Setting author_username creates the post as that user, which requires
+        the API token to belong to a Metaculus superuser.
+        """
+        question_data = self._get_post_create_data(question, author_username)
         self._sleep_between_requests()
         response = requests.post(
             f"{self.base_url}/posts/create/",
@@ -1241,6 +1251,16 @@ class MetaculusClient:
             single_created_question.id_of_post
         )
 
+        if author_username is not None:
+            actual_author = full_created_question.api_json.get("author_username")
+            if actual_author != author_username:
+                raise AuthorMismatchError(
+                    f"Post {full_created_question.id_of_post} was created with author "
+                    f"{actual_author!r} instead of {author_username!r} and has not been "
+                    "approved. Either the API token is not a superuser or the server "
+                    "ignored the author override."
+                )
+
         return full_created_question
 
     @retry_with_exponential_backoff()
@@ -1286,12 +1306,20 @@ class MetaculusClient:
         logger.info(f"Approved question {question.page_url}")
 
     @staticmethod
-    def _get_post_create_data(question: MetaculusQuestion) -> dict:
+    def _get_post_create_data(
+        question: MetaculusQuestion, author_username: str | None = None
+    ) -> dict:
         if question.published_time is None:
             publish_time = question.open_time
         else:
             publish_time = question.published_time
 
+        author_override_fields = (
+            {"author_username": author_username, "is_staff_override": True}
+            if author_username is not None
+            else {}
+        )
+
         return {
             "title": question.question_text,
             "short_title": question.custom_metadata.get("short_name", None)
@@ -1355,4 +1383,5 @@ class MetaculusClient:
                 "fine_print": question.fine_print or "",
                 "group_rank": None,  # only group questions
             },
+            **author_override_fields,
         }

{forecasting_tools-0.3.1 → forecasting_tools-0.3.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "forecasting-tools"
-version = "0.3.1"
+version = "0.3.3"
 description = "AI forecasting and research tools to help humans reason about and forecast the future"
 authors = ["Benjamin Wilson <ben@metaculus.com>"]
 license = "MIT"