forecasting-tools 0.3.2__tar.gz → 0.3.3__tar.gz

{forecasting_tools-0.3.2 → forecasting_tools-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forecasting-tools
-Version: 0.3.2
+Version: 0.3.3
 Summary: AI forecasting and research tools to help humans reason about and forecast the future
 License: MIT
 License-File: LICENSE
@@ -207,7 +207,7 @@ Note: You'll need to have your environment variables set up (see the section bel
 
 ## Customizing the Bot
 ### General Customization
-Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, creating benchmarking interfaces, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
+Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
 
 
 ```python
@@ -380,11 +380,6 @@ file_path = "temp/questions.json"
 DataOrganizer.save_questions_to_file_path(questions, file_path) # Overwrites the file if it already exists
 questions = DataOrganizer.load_questions_from_file_path(file_path)
 
-benchmark_questions = metaculus_client.get_benchmark_questions(
-    num_of_questions_to_return=20
-)
-print(f"Num benchmark questions: {len(benchmark_questions)}")
-
 metaculus_client.post_binary_question_prediction(
     question_id=578, # Note that the question ID is not always the same as the post ID
     prediction_in_decimal=0.012,  # Must be between 0.001 and 0.999
@@ -402,7 +397,6 @@ print("Posted comment")
     Question found with url: https://www.metaculus.com/questions/578
     Num tournament questions: 11
     Num filtered questions: 50
-    Num benchmark questions: 20
     Posted prediction
     Posted comment
 
@@ -413,7 +407,7 @@ Several of the methods above accept a `group_question_mode` parameter that contr
 - `"exclude"` — drop group questions from the result.
 - `"unpack_subquestions"` — turn each subquestion into a separate normal question.
 
-For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, `ApiFilter` (used by `get_questions_matching_filter`), and `get_benchmark_questions` — so group questions don't get overweighted in benchmarks. The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
+For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, and `ApiFilter` (used by `get_questions_matching_filter`). The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
 
 ```python
 from forecasting_tools import MetaculusClient, ApiFilter
@@ -572,8 +566,8 @@ Some features need heavier dependencies that a bot does not, so they are opt-in.
 
 | Extra | You need it for |
 | --- | --- |
-| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `Benchmarker`, `BenchmarkForBot`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
-| `front-end` | The Streamlit app in `forecasting_tools/front_end/` and `run_benchmark_streamlit_page` |
+| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
+| `front-end` | The Streamlit app in `forecasting_tools/front_end/` |
 | `stats` | `forecasting_tools.util.stats` and the calibration adjusters in `forecasting_tools/calibration_adjustment/` |
 | `all` | Everything above |
 
@@ -624,7 +618,7 @@ If you choose not to run Docker, you can use poetry to set up a local virtual en
 
 
 ## Installing dependencies
-Local development uses the optional extras (the tests cover the front end, the agent tools, and benchmarking), so install with:
+Local development uses the optional extras (the tests cover the front end and the agent tools), so install with:
 
 ```bash
 poetry install --all-extras

{forecasting_tools-0.3.2 → forecasting_tools-0.3.3}/README.md

@@ -145,7 +145,7 @@ Note: You'll need to have your environment variables set up (see the section bel
 
 ## Customizing the Bot
 ### General Customization
-Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, creating benchmarking interfaces, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
+Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
 
 
 ```python
@@ -318,11 +318,6 @@ file_path = "temp/questions.json"
 DataOrganizer.save_questions_to_file_path(questions, file_path) # Overwrites the file if it already exists
 questions = DataOrganizer.load_questions_from_file_path(file_path)
 
-benchmark_questions = metaculus_client.get_benchmark_questions(
-    num_of_questions_to_return=20
-)
-print(f"Num benchmark questions: {len(benchmark_questions)}")
-
 metaculus_client.post_binary_question_prediction(
     question_id=578, # Note that the question ID is not always the same as the post ID
     prediction_in_decimal=0.012,  # Must be between 0.001 and 0.999
@@ -340,7 +335,6 @@ print("Posted comment")
     Question found with url: https://www.metaculus.com/questions/578
     Num tournament questions: 11
     Num filtered questions: 50
-    Num benchmark questions: 20
     Posted prediction
     Posted comment
 
@@ -351,7 +345,7 @@ Several of the methods above accept a `group_question_mode` parameter that contr
 - `"exclude"` — drop group questions from the result.
 - `"unpack_subquestions"` — turn each subquestion into a separate normal question.
 
-For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, `ApiFilter` (used by `get_questions_matching_filter`), and `get_benchmark_questions` — so group questions don't get overweighted in benchmarks. The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
+For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, and `ApiFilter` (used by `get_questions_matching_filter`). The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
 
 ```python
 from forecasting_tools import MetaculusClient, ApiFilter
@@ -510,8 +504,8 @@ Some features need heavier dependencies that a bot does not, so they are opt-in.
 
 | Extra | You need it for |
 | --- | --- |
-| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `Benchmarker`, `BenchmarkForBot`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
-| `front-end` | The Streamlit app in `forecasting_tools/front_end/` and `run_benchmark_streamlit_page` |
+| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
+| `front-end` | The Streamlit app in `forecasting_tools/front_end/` |
 | `stats` | `forecasting_tools.util.stats` and the calibration adjusters in `forecasting_tools/calibration_adjustment/` |
 | `all` | Everything above |
 
@@ -562,7 +556,7 @@ If you choose not to run Docker, you can use poetry to set up a local virtual en
 
 
 ## Installing dependencies
-Local development uses the optional extras (the tests cover the front end, the agent tools, and benchmarking), so install with:
+Local development uses the optional extras (the tests cover the front end and the agent tools), so install with:
 
 ```bash
 poetry install --all-extras

{forecasting_tools-0.3.2 → forecasting_tools-0.3.3}/forecasting_tools/__init__.py

@@ -172,6 +172,9 @@ from forecasting_tools.helpers.asknews_searcher import (
 from forecasting_tools.helpers.metaculus_api import MetaculusApi as MetaculusApi
 from forecasting_tools.helpers.metaculus_api import MetaculusClient as MetaculusClient
 from forecasting_tools.helpers.metaculus_client import ApiFilter as ApiFilter
+from forecasting_tools.helpers.metaculus_client import (
+    AuthorMismatchError as AuthorMismatchError,
+)
 from forecasting_tools.helpers.prediction_extractor import (
     PredictionExtractor as PredictionExtractor,
 )

{forecasting_tools-0.3.2 → forecasting_tools-0.3.3}/forecasting_tools/helpers/metaculus_client.py

@@ -101,6 +101,10 @@ class ApiFilter(BaseModel):
         return add_timezone_to_dates_in_base_model(self)
 
 
+class AuthorMismatchError(ValueError):
+    pass
+
+
 class MetaculusClient:
     """
     Documentation for the API can be found at https://www.metaculus.com/api/
@@ -1212,8 +1216,14 @@ class MetaculusClient:
         logger.info(f"Unresolved question ID {question_id}")
 
     @retry_with_exponential_backoff()
-    def create_question(self, question: MetaculusQuestion) -> MetaculusQuestion:
-        question_data = self._get_post_create_data(question)
+    def create_question(
+        self, question: MetaculusQuestion, author_username: str | None = None
+    ) -> MetaculusQuestion:
+        """
+        Setting author_username creates the post as that user, which requires
+        the API token to belong to a Metaculus superuser.
+        """
+        question_data = self._get_post_create_data(question, author_username)
         self._sleep_between_requests()
         response = requests.post(
             f"{self.base_url}/posts/create/",
@@ -1241,6 +1251,16 @@ class MetaculusClient:
             single_created_question.id_of_post
         )
 
+        if author_username is not None:
+            actual_author = full_created_question.api_json.get("author_username")
+            if actual_author != author_username:
+                raise AuthorMismatchError(
+                    f"Post {full_created_question.id_of_post} was created with author "
+                    f"{actual_author!r} instead of {author_username!r} and has not been "
+                    "approved. Either the API token is not a superuser or the server "
+                    "ignored the author override."
+                )
+
         return full_created_question
 
     @retry_with_exponential_backoff()
@@ -1286,12 +1306,20 @@ class MetaculusClient:
         logger.info(f"Approved question {question.page_url}")
 
     @staticmethod
-    def _get_post_create_data(question: MetaculusQuestion) -> dict:
+    def _get_post_create_data(
+        question: MetaculusQuestion, author_username: str | None = None
+    ) -> dict:
         if question.published_time is None:
             publish_time = question.open_time
         else:
             publish_time = question.published_time
 
+        author_override_fields = (
+            {"author_username": author_username, "is_staff_override": True}
+            if author_username is not None
+            else {}
+        )
+
         return {
             "title": question.question_text,
             "short_title": question.custom_metadata.get("short_name", None)
@@ -1355,4 +1383,5 @@ class MetaculusClient:
                 "fine_print": question.fine_print or "",
                 "group_rank": None,  # only group questions
             },
+            **author_override_fields,
         }

{forecasting_tools-0.3.2 → forecasting_tools-0.3.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "forecasting-tools"
-version = "0.3.2"
+version = "0.3.3"
 description = "AI forecasting and research tools to help humans reason about and forecast the future"
 authors = ["Benjamin Wilson <ben@metaculus.com>"]
 license = "MIT"