forecasting-tools 0.3.2__tar.gz → 0.3.4__tar.gz

{forecasting_tools-0.3.2 → forecasting_tools-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forecasting-tools
-Version: 0.3.2
+Version: 0.3.4
 Summary: AI forecasting and research tools to help humans reason about and forecast the future
 License: MIT
 License-File: LICENSE
@@ -207,7 +207,7 @@ Note: You'll need to have your environment variables set up (see the section bel
 
 ## Customizing the Bot
 ### General Customization
-Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, creating benchmarking interfaces, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
+Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
 
 
 ```python
@@ -380,11 +380,6 @@ file_path = "temp/questions.json"
 DataOrganizer.save_questions_to_file_path(questions, file_path) # Overwrites the file if it already exists
 questions = DataOrganizer.load_questions_from_file_path(file_path)
 
-benchmark_questions = metaculus_client.get_benchmark_questions(
-    num_of_questions_to_return=20
-)
-print(f"Num benchmark questions: {len(benchmark_questions)}")
-
 metaculus_client.post_binary_question_prediction(
     question_id=578, # Note that the question ID is not always the same as the post ID
     prediction_in_decimal=0.012,  # Must be between 0.001 and 0.999
@@ -402,7 +397,6 @@ print("Posted comment")
     Question found with url: https://www.metaculus.com/questions/578
     Num tournament questions: 11
     Num filtered questions: 50
-    Num benchmark questions: 20
     Posted prediction
     Posted comment
 
@@ -413,7 +407,7 @@ Several of the methods above accept a `group_question_mode` parameter that contr
 - `"exclude"` — drop group questions from the result.
 - `"unpack_subquestions"` — turn each subquestion into a separate normal question.
 
-For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, `ApiFilter` (used by `get_questions_matching_filter`), and `get_benchmark_questions` — so group questions don't get overweighted in benchmarks. The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
+For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, and `ApiFilter` (used by `get_questions_matching_filter`). The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
 
 ```python
 from forecasting_tools import MetaculusClient, ApiFilter
@@ -572,8 +566,8 @@ Some features need heavier dependencies that a bot does not, so they are opt-in.
 
 | Extra | You need it for |
 | --- | --- |
-| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `Benchmarker`, `BenchmarkForBot`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
-| `front-end` | The Streamlit app in `forecasting_tools/front_end/` and `run_benchmark_streamlit_page` |
+| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
+| `front-end` | The Streamlit app in `forecasting_tools/front_end/` |
 | `stats` | `forecasting_tools.util.stats` and the calibration adjusters in `forecasting_tools/calibration_adjustment/` |
 | `all` | Everything above |
 
@@ -624,7 +618,7 @@ If you choose not to run Docker, you can use poetry to set up a local virtual en
 
 
 ## Installing dependencies
-Local development uses the optional extras (the tests cover the front end, the agent tools, and benchmarking), so install with:
+Local development uses the optional extras (the tests cover the front end and the agent tools), so install with:
 
 ```bash
 poetry install --all-extras

{forecasting_tools-0.3.2 → forecasting_tools-0.3.4}/README.md

@@ -145,7 +145,7 @@ Note: You'll need to have your environment variables set up (see the section bel
 
 ## Customizing the Bot
 ### General Customization
-Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, creating benchmarking interfaces, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
+Generally all you have to do to make your own bot is inherit from the TemplateBot and override any combination of the 3 forecasting methods and the 1 research method. This saves you the headache of interacting with the Metaculus API, implementing aggregation of predictions, etc. Below is an example. It may also be helpful to look at the `TemplateBot` code (`forecasting_tools/forecast_bots/template_bot.py`) for a more complete example.
 
 
 ```python
@@ -318,11 +318,6 @@ file_path = "temp/questions.json"
 DataOrganizer.save_questions_to_file_path(questions, file_path) # Overwrites the file if it already exists
 questions = DataOrganizer.load_questions_from_file_path(file_path)
 
-benchmark_questions = metaculus_client.get_benchmark_questions(
-    num_of_questions_to_return=20
-)
-print(f"Num benchmark questions: {len(benchmark_questions)}")
-
 metaculus_client.post_binary_question_prediction(
     question_id=578, # Note that the question ID is not always the same as the post ID
     prediction_in_decimal=0.012,  # Must be between 0.001 and 0.999
@@ -340,7 +335,6 @@ print("Posted comment")
     Question found with url: https://www.metaculus.com/questions/578
     Num tournament questions: 11
     Num filtered questions: 50
-    Num benchmark questions: 20
     Posted prediction
     Posted comment
 
@@ -351,7 +345,7 @@ Several of the methods above accept a `group_question_mode` parameter that contr
 - `"exclude"` — drop group questions from the result.
 - `"unpack_subquestions"` — turn each subquestion into a separate normal question.
 
-For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, `ApiFilter` (used by `get_questions_matching_filter`), and `get_benchmark_questions` — so group questions don't get overweighted in benchmarks. The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
+For backwards compatibility, the default is `"exclude"` for `get_question_by_post_id`, `get_question_by_url`, and `ApiFilter` (used by `get_questions_matching_filter`). The exception is `get_all_open_questions_from_tournament`, which defaults to `"unpack_subquestions"` so all subquestions are forecasted as normal questions.
 
 ```python
 from forecasting_tools import MetaculusClient, ApiFilter
@@ -510,8 +504,8 @@ Some features need heavier dependencies that a bot does not, so they are opt-in.
 
 | Extra | You need it for |
 | --- | --- |
-| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `Benchmarker`, `BenchmarkForBot`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
-| `front-end` | The Streamlit app in `forecasting_tools/front_end/` and `run_benchmark_streamlit_page` |
+| `agents` | Anything built on the agent SDK: `QuestionDecomposer`, `QuestionOperationalizer`, `TopicGenerator`, `DataAnalyzer`, `BotOptimizer`, `CustomizableBot`, `ComputerUse` (browser-driving agent), and the AI Congress tools |
+| `front-end` | The Streamlit app in `forecasting_tools/front_end/` |
 | `stats` | `forecasting_tools.util.stats` and the calibration adjusters in `forecasting_tools/calibration_adjustment/` |
 | `all` | Everything above |
 
@@ -562,7 +556,7 @@ If you choose not to run Docker, you can use poetry to set up a local virtual en
 
 
 ## Installing dependencies
-Local development uses the optional extras (the tests cover the front end, the agent tools, and benchmarking), so install with:
+Local development uses the optional extras (the tests cover the front end and the agent tools), so install with:
 
 ```bash
 poetry install --all-extras

{forecasting_tools-0.3.2 → forecasting_tools-0.3.4}/forecasting_tools/__init__.py

@@ -172,6 +172,9 @@ from forecasting_tools.helpers.asknews_searcher import (
 from forecasting_tools.helpers.metaculus_api import MetaculusApi as MetaculusApi
 from forecasting_tools.helpers.metaculus_api import MetaculusClient as MetaculusClient
 from forecasting_tools.helpers.metaculus_client import ApiFilter as ApiFilter
+from forecasting_tools.helpers.metaculus_client import (
+    AuthorMismatchError as AuthorMismatchError,
+)
 from forecasting_tools.helpers.prediction_extractor import (
     PredictionExtractor as PredictionExtractor,
 )

{forecasting_tools-0.3.2 → forecasting_tools-0.3.4}/forecasting_tools/helpers/metaculus_client.py

@@ -101,6 +101,10 @@ class ApiFilter(BaseModel):
         return add_timezone_to_dates_in_base_model(self)
 
 
+class AuthorMismatchError(ValueError):
+    pass
+
+
 class MetaculusClient:
     """
     Documentation for the API can be found at https://www.metaculus.com/api/
@@ -1212,8 +1216,15 @@ class MetaculusClient:
         logger.info(f"Unresolved question ID {question_id}")
 
     @retry_with_exponential_backoff()
-    def create_question(self, question: MetaculusQuestion) -> MetaculusQuestion:
-        question_data = self._get_post_create_data(question)
+    def create_question(
+        self, question: MetaculusQuestion, acting_user: str | int | None = None
+    ) -> MetaculusQuestion:
+        """
+        Setting acting_user (a username or user id) creates the post with that user
+        as author. The API token must own that bot, be allowed to act as metac bots,
+        or belong to a superuser.
+        """
+        question_data = self._get_post_create_data(question, acting_user)
         self._sleep_between_requests()
         response = requests.post(
             f"{self.base_url}/posts/create/",
@@ -1241,6 +1252,9 @@ class MetaculusClient:
             single_created_question.id_of_post
         )
 
+        if acting_user is not None:
+            self._raise_if_author_is_not_acting_user(full_created_question, acting_user)
+
         return full_created_question
 
     @retry_with_exponential_backoff()
@@ -1286,12 +1300,35 @@ class MetaculusClient:
         logger.info(f"Approved question {question.page_url}")
 
     @staticmethod
-    def _get_post_create_data(question: MetaculusQuestion) -> dict:
+    def _raise_if_author_is_not_acting_user(
+        question: MetaculusQuestion, acting_user: str | int
+    ) -> None:
+        if str(acting_user).isdigit():
+            actual_author = question.api_json.get("author_id")
+            expected_author = int(acting_user)
+        else:
+            actual_author = question.api_json.get("author_username")
+            expected_author = acting_user
+        if actual_author != expected_author:
+            raise AuthorMismatchError(
+                f"Post {question.id_of_post} was created with author "
+                f"{actual_author!r} instead of {expected_author!r} and has not been "
+                "approved. The server likely ignored the acting_user field."
+            )
+
+    @staticmethod
+    def _get_post_create_data(
+        question: MetaculusQuestion, acting_user: str | int | None = None
+    ) -> dict:
         if question.published_time is None:
             publish_time = question.open_time
         else:
             publish_time = question.published_time
 
+        acting_user_fields = (
+            {"acting_user": acting_user} if acting_user is not None else {}
+        )
+
         return {
             "title": question.question_text,
             "short_title": question.custom_metadata.get("short_name", None)
@@ -1355,4 +1392,5 @@ class MetaculusClient:
                 "fine_print": question.fine_print or "",
                 "group_rank": None,  # only group questions
             },
+            **acting_user_fields,
         }

{forecasting_tools-0.3.2 → forecasting_tools-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "forecasting-tools"
-version = "0.3.2"
+version = "0.3.4"
 description = "AI forecasting and research tools to help humans reason about and forecast the future"
 authors = ["Benjamin Wilson <ben@metaculus.com>"]
 license = "MIT"