forecasting-tools 0.3.3__tar.gz → 0.3.5__tar.gz

{forecasting_tools-0.3.3 → forecasting_tools-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forecasting-tools
-Version: 0.3.3
+Version: 0.3.5
 Summary: AI forecasting and research tools to help humans reason about and forecast the future
 License: MIT
 License-File: LICENSE
@@ -35,7 +35,7 @@ Requires-Dist: exceptiongroup (>=1.2.2,<2.0.0)
 Requires-Dist: faker (>=37.0.0,<41.0.0) ; extra == "agents" or extra == "front-end" or extra == "all"
 Requires-Dist: firecrawl-py (>=4.0,<5.0.0) ; extra == "source-archive" or extra == "all"
 Requires-Dist: hyperbrowser (>=0.53.0,<2.0.0) ; extra == "source-archive" or extra == "agents" or extra == "front-end" or extra == "all"
-Requires-Dist: litellm (>=1.59.1,<2.0.0,!=1.76.*,!=1.77.*,!=1.82.7,!=1.82.8)
+Requires-Dist: litellm (>=1.59.1,<2.0.0,!=1.76.*,!=1.77.*,!=1.82.7,!=1.82.8,!=1.107.*,!=1.108.*,!=1.109.*)
 Requires-Dist: nest-asyncio (>=1.5.8,<2.0.0)
 Requires-Dist: numpy (>=1.26.0,<3.0.0)
 Requires-Dist: openai (>=1.51.0,<4.0.0)
@@ -71,8 +71,6 @@ Description-Content-Type: text/markdown
 # Quick Start
 Install this package with `pip install forecasting-tools`
 
-Demo website: https://forecasting-tools.streamlit.app/
-
 Demo repo (get a Metaculus bot running in 30min): https://github.com/Metaculus/metac-bot-template
 
 # Overview
@@ -87,7 +85,7 @@ Here are the tools most likely to be useful to you:
 Here are some other features of the project (not all are documented yet):
 - **General LLM Wrapper:** A unified interface around litellm with retry logic, the Metaculus proxy, structured outputs, and cost tracking
 - **Monetary Cost Manager:** for tracking AI and API expenses
-- **Other experimental tools:** See the demo site for other AI forecasting tools that this project supports (not all are documented). Also see the `scripts` folder for other common workflows and entry points into the code.
+- **Other experimental tools:** See the Streamlit front end in `forecasting_tools/front_end/` (see "Running the Front End" below) for other AI forecasting tools that this project supports (not all are documented). Also see the `scripts` folder for other common workflows and entry points into the code.
 
 All the examples below are in a Jupyter Notebook called `README.ipynb` which you can run locally to test the package (make sure to run the first cell though).
 

{forecasting_tools-0.3.3 → forecasting_tools-0.3.5}/README.md

@@ -9,8 +9,6 @@
 # Quick Start
 Install this package with `pip install forecasting-tools`
 
-Demo website: https://forecasting-tools.streamlit.app/
-
 Demo repo (get a Metaculus bot running in 30min): https://github.com/Metaculus/metac-bot-template
 
 # Overview
@@ -25,7 +23,7 @@ Here are the tools most likely to be useful to you:
 Here are some other features of the project (not all are documented yet):
 - **General LLM Wrapper:** A unified interface around litellm with retry logic, the Metaculus proxy, structured outputs, and cost tracking
 - **Monetary Cost Manager:** for tracking AI and API expenses
-- **Other experimental tools:** See the demo site for other AI forecasting tools that this project supports (not all are documented). Also see the `scripts` folder for other common workflows and entry points into the code.
+- **Other experimental tools:** See the Streamlit front end in `forecasting_tools/front_end/` (see "Running the Front End" below) for other AI forecasting tools that this project supports (not all are documented). Also see the `scripts` folder for other common workflows and entry points into the code.
 
 All the examples below are in a Jupyter Notebook called `README.ipynb` which you can run locally to test the package (make sure to run the first cell though).
 

{forecasting_tools-0.3.3 → forecasting_tools-0.3.5}/forecasting_tools/data_models/multiple_choice_report.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import math
 from typing import TYPE_CHECKING
 
 from pydantic import BaseModel, Field, model_validator
@@ -33,29 +34,16 @@ class PredictedOptionList(BaseModel):
                 "too far from 1 to be confident that normalization will deliver an "
                 "intended prediction."
             )
-        logger.warning(
-            f"Sum of option probabilities {sum_of_probabilities} is not 1, but is close to 1. "
-            "Normalizing the probabilities."
-        )
-
-        # Step 1: Clamp values
-        max_clamped_prob = 0.99
-        min_clamped_prob = 0.01
-        clamped_list = [
-            max(min(x.probability, max_clamped_prob), min_clamped_prob)
-            for x in self.predicted_options
-        ]
-
-        # Step 2: Calculate the sum of all elements
-        total_sum_decimal = sum(clamped_list)
-
-        # Step 3: Normalize the list so that all elements add up to 1
-        normalized_list = [x / total_sum_decimal for x in clamped_list]
+        if not math.isclose(sum_of_probabilities, 1):
+            logger.warning(
+                f"Sum of option probabilities {sum_of_probabilities} is not 1, but is close to 1. "
+                "Normalizing the probabilities."
+            )
 
-        # Step 4: Adjust for any small floating-point errors
-        adjustment = 1.0 - sum(normalized_list)
-        normalized_list[-1] += adjustment
-        normalized_option_probabilities = normalized_list
+        normalized_option_probabilities = self._normalize_with_minimum(
+            [option.probability for option in self.predicted_options],
+            minimum_probability=0.001,
+        )
 
         new_sum = sum(normalized_option_probabilities)
         assert (
@@ -81,6 +69,56 @@ class PredictedOptionList(BaseModel):
         ]
         return self
 
+    @staticmethod
+    def _normalize_with_minimum(
+        probabilities: list[float], minimum_probability: float
+    ) -> list[float]:
+        """
+        Normalizes to sum to 1 while keeping every option at or above the minimum.
+        Options below the minimum are pinned to it and the excess is taken
+        proportionally from the remaining options, so with 2+ options no option
+        can exceed 1 - minimum_probability.
+        """
+        if minimum_probability * len(probabilities) >= 1:
+            raise ValueError(
+                f"Cannot give all {len(probabilities)} options a probability of at least {minimum_probability}"
+            )
+        total = sum(probabilities)
+        unadjusted = [probability / total for probability in probabilities]
+        normalized = unadjusted
+        pinned_indices: set[int] = set()
+        while True:
+            newly_pinned_indices = {
+                index
+                for index, probability in enumerate(normalized)
+                if probability < minimum_probability and index not in pinned_indices
+            }
+            if not newly_pinned_indices:
+                break
+            pinned_indices |= newly_pinned_indices
+            unpinned_mass = 1 - minimum_probability * len(pinned_indices)
+            unpinned_total = sum(
+                probability
+                for index, probability in enumerate(unadjusted)
+                if index not in pinned_indices
+            )
+            normalized = [
+                (
+                    minimum_probability
+                    if index in pinned_indices
+                    else probability * unpinned_mass / unpinned_total
+                )
+                for index, probability in enumerate(unadjusted)
+            ]
+
+        largest_index = normalized.index(max(normalized))
+        normalized[largest_index] = 1.0 - sum(
+            probability
+            for index, probability in enumerate(normalized)
+            if index != largest_index
+        )
+        return normalized
+
     def to_dict(self) -> dict[str, float]:
         return {
             option.option_name: option.probability for option in self.predicted_options

{forecasting_tools-0.3.3 → forecasting_tools-0.3.5}/forecasting_tools/data_models/numeric_report.py

@@ -585,9 +585,20 @@ class NumericDistribution(BaseModel):
             return float(cap_pmf(scale).sum())
 
         # find the appropriate scale search space
+        # An increasing CDF needs at most ~46 expansions (scale <= cap / min bin = 3800), so hitting
+        # the limit means no scale works (e.g. negative bins from a non-increasing CDF)
+        max_scale_expansions = 100
         lo = hi = scale = 1.0
+        expansions = 0
         while capped_sum(hi) < 1.0:
+            if expansions >= max_scale_expansions:
+                raise ValueError(
+                    "Could not standardize CDF: no scale makes the capped probability mass sum to 1. "
+                    f"Smallest in-range bin probability is {pmf[1:-1].min():.3g} "
+                    "(negative means the CDF is not increasing)."
+                )
             hi *= 1.2
+            expansions += 1
         # hone in on scale value that makes capped sum 1
         for _ in range(100):
             scale = 0.5 * (lo + hi)

{forecasting_tools-0.3.3 → forecasting_tools-0.3.5}/forecasting_tools/helpers/metaculus_client.py

@@ -108,6 +108,16 @@ class AuthorMismatchError(ValueError):
 class MetaculusClient:
     """
     Documentation for the API can be found at https://www.metaculus.com/api/
+
+    Setting acting_user (a username or user id) makes the client forecast,
+    comment, create questions, and read "my forecasts" as that account while
+    authenticating with the token. The token must belong to the acting user's
+    bot owner, an account allowed to act as metac bots, or a superuser.
+    Getting a question by post id is done fully as the acting user, but
+    question lists are still fetched as the token's account, so a question only
+    the acting user can see will not be listed. Public comments and other
+    endpoints (e.g. coherence links, bot management) are not supported for the
+    acting user and act as the token's account or raise.
     """
 
     # NOTE: The tourament slug can be used for ID as well (e.g. "aibq2" or "quarterly-cup" instead of 32721 or 32630)
@@ -174,6 +184,7 @@ class MetaculusClient:
         sleep_seconds_between_requests: float = 3.5,
         sleep_jitter_seconds: float = 1,
         token: str | None = None,
+        acting_user: str | int | None = None,
     ):
         default_base_url = "https://www.metaculus.com/api"
         self.base_url = (
@@ -188,6 +199,7 @@ class MetaculusClient:
             logger.warning(
                 "METACULUS_TOKEN environment variable and/or token field not set"
             )
+        self.acting_user = acting_user
 
     @retry_with_exponential_backoff()
     def get_user_bots(self) -> list[UserResponse]:
@@ -250,15 +262,27 @@ class MetaculusClient:
         is_private: bool = True,
         included_forecast: bool = True,
     ) -> None:
+        comment = {
+            "on_post": post_id,
+            "text": comment_text,
+            "is_private": is_private,
+            "included_forecast": included_forecast,
+        }
+        if self.acting_user is not None:
+            if not is_private:
+                raise ValueError(
+                    "Public comments cannot be posted with acting_user set: the "
+                    "bulk endpoint that supports acting_user only accepts private "
+                    "comments."
+                )
+            self._post_bulk_forecasts_and_comments(comments=[comment])
+            logger.info(f"Posted comment on post {post_id} as {self.acting_user}")
+            return
+
         self._sleep_between_requests()
         response = requests.post(
             f"{self.base_url}/comments/create/",
-            json={
-                "on_post": post_id,
-                "text": comment_text,
-                "is_private": is_private,
-                "included_forecast": included_forecast,
-            },
+            json=comment,
             **self._get_auth_headers(),  # type: ignore
             timeout=self.timeout,
         )
@@ -371,6 +395,7 @@ class MetaculusClient:
         headers = self._get_auth_headers()
         response = requests.get(
             url,
+            params={"acting_user": self.acting_user},
             **headers,  # type: ignore
             timeout=self.timeout,
         )
@@ -614,6 +639,15 @@ class MetaculusClient:
     def _post_question_prediction(
         self, question_id: int, forecast_payload: dict
     ) -> None:
+        if self.acting_user is not None:
+            self._post_bulk_forecasts_and_comments(
+                forecasts=[{"question": question_id, **forecast_payload}]
+            )
+            logger.info(
+                f"Posted prediction on question {question_id} as {self.acting_user}"
+            )
+            return
+
         url = f"{self.base_url}/questions/forecast/"
         self._sleep_between_requests()
         response = requests.post(
@@ -631,6 +665,96 @@ class MetaculusClient:
         logger.info(f"Posted prediction on question {question_id}")
         raise_for_status_with_additional_info(response)
 
+    def _post_bulk_forecasts_and_comments(
+        self,
+        forecasts: list[dict] | None = None,
+        comments: list[dict] | None = None,
+    ) -> None:
+        """
+        Not retried here since callers are already wrapped in
+        retry_with_exponential_backoff.
+        """
+        self._sleep_between_requests()
+        response = requests.post(
+            f"{self.base_url}/questions/bulk-forecast-comment/",
+            json={
+                "acting_user": self.acting_user,
+                "forecasts": forecasts or [],
+                "comments": comments or [],
+            },
+            **self._get_auth_headers(),  # type: ignore
+            timeout=self.timeout,
+        )
+        raise_for_status_with_additional_info(response)
+
+    def _get_acting_user_forecasts(
+        self, question_ids: list[int]
+    ) -> dict[int, list[dict]]:
+        """
+        Returns the acting user's forecasts (oldest first) keyed by question id.
+        Questions the acting user cannot view are left out. Not retried here
+        since callers are already wrapped in retry_with_exponential_backoff.
+        """
+        self._sleep_between_requests()
+        response = requests.post(
+            f"{self.base_url}/questions/bulk-forecast-read/",
+            json={"acting_user": self.acting_user, "question_ids": question_ids},
+            **self._get_auth_headers(),  # type: ignore
+            timeout=self.timeout,
+        )
+        raise_for_status_with_additional_info(response)
+        results: list[dict] = json.loads(response.content)["results"]
+        return {result["question_id"]: result["forecasts"] for result in results}
+
+    def _set_my_forecasts_field_to_acting_users(self, post_jsons: list[dict]) -> None:
+        """
+        The posts list endpoint fills the "my_forecasts" field with the token
+        account's forecasts and, unlike the post detail endpoint, doesn't take
+        acting_user. With an acting user, this overwrites that field in the
+        given jsons with the acting user's forecasts so that
+        already_forecasted, previous_forecasts, etc. describe the acting user.
+        Score data is not available this way and is left empty.
+        """
+        if self.acting_user is None:
+            return
+        question_jsons = [
+            question_json
+            for post_json in post_jsons
+            for question_json in self._get_question_jsons_in_post(post_json)
+        ]
+        if not question_jsons:
+            return
+        forecasts_by_question = self._get_acting_user_forecasts(
+            sorted({question_json["id"] for question_json in question_jsons})
+        )
+        for question_json in question_jsons:
+            history = forecasts_by_question.get(question_json["id"], [])
+            question_json["my_forecasts"] = {
+                "history": history,
+                "latest": history[-1] if history else None,
+                "score_data": {},
+            }
+
+    @staticmethod
+    def _get_question_jsons_in_post(post_json: dict) -> list[dict]:
+        if "group_of_questions" in post_json:
+            return list(post_json["group_of_questions"]["questions"])
+        if "conditional" in post_json:
+            conditional: dict = post_json["conditional"]
+            return [
+                conditional[key]
+                for key in (
+                    "condition",
+                    "condition_child",
+                    "question_yes",
+                    "question_no",
+                )
+                if conditional.get(key)
+            ]
+        if post_json.get("question"):
+            return [post_json["question"]]
+        return []
+
     @retry_with_exponential_backoff()
     def _get_questions_from_api(
         self, params: dict[str, Any], group_question_mode: GroupQuestionMode
@@ -653,6 +777,7 @@ class MetaculusClient:
                 f"Removed {len(removed_posts)} posts that "
                 "are not supported (e.g. notebook or conditional question)"
             )
+        self._set_my_forecasts_field_to_acting_users(supported_posts)
 
         questions: list[MetaculusQuestion] = []
         for q in supported_posts:
@@ -885,7 +1010,9 @@ class MetaculusClient:
         if api_filter.allowed_tournaments:
             url_params["tournaments"] = api_filter.allowed_tournaments
 
-        if api_filter.is_previously_forecasted_by_user:
+        # The server only allows forecaster_id to be the token's own account,
+        # so for an acting user this is filtered locally instead
+        if api_filter.is_previously_forecasted_by_user and self.acting_user is None:
             user_id = self.get_current_user_id()
             if user_id:
                 url_params["forecaster_id"] = user_id
@@ -961,6 +1088,11 @@ class MetaculusClient:
                 api_filter.cp_reveal_time_lt,
             )
 
+        if api_filter.is_previously_forecasted_by_user and self.acting_user is not None:
+            questions = [
+                question for question in questions if question.already_forecasted
+            ]
+
         return questions
 
     @staticmethod
@@ -1217,13 +1349,16 @@ class MetaculusClient:
 
     @retry_with_exponential_backoff()
     def create_question(
-        self, question: MetaculusQuestion, author_username: str | None = None
+        self, question: MetaculusQuestion, acting_user: str | int | None = None
     ) -> MetaculusQuestion:
         """
-        Setting author_username creates the post as that user, which requires
-        the API token to belong to a Metaculus superuser.
+        Setting acting_user (a username or user id) creates the post with that user
+        as author. The API token must own that bot, be allowed to act as metac bots,
+        or belong to a superuser. Defaults to the client's acting_user.
         """
-        question_data = self._get_post_create_data(question, author_username)
+        if acting_user is None:
+            acting_user = self.acting_user
+        question_data = self._get_post_create_data(question, acting_user)
         self._sleep_between_requests()
         response = requests.post(
             f"{self.base_url}/posts/create/",
@@ -1251,15 +1386,8 @@ class MetaculusClient:
             single_created_question.id_of_post
         )
 
-        if author_username is not None:
-            actual_author = full_created_question.api_json.get("author_username")
-            if actual_author != author_username:
-                raise AuthorMismatchError(
-                    f"Post {full_created_question.id_of_post} was created with author "
-                    f"{actual_author!r} instead of {author_username!r} and has not been "
-                    "approved. Either the API token is not a superuser or the server "
-                    "ignored the author override."
-                )
+        if acting_user is not None:
+            self._raise_if_author_is_not_acting_user(full_created_question, acting_user)
 
         return full_created_question
 
@@ -1305,19 +1433,34 @@ class MetaculusClient:
         raise_for_status_with_additional_info(response)
         logger.info(f"Approved question {question.page_url}")
 
+    @staticmethod
+    def _raise_if_author_is_not_acting_user(
+        question: MetaculusQuestion, acting_user: str | int
+    ) -> None:
+        if str(acting_user).isdigit():
+            actual_author = question.api_json.get("author_id")
+            expected_author = int(acting_user)
+        else:
+            actual_author = question.api_json.get("author_username")
+            expected_author = acting_user
+        if actual_author != expected_author:
+            raise AuthorMismatchError(
+                f"Post {question.id_of_post} was created with author "
+                f"{actual_author!r} instead of {expected_author!r} and has not been "
+                "approved. The server likely ignored the acting_user field."
+            )
+
     @staticmethod
     def _get_post_create_data(
-        question: MetaculusQuestion, author_username: str | None = None
+        question: MetaculusQuestion, acting_user: str | int | None = None
     ) -> dict:
         if question.published_time is None:
             publish_time = question.open_time
         else:
             publish_time = question.published_time
 
-        author_override_fields = (
-            {"author_username": author_username, "is_staff_override": True}
-            if author_username is not None
-            else {}
+        acting_user_fields = (
+            {"acting_user": acting_user} if acting_user is not None else {}
         )
 
         return {
@@ -1383,5 +1526,5 @@ class MetaculusClient:
                 "fine_print": question.fine_print or "",
                 "group_rank": None,  # only group questions
             },
-            **author_override_fields,
+            **acting_user_fields,
         }

{forecasting_tools-0.3.3 → forecasting_tools-0.3.5}/forecasting_tools/util/misc.py

@@ -46,7 +46,14 @@ def raise_for_status_with_additional_info(
             f"Response JSON: {response_json}."
         )
         logger.error(error_message)
-        raise requests.exceptions.HTTPError(error_message) from e
+        raise requests.exceptions.HTTPError(error_message, response=response) from e
+
+
+def _is_permanent_client_error(error: Exception) -> bool:
+    if not isinstance(error, requests.exceptions.HTTPError) or error.response is None:
+        return False
+    status_code = error.response.status_code
+    return 400 <= status_code < 500 and status_code != 429
 
 
 def retry_with_exponential_backoff(
@@ -82,6 +89,11 @@ def retry_with_exponential_backoff(
                 try:
                     return func(*args, **kwargs)
                 except retry_on_exceptions as e:
+                    if _is_permanent_client_error(e):
+                        logger.warning(
+                            f"Not retrying {func.__name__} since 4xx errors (other than 429) are not transient. Error: {e}"
+                        )
+                        raise
                     retries += 1
                     if retries > max_retries:
                         logger.error(

{forecasting_tools-0.3.3 → forecasting_tools-0.3.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "forecasting-tools"
-version = "0.3.3"
+version = "0.3.5"
 description = "AI forecasting and research tools to help humans reason about and forecast the future"
 authors = ["Benjamin Wilson <ben@metaculus.com>"]
 license = "MIT"
@@ -38,7 +38,7 @@ pillow = ">=9.0.0,<13.0.0"
 typeguard = "^4.3.0"
 scikit-learn = {version = "^1.5.2", optional = true}
 scipy = {version = ">=1.10.0,<2.0.0", optional = true}
-litellm = ">=1.59.1,<2.0.0,!=1.76.*,!=1.77.*,!=1.82.7,!=1.82.8"
+litellm = ">=1.59.1,<2.0.0,!=1.76.*,!=1.77.*,!=1.82.7,!=1.82.8,!=1.107.*,!=1.108.*,!=1.109.*"
 numpy = ">=1.26.0,<3.0.0"
 exceptiongroup = "^1.2.2"
 faker = {version = ">=37.0.0,<41.0.0", optional = true}