prediction-market-agent-tooling 0.50.1__tar.gz → 0.51.1__tar.gz

{prediction_market_agent_tooling-0.50.1 → prediction_market_agent_tooling-0.51.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prediction-market-agent-tooling
-Version: 0.50.1
+Version: 0.51.1
 Summary: Tools to benchmark, deploy and monitor prediction market agents.
 Author: Gnosis
 Requires-Python: >=3.10,<3.12

{prediction_market_agent_tooling-0.50.1 → prediction_market_agent_tooling-0.51.1}/prediction_market_agent_tooling/deploy/agent.py

@@ -66,6 +66,7 @@ from prediction_market_agent_tooling.monitor.monitor_app import (
 )
 from prediction_market_agent_tooling.tools.hexbytes_custom import HexBytes
 from prediction_market_agent_tooling.tools.ipfs.ipfs_handler import IPFSHandler
+from prediction_market_agent_tooling.tools.is_invalid import is_invalid
 from prediction_market_agent_tooling.tools.is_predictable import is_predictable_binary
 from prediction_market_agent_tooling.tools.langfuse_ import langfuse_context, observe
 from prediction_market_agent_tooling.tools.utils import DatetimeUTC, utcnow
@@ -295,6 +296,7 @@ class DeployableTraderAgent(DeployableAgent):
     bet_on_n_markets_per_run: int = 1
     min_required_balance_to_operate: xDai | None = xdai_type(1)
     min_balance_to_keep_in_native_currency: xDai | None = xdai_type(0.1)
+    allow_invalid_questions: bool = False
 
     def __init__(
         self,
@@ -403,6 +405,9 @@ class DeployableTraderAgent(DeployableAgent):
         if not is_predictable_binary(market.question):
             return False
 
+        if not self.allow_invalid_questions and is_invalid(market.question):
+            return False
+
         return True
 
     def answer_binary_market(self, market: AgentMarket) -> ProbabilisticAnswer | None:

prediction_market_agent_tooling-0.51.1/prediction_market_agent_tooling/markets/metaculus/data_models.py

@@ -0,0 +1,104 @@
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel
+
+from prediction_market_agent_tooling.gtypes import Probability
+from prediction_market_agent_tooling.tools.utils import DatetimeUTC
+
+
+class QuestionType(str, Enum):
+    binary = "binary"
+
+
+class AggregationItem(BaseModel):
+    start_time: DatetimeUTC
+    end_time: DatetimeUTC | None
+    forecast_values: list[float] | None
+    forecaster_count: int
+    interval_lower_bounds: list[float] | None
+    centers: list[float] | None
+    interval_upper_bounds: list[float] | None
+    means: list[float] | None
+    histogram: list[float] | None
+
+
+class Aggregation(BaseModel):
+    history: list[AggregationItem]
+    latest: AggregationItem | None
+    score_data: dict[str, Any]
+
+
+class Aggregations(BaseModel):
+    recency_weighted: Aggregation
+    unweighted: Aggregation
+    single_aggregation: Aggregation
+    metaculus_prediction: Aggregation
+
+
+class MyForecast(BaseModel):
+    start_time: DatetimeUTC
+    end_time: DatetimeUTC | None
+    forecast_values: list[float] | None
+    interval_lower_bounds: list[float] | None
+    centers: list[float] | None
+    interval_upper_bounds: list[float] | None
+
+
+class MyAggregation(BaseModel):
+    history: list[MyForecast]
+    latest: MyForecast | None
+    score_data: dict[str, Any]
+
+
+class Question(BaseModel):
+    aggregations: Aggregations
+    my_forecasts: MyAggregation
+    type: QuestionType
+    possibilities: dict[str, str] | None
+
+
+class MetaculusQuestion(BaseModel):
+    id: int
+    author_id: int
+    author_username: str
+    title: str
+    created_at: DatetimeUTC
+    published_at: DatetimeUTC
+    scheduled_close_time: DatetimeUTC
+    scheduled_resolve_time: DatetimeUTC
+    user_permission: str
+    comment_count: int
+    question: Question
+    # TODO add the rest of the fields https://github.com/gnosis/prediction-market-agent-tooling/issues/301
+
+    @property
+    def page_url(self) -> str:
+        return f"https://www.metaculus.com/questions/{self.id}/"
+
+    @property
+    def p_yes(self) -> Probability:
+        if self.question.type != QuestionType.binary:
+            raise ValueError(f"Only binary markets can have p_yes.")
+        if (
+            self.question.aggregations.recency_weighted is None
+            or self.question.aggregations.recency_weighted.latest is None
+            or self.question.aggregations.recency_weighted.latest.forecast_values
+            is None
+        ):
+            # If no value is provided (i.e. the question is new and has not been answered yet), we default to 0.5.
+            return Probability(0.5)
+        if len(self.question.aggregations.recency_weighted.latest.forecast_values) != 2:
+            raise ValueError(
+                f"Invalid logic, assumed that binary markets will have two forecasts, got: {self.question.aggregations.recency_weighted.latest.forecast_values}"
+            )
+        # Experimentally figured out that they store "Yes" at index 1.
+        return Probability(
+            self.question.aggregations.recency_weighted.latest.forecast_values[1]
+        )
+
+
+class MetaculusQuestions(BaseModel):
+    next: str | None
+    previous: str | None
+    results: list[MetaculusQuestion]

{prediction_market_agent_tooling-0.50.1 → prediction_market_agent_tooling-0.51.1}/prediction_market_agent_tooling/markets/metaculus/metaculus.py

@@ -37,13 +37,12 @@ class MetaculusAgentMarket(AgentMarket):
             question=model.title,
             outcomes=[],
             resolution=None,
-            current_p_yes=Probability(model.community_prediction.full.p_yes),
-            created_time=model.created_time,
-            close_time=model.close_time,
-            url=model.url,
+            current_p_yes=model.p_yes,
+            created_time=model.created_at,
+            close_time=model.scheduled_close_time,
+            url=model.page_url,
             volume=None,
-            have_predicted=model.my_predictions is not None
-            and len(model.my_predictions.predictions) > 0,
+            have_predicted=model.question.my_forecasts.latest is not None,
             outcome_token_pool=None,
         )
 

{prediction_market_agent_tooling-0.50.1 → prediction_market_agent_tooling-0.51.1}/prediction_market_agent_tooling/markets/omen/data_models.py

@@ -352,25 +352,38 @@ class OmenMarket(BaseModel):
     def is_binary(self) -> bool:
         return len(self.outcomes) == 2
 
-    @property
-    def boolean_outcome(self) -> bool:
+    def boolean_outcome_from_answer(self, answer: HexBytes) -> bool:
         if not self.is_binary:
             raise ValueError(
                 f"Market with title {self.title} is not binary, it has {len(self.outcomes)} outcomes."
             )
+        outcome: str = self.outcomes[answer.as_int()]
+        return get_boolean_outcome(outcome)
+
+    @property
+    def boolean_outcome(self) -> bool:
         if not self.is_resolved_with_valid_answer:
             raise ValueError(f"Bet with title {self.title} is not resolved.")
+        return self.boolean_outcome_from_answer(
+            check_not_none(
+                self.currentAnswer, "Can not be None if `is_resolved_with_valid_answer`"
+            )
+        )
 
-        outcome: str = self.outcomes[check_not_none(self.answer_index)]
-        return get_boolean_outcome(outcome)
+    def get_resolution_enum_from_answer(self, answer: HexBytes) -> Resolution:
+        if self.boolean_outcome_from_answer(answer):
+            return Resolution.YES
+        else:
+            return Resolution.NO
 
     def get_resolution_enum(self) -> t.Optional[Resolution]:
         if not self.is_resolved_with_valid_answer:
             return None
-        if self.boolean_outcome:
-            return Resolution.YES
-        else:
-            return Resolution.NO
+        return self.get_resolution_enum_from_answer(
+            check_not_none(
+                self.currentAnswer, "Can not be None if `is_resolved_with_valid_answer`"
+            )
+        )
 
     @property
     def url(self) -> str:

{prediction_market_agent_tooling-0.50.1 → prediction_market_agent_tooling-0.51.1}/prediction_market_agent_tooling/markets/omen/omen_resolving.py

@@ -18,9 +18,6 @@ from prediction_market_agent_tooling.markets.omen.data_models import (
     OmenMarket,
     RealityQuestion,
 )
-from prediction_market_agent_tooling.markets.omen.omen import (
-    OMEN_DEFAULT_REALITIO_BOND_VALUE,
-)
 from prediction_market_agent_tooling.markets.omen.omen_contracts import (
     OmenOracleContract,
     OmenRealitioContract,
@@ -121,6 +118,7 @@ def claim_bonds_on_realitio_question(
 def finalize_markets(
     api_keys: APIKeys,
     markets_with_resolutions: list[tuple[OmenMarket, Resolution | None]],
+    realitio_bond: xDai,
     wait_n_days_before_invalid: int = 30,
     web3: Web3 | None = None,
 ) -> list[HexAddress]:
@@ -140,7 +138,7 @@ def finalize_markets(
                 omen_submit_invalid_answer_market_tx(
                     api_keys,
                     market,
-                    OMEN_DEFAULT_REALITIO_BOND_VALUE,
+                    realitio_bond,
                     web3=web3,
                 )
 
@@ -155,7 +153,7 @@ def finalize_markets(
                 api_keys,
                 market,
                 resolution,
-                OMEN_DEFAULT_REALITIO_BOND_VALUE,
+                realitio_bond,
                 web3=web3,
             )
             finalized_markets.append(market.id)
@@ -168,7 +166,7 @@ def finalize_markets(
             omen_submit_invalid_answer_market_tx(
                 api_keys,
                 market,
-                OMEN_DEFAULT_REALITIO_BOND_VALUE,
+                realitio_bond,
                 web3=web3,
             )
 

{prediction_market_agent_tooling-0.50.1 → prediction_market_agent_tooling-0.51.1}/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py

@@ -71,12 +71,17 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
     def __init__(self) -> None:
         self.sg = Subgrounds()
 
-        # Patch the query_json method to retry on failure.
+        # Patch methods to retry on failure.
         self.sg.query_json = tenacity.retry(
             stop=tenacity.stop_after_attempt(3),
             wait=tenacity.wait_fixed(1),
             after=lambda x: logger.debug(f"query_json failed, {x.attempt_number=}."),
         )(self.sg.query_json)
+        self.sg.load_subgraph = tenacity.retry(
+            stop=tenacity.stop_after_attempt(3),
+            wait=tenacity.wait_fixed(1),
+            after=lambda x: logger.debug(f"load_subgraph failed, {x.attempt_number=}."),
+        )(self.sg.load_subgraph)
 
         keys = APIKeys()
 
@@ -221,7 +226,6 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
         opened_after: t.Optional[DatetimeUTC] = None,
         finalized_before: t.Optional[DatetimeUTC] = None,
         finalized_after: t.Optional[DatetimeUTC] = None,
-        finalized: bool | None = None,
         resolved: bool | None = None,
         liquidity_bigger_than: Wei | None = None,
         condition_id_in: list[HexBytes] | None = None,
@@ -273,12 +277,6 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
             else:
                 where_stms["resolutionTimestamp"] = None
 
-        if finalized is not None:
-            if finalized:
-                where_stms["answerFinalizedTimestamp_not"] = None
-            else:
-                where_stms["answerFinalizedTimestamp"] = None
-
         if opened_after:
             where_stms["question_"]["openingTimestamp_gt"] = to_int_timestamp(
                 opened_after
@@ -358,13 +356,11 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
         Simplified `get_omen_binary_markets` method, which allows to fetch markets based on the filter_by and sort_by values.
         """
         # These values need to be set according to the filter_by value, so they can not be passed as arguments.
-        finalized: bool | None = None
         resolved: bool | None = None
         opened_after: DatetimeUTC | None = None
         liquidity_bigger_than: Wei | None = None
 
         if filter_by == FilterBy.RESOLVED:
-            finalized = True
             resolved = True
         elif filter_by == FilterBy.OPEN:
             # We can not use `resolved=False` + `finalized=False` here,
@@ -381,7 +377,6 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
 
         return self.get_omen_binary_markets(
             limit=limit,
-            finalized=finalized,
             resolved=resolved,
             opened_after=opened_after,
             liquidity_bigger_than=liquidity_bigger_than,
@@ -401,7 +396,6 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
         opened_after: t.Optional[DatetimeUTC] = None,
         finalized_before: t.Optional[DatetimeUTC] = None,
         finalized_after: t.Optional[DatetimeUTC] = None,
-        finalized: bool | None = None,
         resolved: bool | None = None,
         creator: t.Optional[HexAddress] = None,
         creator_in: t.Optional[t.Sequence[HexAddress]] = None,
@@ -429,7 +423,6 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
             opened_after=opened_after,
             finalized_before=finalized_before,
             finalized_after=finalized_after,
-            finalized=finalized,
             resolved=resolved,
             condition_id_in=condition_id_in,
             id_in=id_in,

prediction_market_agent_tooling-0.51.1/prediction_market_agent_tooling/tools/is_invalid.py

@@ -0,0 +1,92 @@
+import tenacity
+
+from prediction_market_agent_tooling.config import APIKeys
+from prediction_market_agent_tooling.loggers import logger
+from prediction_market_agent_tooling.tools.cache import persistent_inmemory_cache
+from prediction_market_agent_tooling.tools.is_predictable import (
+    parse_decision_yes_no_completion,
+)
+from prediction_market_agent_tooling.tools.langfuse_ import (
+    get_langfuse_langchain_config,
+    observe,
+)
+from prediction_market_agent_tooling.tools.utils import (
+    LLM_SEED,
+    LLM_SUPER_LOW_TEMPERATURE,
+)
+
+# I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought.
+# Rules are almost copy-pasted from https://cdn.kleros.link/ipfs/QmZM12kkguXFk2C94ykrKpambt4iUVKsVsxGxDEdLS68ws/omen-rules.pdf,
+# with some small prompting mods and I removed the point about "The outcome of the market must be known by its Resolution Date.", because that can not be verified before-hand.
+# and also point about "in which none of the answers are valid will resolve as invalid" and "in which multiple answers are valid will resolve as invalid.", because before hand we can not know if one of the outcomes happened or not.
+QUESTION_IS_INVALID_PROMPT = """Main signs about an invalid question (sometimes referred to as a "market"):
+- The market's question is about immoral violence, death or assassination.
+- The violent event can be caused by a single conscious being.
+- The violent event is done illegally.
+- The market should not directly incentivize immoral violent (such as murder, rape or unjust imprisonment) actions which could likely be performed by any participant.
+- Invalid: Will Donald Trump be alive on the 01/12/2021? (Anyone could bet on “No” and kill him for a guaranteed profit. Anyone could bet on “Yes” to effectively put a bounty on his head).
+- Invalid: Will Hera be a victim of swatting in 2020? (Anyone could falsely call the emergency services on him in order to win the bet)
+- This does not prevent markets:
+  - Whose topics are violent events not caused by conscious beings.
+  - Valid: How many people will die from COVID19 in 2020? (Viruses don’t use prediction markets).
+  - Whose main source of uncertainty is not related to a potential violent action.
+  - Valid: Will Trump win the 2020 US presidential election? (The main source of uncertainty is the vote of US citizens, not a potential murder of a presidential candidate).
+  - Which could give an incentive only to specific participants to commit an immoral violent action, but are in practice unlikely.
+  - Valid: Will the US be engaged in a military conflict with a UN member state in 2021? (It’s unlikely for the US to declare war in order to win a bet on this market).
+  - Valid: Will Derek Chauvin go to jail for the murder of George Flyod? (It’s unlikely that the jurors would collude to make a wrong verdict in order to win this market).
+- Questions with relative dates will resolve as invalid. Dates must be stated in absolute terms, not relative depending on the current time.
+- Invalid: Who will be the president of the United States in 6 months? (“in 6 months depends on the current time”).
+- Questions about moral values and not facts will be resolved as invalid.
+- Invalid: “Is it ethical to eat meat?”.
+
+Follow a chain of thought to evaluate if the question is invalid:
+
+First, write the parts of the following question:
+
+"{question}"
+
+Then, write down what is the future event of the question, what it refers to and when that event will happen if the question contains it.
+
+Then, explain why do you think it is or isn't invalid.
+
+Finally, write your final decision, write `decision: ` followed by either "yes it is invalid" or "no it isn't invalid" about the question. Don't write anything else after that. You must include "yes" or "no".
+"""
+
+
+@persistent_inmemory_cache
+@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_fixed(1))
+@observe()
+def is_invalid(
+    question: str,
+    engine: str = "gpt-4o",
+    temperature: float = LLM_SUPER_LOW_TEMPERATURE,
+    seed: int = LLM_SEED,
+    prompt_template: str = QUESTION_IS_INVALID_PROMPT,
+    max_tokens: int = 1024,
+) -> bool:
+    """
+    Evaluate if the question is actually answerable.
+    """
+    try:
+        from langchain.prompts import ChatPromptTemplate
+        from langchain_openai import ChatOpenAI
+    except ImportError:
+        logger.error("langchain not installed, skipping is_invalid")
+        return True
+
+    llm = ChatOpenAI(
+        model=engine,
+        temperature=temperature,
+        seed=seed,
+        api_key=APIKeys().openai_api_key_secretstr_v1,
+    )
+
+    prompt = ChatPromptTemplate.from_template(template=prompt_template)
+    messages = prompt.format_messages(question=question)
+    completion = str(
+        llm.invoke(
+            messages, max_tokens=max_tokens, config=get_langfuse_langchain_config()
+        ).content
+    )
+
+    return parse_decision_yes_no_completion(question, completion)

{prediction_market_agent_tooling-0.50.1 → prediction_market_agent_tooling-0.51.1}/prediction_market_agent_tooling/tools/utils.py

@@ -22,8 +22,10 @@ from prediction_market_agent_tooling.loggers import logger
 T = TypeVar("T")
 
 # t=0 is mathematically impossible and it's not clear how OpenAI (and others) handle it, as a result, even with t=0, gpt-4-turbo produces very different outputs,
-# it seems that using a very low temperature is the best way to have as consistent outputs as possible: https://community.openai.com/t/why-the-api-output-is-inconsistent-even-after-the-temperature-is-set-to-0/329541/12
+# see this experiment to figure out if you should use LLM_SUPER_LOW_TEMPERATURE or just 0: https://github.com/gnosis/prediction-market-agent/pull/438.
 LLM_SUPER_LOW_TEMPERATURE = 0.00000001
+# For consistent results, also include seed for models that supports it.
+LLM_SEED = 0
 
 
 def check_not_none(

{prediction_market_agent_tooling-0.50.1 → prediction_market_agent_tooling-0.51.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "prediction-market-agent-tooling"
-version = "0.50.1"
+version = "0.51.1"
 description = "Tools to benchmark, deploy and monitor prediction market agents."
 authors = ["Gnosis"]
 readme = "README.md"

prediction_market_agent_tooling-0.50.1/prediction_market_agent_tooling/markets/metaculus/data_models.py

@@ -1,97 +0,0 @@
-from enum import Enum
-from typing import Any
-
-from pydantic import BaseModel
-
-from prediction_market_agent_tooling.tools.utils import DatetimeUTC
-
-
-class QuestionType(str, Enum):
-    forecast = "forecast"
-    notebook = "notebook"
-    discussion = "discussion"
-    claim = "claim"
-    group = "group"
-    conditional_group = "conditional_group"
-    multiple_choice = "multiple_choice"
-
-
-class CommunityPrediction(BaseModel):
-    y: list[float]
-    q1: float | None = None
-    q2: float | None = None
-    q3: float | None = None
-
-    @property
-    def p_yes(self) -> float:
-        """
-        q2 corresponds to the median, or 'second quartile' of the distribution.
-
-        If no value is provided (i.e. the question is new and has not been
-        answered yet), we default to 0.5.
-        """
-        return self.q2 if self.q2 is not None else 0.5
-
-
-class Prediction(BaseModel):
-    t: DatetimeUTC
-    x: float
-
-
-class UserPredictions(BaseModel):
-    id: int
-    predictions: list[Prediction]
-    points_won: float | None = None
-    user: int
-    username: str
-    question: int
-
-
-class CommunityPredictionStats(BaseModel):
-    full: CommunityPrediction
-    unweighted: CommunityPrediction
-
-
-class MetaculusQuestion(BaseModel):
-    """
-    https://www.metaculus.com/api2/schema/redoc/#tag/questions/operation/questions_retrieve
-    """
-
-    active_state: Any
-    url: str
-    page_url: str
-    id: int
-    author: int
-    author_name: str
-    author_id: int
-    title: str
-    title_short: str
-    group_label: str | None = None
-    resolution: int | None
-    resolved_option: int | None
-    created_time: DatetimeUTC
-    publish_time: DatetimeUTC | None = None
-    close_time: DatetimeUTC | None = None
-    effected_close_time: DatetimeUTC | None
-    resolve_time: DatetimeUTC | None = None
-    possibilities: dict[Any, Any] | None = None
-    scoring: dict[Any, Any] = {}
-    type: QuestionType | None = None
-    user_perms: Any
-    weekly_movement: float | None
-    weekly_movement_direction: int | None = None
-    cp_reveal_time: DatetimeUTC | None = None
-    edited_time: DatetimeUTC
-    last_activity_time: DatetimeUTC
-    activity: float
-    comment_count: int
-    votes: int
-    community_prediction: CommunityPredictionStats
-    my_predictions: UserPredictions | None = None
-    # TODO add the rest of the fields https://github.com/gnosis/prediction-market-agent-tooling/issues/301
-
-
-class MetaculusQuestions(BaseModel):
-    next: str | None
-    previous: str | None
-    results: list[MetaculusQuestion]