prediction-market-agent-tooling 0.51.0__tar.gz → 0.52.0__tar.gz

{prediction_market_agent_tooling-0.51.0 → prediction_market_agent_tooling-0.52.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prediction-market-agent-tooling
-Version: 0.51.0
+Version: 0.52.0
 Summary: Tools to benchmark, deploy and monitor prediction market agents.
 Author: Gnosis
 Requires-Python: >=3.10,<3.12

{prediction_market_agent_tooling-0.51.0 → prediction_market_agent_tooling-0.52.0}/prediction_market_agent_tooling/deploy/agent.py

@@ -66,6 +66,7 @@ from prediction_market_agent_tooling.monitor.monitor_app import (
 )
 from prediction_market_agent_tooling.tools.hexbytes_custom import HexBytes
 from prediction_market_agent_tooling.tools.ipfs.ipfs_handler import IPFSHandler
+from prediction_market_agent_tooling.tools.is_invalid import is_invalid
 from prediction_market_agent_tooling.tools.is_predictable import is_predictable_binary
 from prediction_market_agent_tooling.tools.langfuse_ import langfuse_context, observe
 from prediction_market_agent_tooling.tools.utils import DatetimeUTC, utcnow
@@ -295,6 +296,7 @@ class DeployableTraderAgent(DeployableAgent):
     bet_on_n_markets_per_run: int = 1
     min_required_balance_to_operate: xDai | None = xdai_type(1)
     min_balance_to_keep_in_native_currency: xDai | None = xdai_type(0.1)
+    allow_invalid_questions: bool = False
 
     def __init__(
         self,
@@ -403,6 +405,9 @@ class DeployableTraderAgent(DeployableAgent):
         if not is_predictable_binary(market.question):
             return False
 
+        if not self.allow_invalid_questions and is_invalid(market.question):
+            return False
+
         return True
 
     def answer_binary_market(self, market: AgentMarket) -> ProbabilisticAnswer | None:

prediction_market_agent_tooling-0.52.0/prediction_market_agent_tooling/markets/metaculus/data_models.py

@@ -0,0 +1,104 @@
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel
+
+from prediction_market_agent_tooling.gtypes import Probability
+from prediction_market_agent_tooling.tools.utils import DatetimeUTC
+
+
+class QuestionType(str, Enum):
+    binary = "binary"
+
+
+class AggregationItem(BaseModel):
+    start_time: DatetimeUTC
+    end_time: DatetimeUTC | None
+    forecast_values: list[float] | None
+    forecaster_count: int
+    interval_lower_bounds: list[float] | None
+    centers: list[float] | None
+    interval_upper_bounds: list[float] | None
+    means: list[float] | None
+    histogram: list[float] | None
+
+
+class Aggregation(BaseModel):
+    history: list[AggregationItem]
+    latest: AggregationItem | None
+    score_data: dict[str, Any]
+
+
+class Aggregations(BaseModel):
+    recency_weighted: Aggregation
+    unweighted: Aggregation
+    single_aggregation: Aggregation
+    metaculus_prediction: Aggregation
+
+
+class MyForecast(BaseModel):
+    start_time: DatetimeUTC
+    end_time: DatetimeUTC | None
+    forecast_values: list[float] | None
+    interval_lower_bounds: list[float] | None
+    centers: list[float] | None
+    interval_upper_bounds: list[float] | None
+
+
+class MyAggregation(BaseModel):
+    history: list[MyForecast]
+    latest: MyForecast | None
+    score_data: dict[str, Any]
+
+
+class Question(BaseModel):
+    aggregations: Aggregations
+    my_forecasts: MyAggregation
+    type: QuestionType
+    possibilities: dict[str, str] | None
+
+
+class MetaculusQuestion(BaseModel):
+    id: int
+    author_id: int
+    author_username: str
+    title: str
+    created_at: DatetimeUTC
+    published_at: DatetimeUTC
+    scheduled_close_time: DatetimeUTC
+    scheduled_resolve_time: DatetimeUTC
+    user_permission: str
+    comment_count: int
+    question: Question
+    # TODO add the rest of the fields https://github.com/gnosis/prediction-market-agent-tooling/issues/301
+
+    @property
+    def page_url(self) -> str:
+        return f"https://www.metaculus.com/questions/{self.id}/"
+
+    @property
+    def p_yes(self) -> Probability:
+        if self.question.type != QuestionType.binary:
+            raise ValueError(f"Only binary markets can have p_yes.")
+        if (
+            self.question.aggregations.recency_weighted is None
+            or self.question.aggregations.recency_weighted.latest is None
+            or self.question.aggregations.recency_weighted.latest.forecast_values
+            is None
+        ):
+            # If no value is provided (i.e. the question is new and has not been answered yet), we default to 0.5.
+            return Probability(0.5)
+        if len(self.question.aggregations.recency_weighted.latest.forecast_values) != 2:
+            raise ValueError(
+                f"Invalid logic, assumed that binary markets will have two forecasts, got: {self.question.aggregations.recency_weighted.latest.forecast_values}"
+            )
+        # Experimentally figured out that they store "Yes" at index 1.
+        return Probability(
+            self.question.aggregations.recency_weighted.latest.forecast_values[1]
+        )
+
+
+class MetaculusQuestions(BaseModel):
+    next: str | None
+    previous: str | None
+    results: list[MetaculusQuestion]

{prediction_market_agent_tooling-0.51.0 → prediction_market_agent_tooling-0.52.0}/prediction_market_agent_tooling/markets/metaculus/metaculus.py

@@ -37,13 +37,12 @@ class MetaculusAgentMarket(AgentMarket):
             question=model.title,
             outcomes=[],
             resolution=None,
-            current_p_yes=Probability(model.community_prediction.full.p_yes),
-            created_time=model.created_time,
-            close_time=model.close_time,
-            url=model.url,
+            current_p_yes=model.p_yes,
+            created_time=model.created_at,
+            close_time=model.scheduled_close_time,
+            url=model.page_url,
             volume=None,
-            have_predicted=model.my_predictions is not None
-            and len(model.my_predictions.predictions) > 0,
+            have_predicted=model.question.my_forecasts.latest is not None,
             outcome_token_pool=None,
         )
 

{prediction_market_agent_tooling-0.51.0 → prediction_market_agent_tooling-0.52.0}/prediction_market_agent_tooling/markets/omen/data_models.py

@@ -73,6 +73,10 @@ class Question(BaseModel):
     answerFinalizedTimestamp: t.Optional[DatetimeUTC] = None
     currentAnswer: t.Optional[str] = None
 
+    @property
+    def question_id(self) -> HexBytes:
+        return self.id
+
     @property
     def question_raw(self) -> str:
         # Based on https://github.com/protofire/omen-exchange/blob/2cfdf6bfe37afa8b169731d51fea69d42321d66c/app/src/hooks/graph/useGraphMarketMakerData.tsx#L217.
@@ -583,7 +587,7 @@ class RealityQuestion(BaseModel):
     historyHash: HexBytes | None
     updatedTimestamp: int
     contentHash: HexBytes
-    questionId: HexBytes
+    questionId: HexBytes  # This is the `id` on question from omen subgraph.
     answerFinalizedTimestamp: int
     currentScheduledFinalizationTimestamp: int
 
@@ -660,6 +664,10 @@ def format_realitio_question(
     template_id: int,
 ) -> str:
     """If you add a new template id here, also add to the parsing function below."""
+
+    # Escape characters for JSON troubles on Reality.eth.
+    question = question.replace('"', '\\"')
+
     if template_id == 2:
         return "␟".join(
             [

{prediction_market_agent_tooling-0.51.0 → prediction_market_agent_tooling-0.52.0}/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py

@@ -218,30 +218,45 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
 
     def _build_where_statements(
         self,
-        creator: t.Optional[HexAddress] = None,
-        creator_in: t.Optional[t.Sequence[HexAddress]] = None,
-        outcomes: list[str] = OMEN_BINARY_MARKET_OUTCOMES,
-        created_after: t.Optional[DatetimeUTC] = None,
-        opened_before: t.Optional[DatetimeUTC] = None,
-        opened_after: t.Optional[DatetimeUTC] = None,
-        finalized_before: t.Optional[DatetimeUTC] = None,
-        finalized_after: t.Optional[DatetimeUTC] = None,
-        resolved: bool | None = None,
-        liquidity_bigger_than: Wei | None = None,
-        condition_id_in: list[HexBytes] | None = None,
-        id_in: list[str] | None = None,
-        excluded_questions: set[str] | None = None,
-        collateral_token_address_in: tuple[ChecksumAddress, ...] | None = None,
-        category: str | None = None,
+        creator: HexAddress | None,
+        creator_in: t.Sequence[HexAddress] | None,
+        outcomes: list[str],
+        created_after: DatetimeUTC | None,
+        question_opened_before: DatetimeUTC | None,
+        question_opened_after: DatetimeUTC | None,
+        question_finalized_before: DatetimeUTC | None,
+        question_finalized_after: DatetimeUTC | None,
+        question_with_answers: bool | None,
+        question_id: HexBytes | None,
+        question_id_in: list[HexBytes] | None,
+        question_current_answer_before: DatetimeUTC | None,
+        question_excluded_titles: set[str] | None,
+        resolved: bool | None,
+        liquidity_bigger_than: Wei | None,
+        condition_id_in: list[HexBytes] | None,
+        id_in: list[str] | None,
+        collateral_token_address_in: tuple[ChecksumAddress, ...] | None,
+        category: str | None,
     ) -> dict[str, t.Any]:
         where_stms: dict[str, t.Any] = {
             "isPendingArbitration": False,
             "outcomes": outcomes,
             "title_not": None,
-            "question_": {},
             "condition_": {},
         }
 
+        where_stms["question_"] = self.get_omen_question_filters(
+            question_id=question_id,
+            opened_before=question_opened_before,
+            opened_after=question_opened_after,
+            finalized_before=question_finalized_before,
+            finalized_after=question_finalized_after,
+            with_answers=question_with_answers,
+            current_answer_before=question_current_answer_before,
+            question_id_in=question_id_in,
+            excluded_titles=question_excluded_titles,
+        )
+
         if collateral_token_address_in:
             where_stms["collateralToken_in"] = [
                 x.lower() for x in collateral_token_address_in
@@ -256,11 +271,6 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
         if created_after:
             where_stms["creationTimestamp_gt"] = to_int_timestamp(created_after)
 
-        if opened_before:
-            where_stms["question_"]["openingTimestamp_lt"] = to_int_timestamp(
-                opened_before
-            )
-
         if liquidity_bigger_than is not None:
             where_stms["liquidityParameter_gt"] = liquidity_bigger_than
 
@@ -277,30 +287,9 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
             else:
                 where_stms["resolutionTimestamp"] = None
 
-        if opened_after:
-            where_stms["question_"]["openingTimestamp_gt"] = to_int_timestamp(
-                opened_after
-            )
-
-        if finalized_before:
-            where_stms["answerFinalizedTimestamp_lt"] = to_int_timestamp(
-                finalized_before
-            )
-
-        if finalized_after:
-            where_stms["answerFinalizedTimestamp_gt"] = to_int_timestamp(
-                finalized_after
-            )
-
         if category:
             where_stms["category"] = category
 
-        # `excluded_question_titles` can not be an empty list, otherwise the API bugs out and returns nothing.
-        excluded_question_titles = [""]
-        if excluded_questions:
-            excluded_question_titles = [i for i in excluded_questions]
-
-        where_stms["question_"]["title_not_in"] = excluded_question_titles
         return where_stms
 
     def _build_sort_params(
@@ -378,12 +367,12 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
         return self.get_omen_binary_markets(
             limit=limit,
             resolved=resolved,
-            opened_after=opened_after,
+            question_opened_after=opened_after,
             liquidity_bigger_than=liquidity_bigger_than,
             sort_direction=sort_direction,
             sort_by_field=sort_by_field,
             created_after=created_after,
-            excluded_questions=excluded_questions,
+            question_excluded_titles=excluded_questions,
             collateral_token_address_in=collateral_token_address_in,
             category=category,
         )
@@ -391,18 +380,22 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
     def get_omen_binary_markets(
         self,
         limit: t.Optional[int],
-        created_after: t.Optional[DatetimeUTC] = None,
-        opened_before: t.Optional[DatetimeUTC] = None,
-        opened_after: t.Optional[DatetimeUTC] = None,
-        finalized_before: t.Optional[DatetimeUTC] = None,
-        finalized_after: t.Optional[DatetimeUTC] = None,
+        creator: HexAddress | None = None,
+        creator_in: t.Sequence[HexAddress] | None = None,
+        created_after: DatetimeUTC | None = None,
+        question_opened_before: DatetimeUTC | None = None,
+        question_opened_after: DatetimeUTC | None = None,
+        question_finalized_before: DatetimeUTC | None = None,
+        question_finalized_after: DatetimeUTC | None = None,
+        question_with_answers: bool | None = None,
+        question_id: HexBytes | None = None,
+        question_id_in: list[HexBytes] | None = None,
+        question_current_answer_before: DatetimeUTC | None = None,
+        question_excluded_titles: set[str] | None = None,
         resolved: bool | None = None,
-        creator: t.Optional[HexAddress] = None,
-        creator_in: t.Optional[t.Sequence[HexAddress]] = None,
         liquidity_bigger_than: Wei | None = None,
         condition_id_in: list[HexBytes] | None = None,
         id_in: list[str] | None = None,
-        excluded_questions: set[str] | None = None,  # question titles
         sort_by_field: FieldPath | None = None,
         sort_direction: str | None = None,
         outcomes: list[str] = OMEN_BINARY_MARKET_OUTCOMES,
@@ -419,14 +412,18 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
             creator_in=creator_in,
             outcomes=outcomes,
             created_after=created_after,
-            opened_before=opened_before,
-            opened_after=opened_after,
-            finalized_before=finalized_before,
-            finalized_after=finalized_after,
+            question_opened_before=question_opened_before,
+            question_opened_after=question_opened_after,
+            question_finalized_before=question_finalized_before,
+            question_finalized_after=question_finalized_after,
+            question_with_answers=question_with_answers,
+            question_id=question_id,
+            question_id_in=question_id_in,
+            question_current_answer_before=question_current_answer_before,
+            question_excluded_titles=question_excluded_titles,
             resolved=resolved,
             condition_id_in=condition_id_in,
             id_in=id_in,
-            excluded_questions=excluded_questions,
             liquidity_bigger_than=liquidity_bigger_than,
             collateral_token_address_in=collateral_token_address_in,
             category=category,
@@ -644,15 +641,21 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
 
     @staticmethod
     def get_reality_question_filters(
-        user: HexAddress | None = None,
-        claimed: bool | None = None,
-        current_answer_before: DatetimeUTC | None = None,
-        finalized_before: DatetimeUTC | None = None,
-        finalized_after: DatetimeUTC | None = None,
-        id_in: list[str] | None = None,
-        question_id: HexBytes | None = None,
-        question_id_in: list[HexBytes] | None = None,
+        user: HexAddress | None,
+        claimed: bool | None,
+        current_answer_before: DatetimeUTC | None,
+        finalized_before: DatetimeUTC | None,
+        finalized_after: DatetimeUTC | None,
+        with_answers: bool | None,
+        question_id: HexBytes | None,
+        question_id_in: list[HexBytes] | None,
+        opened_before: t.Optional[DatetimeUTC],
+        opened_after: t.Optional[DatetimeUTC],
+        excluded_titles: set[str] | None,
     ) -> dict[str, t.Any]:
+        """
+        Be aware, both Omen subgraph and Reality subgraph are indexing questions, but their fields are a bit different.
+        """
         where_stms: dict[str, t.Any] = {}
 
         if user is not None:
@@ -672,6 +675,12 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
                 current_answer_before
             )
 
+        if opened_before:
+            where_stms["openingTimestamp_lt"] = to_int_timestamp(opened_before)
+
+        if opened_after:
+            where_stms["openingTimestamp_gt"] = to_int_timestamp(opened_after)
+
         if finalized_before is not None:
             where_stms["answerFinalizedTimestamp_lt"] = to_int_timestamp(
                 finalized_before
@@ -682,12 +691,77 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
                 finalized_after
             )
 
-        if id_in is not None:
-            where_stms["id_in"] = id_in
+        if with_answers is not None:
+            if with_answers:
+                where_stms["answerFinalizedTimestamp_not"] = None
+            else:
+                where_stms["answerFinalizedTimestamp"] = None
 
         if question_id_in is not None:
+            # Be aware: On Omen subgraph, question's `id` represents `questionId` on reality subgraph. And `id` on reality subraph is just a weird concat of multiple things from the question.
             where_stms["questionId_in"] = [x.hex() for x in question_id_in]
 
+        if excluded_titles:
+            # Be aware: This is called `title_not_in` on Omen subgraph.
+            where_stms["qTitle_not_in"] = [i for i in excluded_titles]
+
+        return where_stms
+
+    @staticmethod
+    def get_omen_question_filters(
+        current_answer_before: DatetimeUTC | None,
+        finalized_before: DatetimeUTC | None,
+        finalized_after: DatetimeUTC | None,
+        with_answers: bool | None,
+        question_id: HexBytes | None,
+        question_id_in: list[HexBytes] | None,
+        opened_before: t.Optional[DatetimeUTC],
+        opened_after: t.Optional[DatetimeUTC],
+        excluded_titles: set[str] | None,
+    ) -> dict[str, t.Any]:
+        """
+        Be aware, both Omen subgraph and Reality subgraph are indexing questions, but their fields are a bit different.
+        """
+        where_stms: dict[str, t.Any] = {}
+
+        if question_id is not None:
+            where_stms["questionId"] = question_id.hex()
+
+        if current_answer_before is not None:
+            where_stms["currentAnswerTimestamp_lt"] = to_int_timestamp(
+                current_answer_before
+            )
+
+        if opened_before:
+            where_stms["openingTimestamp_lt"] = to_int_timestamp(opened_before)
+
+        if opened_after:
+            where_stms["openingTimestamp_gt"] = to_int_timestamp(opened_after)
+
+        if finalized_before is not None:
+            where_stms["answerFinalizedTimestamp_lt"] = to_int_timestamp(
+                finalized_before
+            )
+
+        if finalized_after is not None:
+            where_stms["answerFinalizedTimestamp_gt"] = to_int_timestamp(
+                finalized_after
+            )
+
+        if with_answers is not None:
+            if with_answers:
+                where_stms["answerFinalizedTimestamp_not"] = None
+            else:
+                where_stms["answerFinalizedTimestamp"] = None
+
+        if question_id_in is not None:
+            # Be aware: On Omen subgraph, question's `id` represents `questionId` on reality subgraph. And `id` on reality subraph is just a weird concat of multiple things from the question.
+            where_stms["id_in"] = [x.hex() for x in question_id_in]
+
+        if excluded_titles:
+            # Be aware: This is called `qTitle_not_in` on Omen subgraph.
+            where_stms["title_not_in"] = [i for i in excluded_titles]
+
         return where_stms
 
     def get_questions(
@@ -698,17 +772,25 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
         current_answer_before: DatetimeUTC | None = None,
         finalized_before: DatetimeUTC | None = None,
         finalized_after: DatetimeUTC | None = None,
-        id_in: list[str] | None = None,
+        with_answers: bool | None = None,
         question_id_in: list[HexBytes] | None = None,
+        question_id: HexBytes | None = None,
+        opened_before: DatetimeUTC | None = None,
+        opened_after: DatetimeUTC | None = None,
+        excluded_titles: set[str] | None = None,
     ) -> list[RealityQuestion]:
         where_stms: dict[str, t.Any] = self.get_reality_question_filters(
             user=user,
             claimed=claimed,
             finalized_before=finalized_before,
             finalized_after=finalized_after,
+            with_answers=with_answers,
             current_answer_before=current_answer_before,
-            id_in=id_in,
             question_id_in=question_id_in,
+            question_id=question_id,
+            opened_before=opened_before,
+            opened_after=opened_after,
+            excluded_titles=excluded_titles,
         )
         questions = self.realityeth_subgraph.Query.questions(
             first=(
@@ -738,12 +820,17 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
         self,
         limit: int | None,
         user: HexAddress | None = None,
-        question_id: HexBytes | None = None,
+        question_user: HexAddress | None = None,
         question_claimed: bool | None = None,
-        question_finalized_before: DatetimeUTC | None = None,
-        question_finalized_after: DatetimeUTC | None = None,
-        question_current_answer_before: DatetimeUTC | None = None,
+        question_opened_before: t.Optional[DatetimeUTC] = None,
+        question_opened_after: t.Optional[DatetimeUTC] = None,
+        question_finalized_before: t.Optional[DatetimeUTC] = None,
+        question_finalized_after: t.Optional[DatetimeUTC] = None,
+        question_with_answers: bool | None = None,
+        question_id: HexBytes | None = None,
         question_id_in: list[HexBytes] | None = None,
+        question_current_answer_before: DatetimeUTC | None = None,
+        question_excluded_titles: set[str] | None = None,
     ) -> list[RealityResponse]:
         where_stms: dict[str, t.Any] = {}
 
@@ -751,12 +838,17 @@ class OmenSubgraphHandler(metaclass=SingletonMeta):
             where_stms["user"] = user.lower()
 
         where_stms["question_"] = self.get_reality_question_filters(
+            user=question_user,
             question_id=question_id,
             claimed=question_claimed,
+            opened_before=question_opened_before,
+            opened_after=question_opened_after,
             finalized_before=question_finalized_before,
             finalized_after=question_finalized_after,
+            with_answers=question_with_answers,
             current_answer_before=question_current_answer_before,
             question_id_in=question_id_in,
+            excluded_titles=question_excluded_titles,
         )
 
         responses = self.realityeth_subgraph.Query.responses(

prediction_market_agent_tooling-0.52.0/prediction_market_agent_tooling/tools/is_invalid.py

@@ -0,0 +1,92 @@
+import tenacity
+
+from prediction_market_agent_tooling.config import APIKeys
+from prediction_market_agent_tooling.loggers import logger
+from prediction_market_agent_tooling.tools.cache import persistent_inmemory_cache
+from prediction_market_agent_tooling.tools.is_predictable import (
+    parse_decision_yes_no_completion,
+)
+from prediction_market_agent_tooling.tools.langfuse_ import (
+    get_langfuse_langchain_config,
+    observe,
+)
+from prediction_market_agent_tooling.tools.utils import (
+    LLM_SEED,
+    LLM_SUPER_LOW_TEMPERATURE,
+)
+
+# I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought.
+# Rules are almost copy-pasted from https://cdn.kleros.link/ipfs/QmZM12kkguXFk2C94ykrKpambt4iUVKsVsxGxDEdLS68ws/omen-rules.pdf,
+# with some small prompting mods and I removed the point about "The outcome of the market must be known by its Resolution Date.", because that can not be verified before-hand.
+# and also point about "in which none of the answers are valid will resolve as invalid" and "in which multiple answers are valid will resolve as invalid.", because before hand we can not know if one of the outcomes happened or not.
+QUESTION_IS_INVALID_PROMPT = """Main signs about an invalid question (sometimes referred to as a "market"):
+- The market's question is about immoral violence, death or assassination.
+- The violent event can be caused by a single conscious being.
+- The violent event is done illegally.
+- The market should not directly incentivize immoral violent (such as murder, rape or unjust imprisonment) actions which could likely be performed by any participant.
+- Invalid: Will Donald Trump be alive on the 01/12/2021? (Anyone could bet on “No” and kill him for a guaranteed profit. Anyone could bet on “Yes” to effectively put a bounty on his head).
+- Invalid: Will Hera be a victim of swatting in 2020? (Anyone could falsely call the emergency services on him in order to win the bet)
+- This does not prevent markets:
+  - Whose topics are violent events not caused by conscious beings.
+  - Valid: How many people will die from COVID19 in 2020? (Viruses don’t use prediction markets).
+  - Whose main source of uncertainty is not related to a potential violent action.
+  - Valid: Will Trump win the 2020 US presidential election? (The main source of uncertainty is the vote of US citizens, not a potential murder of a presidential candidate).
+  - Which could give an incentive only to specific participants to commit an immoral violent action, but are in practice unlikely.
+  - Valid: Will the US be engaged in a military conflict with a UN member state in 2021? (It’s unlikely for the US to declare war in order to win a bet on this market).
+  - Valid: Will Derek Chauvin go to jail for the murder of George Flyod? (It’s unlikely that the jurors would collude to make a wrong verdict in order to win this market).
+- Questions with relative dates will resolve as invalid. Dates must be stated in absolute terms, not relative depending on the current time.
+- Invalid: Who will be the president of the United States in 6 months? (“in 6 months depends on the current time”).
+- Questions about moral values and not facts will be resolved as invalid.
+- Invalid: “Is it ethical to eat meat?”.
+
+Follow a chain of thought to evaluate if the question is invalid:
+
+First, write the parts of the following question:
+
+"{question}"
+
+Then, write down what is the future event of the question, what it refers to and when that event will happen if the question contains it.
+
+Then, explain why do you think it is or isn't invalid.
+
+Finally, write your final decision, write `decision: ` followed by either "yes it is invalid" or "no it isn't invalid" about the question. Don't write anything else after that. You must include "yes" or "no".
+"""
+
+
+@persistent_inmemory_cache
+@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_fixed(1))
+@observe()
+def is_invalid(
+    question: str,
+    engine: str = "gpt-4o",
+    temperature: float = LLM_SUPER_LOW_TEMPERATURE,
+    seed: int = LLM_SEED,
+    prompt_template: str = QUESTION_IS_INVALID_PROMPT,
+    max_tokens: int = 1024,
+) -> bool:
+    """
+    Evaluate if the question is actually answerable.
+    """
+    try:
+        from langchain.prompts import ChatPromptTemplate
+        from langchain_openai import ChatOpenAI
+    except ImportError:
+        logger.error("langchain not installed, skipping is_invalid")
+        return True
+
+    llm = ChatOpenAI(
+        model=engine,
+        temperature=temperature,
+        seed=seed,
+        api_key=APIKeys().openai_api_key_secretstr_v1,
+    )
+
+    prompt = ChatPromptTemplate.from_template(template=prompt_template)
+    messages = prompt.format_messages(question=question)
+    completion = str(
+        llm.invoke(
+            messages, max_tokens=max_tokens, config=get_langfuse_langchain_config()
+        ).content
+    )
+
+    return parse_decision_yes_no_completion(question, completion)

{prediction_market_agent_tooling-0.51.0 → prediction_market_agent_tooling-0.52.0}/prediction_market_agent_tooling/tools/utils.py

@@ -22,8 +22,10 @@ from prediction_market_agent_tooling.loggers import logger
 T = TypeVar("T")
 
 # t=0 is mathematically impossible and it's not clear how OpenAI (and others) handle it, as a result, even with t=0, gpt-4-turbo produces very different outputs,
-# it seems that using a very low temperature is the best way to have as consistent outputs as possible: https://community.openai.com/t/why-the-api-output-is-inconsistent-even-after-the-temperature-is-set-to-0/329541/12
+# see this experiment to figure out if you should use LLM_SUPER_LOW_TEMPERATURE or just 0: https://github.com/gnosis/prediction-market-agent/pull/438.
 LLM_SUPER_LOW_TEMPERATURE = 0.00000001
+# For consistent results, also include seed for models that supports it.
+LLM_SEED = 0
 
 
 def check_not_none(

{prediction_market_agent_tooling-0.51.0 → prediction_market_agent_tooling-0.52.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "prediction-market-agent-tooling"
-version = "0.51.0"
+version = "0.52.0"
 description = "Tools to benchmark, deploy and monitor prediction market agents."
 authors = ["Gnosis"]
 readme = "README.md"

prediction_market_agent_tooling-0.51.0/prediction_market_agent_tooling/markets/metaculus/data_models.py

@@ -1,97 +0,0 @@
-from enum import Enum
-from typing import Any
-
-from pydantic import BaseModel
-
-from prediction_market_agent_tooling.tools.utils import DatetimeUTC
-
-
-class QuestionType(str, Enum):
-    forecast = "forecast"
-    notebook = "notebook"
-    discussion = "discussion"
-    claim = "claim"
-    group = "group"
-    conditional_group = "conditional_group"
-    multiple_choice = "multiple_choice"
-
-
-class CommunityPrediction(BaseModel):
-    y: list[float]
-    q1: float | None = None
-    q2: float | None = None
-    q3: float | None = None
-
-    @property
-    def p_yes(self) -> float:
-        """
-        q2 corresponds to the median, or 'second quartile' of the distribution.
-
-        If no value is provided (i.e. the question is new and has not been
-        answered yet), we default to 0.5.
-        """
-        return self.q2 if self.q2 is not None else 0.5
-
-
-class Prediction(BaseModel):
-    t: DatetimeUTC
-    x: float
-
-
-class UserPredictions(BaseModel):
-    id: int
-    predictions: list[Prediction]
-    points_won: float | None = None
-    user: int
-    username: str
-    question: int
-
-
-class CommunityPredictionStats(BaseModel):
-    full: CommunityPrediction
-    unweighted: CommunityPrediction
-
-
-class MetaculusQuestion(BaseModel):
-    """
-    https://www.metaculus.com/api2/schema/redoc/#tag/questions/operation/questions_retrieve
-    """
-
-    active_state: Any
-    url: str
-    page_url: str
-    id: int
-    author: int
-    author_name: str
-    author_id: int
-    title: str
-    title_short: str
-    group_label: str | None = None
-    resolution: int | None
-    resolved_option: int | None
-    created_time: DatetimeUTC
-    publish_time: DatetimeUTC | None = None
-    close_time: DatetimeUTC | None = None
-    effected_close_time: DatetimeUTC | None
-    resolve_time: DatetimeUTC | None = None
-    possibilities: dict[Any, Any] | None = None
-    scoring: dict[Any, Any] = {}
-    type: QuestionType | None = None
-    user_perms: Any
-    weekly_movement: float | None
-    weekly_movement_direction: int | None = None
-    cp_reveal_time: DatetimeUTC | None = None
-    edited_time: DatetimeUTC
-    last_activity_time: DatetimeUTC
-    activity: float
-    comment_count: int
-    votes: int
-    community_prediction: CommunityPredictionStats
-    my_predictions: UserPredictions | None = None
-    # TODO add the rest of the fields https://github.com/gnosis/prediction-market-agent-tooling/issues/301
-
-
-class MetaculusQuestions(BaseModel):
-    next: str | None
-    previous: str | None
-    results: list[MetaculusQuestion]