prediction-market-agent-tooling 0.42.1__py3-none-any.whl → 0.43.1__py3-none-any.whl

prediction_market_agent_tooling/markets/agent_market.py

@@ -47,6 +47,7 @@ class AgentMarket(BaseModel):
 
     id: str
     question: str
+    description: str | None
     outcomes: list[str]
     resolution: Resolution | None
     created_time: datetime | None

prediction_market_agent_tooling/markets/manifold/manifold.py

@@ -13,11 +13,12 @@ from prediction_market_agent_tooling.markets.data_models import BetAmount, Curre
 from prediction_market_agent_tooling.markets.manifold.api import (
     get_authenticated_user,
     get_manifold_binary_markets,
+    get_manifold_market,
     place_bet,
 )
 from prediction_market_agent_tooling.markets.manifold.data_models import (
     MANIFOLD_BASE_URL,
-    ManifoldMarket,
+    FullManifoldMarket,
 )
 from prediction_market_agent_tooling.tools.betting_strategies.minimum_bet_to_win import (
     minimum_bet_to_win,
@@ -58,10 +59,11 @@ class ManifoldAgentMarket(AgentMarket):
         )
 
     @staticmethod
-    def from_data_model(model: ManifoldMarket) -> "ManifoldAgentMarket":
+    def from_data_model(model: FullManifoldMarket) -> "ManifoldAgentMarket":
         return ManifoldAgentMarket(
             id=model.id,
             question=model.question,
+            description=model.textDescription,
             outcomes=model.outcomes,
             resolution=model.resolution,
             created_time=model.createdTime,
@@ -100,7 +102,7 @@ class ManifoldAgentMarket(AgentMarket):
             raise ValueError(f"Unknown filter_by: {filter_by}")
 
         return [
-            ManifoldAgentMarket.from_data_model(m)
+            ManifoldAgentMarket.from_data_model(get_manifold_market(m.id))
             for m in get_manifold_binary_markets(
                 limit=limit,
                 sort=sort,

prediction_market_agent_tooling/markets/metaculus/metaculus.py

@@ -25,6 +25,9 @@ class MetaculusAgentMarket(AgentMarket):
 
     have_predicted: bool
     base_url: t.ClassVar[str] = METACULUS_API_BASE_URL
+    description: str | None = (
+        None  # Metaculus markets don't have a description, so just default to None.
+    )
 
     @staticmethod
     def from_data_model(model: MetaculusQuestion) -> "MetaculusAgentMarket":

prediction_market_agent_tooling/markets/omen/omen.py

@@ -91,6 +91,9 @@ class OmenAgentMarket(AgentMarket):
     )
 
     _binary_market_p_yes_history: list[Probability] | None = None
+    description: str | None = (
+        None  # Omen markets don't have a description, so just default to None.
+    )
 
     @property
     def yes_index(self) -> int:

prediction_market_agent_tooling/markets/polymarket/polymarket.py

@@ -31,6 +31,7 @@ class PolymarketAgentMarket(AgentMarket):
         return PolymarketAgentMarket(
             id=model.id,
             question=model.question,
+            description=model.description,
             outcomes=[x.outcome for x in model.tokens],
             resolution=model.resolution,
             current_p_yes=model.p_yes,

prediction_market_agent_tooling/tools/is_predictable.py

@@ -2,6 +2,7 @@ from loguru import logger
 
 from prediction_market_agent_tooling.config import APIKeys
 from prediction_market_agent_tooling.tools.cache import persistent_inmemory_cache
+from prediction_market_agent_tooling.tools.utils import LLM_SUPER_LOW_TEMPERATURE
 
 # I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought.
 QUESTION_IS_PREDICTABLE_BINARY_PROMPT = """Main signs about a fully qualified question (sometimes referred to as a "market"):
@@ -28,6 +29,47 @@ Then, explain why do you think it is or isn't fully qualified.
 Finally, write your final decision, write `decision: ` followed by either "yes it is fully qualified" or "no it isn't fully qualified" about the question. Don't write anything else after that. You must include "yes" or "no".
 """
 
+QUESTION_IS_PREDICTABLE_WITHOUT_DESCRIPTION_PROMPT = """Main signs about a fully self-contained question (sometimes referred to as a "market"):
+- Description of the question can not contain any additional information required to answer the question.
+
+For the question:
+
+```
+{question}
+```
+
+And the description:
+
+```
+{description}
+```
+
+Description refers only to the text above and nothing else. 
+
+Even if the question is somewhat vague, but even the description does not contain enough of extra information, it's okay and the question is fully self-contained. 
+If the question is vague and the description contains the information required to answer the question, it's not fully self-contained and the answer is "no".
+
+Follow a chain of thought to evaluate if the question doesn't need the description to be answered.
+
+Start by examining detaily the question and the description. Write down their parts, what they refer to and what they contain. 
+
+Continue by writing comparison of the question and the description content. Write down what the question contains and what the description contains.
+
+Explain, why do you think it does or doesn't need the description.
+
+Description can contain additional information, but it can not contain any information required to answer the question.
+
+Description can contain additional information about the exact resolution criteria, but the question should be answerable even without it.
+
+As long as the question contains some time frame, it's okay if the description only specifies it in more detail.
+
+Description usually contains the question in more detailed form, but the question on its own should be answerable.
+
+For example, that means, description can not contain date if question doesn't contain it. Description can not contain target if the question doesn't contain it, etc.
+
+Finally, write your final decision, write `decision: ` followed by either "yes it is fully self-contained" or "no it isn't fully self-contained" about the question. Don't write anything else after that. You must include "yes" or "no".
+"""
+
 
 @persistent_inmemory_cache
 def is_predictable_binary(
@@ -42,12 +84,12 @@ def is_predictable_binary(
         from langchain.prompts import ChatPromptTemplate
         from langchain_openai import ChatOpenAI
     except ImportError:
-        logger.info("langchain not installed, skipping is_predictable_binary")
+        logger.error("langchain not installed, skipping is_predictable_binary")
         return True
 
     llm = ChatOpenAI(
         model=engine,
-        temperature=0.0,
+        temperature=LLM_SUPER_LOW_TEMPERATURE,
         api_key=APIKeys().openai_api_key_secretstr_v1,
     )
 
@@ -55,20 +97,54 @@ def is_predictable_binary(
     messages = prompt.format_messages(question=question)
     completion = str(llm(messages, max_tokens=512).content)
 
+    return parse_decision_yes_no_completion(question, completion)
+
+
+@persistent_inmemory_cache
+def is_predictable_without_description(
+    question: str,
+    description: str,
+    engine: str = "gpt-4-1106-preview",
+    prompt_template: str = QUESTION_IS_PREDICTABLE_WITHOUT_DESCRIPTION_PROMPT,
+) -> bool:
+    """
+    Evaluate if the question is fully self-contained.
+    """
+    try:
+        from langchain.prompts import ChatPromptTemplate
+        from langchain_openai import ChatOpenAI
+    except ImportError:
+        logger.error(
+            "langchain not installed, skipping is_predictable_without_description"
+        )
+        return True
+
+    llm = ChatOpenAI(
+        model=engine,
+        temperature=LLM_SUPER_LOW_TEMPERATURE,
+        api_key=APIKeys().openai_api_key_secretstr_v1,
+    )
+
+    prompt = ChatPromptTemplate.from_template(template=prompt_template)
+    messages = prompt.format_messages(
+        question=question,
+        description=description,
+    )
+    completion = str(llm(messages, max_tokens=512).content)
+
+    return parse_decision_yes_no_completion(question, completion)
+
+
+def parse_decision_yes_no_completion(question: str, completion: str) -> bool:
+    logger.debug(completion)
     try:
         decision = completion.lower().rsplit("decision", 1)[1]
     except IndexError as e:
-        raise ValueError(
-            f"Invalid completion in is_predictable for `{question}`: {completion}"
-        ) from e
+        raise ValueError(f"Invalid completion for `{question}`: {completion}") from e
 
     if "yes" in decision:
-        is_predictable = True
+        return True
     elif "no" in decision:
-        is_predictable = False
+        return False
     else:
-        raise ValueError(
-            f"Invalid completion in is_predictable for `{question}`: {completion}"
-        )
-
-    return is_predictable
+        raise ValueError(f"Invalid completion for `{question}`: {completion}")

prediction_market_agent_tooling/tools/parallelism.py

@@ -1,10 +1,12 @@
 import concurrent
 from concurrent.futures import Executor
+from concurrent.futures.process import ProcessPoolExecutor
 from concurrent.futures.thread import ThreadPoolExecutor
 from typing import Callable, Generator, TypeVar
 
 # Max workers to 5 to avoid rate limiting on some APIs, create a custom executor if you need more workers.
 DEFAULT_THREADPOOL_EXECUTOR = ThreadPoolExecutor(max_workers=5)
+DEFAULT_PROCESSPOOL_EXECUTOR = ProcessPoolExecutor(max_workers=5)
 
 A = TypeVar("A")
 B = TypeVar("B")

prediction_market_agent_tooling/tools/utils.py

@@ -19,6 +19,10 @@ from prediction_market_agent_tooling.loggers import logger
 
 T = TypeVar("T")
 
+# t=0 is mathematically impossible and it's not clear how OpenAI (and others) handle it, as a result, even with t=0, gpt-4-turbo produces very different outputs,
+# it seems that using a very low temperature is the best way to have as consistent outputs as possible: https://community.openai.com/t/why-the-api-output-is-inconsistent-even-after-the-temperature-is-set-to-0/329541/12
+LLM_SUPER_LOW_TEMPERATURE = 0.00000001
+
 
 def check_not_none(
     value: Optional[T],

{prediction_market_agent_tooling-0.42.1.dist-info → prediction_market_agent_tooling-0.43.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prediction-market-agent-tooling
-Version: 0.42.1
+Version: 0.43.1
 Summary: Tools to benchmark, deploy and monitor prediction market agents.
 Author: Gnosis
 Requires-Python: >=3.10,<3.12

{prediction_market_agent_tooling-0.42.1.dist-info → prediction_market_agent_tooling-0.43.1.dist-info}/RECORD

@@ -21,28 +21,28 @@ prediction_market_agent_tooling/deploy/gcp/kubernetes_models.py,sha256=qYIHRxQLa
 prediction_market_agent_tooling/deploy/gcp/utils.py,sha256=oyW0jgrUT2Tr49c7GlpcMsYNQjoCSOcWis3q-MmVAhU,6089
 prediction_market_agent_tooling/gtypes.py,sha256=lbV2nsPyhMIRI9olx0_6A06jwTWKYBPGMxyiGVFysag,2467
 prediction_market_agent_tooling/loggers.py,sha256=ua9rynYmsbOJZjxPIFxRBooomeN08zuLSJ7lxZMDS7w,3133
-prediction_market_agent_tooling/markets/agent_market.py,sha256=zUxDOR-fx1roQKdUCN3FsBfnziIL6POTNK1NsMOgLFw,8365
+prediction_market_agent_tooling/markets/agent_market.py,sha256=4xMZSiNdr5NiJxuPWqhA6tnl4VloAxSpLvQdY7ySwAA,8393
 prediction_market_agent_tooling/markets/categorize.py,sha256=HyKSFHXPL7Hfe90ahbF7xszamYREUVrPkLcpifw1V9Y,935
 prediction_market_agent_tooling/markets/data_models.py,sha256=qD0LyFkzimaMkDVE0QO2a4I9fQ8qpO2qPmVzb-0JBik,2085
 prediction_market_agent_tooling/markets/manifold/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 prediction_market_agent_tooling/markets/manifold/api.py,sha256=AC2zmkzpBU3P4kyybs7CgPbDg4hLAx3GY5mjgDi7qDo,7221
 prediction_market_agent_tooling/markets/manifold/data_models.py,sha256=jHqOzOiN21wYvDNyh4VtbGtj4adWr6vA4liOQmh24cc,6239
-prediction_market_agent_tooling/markets/manifold/manifold.py,sha256=EwRL06E2Y_ZAzr8efwS5yD6p6rnykrcBhqmNDUGZ8Aw,4075
+prediction_market_agent_tooling/markets/manifold/manifold.py,sha256=ai82Ozzx-BdtIDvORmepODh3S-EeyAvYBQyQUEarMLg,4179
 prediction_market_agent_tooling/markets/manifold/utils.py,sha256=cPPFWXm3vCYH1jy7_ctJZuQH9ZDaPL4_AgAYzGWkoow,513
 prediction_market_agent_tooling/markets/markets.py,sha256=Hz3E7LJ5HIjCHQtdU5_Bymav2dYT0dDxKOL0i8mV0mg,3142
 prediction_market_agent_tooling/markets/metaculus/api.py,sha256=gvPQVAM5NlCyWzEMt4WML9saRBsK9eiHAZP6jwirVqc,2750
 prediction_market_agent_tooling/markets/metaculus/data_models.py,sha256=6TBy17xntdLBR61QCE5wddwTa_k2D0D8ZgK6p7sGUuc,2448
-prediction_market_agent_tooling/markets/metaculus/metaculus.py,sha256=KMWksTNFCKxqjeMiuRh5yNarPSv7sH0aDkgeC3fm3Dg,3293
+prediction_market_agent_tooling/markets/metaculus/metaculus.py,sha256=uNF7LP4evvubk818g2zbX1VlnFxeUQOkNgx_e_LwaJA,3416
 prediction_market_agent_tooling/markets/omen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 prediction_market_agent_tooling/markets/omen/data_models.py,sha256=oY-5wvFR4lXaTsmXVKrqrAQ8TWd-mAHLaxI1ETatLOc,14477
-prediction_market_agent_tooling/markets/omen/omen.py,sha256=VHZmO5_5i3ydzBvvjSas-8AfkmnQ87pMFStYUETQW2Y,38996
+prediction_market_agent_tooling/markets/omen/omen.py,sha256=GSuUP8gjFKy6zp2dfq5AGlzu5T7B08cFdjjlw07rSdU,39114
 prediction_market_agent_tooling/markets/omen/omen_contracts.py,sha256=JGXCO9MIVr-DGyoH2sxReAw7ZDTC_ev0UxDpq1QBv5Q,21854
 prediction_market_agent_tooling/markets/omen/omen_resolving.py,sha256=g77QsQ5WnSI2rzBlX87L_EhWMwobkyXyfRhHQmpAdzo,9012
 prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py,sha256=kDbeZ8ImLtBSFz0GoGdCqeC1Xd3_eGMAxvmLFT8kp8Y,25222
 prediction_market_agent_tooling/markets/polymarket/api.py,sha256=HXmA1akA0qDj0m3e-GEvWG8x75pm6BX4H7YJPQcST7I,4767
 prediction_market_agent_tooling/markets/polymarket/data_models.py,sha256=9CJzakyEcsn6DQBK2nOXjOMzTZBLAmK_KqevXvW17DI,4292
 prediction_market_agent_tooling/markets/polymarket/data_models_web.py,sha256=f8SRQy0Rn-gIHSEMrJJAI8H3J7l8lzOLj3aCMe0vJv8,11324
-prediction_market_agent_tooling/markets/polymarket/polymarket.py,sha256=zqGfiUOH9f7jmDcQAt30wrHyfmqLMNwCYtotsxFoJmA,2678
+prediction_market_agent_tooling/markets/polymarket/polymarket.py,sha256=xGJejGYoDbLBfXabhOI85ZMNnAMsWTedCCPKP6KfGno,2721
 prediction_market_agent_tooling/markets/polymarket/utils.py,sha256=m4JG6WULh5epCJt4XBMHg0ae5NoVhqlOvAl0A7DR9iM,2023
 prediction_market_agent_tooling/monitor/langfuse/langfuse_wrapper.py,sha256=b6T69YB1x8kSUvW9uRFuSWPLOrXzapZG7m5O5SU0QTQ,895
 prediction_market_agent_tooling/monitor/markets/manifold.py,sha256=GdYpgRX1GahDi-75Mr53jgtEg6nWcs_rHDUkg4o_7dQ,3352
@@ -66,15 +66,15 @@ prediction_market_agent_tooling/tools/google.py,sha256=SfVDxb3oEOUK8mpd0l3mTX9yb
 prediction_market_agent_tooling/tools/hexbytes_custom.py,sha256=Bp94qgPjvjWf1Vb4lNzGFDXRdThw1rJ91vL6r2PWq5E,2096
 prediction_market_agent_tooling/tools/image_gen/image_gen.py,sha256=HzRwBx62hOXBOmrtpkXaP9Qq1Ku03uUGdREocyjLQ_k,1266
 prediction_market_agent_tooling/tools/image_gen/market_thumbnail_gen.py,sha256=g-2BWS11CS7K6QRNbZRMTX_Xo2xOwjyCdhnwaQygX1Y,1104
-prediction_market_agent_tooling/tools/is_predictable.py,sha256=radTwyxfUGfE1HCKWlvKl05Cpv_HD7E1d6A_Wm1K5ek,2930
-prediction_market_agent_tooling/tools/parallelism.py,sha256=8mgkF5sBwFGS5GMvlpzam82Y3p2swPYuNsywpQuy-a4,1508
+prediction_market_agent_tooling/tools/is_predictable.py,sha256=EobE2J0g2mfwD-o3Jv-cK6_EVMlnItZQunqWiNLqOvI,6188
+prediction_market_agent_tooling/tools/parallelism.py,sha256=Rz8QdVUWX8KCbr8UZfaC_b1GBWIb3bXwITUumuvBJ60,1633
 prediction_market_agent_tooling/tools/safe.py,sha256=h0xOO0eNtitClf0fPkn-0oTc6A_bflDTee98V_aiV-A,5195
 prediction_market_agent_tooling/tools/singleton.py,sha256=CiIELUiI-OeS7U7eeHEt0rnVhtQGzwoUdAgn_7u_GBM,729
 prediction_market_agent_tooling/tools/streamlit_user_login.py,sha256=NXEqfjT9Lc9QtliwSGRASIz1opjQ7Btme43H4qJbzgE,3010
-prediction_market_agent_tooling/tools/utils.py,sha256=-G22UEbCRm59bm1RWFdeF55hRsaxgwZVAHvK32-Ye1g,6190
+prediction_market_agent_tooling/tools/utils.py,sha256=JE9YWtPPhnTgLiOyGAZDNG5K8nCwUY9IZEuAlm9UcxA,6611
 prediction_market_agent_tooling/tools/web3_utils.py,sha256=nKRHmdLnWSKd3wpo-cysXGvhhrJ2Yf69sN2FFQfSt6s,10578
-prediction_market_agent_tooling-0.42.1.dist-info/LICENSE,sha256=6or154nLLU6bELzjh0mCreFjt0m2v72zLi3yHE0QbeE,7650
-prediction_market_agent_tooling-0.42.1.dist-info/METADATA,sha256=2sleEbYjQ1DxF34LiETR16iXB1HukqtLjeMVKJYRqNQ,7634
-prediction_market_agent_tooling-0.42.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-prediction_market_agent_tooling-0.42.1.dist-info/entry_points.txt,sha256=m8PukHbeH5g0IAAmOf_1Ahm-sGAMdhSSRQmwtpmi2s8,81
-prediction_market_agent_tooling-0.42.1.dist-info/RECORD,,
+prediction_market_agent_tooling-0.43.1.dist-info/LICENSE,sha256=6or154nLLU6bELzjh0mCreFjt0m2v72zLi3yHE0QbeE,7650
+prediction_market_agent_tooling-0.43.1.dist-info/METADATA,sha256=a-sGAV5KiQixdLUfQXr4NHxJ848IBQ2uzLsKTqNB-Xs,7634
+prediction_market_agent_tooling-0.43.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+prediction_market_agent_tooling-0.43.1.dist-info/entry_points.txt,sha256=m8PukHbeH5g0IAAmOf_1Ahm-sGAMdhSSRQmwtpmi2s8,81
+prediction_market_agent_tooling-0.43.1.dist-info/RECORD,,