prediction-market-agent-tooling 0.42.0__tar.gz → 0.43.0__tar.gz

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prediction-market-agent-tooling
-Version: 0.42.0
+Version: 0.43.0
 Summary: Tools to benchmark, deploy and monitor prediction market agents.
 Author: Gnosis
 Requires-Python: >=3.10,<3.12
@@ -21,9 +21,9 @@ Requires-Dist: google-cloud-functions (>=1.16.0,<2.0.0)
 Requires-Dist: google-cloud-resource-manager (>=1.12.0,<2.0.0)
 Requires-Dist: google-cloud-secret-manager (>=2.18.2,<3.0.0)
 Requires-Dist: isort (>=5.13.2,<6.0.0)
-Requires-Dist: langchain (>=0.1.9,<0.2.0) ; extra == "langchain"
+Requires-Dist: langchain (>=0.2.6,<0.3.0) ; extra == "langchain"
 Requires-Dist: langchain-community (>=0.0.19)
-Requires-Dist: langchain-openai (>=0.0.5,<0.0.6) ; extra == "langchain"
+Requires-Dist: langchain-openai (>=0.1.0,<0.2.0) ; extra == "langchain"
 Requires-Dist: langfuse (>=2.27.1,<3.0.0)
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: numpy (>=1.26.4,<2.0.0)

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/config.py

@@ -3,9 +3,14 @@ import typing as t
 from gnosis.eth import EthereumClient
 from gnosis.safe import Safe
 from pydantic.types import SecretStr
+from pydantic.v1.types import SecretStr as SecretStrV1
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-from prediction_market_agent_tooling.gtypes import ChecksumAddress, PrivateKey
+from prediction_market_agent_tooling.gtypes import (
+    ChecksumAddress,
+    PrivateKey,
+    secretstr_to_v1_secretstr,
+)
 from prediction_market_agent_tooling.markets.manifold.api import get_authenticated_user
 from prediction_market_agent_tooling.tools.utils import check_not_none
 from prediction_market_agent_tooling.tools.web3_utils import private_key_to_public_key
@@ -87,6 +92,10 @@ class APIKeys(BaseSettings):
             self.OPENAI_API_KEY, "OPENAI_API_KEY missing in the environment."
         )
 
+    @property
+    def openai_api_key_secretstr_v1(self) -> SecretStrV1:
+        return secretstr_to_v1_secretstr(self.openai_api_key)
+
     @property
     def graph_api_key(self) -> SecretStr:
         return check_not_none(

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/markets/agent_market.py

@@ -47,6 +47,7 @@ class AgentMarket(BaseModel):
 
     id: str
     question: str
+    description: str | None
     outcomes: list[str]
     resolution: Resolution | None
     created_time: datetime | None

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/markets/categorize.py

@@ -21,7 +21,7 @@ Write only the category itself, nothing else.
         prompt
         | ChatOpenAI(
             model=model,
-            api_key=APIKeys().openai_api_key.get_secret_value(),
+            api_key=APIKeys().openai_api_key_secretstr_v1,
         )
         | StrOutputParser()
     )

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/markets/manifold/manifold.py

@@ -13,11 +13,12 @@ from prediction_market_agent_tooling.markets.data_models import BetAmount, Curre
 from prediction_market_agent_tooling.markets.manifold.api import (
     get_authenticated_user,
     get_manifold_binary_markets,
+    get_manifold_market,
     place_bet,
 )
 from prediction_market_agent_tooling.markets.manifold.data_models import (
     MANIFOLD_BASE_URL,
-    ManifoldMarket,
+    FullManifoldMarket,
 )
 from prediction_market_agent_tooling.tools.betting_strategies.minimum_bet_to_win import (
     minimum_bet_to_win,
@@ -58,10 +59,11 @@ class ManifoldAgentMarket(AgentMarket):
         )
 
     @staticmethod
-    def from_data_model(model: ManifoldMarket) -> "ManifoldAgentMarket":
+    def from_data_model(model: FullManifoldMarket) -> "ManifoldAgentMarket":
         return ManifoldAgentMarket(
             id=model.id,
             question=model.question,
+            description=model.textDescription,
             outcomes=model.outcomes,
             resolution=model.resolution,
             created_time=model.createdTime,
@@ -100,7 +102,7 @@ class ManifoldAgentMarket(AgentMarket):
             raise ValueError(f"Unknown filter_by: {filter_by}")
 
         return [
-            ManifoldAgentMarket.from_data_model(m)
+            ManifoldAgentMarket.from_data_model(get_manifold_market(m.id))
             for m in get_manifold_binary_markets(
                 limit=limit,
                 sort=sort,

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/markets/metaculus/metaculus.py

@@ -25,6 +25,9 @@ class MetaculusAgentMarket(AgentMarket):
 
     have_predicted: bool
     base_url: t.ClassVar[str] = METACULUS_API_BASE_URL
+    description: str | None = (
+        None  # Metaculus markets don't have a description, so just default to None.
+    )
 
     @staticmethod
     def from_data_model(model: MetaculusQuestion) -> "MetaculusAgentMarket":

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/markets/omen/omen.py

@@ -91,6 +91,9 @@ class OmenAgentMarket(AgentMarket):
     )
 
     _binary_market_p_yes_history: list[Probability] | None = None
+    description: str | None = (
+        None  # Omen markets don't have a description, so just default to None.
+    )
 
     @property
     def yes_index(self) -> int:

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/markets/polymarket/polymarket.py

@@ -31,6 +31,7 @@ class PolymarketAgentMarket(AgentMarket):
         return PolymarketAgentMarket(
             id=model.id,
             question=model.question,
+            description=model.description,
             outcomes=[x.outcome for x in model.tokens],
             resolution=model.resolution,
             current_p_yes=model.p_yes,

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/tools/image_gen/market_thumbnail_gen.py

@@ -14,7 +14,7 @@ def rewrite_question_into_image_generation_prompt(question: str) -> str:
     llm = ChatOpenAI(
         model="gpt-4-turbo",
         temperature=0.0,
-        api_key=APIKeys().openai_api_key.get_secret_value(),
+        api_key=APIKeys().openai_api_key_secretstr_v1,
     )
     rewritten = str(
         llm.invoke(

prediction_market_agent_tooling-0.43.0/prediction_market_agent_tooling/tools/is_predictable.py

@@ -0,0 +1,150 @@
+from loguru import logger
+
+from prediction_market_agent_tooling.config import APIKeys
+from prediction_market_agent_tooling.tools.cache import persistent_inmemory_cache
+from prediction_market_agent_tooling.tools.utils import LLM_SUPER_LOW_TEMPERATURE
+
+# I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought.
+QUESTION_IS_PREDICTABLE_BINARY_PROMPT = """Main signs about a fully qualified question (sometimes referred to as a "market"):
+- The market's question needs to be specific, without use of pronouns.
+- The market's question needs to have a clear future event.
+- The market's question needs to have a clear time frame.
+- The event in the market's question doesn't have to be ultra-specific, it will be decided by a crowd later on.
+- If the market's question contains date, but without an year, it's okay.
+- If the market's question contains year, but without an exact date, it's okay.
+- The market's question can not be about itself or refer to itself.
+- The answer is probably Google-able, after the event happened.
+- The potential asnwer can be only "Yes" or "No".
+
+Follow a chain of thought to evaluate if the question is fully qualified:
+
+First, write the parts of the following question:
+
+"{question}"
+
+Then, write down what is the future event of the question, what it refers to and when that event will happen if the question contains it.
+
+Then, explain why do you think it is or isn't fully qualified.
+
+Finally, write your final decision, write `decision: ` followed by either "yes it is fully qualified" or "no it isn't fully qualified" about the question. Don't write anything else after that. You must include "yes" or "no".
+"""
+
+QUESTION_IS_PREDICTABLE_WITHOUT_DESCRIPTION_PROMPT = """Main signs about a fully self-contained question (sometimes referred to as a "market"):
+- Description of the question can not contain any additional information required to answer the question.
+
+For the question:
+
+```
+{question}
+```
+
+And the description:
+
+```
+{description}
+```
+
+Description refers only to the text above and nothing else. 
+
+Even if the question is somewhat vague, but even the description does not contain enough of extra information, it's okay and the question is fully self-contained. 
+If the question is vague and the description contains the information required to answer the question, it's not fully self-contained and the answer is "no".
+
+Follow a chain of thought to evaluate if the question doesn't need the description to be answered.
+
+Start by examining detaily the question and the description. Write down their parts, what they refer to and what they contain. 
+
+Continue by writing comparison of the question and the description content. Write down what the question contains and what the description contains.
+
+Explain, why do you think it does or doesn't need the description.
+
+Description can contain additional information, but it can not contain any information required to answer the question.
+
+Description can contain additional information about the exact resolution criteria, but the question should be answerable even without it.
+
+As long as the question contains some time frame, it's okay if the description only specifies it in more detail.
+
+Description usually contains the question in more detailed form, but the question on its own should be answerable.
+
+For example, that means, description can not contain date if question doesn't contain it. Description can not contain target if the question doesn't contain it, etc.
+
+Finally, write your final decision, write `decision: ` followed by either "yes it is fully self-contained" or "no it isn't fully self-contained" about the question. Don't write anything else after that. You must include "yes" or "no".
+"""
+
+
+@persistent_inmemory_cache
+def is_predictable_binary(
+    question: str,
+    engine: str = "gpt-4-1106-preview",
+    prompt_template: str = QUESTION_IS_PREDICTABLE_BINARY_PROMPT,
+) -> bool:
+    """
+    Evaluate if the question is actually answerable.
+    """
+    try:
+        from langchain.prompts import ChatPromptTemplate
+        from langchain_openai import ChatOpenAI
+    except ImportError:
+        logger.error("langchain not installed, skipping is_predictable_binary")
+        return True
+
+    llm = ChatOpenAI(
+        model=engine,
+        temperature=LLM_SUPER_LOW_TEMPERATURE,
+        api_key=APIKeys().openai_api_key_secretstr_v1,
+    )
+
+    prompt = ChatPromptTemplate.from_template(template=prompt_template)
+    messages = prompt.format_messages(question=question)
+    completion = str(llm(messages, max_tokens=512).content)
+
+    return parse_decision_yes_no_completion(question, completion)
+
+
+@persistent_inmemory_cache
+def is_predictable_without_description(
+    question: str,
+    description: str,
+    engine: str = "gpt-4-1106-preview",
+    prompt_template: str = QUESTION_IS_PREDICTABLE_WITHOUT_DESCRIPTION_PROMPT,
+) -> bool:
+    """
+    Evaluate if the question is fully self-contained.
+    """
+    try:
+        from langchain.prompts import ChatPromptTemplate
+        from langchain_openai import ChatOpenAI
+    except ImportError:
+        logger.error(
+            "langchain not installed, skipping is_predictable_without_description"
+        )
+        return True
+
+    llm = ChatOpenAI(
+        model=engine,
+        temperature=LLM_SUPER_LOW_TEMPERATURE,
+        api_key=APIKeys().openai_api_key_secretstr_v1,
+    )
+
+    prompt = ChatPromptTemplate.from_template(template=prompt_template)
+    messages = prompt.format_messages(
+        question=question,
+        description=description,
+    )
+    completion = str(llm(messages, max_tokens=512).content)
+
+    return parse_decision_yes_no_completion(question, completion)
+
+
+def parse_decision_yes_no_completion(question: str, completion: str) -> bool:
+    logger.debug(completion)
+    try:
+        decision = completion.lower().rsplit("decision", 1)[1]
+    except IndexError as e:
+        raise ValueError(f"Invalid completion for `{question}`: {completion}") from e
+
+    if "yes" in decision:
+        return True
+    elif "no" in decision:
+        return False
+    else:
+        raise ValueError(f"Invalid completion for `{question}`: {completion}")

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/prediction_market_agent_tooling/tools/utils.py

@@ -19,6 +19,10 @@ from prediction_market_agent_tooling.loggers import logger
 
 T = TypeVar("T")
 
+# t=0 is mathematically impossible and it's not clear how OpenAI (and others) handle it, as a result, even with t=0, gpt-4-turbo produces very different outputs,
+# it seems that using a very low temperature is the best way to have as consistent outputs as possible: https://community.openai.com/t/why-the-api-output-is-inconsistent-even-after-the-temperature-is-set-to-0/329541/12
+LLM_SUPER_LOW_TEMPERATURE = 0.00000001
+
 
 def check_not_none(
     value: Optional[T],

{prediction_market_agent_tooling-0.42.0 → prediction_market_agent_tooling-0.43.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "prediction-market-agent-tooling"
-version = "0.42.0"
+version = "0.43.0"
 description = "Tools to benchmark, deploy and monitor prediction market agents."
 authors = ["Gnosis"]
 readme = "README.md"
@@ -31,8 +31,8 @@ scikit-learn = "^1.3.1"
 tabulate = "^0.9.0"
 types-pytz = "^2024.1.0.20240203"
 google-cloud-secret-manager = "^2.18.2"
-langchain = { version = "^0.1.9", optional = true}
-langchain-openai = { version = "^0.0.5", optional = true}
+langchain = { version = "^0.2.6", optional = true}
+langchain-openai = { version = "^0.1.0", optional = true}
 google-api-python-client = { version = "2.95.0", optional = true}
 subgrounds = "^1.9.1"
 loguru = "^0.7.2"

prediction_market_agent_tooling-0.42.0/prediction_market_agent_tooling/tools/is_predictable.py

@@ -1,74 +0,0 @@
-from loguru import logger
-
-from prediction_market_agent_tooling.config import APIKeys
-from prediction_market_agent_tooling.tools.cache import persistent_inmemory_cache
-
-# I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought.
-QUESTION_IS_PREDICTABLE_BINARY_PROMPT = """Main signs about a fully qualified question (sometimes referred to as a "market"):
-- The market's question needs to be specific, without use of pronouns.
-- The market's question needs to have a clear future event.
-- The market's question needs to have a clear time frame.
-- The event in the market's question doesn't have to be ultra-specific, it will be decided by a crowd later on.
-- If the market's question contains date, but without an year, it's okay.
-- If the market's question contains year, but without an exact date, it's okay.
-- The market's question can not be about itself or refer to itself.
-- The answer is probably Google-able, after the event happened.
-- The potential asnwer can be only "Yes" or "No".
-
-Follow a chain of thought to evaluate if the question is fully qualified:
-
-First, write the parts of the following question:
-
-"{question}"
-
-Then, write down what is the future event of the question, what it refers to and when that event will happen if the question contains it.
-
-Then, explain why do you think it is or isn't fully qualified.
-
-Finally, write your final decision, write `decision: ` followed by either "yes it is fully qualified" or "no it isn't fully qualified" about the question. Don't write anything else after that. You must include "yes" or "no".
-"""
-
-
-@persistent_inmemory_cache
-def is_predictable_binary(
-    question: str,
-    engine: str = "gpt-4-1106-preview",
-    prompt_template: str = QUESTION_IS_PREDICTABLE_BINARY_PROMPT,
-) -> bool:
-    """
-    Evaluate if the question is actually answerable.
-    """
-    try:
-        from langchain.prompts import ChatPromptTemplate
-        from langchain_openai import ChatOpenAI
-    except ImportError:
-        logger.info("langchain not installed, skipping is_predictable_binary")
-        return True
-
-    llm = ChatOpenAI(
-        model=engine,
-        temperature=0.0,
-        api_key=APIKeys().openai_api_key.get_secret_value(),
-    )
-
-    prompt = ChatPromptTemplate.from_template(template=prompt_template)
-    messages = prompt.format_messages(question=question)
-    completion = str(llm(messages, max_tokens=512).content)
-
-    try:
-        decision = completion.lower().rsplit("decision", 1)[1]
-    except IndexError as e:
-        raise ValueError(
-            f"Invalid completion in is_predictable for `{question}`: {completion}"
-        ) from e
-
-    if "yes" in decision:
-        is_predictable = True
-    elif "no" in decision:
-        is_predictable = False
-    else:
-        raise ValueError(
-            f"Invalid completion in is_predictable for `{question}`: {completion}"
-        )
-
-    return is_predictable