prediction-market-agent-tooling 0.14.1__tar.gz → 0.15.1__tar.gz

{prediction_market_agent_tooling-0.14.1 → prediction_market_agent_tooling-0.15.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prediction-market-agent-tooling
-Version: 0.14.1
+Version: 0.15.1
 Summary: Tools to benchmark, deploy and monitor prediction market agents.
 Author: Gnosis
 Requires-Python: >=3.10,<3.12
@@ -11,6 +11,7 @@ Provides-Extra: google
 Provides-Extra: langchain
 Requires-Dist: autoflake (>=2.2.1,<3.0.0)
 Requires-Dist: cron-validator (>=1.0.8,<2.0.0)
+Requires-Dist: eth-account (>=0.8.0,<0.9.0)
 Requires-Dist: eth-typing (>=3.0.0,<4.0.0)
 Requires-Dist: functions-framework (>=3.5.0,<4.0.0)
 Requires-Dist: google-api-python-client (==2.95.0) ; extra == "google"
@@ -24,8 +25,11 @@ Requires-Dist: langchain-openai (>=0.0.5,<0.0.6) ; extra == "langchain"
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: mech-client (>=0.2.13,<0.3.0)
 Requires-Dist: numpy (>=1.26.4,<2.0.0)
+Requires-Dist: prompt-toolkit (>=3.0.43,<4.0.0)
 Requires-Dist: pydantic (>=2.6.1,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.1.0,<3.0.0)
+Requires-Dist: safe-cli (>=1.0.0,<2.0.0)
+Requires-Dist: safe-eth-py (>=6.0.0b14,<7.0.0)
 Requires-Dist: scikit-learn (>=1.4.0,<2.0.0)
 Requires-Dist: streamlit (>=1.31.0,<2.0.0)
 Requires-Dist: subgrounds (>=1.8.1,<2.0.0)
@@ -104,6 +108,14 @@ class DeployableCoinFlipAgent(DeployableAgent):
 DeployableCoinFlipAgent().deploy_gcp(...)
 ```
 
+For deploying a Safe manually for a given agent, run the script below:
+
+```commandline
+poetry run python scripts/create_safe_for_agent.py  --from-private-key <YOUR_AGENT_PRIVATE_KEY> --salt-nonce 42
+```
+This will output the newly created Safe in the terminal, and it can then be copied over to the deployment part (e.g. Terraform).
+Note that `salt_nonce` can be passed so that the created safe is deterministically created for each agent, so that, if the same `salt_nonce` is used, the script will not create a new Safe for the agent, instead it will output the previously existent Safe.
+
 ## Monitoring
 
 Monitor the performance of the agents deployed to GCP, as well as meta-metrics of the prediction market platforms they are deployed to.

{prediction_market_agent_tooling-0.14.1 → prediction_market_agent_tooling-0.15.1}/README.md

@@ -65,6 +65,14 @@ class DeployableCoinFlipAgent(DeployableAgent):
 DeployableCoinFlipAgent().deploy_gcp(...)
 ```
 
+For deploying a Safe manually for a given agent, run the script below:
+
+```commandline
+poetry run python scripts/create_safe_for_agent.py  --from-private-key <YOUR_AGENT_PRIVATE_KEY> --salt-nonce 42
+```
+This will output the newly created Safe in the terminal, and it can then be copied over to the deployment part (e.g. Terraform).
+Note that `salt_nonce` can be passed so that the created safe is deterministically created for each agent, so that, if the same `salt_nonce` is used, the script will not create a new Safe for the agent, instead it will output the previously existent Safe.
+
 ## Monitoring
 
 Monitor the performance of the agents deployed to GCP, as well as meta-metrics of the prediction market platforms they are deployed to.

prediction_market_agent_tooling-0.15.1/prediction_market_agent_tooling/abis/erc20.abi.json

@@ -0,0 +1,315 @@
+[
+			{
+				"inputs": [],
+				"stateMutability": "nonpayable",
+				"type": "constructor"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "spender",
+						"type": "address"
+					},
+					{
+						"internalType": "uint256",
+						"name": "allowance",
+						"type": "uint256"
+					},
+					{
+						"internalType": "uint256",
+						"name": "needed",
+						"type": "uint256"
+					}
+				],
+				"name": "ERC20InsufficientAllowance",
+				"type": "error"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "sender",
+						"type": "address"
+					},
+					{
+						"internalType": "uint256",
+						"name": "balance",
+						"type": "uint256"
+					},
+					{
+						"internalType": "uint256",
+						"name": "needed",
+						"type": "uint256"
+					}
+				],
+				"name": "ERC20InsufficientBalance",
+				"type": "error"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "approver",
+						"type": "address"
+					}
+				],
+				"name": "ERC20InvalidApprover",
+				"type": "error"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "receiver",
+						"type": "address"
+					}
+				],
+				"name": "ERC20InvalidReceiver",
+				"type": "error"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "sender",
+						"type": "address"
+					}
+				],
+				"name": "ERC20InvalidSender",
+				"type": "error"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "spender",
+						"type": "address"
+					}
+				],
+				"name": "ERC20InvalidSpender",
+				"type": "error"
+			},
+			{
+				"anonymous": False,
+				"inputs": [
+					{
+						"indexed": True,
+						"internalType": "address",
+						"name": "owner",
+						"type": "address"
+					},
+					{
+						"indexed": True,
+						"internalType": "address",
+						"name": "spender",
+						"type": "address"
+					},
+					{
+						"indexed": False,
+						"internalType": "uint256",
+						"name": "value",
+						"type": "uint256"
+					}
+				],
+				"name": "Approval",
+				"type": "event"
+			},
+			{
+				"anonymous": False,
+				"inputs": [
+					{
+						"indexed": True,
+						"internalType": "address",
+						"name": "from",
+						"type": "address"
+					},
+					{
+						"indexed": True,
+						"internalType": "address",
+						"name": "to",
+						"type": "address"
+					},
+					{
+						"indexed": False,
+						"internalType": "uint256",
+						"name": "value",
+						"type": "uint256"
+					}
+				],
+				"name": "Transfer",
+				"type": "event"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "owner",
+						"type": "address"
+					},
+					{
+						"internalType": "address",
+						"name": "spender",
+						"type": "address"
+					}
+				],
+				"name": "allowance",
+				"outputs": [
+					{
+						"internalType": "uint256",
+						"name": "",
+						"type": "uint256"
+					}
+				],
+				"stateMutability": "view",
+				"type": "function"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "spender",
+						"type": "address"
+					},
+					{
+						"internalType": "uint256",
+						"name": "value",
+						"type": "uint256"
+					}
+				],
+				"name": "approve",
+				"outputs": [
+					{
+						"internalType": "bool",
+						"name": "",
+						"type": "bool"
+					}
+				],
+				"stateMutability": "nonpayable",
+				"type": "function"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "account",
+						"type": "address"
+					}
+				],
+				"name": "balanceOf",
+				"outputs": [
+					{
+						"internalType": "uint256",
+						"name": "",
+						"type": "uint256"
+					}
+				],
+				"stateMutability": "view",
+				"type": "function"
+			},
+			{
+				"inputs": [],
+				"name": "decimals",
+				"outputs": [
+					{
+						"internalType": "uint8",
+						"name": "",
+						"type": "uint8"
+					}
+				],
+				"stateMutability": "view",
+				"type": "function"
+			},
+			{
+				"inputs": [],
+				"name": "name",
+				"outputs": [
+					{
+						"internalType": "string",
+						"name": "",
+						"type": "string"
+					}
+				],
+				"stateMutability": "view",
+				"type": "function"
+			},
+			{
+				"inputs": [],
+				"name": "symbol",
+				"outputs": [
+					{
+						"internalType": "string",
+						"name": "",
+						"type": "string"
+					}
+				],
+				"stateMutability": "view",
+				"type": "function"
+			},
+			{
+				"inputs": [],
+				"name": "totalSupply",
+				"outputs": [
+					{
+						"internalType": "uint256",
+						"name": "",
+						"type": "uint256"
+					}
+				],
+				"stateMutability": "view",
+				"type": "function"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "to",
+						"type": "address"
+					},
+					{
+						"internalType": "uint256",
+						"name": "value",
+						"type": "uint256"
+					}
+				],
+				"name": "transfer",
+				"outputs": [
+					{
+						"internalType": "bool",
+						"name": "",
+						"type": "bool"
+					}
+				],
+				"stateMutability": "nonpayable",
+				"type": "function"
+			},
+			{
+				"inputs": [
+					{
+						"internalType": "address",
+						"name": "from",
+						"type": "address"
+					},
+					{
+						"internalType": "address",
+						"name": "to",
+						"type": "address"
+					},
+					{
+						"internalType": "uint256",
+						"name": "value",
+						"type": "uint256"
+					}
+				],
+				"name": "transferFrom",
+				"outputs": [
+					{
+						"internalType": "bool",
+						"name": "",
+						"type": "bool"
+					}
+				],
+				"stateMutability": "nonpayable",
+				"type": "function"
+			},
+		]

{prediction_market_agent_tooling-0.14.1 → prediction_market_agent_tooling-0.15.1}/prediction_market_agent_tooling/benchmark/agents.py

@@ -9,7 +9,13 @@ from prediction_market_agent_tooling.benchmark.utils import (
 
 
 class AbstractBenchmarkedAgent:
-    def __init__(self, agent_name: str, max_workers: t.Optional[int] = None):
+    def __init__(
+        self,
+        agent_name: str,
+        max_workers: t.Optional[int] = None,
+        model: str | None = None,
+    ):
+        self.model = model
         self.agent_name = agent_name
         self.max_workers = max_workers  # Limit the number of workers that can run this worker in parallel threads
 

{prediction_market_agent_tooling-0.14.1 → prediction_market_agent_tooling-0.15.1}/prediction_market_agent_tooling/benchmark/benchmark.py

@@ -1,12 +1,10 @@
 import concurrent.futures
 import os
-import time
 import typing as t
 from collections import defaultdict
 
 import numpy as np
 import pandas as pd
-from langchain_community.callbacks import get_openai_callback
 from sklearn.metrics import precision_score, recall_score
 from tqdm import tqdm
 
@@ -15,9 +13,9 @@ from prediction_market_agent_tooling.benchmark.utils import (
     Prediction,
     PredictionsCache,
     Resolution,
-    get_llm_api_call_cost,
 )
 from prediction_market_agent_tooling.markets.agent_market import AgentMarket
+from prediction_market_agent_tooling.tools.costs import openai_costs
 from prediction_market_agent_tooling.tools.utils import (
     check_not_none,
     should_not_happen,
@@ -129,6 +127,7 @@ class Benchmarker:
         return self.predictions.get_prediction(agent_name=agent_name, question=question)
 
     def run_agents(self, enable_timing: bool = True) -> None:
+        agent: AbstractBenchmarkedAgent  # Fix for mypy issue with tqdm.
         for agent in tqdm(self.registered_agents, desc="Running agents"):
             # Filter out cached predictions
             markets_to_run = [
@@ -142,27 +141,21 @@ class Benchmarker:
             def get_prediction_result(
                 market: AgentMarket,
             ) -> tuple[str, Prediction]:
-                with get_openai_callback() as cb:
-                    start = time.time()
+                with openai_costs(model=agent.model) as costs:
                     prediction = (
                         agent.check_and_predict(market_question=market.question)
                         if not market.is_resolved()
-                        else agent.check_and_predict_restricted(
-                            market_question=market.question,
-                            time_restriction_up_to=market.created_time,  # TODO: Add support for resolved_at and any time in between.
+                        else (
+                            agent.check_and_predict_restricted(
+                                market_question=market.question,
+                                time_restriction_up_to=market.created_time,  # TODO: Add support for resolved_at and any time in between.
+                            )
+                            if market.created_time is not None
+                            else should_not_happen()
                         )
                     )
-
-                    prediction.time = time.time() - start if enable_timing else None
-
-                    if cb.total_tokens > 0 and cb.total_cost == 0:
-                        # TODO: this is a hack to get the cost for an unsupported model
-                        cb.total_cost = get_llm_api_call_cost(
-                            model=agent.model,
-                            prompt_tokens=cb.prompt_tokens,
-                            completion_tokens=cb.completion_tokens,
-                        )
-                    prediction.cost = cb.total_cost
+                    prediction.time = costs.time
+                    prediction.cost = costs.cost
                 return market.question, prediction
 
             # Run agents in parallel
@@ -208,7 +201,7 @@ class Benchmarker:
             return None
         mse = sum(
             [
-                (check_not_none(p.outcome_prediction).p_yes - m.p_yes) ** 2
+                (check_not_none(p.outcome_prediction).p_yes - m.current_p_yes) ** 2
                 for p, m in zip(predictions, markets)
             ]
         ) / len(predictions)
@@ -262,7 +255,10 @@ class Benchmarker:
 
         within_range_count = 0
         for p, m in zip(predictions, markets):
-            if abs(check_not_none(p.outcome_prediction).p_yes - m.p_yes) <= tolerance:
+            if (
+                abs(check_not_none(p.outcome_prediction).p_yes - m.current_p_yes)
+                <= tolerance
+            ):
                 within_range_count += 1
 
         return (100 * within_range_count) / len(predictions)
@@ -330,7 +326,7 @@ class Benchmarker:
             return None
 
         p_yes_errors = [
-            abs(check_not_none(p.outcome_prediction).p_yes - m.p_yes)
+            abs(check_not_none(p.outcome_prediction).p_yes - m.current_p_yes)
             for p, m in zip(predictions, markets)
         ]
         confidences = [
@@ -420,7 +416,7 @@ class Benchmarker:
                 for p in agent_predictions
             ]
         markets_summary[f"reference p_yes"] = [
-            f"{m.p_yes:.2f} [{m.probable_resolution}]" for m in self.markets
+            f"{m.current_p_yes:.2f} [{m.probable_resolution}]" for m in self.markets
         ]
         return markets_summary
 
@@ -475,7 +471,9 @@ class Benchmarker:
             return None
 
         expected_value = (
-            yes_shares * market.p_yes + no_shares * (1 - market.p_yes) - bet_units
+            yes_shares * market.current_p_yes
+            + no_shares * (1 - market.current_p_yes)
+            - bet_units
         )
         expected_returns_perc = 100 * expected_value / bet_units
 

{prediction_market_agent_tooling-0.14.1 → prediction_market_agent_tooling-0.15.1}/prediction_market_agent_tooling/benchmark/utils.py

@@ -80,6 +80,10 @@ def get_llm_api_call_cost(
             "prompt_tokens": 0.01,
             "completion_tokens": 0.03,
         },
+        "gpt-4-turbo-2024-04-09": {
+            "prompt_tokens": 0.01,
+            "completion_tokens": 0.03,
+        },
         "gpt-3.5-turbo-0125": {
             "prompt_tokens": 0.0005,
             "completion_tokens": 0.0015,

{prediction_market_agent_tooling-0.14.1 → prediction_market_agent_tooling-0.15.1}/prediction_market_agent_tooling/config.py

@@ -1,5 +1,6 @@
 import typing as t
 
+from pydantic import BaseModel
 from pydantic.types import SecretStr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -23,6 +24,7 @@ class APIKeys(BaseSettings):
 
     MANIFOLD_API_KEY: t.Optional[SecretStr] = None
     BET_FROM_PRIVATE_KEY: t.Optional[PrivateKey] = None
+    SAFE_ADDRESS: t.Optional[ChecksumAddress] = None
     OPENAI_API_KEY: t.Optional[SecretStr] = None
 
     GOOGLE_SEARCH_API_KEY: t.Optional[SecretStr] = None
@@ -43,10 +45,6 @@ class APIKeys(BaseSettings):
             self.MANIFOLD_API_KEY, "MANIFOLD_API_KEY missing in the environment."
         )
 
-    @property
-    def bet_from_address(self) -> ChecksumAddress:
-        return private_key_to_public_key(self.bet_from_private_key)
-
     @property
     def bet_from_private_key(self) -> PrivateKey:
         return check_not_none(
@@ -87,3 +85,28 @@ class APIKeys(BaseSettings):
             for k, v in self.model_dump().items()
             if APIKeys.model_fields[k].annotation in SECRET_TYPES and v is not None
         }
+
+
+class PrivateCredentials(BaseModel):
+    private_key: PrivateKey
+    safe_address: ChecksumAddress | None
+
+    @property
+    def public_key(self) -> ChecksumAddress:
+        """If the SAFE is available, we always route transactions via SAFE. Otherwise we use the EOA."""
+        return (
+            self.safe_address
+            if self.safe_address is not None
+            else private_key_to_public_key(self.private_key)
+        )
+
+    @property
+    def has_safe_address(self) -> bool:
+        return self.safe_address is not None
+
+    @staticmethod
+    def from_api_keys(api_keys: APIKeys) -> "PrivateCredentials":
+        return PrivateCredentials(
+            private_key=api_keys.bet_from_private_key,
+            safe_address=api_keys.SAFE_ADDRESS,
+        )

{prediction_market_agent_tooling-0.14.1 → prediction_market_agent_tooling-0.15.1}/prediction_market_agent_tooling/deploy/agent.py

@@ -7,7 +7,7 @@ from datetime import datetime
 
 from loguru import logger
 
-from prediction_market_agent_tooling.config import APIKeys
+from prediction_market_agent_tooling.config import APIKeys, PrivateCredentials
 from prediction_market_agent_tooling.deploy.constants import (
     MARKET_TYPE_KEY,
     REPOSITORY_KEY,
@@ -181,11 +181,11 @@ def {entrypoint_function_name}(request) -> str:
         """
         Executes actions that occur before bets are placed.
         """
-        keys = APIKeys()
+        private_credentials = PrivateCredentials.from_api_keys(APIKeys())
 
         if market_type == MarketType.OMEN:
             # Omen is specific, because the user (agent) needs to manually withdraw winnings from the market.
-            redeem_from_all_user_positions(keys.bet_from_private_key)
+            redeem_from_all_user_positions(private_credentials)
 
     def process_bets(self, market_type: MarketType, _place_bet: bool = True) -> None:
         """

{prediction_market_agent_tooling-0.14.1 → prediction_market_agent_tooling-0.15.1}/prediction_market_agent_tooling/markets/agent_market.py

@@ -46,7 +46,7 @@ class AgentMarket(BaseModel):
     resolution: Resolution | None
     created_time: datetime | None
     close_time: datetime | None
-    p_yes: Probability
+    current_p_yes: Probability
     url: str
     volume: float | None  # Should be in currency of `currency` above.
 
@@ -58,8 +58,8 @@ class AgentMarket(BaseModel):
     )
 
     @property
-    def p_no(self) -> Probability:
-        return Probability(1 - self.p_yes)
+    def current_p_no(self) -> Probability:
+        return Probability(1 - self.current_p_yes)
 
     @property
     def yes_outcome_price(self) -> float:
@@ -67,7 +67,7 @@ class AgentMarket(BaseModel):
         Price at prediction market is equal to the probability of given outcome.
         Keep as an extra property, in case it wouldn't be true for some prediction market platform.
         """
-        return self.p_yes
+        return self.current_p_yes
 
     @property
     def no_outcome_price(self) -> float:
@@ -75,7 +75,7 @@ class AgentMarket(BaseModel):
         Price at prediction market is equal to the probability of given outcome.
         Keep as an extra property, in case it wouldn't be true for some prediction market platform.
         """
-        return self.p_no
+        return self.current_p_no
 
     @property
     def probable_resolution(self) -> Resolution:
@@ -85,7 +85,7 @@ class AgentMarket(BaseModel):
             else:
                 raise ValueError(f"Unknown resolution: {self.resolution}")
         else:
-            return Resolution.YES if self.p_yes > 0.5 else Resolution.NO
+            return Resolution.YES if self.current_p_yes > 0.5 else Resolution.NO
 
     @property
     def boolean_outcome(self) -> bool:
@@ -96,6 +96,20 @@ class AgentMarket(BaseModel):
                 return False
         should_not_happen(f"Market {self.id} does not have a successful resolution.")
 
+    def get_last_trade_p_yes(self) -> Probability | None:
+        """
+        Get the last trade price for the YES outcome. This can be different from the current p_yes, for example if market is closed and it's probabilities are fixed to 0 and 1.
+        Could be None if no trades were made.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def get_last_trade_p_no(self) -> Probability | None:
+        """
+        Get the last trade price for the NO outcome. This can be different from the current p_yes, for example if market is closed and it's probabilities are fixed to 0 and 1.
+        Could be None if no trades were made.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
     def get_bet_amount(self, amount: float) -> BetAmount:
         return BetAmount(amount=amount, currency=self.currency)
 
@@ -148,9 +162,6 @@ class AgentMarket(BaseModel):
         except ValueError:
             raise ValueError(f"Outcome `{outcome}` not found in `{self.outcomes}`.")
 
-    def get_squared_error(self) -> float:
-        return (self.p_yes - self.boolean_outcome) ** 2
-
     def get_token_balance(self, user_id: str, outcome: str) -> TokenAmount:
         raise NotImplementedError("Subclasses must implement this method")