prediction-market-agent-tooling 0.66.5__py3-none-any.whl → 0.67.1__py3-none-any.whl

prediction_market_agent_tooling/deploy/agent.py

@@ -19,9 +19,11 @@ from prediction_market_agent_tooling.deploy.trade_interval import (
 )
 from prediction_market_agent_tooling.gtypes import USD, OutcomeToken, xDai
 from prediction_market_agent_tooling.loggers import logger
+from prediction_market_agent_tooling.markets.agent_market import AgentMarket, FilterBy
+from prediction_market_agent_tooling.markets.agent_market import (
+    MarketType as AgentMarketType,
+)
 from prediction_market_agent_tooling.markets.agent_market import (
-    AgentMarket,
-    FilterBy,
     ProcessedMarket,
     ProcessedTradedMarket,
     SortBy,
@@ -335,6 +337,15 @@ class DeployablePredictionAgent(DeployableAgent):
             return True
         return False
 
+    @property
+    def agent_market_type(self) -> AgentMarketType:
+        if self.fetch_scalar_markets:
+            return AgentMarketType.SCALAR
+        elif self.fetch_categorical_markets:
+            return AgentMarketType.CATEGORICAL
+        else:
+            return AgentMarketType.BINARY
+
     def get_markets(
         self,
         market_type: MarketType,
@@ -343,14 +354,16 @@ class DeployablePredictionAgent(DeployableAgent):
         Override this method to customize what markets will fetch for processing.
         """
         cls = market_type.market_class
+
+        agent_market_type = self.agent_market_type
+
         # Fetch the soonest closing markets to choose from
         available_markets = cls.get_markets(
             limit=self.n_markets_to_fetch,
             sort_by=self.get_markets_sort_by,
             filter_by=self.get_markets_filter_by,
             created_after=self.trade_on_markets_created_after,
-            fetch_categorical_markets=self.fetch_categorical_markets,
-            fetch_scalar_markets=self.fetch_scalar_markets,
+            market_type=agent_market_type,
         )
         return available_markets
 
@@ -628,7 +641,12 @@ class DeployableTraderAgent(DeployablePredictionAgent):
         api_keys = APIKeys()
         user_id = market.get_user_id(api_keys=api_keys)
 
-        existing_position = market.get_position(user_id=user_id)
+        try:
+            existing_position = market.get_position(user_id=user_id)
+        except Exception as e:
+            logger.warning(f"Could not get position for user {user_id}, exception {e}")
+            return None
+
         trades = self.build_trades(
             market=market,
             answer=processed_market.answer,

prediction_market_agent_tooling/markets/agent_market.py

@@ -64,6 +64,13 @@ class FilterBy(str, Enum):
     NONE = "none"
 
 
+class MarketType(str, Enum):
+    ALL = "all"
+    CATEGORICAL = "categorical"
+    SCALAR = "scalar"
+    BINARY = "binary"
+
+
 class AgentMarket(BaseModel):
     """
     Common market class that can be created from vendor specific markets.
@@ -369,8 +376,8 @@ class AgentMarket(BaseModel):
         filter_by: FilterBy = FilterBy.OPEN,
         created_after: t.Optional[DatetimeUTC] = None,
         excluded_questions: set[str] | None = None,
-        fetch_categorical_markets: bool = False,
-        fetch_scalar_markets: bool = False,
+        market_type: MarketType = MarketType.ALL,
+        include_conditional_markets: bool = False,
     ) -> t.Sequence["AgentMarket"]:
         raise NotImplementedError("Subclasses must implement this method")
 

prediction_market_agent_tooling/markets/manifold/manifold.py

@@ -12,6 +12,7 @@ from prediction_market_agent_tooling.markets.agent_market import (
     AgentMarket,
     FilterBy,
     MarketFees,
+    MarketType,
     SortBy,
 )
 from prediction_market_agent_tooling.markets.manifold.api import (
@@ -110,8 +111,8 @@ class ManifoldAgentMarket(AgentMarket):
         filter_by: FilterBy = FilterBy.OPEN,
         created_after: t.Optional[DatetimeUTC] = None,
         excluded_questions: set[str] | None = None,
-        fetch_categorical_markets: bool = False,
-        fetch_scalar_markets: bool = False,
+        market_type: MarketType = MarketType.ALL,
+        include_conditional_markets: bool = False,
     ) -> t.Sequence["ManifoldAgentMarket"]:
         sort: t.Literal["newest", "close-date"] | None
         if sort_by == SortBy.CLOSING_SOONEST:

prediction_market_agent_tooling/markets/markets.py

@@ -3,11 +3,11 @@ from enum import Enum
 
 from prediction_market_agent_tooling.jobs.jobs_models import JobAgentMarket
 from prediction_market_agent_tooling.jobs.omen.omen_jobs import OmenJobAgentMarket
+from prediction_market_agent_tooling.markets.agent_market import AgentMarket, FilterBy
 from prediction_market_agent_tooling.markets.agent_market import (
-    AgentMarket,
-    FilterBy,
-    SortBy,
+    MarketType as AgentMarketType,
 )
+from prediction_market_agent_tooling.markets.agent_market import SortBy
 from prediction_market_agent_tooling.markets.manifold.manifold import (
     ManifoldAgentMarket,
 )
@@ -68,7 +68,7 @@ def get_binary_markets(
     sort_by: SortBy = SortBy.NONE,
     excluded_questions: set[str] | None = None,
     created_after: DatetimeUTC | None = None,
-    fetch_scalar_markets: bool = False,
+    agent_market_type: AgentMarketType = AgentMarketType.BINARY,
 ) -> t.Sequence[AgentMarket]:
     agent_market_class = MARKET_TYPE_TO_AGENT_MARKET[market_type]
     markets = agent_market_class.get_markets(
@@ -77,6 +77,6 @@ def get_binary_markets(
         filter_by=filter_by,
         created_after=created_after,
         excluded_questions=excluded_questions,
-        fetch_scalar_markets=fetch_scalar_markets,
+        market_type=agent_market_type,
     )
     return markets

prediction_market_agent_tooling/markets/metaculus/metaculus.py

@@ -9,6 +9,7 @@ from prediction_market_agent_tooling.markets.agent_market import (
     AgentMarket,
     FilterBy,
     MarketFees,
+    MarketType,
     ProcessedMarket,
     SortBy,
 )
@@ -72,8 +73,9 @@ class MetaculusAgentMarket(AgentMarket):
         filter_by: FilterBy = FilterBy.OPEN,
         created_after: t.Optional[DatetimeUTC] = None,
         excluded_questions: set[str] | None = None,
+        market_type: MarketType = MarketType.ALL,
+        include_conditional_markets: bool = False,
         tournament_id: int | None = None,
-        fetch_scalar_markets: bool = False,
     ) -> t.Sequence["MetaculusAgentMarket"]:
         order_by: str | None
         if sort_by == SortBy.NONE:

prediction_market_agent_tooling/markets/omen/omen.py

@@ -25,6 +25,7 @@ from prediction_market_agent_tooling.markets.agent_market import (
     AgentMarket,
     FilterBy,
     MarketFees,
+    MarketType,
     ProcessedMarket,
     ProcessedTradedMarket,
     SortBy,
@@ -379,9 +380,11 @@ class OmenAgentMarket(AgentMarket):
         filter_by: FilterBy = FilterBy.OPEN,
         created_after: t.Optional[DatetimeUTC] = None,
         excluded_questions: set[str] | None = None,
-        fetch_categorical_markets: bool = False,
-        fetch_scalar_markets: bool = False,
+        market_type: MarketType = MarketType.ALL,
+        include_conditional_markets: bool = False,
     ) -> t.Sequence["OmenAgentMarket"]:
+        fetch_categorical_markets = market_type == MarketType.CATEGORICAL
+
         return [
             OmenAgentMarket.from_data_model(m)
             for m in OmenSubgraphHandler().get_omen_markets_simple(

prediction_market_agent_tooling/markets/polymarket/api.py

@@ -1,140 +1,123 @@
 import typing as t
+from enum import Enum
+from urllib.parse import urljoin
 
-import requests
+import httpx
 import tenacity
 
 from prediction_market_agent_tooling.loggers import logger
 from prediction_market_agent_tooling.markets.polymarket.data_models import (
     POLYMARKET_FALSE_OUTCOME,
     POLYMARKET_TRUE_OUTCOME,
-    MarketsEndpointResponse,
-    PolymarketMarket,
-    PolymarketMarketWithPrices,
-    PolymarketPriceResponse,
-    PolymarketTokenWithPrices,
-    Prices,
+    PolymarketGammaResponse,
+    PolymarketGammaResponseDataItem,
 )
+from prediction_market_agent_tooling.tools.datetime_utc import DatetimeUTC
+from prediction_market_agent_tooling.tools.httpx_cached_client import HttpxCachedClient
 from prediction_market_agent_tooling.tools.utils import response_to_model
 
-POLYMARKET_API_BASE_URL = "https://clob.polymarket.com/"
 MARKETS_LIMIT = 100  # Polymarket will only return up to 100 markets
+POLYMARKET_GAMMA_API_BASE_URL = "https://gamma-api.polymarket.com/"
+
+
+class PolymarketOrderByEnum(str, Enum):
+    LIQUIDITY = "liquidity"
+    START_DATE = "startDate"
+    END_DATE = "endDate"
+    VOLUME_24HR = "volume24hr"
 
 
 @tenacity.retry(
-    stop=tenacity.stop_after_attempt(5),
-    wait=tenacity.wait_chain(*[tenacity.wait_fixed(n) for n in range(1, 6)]),
-    after=lambda x: logger.debug(f"get_polymarkets failed, {x.attempt_number=}."),
+    stop=tenacity.stop_after_attempt(2),
+    wait=tenacity.wait_fixed(1),
+    after=lambda x: logger.debug(
+        f"get_polymarkets_with_pagination failed, {x.attempt_number=}."
+    ),
 )
-def get_polymarkets(
-    limit: int,
-    with_rewards: bool = False,
-    next_cursor: str | None = None,
-) -> MarketsEndpointResponse:
-    url = (
-        f"{POLYMARKET_API_BASE_URL}/{'sampling-markets' if with_rewards else 'markets'}"
-    )
-    params: dict[str, str | int | float | None] = {
-        "limit": min(limit, MARKETS_LIMIT),
-    }
-    if next_cursor is not None:
-        params["next_cursor"] = next_cursor
-    return response_to_model(requests.get(url, params=params), MarketsEndpointResponse)
-
-
-def get_polymarket_binary_markets(
+def get_polymarkets_with_pagination(
     limit: int,
-    closed: bool | None = False,
+    created_after: t.Optional[DatetimeUTC] = None,
+    active: bool | None = None,
+    closed: bool | None = None,
     excluded_questions: set[str] | None = None,
-    with_rewards: bool = False,
-    main_markets_only: bool = True,
-) -> list[PolymarketMarketWithPrices]:
+    only_binary: bool = True,
+    archived: bool = False,
+    ascending: bool = False,
+    order_by: PolymarketOrderByEnum = PolymarketOrderByEnum.VOLUME_24HR,
+) -> list[PolymarketGammaResponseDataItem]:
     """
-    See https://learn.polymarket.com/trading-rewards for information about rewards.
+    Binary markets have len(model.markets) == 1.
+    Categorical markets have len(model.markets) > 1
     """
-
-    all_markets: list[PolymarketMarketWithPrices] = []
-    next_cursor: str | None = None
-
-    while True:
-        response = get_polymarkets(
-            limit, with_rewards=with_rewards, next_cursor=next_cursor
+    client: httpx.Client = HttpxCachedClient(ttl=60).get_client()
+    all_markets: list[PolymarketGammaResponseDataItem] = []
+    offset = 0
+    remaining = limit
+
+    while remaining > 0:
+        # Calculate how many items to request in this batch (up to MARKETS_LIMIT or remaining)
+        # By default we fetch many markets because not possible to filter by binary/categorical
+        batch_size = MARKETS_LIMIT
+
+        # Build query parameters, excluding None values
+        params = {
+            "limit": batch_size,
+            "active": str(active).lower() if active is not None else None,
+            "archived": str(archived).lower(),
+            "closed": str(closed).lower() if closed is not None else None,
+            "order": order_by.value,
+            "ascending": str(ascending).lower(),
+            "offset": offset,
+        }
+        params_not_none = {k: v for k, v in params.items() if v is not None}
+        url = urljoin(
+            POLYMARKET_GAMMA_API_BASE_URL,
+            f"events/pagination",
         )
 
-        for market in response.data:
-            # Closed markets means resolved markets.
-            if closed is not None and market.closed != closed:
-                continue
-
-            # Skip markets that are inactive.
-            # Documentation does not provide more details about this, but if API returns them, website gives "Oops...we didn't forecast this".
-            if not market.active:
-                continue
+        r = client.get(url, params=params_not_none)
+        r.raise_for_status()
 
-            # Skip also those that were archived.
-            # Again nothing about it in documentation and API doesn't seem to return them, but to be safe.
-            if market.archived:
-                continue
+        market_response = response_to_model(r, PolymarketGammaResponse)
 
-            if excluded_questions and market.question in excluded_questions:
+        markets_to_add = []
+        for m in market_response.data:
+            if excluded_questions and m.title in excluded_questions:
                 continue
 
-            # Atm we work with binary markets only.
-            if sorted(token.outcome for token in market.tokens) != [
-                POLYMARKET_FALSE_OUTCOME,
-                POLYMARKET_TRUE_OUTCOME,
-            ]:
-                continue
+            sorted_outcome_list = sorted(m.markets[0].outcomes_list)
+            if only_binary:
+                # We keep markets that are only Yes,No
+                if len(m.markets) > 1 or sorted_outcome_list != [
+                    POLYMARKET_FALSE_OUTCOME,
+                    POLYMARKET_TRUE_OUTCOME,
+                ]:
+                    continue
 
-            # This is pretty slow to do here, but our safest option at the moment. So keep it as the last filter.
-            # TODO: Add support for `description` for `AgentMarket` and if it isn't None, use it in addition to the question in all agents. Then this can be removed.
-            if main_markets_only and not market.fetch_if_its_a_main_market():
+            if created_after and created_after > m.startDate:
                 continue
 
-            tokens_with_price = get_market_tokens_with_prices(market)
-            market_with_prices = PolymarketMarketWithPrices.model_validate(
-                {**market.model_dump(), "tokens": tokens_with_price}
-            )
+            markets_to_add.append(m)
 
-            all_markets.append(market_with_prices)
+        if only_binary:
+            markets_to_add = [
+                market for market in market_response.data if len(market.markets) == 1
+            ]
 
-        if len(all_markets) >= limit:
-            break
+        # Add the markets from this batch to our results
+        all_markets.extend(markets_to_add)
 
-        next_cursor = response.next_cursor
+        # Update counters
+        offset += len(market_response.data)
+        remaining -= len(markets_to_add)
 
-        if next_cursor == "LTE=":
-            # 'LTE=' means the end.
+        # Stop if we've reached our limit or there are no more results
+        if (
+            remaining <= 0
+            or not market_response.pagination.hasMore
+            or len(market_response.data) == 0
+        ):
             break
 
+    # Return exactly the number of items requested (in case we got more due to batch size)
     return all_markets[:limit]
-
-
-def get_polymarket_market(condition_id: str) -> PolymarketMarket:
-    url = f"{POLYMARKET_API_BASE_URL}/markets/{condition_id}"
-    return response_to_model(requests.get(url), PolymarketMarket)
-
-
-def get_token_price(
-    token_id: str, side: t.Literal["buy", "sell"]
-) -> PolymarketPriceResponse:
-    url = f"{POLYMARKET_API_BASE_URL}/price"
-    params = {"token_id": token_id, "side": side}
-    return response_to_model(requests.get(url, params=params), PolymarketPriceResponse)
-
-
-def get_market_tokens_with_prices(
-    market: PolymarketMarket,
-) -> list[PolymarketTokenWithPrices]:
-    tokens_with_prices = [
-        PolymarketTokenWithPrices(
-            token_id=token.token_id,
-            outcome=token.outcome,
-            winner=token.winner,
-            prices=Prices(
-                BUY=get_token_price(token.token_id, "buy").price_dec,
-                SELL=get_token_price(token.token_id, "sell").price_dec,
-            ),
-        )
-        for token in market.tokens
-    ]
-    return tokens_with_prices

prediction_market_agent_tooling/markets/polymarket/data_models.py

@@ -1,3 +1,5 @@
+import json
+
 from pydantic import BaseModel
 
 from prediction_market_agent_tooling.gtypes import USDC, OutcomeStr, Probability
@@ -5,9 +7,9 @@ from prediction_market_agent_tooling.markets.data_models import Resolution
 from prediction_market_agent_tooling.markets.polymarket.data_models_web import (
     POLYMARKET_FALSE_OUTCOME,
     POLYMARKET_TRUE_OUTCOME,
-    PolymarketFullMarket,
     construct_polymarket_url,
 )
+from prediction_market_agent_tooling.tools.hexbytes_custom import HexBytes
 from prediction_market_agent_tooling.tools.utils import DatetimeUTC
 
 
@@ -26,6 +28,63 @@ class PolymarketToken(BaseModel):
     winner: bool
 
 
+class PolymarketGammaMarket(BaseModel):
+    conditionId: HexBytes
+    outcomes: str
+    outcomePrices: str | None = None
+    marketMakerAddress: str
+    createdAt: DatetimeUTC
+    updatedAt: DatetimeUTC | None = None
+    archived: bool
+    questionId: str | None = None
+    clobTokenIds: str | None = None  # int-encoded hex
+
+    @property
+    def outcomes_list(self) -> list[OutcomeStr]:
+        return [OutcomeStr(i) for i in json.loads(self.outcomes)]
+
+    @property
+    def outcome_prices(self) -> list[float] | None:
+        if not self.outcomePrices:
+            return None
+        return [float(i) for i in json.loads(self.outcomePrices)]
+
+
+class PolymarketGammaTag(BaseModel):
+    label: str
+    slug: str
+
+
+class PolymarketGammaResponseDataItem(BaseModel):
+    id: str
+    slug: str
+    volume: float | None = None
+    startDate: DatetimeUTC
+    endDate: DatetimeUTC | None = None
+    liquidity: float | None = None
+    liquidityClob: float | None = None
+    title: str
+    description: str
+    archived: bool
+    closed: bool
+    active: bool
+    markets: list[PolymarketGammaMarket]
+    tags: list[PolymarketGammaTag]
+
+    @property
+    def url(self) -> str:
+        return construct_polymarket_url(self.slug)
+
+
+class PolymarketGammaPagination(BaseModel):
+    hasMore: bool
+
+
+class PolymarketGammaResponse(BaseModel):
+    data: list[PolymarketGammaResponseDataItem]
+    pagination: PolymarketGammaPagination
+
+
 class PolymarketMarket(BaseModel):
     enable_order_book: bool
     active: bool
@@ -89,19 +148,6 @@ class PolymarketMarket(BaseModel):
                 f"Should not happen, invalid winner tokens: {winner_tokens}"
             )
 
-    def fetch_full_market(self) -> PolymarketFullMarket | None:
-        return PolymarketFullMarket.fetch_from_url(self.url)
-
-    def fetch_if_its_a_main_market(self) -> bool:
-        # On Polymarket, there are markets that are actually a group of multiple Yes/No markets, for example https://polymarket.com/event/presidential-election-winner-2024.
-        # But API returns them individually, and then we receive questions such as "Will any other Republican Politician win the 2024 US Presidential Election?",
-        # which are naturally unpredictable without futher details.
-        # This is a heuristic to filter them out.
-        # Warning: This is a very slow operation, as it requires fetching the website. Use it only when necessary.
-        full_market = self.fetch_full_market()
-        # `full_market` can be None, if this class come from a multiple Yes/No market, becase then, the constructed URL is invalid (and there is now way to construct an valid one from the data we have).
-        return full_market is not None and full_market.is_main_market
-
 
 class MarketsEndpointResponse(BaseModel):
     limit: int

prediction_market_agent_tooling/markets/polymarket/data_models_web.py

@@ -5,14 +5,11 @@ Keep in mind that not all fields were used so far, so there might be some bugs.
 These models are based on what Polymarket's website returns in the response, and `prediction_market_agent_tooling/markets/polymarket/data_models.py` are based on what their API returns.
 """
 
-import json
 import typing as t
 
-import requests
 from pydantic import BaseModel, field_validator
 
 from prediction_market_agent_tooling.gtypes import USDC, HexAddress, OutcomeStr
-from prediction_market_agent_tooling.loggers import logger
 from prediction_market_agent_tooling.markets.data_models import Resolution
 from prediction_market_agent_tooling.tools.utils import DatetimeUTC
 
@@ -293,57 +290,6 @@ class PolymarketFullMarket(BaseModel):
             )
         return self.markets[0]
 
-    @staticmethod
-    def fetch_from_url(url: str) -> "PolymarketFullMarket | None":
-        """
-        Get the full market data from the Polymarket website.
-
-        Returns None if this market's url returns "Oops...we didn't forecast this", see `check_if_its_a_main_market` method for more details.
-
-        Warning: This is a very slow operation, as it requires fetching the website. Use it only when necessary.
-        """
-        logger.info(f"Fetching full market from {url}")
-
-        # Fetch the website as a normal browser would.
-        headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
-        }
-        content = requests.get(url, headers=headers).text
-
-        # Find the JSON with the data within the content.
-        start_tag = """<script id="__NEXT_DATA__" type="application/json" crossorigin="anonymous">"""
-        start_idx = content.find(start_tag) + len(start_tag)
-        end_idx = content.find("</script>", start_idx)
-        response_data = content[start_idx:end_idx]
-
-        # Parsing.
-        response_dict = json.loads(response_data)
-        response_model = PolymarketWebResponse.model_validate(response_dict)
-
-        full_market_queries = [
-            q
-            for q in response_model.props.pageProps.dehydratedState.queries
-            if isinstance(q.state.data, PolymarketFullMarket)
-        ]
-
-        # We expect either 0 markets (if it doesn't exist) or 1 market.
-        if len(full_market_queries) not in (0, 1):
-            raise ValueError(
-                f"Unexpected number of queries in the response, please check it out and modify the code accordingly: `{response_dict}`"
-            )
-
-        # It will be `PolymarketFullMarket` thanks to the filter above.
-        market = (
-            t.cast(PolymarketFullMarket, full_market_queries[0].state.data)
-            if full_market_queries
-            else None
-        )
-
-        if market is None:
-            logger.warning(f"No polymarket found for {url}")
-
-        return market
-
 
 class PriceSide(BaseModel):
     price: USDC