prediction-market-agent-tooling 0.64.10__tar.gz → 0.64.11__tar.gz

{prediction_market_agent_tooling-0.64.10 → prediction_market_agent_tooling-0.64.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: prediction-market-agent-tooling
-Version: 0.64.10
+Version: 0.64.11
 Summary: Tools to benchmark, deploy and monitor prediction market agents.
 Author: Gnosis
 Requires-Python: >=3.10,<3.13

prediction_market_agent_tooling-0.64.11/prediction_market_agent_tooling/logprobs_parser.py

@@ -0,0 +1,155 @@
+import math
+from itertools import product
+from typing import Any, Tuple, Type
+
+from pydantic import BaseModel, ValidationError
+from pydantic.fields import FieldInfo
+from pydantic.type_adapter import TypeAdapter
+
+from prediction_market_agent_tooling.loggers import logger
+
+
+class LogprobDetail(BaseModel):
+    token: str
+    logprob: float
+    prob: float
+
+
+class FieldLogprobs(BaseModel):
+    key: str
+    logprobs: list[LogprobDetail]
+
+
+class LogprobsParser:
+    def __init__(
+        self,
+        skip_fields: list[str] | None = None,
+        max_top_logprobs_length: int = 3,
+        max_logprobs_length: int = 5,
+    ):
+        base_skip_fields = ["logprobs"]
+        self.skip_fields = base_skip_fields + (skip_fields or [])
+        self.max_top_logprobs_length = max_top_logprobs_length
+        self.max_logprobs_length = max_logprobs_length
+
+    def _get_logprobs_key_index(
+        self, logprobs: list[dict[str, Any]], field_name: str
+    ) -> int:
+        key_candidate = ""
+        for i, token in enumerate(logprobs):
+            if token["token"] in field_name:
+                key_candidate = key_candidate + token["token"]
+            else:
+                key_candidate = ""
+            if key_candidate == field_name:
+                return i
+
+        return -1
+
+    def _get_logprobs_indexes_for_result(
+        self, logprobs: list[dict[str, Any]], key_index: int
+    ) -> Tuple[int, int]:
+        result_start_index = next(
+            (
+                i
+                for i in range(key_index + 1, len(logprobs))
+                if logprobs[i]["token"] in {":", ",", " ", ' "', '"', "\t", "\u00A0"}
+            ),
+            -1,
+        )
+        result_end_index = next(
+            (
+                i
+                for i in range(result_start_index, len(logprobs))
+                if logprobs[i]["token"] in {",", '"', ",\n", "\",\n'", '",\n'}
+            ),
+            len(logprobs) - 1,
+        )
+        return result_start_index + 1, result_end_index
+
+    def _is_correct_type(self, token: str, key_type: type | None) -> bool:
+        if key_type is None:
+            return True
+
+        try:
+            TypeAdapter(key_type).validate_python(token)
+            return True
+        except ValidationError:
+            return False
+
+    def _parse_valid_tokens_with__agg_probs(
+        self,
+        logprobs_list: list[tuple[dict[str, Any]]],
+        field_info: FieldInfo,
+        top_logprobs: int,
+    ) -> list[dict[str, Any]]:
+        results: list[dict[str, Any]] = [
+            {
+                "token": "".join(str(logprob["token"]) for logprob in logprobs),
+                "logprob": sum(float(logprob["logprob"]) for logprob in logprobs),
+                "prob": math.exp(
+                    sum(float(logprob["logprob"]) for logprob in logprobs)
+                ),
+            }
+            for logprobs in logprobs_list
+        ]
+
+        results_filtered: list[dict[str, Any]] = [
+            result
+            for result in results
+            if self._is_correct_type(result["token"], field_info.annotation)
+        ]
+
+        sorted_results = sorted(
+            results_filtered, key=lambda x: x["logprob"], reverse=True
+        )
+        return (
+            sorted_results[:top_logprobs]
+            if len(sorted_results) > top_logprobs
+            else sorted_results
+        )
+
+    def parse_logprobs(
+        self, logprobs: list[dict[str, Any]], target_model_cls: Type[BaseModel]
+    ) -> list[FieldLogprobs]:
+        results_for_keys = []
+
+        for field_name, field_info in target_model_cls.model_fields.items():
+            if field_name in self.skip_fields:
+                continue
+
+            key_index = self._get_logprobs_key_index(logprobs, field_name)
+
+            if key_index < 0:
+                logger.warning(f"Key {field_name} not found in logprobs")
+                continue
+
+            (
+                result_start_index,
+                result_end_index,
+            ) = self._get_logprobs_indexes_for_result(logprobs, key_index)
+
+            if result_start_index < 0 or result_end_index < 0:
+                logger.warning(f"Error in parsing result for {field_name} in logprobs")
+                continue
+
+            valid_logprobs_raw = [
+                logprobs[i]["top_logprobs"][: self.max_top_logprobs_length]
+                for i in range(result_start_index, result_end_index)
+                if logprobs[i]["top_logprobs"] is not None
+            ]
+
+            parsed_logprobs_data = self._parse_valid_tokens_with__agg_probs(
+                list(product(*valid_logprobs_raw[: self.max_logprobs_length])),
+                field_info,
+                min(len(sublist) for sublist in valid_logprobs_raw),
+            )
+
+            results_for_keys.append(
+                FieldLogprobs(
+                    key=field_name,
+                    logprobs=[LogprobDetail(**item) for item in parsed_logprobs_data],
+                )
+            )
+
+        return results_for_keys

{prediction_market_agent_tooling-0.64.10 → prediction_market_agent_tooling-0.64.11}/prediction_market_agent_tooling/markets/data_models.py

@@ -10,6 +10,7 @@ from prediction_market_agent_tooling.gtypes import (
     OutcomeToken,
     Probability,
 )
+from prediction_market_agent_tooling.logprobs_parser import FieldLogprobs
 from prediction_market_agent_tooling.tools.utils import DatetimeUTC
 
 
@@ -77,6 +78,7 @@ class ProbabilisticAnswer(BaseModel):
     p_yes: Probability
     confidence: float
     reasoning: str | None = None
+    logprobs: list[FieldLogprobs] | None = None
 
     @property
     def p_no(self) -> Probability:

{prediction_market_agent_tooling-0.64.10 → prediction_market_agent_tooling-0.64.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "prediction-market-agent-tooling"
-version = "0.64.10"
+version = "0.64.11"
 description = "Tools to benchmark, deploy and monitor prediction market agents."
 authors = ["Gnosis"]
 readme = "README.md"

prediction_market_agent_tooling-0.64.10/prediction_market_agent_tooling/logprobs_parser.py

@@ -1,117 +0,0 @@
-import math
-from itertools import product
-from typing import Any, Tuple
-
-from pydantic import BaseModel
-
-from prediction_market_agent_tooling.loggers import logger
-
-
-class LogprobKey(BaseModel):
-    name: str
-    key_type: type
-    valid_values: set[Any] | None
-
-
-class LogprobsParser:
-    def _get_logprobs_key_index(
-        self, logprobs: list[dict[str, Any]], key: LogprobKey
-    ) -> int:
-        key_candidate = ""
-        for i, token in enumerate(logprobs):
-            if token["token"] in key.name:
-                key_candidate = key_candidate + token["token"]
-            else:
-                key_candidate = ""
-            if key_candidate == key.name:
-                return i
-
-        return -1
-
-    def _get_logprobs_indexes_for_result(
-        self, logprobs: list[dict[str, Any]], key_index: int
-    ) -> Tuple[int, int]:
-        result_start_index = next(
-            (
-                i
-                for i in range(key_index + 1, len(logprobs))
-                if logprobs[i]["token"] in {":", ",", " ", ' "', '"', "\t", "\u00A0"}
-            ),
-            -1,
-        )
-        result_end_index = next(
-            (
-                i
-                for i in range(result_start_index, len(logprobs))
-                if logprobs[i]["token"] in {",", '"', ",\n", "\",\n'", '",\n'}
-            ),
-            -1,
-        )
-        return result_start_index + 1, result_end_index
-
-    def _is_correct_type(self, token: str, key_type: type) -> bool:
-        try:
-            key_type(token)
-            return True
-        except ValueError:
-            return False
-
-    def _parse_valid_tokens_with__agg_probs(
-        self, logprobs_list: list[tuple[dict[str, Any]]], key: LogprobKey
-    ) -> list[dict[str, Any]]:
-        results: list[dict[str, Any]] = [
-            {
-                "token": "".join(str(logprob["token"]) for logprob in logprobs),
-                "logprob": sum(float(logprob["logprob"]) for logprob in logprobs),
-                "prob": math.exp(
-                    sum(float(logprob["logprob"]) for logprob in logprobs)
-                ),
-            }
-            for logprobs in logprobs_list
-        ]
-
-        results_filtered: list[dict[str, Any]] = [
-            result
-            for result in results
-            if self._is_correct_type(result["token"], key.key_type)
-            and (key.valid_values is None or result["token"] in key.valid_values)
-        ]
-
-        return sorted(results_filtered, key=lambda x: x["logprob"], reverse=True)[
-            : len(logprobs_list[0])
-        ]
-
-    def parse_logprobs(
-        self, logprobs: list[dict[str, Any]], keys: list[LogprobKey]
-    ) -> list[dict[str, Any]]:
-        results_for_keys = []
-
-        for key in keys:
-            key_index = self._get_logprobs_key_index(logprobs, key)
-            if key_index < 0:
-                logger.warning(f"Key {key.name} not found in logprobs")
-                continue
-
-            (
-                result_start_index,
-                result_end_index,
-            ) = self._get_logprobs_indexes_for_result(logprobs, key_index)
-            if result_start_index < 0 or result_end_index < 0:
-                logger.warning(f"Error in parsing result for {key.name} in logprobs")
-                continue
-
-            valid_logprobs = [
-                logprobs[i]["top_logprobs"]
-                for i in range(result_start_index, result_end_index)
-            ]
-
-            results_for_keys.append(
-                {
-                    "key": key.name,
-                    "logprobs": self._parse_valid_tokens_with__agg_probs(
-                        list(product(*valid_logprobs)), key
-                    ),
-                }
-            )
-
-        return results_for_keys