edsl 0.1.55__py3-none-any.whl → 0.1.56__py3-none-any.whl

edsl/inference_services/services/google_service.py

@@ -7,11 +7,13 @@ from google.api_core.exceptions import InvalidArgument
 
 # from ...exceptions.general import MissingAPIKeyError
 from ..inference_service_abc import InferenceServiceABC
+
 # Use TYPE_CHECKING to avoid circular imports at runtime
 if TYPE_CHECKING:
     from ...language_models import LanguageModel
     from ....scenarios.file_store import FileStore as Files
-#from ...coop import Coop
+# from ...coop import Coop
+import asyncio
 
 safety_settings = [
     {
@@ -61,7 +63,7 @@ class GoogleService(InferenceServiceABC):
     @classmethod
     def create_model(
         cls, model_name: str = "gemini-pro", model_class_name=None
-    ) -> 'LanguageModel':
+    ) -> "LanguageModel":
         if model_class_name is None:
             model_class_name = cls.to_class_name(model_name)
 
@@ -138,6 +140,7 @@ class GoogleService(InferenceServiceABC):
                     gen_ai_file = google.generativeai.types.file_types.File(
                         file.external_locations["google"]
                     )
+
                     combined_prompt.append(gen_ai_file)
 
                 try:

edsl/interviews/request_token_estimator.py

@@ -124,6 +124,14 @@ class RequestTokenEstimator:
                             width, height = file.get_image_dimensions()
                             token_usage = estimate_tokens(model_name, width, height)
                             file_tokens += token_usage
+                        if file.is_video():
+                            model_name = self.interview.model.model
+                            duration = file.get_video_metadata()["simplified"][
+                                "duration_seconds"
+                            ]
+                            file_tokens += (
+                                duration * 295
+                            )  # (295 tokens per second for video + audio)
                         else:
                             file_tokens += file.size * 0.25
             else:

edsl/invigilators/invigilators.py

@@ -1,4 +1,5 @@
 """Module for creating Invigilators, which are objects to administer a question to an Agent."""
+
 from abc import ABC, abstractmethod
 import asyncio
 from typing import Coroutine, Dict, Any, Optional, TYPE_CHECKING
@@ -395,17 +396,21 @@ class InvigilatorAI(InvigilatorBase):
 
         if agent_response_dict.model_outputs.cache_used and False:
             data = {
-                "answer": agent_response_dict.edsl_dict.answer
-                if type(agent_response_dict.edsl_dict.answer) is str
-                or type(agent_response_dict.edsl_dict.answer) is dict
-                or type(agent_response_dict.edsl_dict.answer) is list
-                or type(agent_response_dict.edsl_dict.answer) is int
-                or type(agent_response_dict.edsl_dict.answer) is float
-                or type(agent_response_dict.edsl_dict.answer) is bool
-                else "",
-                "comment": agent_response_dict.edsl_dict.comment
-                if agent_response_dict.edsl_dict.comment
-                else "",
+                "answer": (
+                    agent_response_dict.edsl_dict.answer
+                    if type(agent_response_dict.edsl_dict.answer) is str
+                    or type(agent_response_dict.edsl_dict.answer) is dict
+                    or type(agent_response_dict.edsl_dict.answer) is list
+                    or type(agent_response_dict.edsl_dict.answer) is int
+                    or type(agent_response_dict.edsl_dict.answer) is float
+                    or type(agent_response_dict.edsl_dict.answer) is bool
+                    else ""
+                ),
+                "comment": (
+                    agent_response_dict.edsl_dict.comment
+                    if agent_response_dict.edsl_dict.comment
+                    else ""
+                ),
                 "generated_tokens": agent_response_dict.edsl_dict.generated_tokens,
                 "question_name": self.question.question_name,
                 "prompts": self.get_prompts(),
@@ -415,7 +420,11 @@ class InvigilatorAI(InvigilatorBase):
                 "cache_key": agent_response_dict.model_outputs.cache_key,
                 "validated": True,
                 "exception_occurred": exception_occurred,
-                "cost": agent_response_dict.model_outputs.cost,
+                "input_tokens": agent_response_dict.model_outputs.input_tokens,
+                "output_tokens": agent_response_dict.model_outputs.output_tokens,
+                "input_price_per_million_tokens": agent_response_dict.model_outputs.input_price_per_million_tokens,
+                "output_price_per_million_tokens": agent_response_dict.model_outputs.output_price_per_million_tokens,
+                "total_cost": agent_response_dict.model_outputs.total_cost,
             }
 
             result = EDSLResultObjectInput(**data)
@@ -480,7 +489,11 @@ class InvigilatorAI(InvigilatorBase):
                 "cache_key": agent_response_dict.model_outputs.cache_key,
                 "validated": validated,
                 "exception_occurred": exception_occurred,
-                "cost": agent_response_dict.model_outputs.cost,
+                "input_tokens": agent_response_dict.model_outputs.input_tokens,
+                "output_tokens": agent_response_dict.model_outputs.output_tokens,
+                "input_price_per_million_tokens": agent_response_dict.model_outputs.input_price_per_million_tokens,
+                "output_price_per_million_tokens": agent_response_dict.model_outputs.output_price_per_million_tokens,
+                "total_cost": agent_response_dict.model_outputs.total_cost,
             }
             result = EDSLResultObjectInput(**data)
             return result

edsl/jobs/jobs_pricing_estimation.py

@@ -1,7 +1,8 @@
 import logging
 import math
 
-from typing import List, TYPE_CHECKING, Union, Literal
+from typing import List, TYPE_CHECKING, Union, Literal, Dict
+from collections import namedtuple
 
 if TYPE_CHECKING:
     from .jobs import Jobs
@@ -20,8 +21,8 @@ logger = logging.getLogger(__name__)
 
 class PromptCostEstimator:
 
-    DEFAULT_INPUT_PRICE_PER_TOKEN = 0.000001
-    DEFAULT_OUTPUT_PRICE_PER_TOKEN = 0.000001
+    DEFAULT_INPUT_PRICE_PER_MILLION_TOKENS = 1.0
+    DEFAULT_OUTPUT_PRICE_PER_MILLION_TOKENS = 1.0
     CHARS_PER_TOKEN = 4
     OUTPUT_TOKENS_PER_INPUT_TOKEN = 0.75
     PIPING_MULTIPLIER = 2
@@ -48,81 +49,90 @@ class PromptCostEstimator:
             return PromptCostEstimator.PIPING_MULTIPLIER
         return 1
 
-    @property
-    def key(self):
-        return (self.inference_service, self.model)
-
-    @property
-    def relevant_prices(self):
-        try:
-            return self.price_lookup[self.key]
-        except KeyError:
-            return {}
-
-    def _get_highest_price_for_service(self, price_type: str) -> Union[float, None]:
-        """Returns the highest price per token for a given service and price type (input/output).
+    def _get_fallback_price(self, inference_service: str) -> Dict:
+        """
+        Get fallback prices for a service.
+        - First fallback: The highest input and output prices for that service from the price lookup.
+        - Second fallback: $1.00 per million tokens (for both input and output).
 
         Args:
-            price_type: Either "input" or "output"
+            inference_service (str): The inference service name
 
         Returns:
-            float | None: The highest price per token for the service, or None if not found
+            Dict: Price information
         """
-        prices_for_service = [
-            prices[price_type]["service_stated_token_price"]
-            / prices[price_type]["service_stated_token_qty"]
+        PriceEntry = namedtuple("PriceEntry", ["tokens_per_usd", "price_info"])
+
+        service_prices = [
+            prices
             for (service, _), prices in self.price_lookup.items()
-            if service == self.inference_service and price_type in prices
+            if service == inference_service
         ]
-        return max(prices_for_service) if prices_for_service else None
 
-    def input_price_per_token(
-        self,
-    ) -> tuple[float, Literal["price_lookup", "highest_price_for_service", "default"]]:
-        try:
-            return (
-                self.relevant_prices["input"]["service_stated_token_price"]
-                / self.relevant_prices["input"]["service_stated_token_qty"]
-            ), "price_lookup"
-        except KeyError:
-            highest_price = self._get_highest_price_for_service("input")
-            if highest_price is not None:
-                import warnings
-
-                warnings.warn(
-                    f"Price data not found for {self.key}. Using highest available input price for {self.inference_service}: ${highest_price:.6f} per token"
-                )
-                return highest_price, "highest_price_for_service"
-            import warnings
+        default_input_price_info = {
+            "one_usd_buys": 1_000_000,
+            "service_stated_token_qty": 1_000_000,
+            "service_stated_token_price": self.DEFAULT_INPUT_PRICE_PER_MILLION_TOKENS,
+        }
+        default_output_price_info = {
+            "one_usd_buys": 1_000_000,
+            "service_stated_token_qty": 1_000_000,
+            "service_stated_token_price": self.DEFAULT_OUTPUT_PRICE_PER_MILLION_TOKENS,
+        }
 
-            warnings.warn(
-                f"Price data not found for {self.inference_service}. Using default estimate for input token price: $1.00 / 1M tokens"
-            )
-            return self.DEFAULT_INPUT_PRICE_PER_TOKEN, "default"
+        # Find the most expensive price entries (lowest tokens per USD)
+        input_price_info = default_input_price_info
+        output_price_info = default_output_price_info
 
-    def output_price_per_token(
-        self,
-    ) -> tuple[float, Literal["price_lookup", "highest_price_for_service", "default"]]:
-        try:
-            return (
-                self.relevant_prices["output"]["service_stated_token_price"]
-                / self.relevant_prices["output"]["service_stated_token_qty"]
-            ), "price_lookup"
-        except KeyError:
-            highest_price = self._get_highest_price_for_service("output")
-            if highest_price is not None:
-                import warnings
-
-                warnings.warn(
-                    f"Price data not found for {self.key}. Using highest available output price for {self.inference_service}: ${highest_price:.6f} per token"
-                )
-                return highest_price, "highest_price_for_service"
-            import warnings
+        input_prices = [
+            PriceEntry(float(p["input"]["one_usd_buys"]), p["input"])
+            for p in service_prices
+            if "input" in p
+        ]
+        if input_prices:
+            input_price_info = min(
+                input_prices, key=lambda price: price.tokens_per_usd
+            ).price_info
+
+        output_prices = [
+            PriceEntry(float(p["output"]["one_usd_buys"]), p["output"])
+            for p in service_prices
+            if "output" in p
+        ]
+        if output_prices:
+            output_price_info = min(
+                output_prices, key=lambda price: price.tokens_per_usd
+            ).price_info
 
-            warnings.warn(
-                f"Price data not found for {self.inference_service}. Using default estimate for output token price: $1.00 / 1M tokens"
-            )
-            return self.DEFAULT_OUTPUT_PRICE_PER_TOKEN, "default"
+        return {
+            "input": input_price_info,
+            "output": output_price_info,
+        }
+
+    def get_price(self, inference_service: str, model: str) -> Dict:
+        """Get the price information for a specific service and model."""
+        key = (inference_service, model)
+        return self.price_lookup.get(key) or self._get_fallback_price(inference_service)
+
+    def get_price_per_million_tokens(
+        self,
+        relevant_prices: Dict,
+        token_type: Literal["input", "output"],
+    ) -> Dict:
+        """
+        Get the price per million tokens for a specific service, model, and token type.
+        """
+        service_price = relevant_prices[token_type]["service_stated_token_price"]
+        service_qty = relevant_prices[token_type]["service_stated_token_qty"]
+
+        if service_qty == 1_000_000:
+            price_per_million_tokens = service_price
+        elif service_qty == 1_000:
+            price_per_million_tokens = service_price * 1_000
+        else:
+            price_per_token = service_price / service_qty
+            price_per_million_tokens = round(price_per_token * 1_000_000, 10)
+        return price_per_million_tokens
 
     def __call__(self):
         user_prompt_chars = len(str(self.user_prompt)) * self.get_piping_multiplier(
@@ -135,20 +145,28 @@ class PromptCostEstimator:
         input_tokens = (user_prompt_chars + system_prompt_chars) // self.CHARS_PER_TOKEN
         output_tokens = math.ceil(self.OUTPUT_TOKENS_PER_INPUT_TOKEN * input_tokens)
 
-        input_price_per_token, input_price_source = self.input_price_per_token()
-        output_price_per_token, output_price_source = self.output_price_per_token()
+        relevant_prices = self.get_price(self.inference_service, self.model)
 
-        cost = (
-            input_tokens * input_price_per_token
-            + output_tokens * output_price_per_token
+        input_price_per_million_tokens = self.get_price_per_million_tokens(
+            relevant_prices, "input"
         )
+        output_price_per_million_tokens = self.get_price_per_million_tokens(
+            relevant_prices, "output"
+        )
+
+        input_price_per_token = input_price_per_million_tokens / 1_000_000
+        output_price_per_token = output_price_per_million_tokens / 1_000_000
+
+        input_cost = input_tokens * input_price_per_token
+        output_cost = output_tokens * output_price_per_token
+        cost = input_cost + output_cost
         return {
-            "input_price_source": input_price_source,
-            "input_price_per_token": input_price_per_token,
+            "input_price_per_million_tokens": input_price_per_million_tokens,
+            "output_price_per_million_tokens": output_price_per_million_tokens,
             "input_tokens": input_tokens,
-            "output_price_source": output_price_source,
             "output_tokens": output_tokens,
-            "output_price_per_token": output_price_per_token,
+            "input_cost_usd": input_cost,
+            "output_cost_usd": output_cost,
             "cost_usd": cost,
         }
 
@@ -328,6 +346,26 @@ class JobsPrompts:
             "model": model,
         }
 
+    def process_token_type(self, item: dict, token_type: str) -> tuple:
+        """
+        Helper function to process a single token type (input or output) for price estimation.
+        """
+        price = item[f"estimated_{token_type}_price_per_million_tokens"]
+        tokens = item[f"estimated_{token_type}_tokens"]
+        cost = item[f"estimated_{token_type}_cost_usd"]
+
+        return (
+            (item["inference_service"], item["model"], token_type, price),
+            {
+                "inference_service": item["inference_service"],
+                "model": item["model"],
+                "token_type": token_type,
+                "price_per_million_tokens": price,
+                "tokens": tokens,
+                "cost_usd": cost,
+            },
+        )
+
     def estimate_job_cost_from_external_prices(
         self, price_lookup: dict, iterations: int = 1
     ) -> dict:
@@ -341,9 +379,9 @@ class JobsPrompts:
         - 1 token = 4 characters.
         - For each prompt, output tokens = input tokens * 0.75, rounded up to the nearest integer.
         """
-        interviews = self.interviews
+        # Collect all prompt data
         data = []
-        for interview in interviews:
+        for interview in self.interviews:
             invigilators = [
                 FetchInvigilator(interview)(question)
                 for question in self.survey.questions
@@ -354,59 +392,62 @@ class JobsPrompts:
                     **prompt_details, price_lookup=price_lookup
                 )
                 price_estimates = {
+                    "estimated_input_price_per_million_tokens": prompt_cost[
+                        "input_price_per_million_tokens"
+                    ],
+                    "estimated_output_price_per_million_tokens": prompt_cost[
+                        "output_price_per_million_tokens"
+                    ],
                     "estimated_input_tokens": prompt_cost["input_tokens"],
                     "estimated_output_tokens": prompt_cost["output_tokens"],
+                    "estimated_input_cost_usd": prompt_cost["input_cost_usd"],
+                    "estimated_output_cost_usd": prompt_cost["output_cost_usd"],
                     "estimated_cost_usd": prompt_cost["cost_usd"],
                 }
-                data.append({**price_estimates, **prompt_details})
+                data.append(
+                    {
+                        **prompt_details,
+                        **price_estimates,
+                    }
+                )
 
-        model_groups = {}
+        # Group by service, model, token type, and price
+        detailed_groups = {}
         for item in data:
-            key = (item["inference_service"], item["model"])
-            if key not in model_groups:
-                model_groups[key] = {
-                    "inference_service": item["inference_service"],
-                    "model": item["model"],
-                    "estimated_cost_usd": 0,
-                    "estimated_input_tokens": 0,
-                    "estimated_output_tokens": 0,
-                }
-
-            # Accumulate values
-            model_groups[key]["estimated_cost_usd"] += item["estimated_cost_usd"]
-            model_groups[key]["estimated_input_tokens"] += item[
-                "estimated_input_tokens"
-            ]
-            model_groups[key]["estimated_output_tokens"] += item[
-                "estimated_output_tokens"
-            ]
-
-        # Apply iterations and convert to list
-        estimated_costs_by_model = []
-        for group_data in model_groups.values():
-            group_data["estimated_cost_usd"] *= iterations
-            group_data["estimated_input_tokens"] *= iterations
-            group_data["estimated_output_tokens"] *= iterations
-            estimated_costs_by_model.append(group_data)
+            for token_type in ["input", "output"]:
+                key, group_data = self.process_token_type(item, token_type)
+                if key not in detailed_groups:
+                    detailed_groups[key] = group_data
+                else:
+                    detailed_groups[key]["tokens"] += group_data["tokens"]
+                    detailed_groups[key]["cost_usd"] += group_data["cost_usd"]
+
+        # Apply iterations and prepare final output
+        detailed_costs = []
+        for group in detailed_groups.values():
+            group["tokens"] *= iterations
+            group["cost_usd"] *= iterations
+            detailed_costs.append(group)
 
         # Calculate totals
-        estimated_total_cost = sum(
-            model["estimated_cost_usd"] for model in estimated_costs_by_model
-        )
+        estimated_total_cost = sum(group["cost_usd"] for group in detailed_costs)
         estimated_total_input_tokens = sum(
-            model["estimated_input_tokens"] for model in estimated_costs_by_model
+            group["tokens"]
+            for group in detailed_costs
+            if group["token_type"] == "input"
         )
         estimated_total_output_tokens = sum(
-            model["estimated_output_tokens"] for model in estimated_costs_by_model
+            group["tokens"]
+            for group in detailed_costs
+            if group["token_type"] == "output"
         )
 
         output = {
             "estimated_total_cost_usd": estimated_total_cost,
             "estimated_total_input_tokens": estimated_total_input_tokens,
             "estimated_total_output_tokens": estimated_total_output_tokens,
-            "model_costs": estimated_costs_by_model,
+            "detailed_costs": detailed_costs,
         }
-
         return output
 
     def estimate_job_cost(self, iterations: int = 1) -> dict:

edsl/language_models/language_model.py

@@ -49,6 +49,7 @@ from ..data_transfer_models import (
 )
 
 if TYPE_CHECKING:
+    from .price_manager import ResponseCost
     from ..caching import Cache
     from ..scenarios import FileStore
     from ..questions import QuestionBase
@@ -782,13 +783,18 @@ class LanguageModel(
         # Calculate cost for the response
         cost = self.cost(response)
         # Return a structured response with metadata
-        return ModelResponse(
+        response = ModelResponse(
             response=response,
             cache_used=cache_used,
             cache_key=cache_key,
             cached_response=cached_response,
-            cost=cost,
+            input_tokens=cost.input_tokens,
+            output_tokens=cost.output_tokens,
+            input_price_per_million_tokens=cost.input_price_per_million_tokens,
+            output_price_per_million_tokens=cost.output_price_per_million_tokens,
+            total_cost=cost.total_cost,
         )
+        return response
 
     _get_intended_model_call_outcome = sync_wrapper(
         _async_get_intended_model_call_outcome
@@ -881,7 +887,7 @@ class LanguageModel(
 
     get_response = sync_wrapper(async_get_response)
 
-    def cost(self, raw_response: dict[str, Any]) -> Union[float, str]:
+    def cost(self, raw_response: dict[str, Any]) -> ResponseCost:
         """Calculate the monetary cost of a model API call.
 
         This method extracts token usage information from the response and
@@ -892,7 +898,7 @@ class LanguageModel(
             raw_response: The complete response dictionary from the model API
 
         Returns:
-            Union[float, str]: The calculated cost in dollars, or an error message
+            ResponseCost: Object containing token counts and total cost
         """
         # Extract token usage data from the response
         usage = self.get_usage_dict(raw_response)
@@ -1147,13 +1153,25 @@ class LanguageModel(
             }
             cached_response, cache_key = cache.fetch(**cache_call_params)
             response = json.loads(cached_response)
-            cost = 0
+
+            try:
+                usage = self.get_usage_dict(response)
+                input_tokens = int(usage[self.input_token_name])
+                output_tokens = int(usage[self.output_token_name])
+            except Exception as e:
+                print(f"Could not fetch tokens from model response: {e}")
+                input_tokens = None
+                output_tokens = None
             return ModelResponse(
                 response=response,
                 cache_used=True,
                 cache_key=cache_key,
                 cached_response=cached_response,
-                cost=cost,
+                input_tokens=input_tokens,
+                output_tokens=output_tokens,
+                input_price_per_million_tokens=0,
+                output_price_per_million_tokens=0,
+                total_cost=0,
             )
 
         # Bind the new method to the copied instance