edsl 0.1.34.dev1__tar.gz → 0.1.35__tar.gz

{edsl-0.1.34.dev1 → edsl-0.1.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.34.dev1
+Version: 0.1.35
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT

edsl-0.1.35/edsl/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.35"

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/agents/PromptConstructor.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from typing import Dict, Any, Optional, Set
 from collections import UserList
-import enum
+import pdb
 
 from jinja2 import Environment, meta
 
@@ -172,7 +172,6 @@ class PromptConstructor:
         """
         # The user might have passed a custom prompt, which would be stored in _question_instructions_prompt
         if not hasattr(self, "_question_instructions_prompt"):
-
             # Gets the instructions for the question - this is how the question should be answered
             question_prompt = self.question.get_instructions(model=self.model.model)
 
@@ -198,14 +197,15 @@ class PromptConstructor:
                     self.question.question_options = question_options
 
                 # might be getting it from the prior answers
-                if isinstance(
-                    question_options := self.prior_answers_dict()
-                    .get(question_option_key)
-                    .answer,
-                    list,
-                ):
-                    question_data["question_options"] = question_options
-                    self.question.question_options = question_options
+                if self.prior_answers_dict().get(question_option_key) is not None:
+                    if isinstance(
+                        question_options := self.prior_answers_dict()
+                        .get(question_option_key)
+                        .answer,
+                        list,
+                    ):
+                        question_data["question_options"] = question_options
+                        self.question.question_options = question_options
 
             replacement_dict = (
                 {key: f"<see file {key}>" for key in self.scenario_file_keys}
@@ -241,10 +241,13 @@ class PromptConstructor:
                     )
 
             if undefined_template_variables:
-                # breakpoint()
-                raise QuestionScenarioRenderError(
-                    f"Question instructions still has variables: {undefined_template_variables}."
-                )
+                msg = f"Question instructions still has variables: {undefined_template_variables}."
+                import warnings
+
+                warnings.warn(msg)
+                # raise QuestionScenarioRenderError(
+                #     f"Question instructions still has variables: {undefined_template_variables}."
+                # )
 
             ####################################
             # Check if question has instructions - these are instructions in a Survey that can apply to multiple follow-on questions

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/inference_services/OpenAIService.py

@@ -188,12 +188,16 @@ class OpenAIService(InferenceServiceABC):
                 else:
                     content = user_prompt
                 client = self.async_client()
+
                 messages = [
                     {"role": "system", "content": system_prompt},
                     {"role": "user", "content": content},
                 ]
-                if system_prompt == "" and self.omit_system_prompt_if_empty:
+                if (
+                    system_prompt == "" and self.omit_system_prompt_if_empty
+                ) or "o1" in self.model:
                     messages = messages[1:]
+
                 params = {
                     "model": self.model,
                     "messages": messages,
@@ -205,7 +209,14 @@ class OpenAIService(InferenceServiceABC):
                     "logprobs": self.logprobs,
                     "top_logprobs": self.top_logprobs if self.logprobs else None,
                 }
-                response = await client.chat.completions.create(**params)
+                if "o1" in self.model:
+                    params.pop("max_tokens")
+                    params["max_completion_tokens"] = self.max_tokens
+                    params["temperature"] = 1
+                try:
+                    response = await client.chat.completions.create(**params)
+                except Exception as e:
+                    print(e)
                 return response.model_dump()
 
         LLM.__name__ = "LanguageModel"

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/jobs/Jobs.py

@@ -145,14 +145,21 @@ class Jobs(Base):
         >>> Jobs.example().prompts()
         Dataset(...)
         """
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
 
         interviews = self.interviews()
         # data = []
         interview_indices = []
-        question_indices = []
+        question_names = []
         user_prompts = []
         system_prompts = []
         scenario_indices = []
+        agent_indices = []
+        models = []
+        costs = []
         from edsl.results.Dataset import Dataset
 
         for interview_index, interview in enumerate(interviews):
@@ -160,23 +167,97 @@ class Jobs(Base):
                 interview._get_invigilator(question)
                 for question in self.survey.questions
             ]
-            # list(interview._build_invigilators(debug=False))
             for _, invigilator in enumerate(invigilators):
                 prompts = invigilator.get_prompts()
-                user_prompts.append(prompts["user_prompt"])
-                system_prompts.append(prompts["system_prompt"])
+                user_prompt = prompts["user_prompt"]
+                system_prompt = prompts["system_prompt"]
+                user_prompts.append(user_prompt)
+                system_prompts.append(system_prompt)
+                agent_index = self.agents.index(invigilator.agent)
+                agent_indices.append(agent_index)
                 interview_indices.append(interview_index)
-                scenario_indices.append(invigilator.scenario)
-                question_indices.append(invigilator.question.question_name)
-        return Dataset(
+                scenario_index = self.scenarios.index(invigilator.scenario)
+                scenario_indices.append(scenario_index)
+                models.append(invigilator.model.model)
+                question_names.append(invigilator.question.question_name)
+                # cost calculation
+                key = (invigilator.model._inference_service_, invigilator.model.model)
+                relevant_prices = price_lookup[key]
+                inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+                inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+                input_tokens = len(str(user_prompt) + str(system_prompt)) // 4
+                output_tokens = len(str(user_prompt) + str(system_prompt)) // 4
+                cost = input_tokens / float(
+                    inverse_input_price
+                ) + output_tokens / float(inverse_output_price)
+                costs.append(cost)
+
+        d = Dataset(
             [
-                {"interview_index": interview_indices},
-                {"question_index": question_indices},
                 {"user_prompt": user_prompts},
-                {"scenario_index": scenario_indices},
                 {"system_prompt": system_prompts},
+                {"interview_index": interview_indices},
+                {"question_name": question_names},
+                {"scenario_index": scenario_indices},
+                {"agent_index": agent_indices},
+                {"model": models},
+                {"estimated_cost": costs},
             ]
         )
+        return d
+        # if table:
+        #     d.to_scenario_list().print(format="rich")
+        # else:
+        #     return d
+
+    def show_prompts(self) -> None:
+        """Print the prompts."""
+        self.prompts().to_scenario_list().print(format="rich")
+
+    def estimate_job_cost(self):
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
+
+        prompts = self.prompts()
+
+        text_len = 0
+        for prompt in prompts:
+            text_len += len(str(prompt))
+
+        input_token_aproximations = text_len // 4
+
+        aproximation_cost = {}
+        total_cost = 0
+        for model in self.models:
+            key = (model._inference_service_, model.model)
+            relevant_prices = price_lookup[key]
+            inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+            inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+
+            aproximation_cost[key] = {
+                "input": input_token_aproximations / float(inverse_input_price),
+                "output": input_token_aproximations / float(inverse_output_price),
+            }
+            ##TODO curenlty we approximate the number of output tokens with the number
+            # of input tokens. A better solution will be to compute the quesiton answer options length and sum them
+            # to compute the output tokens
+
+            total_cost += input_token_aproximations / float(inverse_input_price)
+            total_cost += input_token_aproximations / float(inverse_output_price)
+
+        # multiply_factor = len(self.agents or [1]) * len(self.scenarios or [1])
+        multiply_factor = 1
+        out = {
+            "input_token_aproximations": input_token_aproximations,
+            "models_costs": aproximation_cost,
+            "estimated_total_cost": total_cost * multiply_factor,
+            "multiply_factor": multiply_factor,
+            "single_config_cost": total_cost,
+        }
+
+        return out
 
     @staticmethod
     def _get_container_class(object):

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/jobs/interviews/Interview.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 import asyncio
 from typing import Any, Type, List, Generator, Optional, Union
+import copy
 
 from tenacity import (
     retry,
@@ -99,15 +100,17 @@ class Interview(InterviewStatusMixin):
 
         """
         self.agent = agent
-        self.survey = survey
+        # what I would like to do
+        self.survey = copy.deepcopy(survey)  # survey  copy.deepcopy(survey)
+        # self.survey = survey
         self.scenario = scenario
         self.model = model
         self.debug = debug
         self.iteration = iteration
         self.cache = cache
-        self.answers: dict[str, str] = (
-            Answers()
-        )  # will get filled in as interview progresses
+        self.answers: dict[
+            str, str
+        ] = Answers()  # will get filled in as interview progresses
         self.sidecar_model = sidecar_model
 
         # self.stop_on_exception = False
@@ -428,11 +431,11 @@ class Interview(InterviewStatusMixin):
         """
         current_question_index: int = self.to_index[current_question.question_name]
 
-        next_question: Union[int, EndOfSurvey] = (
-            self.survey.rule_collection.next_question(
-                q_now=current_question_index,
-                answers=self.answers | self.scenario | self.agent["traits"],
-            )
+        next_question: Union[
+            int, EndOfSurvey
+        ] = self.survey.rule_collection.next_question(
+            q_now=current_question_index,
+            answers=self.answers | self.scenario | self.agent["traits"],
         )
 
         next_question_index = next_question.next_q

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -173,19 +173,19 @@ class JobsRunnerAsyncio:
 
         prompt_dictionary = {}
         for answer_key_name in answer_key_names:
-            prompt_dictionary[answer_key_name + "_user_prompt"] = (
-                question_name_to_prompts[answer_key_name]["user_prompt"]
-            )
-            prompt_dictionary[answer_key_name + "_system_prompt"] = (
-                question_name_to_prompts[answer_key_name]["system_prompt"]
-            )
+            prompt_dictionary[
+                answer_key_name + "_user_prompt"
+            ] = question_name_to_prompts[answer_key_name]["user_prompt"]
+            prompt_dictionary[
+                answer_key_name + "_system_prompt"
+            ] = question_name_to_prompts[answer_key_name]["system_prompt"]
 
         raw_model_results_dictionary = {}
         for result in valid_results:
             question_name = result.question_name
-            raw_model_results_dictionary[question_name + "_raw_model_response"] = (
-                result.raw_model_response
-            )
+            raw_model_results_dictionary[
+                question_name + "_raw_model_response"
+            ] = result.raw_model_response
             raw_model_results_dictionary[question_name + "_cost"] = result.cost
             one_use_buys = (
                 "NA"

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/language_models/LanguageModel.py

@@ -490,7 +490,7 @@ class LanguageModel(
                 "user_prompt": user_prompt,
                 "system_prompt": system_prompt,
                 "files_list": files_list
-                #**({"encoded_image": encoded_image} if encoded_image else {}),
+                # **({"encoded_image": encoded_image} if encoded_image else {}),
             }
             # response = await f(**params)
             response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
@@ -534,7 +534,7 @@ class LanguageModel(
         system_prompt: str,
         cache: "Cache",
         iteration: int = 1,
-        files_list: Optional[List['File']] = None,
+        files_list: Optional[List["File"]] = None,
     ) -> dict:
         """Get response, parse, and return as string.
 

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/language_models/utilities.py

@@ -40,8 +40,10 @@ def create_language_model(
         _tpm = 1000000000000
 
         async def async_execute_model_call(
-            self, user_prompt: str, system_prompt: str, 
-            files_list: Optional[List[Any]] = None
+            self,
+            user_prompt: str,
+            system_prompt: str,
+            files_list: Optional[List[Any]] = None,
         ) -> dict[str, Any]:
             question_number = int(
                 user_prompt.split("XX")[1]

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/questions/QuestionBase.py

@@ -82,7 +82,8 @@ class QuestionBase(
         if not hasattr(self, "_fake_data_factory"):
             from polyfactory.factories.pydantic_factory import ModelFactory
 
-            class FakeData(ModelFactory[self.response_model]): ...
+            class FakeData(ModelFactory[self.response_model]):
+                ...
 
             self._fake_data_factory = FakeData
         return self._fake_data_factory

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/questions/QuestionMultipleChoice.py

@@ -120,9 +120,9 @@ class QuestionMultipleChoice(QuestionBase):
 
     question_type = "multiple_choice"
     purpose = "When options are known and limited"
-    question_options: Union[list[str], list[list], list[float], list[int]] = (
-        QuestionOptionsDescriptor()
-    )
+    question_options: Union[
+        list[str], list[list], list[float], list[int]
+    ] = QuestionOptionsDescriptor()
     _response_model = None
     response_validator_class = MultipleChoiceResponseValidator
 

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/questions/templates/numerical/answering_instructions.jinja

@@ -1,7 +1,6 @@
 This question requires a numerical response in the form of an integer or decimal (e.g., -12, 0, 1, 2, 3.45, ...).
 Respond with just your number on a single line.
 If your response is equivalent to zero, report '0'
-If you cannot determine the answer, report 'None'
 
 {% if include_comment %}
 After the answer, put a comment explaining your choice on the next line.

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/scenarios/FileStore.py

@@ -263,7 +263,6 @@ class FileStore(Scenario):
 
 
 class CSVFileStore(FileStore):
-
     @classmethod
     def example(cls):
         from edsl.results.Results import Results
@@ -282,7 +281,6 @@ class CSVFileStore(FileStore):
 
 
 class PDFFileStore(FileStore):
-
     def view(self):
         pdf_path = self.to_tempfile()
         print(f"PDF path: {pdf_path}")  # Print the path to ensure it exists
@@ -358,7 +356,6 @@ class PDFFileStore(FileStore):
 
 
 class PNGFileStore(FileStore):
-
     @classmethod
     def example(cls):
         import textwrap
@@ -382,7 +379,6 @@ class PNGFileStore(FileStore):
 
 
 class SQLiteFileStore(FileStore):
-
     @classmethod
     def example(cls):
         import sqlite3
@@ -405,7 +401,6 @@ class SQLiteFileStore(FileStore):
 
 
 class HTMLFileStore(FileStore):
-
     @classmethod
     def example(cls):
         import tempfile

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/scenarios/ScenarioList.py

@@ -39,6 +39,15 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             super().__init__([])
         self.codebook = codebook or {}
 
+    def unique(self) -> ScenarioList:
+        """Return a list of unique scenarios.
+
+        >>> s = ScenarioList([Scenario({'a': 1}), Scenario({'a': 1}), Scenario({'a': 2})])
+        >>> s.unique()
+        ScenarioList([Scenario({'a': 1}), Scenario({'a': 2})])
+        """
+        return ScenarioList(list(set(self)))
+
     @property
     def has_jinja_braces(self) -> bool:
         """Check if the ScenarioList has Jinja braces."""

{edsl-0.1.34.dev1 → edsl-0.1.35}/edsl/surveys/base.py

@@ -36,6 +36,10 @@ class EndOfSurveyParent:
         """
         return self
 
+    def __deepcopy__(self, memo):
+        # Return the same instance when deepcopy is called
+        return self
+
     def __radd__(self, other):
         """Add the object to another object.
 

{edsl-0.1.34.dev1 → edsl-0.1.35}/pyproject.toml

@@ -1,28 +1,19 @@
 [build-system]
 build-backend = "poetry.core.masonry.api"
-requires = [ "poetry-core", "setuptools<72.0",]
+requires = ["poetry-core", "setuptools<72.0"]
 
 [tool.poetry]
-authors = [ "Apostolos Filippas <apostolos@expectedparrot.com>", "John Horton <john@expectedparrot.com>", "Robin Horton <robin@expectedparrot.com>",]
-classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules",]
+authors = ["Apostolos Filippas <apostolos@expectedparrot.com>", "John Horton <john@expectedparrot.com>", "Robin Horton <robin@expectedparrot.com>"]
+classifiers = ["Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules"]
 description = "Create and analyze LLM-based surveys"
 documentation = "https://docs.expectedparrot.com"
 homepage = "https://www.expectedparrot.com/"
-include = [ "edsl/questions/templates/**/*",]
-keywords = [ "LLM", "social science", "surveys", "user research",]
+include = ["edsl/questions/templates/**/*"]
+keywords = ["LLM", "social science", "surveys", "user research"]
 license = "MIT"
 name = "edsl"
 readme = "README.md"
-version = "0.1.34.dev1"
-[[tool.poetry.packages]]
-include = "edsl"
-
-[tool.tomlsort]
-all = true
-in_place = true
-spaces_before_inline_comment = 2
-spaces_indent_inline_array = 4
-trailing_comma_inline_array = true
+version = "0.1.35"
 
 [tool.poetry.dependencies]
 python = ">=3.9.1,<3.13"
@@ -58,12 +49,9 @@ urllib3 = ">=1.25.4,<1.27"
 google-generativeai = "^0.8.2"
 
 [tool.poetry.dependencies.black]
-extras = [ "jupyter",]
+extras = ["jupyter"]
 version = "^24.4.2"
 
-[tool.tomlsort.overrides."tool.poetry.dependencies"]
-table_keys = false
-
 [tool.poetry.group.dev.dependencies]
 coverage = "^7.3.3"
 fastapi = "^0.112.1"
@@ -89,3 +77,16 @@ sphinx-rtd-theme = "^2.0.0"
 toml = "^0.10.2"
 toml-sort = "^0.23.1"
 uvicorn = "^0.30.6"
+
+[[tool.poetry.packages]]
+include = "edsl"
+
+[tool.tomlsort]
+all = true
+in_place = true
+spaces_before_inline_comment = 2
+spaces_indent_inline_array = 4
+trailing_comma_inline_array = true
+
+[tool.tomlsort.overrides."tool.poetry.dependencies"]
+table_keys = false

edsl-0.1.34.dev1/edsl/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.1.34.dev1"