edsl 0.1.34.dev1__py3-none-any.whl → 0.1.35__py3-none-any.whl

edsl/__version__.py

@@ -1 +1 @@
-__version__ = "0.1.34.dev1"
+__version__ = "0.1.35"

edsl/agents/PromptConstructor.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from typing import Dict, Any, Optional, Set
 from collections import UserList
-import enum
+import pdb
 
 from jinja2 import Environment, meta
 
@@ -172,7 +172,6 @@ class PromptConstructor:
         """
         # The user might have passed a custom prompt, which would be stored in _question_instructions_prompt
         if not hasattr(self, "_question_instructions_prompt"):
-
             # Gets the instructions for the question - this is how the question should be answered
             question_prompt = self.question.get_instructions(model=self.model.model)
 
@@ -198,14 +197,15 @@ class PromptConstructor:
                     self.question.question_options = question_options
 
                 # might be getting it from the prior answers
-                if isinstance(
-                    question_options := self.prior_answers_dict()
-                    .get(question_option_key)
-                    .answer,
-                    list,
-                ):
-                    question_data["question_options"] = question_options
-                    self.question.question_options = question_options
+                if self.prior_answers_dict().get(question_option_key) is not None:
+                    if isinstance(
+                        question_options := self.prior_answers_dict()
+                        .get(question_option_key)
+                        .answer,
+                        list,
+                    ):
+                        question_data["question_options"] = question_options
+                        self.question.question_options = question_options
 
             replacement_dict = (
                 {key: f"<see file {key}>" for key in self.scenario_file_keys}
@@ -241,10 +241,13 @@ class PromptConstructor:
                     )
 
             if undefined_template_variables:
-                # breakpoint()
-                raise QuestionScenarioRenderError(
-                    f"Question instructions still has variables: {undefined_template_variables}."
-                )
+                msg = f"Question instructions still has variables: {undefined_template_variables}."
+                import warnings
+
+                warnings.warn(msg)
+                # raise QuestionScenarioRenderError(
+                #     f"Question instructions still has variables: {undefined_template_variables}."
+                # )
 
             ####################################
             # Check if question has instructions - these are instructions in a Survey that can apply to multiple follow-on questions

edsl/inference_services/OpenAIService.py

@@ -188,12 +188,16 @@ class OpenAIService(InferenceServiceABC):
                 else:
                     content = user_prompt
                 client = self.async_client()
+
                 messages = [
                     {"role": "system", "content": system_prompt},
                     {"role": "user", "content": content},
                 ]
-                if system_prompt == "" and self.omit_system_prompt_if_empty:
+                if (
+                    system_prompt == "" and self.omit_system_prompt_if_empty
+                ) or "o1" in self.model:
                     messages = messages[1:]
+
                 params = {
                     "model": self.model,
                     "messages": messages,
@@ -205,7 +209,14 @@ class OpenAIService(InferenceServiceABC):
                     "logprobs": self.logprobs,
                     "top_logprobs": self.top_logprobs if self.logprobs else None,
                 }
-                response = await client.chat.completions.create(**params)
+                if "o1" in self.model:
+                    params.pop("max_tokens")
+                    params["max_completion_tokens"] = self.max_tokens
+                    params["temperature"] = 1
+                try:
+                    response = await client.chat.completions.create(**params)
+                except Exception as e:
+                    print(e)
                 return response.model_dump()
 
         LLM.__name__ = "LanguageModel"

edsl/jobs/Jobs.py

@@ -145,14 +145,21 @@ class Jobs(Base):
         >>> Jobs.example().prompts()
         Dataset(...)
         """
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
 
         interviews = self.interviews()
         # data = []
         interview_indices = []
-        question_indices = []
+        question_names = []
         user_prompts = []
         system_prompts = []
         scenario_indices = []
+        agent_indices = []
+        models = []
+        costs = []
         from edsl.results.Dataset import Dataset
 
         for interview_index, interview in enumerate(interviews):
@@ -160,23 +167,97 @@ class Jobs(Base):
                 interview._get_invigilator(question)
                 for question in self.survey.questions
             ]
-            # list(interview._build_invigilators(debug=False))
             for _, invigilator in enumerate(invigilators):
                 prompts = invigilator.get_prompts()
-                user_prompts.append(prompts["user_prompt"])
-                system_prompts.append(prompts["system_prompt"])
+                user_prompt = prompts["user_prompt"]
+                system_prompt = prompts["system_prompt"]
+                user_prompts.append(user_prompt)
+                system_prompts.append(system_prompt)
+                agent_index = self.agents.index(invigilator.agent)
+                agent_indices.append(agent_index)
                 interview_indices.append(interview_index)
-                scenario_indices.append(invigilator.scenario)
-                question_indices.append(invigilator.question.question_name)
-        return Dataset(
+                scenario_index = self.scenarios.index(invigilator.scenario)
+                scenario_indices.append(scenario_index)
+                models.append(invigilator.model.model)
+                question_names.append(invigilator.question.question_name)
+                # cost calculation
+                key = (invigilator.model._inference_service_, invigilator.model.model)
+                relevant_prices = price_lookup[key]
+                inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+                inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+                input_tokens = len(str(user_prompt) + str(system_prompt)) // 4
+                output_tokens = len(str(user_prompt) + str(system_prompt)) // 4
+                cost = input_tokens / float(
+                    inverse_input_price
+                ) + output_tokens / float(inverse_output_price)
+                costs.append(cost)
+
+        d = Dataset(
             [
-                {"interview_index": interview_indices},
-                {"question_index": question_indices},
                 {"user_prompt": user_prompts},
-                {"scenario_index": scenario_indices},
                 {"system_prompt": system_prompts},
+                {"interview_index": interview_indices},
+                {"question_name": question_names},
+                {"scenario_index": scenario_indices},
+                {"agent_index": agent_indices},
+                {"model": models},
+                {"estimated_cost": costs},
             ]
         )
+        return d
+        # if table:
+        #     d.to_scenario_list().print(format="rich")
+        # else:
+        #     return d
+
+    def show_prompts(self) -> None:
+        """Print the prompts."""
+        self.prompts().to_scenario_list().print(format="rich")
+
+    def estimate_job_cost(self):
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
+
+        prompts = self.prompts()
+
+        text_len = 0
+        for prompt in prompts:
+            text_len += len(str(prompt))
+
+        input_token_aproximations = text_len // 4
+
+        aproximation_cost = {}
+        total_cost = 0
+        for model in self.models:
+            key = (model._inference_service_, model.model)
+            relevant_prices = price_lookup[key]
+            inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+            inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+
+            aproximation_cost[key] = {
+                "input": input_token_aproximations / float(inverse_input_price),
+                "output": input_token_aproximations / float(inverse_output_price),
+            }
+            ##TODO curenlty we approximate the number of output tokens with the number
+            # of input tokens. A better solution will be to compute the quesiton answer options length and sum them
+            # to compute the output tokens
+
+            total_cost += input_token_aproximations / float(inverse_input_price)
+            total_cost += input_token_aproximations / float(inverse_output_price)
+
+        # multiply_factor = len(self.agents or [1]) * len(self.scenarios or [1])
+        multiply_factor = 1
+        out = {
+            "input_token_aproximations": input_token_aproximations,
+            "models_costs": aproximation_cost,
+            "estimated_total_cost": total_cost * multiply_factor,
+            "multiply_factor": multiply_factor,
+            "single_config_cost": total_cost,
+        }
+
+        return out
 
     @staticmethod
     def _get_container_class(object):

edsl/jobs/interviews/Interview.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 import asyncio
 from typing import Any, Type, List, Generator, Optional, Union
+import copy
 
 from tenacity import (
     retry,
@@ -99,15 +100,17 @@ class Interview(InterviewStatusMixin):
 
         """
         self.agent = agent
-        self.survey = survey
+        # what I would like to do
+        self.survey = copy.deepcopy(survey)  # survey  copy.deepcopy(survey)
+        # self.survey = survey
         self.scenario = scenario
         self.model = model
         self.debug = debug
         self.iteration = iteration
         self.cache = cache
-        self.answers: dict[str, str] = (
-            Answers()
-        )  # will get filled in as interview progresses
+        self.answers: dict[
+            str, str
+        ] = Answers()  # will get filled in as interview progresses
         self.sidecar_model = sidecar_model
 
         # self.stop_on_exception = False
@@ -428,11 +431,11 @@ class Interview(InterviewStatusMixin):
         """
         current_question_index: int = self.to_index[current_question.question_name]
 
-        next_question: Union[int, EndOfSurvey] = (
-            self.survey.rule_collection.next_question(
-                q_now=current_question_index,
-                answers=self.answers | self.scenario | self.agent["traits"],
-            )
+        next_question: Union[
+            int, EndOfSurvey
+        ] = self.survey.rule_collection.next_question(
+            q_now=current_question_index,
+            answers=self.answers | self.scenario | self.agent["traits"],
         )
 
         next_question_index = next_question.next_q

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -173,19 +173,19 @@ class JobsRunnerAsyncio:
 
         prompt_dictionary = {}
         for answer_key_name in answer_key_names:
-            prompt_dictionary[answer_key_name + "_user_prompt"] = (
-                question_name_to_prompts[answer_key_name]["user_prompt"]
-            )
-            prompt_dictionary[answer_key_name + "_system_prompt"] = (
-                question_name_to_prompts[answer_key_name]["system_prompt"]
-            )
+            prompt_dictionary[
+                answer_key_name + "_user_prompt"
+            ] = question_name_to_prompts[answer_key_name]["user_prompt"]
+            prompt_dictionary[
+                answer_key_name + "_system_prompt"
+            ] = question_name_to_prompts[answer_key_name]["system_prompt"]
 
         raw_model_results_dictionary = {}
         for result in valid_results:
             question_name = result.question_name
-            raw_model_results_dictionary[question_name + "_raw_model_response"] = (
-                result.raw_model_response
-            )
+            raw_model_results_dictionary[
+                question_name + "_raw_model_response"
+            ] = result.raw_model_response
             raw_model_results_dictionary[question_name + "_cost"] = result.cost
             one_use_buys = (
                 "NA"

edsl/language_models/LanguageModel.py

@@ -490,7 +490,7 @@ class LanguageModel(
                 "user_prompt": user_prompt,
                 "system_prompt": system_prompt,
                 "files_list": files_list
-                #**({"encoded_image": encoded_image} if encoded_image else {}),
+                # **({"encoded_image": encoded_image} if encoded_image else {}),
             }
             # response = await f(**params)
             response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
@@ -534,7 +534,7 @@ class LanguageModel(
         system_prompt: str,
         cache: "Cache",
         iteration: int = 1,
-        files_list: Optional[List['File']] = None,
+        files_list: Optional[List["File"]] = None,
     ) -> dict:
         """Get response, parse, and return as string.
 

edsl/language_models/utilities.py

@@ -40,8 +40,10 @@ def create_language_model(
         _tpm = 1000000000000
 
         async def async_execute_model_call(
-            self, user_prompt: str, system_prompt: str, 
-            files_list: Optional[List[Any]] = None
+            self,
+            user_prompt: str,
+            system_prompt: str,
+            files_list: Optional[List[Any]] = None,
         ) -> dict[str, Any]:
             question_number = int(
                 user_prompt.split("XX")[1]

edsl/questions/QuestionBase.py

@@ -82,7 +82,8 @@ class QuestionBase(
         if not hasattr(self, "_fake_data_factory"):
             from polyfactory.factories.pydantic_factory import ModelFactory
 
-            class FakeData(ModelFactory[self.response_model]): ...
+            class FakeData(ModelFactory[self.response_model]):
+                ...
 
             self._fake_data_factory = FakeData
         return self._fake_data_factory

edsl/questions/QuestionMultipleChoice.py

@@ -120,9 +120,9 @@ class QuestionMultipleChoice(QuestionBase):
 
     question_type = "multiple_choice"
     purpose = "When options are known and limited"
-    question_options: Union[list[str], list[list], list[float], list[int]] = (
-        QuestionOptionsDescriptor()
-    )
+    question_options: Union[
+        list[str], list[list], list[float], list[int]
+    ] = QuestionOptionsDescriptor()
     _response_model = None
     response_validator_class = MultipleChoiceResponseValidator
 

edsl/questions/templates/numerical/answering_instructions.jinja

@@ -1,7 +1,6 @@
 This question requires a numerical response in the form of an integer or decimal (e.g., -12, 0, 1, 2, 3.45, ...).
 Respond with just your number on a single line.
 If your response is equivalent to zero, report '0'
-If you cannot determine the answer, report 'None'
 
 {% if include_comment %}
 After the answer, put a comment explaining your choice on the next line.

edsl/scenarios/FileStore.py

@@ -263,7 +263,6 @@ class FileStore(Scenario):
 
 
 class CSVFileStore(FileStore):
-
     @classmethod
     def example(cls):
         from edsl.results.Results import Results
@@ -282,7 +281,6 @@ class CSVFileStore(FileStore):
 
 
 class PDFFileStore(FileStore):
-
     def view(self):
         pdf_path = self.to_tempfile()
         print(f"PDF path: {pdf_path}")  # Print the path to ensure it exists
@@ -358,7 +356,6 @@ class PDFFileStore(FileStore):
 
 
 class PNGFileStore(FileStore):
-
     @classmethod
     def example(cls):
         import textwrap
@@ -382,7 +379,6 @@ class PNGFileStore(FileStore):
 
 
 class SQLiteFileStore(FileStore):
-
     @classmethod
     def example(cls):
         import sqlite3
@@ -405,7 +401,6 @@ class SQLiteFileStore(FileStore):
 
 
 class HTMLFileStore(FileStore):
-
     @classmethod
     def example(cls):
         import tempfile

edsl/scenarios/ScenarioList.py

@@ -39,6 +39,15 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             super().__init__([])
         self.codebook = codebook or {}
 
+    def unique(self) -> ScenarioList:
+        """Return a list of unique scenarios.
+
+        >>> s = ScenarioList([Scenario({'a': 1}), Scenario({'a': 1}), Scenario({'a': 2})])
+        >>> s.unique()
+        ScenarioList([Scenario({'a': 1}), Scenario({'a': 2})])
+        """
+        return ScenarioList(list(set(self)))
+
     @property
     def has_jinja_braces(self) -> bool:
         """Check if the ScenarioList has Jinja braces."""

edsl/surveys/base.py

@@ -36,6 +36,10 @@ class EndOfSurveyParent:
         """
         return self
 
+    def __deepcopy__(self, memo):
+        # Return the same instance when deepcopy is called
+        return self
+
     def __radd__(self, other):
         """Add the object to another object.
 

{edsl-0.1.34.dev1.dist-info → edsl-0.1.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.34.dev1
+Version: 0.1.35
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT

{edsl-0.1.34.dev1.dist-info → edsl-0.1.35.dist-info}/RECORD

@@ -2,12 +2,12 @@ edsl/Base.py,sha256=wdFpHWlQlGNL4XfOmYA0AK9YupMDxK3G7mDHCQp60o4,9295
 edsl/BaseDiff.py,sha256=RoVEh52UJs22yMa7k7jv8se01G62jJNWnBzaZngo-Ug,8260
 edsl/TemplateLoader.py,sha256=sDBlSMt7EfOduM7w3h6v03gvh_Rzn9hVrlS-iLSQdZA,849
 edsl/__init__.py,sha256=UZcx9RHSi3Dslh2lWvCOeppdMW9Xzw_YLs-kFaNW1MU,1770
-edsl/__version__.py,sha256=XhBgcCdTjSpYn65HiKUqna67rnO8Ic__tIbjoMBXW6g,28
+edsl/__version__.py,sha256=ABqgofsnbWf7823vTBbZNQ81eKQbWwrVToAU6T3z-6s,23
 edsl/agents/Agent.py,sha256=ww6DK177eHQUlYkzgnt1b-MBDKXCdhVx3HezAZZ7TKk,28473
 edsl/agents/AgentList.py,sha256=qo8VK3Ov0YOSbsBcHmlwLZBH81CcDfy5IEcx9AVH78M,10963
 edsl/agents/Invigilator.py,sha256=6xd4sJ6Jzxld8LZDWZuSCZLL_MfaSux4LJCAm_RLEOM,9077
 edsl/agents/InvigilatorBase.py,sha256=qIdAiriXAbnJH_SN9w2UAXHcDgQvk8Ar3QerKFjtPwM,10341
-edsl/agents/PromptConstructor.py,sha256=BmScHm5y67vnPKMgo-Diq4p_nyVf_G7MB9IHFY7rml8,14599
+edsl/agents/PromptConstructor.py,sha256=qVCfRyYR-vKcpGJ_It2SOUvBfHtF8M6qYcf9ZCsHBl8,14855
 edsl/agents/__init__.py,sha256=B1dWfV4QWOo8cc2KeuetdFGeNhZ8XHc0Q8YhQW9k7BE,136
 edsl/agents/descriptors.py,sha256=m8ND3-2-JbgNX1HGakBNLIeemwsgYa1mQxYO9GW33hw,2934
 edsl/agents/prompt_helpers.py,sha256=rHUxM_F0kCOkJmnhCyK-amFKViAYvpRRLD8LHFLGqQw,5023
@@ -78,7 +78,7 @@ edsl/inference_services/InferenceServiceABC.py,sha256=rXqorwbKqzlwui2cxum8_TRrBc
 edsl/inference_services/InferenceServicesCollection.py,sha256=EDyxnoSjGXhWob_ost7U8WUYjn1jgL_noB0-VlXBnOo,2810
 edsl/inference_services/MistralAIService.py,sha256=7mUsBEZdEWIjfh4qMNemTT2xYMq7k0yuMLGtDTdfp4Y,3878
 edsl/inference_services/OllamaService.py,sha256=oro9CRl8IUE2Ro-zE69Cr4Zaf6Gdw29XW5CFU-46E0k,498
-edsl/inference_services/OpenAIService.py,sha256=2rGW0iGD_hnM1NHBJQCDI59Ti-9wFQu7gh8hDB95Cr4,7264
+edsl/inference_services/OpenAIService.py,sha256=wraTu62bZojmgAXHNG6pJMMPZiyO1pSDfY73LVaBR_I,7621
 edsl/inference_services/TestService.py,sha256=-jTXkl_qLt1k8gJjRb0SMgTb9EY-XMTP-ZUL9AJcUCA,3009
 edsl/inference_services/TogetherAIService.py,sha256=p_31ccrfN25kZF2xlAlUkb7w1EL4lGjmkSv-5qZ7TtY,6301
 edsl/inference_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -88,12 +88,12 @@ edsl/inference_services/registry.py,sha256=CwdaQ-A5PTb5lFKMQdOfl8IqCw2SVJ8HlC-_2
 edsl/inference_services/write_available.py,sha256=NNwhATlaMp8IYY635MSx-oYxt5X15acjAfaqYCo_I1Y,285
 edsl/jobs/Answers.py,sha256=c4LpigQjdnMr7iJu8571C4FggGPVudfT7hbJgmgKW40,1821
 edsl/jobs/FailedQuestion.py,sha256=3D5Vcmv1t2_dzBWbkUUIia3_jXHdzQDdyg-4TEIWU2Q,2701
-edsl/jobs/Jobs.py,sha256=a2Y-goWT9vGi00OqW7t9L8Oi6SXXiSllymd3s1frX1g,34035
+edsl/jobs/Jobs.py,sha256=oryxhrBuXs9Ba8mt5BSIjX3gT37hdzothL5xH0ygrkU,37227
 edsl/jobs/__init__.py,sha256=aKuAyd_GoalGj-k7djOoVwEbFUE2XLPlikXaA1_8yAg,32
 edsl/jobs/buckets/BucketCollection.py,sha256=11CRisE1WAPcAlI3YJK3DVvu0AqSvv8KskXo4Q1waSk,2286
 edsl/jobs/buckets/ModelBuckets.py,sha256=hxw_tzc0V42CiB7mh5jIxlgwDVJ-zFZhlLtKrHEg8ho,2419
 edsl/jobs/buckets/TokenBucket.py,sha256=7fG4omzTcj5xC2iJLO9bfBkdAGs6Y3weXzlA3BgPr0E,9090
-edsl/jobs/interviews/Interview.py,sha256=Vt5-6zNazBnp8EZRTA552hy3MyaFxMPkLGgAaIhikzI,23774
+edsl/jobs/interviews/Interview.py,sha256=iw0thEGwENOl8uzHRJ88zjYHEDSlM_X96Fyse9wZCwM,23883
 edsl/jobs/interviews/InterviewExceptionCollection.py,sha256=Ez8BCZUD3odqoY9h-gzYKKM8yaHynQ-eYw2uMDh7t98,3279
 edsl/jobs/interviews/InterviewExceptionEntry.py,sha256=BGGjj0sb1wJJ0QmYklt1DyEYKD8mUGygllGfN2WXKbY,4903
 edsl/jobs/interviews/InterviewStatistic.py,sha256=hY5d2EkIJ96NilPpZAvZZzZoxLXM7ss3xx5MIcKtTPs,1856
@@ -103,7 +103,7 @@ edsl/jobs/interviews/InterviewStatusLog.py,sha256=6u0F8gf5tha39VQL-IK_QPkCsQAYVO
 edsl/jobs/interviews/InterviewStatusMixin.py,sha256=VV0Pel-crUsLoGpTifeIIkXsLGj0bfuO--UtpRnH-dU,1251
 edsl/jobs/interviews/ReportErrors.py,sha256=RSzDU2rWwtjfztj7sqaMab0quCiY-X2bG3AEOxhTim8,1745
 edsl/jobs/interviews/interview_status_enum.py,sha256=KJ-1yLAHdX-p8TiFnM0M3v1tnBwkq4aMCuBX6-ytrI8,229
-edsl/jobs/runners/JobsRunnerAsyncio.py,sha256=CxO93PO7UGtr9yzrIWDlBjHdq1uAaKoaevsJsYV_3zA,12856
+edsl/jobs/runners/JobsRunnerAsyncio.py,sha256=SXWFTgTYQzqIaBjPQ2mRrxR3aGFMQyw2eD9wX8tZNkQ,12850
 edsl/jobs/runners/JobsRunnerStatus.py,sha256=4eCh9sRpswGdKeSMW9pCGCAjJZa-OrWUPI7tsxIy_g4,12112
 edsl/jobs/runners/JobsRunnerStatusData.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edsl/jobs/tasks/QuestionTaskCreator.py,sha256=f26rokFVXVdyKR2M8of_jlZqP6TPcTAINk9tEvb35pw,10122
@@ -114,7 +114,7 @@ edsl/jobs/tasks/task_management.py,sha256=KMToZuXzMlnHRHUF_VHL0-lHMTGhklf2GHVuwE
 edsl/jobs/tasks/task_status_enum.py,sha256=DOyrz61YlIS8R1W7izJNphcLrJ7I_ReUlfdRmk23h0Q,5333
 edsl/jobs/tokens/InterviewTokenUsage.py,sha256=u_6-IHpGFwZ6qMEXr24-jyLVUSSp4dSs_4iAZsBv7O4,1100
 edsl/jobs/tokens/TokenUsage.py,sha256=odj2-wDNEbHl9noyFAQ0DSKV0D9cv3aDOpmXufKZ8O4,1323
-edsl/language_models/LanguageModel.py,sha256=_f3Gv-1n4ttKWs7aSJG13xRklzxL-wOAn2Gl1jrWh5M,25193
+edsl/language_models/LanguageModel.py,sha256=lGmflHSq0vLt4zBzYSKxH5dFccfA4Mhj4NM-XhTNZxY,25194
 edsl/language_models/ModelList.py,sha256=GhjRV7y1jRvP_Yjgwv6fxksTVb8sFPBiiRRfdqW-hgg,2852
 edsl/language_models/RegisterLanguageModelsMeta.py,sha256=eMtBSAnlRnC4c-0_o2QkSNyzv-uAce4BEGMXq2PLj2E,7523
 edsl/language_models/__init__.py,sha256=bvY7Gy6VkX1gSbNkRbGPS-M1kUnb0EohL0FSagaEaTs,109
@@ -123,7 +123,7 @@ edsl/language_models/fake_openai_service.py,sha256=2AAsAinELbMZRqiepwBkWhWcLuMe5
 edsl/language_models/registry.py,sha256=hfOlKbTkXrXGpZHQQPKE9uyyUCgOxoUyyIaKL2kf53U,4369
 edsl/language_models/repair.py,sha256=d0i2S3kJfX7JtuCYhlIyT0QP8hcZkRPLanC09lOW_xo,5353
 edsl/language_models/unused/ReplicateBase.py,sha256=J1oqf7mEyyKhRwNUomnptVqAsVFYCbS3iTW0EXpKtXo,3331
-edsl/language_models/utilities.py,sha256=ZEwldu1n4M0YRxcQ1s5q6SOkgc7ld4SLuFMq6hjiHIk,2256
+edsl/language_models/utilities.py,sha256=GWON2ahCpB-_-hhqmQ5Yi7_rKB4cd8GlucWuq6EnGZQ,2280
 edsl/notebooks/Notebook.py,sha256=xi9xkxmkQ6-DwhqbjjMLpYKB0VJV20AtwEonJ6mnqjo,7739
 edsl/notebooks/__init__.py,sha256=VNUA3nNq04slWNbYaNrzOhQJu3AZANpvBniyCJSzJ7U,45
 edsl/prompts/Prompt.py,sha256=KsIz0tZnIqZlnbzNWO57C3MwJ0OCcUrZPmu7DBApc4s,11887
@@ -143,7 +143,7 @@ edsl/prompts/library/question_rank.py,sha256=WDgXyph0EKWJrSgsW2eqcx3xdU-WA1LEvB2
 edsl/prompts/prompt_config.py,sha256=O3Y5EvBsCeKs9m9IjXiRXOcHWlWvQV0yqsNb2oSR1ow,974
 edsl/prompts/registry.py,sha256=XOsqGsvNOmIG83jqoszqI72yNZkkszKR4FrEhwSzj_Q,8093
 edsl/questions/AnswerValidatorMixin.py,sha256=t_ABep50KP02GSc48Y8VAEjp35drVOngfrWXU5aVhgk,11505
-edsl/questions/QuestionBase.py,sha256=NY1M1QT3v2ToZJMbzsER4zejOPmczPzxEA_LKmdbsPk,21153
+edsl/questions/QuestionBase.py,sha256=QsdYFHnWYh04JpCdx3o3WGgohBfE5X_7idQRRYrx_SA,21169
 edsl/questions/QuestionBaseGenMixin.py,sha256=CPxjWZjrxuSO8YWelz6dbp7fm788gN7-T8z7jXStboQ,6017
 edsl/questions/QuestionBasePromptsMixin.py,sha256=Km1P6PpkVJ9O3dS8pJZHNJ6aQABhYGLwdef4Z2vSrqk,9516
 edsl/questions/QuestionBudget.py,sha256=TJgPsyqafJdJw5if0zVxh7zHloourINUqUWfWIlRq9Y,8131
@@ -152,7 +152,7 @@ edsl/questions/QuestionExtract.py,sha256=PlXwMeZgPAFBXIHSXpFMYTToag-HwA9C7u6-Z3b
 edsl/questions/QuestionFreeText.py,sha256=uXJxrrAGCq0-J6WpO6TBNFBNOC2ztoEVa-2UMXuwlak,3384
 edsl/questions/QuestionFunctional.py,sha256=yZQFLQAxfNsAIETffFoBr-Ltb2oFPeywu-nhm9qjYRc,5133
 edsl/questions/QuestionList.py,sha256=vs2AE8OnbwVsly-sorb9dfIibdF1BpOaCRYyvwXYSzY,7209
-edsl/questions/QuestionMultipleChoice.py,sha256=SrvT8vS9KOCBogcABOFBhO1gb5Lr0vvmOfoxUkSw-h0,10294
+edsl/questions/QuestionMultipleChoice.py,sha256=Yj94-6fwFqDI9UvjwSCOfKnp4gBB86XMmCsL7lbX-t4,10292
 edsl/questions/QuestionNumerical.py,sha256=_jMZ28DZHYAv_g3Y3vCnmzerMs995on0Ng6j4pDcfHo,4959
 edsl/questions/QuestionRank.py,sha256=kYHTYXU88X2Uv-zeCiI9w5aEFYTxvg2p7DoGaaER4ik,11596
 edsl/questions/Quick.py,sha256=h6h4fEvIkLIFJX2JiqfOUEXzku9azWxEpI5o2A4RmVs,1731
@@ -205,7 +205,7 @@ edsl/questions/templates/multiple_choice/answering_instructions.jinja,sha256=eSc
 edsl/questions/templates/multiple_choice/html.jinja,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edsl/questions/templates/multiple_choice/question_presentation.jinja,sha256=hoEVj4GQD3EYnR2AStXkMFOJeqISNoEVzBd8-cx2yWg,273
 edsl/questions/templates/numerical/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-edsl/questions/templates/numerical/answering_instructions.jinja,sha256=BfGAeKFJWEzzvB91x8DrpRUERSvDv6kXf3Y0szL27LY,373
+edsl/questions/templates/numerical/answering_instructions.jinja,sha256=70CnuIbdRKGiYJk7316-FFloRh3Oeg_RvUu4tthNKqs,323
 edsl/questions/templates/numerical/question_presentation.jinja,sha256=8lMUWtEPHD4XOAyVEfCmWSwRFrdUa3lo8sxzogDyzWE,183
 edsl/questions/templates/rank/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edsl/questions/templates/rank/answering_instructions.jinja,sha256=-LWrhVJ0ZwQW_DXhARh5GweqlarWbhZzoqwr6tE-33s,433
@@ -229,10 +229,10 @@ edsl/results/ResultsToolsMixin.py,sha256=mseEFxJCf9sjXdIxpjITt_UZBwdXxw2o2VLg5jM
 edsl/results/Selector.py,sha256=4AsFD71FKTFY1a0_AImsYWcYKx-RXPG6RgwsAvuNW3k,4403
 edsl/results/__init__.py,sha256=2YcyiVtXi-3vIV0ZzOy1PqBLm2gaziufJVi4fdNrAt8,80
 edsl/results/tree_explore.py,sha256=hQjiO4E71rIOPDgEHgK8T8ukxqoNdgX_tvyiDlG4_9U,4624
-edsl/scenarios/FileStore.py,sha256=mk8jVl2vvP31ufWPruS5ULT4TmbmMWd7NgQbtRIrJE0,13934
+edsl/scenarios/FileStore.py,sha256=V_zn2RmVclcBQTvqghotnTO97YHWUXqeVfN7eMfpfUM,13929
 edsl/scenarios/Scenario.py,sha256=ZG1x4_MmWP9j0gakLwsxOZ7ESMy3ifwBMhgQlHvsYo8,16809
 edsl/scenarios/ScenarioHtmlMixin.py,sha256=EmugmbPJYW5eZS30rM6pDMDQD9yrrvHjmgZWB1qBfq4,1882
-edsl/scenarios/ScenarioList.py,sha256=v5zmM3AOxbVFN9kez6h8GVbFFAxDlykycXqREDc2h8c,40622
+edsl/scenarios/ScenarioList.py,sha256=1zbwDICXbvtUGA5bDlhThNp9pfhBGIHIqhK2cX-th50,40942
 edsl/scenarios/ScenarioListExportMixin.py,sha256=wfffY9xy_1QyIM-1xnisr64izSLjmyuotUYY5iDLodc,1681
 edsl/scenarios/ScenarioListPdfMixin.py,sha256=z_H2sZn5SCSq6nRLSU5jefaOlh4sqJLyOY_Ld0XCR18,8332
 edsl/scenarios/__init__.py,sha256=KRwZCLf2R0qyJvv1NGbd8M51Bt6Ia6Iylg-Xq_2Fa6M,98
@@ -253,7 +253,7 @@ edsl/surveys/SurveyExportMixin.py,sha256=Kvkd2ku2Kemsn2Nw-Yt8GTnGFcUqfEiKznmisAe
 edsl/surveys/SurveyFlowVisualizationMixin.py,sha256=dEG_f-L0ZAyWU5Ta584IX5GZurjVt1tbIISo5z61Jvg,4004
 edsl/surveys/SurveyQualtricsImport.py,sha256=SSZv53D1zVhQSfSw-X0_cte0QnkWhE9v922wLn6RMkI,9771
 edsl/surveys/__init__.py,sha256=vjMYVlP95fHVqqw2FfKXRuYbTArZkZr1nK4FnXzZWzs,129
-edsl/surveys/base.py,sha256=n5PBx0BF0powzBXCsufpUekfNK_9huf3rohtU1mMCq0,1001
+edsl/surveys/base.py,sha256=XJHGEbbsH6hlYYkmI4isVLD8guLz8BdhR-eQRL78mc4,1115
 edsl/surveys/descriptors.py,sha256=3B-hBVvGpLlVBCyOnPuxkLjesvpr0QIuATbggp_MJ7o,2076
 edsl/surveys/instructions/ChangeInstruction.py,sha256=XDLuI5nVI60iJz1w1kLaKmYryAYE0XIyRbElBtNjVVM,1265
 edsl/surveys/instructions/Instruction.py,sha256=WaTGihAQ6lCtm5W4vknTamkPzDp-eIAirdtGV37fdbc,925
@@ -289,7 +289,7 @@ edsl/utilities/interface.py,sha256=AaKpWiwWBwP2swNXmnFlIf3ZFsjfsR5bjXQAW47tD-8,1
 edsl/utilities/repair_functions.py,sha256=tftmklAqam6LOQQu_-9U44N-llycffhW8LfO63vBmNw,929
 edsl/utilities/restricted_python.py,sha256=5-_zUhrNbos7pLhDl9nr8d24auRlquR6w-vKkmNjPiA,2060
 edsl/utilities/utilities.py,sha256=gqMtWWNEZkWLiRR9vHW-VRNy2bStEPlJ-I2aK9CwFiQ,11367
-edsl-0.1.34.dev1.dist-info/LICENSE,sha256=_qszBDs8KHShVYcYzdMz3HNMtH-fKN_p5zjoVAVumFc,1111
-edsl-0.1.34.dev1.dist-info/METADATA,sha256=S-k9GEY61HYIUZ60rtZxkPBP0g9V2lb6YfG70P41PFk,4476
-edsl-0.1.34.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-edsl-0.1.34.dev1.dist-info/RECORD,,
+edsl-0.1.35.dist-info/LICENSE,sha256=_qszBDs8KHShVYcYzdMz3HNMtH-fKN_p5zjoVAVumFc,1111
+edsl-0.1.35.dist-info/METADATA,sha256=eyPEBXgJt4VgO_f7yPO8rIjvRDT2_RFjbCzF3bvVftk,4471
+edsl-0.1.35.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+edsl-0.1.35.dist-info/RECORD,,