edsl 0.1.45__py3-none-any.whl → 0.1.47__py3-none-any.whl

edsl/data/Cache.py

@@ -173,6 +173,7 @@ class Cache(Base):
         user_prompt: str,
         response: dict,
         iteration: int,
+        service: str,
     ) -> str:
         """
         Add a new key-value pair to the cache.
@@ -204,6 +205,7 @@ class Cache(Base):
             user_prompt=user_prompt,
             output=json.dumps(response),
             iteration=iteration,
+            service=service,
         )
         key = entry.key
         self.new_entries[key] = entry

edsl/data/CacheEntry.py

@@ -16,7 +16,7 @@ class CacheEntry(RepresentationMixin):
     """
 
     key_fields = ["model", "parameters", "system_prompt", "user_prompt", "iteration"]
-    all_fields = key_fields + ["timestamp", "output"]
+    all_fields = key_fields + ["timestamp", "output", "service"]
 
     def __init__(
         self,
@@ -28,6 +28,7 @@ class CacheEntry(RepresentationMixin):
         iteration: Optional[int] = None,
         output: str,
         timestamp: Optional[int] = None,
+        service: Optional[str] = None,
     ):
         self.model = model
         self.parameters = parameters
@@ -38,6 +39,7 @@ class CacheEntry(RepresentationMixin):
         self.timestamp = timestamp or int(
             datetime.datetime.now(datetime.timezone.utc).timestamp()
         )
+        self.service = service
         self._check_types()
 
     def _check_types(self):
@@ -59,6 +61,8 @@ class CacheEntry(RepresentationMixin):
         # TODO: should probably be float
         if not isinstance(self.timestamp, int):
             raise TypeError(f"`timestamp` should be an integer")
+        if self.service is not None and not isinstance(self.service, str):
+            raise TypeError("`service` should be either a string or None")
 
     @classmethod
     def gen_key(
@@ -94,6 +98,7 @@ class CacheEntry(RepresentationMixin):
             "output": self.output,
             "iteration": self.iteration,
             "timestamp": self.timestamp,
+            "service": self.service,
         }
         # if add_edsl_version:
         #     from edsl import __version__
@@ -144,7 +149,8 @@ class CacheEntry(RepresentationMixin):
             f"user_prompt={repr(self.user_prompt)}, "
             f"output={repr(self.output)}, "
             f"iteration={self.iteration}, "
-            f"timestamp={self.timestamp})"
+            f"timestamp={self.timestamp}, "
+            f"service={repr(self.service)})"
         )
 
     @classmethod
@@ -164,6 +170,7 @@ class CacheEntry(RepresentationMixin):
             output="The fox says 'hello'",
             iteration=1,
             timestamp=int(datetime.datetime.now(datetime.timezone.utc).timestamp()),
+            service="openai",
         )
 
     @classmethod
@@ -184,6 +191,7 @@ class CacheEntry(RepresentationMixin):
         input = cls.example().to_dict()
         _ = input.pop("timestamp")
         _ = input.pop("output")
+        _ = input.pop("service")
         return input
 
     @classmethod

edsl/data/RemoteCacheSync.py

@@ -100,7 +100,7 @@ class RemoteCacheSync(AbstractContextManager):
 
     def _get_cache_difference(self) -> CacheDifference:
         """Retrieves differences between local and remote caches."""
-        diff = self.coop.remote_cache_get_diff(self.cache.keys())
+        diff = self.coop.legacy_remote_cache_get_diff(self.cache.keys())
         return CacheDifference(
             client_missing_entries=diff.get("client_missing_cacheentries", []),
             server_missing_keys=diff.get("server_missing_cacheentry_keys", []),
@@ -112,7 +112,7 @@ class RemoteCacheSync(AbstractContextManager):
         missing_count = len(diff.client_missing_entries)
 
         if missing_count == 0:
-        #     self._output("No new entries to add to local cache.")
+            #     self._output("No new entries to add to local cache.")
             return
 
         # self._output(
@@ -154,22 +154,23 @@ class RemoteCacheSync(AbstractContextManager):
         upload_count = len(entries_to_upload)
 
         if upload_count > 0:
+            pass
             # self._output(
             #     f"Updating remote cache with {upload_count:,} new "
             #     f"{'entry' if upload_count == 1 else 'entries'}..."
             # )
 
-            self.coop.remote_cache_create_many(
-                entries_to_upload,
-                visibility="private",
-                description=self.remote_cache_description,
-            )
+            # self.coop.remote_cache_create_many(
+            #     entries_to_upload,
+            #     visibility="private",
+            #     description=self.remote_cache_description,
+            # )
             # self._output("Remote cache updated!")
         # else:
-            # self._output("No new entries to add to remote cache.")
+        # self._output("No new entries to add to remote cache.")
 
         # self._output(
-            # f"There are {len(self.cache.keys()):,} entries in the local cache."
+        # f"There are {len(self.cache.keys()):,} entries in the local cache."
         # )
 
 

edsl/inference_services/AvailableModelFetcher.py

@@ -69,7 +69,7 @@ class AvailableModelFetcher:
 
         Returns a list of [model, service_name, index] entries.
         """
-        if service == "azure":
+        if service == "azure" or service == "bedrock":
             force_refresh = True  # Azure models are listed inside the .env AZURE_ENDPOINT_URL_AND_KEY variable
 
         if service:  # they passed a specific service

edsl/inference_services/PerplexityService.py

@@ -29,9 +29,12 @@ class PerplexityService(OpenAIService):
     @classmethod
     def available(cls) -> List[str]:
         return [
-            "llama-3.1-sonar-huge-128k-online",
-            "llama-3.1-sonar-large-128k-online",
-            "llama-3.1-sonar-small-128k-online",
+            "sonar-deep-research",
+            "sonar-reasoning-pro",
+            "sonar-reasoning",
+            "sonar-pro",
+            "sonar",
+            "r1-1776",
         ]
 
     @classmethod
@@ -65,10 +68,10 @@ class PerplexityService(OpenAIService):
             }
 
             def sync_client(self):
-                return cls.sync_client()
+                return cls.sync_client(api_key=self.api_token)
 
             def async_client(self):
-                return cls.async_client()
+                return cls.async_client(api_key=self.api_token)
 
             @classmethod
             def available(cls) -> list[str]:
@@ -149,6 +152,7 @@ class PerplexityService(OpenAIService):
                     # "logprobs": self.logprobs,
                     # "top_logprobs": self.top_logprobs if self.logprobs else None,
                 }
+                print("calling the model", flush=True)
                 try:
                     response = await client.chat.completions.create(**params)
                 except Exception as e:

edsl/jobs/AnswerQuestionFunctionConstructor.py

@@ -66,10 +66,14 @@ class SkipHandler:
             )
         )
 
+
         def cancel_between(start, end):
             """Cancel the tasks for questions between the start and end indices."""
             for i in range(start, end):
-                self.interview.tasks[i].cancel()
+                #print(f"Cancelling task {i}")
+                #self.interview.tasks[i].cancel()
+                #self.interview.tasks[i].set_result("skipped")
+                self.interview.skip_flags[self.interview.survey.questions[i].question_name] = True
 
         if (next_question_index := next_question.next_q) == EndOfSurvey:
             cancel_between(
@@ -80,6 +84,8 @@ class SkipHandler:
         if next_question_index > (current_question_index + 1):
             cancel_between(current_question_index + 1, next_question_index)
 
+        
+
 
 class AnswerQuestionFunctionConstructor:
     """Constructs a function that answers a question and records the answer."""
@@ -161,6 +167,11 @@ class AnswerQuestionFunctionConstructor:
         async def attempt_answer():
             invigilator = self.invigilator_fetcher(question)
 
+            if self.interview.skip_flags.get(question.question_name, False):
+                return invigilator.get_failed_task_result(
+                    failure_reason="Question skipped."
+                )
+
             if self.skip_handler.should_skip(question):
                 return invigilator.get_failed_task_result(
                     failure_reason="Question skipped."

edsl/jobs/Jobs.py

@@ -119,6 +119,19 @@ class Jobs(Base):
         :param agents: a list of agents
         :param models: a list of models
         :param scenarios: a list of scenarios
+
+
+        >>> from edsl.surveys.Survey import Survey
+        >>> from edsl.questions.QuestionFreeText import QuestionFreeText
+        >>> q = QuestionFreeText(question_name="name", question_text="What is your name?")
+        >>> s = Survey(questions=[q])
+        >>> j = Jobs(survey = s)
+        >>> q = QuestionFreeText(question_name="{{ bad_name }}", question_text="What is your name?")
+        >>> s = Survey(questions=[q])
+        >>> j = Jobs(survey = s)
+        Traceback (most recent call last):
+        ...
+        ValueError: At least some question names are not valid: ['{{ bad_name }}']
         """
         self.run_config = RunConfig(
             environment=RunEnvironment(), parameters=RunParameters()
@@ -129,6 +142,13 @@ class Jobs(Base):
         self.scenarios: ScenarioList = scenarios
         self.models: ModelList = models
 
+        try:
+            assert self.survey.question_names_valid()
+        except Exception as e:
+            invalid_question_names = [q.question_name for q in self.survey.questions if not q.is_valid_question_name()]
+            raise ValueError(f"At least some question names are not valid: {invalid_question_names}")
+        
+
     def add_running_env(self, running_env: RunEnvironment):
         self.run_config.add_environment(running_env)
         return self
@@ -277,7 +297,7 @@ class Jobs(Base):
 
         return JobsComponentConstructor(self).by(*args)
 
-    def prompts(self) -> "Dataset":
+    def prompts(self, iterations=1) -> "Dataset":
         """Return a Dataset of prompts that will be used.
 
 
@@ -285,7 +305,7 @@ class Jobs(Base):
         >>> Jobs.example().prompts()
         Dataset(...)
         """
-        return JobsPrompts(self).prompts()
+        return JobsPrompts(self).prompts(iterations=iterations)
 
     def show_prompts(self, all: bool = False) -> None:
         """Print the prompts."""
@@ -418,11 +438,9 @@ class Jobs(Base):
         BucketCollection(...)
         """
         bc = BucketCollection.from_models(self.models)
-        
+
         if self.run_config.environment.key_lookup is not None:
-            bc.update_from_key_lookup(
-                self.run_config.environment.key_lookup
-            )
+            bc.update_from_key_lookup(self.run_config.environment.key_lookup)
         return bc
 
     def html(self):
@@ -484,25 +502,24 @@ class Jobs(Base):
     def _start_remote_inference_job(
         self, job_handler: Optional[JobsRemoteInferenceHandler] = None
     ) -> Union["Results", None]:
-
         if job_handler is None:
             job_handler = self._create_remote_inference_handler()
-            
+
         job_info = job_handler.create_remote_inference_job(
-                iterations=self.run_config.parameters.n,
-                remote_inference_description=self.run_config.parameters.remote_inference_description,
-                remote_inference_results_visibility=self.run_config.parameters.remote_inference_results_visibility,
+            iterations=self.run_config.parameters.n,
+            remote_inference_description=self.run_config.parameters.remote_inference_description,
+            remote_inference_results_visibility=self.run_config.parameters.remote_inference_results_visibility,
+            fresh=self.run_config.parameters.fresh,
         )
         return job_info
-    
-    def _create_remote_inference_handler(self) -> JobsRemoteInferenceHandler:
 
+    def _create_remote_inference_handler(self) -> JobsRemoteInferenceHandler:
         from edsl.jobs.JobsRemoteInferenceHandler import JobsRemoteInferenceHandler
-        
+
         return JobsRemoteInferenceHandler(
             self, verbose=self.run_config.parameters.verbose
         )
-        
+
     def _remote_results(
         self,
         config: RunConfig,
@@ -516,7 +533,8 @@ class Jobs(Base):
         if jh.use_remote_inference(self.run_config.parameters.disable_remote_inference):
             job_info: RemoteJobInfo = self._start_remote_inference_job(jh)
             if background:
-                from edsl.results.Results import Results 
+                from edsl.results.Results import Results
+
                 results = Results.from_job_info(job_info)
                 return results
             else:
@@ -603,7 +621,7 @@ class Jobs(Base):
         # first try to run the job remotely
         if (results := self._remote_results(config)) is not None:
             return results
-        
+
         self._check_if_local_keys_ok()
 
         if config.environment.bucket_collection is None:

edsl/jobs/JobsComponentConstructor.py

@@ -153,7 +153,8 @@ class JobsComponentConstructor:
         For example, if the user passes in 3 agents,
         and there are 2 existing agents, this will create 6 new agents
         >>> from edsl.jobs import Jobs
-        >>> JobsComponentConstructor(Jobs(survey = []))._merge_objects([1,2,3], [4,5,6])
+        >>> from edsl.surveys.Survey import Survey
+        >>> JobsComponentConstructor(Jobs(survey = Survey.example()))._merge_objects([1,2,3], [4,5,6])
         [5, 6, 7, 6, 7, 8, 7, 8, 9]
         """
         new_objects = JobsComponentConstructor._get_empty_container_object(

edsl/jobs/JobsPrompts.py

@@ -18,6 +18,7 @@ from edsl.data.CacheEntry import CacheEntry
 
 logger = logging.getLogger(__name__)
 
+
 class JobsPrompts:
     def __init__(self, jobs: "Jobs"):
         self.interviews = jobs.interviews()
@@ -26,7 +27,9 @@ class JobsPrompts:
         self.survey = jobs.survey
         self._price_lookup = None
         self._agent_lookup = {agent: idx for idx, agent in enumerate(self.agents)}
-        self._scenario_lookup = {scenario: idx for idx, scenario in enumerate(self.scenarios)}
+        self._scenario_lookup = {
+            scenario: idx for idx, scenario in enumerate(self.scenarios)
+        }
 
     @property
     def price_lookup(self):
@@ -37,7 +40,7 @@ class JobsPrompts:
             self._price_lookup = c.fetch_prices()
         return self._price_lookup
 
-    def prompts(self) -> "Dataset":
+    def prompts(self, iterations=1) -> "Dataset":
         """Return a Dataset of prompts that will be used.
 
         >>> from edsl.jobs import Jobs
@@ -54,11 +57,11 @@ class JobsPrompts:
         models = []
         costs = []
         cache_keys = []
-        
+
         for interview_index, interview in enumerate(interviews):
             logger.info(f"Processing interview {interview_index} of {len(interviews)}")
             interview_start = time.time()
-            
+
             # Fetch invigilators timing
             invig_start = time.time()
             invigilators = [
@@ -66,8 +69,10 @@ class JobsPrompts:
                 for question in interview.survey.questions
             ]
             invig_end = time.time()
-            logger.debug(f"Time taken to fetch invigilators: {invig_end - invig_start:.4f}s")
-            
+            logger.debug(
+                f"Time taken to fetch invigilators: {invig_end - invig_start:.4f}s"
+            )
+
             # Process prompts timing
             prompts_start = time.time()
             for _, invigilator in enumerate(invigilators):
@@ -75,13 +80,15 @@ class JobsPrompts:
                 get_prompts_start = time.time()
                 prompts = invigilator.get_prompts()
                 get_prompts_end = time.time()
-                logger.debug(f"Time taken to get prompts: {get_prompts_end - get_prompts_start:.4f}s")
-                
+                logger.debug(
+                    f"Time taken to get prompts: {get_prompts_end - get_prompts_start:.4f}s"
+                )
+
                 user_prompt = prompts["user_prompt"]
                 system_prompt = prompts["system_prompt"]
                 user_prompts.append(user_prompt)
                 system_prompts.append(system_prompt)
-                
+
                 # Index lookups timing
                 index_start = time.time()
                 agent_index = self._agent_lookup[invigilator.agent]
@@ -90,14 +97,18 @@ class JobsPrompts:
                 scenario_index = self._scenario_lookup[invigilator.scenario]
                 scenario_indices.append(scenario_index)
                 index_end = time.time()
-                logger.debug(f"Time taken for index lookups: {index_end - index_start:.4f}s")
-                
+                logger.debug(
+                    f"Time taken for index lookups: {index_end - index_start:.4f}s"
+                )
+
                 # Model and question name assignment timing
                 assign_start = time.time()
                 models.append(invigilator.model.model)
                 question_names.append(invigilator.question.question_name)
                 assign_end = time.time()
-                logger.debug(f"Time taken for assignments: {assign_end - assign_start:.4f}s")
+                logger.debug(
+                    f"Time taken for assignments: {assign_end - assign_start:.4f}s"
+                )
 
                 # Cost estimation timing
                 cost_start = time.time()
@@ -109,32 +120,44 @@ class JobsPrompts:
                     model=invigilator.model.model,
                 )
                 cost_end = time.time()
-                logger.debug(f"Time taken to estimate prompt cost: {cost_end - cost_start:.4f}s")
+                logger.debug(
+                    f"Time taken to estimate prompt cost: {cost_end - cost_start:.4f}s"
+                )
                 costs.append(prompt_cost["cost_usd"])
 
                 # Cache key generation timing
                 cache_key_gen_start = time.time()
-                cache_key = CacheEntry.gen_key(
-                    model=invigilator.model.model,
-                    parameters=invigilator.model.parameters,
-                    system_prompt=system_prompt,
-                    user_prompt=user_prompt,
-                    iteration=0,
-                )
+                for iteration in range(iterations):
+                    cache_key = CacheEntry.gen_key(
+                        model=invigilator.model.model,
+                        parameters=invigilator.model.parameters,
+                        system_prompt=system_prompt,
+                        user_prompt=user_prompt,
+                        iteration=iteration,
+                    )
+                    cache_keys.append(cache_key)
+
                 cache_key_gen_end = time.time()
-                cache_keys.append(cache_key)
-                logger.debug(f"Time taken to generate cache key: {cache_key_gen_end - cache_key_gen_start:.4f}s")
+                logger.debug(
+                    f"Time taken to generate cache key: {cache_key_gen_end - cache_key_gen_start:.4f}s"
+                )
                 logger.debug("-" * 50)  # Separator between iterations
 
             prompts_end = time.time()
-            logger.info(f"Time taken to process prompts: {prompts_end - prompts_start:.4f}s")
-            
+            logger.info(
+                f"Time taken to process prompts: {prompts_end - prompts_start:.4f}s"
+            )
+
             interview_end = time.time()
-            logger.info(f"Overall time taken for interview: {interview_end - interview_start:.4f}s")
+            logger.info(
+                f"Overall time taken for interview: {interview_end - interview_start:.4f}s"
+            )
             logger.info("Time breakdown:")
             logger.info(f"  Invigilators: {invig_end - invig_start:.4f}s")
             logger.info(f"  Prompts processing: {prompts_end - prompts_start:.4f}s")
-            logger.info(f"  Other overhead: {(interview_end - interview_start) - ((invig_end - invig_start) + (prompts_end - prompts_start)):.4f}s")
+            logger.info(
+                f"  Other overhead: {(interview_end - interview_start) - ((invig_end - invig_start) + (prompts_end - prompts_start)):.4f}s"
+            )
 
         d = Dataset(
             [

edsl/jobs/JobsRemoteInferenceHandler.py

@@ -24,7 +24,7 @@ from edsl.jobs.JobsRemoteInferenceLogger import JobLogger
 class RemoteJobConstants:
     """Constants for remote job handling."""
 
-    REMOTE_JOB_POLL_INTERVAL = 1
+    REMOTE_JOB_POLL_INTERVAL = 4
     REMOTE_JOB_VERBOSE = False
     DISCORD_URL = "https://discord.com/invite/mxAYkjfy9m"
 
@@ -88,8 +88,8 @@ class JobsRemoteInferenceHandler:
         iterations: int = 1,
         remote_inference_description: Optional[str] = None,
         remote_inference_results_visibility: Optional[VisibilityType] = "unlisted",
+        fresh: Optional[bool] = False,
     ) -> RemoteJobInfo:
-
         from edsl.config import CONFIG
         from edsl.coop.coop import Coop
 
@@ -106,6 +106,7 @@ class JobsRemoteInferenceHandler:
             status="queued",
             iterations=iterations,
             initial_results_visibility=remote_inference_results_visibility,
+            fresh=fresh,
         )
         logger.update(
             "Your survey is running at the Expected Parrot server...",
@@ -277,9 +278,7 @@ class JobsRemoteInferenceHandler:
         job_in_queue = True
         while job_in_queue:
             result = self._attempt_fetch_job(
-                job_info,
-                remote_job_data_fetcher,
-                object_fetcher
+                job_info, remote_job_data_fetcher, object_fetcher
             )
             if result != "continue":
                 return result

edsl/jobs/data_structures.py

@@ -36,6 +36,9 @@ class RunParameters(Base):
     disable_remote_cache: bool = False
     disable_remote_inference: bool = False
     job_uuid: Optional[str] = None
+    fresh: Optional[
+        bool
+    ] = False  # if True, will not use cache and will save new results to cache
 
     def to_dict(self, add_edsl_version=False) -> dict:
         d = asdict(self)

edsl/jobs/interviews/Interview.py

@@ -238,9 +238,6 @@ class Interview:
         >>> run_config = RunConfig(parameters = RunParameters(), environment = RunEnvironment())
         >>> run_config.parameters.stop_on_exception = True
         >>> result, _ = asyncio.run(i.async_conduct_interview(run_config))
-        Traceback (most recent call last):
-        ...
-        asyncio.exceptions.CancelledError
         """
         from edsl.jobs.Jobs import RunConfig, RunParameters, RunEnvironment
 
@@ -262,6 +259,8 @@ class Interview:
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
             model_buckets = ModelBuckets.infinity_bucket()
 
+        self.skip_flags = {q.question_name: False for q in self.survey.questions}
+
         # was "self.tasks" - is that necessary?
         self.tasks = self.task_manager.build_question_tasks(
             answer_func=AnswerQuestionFunctionConstructor(
@@ -310,6 +309,10 @@ class Interview:
         def handle_task(task, invigilator):
             try:
                 result: Answers = task.result()
+                if result == "skipped":
+                    result = invigilator.get_failed_task_result(
+                        failure_reason="Task was skipped."
+                    )
             except asyncio.CancelledError as e:  # task was cancelled
                 result = invigilator.get_failed_task_result(
                     failure_reason="Task was cancelled."

edsl/language_models/LanguageModel.py

@@ -379,8 +379,10 @@ class LanguageModel(
         cached_response, cache_key = cache.fetch(**cache_call_params)
 
         if cache_used := cached_response is not None:
+            #           print("cache used")
             response = json.loads(cached_response)
         else:
+            #            print("cache not used")
             f = (
                 self.remote_async_execute_model_call
                 if hasattr(self, "remote") and self.remote
@@ -396,11 +398,14 @@ class LanguageModel(
             TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
             response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
             new_cache_key = cache.store(
-                **cache_call_params, response=response
+                **cache_call_params, response=response, service=self._inference_service_
             )  # store the response in the cache
             assert new_cache_key == cache_key  # should be the same
 
+        # breakpoint()
+
         cost = self.cost(response)
+        # breakpoint()
         return ModelResponse(
             response=response,
             cache_used=cache_used,
@@ -465,6 +470,7 @@ class LanguageModel(
             model_outputs=model_outputs,
             edsl_dict=edsl_dict,
         )
+        # breakpoint()
         return agent_response_dict
 
     get_response = sync_wrapper(async_get_response)

edsl/questions/QuestionBase.py

@@ -18,6 +18,7 @@ from edsl.questions.SimpleAskMixin import SimpleAskMixin
 from edsl.questions.QuestionBasePromptsMixin import QuestionBasePromptsMixin
 from edsl.questions.question_base_gen_mixin import QuestionBaseGenMixin
 from edsl.utilities.remove_edsl_version import remove_edsl_version
+from edsl.utilities.utilities import is_valid_variable_name
 
 if TYPE_CHECKING:
     from edsl.questions.response_validator_abc import ResponseValidatorABC
@@ -56,6 +57,10 @@ class QuestionBase(
     _answering_instructions = None
     _question_presentation = None
 
+    def is_valid_question_name(self) -> bool:
+        """Check if the question name is valid."""
+        return is_valid_variable_name(self.question_name)
+
     @property
     def response_validator(self) -> "ResponseValidatorABC":
         """Return the response validator."""

edsl/questions/question_base_gen_mixin.py

@@ -140,6 +140,8 @@ class QuestionBaseGenMixin:
             k: v for k, v in replacement_dict.items() if not isinstance(v, Scenario)
         }
 
+        strings_only_replacement_dict['scenario'] = strings_only_replacement_dict
+
         def _has_unrendered_variables(template_str: str, env: Environment) -> bool:
             """Check if the template string has any unrendered variables."""
             if not isinstance(template_str, str):