edsl 0.1.44__py3-none-any.whl → 0.1.46__py3-none-any.whl

edsl/jobs/JobsRemoteInferenceHandler.py

@@ -24,7 +24,7 @@ from edsl.jobs.JobsRemoteInferenceLogger import JobLogger
 class RemoteJobConstants:
     """Constants for remote job handling."""
 
-    REMOTE_JOB_POLL_INTERVAL = 1
+    REMOTE_JOB_POLL_INTERVAL = 4
     REMOTE_JOB_VERBOSE = False
     DISCORD_URL = "https://discord.com/invite/mxAYkjfy9m"
 
@@ -88,8 +88,8 @@ class JobsRemoteInferenceHandler:
         iterations: int = 1,
         remote_inference_description: Optional[str] = None,
         remote_inference_results_visibility: Optional[VisibilityType] = "unlisted",
+        fresh: Optional[bool] = False,
     ) -> RemoteJobInfo:
-
         from edsl.config import CONFIG
         from edsl.coop.coop import Coop
 
@@ -106,6 +106,7 @@ class JobsRemoteInferenceHandler:
             status="queued",
             iterations=iterations,
             initial_results_visibility=remote_inference_results_visibility,
+            fresh=fresh,
         )
         logger.update(
             "Your survey is running at the Expected Parrot server...",
@@ -277,9 +278,7 @@ class JobsRemoteInferenceHandler:
         job_in_queue = True
         while job_in_queue:
             result = self._attempt_fetch_job(
-                job_info,
-                remote_job_data_fetcher,
-                object_fetcher
+                job_info, remote_job_data_fetcher, object_fetcher
             )
             if result != "continue":
                 return result

edsl/jobs/async_interview_runner.py

@@ -7,6 +7,8 @@ from edsl.data_transfer_models import EDSLResultObjectInput
 
 from edsl.results.Result import Result
 from edsl.jobs.interviews.Interview import Interview
+from edsl.config import Config
+config = Config()
 
 if TYPE_CHECKING:
     from edsl.jobs.Jobs import Jobs
@@ -23,7 +25,7 @@ from edsl.jobs.data_structures import RunConfig
 
 
 class AsyncInterviewRunner:
-    MAX_CONCURRENT = 5
+    MAX_CONCURRENT = int(config.EDSL_MAX_CONCURRENT_TASKS)
 
     def __init__(self, jobs: "Jobs", run_config: RunConfig):
         self.jobs = jobs

edsl/jobs/check_survey_scenario_compatibility.py

@@ -72,11 +72,11 @@ class CheckSurveyScenarioCompatibility:
                 if warn:
                     warnings.warn(message)
 
-        if self.scenarios.has_jinja_braces:
-            warnings.warn(
-                "The scenarios have Jinja braces ({{ and }}). Converting to '<<' and '>>'. If you want a different conversion, use the convert_jinja_braces method first to modify the scenario."
-            )
-            self.scenarios = self.scenarios._convert_jinja_braces()
+        # if self.scenarios.has_jinja_braces:
+        #     warnings.warn(
+        #         "The scenarios have Jinja braces ({{ and }}). Converting to '<<' and '>>'. If you want a different conversion, use the convert_jinja_braces method first to modify the scenario."
+        #     )
+        #     self.scenarios = self.scenarios._convert_jinja_braces()
 
 
 if __name__ == "__main__":

edsl/jobs/data_structures.py

@@ -36,6 +36,9 @@ class RunParameters(Base):
     disable_remote_cache: bool = False
     disable_remote_inference: bool = False
     job_uuid: Optional[str] = None
+    fresh: Optional[
+        bool
+    ] = False  # if True, will not use cache and will save new results to cache
 
     def to_dict(self, add_edsl_version=False) -> dict:
         d = asdict(self)

edsl/jobs/interviews/Interview.py

@@ -238,9 +238,6 @@ class Interview:
         >>> run_config = RunConfig(parameters = RunParameters(), environment = RunEnvironment())
         >>> run_config.parameters.stop_on_exception = True
         >>> result, _ = asyncio.run(i.async_conduct_interview(run_config))
-        Traceback (most recent call last):
-        ...
-        asyncio.exceptions.CancelledError
         """
         from edsl.jobs.Jobs import RunConfig, RunParameters, RunEnvironment
 
@@ -262,6 +259,8 @@ class Interview:
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
             model_buckets = ModelBuckets.infinity_bucket()
 
+        self.skip_flags = {q.question_name: False for q in self.survey.questions}
+
         # was "self.tasks" - is that necessary?
         self.tasks = self.task_manager.build_question_tasks(
             answer_func=AnswerQuestionFunctionConstructor(
@@ -310,6 +309,10 @@ class Interview:
         def handle_task(task, invigilator):
             try:
                 result: Answers = task.result()
+                if result == "skipped":
+                    result = invigilator.get_failed_task_result(
+                        failure_reason="Task was skipped."
+                    )
             except asyncio.CancelledError as e:  # task was cancelled
                 result = invigilator.get_failed_task_result(
                     failure_reason="Task was cancelled."

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -166,6 +166,9 @@ class InterviewExceptionEntry:
         >>> entry = InterviewExceptionEntry.example()
         >>> _ = entry.to_dict()
         """
+        import json
+        from edsl.exceptions.questions import QuestionAnswerValidationError
+
         invigilator = (
             self.invigilator.to_dict() if self.invigilator is not None else None
         )
@@ -174,7 +177,16 @@ class InterviewExceptionEntry:
             "time": self.time,
             "traceback": self.traceback,
             "invigilator": invigilator,
+            "additional_data": {},
         }
+
+        if isinstance(self.exception, QuestionAnswerValidationError):
+            d["additional_data"]["edsl_response"] = json.dumps(self.exception.data)
+            d["additional_data"]["validating_model"] = json.dumps(
+                self.exception.model.model_json_schema()
+            )
+            d["additional_data"]["error_message"] = str(self.exception.message)
+
         return d
 
     @classmethod

edsl/jobs/tasks/TaskHistory.py

@@ -419,7 +419,7 @@ class TaskHistory(RepresentationMixin):
         filename: Optional[str] = None,
         return_link=False,
         css=None,
-        cta="\nClick to open the report in a new tab\n",
+        cta="<br><span style='font-size: 18px; font-weight: medium-bold; text-decoration: underline;'>Click to open the report in a new tab</span><br><br>",
         open_in_browser=False,
     ):
         """Return an HTML report."""

edsl/language_models/LanguageModel.py

@@ -379,8 +379,10 @@ class LanguageModel(
         cached_response, cache_key = cache.fetch(**cache_call_params)
 
         if cache_used := cached_response is not None:
+ #           print("cache used")
             response = json.loads(cached_response)
         else:
+#            print("cache not used")
             f = (
                 self.remote_async_execute_model_call
                 if hasattr(self, "remote") and self.remote
@@ -394,14 +396,16 @@ class LanguageModel(
             from edsl.config import CONFIG
 
             TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
-
             response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
             new_cache_key = cache.store(
                 **cache_call_params, response=response
             )  # store the response in the cache
             assert new_cache_key == cache_key  # should be the same
 
+        #breakpoint()
+
         cost = self.cost(response)
+        #breakpoint()
         return ModelResponse(
             response=response,
             cache_used=cache_used,
@@ -466,6 +470,7 @@ class LanguageModel(
             model_outputs=model_outputs,
             edsl_dict=edsl_dict,
         )
+        #breakpoint()
         return agent_response_dict
 
     get_response = sync_wrapper(async_get_response)
@@ -518,8 +523,6 @@ class LanguageModel(
         """
         from edsl.language_models.model import get_model_class
 
-        # breakpoint()
-
         model_class = get_model_class(
             data["model"], service_name=data.get("inference_service", None)
         )

edsl/language_models/PriceManager.py

@@ -30,19 +30,22 @@ class PriceManager:
         except Exception as e:
             print(f"Error fetching prices: {str(e)}")
 
-    def get_price(self, inference_service: str, model: str) -> Optional[Dict]:
+    def get_price(self, inference_service: str, model: str) -> Dict:
         """
         Get the price information for a specific service and model combination.
+        If no specific price is found, returns a fallback price.
 
         Args:
             inference_service (str): The name of the inference service
             model (str): The model identifier
 
         Returns:
-            Optional[Dict]: Price information if found, None otherwise
+            Dict: Price information (either actual or fallback prices)
         """
         key = (inference_service, model)
-        return self._price_lookup.get(key)
+        return self._price_lookup.get(key) or self._get_fallback_price(
+            inference_service
+        )
 
     def get_all_prices(self) -> Dict[Tuple[str, str], Dict]:
         """
@@ -53,6 +56,45 @@ class PriceManager:
         """
         return self._price_lookup.copy()
 
+    def _get_fallback_price(self, inference_service: str) -> Dict:
+        """
+        Get fallback prices for a service.
+        - First fallback: The highest input and output prices for that service from the price lookup.
+        - Second fallback: $1.00 per million tokens (for both input and output).
+
+        Args:
+            inference_service (str): The inference service name
+
+        Returns:
+            Dict: Price information
+        """
+        service_prices = [
+            prices
+            for (service, _), prices in self._price_lookup.items()
+            if service == inference_service
+        ]
+
+        input_tokens_per_usd = [
+            float(p["input"]["one_usd_buys"]) for p in service_prices if "input" in p
+        ]
+        if input_tokens_per_usd:
+            min_input_tokens = min(input_tokens_per_usd)
+        else:
+            min_input_tokens = 1_000_000
+
+        output_tokens_per_usd = [
+            float(p["output"]["one_usd_buys"]) for p in service_prices if "output" in p
+        ]
+        if output_tokens_per_usd:
+            min_output_tokens = min(output_tokens_per_usd)
+        else:
+            min_output_tokens = 1_000_000
+
+        return {
+            "input": {"one_usd_buys": min_input_tokens},
+            "output": {"one_usd_buys": min_output_tokens},
+        }
+
     def calculate_cost(
         self,
         inference_service: str,
@@ -75,8 +117,6 @@ class PriceManager:
             Union[float, str]: Total cost if calculation successful, error message string if not
         """
         relevant_prices = self.get_price(inference_service, model)
-        if relevant_prices is None:
-            return f"Could not find price for model {model} in the price lookup."
 
         # Extract token counts
         try:

edsl/language_models/model.py

@@ -17,7 +17,11 @@ if TYPE_CHECKING:
     from edsl.results.Dataset import Dataset
 
 
-def get_model_class(model_name, registry: Optional[InferenceServicesCollection] = None, service_name: Optional[InferenceServiceLiteral] = None):
+def get_model_class(
+    model_name,
+    registry: Optional[InferenceServicesCollection] = None,
+    service_name: Optional[InferenceServiceLiteral] = None,
+):
     from edsl.inference_services.registry import default
 
     registry = registry or default
@@ -40,6 +44,9 @@ class Meta(type):
         To get the default model, you can leave out the model name. 
         To see the available models, you can do:
         >>> Model.available()
+
+        Or to see the models for a specific service, you can do:
+        >>> Model.available(service='openai')
         """
         )
 
@@ -97,7 +104,10 @@ class Model(metaclass=Meta):
         *args,
         **kwargs,
     ):
-        "Instantiate a new language model."
+        """Instantiate a new language model.
+        >>> Model()
+        Model(...)
+        """
         # Map index to the respective subclass
         if model_name is None:
             model_name = cls.default_model
@@ -127,28 +137,25 @@ class Model(metaclass=Meta):
         >>> Model.service_classes()
         [...]
         """
-        return [r for r in cls.services(name_only=True)]
+        return [r for r in cls.services()]
 
     @classmethod
     def services(cls, name_only: bool = False) -> List[str]:
-        """Returns a list of services, annotated with whether the user has local keys for them."""
-        services_with_local_keys = set(cls.key_info().select("service").to_list())
-        f = lambda service_name: (
-            "yes" if service_name in services_with_local_keys else " "
-        )
-        if name_only:
-            return PrettyList(
-                [r._inference_service_ for r in cls.get_registry().services],
-                columns=["Service Name"],
-            )
-        else:
-            return PrettyList(
+        """Returns a list of services excluding 'test', sorted alphabetically.
+
+        >>> Model.services()
+        [...]
+        """
+        return PrettyList(
+            sorted(
                 [
-                    (r._inference_service_, f(r._inference_service_))
+                    [r._inference_service_]
                     for r in cls.get_registry().services
-                ],
-                columns=["Service Name", "Local key?"],
-            )
+                    if r._inference_service_.lower() != "test"
+                ]
+            ),
+            columns=["Service Name"],
+        )
 
     @classmethod
     def services_with_local_keys(cls) -> set:
@@ -198,7 +205,15 @@ class Model(metaclass=Meta):
         search_term: str = None,
         name_only: bool = False,
         service: Optional[str] = None,
+        force_refresh: bool = False,
     ):
+        """Get available models
+
+        >>> Model.available()
+        [...]
+        >>> Model.available(service='openai')
+        [...]
+        """
         # if search_term is None and service is None:
         #     print("Getting available models...")
         #     print("You have local keys for the following services:")
@@ -209,13 +224,16 @@ class Model(metaclass=Meta):
         #     return None
 
         if service is not None:
-            if service not in cls.services(name_only=True):
+            known_services = [x[0] for x in cls.services(name_only=True)]
+            if service not in known_services:
                 raise ValueError(
                     f"Service {service} not found in available services.",
-                    f"Available services are: {cls.services()}",
+                    f"Available services are: {known_services}",
                 )
 
-        full_list = cls.get_registry().available(service=service)
+        full_list = cls.get_registry().available(
+            service=service, force_refresh=force_refresh
+        )
 
         if search_term is None:
             if name_only:
@@ -319,6 +337,9 @@ class Model(metaclass=Meta):
         """
         Returns an example Model instance.
 
+        >>> Model.example()
+        Model(...)
+
         :param randomize: If True, the temperature is set to a random decimal between 0 and 1.
         """
         temperature = 0.5 if not randomize else round(random(), 2)
@@ -331,7 +352,7 @@ if __name__ == "__main__":
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)
 
-    available = Model.available()
-    m = Model("gpt-4-1106-preview")
-    results = m.execute_model_call("Hello world")
-    print(results)
+    # available = Model.available()
+    # m = Model("gpt-4-1106-preview")
+    # results = m.execute_model_call("Hello world")
+    # print(results)

edsl/questions/QuestionBase.py

@@ -85,6 +85,9 @@ class QuestionBase(
         >>> Q.example()._simulate_answer()
         {'answer': '...', 'generated_tokens': ...}
         """
+        if self.question_type == "free_text":
+            return {"answer": "Hello, how are you?", 'generated_tokens': "Hello, how are you?"}
+        
         simulated_answer = self.fake_data_factory.build().dict()
         if human_readable and hasattr(self, "question_options") and self.use_code:
             simulated_answer["answer"] = [
@@ -432,6 +435,24 @@ class QuestionBase(
 
         return Survey([self])
 
+    def humanize(
+        self,
+        project_name: str = "Project",
+        survey_description: Optional[str] = None,
+        survey_alias: Optional[str] = None,
+        survey_visibility: Optional["VisibilityType"] = "unlisted",
+    ) -> dict:
+        """
+        Turn a single question into a survey and send the survey to Coop.
+
+        Then, create a project on Coop so you can share the survey with human respondents.
+        """
+        s = self.to_survey()
+        project_details = s.humanize(
+            project_name, survey_description, survey_alias, survey_visibility
+        )
+        return project_details
+
     def by(self, *args) -> "Jobs":
         """Turn a single question into a survey and then a Job."""
         from edsl.surveys.Survey import Survey

edsl/questions/QuestionBasePromptsMixin.py

@@ -187,6 +187,73 @@ class QuestionBasePromptsMixin:
         from edsl.prompts import Prompt
 
         return Prompt(self.question_presentation) + Prompt(self.answering_instructions)
+    
+
+    def detailed_parameters_by_key(self) -> dict[str, set[tuple[str, ...]]]:
+        """
+        Return a dictionary of parameters by key.
+
+        >>> from edsl import QuestionMultipleChoice
+        >>> QuestionMultipleChoice.example().detailed_parameters_by_key()
+        {'question_name': set(), 'question_text': set()}
+
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "example", question_text = "What is your name, {{ nickname }}, based on {{ q0.answer }}?")
+        >>> r = q.detailed_parameters_by_key()
+        >>> r == {'question_name': set(), 'question_text': {('q0', 'answer'), ('nickname',)}}
+        True
+        """
+        params_by_key = {}
+        for key, value in self.data.items():
+            if isinstance(value, str):
+                params_by_key[key] = self.extract_parameters(value)
+        return params_by_key
+
+    @staticmethod
+    def extract_parameters(txt: str) -> set[tuple[str, ...]]:
+        """Return all parameters of the question as tuples representing their full paths.
+        
+        :param txt: The text to extract parameters from.
+        :return: A set of tuples representing the parameters.
+
+        >>> from edsl.questions import QuestionMultipleChoice
+        >>> d = QuestionMultipleChoice.example().extract_parameters("What is your name, {{ nickname }}, based on {{ q0.answer }}?")
+        >>> d =={('nickname',), ('q0', 'answer')}
+        True
+        """
+        from jinja2 import Environment, nodes
+
+        env = Environment()
+        #txt = self._all_text()
+        ast = env.parse(txt)
+        
+        variables = set()
+        processed_nodes = set()  # Keep track of nodes we've processed
+        
+        def visit_node(node, path=()):
+            if id(node) in processed_nodes:
+                return
+            processed_nodes.add(id(node))
+            
+            if isinstance(node, nodes.Name):
+                # Only add the name if we're not in the middle of building a longer path
+                if not path:
+                    variables.add((node.name,))
+                else:
+                    variables.add((node.name,) + path)
+            elif isinstance(node, nodes.Getattr):
+                # Build path from bottom up
+                new_path = (node.attr,) + path
+                visit_node(node.node, new_path)
+        
+        for node in ast.find_all((nodes.Name, nodes.Getattr)):
+            visit_node(node)
+
+        return variables
+    
+    @property
+    def detailed_parameters(self):
+        return [".".join(p) for p in self.extract_parameters(self._all_text())]
 
     @property
     def parameters(self) -> set[str]:
@@ -219,3 +286,39 @@ class QuestionBasePromptsMixin:
                 return self.new_default_instructions
             else:
                 return self.applicable_prompts(model)[0]()
+
+    @staticmethod
+    def sequence_in_dict(d: dict, path: tuple[str, ...]) -> tuple[bool, any]:
+        """Check if a sequence of nested keys exists in a dictionary and return the value.
+        
+        Args:
+            d: The dictionary to check
+            path: Tuple of keys representing the nested path
+            
+        Returns:
+            tuple[bool, any]: (True, value) if the path exists, (False, None) otherwise
+            
+        Example:
+            >>> sequence_in_dict = QuestionBasePromptsMixin.sequence_in_dict
+            >>> d = {'a': {'b': {'c': 1}}}
+            >>> sequence_in_dict(d, ('a', 'b', 'c'))
+            (True, 1)
+            >>> sequence_in_dict(d, ('a', 'b', 'd'))
+            (False, None)
+            >>> sequence_in_dict(d, ('x',))
+            (False, None)
+        """
+        try:
+            current = d
+            for key in path:
+                current = current.get(key)
+                if current is None:
+                    return (False, None)
+            return (True, current)
+        except (AttributeError, TypeError):
+            return (False, None)
+
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()

edsl/questions/QuestionFreeText.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 from typing import Any, Optional
 from uuid import uuid4
 
-from pydantic import field_validator
+from pydantic import field_validator, model_validator
 
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.response_validator_abc import ResponseValidatorABC
@@ -24,6 +24,17 @@ class FreeTextResponse(BaseModel):
     answer: str
     generated_tokens: Optional[str] = None
 
+    @model_validator(mode='after')
+    def validate_tokens_match_answer(self):
+        if self.generated_tokens is not None:  # If generated_tokens exists
+            # Ensure exact string equality
+            if self.answer.strip() != self.generated_tokens.strip():  # They MUST match exactly
+                raise ValueError(
+                    f"answer '{self.answer}' must exactly match generated_tokens '{self.generated_tokens}'. "
+                    f"Type of answer: {type(self.answer)}, Type of tokens: {type(self.generated_tokens)}"
+                )
+        return self
+
 
 class FreeTextResponseValidator(ResponseValidatorABC):
     required_params = []
@@ -37,10 +48,16 @@ class FreeTextResponseValidator(ResponseValidatorABC):
     ]
 
     def fix(self, response, verbose=False):
-        return {
-            "answer": str(response.get("generated_tokens")),
-            "generated_tokens": str(response.get("generated_tokens")),
-        }
+        if response.get("generated_tokens") != response.get("answer"):
+            return {
+                "answer": str(response.get("generated_tokens")),
+                "generated_tokens": str(response.get("generated_tokens")),
+            }
+        else:
+            return {
+                "answer": str(response.get("generated_tokens")),
+                "generated_tokens": str(response.get("generated_tokens")),
+            }
 
 
 class QuestionFreeText(QuestionBase):

edsl/questions/descriptors.py

@@ -2,6 +2,7 @@
 
 from abc import ABC, abstractmethod
 import re
+import textwrap
 from typing import Any, Callable, List, Optional
 from edsl.exceptions.questions import (
     QuestionCreationValidationError,
@@ -404,6 +405,9 @@ class QuestionTextDescriptor(BaseDescriptor):
             raise Exception("Question is too short!")
         if not isinstance(value, str):
             raise Exception("Question must be a string!")
+        
+        #value = textwrap.dedent(value).strip()
+
         if contains_single_braced_substring(value):
             import warnings