edsl 0.1.41__py3-none-any.whl → 0.1.43__py3-none-any.whl

edsl/language_models/LanguageModel.py

@@ -244,7 +244,7 @@ class LanguageModel(
 
         >>> m = LanguageModel.example()
         >>> hash(m)
-        1811901442659237949
+        325654563661254408
         """
         from edsl.utilities.utilities import dict_hash
 
@@ -495,11 +495,12 @@ class LanguageModel(
 
         >>> m = LanguageModel.example()
         >>> m.to_dict()
-        {'model': '...', 'parameters': {'temperature': ..., 'max_tokens': ..., 'top_p': ..., 'frequency_penalty': ..., 'presence_penalty': ..., 'logprobs': False, 'top_logprobs': ...}, 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
+        {'model': '...', 'parameters': {'temperature': ..., 'max_tokens': ..., 'top_p': ..., 'frequency_penalty': ..., 'presence_penalty': ..., 'logprobs': False, 'top_logprobs': ...}, 'inference_service': 'openai', 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
         """
         d = {
             "model": self.model,
             "parameters": self.parameters,
+            "inference_service": self._inference_service_,
         }
         if add_edsl_version:
             from edsl import __version__
@@ -511,7 +512,10 @@ class LanguageModel(
     @classmethod
     @remove_edsl_version
     def from_dict(cls, data: dict) -> Type[LanguageModel]:
-        """Convert dictionary to a LanguageModel child instance."""
+        """Convert dictionary to a LanguageModel child instance.
+
+        NB: This method does not use the stores inference_service but rather just fetches a model class based on the name.
+        """
         from edsl.language_models.model import get_model_class
 
         model_class = get_model_class(data["model"])
@@ -558,7 +562,6 @@ class LanguageModel(
         >>> m = LanguageModel.example(test_model = True, canned_response = "WOWZA!", throw_exception = True)
         >>> r = q.by(m).run(cache = False, disable_remote_cache = True, disable_remote_inference = True, print_exceptions = True)
         Exception report saved to ...
-        Also see: ...
         """
         from edsl.language_models.model import Model
 

edsl/language_models/ModelList.py

@@ -60,7 +60,7 @@ class ModelList(Base, UserList):
 
         sl = ScenarioList()
         for model in self:
-            d = {"model": model.model}
+            d = {"model": model.model, "inference_service": model._inference_service_}
             d.update(model.parameters)
             sl.append(Scenario(d))
         return sl

edsl/language_models/key_management/KeyLookupBuilder.py

@@ -61,7 +61,14 @@ class KeyLookupBuilder:
     DEFAULT_RPM = int(CONFIG.get("EDSL_SERVICE_RPM_BASELINE"))
     DEFAULT_TPM = int(CONFIG.get("EDSL_SERVICE_TPM_BASELINE"))
 
-    def __init__(self, fetch_order: Optional[tuple[str]] = None):
+    def __init__(
+        self,
+        fetch_order: Optional[tuple[str]] = None,
+        coop: Optional["Coop"] = None,
+    ):
+        from edsl.coop import Coop
+
+        # Fetch order goes from lowest priority to highest priority
         if fetch_order is None:
             self.fetch_order = ("config", "env")
         else:
@@ -70,6 +77,11 @@ class KeyLookupBuilder:
         if not isinstance(self.fetch_order, tuple):
             raise ValueError("fetch_order must be a tuple")
 
+        if coop is None:
+            self.coop = Coop()
+        else:
+            self.coop = coop
+
         self.limit_data = {}
         self.key_data = {}
         self.id_data = {}
@@ -131,7 +143,8 @@ class KeyLookupBuilder:
                 service=service,
                 rpm=self.DEFAULT_RPM,
                 tpm=self.DEFAULT_TPM,
-                source="default",
+                rpm_source="default",
+                tpm_source="default",
             )
 
         if limit_entry.rpm is None:
@@ -145,7 +158,8 @@ class KeyLookupBuilder:
             tpm=int(limit_entry.tpm),
             api_id=api_id,
             token_source=api_key_entry.source,
-            limit_source=limit_entry.source,
+            rpm_source=limit_entry.rpm_source,
+            tpm_source=limit_entry.tpm_source,
             id_source=id_source,
         )
 
@@ -156,10 +170,7 @@ class KeyLookupBuilder:
         return dict(list(os.environ.items()))
 
     def _coop_key_value_pairs(self):
-        from edsl.coop import Coop
-
-        c = Coop()
-        return dict(list(c.fetch_rate_limit_config_vars().items()))
+        return dict(list(self.coop.fetch_rate_limit_config_vars().items()))
 
     def _config_key_value_pairs(self):
         from edsl.config import CONFIG
@@ -169,7 +180,7 @@ class KeyLookupBuilder:
     @staticmethod
     def extract_service(key: str) -> str:
         """Extract the service and limit type from the key"""
-        limit_type, service_raw = key.replace("EDSL_SERVICE_", "").split("_")
+        limit_type, service_raw = key.replace("EDSL_SERVICE_", "").split("_", 1)
         return service_raw.lower(), limit_type.lower()
 
     def get_key_value_pairs(self) -> dict:
@@ -187,17 +198,17 @@ class KeyLookupBuilder:
                 d[k] = (v, source)
         return d
 
-    def _entry_type(self, key, value) -> str:
+    def _entry_type(self, key: str) -> str:
         """Determine the type of entry from a key.
 
         >>> builder = KeyLookupBuilder()
-        >>> builder._entry_type("EDSL_SERVICE_RPM_OPENAI", "60")
+        >>> builder._entry_type("EDSL_SERVICE_RPM_OPENAI")
         'limit'
-        >>> builder._entry_type("OPENAI_API_KEY", "sk-1234")
+        >>> builder._entry_type("OPENAI_API_KEY")
         'api_key'
-        >>> builder._entry_type("AWS_ACCESS_KEY_ID", "AKIA1234")
+        >>> builder._entry_type("AWS_ACCESS_KEY_ID")
         'api_id'
-        >>> builder._entry_type("UNKNOWN_KEY", "value")
+        >>> builder._entry_type("UNKNOWN_KEY")
         'unknown'
         """
         if key.startswith("EDSL_SERVICE_"):
@@ -243,11 +254,13 @@ class KeyLookupBuilder:
         service, limit_type = self.extract_service(key)
         if service in self.limit_data:
             setattr(self.limit_data[service], limit_type.lower(), value)
+            setattr(self.limit_data[service], f"{limit_type}_source", source)
         else:
             new_limit_entry = LimitEntry(
-                service=service, rpm=None, tpm=None, source=source
+                service=service, rpm=None, tpm=None, rpm_source=None, tpm_source=None
             )
             setattr(new_limit_entry, limit_type.lower(), value)
+            setattr(new_limit_entry, f"{limit_type}_source", source)
             self.limit_data[service] = new_limit_entry
 
     def _add_api_key(self, key: str, value: str, source: str) -> None:
@@ -265,13 +278,27 @@ class KeyLookupBuilder:
         else:
             self.key_data[service].append(new_entry)
 
-    def process_key_value_pairs(self) -> None:
-        """Process all key-value pairs from the configured sources."""
-        for key, value_pair in self.get_key_value_pairs().items():
+    def update_from_dict(self, d: dict) -> None:
+        """
+        Update data from a dictionary of key-value pairs.
+        Each key is a key name, and each value is a tuple of (value, source).
+
+        >>> builder = KeyLookupBuilder()
+        >>> builder.update_from_dict({"OPENAI_API_KEY": ("sk-1234", "custodial_keys")})
+        >>> 'sk-1234' == builder.key_data["openai"][-1].value
+        True
+        >>> 'custodial_keys' == builder.key_data["openai"][-1].source
+        True
+        """
+        for key, value_pair in d.items():
             value, source = value_pair
-            if (entry_type := self._entry_type(key, value)) == "limit":
+            if self._entry_type(key) == "limit":
                 self._add_limit(key, value, source)
-            elif entry_type == "api_key":
+            elif self._entry_type(key) == "api_key":
                 self._add_api_key(key, value, source)
-            elif entry_type == "api_id":
+            elif self._entry_type(key) == "api_id":
                 self._add_id(key, value, source)
+
+    def process_key_value_pairs(self) -> None:
+        """Process all key-value pairs from the configured sources."""
+        self.update_from_dict(self.get_key_value_pairs())

edsl/language_models/key_management/models.py

@@ -40,18 +40,23 @@ class LimitEntry:
     60
     >>> limit.tpm
     100000
-    >>> limit.source
+    >>> limit.rpm_source
     'config'
+    >>> limit.tpm_source
+    'env'
     """
 
     service: str
     rpm: int
     tpm: int
-    source: Optional[str] = None
+    rpm_source: Optional[str] = None
+    tpm_source: Optional[str] = None
 
     @classmethod
     def example(cls):
-        return LimitEntry(service="openai", rpm=60, tpm=100000, source="config")
+        return LimitEntry(
+            service="openai", rpm=60, tpm=100000, rpm_source="config", tpm_source="env"
+        )
 
 
 @dataclass
@@ -108,7 +113,8 @@ class LanguageModelInput:
     tpm: int
     api_id: Optional[str] = None
     token_source: Optional[str] = None
-    limit_source: Optional[str] = None
+    rpm_source: Optional[str] = None
+    tpm_source: Optional[str] = None
     id_source: Optional[str] = None
 
     def to_dict(self):

edsl/language_models/model.py

@@ -233,6 +233,55 @@ class Model(metaclass=Meta):
             print("OK!")
             print("\n")
 
+    @classmethod
+    def check_working_models(
+        cls,
+        service: Optional[str] = None,
+        works_with_text: Optional[bool] = None,
+        works_with_images: Optional[bool] = None,
+    ) -> list[dict]:
+        from edsl.coop import Coop
+
+        c = Coop()
+        working_models = c.fetch_working_models()
+
+        if service is not None:
+            working_models = [m for m in working_models if m["service"] == service]
+        if works_with_text is not None:
+            working_models = [
+                m for m in working_models if m["works_with_text"] == works_with_text
+            ]
+        if works_with_images is not None:
+            working_models = [
+                m for m in working_models if m["works_with_images"] == works_with_images
+            ]
+
+        if len(working_models) == 0:
+            return []
+
+        else:
+            return PrettyList(
+                [
+                    [
+                        m["service"],
+                        m["model"],
+                        m["works_with_text"],
+                        m["works_with_images"],
+                        m["usd_per_1M_input_tokens"],
+                        m["usd_per_1M_output_tokens"],
+                    ]
+                    for m in working_models
+                ],
+                columns=[
+                    "Service",
+                    "Model",
+                    "Works with text",
+                    "Works with images",
+                    "Price per 1M input tokens (USD)",
+                    "Price per 1M output tokens (USD)",
+                ],
+            )
+
     @classmethod
     def example(cls, randomize: bool = False) -> "Model":
         """

edsl/prompts/Prompt.py

@@ -10,6 +10,48 @@ from edsl.Base import PersistenceMixin, RepresentationMixin
 
 MAX_NESTING = 100
 
+from jinja2 import Environment, meta, TemplateSyntaxError, Undefined
+from functools import lru_cache
+
+class PreserveUndefined(Undefined):
+    def __str__(self):
+        return "{{ " + str(self._undefined_name) + " }}"
+
+# Create environment once at module level
+_env = Environment(undefined=PreserveUndefined)
+
+@lru_cache(maxsize=1024)
+def _compile_template(text: str):
+    return _env.from_string(text)
+
+@lru_cache(maxsize=1024)
+def _find_template_variables(template: str) -> list[str]:
+    """Find and return the template variables."""
+    ast = _env.parse(template)
+    return list(meta.find_undeclared_variables(ast))
+
+def _make_hashable(value):
+    """Convert unhashable types to hashable ones."""
+    if isinstance(value, list):
+        return tuple(_make_hashable(item) for item in value)
+    if isinstance(value, dict):
+        return frozenset((k, _make_hashable(v)) for k, v in value.items())
+    return value
+
+@lru_cache(maxsize=1024)
+def _cached_render(text: str, frozen_replacements: frozenset) -> str:
+    """Cached version of template rendering with frozen replacements."""
+    # Print cache info on every call
+    cache_info = _cached_render.cache_info()
+    print(f"\t\t\t\t\t Cache status - hits: {cache_info.hits}, misses: {cache_info.misses}, current size: {cache_info.currsize}")
+    
+    # Convert back to dict with original types for rendering
+    replacements = {k: v for k, v in frozen_replacements}
+    
+    template = _compile_template(text)
+    result = template.render(replacements)
+    
+    return result
 
 class Prompt(PersistenceMixin, RepresentationMixin):
     """Class for creating a prompt to be used in a survey."""
@@ -145,33 +187,8 @@ class Prompt(PersistenceMixin, RepresentationMixin):
         return f'Prompt(text="""{self.text}""")'
 
     def template_variables(self) -> list[str]:
-        """Return the the variables in the template.
-
-        Example:
-
-        >>> p = Prompt("Hello, {{person}}")
-        >>> p.template_variables()
-        ['person']
-
-        """
-        return self._template_variables(self.text)
-
-    @staticmethod
-    def _template_variables(template: str) -> list[str]:
-        """Find and return the template variables.
-
-        :param template: The template to find the variables in.
-
-        """
-        from jinja2 import Environment, meta, Undefined
-
-        class PreserveUndefined(Undefined):
-            def __str__(self):
-                return "{{ " + str(self._undefined_name) + " }}"
-
-        env = Environment(undefined=PreserveUndefined)
-        ast = env.parse(template)
-        return list(meta.find_undeclared_variables(ast))
+        """Return the variables in the template."""
+        return _find_template_variables(self.text)
 
     def undefined_template_variables(self, replacement_dict: dict):
         """Return the variables in the template that are not in the replacement_dict.
@@ -239,45 +256,39 @@ class Prompt(PersistenceMixin, RepresentationMixin):
             return self
 
     @staticmethod
-    def _render(
-        text: str, primary_replacement, **additional_replacements
-    ) -> "PromptBase":
-        """Render the template text with variables replaced from the provided named dictionaries.
-
-        :param text: The text to render.
-        :param primary_replacement: The primary replacement dictionary.
-        :param additional_replacements: Additional replacement dictionaries.
-
-        Allows for nested variable resolution up to a specified maximum nesting depth.
-
-        Example:
-
-        >>> codebook = {"age": "Age"}
-        >>> p = Prompt("You are an agent named {{ name }}. {{ codebook['age']}}: {{ age }}")
-        >>> p.render({"name": "John", "age": 44}, codebook=codebook)
-        Prompt(text=\"""You are an agent named John. Age: 44\""")
-        """
-        from jinja2 import Environment, meta, TemplateSyntaxError, Undefined
-
-        class PreserveUndefined(Undefined):
-            def __str__(self):
-                return "{{ " + str(self._undefined_name) + " }}"
-
-        env = Environment(undefined=PreserveUndefined)
+    def _render(text: str, primary_replacement, **additional_replacements) -> "PromptBase":
+        """Render the template text with variables replaced."""
+        import time
+        
+        # if there are no replacements, return the text
+        if not primary_replacement and not additional_replacements:
+            return text
+     
         try:
+            variables = _find_template_variables(text)
+            
+            if not variables: # if there are no variables, return the text
+                return text
+            
+            # Combine all replacements
+            all_replacements = {**primary_replacement, **additional_replacements}
+            
             previous_text = None
+            current_text = text
+            iteration = 0
+            
             for _ in range(MAX_NESTING):
-                # breakpoint()
-                rendered_text = env.from_string(text).render(
-                    primary_replacement, **additional_replacements
-                )
-                if rendered_text == previous_text:
-                    # No more changes, so return the rendered text
+                iteration += 1
+                
+                template = _compile_template(current_text)
+                rendered_text = template.render(all_replacements)
+                
+                if rendered_text == current_text:
                     return rendered_text
-                previous_text = text
-                text = rendered_text
+                    
+                previous_text = current_text
+                current_text = rendered_text
 
-            # If the loop exits without returning, it indicates too much nesting
             raise TemplateRenderError(
                 "Too much nesting - you created an infinite loop here, pal"
             )
@@ -331,6 +342,58 @@ class Prompt(PersistenceMixin, RepresentationMixin):
         """Return an example of the prompt."""
         return cls(cls.default_instructions)
 
+    def get_prompts(self) -> Dict[str, Any]:
+        """Get the prompts for the question."""
+        start = time.time()
+        
+        # Build all the components
+        instr_start = time.time()
+        agent_instructions = self.agent_instructions_prompt
+        instr_end = time.time()
+        logger.debug(f"Time taken for agent instructions: {instr_end - instr_start:.4f}s")
+        
+        persona_start = time.time()
+        agent_persona = self.agent_persona_prompt
+        persona_end = time.time()
+        logger.debug(f"Time taken for agent persona: {persona_end - persona_start:.4f}s")
+        
+        q_instr_start = time.time()
+        question_instructions = self.question_instructions_prompt
+        q_instr_end = time.time()
+        logger.debug(f"Time taken for question instructions: {q_instr_end - q_instr_start:.4f}s")
+        
+        memory_start = time.time()
+        prior_question_memory = self.prior_question_memory_prompt
+        memory_end = time.time()
+        logger.debug(f"Time taken for prior question memory: {memory_end - memory_start:.4f}s")
+
+        # Get components dict
+        components = {
+            "agent_instructions": agent_instructions.text,
+            "agent_persona": agent_persona.text,
+            "question_instructions": question_instructions.text,
+            "prior_question_memory": prior_question_memory.text,
+        }
+
+        # Use PromptPlan's get_prompts method
+        plan_start = time.time()
+        prompts = self.prompt_plan.get_prompts(**components)
+        plan_end = time.time()
+        logger.debug(f"Time taken for prompt processing: {plan_end - plan_start:.4f}s")
+        
+        # Handle file keys if present
+        if hasattr(self, 'question_file_keys') and self.question_file_keys:
+            files_start = time.time()
+            files_list = []
+            for key in self.question_file_keys:
+                files_list.append(self.scenario[key])
+            prompts["files_list"] = files_list
+            files_end = time.time()
+            logger.debug(f"Time taken for file key processing: {files_end - files_start:.4f}s")
+        
+        end = time.time()
+        logger.debug(f"Total time in get_prompts: {end - start:.4f}s")
+        return prompts
 
 if __name__ == "__main__":
     print("Running doctests...")

edsl/questions/descriptors.py

@@ -249,7 +249,28 @@ class QuestionNameDescriptor(BaseDescriptor):
 
 
 class QuestionOptionsDescriptor(BaseDescriptor):
-    """Validate that `question_options` is a list, does not exceed the min/max lengths, and has unique items."""
+    """Validate that `question_options` is a list, does not exceed the min/max lengths, and has unique items.
+
+    >>> import warnings
+    >>> q_class = QuestionOptionsDescriptor.example()
+    >>> with warnings.catch_warnings(record=True) as w:
+    ...     _ = q_class(["a ", "b", "c"])  # Has trailing space
+    ...     assert len(w) == 1
+    ...     assert "trailing whitespace" in str(w[0].message)
+
+    >>> _ = q_class(["a", "b", "c", "d", "d"])
+    Traceback (most recent call last):
+    ...
+    edsl.exceptions.questions.QuestionCreationValidationError: Question options must be unique (got ['a', 'b', 'c', 'd', 'd']).
+
+    We allow dynamic question options, which are strings of the form '{{ question_options }}'.
+
+    >>> _ = q_class("{{dynamic_options}}")
+    >>> _ = q_class("dynamic_options")
+    Traceback (most recent call last):
+    ...
+    edsl.exceptions.questions.QuestionCreationValidationError: ...
+    """
 
     @classmethod
     def example(cls):
@@ -273,23 +294,7 @@ class QuestionOptionsDescriptor(BaseDescriptor):
         self.q_budget = q_budget
 
     def validate(self, value: Any, instance) -> None:
-        """Validate the question options.
-
-        >>> q_class = QuestionOptionsDescriptor.example()
-        >>> _ = q_class(["a", "b", "c"])
-        >>> _ = q_class(["a", "b", "c", "d", "d"])
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionCreationValidationError: Question options must be unique (got ['a', 'b', 'c', 'd', 'd']).
-
-        We allow dynamic question options, which are strings of the form '{{ question_options }}'.
-
-        >>> _ = q_class("{{dynamic_options}}")
-        >>> _ = q_class("dynamic_options")
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionCreationValidationError: ...
-        """
+        """Validate the question options."""
         if isinstance(value, str):
             # Check if the string is a dynamic question option
             if "{{" in value and "}}" in value:
@@ -302,10 +307,10 @@ class QuestionOptionsDescriptor(BaseDescriptor):
             raise QuestionCreationValidationError(
                 f"Question options must be a list (got {value})."
             )
-        if len(value) > Settings.MAX_NUM_OPTIONS:
-            raise QuestionCreationValidationError(
-                f"Too many question options (got {value})."
-            )
+        # if len(value) > Settings.MAX_NUM_OPTIONS:
+        #     raise QuestionCreationValidationError(
+        #         f"Too many question options (got {value})."
+        #     )
         if len(value) < Settings.MIN_NUM_OPTIONS:
             raise QuestionCreationValidationError(
                 f"Too few question options (got {value})."
@@ -343,6 +348,15 @@ class QuestionOptionsDescriptor(BaseDescriptor):
                     f"All question options must be at least 1 character long but less than {Settings.MAX_OPTION_LENGTH} characters long (got {value})."
                 )
 
+            # Check for trailing whitespace in string options
+            if any(isinstance(x, str) and (x != x.strip()) for x in value):
+                import warnings
+
+                warnings.warn(
+                    "Some question options contain trailing whitespace. This may cause unexpected behavior.",
+                    UserWarning,
+                )
+
         if hasattr(instance, "min_selections") and instance.min_selections != None:
             if instance.min_selections > len(value):
                 raise QuestionCreationValidationError(
@@ -408,7 +422,7 @@ class QuestionTextDescriptor(BaseDescriptor):
             # Automatically replace single braces with double braces
             # This is here because if the user is using an f-string, the double brace will get converted to a single brace.
             # This undoes that.
-            value = re.sub(r"\{([^\{\}]+)\}", r"{{\1}}", value)
+            # value = re.sub(r"\{([^\{\}]+)\}", r"{{\1}}", value)
             return value
 
         # iterate through all doubles braces and check if they are valid python identifiers

edsl/questions/question_base_gen_mixin.py

@@ -114,6 +114,7 @@ class QuestionBaseGenMixin:
                         .render(strings_only_replacement_dict)
                     )
                 except Exception as e:
+                    #breakpoint()
                     import warnings
 
                     warnings.warn("Failed to render string: " + value)