PyPI - edsl - Versions diffs - 0.1.31__py3-none-any.whl → 0.1.31.dev2__py3-none-any.whl - Mend

edsl 0.1.31py3-none-any.whl → 0.1.31.dev2py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (37) hide show

edsl/__version__.py +1 -1
edsl/agents/Invigilator.py +2 -7
edsl/agents/PromptConstructionMixin.py +4 -9
edsl/config.py +0 -4
edsl/conjure/Conjure.py +0 -6
edsl/coop/coop.py +0 -4
edsl/data/CacheHandler.py +4 -3
edsl/enums.py +0 -2
edsl/inference_services/DeepInfraService.py +91 -6
edsl/inference_services/InferenceServicesCollection.py +8 -13
edsl/inference_services/OpenAIService.py +21 -64
edsl/inference_services/registry.py +1 -2
edsl/jobs/Jobs.py +5 -29
edsl/jobs/buckets/TokenBucket.py +4 -12
edsl/jobs/interviews/Interview.py +9 -31
edsl/jobs/interviews/InterviewTaskBuildingMixin.py +33 -49
edsl/jobs/interviews/interview_exception_tracking.py +10 -68
edsl/jobs/runners/JobsRunnerAsyncio.py +81 -112
edsl/jobs/runners/JobsRunnerStatusData.py +237 -0
edsl/jobs/runners/JobsRunnerStatusMixin.py +35 -291
edsl/jobs/tasks/TaskCreators.py +2 -8
edsl/jobs/tasks/TaskHistory.py +1 -145
edsl/language_models/LanguageModel.py +32 -49
edsl/language_models/registry.py +0 -4
edsl/questions/QuestionMultipleChoice.py +1 -1
edsl/questions/QuestionNumerical.py +1 -0
edsl/results/DatasetExportMixin.py +3 -12
edsl/scenarios/Scenario.py +0 -14
edsl/scenarios/ScenarioList.py +2 -15
edsl/scenarios/ScenarioListExportMixin.py +4 -15
edsl/scenarios/ScenarioListPdfMixin.py +0 -3
{edsl-0.1.31.dist-info → edsl-0.1.31.dev2.dist-info}/METADATA +1 -2
{edsl-0.1.31.dist-info → edsl-0.1.31.dev2.dist-info}/RECORD +35 -37
edsl/inference_services/GroqService.py +0 -18
edsl/jobs/interviews/InterviewExceptionEntry.py +0 -101
{edsl-0.1.31.dist-info → edsl-0.1.31.dev2.dist-info}/LICENSE +0 -0
{edsl-0.1.31.dist-info → edsl-0.1.31.dev2.dist-info}/WHEEL +0 -0

edsl/jobs/tasks/TaskHistory.py CHANGED Viewed

@@ -11,8 +11,6 @@ class TaskHistory:
         [Interview.exceptions, Interview.exceptions, Interview.exceptions, ...]
-        >>> _ = TaskHistory.example()
-        ...
         """
         self.total_interviews = interviews
@@ -20,26 +18,8 @@ class TaskHistory:
         self._interviews = {index: i for index, i in enumerate(self.total_interviews)}
-    @classmethod
-    def example(cls):
-        from edsl.jobs.interviews.Interview import Interview
-        from edsl.jobs.Jobs import Jobs
-        j = Jobs.example(throw_exception_probability=1, test_model=True)
-        from edsl.config import CONFIG
-        results = j.run(print_exceptions=False, skip_retry=True, cache = False)
-        return cls(results.task_history.total_interviews)
     @property
     def exceptions(self):
-        """
-        >>> len(TaskHistory.example().exceptions)
-        4
-        """
         return [i.exceptions for k, i in self._interviews.items() if i.exceptions != {}]
     @property
@@ -62,12 +42,7 @@ class TaskHistory:
     @property
     def has_exceptions(self) -> bool:
-        """Return True if there are any exceptions.
-        >>> TaskHistory.example().has_exceptions
-        True
-        """
+        """Return True if there are any exceptions."""
         return len(self.exceptions) > 0
     def _repr_html_(self):
@@ -241,47 +216,6 @@ class TaskHistory:
         }
         """
-    @property
-    def exceptions_by_type(self) -> dict:
-        """Return a dictionary of exceptions by type."""
-        exceptions_by_type = {}
-        for interview in self.total_interviews:
-            for question_name, exceptions in interview.exceptions.items():
-                for exception in exceptions:
-                    exception_type = exception["exception"]
-                    if exception_type in exceptions_by_type:
-                        exceptions_by_type[exception_type] += 1
-                    else:
-                        exceptions_by_type[exception_type] = 1
-        return exceptions_by_type
-    @property
-    def exceptions_by_question_name(self) -> dict:
-        """Return a dictionary of exceptions tallied by question name."""
-        exceptions_by_question_name = {}
-        for interview in self.total_interviews:
-            for question_name, exceptions in interview.exceptions.items():
-                if question_name not in exceptions_by_question_name:
-                    exceptions_by_question_name[question_name] = 0
-                exceptions_by_question_name[question_name] += len(exceptions)
-        for question in self.total_interviews[0].survey.questions:
-            if question.question_name not in exceptions_by_question_name:
-                exceptions_by_question_name[question.question_name] = 0
-        return exceptions_by_question_name
-    @property
-    def exceptions_by_model(self) -> dict:
-        """Return a dictionary of exceptions tallied by model and question name."""
-        exceptions_by_model = {}
-        for interview in self.total_interviews:
-            model = interview.model
-            if model not in exceptions_by_model:
-                exceptions_by_model[model.model] = 0
-            if interview.exceptions != {}:
-                exceptions_by_model[model.model] += len(interview.exceptions)
-        return exceptions_by_model
     def html(
         self,
         filename: Optional[str] = None,
@@ -302,8 +236,6 @@ class TaskHistory:
         if css is None:
             css = self.css()
-        models_used = set([i.model for index, i in self._interviews.items()])
         template = Template(
             """
         <!DOCTYPE html>
@@ -317,69 +249,6 @@ class TaskHistory:
         </style>
         </head>
         <body>
-            <h1>Overview</h1>
-            <p>There were {{ interviews|length }} total interviews. The number of interviews with exceptions was {{ num_exceptions }}.</p>
-            <p>The models used were: {{ models_used }}.</p>
-            <p>For documentation on dealing with exceptions on Expected Parrot,
-            see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>
-            <h2>Exceptions by Type</h2>
-            <table>
-                <thead>
-                    <tr>
-                        <th>Exception Type</th>
-                        <th>Number</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for exception_type, exceptions in exceptions_by_type.items() %}
-                        <tr>
-                            <td>{{ exception_type }}</td>
-                            <td>{{ exceptions }}</td>
-                        </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
-            <h2>Exceptions by Model</h2>
-            <table>
-                <thead>
-                    <tr>
-                        <th>Model</th>
-                        <th>Number</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for model, exceptions in exceptions_by_model.items() %}
-                        <tr>
-                            <td>{{ model }}</td>
-                            <td>{{ exceptions }}</td>
-                        </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
-            <h2>Exceptions by Question Name</h2>
-            <table>
-                <thead>
-                    <tr>
-                        <th>Question Name</th>
-                        <th>Number of Exceptions</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for question_name, exception_count in exceptions_by_question_name.items() %}
-                        <tr>
-                            <td>{{ question_name }}</td>
-                            <td>{{ exception_count }}</td>
-                        </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
             {% for index, interview in interviews.items() %}
                 {% if interview.exceptions != {} %}
                    <div class="interview">Interview: {{ index }} </div>
@@ -427,18 +296,11 @@ class TaskHistory:
         """
         )
-        # breakpoint()
         # Render the template with data
         output = template.render(
             interviews=self._interviews,
             css=css,
-            num_exceptions=len(self.exceptions),
             performance_plot_html=performance_plot_html,
-            exceptions_by_type=self.exceptions_by_type,
-            exceptions_by_question_name=self.exceptions_by_question_name,
-            exceptions_by_model=self.exceptions_by_model,
-            models_used=models_used,
         )
         # Save the rendered output to a file
@@ -482,9 +344,3 @@ class TaskHistory:
         if return_link:
             return filename
-if __name__ == "__main__":
-    import doctest
-    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/language_models/LanguageModel.py CHANGED Viewed

@@ -11,7 +11,6 @@ import hashlib
 from typing import Coroutine, Any, Callable, Type, List, get_type_hints
 from abc import ABC, abstractmethod
 class IntendedModelCallOutcome:
     "This is a tuple-like class that holds the response, cache_used, and cache_key."
@@ -22,7 +21,7 @@ class IntendedModelCallOutcome:
     def __iter__(self):
         """Iterate over the class attributes.
         >>> a, b, c = IntendedModelCallOutcome({'answer': "yes"}, True, 'x1289')
         >>> a
         {'answer': 'yes'}
@@ -33,11 +32,10 @@ class IntendedModelCallOutcome:
     def __len__(self):
         return 3
     def __repr__(self):
         return f"IntendedModelCallOutcome(response = {self.response}, cache_used = {self.cache_used}, cache_key = '{self.cache_key}')"
 from edsl.config import CONFIG
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
@@ -124,11 +122,6 @@ class LanguageModel(
             # Skip the API key check. Sometimes this is useful for testing.
             self._api_token = None
-    def ask_question(self, question):
-        user_prompt = question.get_instructions().render(question.data).text
-        system_prompt = "You are a helpful agent pretending to be a human."
-        return self.execute_model_call(user_prompt, system_prompt)
     @property
     def api_token(self) -> str:
         if not hasattr(self, "_api_token"):
@@ -209,6 +202,8 @@ class LanguageModel(
         """
         self._set_rate_limits(rpm=rpm, tpm=tpm)
     def _set_rate_limits(self, rpm=None, tpm=None) -> None:
         """Set the rate limits for the model.
@@ -249,16 +244,14 @@ class LanguageModel(
         >>> LanguageModel._overide_default_parameters(passed_parameter_dict={"temperature": 0.5}, default_parameter_dict={"temperature":0.9, "max_tokens": 1000})
         {'temperature': 0.5, 'max_tokens': 1000}
         """
-        # parameters = dict({})
-        return {
-            parameter_name: passed_parameter_dict.get(parameter_name, default_value)
-            for parameter_name, default_value in default_parameter_dict.items()
-        }
+        #parameters = dict({})
-    def __call__(self, user_prompt: str, system_prompt: str):
+        return {parameter_name: passed_parameter_dict.get(parameter_name, default_value)
+                for parameter_name, default_value in default_parameter_dict.items()}
+    def __call__(self, user_prompt:str, system_prompt:str):
         return self.execute_model_call(user_prompt, system_prompt)
     @abstractmethod
     async def async_execute_model_call(user_prompt: str, system_prompt: str):
         """Execute the model call and returns a coroutine.
@@ -317,10 +310,8 @@ class LanguageModel(
         data["choices[0]"]["message"]["content"].
         """
         raise NotImplementedError
-    async def _async_prepare_response(
-        self, model_call_outcome: IntendedModelCallOutcome, cache: "Cache"
-    ) -> dict:
+    async def _async_prepare_response(self, model_call_outcome: IntendedModelCallOutcome, cache: "Cache") -> dict:
         """Prepare the response for return."""
         model_response = {
@@ -330,19 +321,21 @@ class LanguageModel(
             "raw_model_response": model_call_outcome.response,
         }
-        answer_portion = self.parse_response(model_call_outcome.response)
+        answer_portion = self.parse_response(model_call_outcome.response)
         try:
             answer_dict = json.loads(answer_portion)
         except json.JSONDecodeError as e:
             # TODO: Turn into logs to generate issues
             answer_dict, success = await repair(
-                bad_json=answer_portion, error_message=str(e), cache=cache
+                bad_json=answer_portion,
+                error_message=str(e),
+                cache=cache
             )
             if not success:
                 raise Exception(
                     f"""Even the repair failed. The error was: {e}. The response was: {answer_portion}."""
                 )
         return {**model_response, **answer_dict}
     async def async_get_raw_response(
@@ -354,18 +347,16 @@ class LanguageModel(
         encoded_image=None,
     ) -> IntendedModelCallOutcome:
         import warnings
-        warnings.warn(
-            "This method is deprecated. Use async_get_intended_model_call_outcome."
-        )
+        warnings.warn("This method is deprecated. Use async_get_intended_model_call_outcome.")
         return await self._async_get_intended_model_call_outcome(
             user_prompt=user_prompt,
             system_prompt=system_prompt,
             cache=cache,
             iteration=iteration,
-            encoded_image=encoded_image,
+            encoded_image=encoded_image
         )
     async def _async_get_intended_model_call_outcome(
         self,
         user_prompt: str,
@@ -407,8 +398,8 @@ class LanguageModel(
             "iteration": iteration,
         }
         cached_response, cache_key = cache.fetch(**cache_call_params)
-        if cache_used := cached_response is not None:
+        if (cache_used := cached_response is not None):
             response = json.loads(cached_response)
         else:
             f = (
@@ -416,24 +407,16 @@ class LanguageModel(
                 if hasattr(self, "remote") and self.remote
                 else self.async_execute_model_call
             )
-            params = {
-                "user_prompt": user_prompt,
-                "system_prompt": system_prompt,
-                **({"encoded_image": encoded_image} if encoded_image else {}),
+            params = {"user_prompt": user_prompt, "system_prompt": system_prompt,
+            **({"encoded_image": encoded_image} if encoded_image else {})
             }
             response = await f(**params)
-            new_cache_key = cache.store(
-                **cache_call_params, response=response
-            )  # store the response in the cache
-            assert new_cache_key == cache_key  # should be the same
-        return IntendedModelCallOutcome(
-            response=response, cache_used=cache_used, cache_key=cache_key
-        )
+            new_cache_key = cache.store(**cache_call_params, response=response) # store the response in the cache
+            assert new_cache_key == cache_key # should be the same
+        return IntendedModelCallOutcome(response = response, cache_used = cache_used, cache_key = cache_key)
-    _get_intended_model_call_outcome = sync_wrapper(
-        _async_get_intended_model_call_outcome
-    )
+    _get_intended_model_call_outcome = sync_wrapper(_async_get_intended_model_call_outcome)
     get_raw_response = sync_wrapper(async_get_raw_response)
@@ -454,7 +437,7 @@ class LanguageModel(
         self,
         user_prompt: str,
         system_prompt: str,
-        cache: "Cache",
+        cache: 'Cache',
         iteration: int = 1,
         encoded_image=None,
     ) -> dict:
@@ -472,8 +455,8 @@ class LanguageModel(
             "system_prompt": system_prompt,
             "iteration": iteration,
             "cache": cache,
-            **({"encoded_image": encoded_image} if encoded_image else {}),
-        }
+            **({"encoded_image": encoded_image} if encoded_image else {})
+        }
         model_call_outcome = await self._async_get_intended_model_call_outcome(**params)
         return await self._async_prepare_response(model_call_outcome, cache=cache)

edsl/language_models/registry.py CHANGED Viewed

@@ -36,10 +36,6 @@ class Model(metaclass=Meta):
         from edsl.inference_services.registry import default
         registry = registry or default
-        if isinstance(model_name, int):
-            model_name = cls.available(name_only=True)[model_name]
         factory = registry.create_model_factory(model_name)
         return factory(*args, **kwargs)

edsl/questions/QuestionMultipleChoice.py CHANGED Viewed

@@ -96,7 +96,7 @@ class QuestionMultipleChoice(QuestionBase):
             question_option_key = list(meta.find_undeclared_variables(parsed_content))[
                 0
             ]
-            # breakpoint()
+            #breakpoint()
             translated_options = scenario.get(question_option_key)
         else:
             translated_options = [

edsl/questions/QuestionNumerical.py CHANGED Viewed

@@ -26,6 +26,7 @@ class QuestionNumerical(QuestionBase):
         :param question_name: The name of the question.
         :param question_text: The text of the question.
+        :param instructions: Instructions for the question. If not provided, the default instructions are used. To view them, run `QuestionNumerical.default_instructions`.
         :param min_value: The minimum value of the answer.
         :param max_value: The maximum value of the answer.
         """

edsl/results/DatasetExportMixin.py CHANGED Viewed

@@ -27,10 +27,6 @@ class DatasetExportMixin:
         >>> d.relevant_columns(remove_prefix=True)
         ['b']
-        >>> d = Dataset([{'a':[1,2,3,4]}, {'b':[5,6,7,8]}])
-        >>> d.relevant_columns()
-        ['a', 'b']
         >>> from edsl.results import Results; Results.example().select('how_feeling', 'how_feeling_yesterday').relevant_columns()
         ['answer.how_feeling', 'answer.how_feeling_yesterday']
@@ -597,7 +593,7 @@ class DatasetExportMixin:
             return filename
     def tally(
-        self, *fields: Optional[str], top_n: Optional[int] = None, output="Dataset"
+        self, *fields: Optional[str], top_n: Optional[int] = None, output="dict"
     ) -> Union[dict, "Dataset"]:
         """Tally the values of a field or perform a cross-tab of multiple fields.
@@ -605,11 +601,9 @@ class DatasetExportMixin:
         >>> from edsl.results import Results
         >>> r = Results.example()
-        >>> r.select('how_feeling').tally('answer.how_feeling', output = "dict")
+        >>> r.select('how_feeling').tally('answer.how_feeling')
         {'OK': 2, 'Great': 1, 'Terrible': 1}
-        >>> r.select('how_feeling').tally('answer.how_feeling', output = "Dataset")
-        Dataset([{'value': ['OK', 'Great', 'Terrible']}, {'count': [2, 1, 1]}])
-        >>> r.select('how_feeling', 'period').tally('how_feeling', 'period', output = "dict")
+        >>> r.select('how_feeling', 'period').tally('how_feeling', 'period')
         {('OK', 'morning'): 1, ('Great', 'afternoon'): 1, ('Terrible', 'morning'): 1, ('OK', 'afternoon'): 1}
         """
         from collections import Counter
@@ -621,8 +615,6 @@ class DatasetExportMixin:
             column.split(".")[-1] for column in self.relevant_columns()
         ]
-        # breakpoint()
         if not all(
             f in self.relevant_columns() or f in relevant_columns_without_prefix
             for f in fields
@@ -649,7 +641,6 @@ class DatasetExportMixin:
         from edsl.results.Dataset import Dataset
         if output == "dict":
-            # why did I do this?
             warnings.warn(
                 textwrap.dedent(
                     """\

edsl/scenarios/Scenario.py CHANGED Viewed

@@ -182,19 +182,6 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
                 new_scenario[key] = self[key]
         return new_scenario
-    @classmethod
-    def from_url(cls, url: str, field_name: Optional[str] = "text") -> "Scenario":
-        """Creates a scenario from a URL.
-        :param url: The URL to create the scenario from.
-        :param field_name: The field name to use for the text.
-        """
-        import requests
-        text = requests.get(url).text
-        return cls({"url": url, field_name: text})
     @classmethod
     def from_image(cls, image_path: str) -> str:
         """Creates a scenario with a base64 encoding of an image.
@@ -220,7 +207,6 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
     @classmethod
     def from_pdf(cls, pdf_path):
         import fitz  # PyMuPDF
-        from edsl import Scenario
         # Ensure the file exists
         if not os.path.exists(pdf_path):

edsl/scenarios/ScenarioList.py CHANGED Viewed

@@ -242,16 +242,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         return ScenarioList(new_data)
-    def from_urls(self, urls: list[str], field_name: Optional[str] = "text") -> ScenarioList:
-        """Create a ScenarioList from a list of URLs.
-        :param urls: A list of URLs.
-        :param field_name: The name of the field to store the text from the URLs.
-        """
-        return ScenarioList([Scenario.from_url(url, field_name) for url in urls])
     def select(self, *fields) -> ScenarioList:
         """
         Selects scenarios with only the references fields.
@@ -298,15 +288,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s = ScenarioList.from_list("a", [1,2,3])
         >>> s.to_dataset()
         Dataset([{'a': [1, 2, 3]}])
-        >>> s = ScenarioList.from_list("a", [1,2,3]).add_list("b", [4,5,6])
-        >>> s.to_dataset()
-        Dataset([{'a': [1, 2, 3]}, {'b': [4, 5, 6]}])
         """
         from edsl.results.Dataset import Dataset
         keys = self[0].keys()
-        data = [{key: [scenario[key] for scenario in self.data]} for key in keys]
-        return Dataset(data)
+        data = {key: [scenario[key] for scenario in self.data] for key in keys}
+        return Dataset([data])
     def add_list(self, name, values) -> ScenarioList:
         """Add a list of values to a ScenarioList.

edsl/scenarios/ScenarioListExportMixin.py CHANGED Viewed

@@ -20,24 +20,13 @@ def to_dataset(func):
     return wrapper
-def decorate_methods_from_mixin(cls, mixin_cls):
-    for attr_name, attr_value in mixin_cls.__dict__.items():
-        if callable(attr_value) and not attr_name.startswith("__"):
+def decorate_all_methods(cls):
+    for attr_name, attr_value in cls.__dict__.items():
+        if callable(attr_value):
             setattr(cls, attr_name, to_dataset(attr_value))
     return cls
-# def decorate_all_methods(cls):
-#     for attr_name, attr_value in cls.__dict__.items():
-#         if callable(attr_value):
-#             setattr(cls, attr_name, to_dataset(attr_value))
-#     return cls
-# @decorate_all_methods
+@decorate_all_methods
 class ScenarioListExportMixin(DatasetExportMixin):
     """Mixin class for exporting Results objects."""
-    def __init_subclass__(cls, **kwargs):
-        super().__init_subclass__(**kwargs)
-        decorate_methods_from_mixin(cls, DatasetExportMixin)

edsl/scenarios/ScenarioListPdfMixin.py CHANGED Viewed

@@ -43,9 +43,6 @@ class ScenarioListPdfMixin:
     @staticmethod
     def extract_text_from_pdf(pdf_path):
-        from edsl import Scenario
-        # TODO: Add test case
         # Ensure the file exists
         if not os.path.exists(pdf_path):
             raise FileNotFoundError(f"The file {pdf_path} does not exist.")

{edsl-0.1.31.dist-info → edsl-0.1.31.dev2.dist-info}/METADATA RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.31
+Version: 0.1.31.dev2
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT
@@ -19,7 +19,6 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: aiohttp (>=3.9.1,<4.0.0)
 Requires-Dist: anthropic (>=0.23.1,<0.24.0)
 Requires-Dist: black[jupyter] (>=24.4.2,<25.0.0)
-Requires-Dist: groq (>=0.9.0,<0.10.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: jupyter (>=1.0.0,<2.0.0)
 Requires-Dist: markdown2 (>=2.4.11,<3.0.0)

edsl 0.1.31__py3-none-any.whl → 0.1.31.dev2__py3-none-any.whl

edsl 0.1.31py3-none-any.whl → 0.1.31.dev2py3-none-any.whl