edsl 0.1.30.dev4__tar.gz → 0.1.31__tar.gz

{edsl-0.1.30.dev4 → edsl-0.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.30.dev4
+Version: 0.1.31
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT
@@ -19,10 +19,11 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: aiohttp (>=3.9.1,<4.0.0)
 Requires-Dist: anthropic (>=0.23.1,<0.24.0)
 Requires-Dist: black[jupyter] (>=24.4.2,<25.0.0)
+Requires-Dist: groq (>=0.9.0,<0.10.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: jupyter (>=1.0.0,<2.0.0)
 Requires-Dist: markdown2 (>=2.4.11,<3.0.0)
-Requires-Dist: matplotlib (>=3.8.4,<4.0.0)
+Requires-Dist: matplotlib (>=3.8,<3.9)
 Requires-Dist: nest-asyncio (>=1.5.9,<2.0.0)
 Requires-Dist: numpy (>=1.22,<2.0)
 Requires-Dist: openai (>=1.4.0,<2.0.0)
@@ -35,6 +36,7 @@ Requires-Dist: python-docx (>=1.1.0,<2.0.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: restrictedpython (>=7.1,<8.0)
 Requires-Dist: rich (>=13.7.0,<14.0.0)
+Requires-Dist: setuptools (<72.0)
 Requires-Dist: simpleeval (>=0.9.13,<0.10.0)
 Requires-Dist: sqlalchemy (>=2.0.23,<3.0.0)
 Requires-Dist: tenacity (>=8.2.3,<9.0.0)

edsl-0.1.31/edsl/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.31"

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/agents/Invigilator.py

@@ -18,7 +18,12 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
     """An invigilator that uses an AI model to answer questions."""
 
     async def async_answer_question(self) -> AgentResponseDict:
-        """Answer a question using the AI model."""
+        """Answer a question using the AI model.
+
+        >>> i = InvigilatorAI.example()
+        >>> i.answer_question()
+        {'message': '{"answer": "SPAM!"}'}
+        """
         params = self.get_prompts() | {"iteration": self.iteration}
         raw_response = await self.async_get_response(**params)
         data = {
@@ -29,6 +34,7 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
             "raw_model_response": raw_response["raw_model_response"],
         }
         response = self._format_raw_response(**data)
+        # breakpoint()
         return AgentResponseDict(**response)
 
     async def async_get_response(
@@ -97,7 +103,6 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
         answer = question._translate_answer_code_to_answer(
             response["answer"], combined_dict
         )
-        # breakpoint()
         data = {
             "answer": answer,
             "comment": response.get(

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/agents/PromptConstructionMixin.py

@@ -275,8 +275,25 @@ class PromptConstructorMixin:
                     if (new_question := question.split("_comment")[0]) in d:
                         d[new_question].comment = answer
 
+            question_data = self.question.data.copy()
+
+            # check to see if the questio_options is actuall a string
+            if "question_options" in question_data:
+                if isinstance(self.question.data["question_options"], str):
+                    from jinja2 import Environment, meta
+
+                    env = Environment()
+                    parsed_content = env.parse(self.question.data["question_options"])
+                    question_option_key = list(
+                        meta.find_undeclared_variables(parsed_content)
+                    )[0]
+                    question_data["question_options"] = self.scenario.get(
+                        question_option_key
+                    )
+
+            # breakpoint()
             rendered_instructions = question_prompt.render(
-                self.question.data | self.scenario | d | {"agent": self.agent}
+                question_data | self.scenario | d | {"agent": self.agent}
             )
 
             undefined_template_variables = (

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/config.py

@@ -65,6 +65,10 @@ CONFIG_MAP = {
     #     "default": None,
     #     "info": "This env var holds your Anthropic API key (https://www.anthropic.com/).",
     # },
+    # "GROQ_API_KEY": {
+    #     "default": None,
+    #     "info": "This env var holds your GROQ API key (https://console.groq.com/login).",
+    # },
 }
 
 

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/conjure/Conjure.py

@@ -35,6 +35,12 @@ class Conjure:
         # The __init__ method in Conjure won't be called because __new__ returns a different class instance.
         pass
 
+    @classmethod
+    def example(cls):
+        from edsl.conjure.InputData import InputDataABC
+
+        return InputDataABC.example()
+
 
 if __name__ == "__main__":
     pass

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/coop/coop.py

@@ -465,6 +465,7 @@ class Coop:
         description: Optional[str] = None,
         status: RemoteJobStatus = "queued",
         visibility: Optional[VisibilityType] = "unlisted",
+        iterations: Optional[int] = 1,
     ) -> dict:
         """
         Send a remote inference job to the server.
@@ -473,6 +474,7 @@ class Coop:
         :param optional description: A description for this entry in the remote cache.
         :param status: The status of the job. Should be 'queued', unless you are debugging.
         :param visibility: The visibility of the cache entry.
+        :param iterations: The number of times to run each interview.
 
         >>> job = Jobs.example()
         >>> coop.remote_inference_create(job=job, description="My job")
@@ -488,6 +490,7 @@ class Coop:
                 ),
                 "description": description,
                 "status": status,
+                "iterations": iterations,
                 "visibility": visibility,
                 "version": self._edsl_version,
             },
@@ -498,6 +501,7 @@ class Coop:
             "uuid": response_json.get("jobs_uuid"),
             "description": response_json.get("description"),
             "status": response_json.get("status"),
+            "iterations": response_json.get("iterations"),
             "visibility": response_json.get("visibility"),
             "version": self._edsl_version,
         }

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/coop/utils.py

@@ -2,6 +2,7 @@ from edsl import (
     Agent,
     AgentList,
     Cache,
+    ModelList,
     Notebook,
     Results,
     Scenario,
@@ -9,6 +10,7 @@ from edsl import (
     Survey,
     Study,
 )
+from edsl.language_models import LanguageModel
 from edsl.questions import QuestionBase
 from typing import Literal, Optional, Type, Union
 
@@ -16,6 +18,8 @@ EDSLObject = Union[
     Agent,
     AgentList,
     Cache,
+    LanguageModel,
+    ModelList,
     Notebook,
     Type[QuestionBase],
     Results,
@@ -29,6 +33,8 @@ ObjectType = Literal[
     "agent",
     "agent_list",
     "cache",
+    "model",
+    "model_list",
     "notebook",
     "question",
     "results",
@@ -62,8 +68,10 @@ class ObjectRegistry:
         {"object_type": "agent", "edsl_class": Agent},
         {"object_type": "agent_list", "edsl_class": AgentList},
         {"object_type": "cache", "edsl_class": Cache},
-        {"object_type": "question", "edsl_class": QuestionBase},
+        {"object_type": "model", "edsl_class": LanguageModel},
+        {"object_type": "model_list", "edsl_class": ModelList},
         {"object_type": "notebook", "edsl_class": Notebook},
+        {"object_type": "question", "edsl_class": QuestionBase},
         {"object_type": "results", "edsl_class": Results},
         {"object_type": "scenario", "edsl_class": Scenario},
         {"object_type": "scenario_list", "edsl_class": ScenarioList},

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/data/CacheHandler.py

@@ -41,7 +41,7 @@ class CacheHandler:
         old_data = self.from_old_sqlite_cache()
         self.cache.add_from_dict(old_data)
 
-    def create_cache_directory(self) -> None:
+    def create_cache_directory(self, notify = False) -> None:
         """
         Create the cache directory if one is required and it does not exist.
         """
@@ -49,9 +49,8 @@ class CacheHandler:
         dir_path = os.path.dirname(path)
         if dir_path and not os.path.exists(dir_path):
             os.makedirs(dir_path)
-            import warnings
-
-            warnings.warn(f"Created cache directory: {dir_path}")
+            if notify:
+                print(f"Created cache directory: {dir_path}")
 
     def gen_cache(self) -> Cache:
         """

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/enums.py

@@ -59,6 +59,7 @@ class InferenceServiceType(EnumWithChecks):
     GOOGLE = "google"
     TEST = "test"
     ANTHROPIC = "anthropic"
+    GROQ = "groq"
 
 
 service_to_api_keyname = {
@@ -69,6 +70,7 @@ service_to_api_keyname = {
     InferenceServiceType.GOOGLE.value: "GOOGLE_API_KEY",
     InferenceServiceType.TEST.value: "TBD",
     InferenceServiceType.ANTHROPIC.value: "ANTHROPIC_API_KEY",
+    InferenceServiceType.GROQ.value: "GROQ_API_KEY",
 }
 
 

edsl-0.1.31/edsl/inference_services/DeepInfraService.py

@@ -0,0 +1,18 @@
+import aiohttp
+import json
+import requests
+from typing import Any, List
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+
+from edsl.inference_services.OpenAIService import OpenAIService
+
+
+class DeepInfraService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "deep_infra"
+    _env_key_name_ = "DEEP_INFRA_API_KEY"
+    _base_url_ = "https://api.deepinfra.com/v1/openai"
+    _models_list_cache: List[str] = []

edsl-0.1.31/edsl/inference_services/GroqService.py

@@ -0,0 +1,18 @@
+from typing import Any, List
+from edsl.inference_services.OpenAIService import OpenAIService
+
+import groq
+
+
+class GroqService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "groq"
+    _env_key_name_ = "GROQ_API_KEY"
+
+    _sync_client_ = groq.Groq
+    _async_client_ = groq.AsyncGroq
+
+    # _base_url_ = "https://api.deepinfra.com/v1/openai"
+    _base_url_ = None
+    _models_list_cache: List[str] = []

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/inference_services/InferenceServicesCollection.py

@@ -15,15 +15,19 @@ class InferenceServicesCollection:
         cls.added_models[service_name].append(model_name)
 
     @staticmethod
-    def _get_service_available(service) -> list[str]:
+    def _get_service_available(service, warn: bool = False) -> list[str]:
         from_api = True
         try:
             service_models = service.available()
         except Exception as e:
-            warnings.warn(
-                f"Error getting models for {service._inference_service_}. Relying on cache.",
-                UserWarning,
-            )
+            if warn:
+                warnings.warn(
+                    f"""Error getting models for {service._inference_service_}. 
+                    Check that you have properly stored your Expected Parrot API key and activated remote inference, or stored your own API keys for the language models that you want to use.
+                    See https://docs.expectedparrot.com/en/latest/api_keys.html for instructions on storing API keys.
+                    Relying on cache.""",
+                    UserWarning,
+                )
             from edsl.inference_services.models_available_cache import models_available
 
             service_models = models_available.get(service._inference_service_, [])
@@ -57,4 +61,8 @@ class InferenceServicesCollection:
                 if service_name is None or service_name == service._inference_service_:
                     return service.create_model(model_name)
 
+        # if model_name == "test":
+        #     from edsl.language_models import LanguageModel
+        #     return LanguageModel(test = True)
+
         raise Exception(f"Model {model_name} not found in any of the services")

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/inference_services/OpenAIService.py

@@ -1,6 +1,9 @@
 from typing import Any, List
 import re
-from openai import AsyncOpenAI
+import os
+
+# from openai import AsyncOpenAI
+import openai
 
 from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
 from edsl.language_models import LanguageModel
@@ -12,6 +15,22 @@ class OpenAIService(InferenceServiceABC):
 
     _inference_service_ = "openai"
     _env_key_name_ = "OPENAI_API_KEY"
+    _base_url_ = None
+
+    _sync_client_ = openai.OpenAI
+    _async_client_ = openai.AsyncOpenAI
+
+    @classmethod
+    def sync_client(cls):
+        return cls._sync_client_(
+            api_key=os.getenv(cls._env_key_name_), base_url=cls._base_url_
+        )
+
+    @classmethod
+    def async_client(cls):
+        return cls._async_client_(
+            api_key=os.getenv(cls._env_key_name_), base_url=cls._base_url_
+        )
 
     # TODO: Make this a coop call
     model_exclude_list = [
@@ -31,16 +50,24 @@ class OpenAIService(InferenceServiceABC):
     ]
     _models_list_cache: List[str] = []
 
+    @classmethod
+    def get_model_list(cls):
+        raw_list = cls.sync_client().models.list()
+        if hasattr(raw_list, "data"):
+            return raw_list.data
+        else:
+            return raw_list
+
     @classmethod
     def available(cls) -> List[str]:
-        from openai import OpenAI
+        # from openai import OpenAI
 
         if not cls._models_list_cache:
             try:
-                client = OpenAI()
+                # client = OpenAI(api_key = os.getenv(cls._env_key_name_), base_url = cls._base_url_)
                 cls._models_list_cache = [
                     m.id
-                    for m in client.models.list()
+                    for m in cls.get_model_list()
                     if m.id not in cls.model_exclude_list
                 ]
             except Exception as e:
@@ -78,15 +105,24 @@ class OpenAIService(InferenceServiceABC):
                 "top_logprobs": 3,
             }
 
+            def sync_client(self):
+                return cls.sync_client()
+
+            def async_client(self):
+                return cls.async_client()
+
             @classmethod
             def available(cls) -> list[str]:
-                client = openai.OpenAI()
-                return client.models.list()
+                # import openai
+                # client = openai.OpenAI(api_key = os.getenv(cls._env_key_name_), base_url = cls._base_url_)
+                # return client.models.list()
+                return cls.sync_client().models.list()
 
             def get_headers(self) -> dict[str, Any]:
-                from openai import OpenAI
+                # from openai import OpenAI
 
-                client = OpenAI()
+                # client = OpenAI(api_key = os.getenv(cls._env_key_name_), base_url = cls._base_url_)
+                client = self.sync_client()
                 response = client.chat.completions.with_raw_response.create(
                     messages=[
                         {
@@ -124,8 +160,8 @@ class OpenAIService(InferenceServiceABC):
                 encoded_image=None,
             ) -> dict[str, Any]:
                 """Calls the OpenAI API and returns the API response."""
-                content = [{"type": "text", "text": user_prompt}]
                 if encoded_image:
+                    content = [{"type": "text", "text": user_prompt}]
                     content.append(
                         {
                             "type": "image_url",
@@ -134,21 +170,28 @@ class OpenAIService(InferenceServiceABC):
                             },
                         }
                     )
-                self.client = AsyncOpenAI()
-                response = await self.client.chat.completions.create(
-                    model=self.model,
-                    messages=[
+                else:
+                    content = user_prompt
+                # self.client = AsyncOpenAI(
+                #     api_key = os.getenv(cls._env_key_name_),
+                #     base_url = cls._base_url_
+                #     )
+                client = self.async_client()
+                params = {
+                    "model": self.model,
+                    "messages": [
                         {"role": "system", "content": system_prompt},
                         {"role": "user", "content": content},
                     ],
-                    temperature=self.temperature,
-                    max_tokens=self.max_tokens,
-                    top_p=self.top_p,
-                    frequency_penalty=self.frequency_penalty,
-                    presence_penalty=self.presence_penalty,
-                    logprobs=self.logprobs,
-                    top_logprobs=self.top_logprobs if self.logprobs else None,
-                )
+                    "temperature": self.temperature,
+                    "max_tokens": self.max_tokens,
+                    "top_p": self.top_p,
+                    "frequency_penalty": self.frequency_penalty,
+                    "presence_penalty": self.presence_penalty,
+                    "logprobs": self.logprobs,
+                    "top_logprobs": self.top_logprobs if self.logprobs else None,
+                }
+                response = await client.chat.completions.create(**params)
                 return response.model_dump()
 
             @staticmethod

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/inference_services/registry.py

@@ -6,7 +6,8 @@ from edsl.inference_services.OpenAIService import OpenAIService
 from edsl.inference_services.AnthropicService import AnthropicService
 from edsl.inference_services.DeepInfraService import DeepInfraService
 from edsl.inference_services.GoogleService import GoogleService
+from edsl.inference_services.GroqService import GroqService
 
 default = InferenceServicesCollection(
-    [OpenAIService, AnthropicService, DeepInfraService, GoogleService]
+    [OpenAIService, AnthropicService, DeepInfraService, GoogleService, GroqService]
 )

{edsl-0.1.30.dev4 → edsl-0.1.31}/edsl/jobs/Jobs.py

@@ -3,9 +3,7 @@ from __future__ import annotations
 import warnings
 from itertools import product
 from typing import Optional, Union, Sequence, Generator
-
 from edsl.Base import Base
-
 from edsl.exceptions import MissingAPIKeyError
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.jobs.interviews.Interview import Interview
@@ -321,7 +319,11 @@ class Jobs(Base):
         self.scenarios = self.scenarios or [Scenario()]
         for agent, scenario, model in product(self.agents, self.scenarios, self.models):
             yield Interview(
-                survey=self.survey, agent=agent, scenario=scenario, model=model
+                survey=self.survey,
+                agent=agent,
+                scenario=scenario,
+                model=model,
+                skip_retry=self.skip_retry,
             )
 
     def create_bucket_collection(self) -> BucketCollection:
@@ -411,6 +413,12 @@ class Jobs(Base):
                 if warn:
                     warnings.warn(message)
 
+    @property
+    def skip_retry(self):
+        if not hasattr(self, "_skip_retry"):
+            return False
+        return self._skip_retry
+
     def run(
         self,
         n: int = 1,
@@ -425,6 +433,7 @@ class Jobs(Base):
         print_exceptions=True,
         remote_cache_description: Optional[str] = None,
         remote_inference_description: Optional[str] = None,
+        skip_retry: bool = False,
     ) -> Results:
         """
         Runs the Job: conducts Interviews and returns their results.
@@ -443,6 +452,7 @@ class Jobs(Base):
         from edsl.coop.coop import Coop
 
         self._check_parameters()
+        self._skip_retry = skip_retry
 
         if batch_mode is not None:
             raise NotImplementedError(
@@ -461,12 +471,11 @@ class Jobs(Base):
             remote_inference = False
 
         if remote_inference:
-            from edsl.agents.Agent import Agent
-            from edsl.language_models.registry import Model
-            from edsl.results.Result import Result
-            from edsl.results.Results import Results
-            from edsl.scenarios.Scenario import Scenario
-            from edsl.surveys.Survey import Survey
+            import time
+            from datetime import datetime
+            from edsl.config import CONFIG
+
+            expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
 
             self._output("Remote inference activated. Sending job to server...")
             if remote_cache:
@@ -474,33 +483,60 @@ class Jobs(Base):
                     "Remote caching activated. The remote cache will be used for this job."
                 )
 
-            remote_job_data = coop.remote_inference_create(
+            remote_job_creation_data = coop.remote_inference_create(
                 self,
                 description=remote_inference_description,
                 status="queued",
+                iterations=n,
             )
-            self._output("Job sent!")
-            # Create mock results object to store job data
-            results = Results(
-                survey=Survey(),
-                data=[
-                    Result(
-                        agent=Agent.example(),
-                        scenario=Scenario.example(),
-                        model=Model(),
-                        iteration=1,
-                        answer={"info": "Remote job details"},
+            time_queued = datetime.now().strftime("%m/%d/%Y %I:%M:%S %p")
+            job_uuid = remote_job_creation_data.get("uuid")
+            print(f"Remote inference started (Job uuid={job_uuid}).")
+            # print(f"Job queued at {time_queued}.")
+            job_in_queue = True
+            while job_in_queue:
+                remote_job_data = coop.remote_inference_get(job_uuid)
+                status = remote_job_data.get("status")
+                if status == "cancelled":
+                    print("\r" + " " * 80 + "\r", end="")
+                    print("Job cancelled by the user.")
+                    print(
+                        f"See {expected_parrot_url}/home/remote-inference for more details."
                     )
-                ],
-            )
-            results.add_columns_from_dict([remote_job_data])
-            if self.verbose:
-                results.select(["info", "uuid", "status", "version"]).print(
-                    format="rich"
-                )
-            return results
+                    return None
+                elif status == "failed":
+                    print("\r" + " " * 80 + "\r", end="")
+                    print("Job failed.")
+                    print(
+                        f"See {expected_parrot_url}/home/remote-inference for more details."
+                    )
+                    return None
+                elif status == "completed":
+                    results_uuid = remote_job_data.get("results_uuid")
+                    results = coop.get(results_uuid, expected_object_type="results")
+                    print("\r" + " " * 80 + "\r", end="")
+                    print(
+                        f"Job completed and Results stored on Coop (Results uuid={results_uuid})."
+                    )
+                    return results
+                else:
+                    duration = 10 if len(self) < 10 else 60
+                    time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
+                    frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+                    start_time = time.time()
+                    i = 0
+                    while time.time() - start_time < duration:
+                        print(
+                            f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
+                            end="",
+                            flush=True,
+                        )
+                        time.sleep(0.1)
+                        i += 1
         else:
             if check_api_keys:
+                from edsl import Model
+
                 for model in self.models + [Model()]:
                     if not model.has_valid_api_key():
                         raise MissingAPIKeyError(
@@ -606,9 +642,9 @@ class Jobs(Base):
         results = JobsRunnerAsyncio(self).run(*args, **kwargs)
         return results
 
-    async def run_async(self, cache=None, **kwargs):
+    async def run_async(self, cache=None, n=1, **kwargs):
         """Run the job asynchronously."""
-        results = await JobsRunnerAsyncio(self).run_async(cache=cache, **kwargs)
+        results = await JobsRunnerAsyncio(self).run_async(cache=cache, n=n, **kwargs)
         return results
 
     def all_question_parameters(self):
@@ -688,7 +724,10 @@ class Jobs(Base):
     #######################
     @classmethod
     def example(
-        cls, throw_exception_probability: int = 0, randomize: bool = False
+        cls,
+        throw_exception_probability: int = 0,
+        randomize: bool = False,
+        test_model=False,
     ) -> Jobs:
         """Return an example Jobs instance.
 
@@ -706,6 +745,11 @@ class Jobs(Base):
 
         addition = "" if not randomize else str(uuid4())
 
+        if test_model:
+            from edsl.language_models import LanguageModel
+
+            m = LanguageModel.example(test_model=True)
+
         # (status, question, period)
         agent_answers = {
             ("Joyful", "how_feeling", "morning"): "OK",
@@ -753,7 +797,10 @@ class Jobs(Base):
                 Scenario({"period": "afternoon"}),
             ]
         )
-        job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
+        if test_model:
+            job = base_survey.by(m).by(scenario_list).by(joy_agent, sad_agent)
+        else:
+            job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
 
         return job