edsl 0.1.33.dev3__py3-none-any.whl → 0.1.34__py3-none-any.whl

edsl/language_models/LanguageModel.py

@@ -440,7 +440,7 @@ class LanguageModel(
         system_prompt: str,
         cache: "Cache",
         iteration: int = 0,
-        encoded_image=None,
+        files_list=None,
     ) -> ModelResponse:
         """Handle caching of responses.
 
@@ -462,16 +462,18 @@ class LanguageModel(
         >>> m._get_intended_model_call_outcome(user_prompt = "Hello", system_prompt = "hello", cache = Cache())
         ModelResponse(...)"""
 
-        if encoded_image:
-            # the image has is appended to the user_prompt for hash-lookup purposes
-            image_hash = hashlib.md5(encoded_image.encode()).hexdigest()
-            user_prompt += f" {image_hash}"
+        if files_list:
+            files_hash = "+".join([str(hash(file)) for file in files_list])
+            # print(f"Files hash: {files_hash}")
+            user_prompt_with_hashes = user_prompt + f" {files_hash}"
+        else:
+            user_prompt_with_hashes = user_prompt
 
         cache_call_params = {
             "model": str(self.model),
             "parameters": self.parameters,
             "system_prompt": system_prompt,
-            "user_prompt": user_prompt,
+            "user_prompt": user_prompt_with_hashes,
             "iteration": iteration,
         }
         cached_response, cache_key = cache.fetch(**cache_call_params)
@@ -487,7 +489,8 @@ class LanguageModel(
             params = {
                 "user_prompt": user_prompt,
                 "system_prompt": system_prompt,
-                **({"encoded_image": encoded_image} if encoded_image else {}),
+                "files_list": files_list
+                # **({"encoded_image": encoded_image} if encoded_image else {}),
             }
             # response = await f(**params)
             response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
@@ -531,7 +534,7 @@ class LanguageModel(
         system_prompt: str,
         cache: "Cache",
         iteration: int = 1,
-        encoded_image=None,
+        files_list: Optional[List["File"]] = None,
     ) -> dict:
         """Get response, parse, and return as string.
 
@@ -547,7 +550,7 @@ class LanguageModel(
             "system_prompt": system_prompt,
             "iteration": iteration,
             "cache": cache,
-            **({"encoded_image": encoded_image} if encoded_image else {}),
+            "files_list": files_list,
         }
         model_inputs = ModelInputs(user_prompt=user_prompt, system_prompt=system_prompt)
         model_outputs = await self._async_get_intended_model_call_outcome(**params)

edsl/language_models/utilities.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import Any
+from typing import Any, Optional, List
 from edsl import Survey
 from edsl.config import CONFIG
 from edsl.enums import InferenceServiceType
@@ -40,7 +40,10 @@ def create_language_model(
         _tpm = 1000000000000
 
         async def async_execute_model_call(
-            self, user_prompt: str, system_prompt: str
+            self,
+            user_prompt: str,
+            system_prompt: str,
+            files_list: Optional[List[Any]] = None,
         ) -> dict[str, Any]:
             question_number = int(
                 user_prompt.split("XX")[1]

edsl/questions/QuestionBase.py

@@ -44,6 +44,13 @@ class QuestionBase(
     _answering_instructions = None
     _question_presentation = None
 
+    @property
+    def response_model(self) -> type["BaseModel"]:
+        if self._response_model is not None:
+            return self._response_model
+        else:
+            return self.create_response_model()
+
     # region: Validation and simulation methods
     @property
     def response_validator(self) -> "ResponseValidatorBase":
@@ -75,7 +82,8 @@ class QuestionBase(
         if not hasattr(self, "_fake_data_factory"):
             from polyfactory.factories.pydantic_factory import ModelFactory
 
-            class FakeData(ModelFactory[self.response_model]): ...
+            class FakeData(ModelFactory[self.response_model]):
+                ...
 
             self._fake_data_factory = FakeData
         return self._fake_data_factory
@@ -98,7 +106,9 @@ class QuestionBase(
         comment: Optional[str]
         generated_tokens: Optional[str]
 
-    def _validate_answer(self, answer: dict) -> ValidatedAnswer:
+    def _validate_answer(
+        self, answer: dict, replacement_dict: dict = None
+    ) -> ValidatedAnswer:
         """Validate the answer.
         >>> from edsl.exceptions import QuestionAnswerValidationError
         >>> from edsl import QuestionFreeText as Q
@@ -106,7 +116,7 @@ class QuestionBase(
         {'answer': 'Hello', 'generated_tokens': 'Hello'}
         """
 
-        return self.response_validator.validate(answer)
+        return self.response_validator.validate(answer, replacement_dict)
 
     # endregion
 

edsl/questions/QuestionBaseGenMixin.py

@@ -95,6 +95,34 @@ class QuestionBaseGenMixin:
             questions.append(QuestionBase.from_dict(new_data))
         return questions
 
+    def render(self, replacement_dict: dict) -> "QuestionBase":
+        """Render the question components as jinja2 templates with the replacement dictionary."""
+        from jinja2 import Environment
+        from edsl import Scenario
+
+        strings_only_replacement_dict = {
+            k: v for k, v in replacement_dict.items() if not isinstance(v, Scenario)
+        }
+
+        def render_string(value: str) -> str:
+            if value is None or not isinstance(value, str):
+                return value
+            else:
+                try:
+                    return (
+                        Environment()
+                        .from_string(value)
+                        .render(strings_only_replacement_dict)
+                    )
+                except Exception as e:
+                    import warnings
+
+                    warnings.warn("Failed to render string: " + value)
+                    # breakpoint()
+                    return value
+
+        return self.apply_function(render_string)
+
     def apply_function(self, func: Callable, exclude_components=None) -> QuestionBase:
         """Apply a function to the question parts
 

edsl/questions/QuestionCheckBox.py

@@ -245,7 +245,7 @@ class QuestionCheckBox(QuestionBase):
 
         scenario = scenario or Scenario()
         translated_options = [
-            Template(option).render(scenario) for option in self.question_options
+            Template(str(option)).render(scenario) for option in self.question_options
         ]
         translated_codes = []
         for answer_code in answer_codes:

edsl/questions/QuestionMultipleChoice.py

@@ -120,9 +120,9 @@ class QuestionMultipleChoice(QuestionBase):
 
     question_type = "multiple_choice"
     purpose = "When options are known and limited"
-    question_options: Union[list[str], list[list], list[float], list[int]] = (
-        QuestionOptionsDescriptor()
-    )
+    question_options: Union[
+        list[str], list[list], list[float], list[int]
+    ] = QuestionOptionsDescriptor()
     _response_model = None
     response_validator_class = MultipleChoiceResponseValidator
 
@@ -163,7 +163,11 @@ class QuestionMultipleChoice(QuestionBase):
     # Answer methods
     ################
 
-    def create_response_model(self):
+    def create_response_model(self, replacement_dict: dict = None):
+        if replacement_dict is None:
+            replacement_dict = {}
+            # The replacement dict that could be from scenario, current answers, etc. to populate the response model
+
         if self.use_code:
             return create_response_model(
                 list(range(len(self.question_options))), self.permissive

edsl/questions/ResponseValidatorABC.py

@@ -92,7 +92,11 @@ class ResponseValidatorABC(ABC):
         generated_tokens: Optional[str]
 
     def validate(
-        self, raw_edsl_answer_dict: RawEdslAnswerDict, fix=False, verbose=False
+        self,
+        raw_edsl_answer_dict: RawEdslAnswerDict,
+        fix=False,
+        verbose=False,
+        replacement_dict: dict = None,
     ) -> EdslAnswerDict:
         """This is the main validation function.
 

edsl/questions/descriptors.py

@@ -303,7 +303,7 @@ class QuestionOptionsDescriptor(BaseDescriptor):
                 return None
             else:
                 raise QuestionCreationValidationError(
-                    f"Dynamic question options must have jina2 braces - instead received: {value}."
+                    f"Dynamic question options must have jinja2 braces - instead received: {value}."
                 )
         if not isinstance(value, list):
             raise QuestionCreationValidationError(
@@ -325,14 +325,15 @@ class QuestionOptionsDescriptor(BaseDescriptor):
             )
         if not self.linear_scale:
             if not self.q_budget:
-                if not (
-                    value
-                    and all(type(x) == type(value[0]) for x in value)
-                    and isinstance(value[0], (str, list, int, float))
-                ):
-                    raise QuestionCreationValidationError(
-                        f"Question options must be all same type (got {value}).)"
-                    )
+                pass
+            #     if not (
+            #         value
+            #         and all(type(x) == type(value[0]) for x in value)
+            #         and isinstance(value[0], (str, list, int, float))
+            #     ):
+            #         raise QuestionCreationValidationError(
+            #             f"Question options must be all same type (got {value}).)"
+            #         )
             else:
                 if not all(isinstance(x, (str)) for x in value):
                     raise QuestionCreationValidationError(
@@ -390,8 +391,8 @@ class QuestionTextDescriptor(BaseDescriptor):
 
     def validate(self, value, instance):
         """Validate the value is a string."""
-        if len(value) > Settings.MAX_QUESTION_LENGTH:
-            raise Exception("Question is too long!")
+        # if len(value) > Settings.MAX_QUESTION_LENGTH:
+        #     raise Exception("Question is too long!")
         if len(value) < 1:
             raise Exception("Question is too short!")
         if not isinstance(value, str):

edsl/questions/templates/numerical/answering_instructions.jinja

@@ -1,7 +1,6 @@
 This question requires a numerical response in the form of an integer or decimal (e.g., -12, 0, 1, 2, 3.45, ...).
 Respond with just your number on a single line.
 If your response is equivalent to zero, report '0'
-If you cannot determine the answer, report 'None'
 
 {% if include_comment %}
 After the answer, put a comment explaining your choice on the next line.

edsl/questions/templates/yes_no/answering_instructions.jinja

@@ -1,6 +1,6 @@
 {# Answering Instructions #}
-Please reponse with just your answer. 
+Please respond with just your answer. 
 
 {% if include_comment %}
-After the answer, you can put a comment explaining your reponse.
+After the answer, you can put a comment explaining your response.
 {% endif %}

edsl/scenarios/FileStore.py

@@ -1,41 +1,101 @@
-from edsl import Scenario
 import base64
 import io
 import tempfile
-from typing import Optional
+import mimetypes
+import os
+from typing import Dict, Any, IO, Optional
+import requests
+from urllib.parse import urlparse
+
+import google.generativeai as genai
+
+from edsl import Scenario
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+from edsl.utilities.utilities import is_notebook
+
+
+def view_pdf(pdf_path):
+    import os
+    import subprocess
+
+    if is_notebook():
+        from IPython.display import IFrame
+        from IPython.display import display, HTML
+
+        # Replace 'path/to/your/file.pdf' with the actual path to your PDF file
+        IFrame(pdf_path, width=700, height=600)
+        display(HTML(f'<a href="{pdf_path}" target="_blank">Open PDF</a>'))
+        return
+
+    if os.path.exists(pdf_path):
+        try:
+            if (os_name := os.name) == "posix":
+                # for cool kids
+                subprocess.run(["open", pdf_path], check=True)  # macOS
+            elif os_name == "nt":
+                os.startfile(pdf_path)  # Windows
+            else:
+                subprocess.run(["xdg-open", pdf_path], check=True)  # Linux
+        except Exception as e:
+            print(f"Error opening PDF: {e}")
+    else:
+        print("PDF file was not created successfully.")
 
 
 class FileStore(Scenario):
     def __init__(
         self,
-        filename: str,
+        path: Optional[str] = None,
+        mime_type: Optional[str] = None,
         binary: Optional[bool] = None,
         suffix: Optional[str] = None,
         base64_string: Optional[str] = None,
+        external_locations: Optional[Dict[str, str]] = None,
+        **kwargs,
     ):
-        self.filename = filename
-        self.suffix = suffix or "." + filename.split(".")[-1]
+        if path is None and "filename" in kwargs:
+            path = kwargs["filename"]
+        self.path = path
+        self.suffix = suffix or path.split(".")[-1]
         self.binary = binary or False
-        self.base64_string = base64_string or self.encode_file_to_base64_string(
-            filename
+        self.mime_type = (
+            mime_type or mimetypes.guess_type(path)[0] or "application/octet-stream"
         )
+        self.base64_string = base64_string or self.encode_file_to_base64_string(path)
+        self.external_locations = external_locations or {}
         super().__init__(
             {
-                "filename": self.filename,
+                "path": self.path,
                 "base64_string": self.base64_string,
                 "binary": self.binary,
                 "suffix": self.suffix,
+                "mime_type": self.mime_type,
+                "external_locations": self.external_locations,
             }
         )
 
+    def __str__(self):
+        return "FileStore: self.path"
+
+    @property
+    def size(self) -> int:
+        return os.path.getsize(self.path)
+
+    def upload_google(self, refresh: bool = False) -> None:
+        genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+        google_info = genai.upload_file(self.path, mime_type=self.mime_type)
+        self.external_locations["google"] = google_info.to_dict()
+
     @classmethod
+    @remove_edsl_version
     def from_dict(cls, d):
-        return cls(d["filename"], d["binary"], d["suffix"], d["base64_string"])
+        # return cls(d["filename"], d["binary"], d["suffix"], d["base64_string"])
+        return cls(**d)
 
     def __repr__(self):
-        return f"FileStore(filename='{self.filename}', binary='{self.binary}', 'suffix'={self.suffix})"
+        return f"FileStore({self.path})"
 
-    def encode_file_to_base64_string(self, file_path):
+    def encode_file_to_base64_string(self, file_path: str):
         try:
             # Attempt to open the file in text mode
             with open(file_path, "r") as text_file:
@@ -56,14 +116,14 @@ class FileStore(Scenario):
 
         return base64_string
 
-    def open(self):
+    def open(self) -> "IO":
         if self.binary:
             return self.base64_to_file(self["base64_string"], is_binary=True)
         else:
             return self.base64_to_text_file(self["base64_string"])
 
     @staticmethod
-    def base64_to_text_file(base64_string):
+    def base64_to_text_file(base64_string) -> "IO":
         # Decode the base64 string to bytes
         text_data_bytes = base64.b64decode(base64_string)
 
@@ -101,7 +161,9 @@ class FileStore(Scenario):
 
         # Create a named temporary file
         mode = "wb" if self.binary else "w"
-        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=suffix, mode=mode)
+        temp_file = tempfile.NamedTemporaryFile(
+            delete=False, suffix="." + suffix, mode=mode
+        )
 
         if self.binary:
             temp_file.write(file_like_object.read())
@@ -112,31 +174,95 @@ class FileStore(Scenario):
 
         return temp_file.name
 
-    def push(self, description=None):
+    def view(self, max_size: int = 300) -> None:
+        if self.suffix == "pdf":
+            view_pdf(self.path)
+
+        if self.suffix == "png" or self.suffix == "jpg" or self.suffix == "jpeg":
+            if is_notebook():
+                from IPython.display import Image
+                from PIL import Image as PILImage
+
+                if max_size:
+                    # Open the image using Pillow
+                    with PILImage.open(self.path) as img:
+                        # Get original width and height
+                        original_width, original_height = img.size
+
+                        # Calculate the scaling factor
+                        scale = min(
+                            max_size / original_width, max_size / original_height
+                        )
+
+                        # Calculate new dimensions
+                        new_width = int(original_width * scale)
+                        new_height = int(original_height * scale)
+
+                        return Image(self.path, width=new_width, height=new_height)
+                else:
+                    return Image(self.path)
+
+    def push(
+        self, description: Optional[str] = None, visibility: str = "unlisted"
+    ) -> dict:
+        """
+        Push the object to Coop.
+        :param description: The description of the object to push.
+        :param visibility: The visibility of the object to push.
+        """
         scenario_version = Scenario.from_dict(self.to_dict())
         if description is None:
-            description = "File: " + self["filename"]
-        info = scenario_version.push(description=description)
+            description = "File: " + self.path
+        info = scenario_version.push(description=description, visibility=visibility)
         return info
 
     @classmethod
-    def pull(cls, uuid, expected_parrot_url: Optional[str] = None):
+    def pull(cls, uuid: str, expected_parrot_url: Optional[str] = None) -> "FileStore":
+        """
+        :param uuid: The UUID of the object to pull.
+        :param expected_parrot_url: The URL of the Parrot server to use.
+        :return: The object pulled from the Parrot server.
+        """
         scenario_version = Scenario.pull(uuid, expected_parrot_url=expected_parrot_url)
         return cls.from_dict(scenario_version.to_dict())
 
+    @classmethod
+    def from_url(
+        cls,
+        url: str,
+        download_path: Optional[str] = None,
+        mime_type: Optional[str] = None,
+    ) -> "FileStore":
+        """
+        :param url: The URL of the file to download.
+        :param download_path: The path to save the downloaded file.
+        :param mime_type: The MIME type of the file. If None, it will be guessed from the file extension.
+        """
+
+        response = requests.get(url, stream=True)
+        response.raise_for_status()  # Raises an HTTPError for bad responses
+
+        # Get the filename from the URL if download_path is not provided
+        if download_path is None:
+            filename = os.path.basename(urlparse(url).path)
+            if not filename:
+                filename = "downloaded_file"
+            # download_path = filename
+            download_path = os.path.join(os.getcwd(), filename)
+
+        # Ensure the directory exists
+        os.makedirs(os.path.dirname(download_path), exist_ok=True)
+
+        # Write the file
+        with open(download_path, "wb") as file:
+            for chunk in response.iter_content(chunk_size=8192):
+                file.write(chunk)
+
+        # Create and return a new File instance
+        return cls(download_path, mime_type=mime_type)
 
-class CSVFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".csv"
-        )
 
+class CSVFileStore(FileStore):
     @classmethod
     def example(cls):
         from edsl.results.Results import Results
@@ -155,17 +281,6 @@ class CSVFileStore(FileStore):
 
 
 class PDFFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".pdf"
-        )
-
     def view(self):
         pdf_path = self.to_tempfile()
         print(f"PDF path: {pdf_path}")  # Print the path to ensure it exists
@@ -241,17 +356,6 @@ class PDFFileStore(FileStore):
 
 
 class PNGFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".png"
-        )
-
     @classmethod
     def example(cls):
         import textwrap
@@ -275,17 +379,6 @@ class PNGFileStore(FileStore):
 
 
 class SQLiteFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".sqlite"
-        )
-
     @classmethod
     def example(cls):
         import sqlite3
@@ -308,17 +401,6 @@ class SQLiteFileStore(FileStore):
 
 
 class HTMLFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".html"
-        )
-
     @classmethod
     def example(cls):
         import tempfile
@@ -350,9 +432,10 @@ if __name__ == "__main__":
     # fs = PDFFileStore("paper.pdf")
     # fs.view()
     # from edsl import Conjure
-
-    fs = PNGFileStore("robot.png")
-    fs.view()
+    pass
+    # fs = PNGFileStore("logo.png")
+    # fs.view()
+    # fs.upload_google()
 
     # c = Conjure(datafile_name=fs.to_tempfile())
     # f = PDFFileStore("paper.pdf")