edsl 0.1.27.dev2__py3-none-any.whl → 0.1.29__py3-none-any.whl

edsl/surveys/Survey.py

@@ -2,10 +2,8 @@
 
 from __future__ import annotations
 import re
-from typing import Any, Generator, Optional, Union, List, Literal, Callable
 
-from rich import print
-from rich.table import Table
+from typing import Any, Generator, Optional, Union, List, Literal, Callable
 
 from edsl.exceptions import SurveyCreationError, SurveyHasNoRulesError
 from edsl.questions.QuestionBase import QuestionBase
@@ -17,8 +15,8 @@ from edsl.Base import Base
 from edsl.surveys.SurveyExportMixin import SurveyExportMixin
 from edsl.surveys.descriptors import QuestionsDescriptor
 from edsl.surveys.MemoryPlan import MemoryPlan
+
 from edsl.surveys.DAG import DAG
-from edsl.utilities import is_notebook
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 from edsl.surveys.SurveyFlowVisualizationMixin import SurveyFlowVisualizationMixin
 
@@ -88,19 +86,37 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
 
         >>> s = Survey.example()
         >>> s.get_question("q0")
-        Question('multiple_choice', question_name = 'q0', question_text = 'Do you like school?', question_options = ['yes', 'no'])
+        Question('multiple_choice', question_name = \"""q0\""", question_text = \"""Do you like school?\""", question_options = ['yes', 'no'])
         """
         if question_name not in self.question_name_to_index:
             raise KeyError(f"Question name {question_name} not found in survey.")
         index = self.question_name_to_index[question_name]
         return self._questions[index]
 
+    def question_names_to_questions(self) -> dict:
+        """Return a dictionary mapping question names to question attributes."""
+        return {q.question_name: q for q in self.questions}
+
     def get_question(self, question_name: str) -> QuestionBase:
         """Return the question object given the question name."""
         # import warnings
         # warnings.warn("survey.get_question is deprecated. Use subscript operator instead.")
         return self.get(question_name)
 
+    def __hash__(self) -> int:
+        """Return a hash of the question."""
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
+    @property
+    def parameters(self):
+        return set.union(*[q.parameters for q in self.questions])
+
+    @property
+    def parameters_by_question(self):
+        return {q.question_name: q.parameters for q in self.questions}
+
     @property
     def question_names(self) -> list[str]:
         """Return a list of question names in the survey.
@@ -528,7 +544,7 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
     ###################
     # FORWARD METHODS
     ###################
-    def by(self, *args: Union[Agent, Scenario, LanguageModel]) -> Jobs:
+    def by(self, *args: Union["Agent", "Scenario", "LanguageModel"]) -> "Jobs":
         """Add Agents, Scenarios, and LanguageModels to a survey and returns a runnable Jobs object.
 
         :param args: The Agents, Scenarios, and LanguageModels to add to the survey.
@@ -544,6 +560,12 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         job = Jobs(survey=self)
         return job.by(*args)
 
+    def to_jobs(self):
+        """Convert the survey to a Jobs object."""
+        from edsl.jobs.Jobs import Jobs
+
+        return Jobs(survey=self)
+
     def run(self, *args, **kwargs) -> "Results":
         """Turn the survey into a Job and runs it.
 
@@ -551,9 +573,7 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
 
         >>> from edsl import QuestionFreeText
         >>> s = Survey([QuestionFreeText.example()])
-        >>> results = s.run(debug = True)
-        >>> results
-        Results(...)
+        >>> results = s.run(debug = True, cache = False)
         >>> results.select('answer.*').print(format = "rich")
         ┏━━━━━━━━━━━━━━┓
         ┃ answer       ┃
@@ -640,18 +660,17 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
 
         >>> i = s.gen_path_through_survey()
         >>> next(i)
-        Question('multiple_choice', question_name = 'q0', question_text = 'Do you like school?', question_options = ['yes', 'no'])
+        Question('multiple_choice', question_name = \"""q0\""", question_text = \"""Do you like school?\""", question_options = ['yes', 'no'])
         >>> i.send({"q0": "yes"})
-        Question('multiple_choice', question_name = 'q2', question_text = 'Why?', question_options = ['**lack*** of killer bees in cafeteria', 'other'])
+        Question('multiple_choice', question_name = \"""q2\""", question_text = \"""Why?\""", question_options = ['**lack*** of killer bees in cafeteria', 'other'])
 
         And here is the path through the survey if the answer to q0 is 'no':
 
         >>> i2 = s.gen_path_through_survey()
         >>> next(i2)
-        Question('multiple_choice', question_name = 'q0', question_text = 'Do you like school?', question_options = ['yes', 'no'])
+        Question('multiple_choice', question_name = \"""q0\""", question_text = \"""Do you like school?\""", question_options = ['yes', 'no'])
         >>> i2.send({"q0": "no"})
-        Question('multiple_choice', question_name = 'q1', question_text = 'Why not?', question_options = ['killer bees in cafeteria', 'other'])
-
+        Question('multiple_choice', question_name = \"""q1\""", question_text = \"""Why not?\""", question_options = ['killer bees in cafeteria', 'other'])
         """
         question = self._first_question()
         while not question == EndOfSurvey:
@@ -724,6 +743,22 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         except IndexError:
             raise
 
+    @property
+    def piping_dag(self) -> DAG:
+        d = {}
+        for question_name, depenencies in self.parameters_by_question.items():
+            if depenencies:
+                question_index = self.question_name_to_index[question_name]
+                for dependency in depenencies:
+                    if dependency not in self.question_name_to_index:
+                        pass
+                    else:
+                        dependency_index = self.question_name_to_index[dependency]
+                        if question_index not in d:
+                            d[question_index] = set()
+                        d[question_index].add(dependency_index)
+        return d
+
     def dag(self, textify: bool = False) -> DAG:
         """Return the DAG of the survey, which reflects both skip-logic and memory.
 
@@ -737,10 +772,12 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         """
         memory_dag = self.memory_plan.dag
         rule_dag = self.rule_collection.dag
+        piping_dag = self.piping_dag
         if textify:
             memory_dag = DAG(self.textify(memory_dag))
             rule_dag = DAG(self.textify(rule_dag))
-        return memory_dag + rule_dag
+            piping_dag = DAG(self.textify(piping_dag))
+        return memory_dag + rule_dag + piping_dag
 
     ###################
     # DUNDER METHODS
@@ -761,7 +798,7 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
 
         >>> s = Survey.example()
         >>> s[0]
-        Question('multiple_choice', question_name = 'q0', question_text = 'Do you like school?', question_options = ['yes', 'no'])
+        Question('multiple_choice', question_name = \"""q0\""", question_text = \"""Do you like school?\""", question_options = ['yes', 'no'])
 
         """
         if isinstance(index, int):
@@ -802,22 +839,33 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
     ###################
     # SERIALIZATION METHODS
     ###################
-    @add_edsl_version
-    def to_dict(self) -> dict[str, Any]:
+
+    def _to_dict(self) -> dict[str, Any]:
         """Serialize the Survey object to a dictionary.
 
         >>> s = Survey.example()
-        >>> s.to_dict().keys()
-        dict_keys(['questions', 'memory_plan', 'rule_collection', 'question_groups', 'edsl_version', 'edsl_class_name'])
+        >>> s._to_dict().keys()
+        dict_keys(['questions', 'memory_plan', 'rule_collection', 'question_groups'])
 
         """
         return {
-            "questions": [q.to_dict() for q in self._questions],
+            "questions": [q._to_dict() for q in self._questions],
             "memory_plan": self.memory_plan.to_dict(),
             "rule_collection": self.rule_collection.to_dict(),
             "question_groups": self.question_groups,
         }
 
+    @add_edsl_version
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the Survey object to a dictionary.
+
+        >>> s = Survey.example()
+        >>> s.to_dict().keys()
+        dict_keys(['questions', 'memory_plan', 'rule_collection', 'question_groups', 'edsl_version', 'edsl_class_name'])
+
+        """
+        return self._to_dict()
+
     @classmethod
     @remove_edsl_version
     def from_dict(cls, data: dict) -> Survey:
@@ -891,7 +939,7 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         """Print the survey in a rich format.
 
         >>> t = Survey.example().rich_print()
-        >>> print(t)
+        >>> print(t) # doctest: +SKIP
         ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
         ┃ Questions                                                                                          ┃
         ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
@@ -912,6 +960,8 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         │ └───────────────┴─────────────────┴───────────────┴──────────────────────────────────────────────┘ │
         └────────────────────────────────────────────────────────────────────────────────────────────────────┘
         """
+        from rich.table import Table
+
         table = Table(show_header=True, header_style="bold magenta")
         table.add_column("Questions", style="dim")
 
@@ -938,7 +988,7 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         :param filename: The name of the file to save the CSV to.
 
         >>> s = Survey.example()
-        >>> s.to_csv()
+        >>> s.to_csv() # doctest: +SKIP
            index question_name        question_text                                question_options    question_type
         0      0            q0  Do you like school?                                       [yes, no]  multiple_choice
         1      1            q1             Why not?               [killer bees in cafeteria, other]  multiple_choice
@@ -975,7 +1025,7 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         return res
 
     @classmethod
-    def example(cls) -> Survey:
+    def example(cls, params=False) -> Survey:
         """Return an example survey.
 
         >>> s = Survey.example()
@@ -999,34 +1049,71 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
             question_options=["**lack*** of killer bees in cafeteria", "other"],
             question_name="q2",
         )
+        if params:
+            q3 = QuestionMultipleChoice(
+                question_text="To the question '{{ q0.question_text}}', you said '{{ q0.answer }}'. Do you still feel this way?",
+                question_options=["yes", "no"],
+                question_name="q3",
+            )
+            s = cls(questions=[q0, q1, q2, q3])
+            return s
         s = cls(questions=[q0, q1, q2])
         s = s.add_rule(q0, "q0 == 'yes'", q2)
         return s
 
-    def __call__(self, *args, **kwargs):
+    def get_job(self, model=None, agent=None, **kwargs):
+        if model is None:
+            from edsl import Model
+
+            model = Model()
+
+        from edsl.scenarios.Scenario import Scenario
+
+        s = Scenario(kwargs)
+
+        if not agent:
+            from edsl import Agent
+
+            agent = Agent()
+
+        return self.by(s).by(agent).by(model)
+
+    def __call__(self, model=None, agent=None, cache=None, **kwargs):
         """Run the survey with default model, taking the required survey as arguments.
 
         >>> from edsl.questions import QuestionFunctional
         >>> def f(scenario, agent_traits): return "yes" if scenario["period"] == "morning" else "no"
         >>> q = QuestionFunctional(question_name = "q0", func = f)
         >>> s = Survey([q])
-        >>> s(period = "morning").select("answer.q0").first()
+        >>> s(period = "morning", cache = False).select("answer.q0").first()
         'yes'
-        >>> s(period = "evening").select("answer.q0").first()
+        >>> s(period = "evening", cache = False).select("answer.q0").first()
         'no'
         """
-        if "model" in kwargs:
-            model = kwargs["model"]
-            del kwargs["model"]
-        else:
-            from edsl import Model
+        job = self.get_job(model, agent, **kwargs)
+        return job.run(cache=cache)
 
-            model = Model()
-        from edsl.scenarios.Scenario import Scenario
+    async def run_async(self, model=None, agent=None, cache=None, **kwargs):
+        """Run the survey with default model, taking the required survey as arguments.
 
-        s = Scenario(kwargs)
+        >>> from edsl.questions import QuestionFunctional
+        >>> def f(scenario, agent_traits): return "yes" if scenario["period"] == "morning" else "no"
+        >>> q = QuestionFunctional(question_name = "q0", func = f)
+        >>> s = Survey([q])
+        >>> s(period = "morning").select("answer.q0").first()
+        'yes'
+        >>> s(period = "evening").select("answer.q0").first()
+        'no'
+        """
+        # TODO: temp fix by creating a cache
+        if cache is None:
+            from edsl.data import Cache
 
-        return self.by(s).by(model).run()
+            c = Cache()
+        else:
+            c = cache
+        jobs: "Jobs" = self.get_job(model, agent, **kwargs)
+        return await jobs.run_async(cache=c)
 
 
 def main():
@@ -1066,4 +1153,4 @@ def main():
 if __name__ == "__main__":
     import doctest
 
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
+    doctest.testmod(optionflags=doctest.ELLIPSIS | doctest.SKIP)

edsl/surveys/SurveyExportMixin.py

@@ -1,8 +1,6 @@
 """A mixin class for exporting surveys to different formats."""
 
-from docx import Document
 from typing import Union, Optional
-import black
 
 
 class SurveyExportMixin:
@@ -13,8 +11,24 @@ class SurveyExportMixin:
 
         return SurveyCSS.default_style().generate_css()
 
+    def get_description(self) -> str:
+        """Return the description of the survey."""
+        from edsl import QuestionFreeText
+
+        question_texts = "\n".join([q.question_text for q in self._questions])
+        q = QuestionFreeText(
+            question_name="description",
+            question_text=f"""A survey was conducted with the following questions: 
+                             {question_texts}
+                             Please write a description of the survey.
+                             """,
+        )
+        return q.run().select("description").first()
+
     def docx(self, filename=None) -> Union["Document", None]:
         """Generate a docx document for the survey."""
+        from docx import Document
+
         doc = Document()
         doc.add_heading("EDSL Survey")
         doc.add_paragraph(f"\n")
@@ -60,11 +74,17 @@ class SurveyExportMixin:
         :param filename: The name of the file to save the code to.
         :param survey_var_name: The name of the survey variable.
 
+        >>> from edsl.surveys import Survey
         >>> survey = Survey.example()
-        >>> survey.code()
-
-
+        >>> print(survey.code())
+        from edsl.surveys.Survey import Survey
+        ...
+        ...
+        survey = Survey(questions=[q0, q1, q2])
+        ...
         """
+        import black
+
         header_lines = ["from edsl.surveys.Survey import Survey"]
         header_lines.append("from edsl import Question")
         lines = ["\n".join(header_lines)]

edsl/surveys/SurveyFlowVisualizationMixin.py

@@ -1,10 +1,7 @@
 """A mixin for visualizing the flow of a survey."""
 
-import pydot
-import tempfile
-from IPython.display import Image
-from edsl.utilities import is_notebook
 from edsl.surveys.base import RulePriority, EndOfSurvey
+import tempfile
 
 
 class SurveyFlowVisualizationMixin:
@@ -13,6 +10,8 @@ class SurveyFlowVisualizationMixin:
     def show_flow(self, filename: str = None):
         """Create an image showing the flow of users through the survey."""
         # Create a graph object
+        import pydot
+
         graph = pydot.Dot(graph_type="digraph")
 
         # Add nodes for each question
@@ -101,8 +100,11 @@ class SurveyFlowVisualizationMixin:
                         on Ubuntu.
                     """
                 )
+            from edsl.utilities.utilities import is_notebook
 
             if is_notebook():
+                from IPython.display import Image
+
                 display(Image(tmp_file.name))
             else:
                 import os

edsl/tools/plotting.py

@@ -93,8 +93,10 @@ def theme_plot(results, field, context, themes=None, progress_bar=False):
           SELECT json_each.value AS theme 
           FROM self, 
           json_each({ field }_themes)
-          ) GROUP BY theme
-          ORDER BY mentions DESC
+          ) 
+    GROUP BY theme
+    HAVING theme <> 'Other'
+    ORDER BY mentions DESC
              """
     themes = results.sql(themes_query, to_list=True)
 

edsl/utilities/__init__.py

@@ -1,21 +1,22 @@
-from edsl.utilities.interface import (
-    print_dict_as_html_table,
-    print_dict_with_rich,
-    print_list_of_dicts_as_html_table,
-    print_table_with_rich,
-    print_public_methods_with_doc,
-    print_list_of_dicts_as_markdown_table,
-)
+# from edsl.utilities.interface import (
+#     print_dict_as_html_table,
+#     print_dict_with_rich,
+#     print_list_of_dicts_as_html_table,
+#     print_table_with_rich,
+#     print_public_methods_with_doc,
+#     print_list_of_dicts_as_markdown_table,
+# )
 
-from edsl.utilities.utilities import (
-    create_valid_var_name,
-    dict_to_html,
-    hash_value,
-    HTMLSnippet,
-    is_notebook,
-    is_gzipped,
-    is_valid_variable_name,
-    random_string,
-    repair_json,
-    shorten_string,
-)
+# from edsl.utilities.utilities import (
+#     create_valid_var_name,
+#     dict_to_html,
+#     hash_value,
+#     HTMLSnippet,
+#     is_notebook,
+#     is_gzipped,
+#     is_valid_variable_name,
+#     random_string,
+#     repair_json,
+#     shorten_string,
+#     time_all_functions,
+# )

edsl/utilities/gcp_bucket/cloud_storage.py

@@ -0,0 +1,96 @@
+import requests
+
+
+class CloudStorageManager:
+    def __init__(self, secret_token=None):
+        self.api_url = "https://bucket-server-tte53lsfxq-uc.a.run.app"
+        self.secret_token = secret_token
+
+    def get_signed_url(self, file_name, operation="upload"):
+        """Get a signed URL for uploading or downloading a file."""
+
+        if operation == "upload":
+            if self.secret_token == None:
+                raise "Set secret_token for upload permissions"
+            headers = {
+                "Authorization": self.secret_token,
+                "Content-Type": "application/json",
+            }
+        else:
+            headers = {
+                "Content-Type": "application/json",
+            }
+        data = {"file_name": file_name}
+        endpoint = f"{self.api_url}/generate-{operation}-signed-url"
+        response = requests.post(endpoint, json=data, headers=headers)
+
+        if response.status_code == 200:
+            return response.json().get("signed_url")
+        else:
+            raise Exception(
+                f"Failed to get signed URL: {response.status_code} {response.text}"
+            )
+
+    def upload_file(self, file_path, upload_file_name):
+        """Upload a file to the signed URL."""
+        signed_url = self.get_signed_url(upload_file_name, operation="upload")
+
+        with open(file_path, "rb") as file:
+            upload_response = requests.put(
+                signed_url,
+                data=file,
+                headers={"Content-Type": "application/octet-stream"},
+            )
+
+            if upload_response.status_code == 200:
+                print("File uploaded successfully")
+            else:
+                raise Exception(
+                    f"Failed to upload file: {upload_response.status_code} {upload_response.text}"
+                )
+
+    def download_file(self, file_name, save_name):
+        """Download a file from the signed URL."""
+
+        signed_url = self.get_signed_url(file_name, operation="download")
+        download_response = requests.get(signed_url, stream=True)
+
+        if download_response.status_code == 200:
+            with open(save_name, "wb") as file:
+                for chunk in download_response.iter_content(chunk_size=8192):
+                    file.write(chunk)
+            print("File downloaded successfully")
+        else:
+            raise Exception(
+                f"Failed to download file: {download_response.status_code} {download_response.text}"
+            )
+
+    def delete_file(self, file_name):
+        """Delete a file from the cloud storage."""
+        headers = {
+            "Authorization": self.secret_token,
+            "Content-Type": "application/json",
+        }
+        data = {"file_name": file_name}
+        endpoint = f"{self.api_url}/delete-file"
+        response = requests.delete(endpoint, params=data, headers=headers)
+
+        if response.status_code == 200:
+            print("File deleted successfully")
+        else:
+            raise Exception(
+                f"Failed to delete file: {response.status_code} {response.text}"
+            )
+
+    def list_files(self):
+        url = self.api_url + "/list_files"
+        headers = {
+            "Authorization": self.secret_token,
+            "Content-Type": "application/json",
+        }
+        res = requests.get(url, headers=headers)
+        data = res.json()
+        for x in data["data"]:
+            x["url"] = self.api_url + "/file/" + x["shaKey"]
+
+        return data

edsl/utilities/gcp_bucket/simple_example.py

@@ -0,0 +1,9 @@
+from edsl.utilities.gcp_bucket.cloud_storage import CloudStorageManager
+
+manager = CloudStorageManager()
+
+# Download Process
+file_name = "GSS2022.dta"  # Name for the downloaded file
+save_name = "test.dta"
+
+manager.download_file(file_name, save_name)