edsl 0.1.31.dev3__py3-none-any.whl → 0.1.32__py3-none-any.whl

edsl/scenarios/ScenarioList.py

@@ -242,6 +242,18 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         return ScenarioList(new_data)
 
+    def from_urls(
+        self, urls: list[str], field_name: Optional[str] = "text"
+    ) -> ScenarioList:
+        """Create a ScenarioList from a list of URLs.
+
+        :param urls: A list of URLs.
+        :param field_name: The name of the field to store the text from the URLs.
+
+
+        """
+        return ScenarioList([Scenario.from_url(url, field_name) for url in urls])
+
     def select(self, *fields) -> ScenarioList:
         """
         Selects scenarios with only the references fields.
@@ -288,12 +300,15 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s = ScenarioList.from_list("a", [1,2,3])
         >>> s.to_dataset()
         Dataset([{'a': [1, 2, 3]}])
+        >>> s = ScenarioList.from_list("a", [1,2,3]).add_list("b", [4,5,6])
+        >>> s.to_dataset()
+        Dataset([{'a': [1, 2, 3]}, {'b': [4, 5, 6]}])
         """
         from edsl.results.Dataset import Dataset
 
         keys = self[0].keys()
-        data = {key: [scenario[key] for scenario in self.data] for key in keys}
-        return Dataset([data])
+        data = [{key: [scenario[key] for scenario in self.data]} for key in keys]
+        return Dataset(data)
 
     def add_list(self, name, values) -> ScenarioList:
         """Add a list of values to a ScenarioList.
@@ -352,6 +367,99 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             data = cursor.fetchall()
         return cls([Scenario(dict(zip(columns, row))) for row in data])
 
+    @classmethod
+    def from_latex(cls, tex_file_path: str):
+        with open(tex_file_path, "r") as file:
+            lines = file.readlines()
+
+        processed_lines = []
+        non_blank_lines = [
+            (i, line.strip()) for i, line in enumerate(lines) if line.strip()
+        ]
+
+        for index, (line_no, text) in enumerate(non_blank_lines):
+            entry = {
+                "line_no": line_no + 1,  # Using 1-based index for line numbers
+                "text": text,
+                "line_before": non_blank_lines[index - 1][1] if index > 0 else None,
+                "line_after": (
+                    non_blank_lines[index + 1][1]
+                    if index < len(non_blank_lines) - 1
+                    else None
+                ),
+            }
+            processed_lines.append(entry)
+
+        return ScenarioList([Scenario(entry) for entry in processed_lines])
+
+    @classmethod
+    def from_docx(cls, docx_file_path: str):
+        from docx import Document
+
+        doc = Document(docx_file_path)
+        lines = []
+
+        # Extract text from paragraphs, treating each paragraph as a line
+        for para in doc.paragraphs:
+            lines.extend(para.text.splitlines())
+
+        processed_lines = []
+        non_blank_lines = [
+            (i, line.strip()) for i, line in enumerate(lines) if line.strip()
+        ]
+
+        for index, (line_no, text) in enumerate(non_blank_lines):
+            entry = {
+                "line_no": line_no + 1,  # Using 1-based index for line numbers
+                "text": text,
+                "line_before": non_blank_lines[index - 1][1] if index > 0 else None,
+                "line_after": (
+                    non_blank_lines[index + 1][1]
+                    if index < len(non_blank_lines) - 1
+                    else None
+                ),
+            }
+            processed_lines.append(entry)
+
+        return ScenarioList([Scenario(entry) for entry in processed_lines])
+
+    @classmethod
+    def from_google_doc(cls, url: str) -> ScenarioList:
+        """Create a ScenarioList from a Google Doc.
+
+        This method downloads the Google Doc as a Word file (.docx), saves it to a temporary file,
+        and then reads it using the from_docx class method.
+
+        Args:
+            url (str): The URL to the Google Doc.
+
+        Returns:
+            ScenarioList: An instance of the ScenarioList class.
+
+        """
+        import tempfile
+        import requests
+        from docx import Document
+
+        if "/edit" in url:
+            doc_id = url.split("/d/")[1].split("/edit")[0]
+        else:
+            raise ValueError("Invalid Google Doc URL format.")
+
+        export_url = f"https://docs.google.com/document/d/{doc_id}/export?format=docx"
+
+        # Download the Google Doc as a Word file (.docx)
+        response = requests.get(export_url)
+        response.raise_for_status()  # Ensure the request was successful
+
+        # Save the Word file to a temporary file
+        with tempfile.NamedTemporaryFile(suffix=".docx", delete=False) as temp_file:
+            temp_file.write(response.content)
+            temp_filename = temp_file.name
+
+        # Call the from_docx class method with the temporary file
+        return cls.from_docx(temp_filename)
+
     @classmethod
     def from_pandas(cls, df) -> ScenarioList:
         """Create a ScenarioList from a pandas DataFrame.
@@ -379,6 +487,112 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         else:
             return {scenario[field]: scenario[value] for scenario in self}
 
+    @classmethod
+    def from_excel(
+        cls, filename: str, sheet_name: Optional[str] = None
+    ) -> ScenarioList:
+        """Create a ScenarioList from an Excel file.
+
+        If the Excel file contains multiple sheets and no sheet_name is provided,
+        the method will print the available sheets and require the user to specify one.
+
+        Example:
+
+        >>> import tempfile
+        >>> import os
+        >>> import pandas as pd
+        >>> with tempfile.NamedTemporaryFile(delete=False, suffix='.xlsx') as f:
+        ...     df1 = pd.DataFrame({
+        ...         'name': ['Alice', 'Bob'],
+        ...         'age': [30, 25],
+        ...         'location': ['New York', 'Los Angeles']
+        ...     })
+        ...     df2 = pd.DataFrame({
+        ...         'name': ['Charlie', 'David'],
+        ...         'age': [35, 40],
+        ...         'location': ['Chicago', 'Boston']
+        ...     })
+        ...     with pd.ExcelWriter(f.name) as writer:
+        ...         df1.to_excel(writer, sheet_name='Sheet1', index=False)
+        ...         df2.to_excel(writer, sheet_name='Sheet2', index=False)
+        ...     temp_filename = f.name
+        >>> scenario_list = ScenarioList.from_excel(temp_filename, sheet_name='Sheet1')
+        >>> len(scenario_list)
+        2
+        >>> scenario_list[0]['name']
+        'Alice'
+        >>> scenario_list = ScenarioList.from_excel(temp_filename)  # Should raise an error and list sheets
+        Traceback (most recent call last):
+        ...
+        ValueError: Please provide a sheet name to load data from.
+        """
+        from edsl.scenarios.Scenario import Scenario
+        import pandas as pd
+
+        # Get all sheets
+        all_sheets = pd.read_excel(filename, sheet_name=None)
+
+        # If no sheet_name is provided and there is more than one sheet, print available sheets
+        if sheet_name is None:
+            if len(all_sheets) > 1:
+                print("The Excel file contains multiple sheets:")
+                for name in all_sheets.keys():
+                    print(f"- {name}")
+                raise ValueError("Please provide a sheet name to load data from.")
+            else:
+                # If there is only one sheet, use it
+                sheet_name = list(all_sheets.keys())[0]
+
+        # Load the specified or determined sheet
+        df = pd.read_excel(filename, sheet_name=sheet_name)
+
+        observations = []
+        for _, row in df.iterrows():
+            observations.append(Scenario(row.to_dict()))
+
+        return cls(observations)
+
+    @classmethod
+    def from_google_sheet(cls, url: str, sheet_name: str = None) -> ScenarioList:
+        """Create a ScenarioList from a Google Sheet.
+
+        This method downloads the Google Sheet as an Excel file, saves it to a temporary file,
+        and then reads it using the from_excel class method.
+
+        Args:
+            url (str): The URL to the Google Sheet.
+            sheet_name (str, optional): The name of the sheet to load. If None, the method will behave
+                                        the same as from_excel regarding multiple sheets.
+
+        Returns:
+            ScenarioList: An instance of the ScenarioList class.
+
+        """
+        import pandas as pd
+        import tempfile
+        import requests
+
+        if "/edit" in url:
+            sheet_id = url.split("/d/")[1].split("/edit")[0]
+        else:
+            raise ValueError("Invalid Google Sheet URL format.")
+
+        export_url = (
+            f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=xlsx"
+        )
+
+        # Download the Google Sheet as an Excel file
+        response = requests.get(export_url)
+        response.raise_for_status()  # Ensure the request was successful
+
+        # Save the Excel file to a temporary file
+        with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as temp_file:
+            temp_file.write(response.content)
+            temp_filename = temp_file.name
+
+        # Call the from_excel class method with the temporary file
+        return cls.from_excel(temp_filename, sheet_name=sheet_name)
+
     @classmethod
     def from_csv(cls, filename: str) -> ScenarioList:
         """Create a ScenarioList from a CSV file.
@@ -484,17 +698,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             table.add_row(str(i), s.rich_print())
         return table
 
-    # def print(
-    #     self,
-    #     format: Optional[str] = None,
-    #     max_rows: Optional[int] = None,
-    #     pretty_labels: Optional[dict] = None,
-    #     filename: str = None,
-    # ):
-    #     from edsl.utilities.interface import print_scenario_list
-
-    #     print_scenario_list(self[:max_rows])
-
     def __getitem__(self, key: Union[int, slice]) -> Any:
         """Return the item at the given index.
 

edsl/scenarios/ScenarioListExportMixin.py

@@ -20,13 +20,24 @@ def to_dataset(func):
     return wrapper
 
 
-def decorate_all_methods(cls):
-    for attr_name, attr_value in cls.__dict__.items():
-        if callable(attr_value):
+def decorate_methods_from_mixin(cls, mixin_cls):
+    for attr_name, attr_value in mixin_cls.__dict__.items():
+        if callable(attr_value) and not attr_name.startswith("__"):
             setattr(cls, attr_name, to_dataset(attr_value))
     return cls
 
 
-@decorate_all_methods
+# def decorate_all_methods(cls):
+#     for attr_name, attr_value in cls.__dict__.items():
+#         if callable(attr_value):
+#             setattr(cls, attr_name, to_dataset(attr_value))
+#     return cls
+
+
+# @decorate_all_methods
 class ScenarioListExportMixin(DatasetExportMixin):
     """Mixin class for exporting Results objects."""
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        decorate_methods_from_mixin(cls, DatasetExportMixin)

edsl/scenarios/ScenarioListPdfMixin.py

@@ -43,6 +43,9 @@ class ScenarioListPdfMixin:
 
     @staticmethod
     def extract_text_from_pdf(pdf_path):
+        from edsl import Scenario
+
+        # TODO: Add test case
         # Ensure the file exists
         if not os.path.exists(pdf_path):
             raise FileNotFoundError(f"The file {pdf_path} does not exist.")

edsl/surveys/Rule.py

@@ -117,13 +117,15 @@ class Rule:
     def _checks(self):
         pass
 
-    @add_edsl_version
+    # def _to_dict(self):
+
+    # @add_edsl_version
     def to_dict(self):
         """Convert the rule to a dictionary for serialization.
 
         >>> r = Rule.example()
         >>> r.to_dict()
-        {'current_q': 1, 'expression': "q1 == 'yes'", 'next_q': 2, 'priority': 0, 'question_name_to_index': {'q1': 1}, 'before_rule': False, 'edsl_version': '...', 'edsl_class_name': 'Rule'}
+        {'current_q': 1, 'expression': "q1 == 'yes'", 'next_q': 2, 'priority': 0, 'question_name_to_index': {'q1': 1}, 'before_rule': False}
         """
         return {
             "current_q": self.current_q,
@@ -133,6 +135,7 @@ class Rule:
             "question_name_to_index": self.question_name_to_index,
             "before_rule": self.before_rule,
         }
+        # return self._to_dict()
 
     @classmethod
     @remove_edsl_version

edsl/surveys/Survey.py

@@ -2,6 +2,9 @@
 
 from __future__ import annotations
 import re
+import tempfile
+import requests
+
 from typing import Any, Generator, Optional, Union, List, Literal, Callable
 from uuid import uuid4
 from edsl.Base import Base
@@ -75,6 +78,41 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
 
             warnings.warn("name parameter to a survey is deprecated.")
 
+    def simulate(self) -> dict:
+        """Simulate the survey and return the answers."""
+        i = self.gen_path_through_survey()
+        q = next(i)
+        while True:
+            try:
+                answer = q._simulate_answer()
+                q = i.send({q.question_name: answer["answer"]})
+            except StopIteration:
+                break
+        return self.answers
+
+    def create_agent(self) -> "Agent":
+        """Create an agent from the simulated answers."""
+        answers_dict = self.simulate()
+        from edsl.agents.Agent import Agent
+
+        a = Agent(traits=answers_dict)
+
+        def construct_answer_dict_function(traits: dict) -> Callable:
+            def func(self, question: "QuestionBase", scenario=None):
+                return traits.get(question.question_name, None)
+
+            return func
+
+        a.add_direct_question_answering_method(
+            construct_answer_dict_function(answers_dict)
+        )
+        return a
+
+    def simulate_results(self) -> "Results":
+        """Simulate the survey and return the results."""
+        a = self.create_agent()
+        return self.by([a]).run()
+
     def get(self, question_name: str) -> QuestionBase:
         """
         Return the question object given the question name.
@@ -141,6 +179,12 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
 
     @property
     def parameters(self):
+        """Return a set of parameters in the survey.
+
+        >>> s = Survey.example()
+        >>> s.parameters
+        set()
+        """
         return set.union(*[q.parameters for q in self.questions])
 
     @property
@@ -702,9 +746,13 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         >>> i2.send({"q0": "no"})
         Question('multiple_choice', question_name = \"""q1\""", question_text = \"""Why not?\""", question_options = ['killer bees in cafeteria', 'other'])
         """
+        self.answers = {}
         question = self._first_question()
         while not question == EndOfSurvey:
-            self.answers = yield question
+            # breakpoint()
+            answer = yield question
+            self.answers.update(answer)
+            # print(f"Answers: {self.answers}")
             ## TODO: This should also include survey and agent attributes
             question = self.next_question(question, self.answers)
 
@@ -775,6 +823,15 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
 
     @property
     def piping_dag(self) -> DAG:
+        """Figures out the DAG of piping dependencies.
+
+        >>> from edsl import QuestionFreeText
+        >>> q0 = QuestionFreeText(question_text="Here is a question", question_name="q0")
+        >>> q1 = QuestionFreeText(question_text="You previously answered {{ q0 }}---how do you feel now?", question_name="q1")
+        >>> s = Survey([q0, q1])
+        >>> s.piping_dag
+        {1: {0}}
+        """
         d = {}
         for question_name, depenencies in self.parameters_by_question.items():
             if depenencies:
@@ -919,6 +976,32 @@ class Survey(SurveyExportMixin, SurveyFlowVisualizationMixin, Base):
         )
         return survey
 
+    @classmethod
+    def from_qsf(
+        cls, qsf_file: Optional[str] = None, url: Optional[str] = None
+    ) -> Survey:
+        """Create a Survey object from a Qualtrics QSF file."""
+
+        if url and qsf_file:
+            raise ValueError("Only one of url or qsf_file can be provided.")
+
+        if (not url) and (not qsf_file):
+            raise ValueError("Either url or qsf_file must be provided.")
+
+        if url:
+            response = requests.get(url)
+            response.raise_for_status()  # Ensure the request was successful
+
+            # Save the Excel file to a temporary file
+            with tempfile.NamedTemporaryFile(suffix=".qsf", delete=False) as temp_file:
+                temp_file.write(response.content)
+                qsf_file = temp_file.name
+
+        from edsl.surveys.SurveyQualtricsImport import SurveyQualtricsImport
+
+        so = SurveyQualtricsImport(qsf_file)
+        return so.create_survey()
+
     ###################
     # DISPLAY METHODS
     ###################

edsl/surveys/SurveyQualtricsImport.py

@@ -0,0 +1,213 @@
+import json
+import html
+import re
+
+from edsl import Question
+from edsl import Survey
+
+qualtrics_codes = {
+    "TE": "free_text",
+    "MC": "multiple_choice",
+    "Matrix": "matrix",
+    "DB": "free_text",  # not quite right, but for now
+    "Timing": "free_text",  # not quite right, but for now
+}
+# TE (Text Entry): Allows respondents to input a text response.
+# MC (Multiple Choice): Provides respondents with a list of options to choose from.
+# DB (Descriptive Text or Information): Displays text or information without requiring a response.
+# Matrix: A grid-style question where respondents can evaluate multiple items using the same set of response options.
+
+
+def clean_html(raw_html):
+    # Unescape HTML entities
+    clean_text = html.unescape(raw_html)
+    # Remove HTML tags
+    clean_text = re.sub(r"<.*?>", "", clean_text)
+    # Replace non-breaking spaces with regular spaces
+    clean_text = clean_text.replace("\xa0", " ")
+    # Optionally, strip leading/trailing spaces
+    clean_text = clean_text.strip()
+    return clean_text
+
+
+class QualtricsQuestion:
+    def __init__(self, question_json, debug=False):
+        self.debug = debug
+        self.question_json = question_json
+        if self.element != "SQ":
+            raise ValueError("Invalid question element type")
+
+    @property
+    def element(self):
+        return self.question_json["Element"]
+
+    @property
+    def selector(self):
+        return self.question_json.get("Selector", None)
+
+    @property
+    def question_name(self):
+        return self.question_json["PrimaryAttribute"]
+
+    @property
+    def question_text(self):
+        return clean_html(self.question_json["Payload"]["QuestionText"])
+
+    @property
+    def raw_question_type(self):
+        return self.question_json["Payload"]["QuestionType"]
+
+    @property
+    def question_type(self):
+        q_type = qualtrics_codes.get(self.raw_question_type, None)
+        if q_type is None:
+            print(f"Unknown question type: {self.raw_question_type}")
+            return None
+        return q_type
+
+    @property
+    def choices(self):
+        if "Choices" in self.question_json["Payload"]:
+            return [
+                choice["Display"]
+                for choice in self.question_json["Payload"]["Choices"].values()
+            ]
+        return None
+
+    @property
+    def answers(self):
+        if "Answers" in self.question_json["Payload"]:
+            return [
+                choice["Display"]
+                for choice in self.question_json["Payload"]["Choices"].values()
+            ]
+        return None
+
+    def to_edsl(self):
+        if self.question_type == "free_text":
+            try:
+                q = Question(
+                    **{
+                        "question_type": self.question_type,
+                        "question_text": self.question_text,
+                        "question_name": self.question_name,
+                    }
+                )
+                return [q]
+            except Exception as e:
+                return []
+
+        if self.question_type == "multiple_choice":
+            # Let's figure of it it's actually a checkbox question
+            if self.selector == "MAVR" or self.selector == "MULTIPLE":
+                try:
+                    q = Question(
+                        **{
+                            "question_type": "checkbox",
+                            "question_text": self.question_text,
+                            "question_name": self.question_name,
+                            "question_options": self.choices,
+                        }
+                    )
+                    return [q]
+                except Exception as e:
+                    return []
+
+            # maybe it's a linear scale!
+            if "<br>" in self.choices[0]:
+                option_labels = {}
+                question_options = []
+                for choice in self.choices:
+                    if "<br>" in choice:
+                        option_label, question_option = choice.split("<br>")
+                        option_labels[int(question_option)] = option_label
+                        question_options.append(int(question_option))
+                    else:
+                        question_options.append(int(choice))
+                try:
+                    q = Question(
+                        **{
+                            "question_type": "linear_scale",
+                            "question_text": self.question_text,
+                            "question_name": self.question_name,
+                            "question_options": question_options,
+                            "option_labels": option_labels,
+                        }
+                    )
+                    return [q]
+                except Exception as e:
+                    if self.debug:
+                        raise e
+                    else:
+                        print(e)
+                        return []
+
+            try:
+                q = Question(
+                    **{
+                        "question_type": self.question_type,
+                        "question_text": self.question_text,
+                        "question_name": self.question_name,
+                        "question_options": self.choices,
+                    }
+                )
+                return [q]
+            except Exception as e:
+                return []
+
+        if self.question_type == "matrix":
+            questions = []
+            for index, choice in enumerate(self.choices):
+                try:
+                    q = Question(
+                        **{
+                            "question_type": "multiple_choice",
+                            "question_text": self.question_text + f" ({choice})",
+                            "question_name": self.question_name + f"_{index}",
+                            "question_options": self.answers,
+                        }
+                    )
+                    questions.append(q)
+                except Exception as e:
+                    continue
+
+            return questions
+
+        raise ValueError(f"Invalid question type: {self.question_type}")
+
+
+class SurveyQualtricsImport:
+    def __init__(self, qsf_file_name: str):
+        self.qsf_file_name = qsf_file_name
+        self.question_data = self.extract_questions_from_json()
+
+    def create_survey(self):
+        questions = []
+        for qualtrics_questions in self.question_data:
+            questions.extend(qualtrics_questions.to_edsl())
+        return Survey(questions)
+
+    def extract_questions_from_json(self):
+        with open(self.qsf_file_name, "r") as f:
+            survey_data = json.load(f)
+
+        questions = survey_data["SurveyElements"]
+
+        extracted_questions = []
+
+        for question in questions:
+            if question["Element"] == "SQ":
+                extracted_questions.append(QualtricsQuestion(question))
+
+        return extracted_questions
+
+
+if __name__ == "__main__":
+    survey_creator = SurveyQualtricsImport("example.qsf")
+    # print(survey_creator.question_data)
+    survey = survey_creator.create_survey()
+    # info = survey.push()
+    # print(info)
+    # questions = survey.extract_questions_from_json()
+    # for question in questions:
+    #    print(question)