edsl 0.1.37.dev5__py3-none-any.whl → 0.1.38__py3-none-any.whl

edsl/conjure/InputDataMixinQuestionStats.py

@@ -1,182 +0,0 @@
-import functools
-from typing import List
-from edsl.conjure.utilities import Missing
-from collections import Counter
-
-
-class InputDataMixinQuestionStats:
-    def question_statistics(self, question_name: str) -> "QuestionStats":
-        """Return statistics for a question."""
-        return self.QuestionStats(**self._compute_question_statistics(question_name))
-
-    def _compute_question_statistics(self, question_name: str) -> dict:
-        """
-        Return a dictionary of statistics for a question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id._compute_question_statistics('morning')
-        {'num_responses': 2, 'num_unique_responses': 2, 'missing': 0, 'unique_responses': ..., 'frac_numerical': 0.0, 'top_5': [('1', 1), ('4', 1)], 'frac_obs_from_top_5': 1.0}
-        """
-        idx = self.question_names.index(question_name)
-        return {attr: getattr(self, attr)[idx] for attr in self.question_attributes}
-
-    @property
-    def num_responses(self) -> List[int]:
-        """
-        Return the number of responses for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id.num_responses
-        [2, 2]
-        """
-        return self.compute_num_responses()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_num_responses(self):
-        return [len(responses) for responses in self.raw_data]
-
-    @property
-    def num_unique_responses(self) -> List[int]:
-        """
-        The number of unique responses for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id.num_unique_responses
-        [2, 2]
-        """
-        return self.compute_num_unique_responses()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_num_unique_responses(self):
-        return [len(set(responses)) for responses in self.raw_data]
-
-    @property
-    def missing(self) -> List[int]:
-        """The number of observations that are missing.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> input_data = InputDataABC.example(raw_data = [[1,2,Missing().value()]], question_texts = ['A question'])
-        >>> input_data.missing
-        [1]
-
-        """
-        return self.compute_missing()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_missing(self):
-        return [sum([1 for x in v if x == Missing().value()]) for v in self.raw_data]
-
-    @property
-    def frac_numerical(self) -> List[float]:
-        """
-        The fraction of responses that are numerical for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> input_data = InputDataABC.example(raw_data = [[1,2,"Poop", 3]], question_texts = ['A question'])
-        >>> input_data.frac_numerical
-        [0.75]
-        """
-        return self.compute_frac_numerical()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_frac_numerical(self):
-        return [
-            sum([1 for x in v if isinstance(x, (int, float))]) / len(v)
-            for v in self.raw_data
-        ]
-
-    @functools.lru_cache(maxsize=1)
-    def top_k(self, k: int) -> List[List[tuple]]:
-        """
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> input_data = InputDataABC.example(raw_data = [[1,1,1,1,1,2]], question_texts = ['A question'])
-        >>> input_data.top_k(1)
-        [[(1, 5)]]
-        >>> input_data.top_k(2)
-        [[(1, 5), (2, 1)]]
-        """
-        return [Counter(value).most_common(k) for value in self.raw_data]
-
-    @functools.lru_cache(maxsize=1)
-    def frac_obs_from_top_k(self, k):
-        """
-        Return the fraction of observations that are in the top k for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> input_data = InputDataABC.example(raw_data = [[1,1,1,1,1,1,1,1,2, 3]], question_names = ['a'])
-        >>> input_data.frac_obs_from_top_k(1)
-        [0.8]
-        """
-        return [
-            round(
-                sum([x[1] for x in Counter(value).most_common(k) if x[0] != "missing"])
-                / len(value),
-                2,
-            )
-            for value in self.raw_data
-        ]
-
-    @property
-    def frac_obs_from_top_5(self):
-        """The fraction of observations that are in the top 5 for each question."""
-        return self.frac_obs_from_top_k(5)
-
-    @property
-    def top_5(self):
-        """The top 5 responses for each question."""
-        return self.top_k(5)
-
-    @property
-    def unique_responses(self) -> List[List[str]]:
-        """Return a list of unique responses for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id.unique_responses
-        [..., ...]
-        """
-        return self.compute_unique_responses()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_unique_responses(self):
-        return [
-            list(set(self.filter_missing(responses))) for responses in self.raw_data
-        ]
-
-    @staticmethod
-    def filter_missing(responses) -> List[str]:
-        """Return a list of responses with missing values removed."""
-        return [
-            v
-            for v in responses
-            if v != Missing().value() and v != "missing" and v != ""
-        ]
-
-    def unique_responses_more_than_k(self, k, remove_missing=True) -> List[List[str]]:
-        """Return a list of unique responses that occur more than k times for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id.unique_responses_more_than_k(1)
-        [[...], [...]]
-
-        """
-        counters = [Counter(responses) for responses in self.raw_data]
-        new_counters = []
-        for question in counters:
-            top_options = []
-            for option, count in question.items():
-                if count > k and (option != "missing" or not remove_missing):
-                    top_options.append(option)
-            new_counters.append(top_options)
-        return new_counters
-
-
-if __name__ == "__main__":
-    from edsl.conjure.InputData import InputDataABC
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/conjure/InputDataPyRead.py

@@ -1,91 +0,0 @@
-import pandas as pd
-from typing import List
-
-from edsl.conjure.InputData import InputDataABC
-from edsl.conjure.utilities import convert_value
-from edsl.utilities.utilities import is_valid_variable_name
-
-try:
-    import pyreadstat
-except ImportError as e:
-    raise ImportError(
-        "The 'pyreadstat' package is required for this feature. Please install it by running:\n"
-        "pip install pyreadstat\n"
-    ) from e
-
-
-class InputDataPyRead(InputDataABC):
-    def pyread_function(self, datafile_name):
-        raise NotImplementedError
-
-    def _parse(self) -> None:
-        try:
-            df, meta = self.pyread_function(self.datafile_name)
-        except Exception as e:
-            raise ValueError(
-                f"An error occurred while reading the file {self.datafile_name}."
-            ) from e
-        float_columns = df.select_dtypes(include=["float64"]).columns
-        df[float_columns] = df[float_columns].astype(str)
-
-        df.fillna("", inplace=True)
-        df = df.astype(str)
-        self._df = df
-        self._meta = meta
-
-    def get_df(self) -> pd.DataFrame:
-        if not hasattr(self, "_df"):
-            self._parse()
-        return self._df
-
-    def get_answer_codebook(self):
-        if not hasattr(self, "_meta"):
-            self._parse()
-
-        question_name_to_label_name = self._meta.variable_to_label
-        label_name_to_labels = self._meta.value_labels
-        return {
-            qn: label_name_to_labels[label_name]
-            for qn, label_name in question_name_to_label_name.items()
-        }
-
-    def get_raw_data(self) -> List[List[str]]:
-        df = self.get_df()
-        data = [
-            [convert_value(obs) for obs in v]
-            for k, v in df.to_dict(orient="list").items()
-        ]
-        return data
-
-    @property
-    def question_names_to_question_texts(self):
-        """Return a dictionary of question names to question texts.
-        This will repair the question names if they are not valid Python identifiers using the
-        same question_name_repair_func that was passed in.
-        """
-        if not hasattr(self, "_meta"):
-            self._parse()
-        d = {}
-        for qn, label in self._meta.column_names_to_labels.items():
-            new_name = qn
-            if not is_valid_variable_name(qn):
-                new_name = self.question_name_repair_func(qn)
-                if not is_valid_variable_name(new_name):
-                    raise ValueError(
-                        f"""Question names must be valid Python identifiers. '{qn}' is not.""",
-                        """You can pass an entry in question_name_repair_dict to fix this.""",
-                    )
-            if label is not None:
-                d[new_name] = label
-        return d
-
-    def get_question_texts(self):
-        if not hasattr(self, "_meta"):
-            self._parse()
-        return [
-            self.question_names_to_question_texts.get(qn, qn)
-            for qn in self.question_names
-        ]
-
-    def get_question_names(self):
-        return self.get_df().columns.tolist()

edsl/conjure/InputDataSPSS.py

@@ -1,8 +0,0 @@
-from edsl.conjure.InputDataPyRead import InputDataPyRead
-
-
-class InputDataSPSS(InputDataPyRead):
-    def pyread_function(self, datafile_name):
-        from pyreadstat import read_sav
-
-        return read_sav(datafile_name)

edsl/conjure/InputDataStata.py

@@ -1,8 +0,0 @@
-from edsl.conjure.InputDataPyRead import InputDataPyRead
-
-
-class InputDataStata(InputDataPyRead):
-    def pyread_function(self, datafile_name):
-        from pyreadstat import read_dta
-
-        return read_dta(datafile_name)

edsl/conjure/QuestionOptionMixin.py

@@ -1,76 +0,0 @@
-from typing import Union, List
-
-
-class QuestionOptionMixin:
-    @property
-    def question_options(self):
-        if not hasattr(self, "_question_options"):
-            self.question_options = None
-        return self._question_options
-
-    @question_options.setter
-    def question_options(self, value):
-        if value is None:
-            value = [self._get_question_options(qn) for qn in self.question_names]
-        self._question_options = value
-
-    def _get_question_options(self, question_name) -> Union[List[str], None]:
-        """Return the options for a question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> sorted(id._get_question_options('morning'))
-        ['1', '4']
-
-        """
-        qt = self.question_statistics(question_name)
-        idx = self.question_names.index(question_name)
-        question_type = self.question_types[idx]
-        if question_type == "multiple_choice":
-            return [str(o) for o in qt.unique_responses]
-        else:
-            if question_type == "multiple_choice_with_other":
-                options = self.unique_responses_more_than_k(2)[
-                    self.question_names.index(question_name)
-                ] + [self.OTHER_STRING]
-                return [str(o) for o in options]
-            else:
-                return None
-
-    def order_options(self) -> None:
-        """Order the options for multiple choice questions using an LLM."""
-        from edsl import QuestionList, ScenarioList
-        import textwrap
-
-        scenarios = (
-            ScenarioList.from_list("example_question_name", self.question_names)
-            .add_list("example_question_text", self.question_texts)
-            .add_list("example_question_type", self.question_types)
-            .add_list("example_question_options", self.question_options)
-        ).filter(
-            'example_question_type == "multiple_choice" or example_question_type == "multiple_choice_with_other"'
-        )
-
-        question = QuestionList(
-            question_text=textwrap.dedent(
-                """\
-            We have a survey question: `{{ example_question_text }}`.
-            
-            The survey had following options: '{{ example_question_options }}'.
-            The options might be out of order. Please put them in the correct order.
-            If there is not natural order, just put then in order they were presented.
-            """
-            ),
-            question_name="ordering",
-        )
-        proposed_ordering = question.by(scenarios).run()
-        d = dict(
-            proposed_ordering.select("example_question_name", "ordering").to_list()
-        )
-        self._question_options = [d.get(qn, None) for qn in self.question_names]
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/conjure/QuestionTypeMixin.py

@@ -1,23 +0,0 @@
-class QuestionTypeMixin:
-    @property
-    def question_types(self):
-        if not hasattr(self, "_question_types"):
-            self.question_types = None
-        return self._question_types
-
-    @question_types.setter
-    def question_types(self, value):
-        if value is None:
-            value = [self._infer_question_type(qn) for qn in self.question_names]
-        self._question_types = value
-
-    def _infer_question_type(self, question_name) -> str:
-        qt = self.question_statistics(question_name)
-        if qt.num_unique_responses > self.NUM_UNIQUE_THRESHOLD:
-            if qt.frac_numerical > self.FRAC_NUMERICAL_THRESHOLD:
-                return "numerical"
-            if qt.frac_obs_from_top_5 > self.MULTIPLE_CHOICE_OTHER_THRESHOLD:
-                return "multiple_choice_with_other"
-            return "free_text"
-        else:
-            return "multiple_choice"

edsl/conjure/RawQuestion.py

@@ -1,65 +0,0 @@
-from dataclasses import dataclass, field
-from typing import List, Optional, Union
-from edsl.questions import QuestionBase
-from edsl import Question
-
-from edsl.conjure.utilities import convert_value
-
-
-@dataclass
-class RawQuestion:
-    """
-    A class to represent a question before it is converted to edsl class.
-
-    >>> rq = RawQuestion.example()
-    >>> rq.to_question()
-    Question('multiple_choice', question_name = \"""how_are_you\""", question_text = \"""How are you doing?\""", question_options = ['Good', 'Bad'])
-    """
-
-    question_type: str
-    question_name: str
-    question_text: str
-    responses: List[str] = field(default_factory=list)
-    question_options: Optional[List[str]] = None
-
-    @classmethod
-    def example(cls):
-        return cls(
-            question_type="multiple_choice",
-            question_name="how_are_you",
-            question_text="How are you doing?",
-            responses=["Good", "Bad", "Bad", "Good"],
-            question_options=["Good", "Bad"],
-        )
-
-    def __post_init__(self):
-        self.responses = [convert_value(r) for r in self.responses]
-
-    def to_question(self) -> QuestionBase:
-        """Return a Question object from the RawQuestion."""
-
-        # TODO: Remove this once we have a better way to handle multiple_choice_with_other
-        if self.question_type == "multiple_choice_with_other":
-            question_type = "multiple_choice"
-        else:
-            question_type = self.question_type
-
-        # exclude responses from the dictionary if they have a None value; don't inlcude responses in the dictionary
-        d = {
-            k: v
-            for k, v in {
-                "question_type": question_type,
-                "question_name": self.question_name,
-                "question_text": self.question_text,
-                "responses": self.responses,
-                "question_options": self.question_options,
-            }.items()
-            if v is not None and k != "responses"
-        }
-        return Question(**d)
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/conjure/SurveyResponses.py

@@ -1,7 +0,0 @@
-from typing import Dict, List
-from collections import UserDict
-
-
-class SurveyResponses(UserDict):
-    def __init__(self, responses: Dict[str, List[str]]):
-        super().__init__(responses)

edsl/conjure/__init__.py

@@ -1,9 +0,0 @@
-# from edsl.conjure.SurveyBuilder import SurveyBuilder
-# from edsl.conjure.SurveyBuilderCSV import SurveyBuilderCSV
-# from edsl.conjure.SurveyBuilderCSV import SurveyBuilderStata
-# from edsl.conjure.SurveyBuilderSPSS import SurveyBuilderSPSS
-# from edsl.conjure.InputData.InputDataSPSS import InputDataSPSS
-# from edsl.conjure.InputData.InputDataCSV import InputDataCSV
-# from edsl.conjure.InputData.InputDataStata import InputDataStata
-# from edsl.conjure.InputData import InputDataSPSS
-# from edsl.conjure.InputData import InputData

edsl/conjure/utilities.py

@@ -1,201 +0,0 @@
-import requests
-import subprocess
-from io import StringIO
-import os
-import pandas as pd
-
-
-class ValidFilename:
-    """A descriptor that checks if a file exists.
-
-
-    >>> f = ValidFilename()
-    >>> f = "hello"
-    """
-
-    def __set_name__(self, owner, name):
-        self.name = name
-
-    def __get__(self, instance, owner):
-        return instance.__dict__.get(self.name, None)
-
-    def __set__(self, instance, value):
-        if not isinstance(value, str):
-            raise ValueError(
-                f"The filename must be a string, not {type(value).__name__}"
-            )
-
-        if not os.path.exists(value):
-            raise ValueError(f"The file '{value}' does not exist.")
-
-        instance.__dict__[self.name] = value
-
-
-class DummyClassToTestDescriptor:
-    """
-
-    >>> d = DummyClassToTestDescriptor(1)
-    Traceback (most recent call last):
-    ...
-    ValueError: The filename must be a string, not int
-
-    >>> d = DummyClassToTestDescriptor("hello")
-    Traceback (most recent call last):
-    ...
-    ValueError: The file 'hello' does not exist.
-    """
-
-    filename = ValidFilename()
-
-    def __init__(self, filename):
-        self.filename = filename
-
-    def __repr__(self):
-        return f"DummyClassToTestDescriptor({self.filename})"
-
-
-class Missing:
-    def __repr__(self):
-        return "Missing()"
-
-    def __str__(self):
-        return "Missing()"
-
-    def value(self):
-        return "missing"
-
-
-def convert_value(x):
-    """Takes a string and tries to convert it.
-
-    >>> convert_value('1')
-    1
-    >>> convert_value('1.2')
-    1.2
-    >>> convert_value("how are you?")
-    'how are you?'
-    >>> convert_value("")
-    'missing'
-
-    """
-    try:
-        float_val = float(x)
-        if float_val.is_integer():
-            return int(float_val)
-        else:
-            return float_val
-    except ValueError:
-        if len(x) == 0:
-            return Missing().value()
-        else:
-            return str(x)
-
-
-# class RCodeSnippet:
-#     def __init__(self, r_code):
-#         self.r_code = r_code
-
-#     def __call__(self, data_file_name):
-#         return self.run_R_stdin(self.r_code, data_file_name)
-
-#     def __add__(self, other):
-#         return RCodeSnippet(self.r_code + other.r_code)
-
-#     def write_to_file(self, filename) -> None:
-#         """Writes the R code to a file; useful for debugging."""
-#         if filename.endswith(".R") or filename.endswith(".r"):
-#             pass
-#         else:
-#             filename += ".R"
-
-#         with open(filename, "w") as f:
-#             f.write(self.r_code)
-
-#     @staticmethod
-#     def run_R_stdin(r_code, data_file_name, transform_func=lambda x: pd.read_csv(x)):
-#         """Runs an R script and returns the stdout as a string."""
-#         cmd = ["Rscript", "-e", r_code, data_file_name]
-#         process = subprocess.Popen(
-#             cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
-#         )
-#         stdout, stderr = process.communicate()
-#         if stderr != "":
-#             print("Warning: stderr is not empty.")
-#             print(f"Problem running: {r_code}")
-#             raise Exception(stderr)
-#         return transform_func(StringIO(stdout))
-
-
-def infer_question_type(question_text, responses, sample_size=15):
-    from edsl.questions import QuestionMultipleChoice
-
-    q = QuestionMultipleChoice(
-        question_text="""We have a survey question and we are trying to infer its type.
-                        The question text is: '{{question_text}}'.                                   
-                        The first {{ sample_size }} responses are: '{{responses}}'.
-                        There are {{ total }} responses in total.
-                        If a response is a command-separated list, it is likely a checkbox question.
-                        """,
-        question_name="infer_question_type",
-        question_options=[
-            "budget",
-            "checkbox",
-            "extract",
-            "free_text",
-            "likert_five",
-            "linear_scale",
-            "list",
-            "multiple_choice",
-            "numerical",
-            "rank",
-            "top_k",
-            "yes_no",
-        ],
-    )
-    response = (
-        q.to_survey()(
-            question_text=question_text,
-            sample_zize=sample_size,
-            responses=responses[:sample_size],
-        )
-        .select("infer_question_type")
-        .first()
-    )
-    return response
-
-
-def download_file(url, filename):
-    """
-    Downloads a file from a given URL and saves it to the specified filename.
-
-    Parameters:
-    url (str): The URL of the file to download.
-    filename (str): The name of the file to save the downloaded content.
-
-    Returns:
-    str: The path to the saved file.
-    """
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    }
-
-    # Sending the GET request
-    response = requests.get(url, headers=headers)
-
-    # Checking if the request was successful
-    if response.status_code == 200:
-        # Writing the content to the specified file
-        with open(filename, "wb") as file:
-            file.write(response.content)
-        print(f"File downloaded successfully and saved to {filename}")
-        return filename
-    else:
-        print(f"Failed to download file: {response.status_code}")
-        return None
-
-
-# Example usage
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)