edsl 0.1.27.dev2__py3-none-any.whl → 0.1.29__py3-none-any.whl

edsl/conjure/InputDataMixinQuestionStats.py

@@ -0,0 +1,182 @@
+import functools
+from typing import List
+from edsl.conjure.utilities import Missing
+from collections import Counter
+
+
+class InputDataMixinQuestionStats:
+    def question_statistics(self, question_name: str) -> "QuestionStats":
+        """Return statistics for a question."""
+        return self.QuestionStats(**self._compute_question_statistics(question_name))
+
+    def _compute_question_statistics(self, question_name: str) -> dict:
+        """
+        Return a dictionary of statistics for a question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id._compute_question_statistics('morning')
+        {'num_responses': 2, 'num_unique_responses': 2, 'missing': 0, 'unique_responses': ..., 'frac_numerical': 0.0, 'top_5': [('1', 1), ('4', 1)], 'frac_obs_from_top_5': 1.0}
+        """
+        idx = self.question_names.index(question_name)
+        return {attr: getattr(self, attr)[idx] for attr in self.question_attributes}
+
+    @property
+    def num_responses(self) -> List[int]:
+        """
+        Return the number of responses for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id.num_responses
+        [2, 2]
+        """
+        return self.compute_num_responses()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_num_responses(self):
+        return [len(responses) for responses in self.raw_data]
+
+    @property
+    def num_unique_responses(self) -> List[int]:
+        """
+        The number of unique responses for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id.num_unique_responses
+        [2, 2]
+        """
+        return self.compute_num_unique_responses()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_num_unique_responses(self):
+        return [len(set(responses)) for responses in self.raw_data]
+
+    @property
+    def missing(self) -> List[int]:
+        """The number of observations that are missing.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> input_data = InputDataABC.example(raw_data = [[1,2,Missing().value()]], question_texts = ['A question'])
+        >>> input_data.missing
+        [1]
+
+        """
+        return self.compute_missing()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_missing(self):
+        return [sum([1 for x in v if x == Missing().value()]) for v in self.raw_data]
+
+    @property
+    def frac_numerical(self) -> List[float]:
+        """
+        The fraction of responses that are numerical for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> input_data = InputDataABC.example(raw_data = [[1,2,"Poop", 3]], question_texts = ['A question'])
+        >>> input_data.frac_numerical
+        [0.75]
+        """
+        return self.compute_frac_numerical()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_frac_numerical(self):
+        return [
+            sum([1 for x in v if isinstance(x, (int, float))]) / len(v)
+            for v in self.raw_data
+        ]
+
+    @functools.lru_cache(maxsize=1)
+    def top_k(self, k: int) -> List[List[tuple]]:
+        """
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> input_data = InputDataABC.example(raw_data = [[1,1,1,1,1,2]], question_texts = ['A question'])
+        >>> input_data.top_k(1)
+        [[(1, 5)]]
+        >>> input_data.top_k(2)
+        [[(1, 5), (2, 1)]]
+        """
+        return [Counter(value).most_common(k) for value in self.raw_data]
+
+    @functools.lru_cache(maxsize=1)
+    def frac_obs_from_top_k(self, k):
+        """
+        Return the fraction of observations that are in the top k for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> input_data = InputDataABC.example(raw_data = [[1,1,1,1,1,1,1,1,2, 3]], question_names = ['a'])
+        >>> input_data.frac_obs_from_top_k(1)
+        [0.8]
+        """
+        return [
+            round(
+                sum([x[1] for x in Counter(value).most_common(k) if x[0] != "missing"])
+                / len(value),
+                2,
+            )
+            for value in self.raw_data
+        ]
+
+    @property
+    def frac_obs_from_top_5(self):
+        """The fraction of observations that are in the top 5 for each question."""
+        return self.frac_obs_from_top_k(5)
+
+    @property
+    def top_5(self):
+        """The top 5 responses for each question."""
+        return self.top_k(5)
+
+    @property
+    def unique_responses(self) -> List[List[str]]:
+        """Return a list of unique responses for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id.unique_responses
+        [..., ...]
+        """
+        return self.compute_unique_responses()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_unique_responses(self):
+        return [
+            list(set(self.filter_missing(responses))) for responses in self.raw_data
+        ]
+
+    @staticmethod
+    def filter_missing(responses) -> List[str]:
+        """Return a list of responses with missing values removed."""
+        return [
+            v
+            for v in responses
+            if v != Missing().value() and v != "missing" and v != ""
+        ]
+
+    def unique_responses_more_than_k(self, k, remove_missing=True) -> List[List[str]]:
+        """Return a list of unique responses that occur more than k times for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id.unique_responses_more_than_k(1)
+        [[...], [...]]
+
+        """
+        counters = [Counter(responses) for responses in self.raw_data]
+        new_counters = []
+        for question in counters:
+            top_options = []
+            for option, count in question.items():
+                if count > k and (option != "missing" or not remove_missing):
+                    top_options.append(option)
+            new_counters.append(top_options)
+        return new_counters
+
+
+if __name__ == "__main__":
+    from edsl.conjure.InputData import InputDataABC
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/conjure/InputDataPyRead.py

@@ -0,0 +1,91 @@
+import pandas as pd
+from typing import List
+
+from edsl.conjure.InputData import InputDataABC
+from edsl.conjure.utilities import convert_value
+from edsl.utilities.utilities import is_valid_variable_name
+
+try:
+    import pyreadstat
+except ImportError as e:
+    raise ImportError(
+        "The 'pyreadstat' package is required for this feature. Please install it by running:\n"
+        "pip install pyreadstat\n"
+    ) from e
+
+
+class InputDataPyRead(InputDataABC):
+    def pyread_function(self, datafile_name):
+        raise NotImplementedError
+
+    def _parse(self) -> None:
+        try:
+            df, meta = self.pyread_function(self.datafile_name)
+        except Exception as e:
+            raise ValueError(
+                f"An error occurred while reading the file {self.datafile_name}."
+            ) from e
+        float_columns = df.select_dtypes(include=["float64"]).columns
+        df[float_columns] = df[float_columns].astype(str)
+
+        df.fillna("", inplace=True)
+        df = df.astype(str)
+        self._df = df
+        self._meta = meta
+
+    def get_df(self) -> pd.DataFrame:
+        if not hasattr(self, "_df"):
+            self._parse()
+        return self._df
+
+    def get_answer_codebook(self):
+        if not hasattr(self, "_meta"):
+            self._parse()
+
+        question_name_to_label_name = self._meta.variable_to_label
+        label_name_to_labels = self._meta.value_labels
+        return {
+            qn: label_name_to_labels[label_name]
+            for qn, label_name in question_name_to_label_name.items()
+        }
+
+    def get_raw_data(self) -> List[List[str]]:
+        df = self.get_df()
+        data = [
+            [convert_value(obs) for obs in v]
+            for k, v in df.to_dict(orient="list").items()
+        ]
+        return data
+
+    @property
+    def question_names_to_question_texts(self):
+        """Return a dictionary of question names to question texts.
+        This will repair the question names if they are not valid Python identifiers using the
+        same question_name_repair_func that was passed in.
+        """
+        if not hasattr(self, "_meta"):
+            self._parse()
+        d = {}
+        for qn, label in self._meta.column_names_to_labels.items():
+            new_name = qn
+            if not is_valid_variable_name(qn):
+                new_name = self.question_name_repair_func(qn)
+                if not is_valid_variable_name(new_name):
+                    raise ValueError(
+                        f"""Question names must be valid Python identifiers. '{qn}' is not.""",
+                        """You can pass an entry in question_name_repair_dict to fix this.""",
+                    )
+            if label is not None:
+                d[new_name] = label
+        return d
+
+    def get_question_texts(self):
+        if not hasattr(self, "_meta"):
+            self._parse()
+        return [
+            self.question_names_to_question_texts.get(qn, qn)
+            for qn in self.question_names
+        ]
+
+    def get_question_names(self):
+        return self.get_df().columns.tolist()

edsl/conjure/InputDataSPSS.py

@@ -0,0 +1,8 @@
+from edsl.conjure.InputDataPyRead import InputDataPyRead
+
+
+class InputDataSPSS(InputDataPyRead):
+    def pyread_function(self, datafile_name):
+        from pyreadstat import read_sav
+
+        return read_sav(datafile_name)

edsl/conjure/InputDataStata.py

@@ -0,0 +1,8 @@
+from edsl.conjure.InputDataPyRead import InputDataPyRead
+
+
+class InputDataStata(InputDataPyRead):
+    def pyread_function(self, datafile_name):
+        from pyreadstat import read_dta
+
+        return read_dta(datafile_name)

edsl/conjure/QuestionOptionMixin.py

@@ -0,0 +1,76 @@
+from typing import Union, List
+
+
+class QuestionOptionMixin:
+    @property
+    def question_options(self):
+        if not hasattr(self, "_question_options"):
+            self.question_options = None
+        return self._question_options
+
+    @question_options.setter
+    def question_options(self, value):
+        if value is None:
+            value = [self._get_question_options(qn) for qn in self.question_names]
+        self._question_options = value
+
+    def _get_question_options(self, question_name) -> Union[List[str], None]:
+        """Return the options for a question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> sorted(id._get_question_options('morning'))
+        ['1', '4']
+
+        """
+        qt = self.question_statistics(question_name)
+        idx = self.question_names.index(question_name)
+        question_type = self.question_types[idx]
+        if question_type == "multiple_choice":
+            return [str(o) for o in qt.unique_responses]
+        else:
+            if question_type == "multiple_choice_with_other":
+                options = self.unique_responses_more_than_k(2)[
+                    self.question_names.index(question_name)
+                ] + [self.OTHER_STRING]
+                return [str(o) for o in options]
+            else:
+                return None
+
+    def order_options(self) -> None:
+        """Order the options for multiple choice questions using an LLM."""
+        from edsl import QuestionList, ScenarioList
+        import textwrap
+
+        scenarios = (
+            ScenarioList.from_list("example_question_name", self.question_names)
+            .add_list("example_question_text", self.question_texts)
+            .add_list("example_question_type", self.question_types)
+            .add_list("example_question_options", self.question_options)
+        ).filter(
+            'example_question_type == "multiple_choice" or example_question_type == "multiple_choice_with_other"'
+        )
+
+        question = QuestionList(
+            question_text=textwrap.dedent(
+                """\
+            We have a survey question: `{{ example_question_text }}`.
+            
+            The survey had following options: '{{ example_question_options }}'.
+            The options might be out of order. Please put them in the correct order.
+            If there is not natural order, just put then in order they were presented.
+            """
+            ),
+            question_name="ordering",
+        )
+        proposed_ordering = question.by(scenarios).run()
+        d = dict(
+            proposed_ordering.select("example_question_name", "ordering").to_list()
+        )
+        self._question_options = [d.get(qn, None) for qn in self.question_names]
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/conjure/QuestionTypeMixin.py

@@ -0,0 +1,23 @@
+class QuestionTypeMixin:
+    @property
+    def question_types(self):
+        if not hasattr(self, "_question_types"):
+            self.question_types = None
+        return self._question_types
+
+    @question_types.setter
+    def question_types(self, value):
+        if value is None:
+            value = [self._infer_question_type(qn) for qn in self.question_names]
+        self._question_types = value
+
+    def _infer_question_type(self, question_name) -> str:
+        qt = self.question_statistics(question_name)
+        if qt.num_unique_responses > self.NUM_UNIQUE_THRESHOLD:
+            if qt.frac_numerical > self.FRAC_NUMERICAL_THRESHOLD:
+                return "numerical"
+            if qt.frac_obs_from_top_5 > self.MULTIPLE_CHOICE_OTHER_THRESHOLD:
+                return "multiple_choice_with_other"
+            return "free_text"
+        else:
+            return "multiple_choice"

edsl/conjure/RawQuestion.py

@@ -0,0 +1,65 @@
+from dataclasses import dataclass, field
+from typing import List, Optional, Union
+from edsl.questions import QuestionBase
+from edsl import Question
+
+from edsl.conjure.utilities import convert_value
+
+
+@dataclass
+class RawQuestion:
+    """
+    A class to represent a question before it is converted to edsl class.
+
+    >>> rq = RawQuestion.example()
+    >>> rq.to_question()
+    Question('multiple_choice', question_name = \"""how_are_you\""", question_text = \"""How are you doing?\""", question_options = ['Good', 'Bad'])
+    """
+
+    question_type: str
+    question_name: str
+    question_text: str
+    responses: List[str] = field(default_factory=list)
+    question_options: Optional[List[str]] = None
+
+    @classmethod
+    def example(cls):
+        return cls(
+            question_type="multiple_choice",
+            question_name="how_are_you",
+            question_text="How are you doing?",
+            responses=["Good", "Bad", "Bad", "Good"],
+            question_options=["Good", "Bad"],
+        )
+
+    def __post_init__(self):
+        self.responses = [convert_value(r) for r in self.responses]
+
+    def to_question(self) -> QuestionBase:
+        """Return a Question object from the RawQuestion."""
+
+        # TODO: Remove this once we have a better way to handle multiple_choice_with_other
+        if self.question_type == "multiple_choice_with_other":
+            question_type = "multiple_choice"
+        else:
+            question_type = self.question_type
+
+        # exclude responses from the dictionary if they have a None value; don't inlcude responses in the dictionary
+        d = {
+            k: v
+            for k, v in {
+                "question_type": question_type,
+                "question_name": self.question_name,
+                "question_text": self.question_text,
+                "responses": self.responses,
+                "question_options": self.question_options,
+            }.items()
+            if v is not None and k != "responses"
+        }
+        return Question(**d)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/conjure/SurveyResponses.py

@@ -0,0 +1,7 @@
+from typing import Dict, List
+from collections import UserDict
+
+
+class SurveyResponses(UserDict):
+    def __init__(self, responses: Dict[str, List[str]]):
+        super().__init__(responses)

edsl/conjure/__init__.py

@@ -1,4 +1,9 @@
-from edsl.conjure.SurveyBuilder import SurveyBuilder
-from edsl.conjure.SurveyBuilderCSV import SurveyBuilderCSV
-from edsl.conjure.SurveyBuilderCSV import SurveyBuilderStata
-from edsl.conjure.SurveyBuilderSPSS import SurveyBuilderSPSS
+# from edsl.conjure.SurveyBuilder import SurveyBuilder
+# from edsl.conjure.SurveyBuilderCSV import SurveyBuilderCSV
+# from edsl.conjure.SurveyBuilderCSV import SurveyBuilderStata
+# from edsl.conjure.SurveyBuilderSPSS import SurveyBuilderSPSS
+# from edsl.conjure.InputData.InputDataSPSS import InputDataSPSS
+# from edsl.conjure.InputData.InputDataCSV import InputDataCSV
+# from edsl.conjure.InputData.InputDataStata import InputDataStata
+# from edsl.conjure.InputData import InputDataSPSS
+# from edsl.conjure.InputData import InputData