edsl 0.1.36.dev6__py3-none-any.whl → 0.1.37__py3-none-any.whl

edsl/conjure/InputDataCSV.py

@@ -1,48 +1,48 @@
-from typing import List, Optional
-import pandas as pd
-from edsl.conjure.InputData import InputDataABC
-from edsl.conjure.utilities import convert_value
-
-
-class InputDataCSV(InputDataABC):
-    def __init__(self, datafile_name: str, config: Optional[dict] = None, **kwargs):
-        if config is None:
-            config = {"skiprows": None, "delimiter": ","}
-
-        super().__init__(datafile_name, config, **kwargs)
-
-    def get_df(self) -> pd.DataFrame:
-        if not hasattr(self, "_df"):
-            self._df = pd.read_csv(
-                self.datafile_name,
-                skiprows=self.config["skiprows"],
-                encoding_errors="ignore",
-            )
-            float_columns = self._df.select_dtypes(include=["float64"]).columns
-            self._df[float_columns] = self._df[float_columns].astype(str)
-            self._df.fillna("", inplace=True)
-            self._df = self._df.astype(str)
-        return self._df
-
-    def get_raw_data(self) -> List[List[str]]:
-        data = [
-            [convert_value(obs) for obs in v]
-            for k, v in self.get_df().to_dict(orient="list").items()
-        ]
-        return data
-
-    def get_question_texts(self):
-        return list(self.get_df().columns)
-
-    def get_question_names(self):
-        new_names = [self.naming_function(q) for q in self.question_texts]
-
-        if len(new_names) > len(set(new_names)):
-            from collections import Counter
-
-            counter = Counter(new_names)
-            for i, name in enumerate(new_names):
-                if counter[name] > 1:
-                    new_names[i] = name + str(counter[name])
-                    counter[name] -= 1
-        return new_names
+from typing import List, Optional
+import pandas as pd
+from edsl.conjure.InputData import InputDataABC
+from edsl.conjure.utilities import convert_value
+
+
+class InputDataCSV(InputDataABC):
+    def __init__(self, datafile_name: str, config: Optional[dict] = None, **kwargs):
+        if config is None:
+            config = {"skiprows": None, "delimiter": ","}
+
+        super().__init__(datafile_name, config, **kwargs)
+
+    def get_df(self) -> pd.DataFrame:
+        if not hasattr(self, "_df"):
+            self._df = pd.read_csv(
+                self.datafile_name,
+                skiprows=self.config["skiprows"],
+                encoding_errors="ignore",
+            )
+            float_columns = self._df.select_dtypes(include=["float64"]).columns
+            self._df[float_columns] = self._df[float_columns].astype(str)
+            self._df.fillna("", inplace=True)
+            self._df = self._df.astype(str)
+        return self._df
+
+    def get_raw_data(self) -> List[List[str]]:
+        data = [
+            [convert_value(obs) for obs in v]
+            for k, v in self.get_df().to_dict(orient="list").items()
+        ]
+        return data
+
+    def get_question_texts(self):
+        return list(self.get_df().columns)
+
+    def get_question_names(self):
+        new_names = [self.naming_function(q) for q in self.question_texts]
+
+        if len(new_names) > len(set(new_names)):
+            from collections import Counter
+
+            counter = Counter(new_names)
+            for i, name in enumerate(new_names):
+                if counter[name] > 1:
+                    new_names[i] = name + str(counter[name])
+                    counter[name] -= 1
+        return new_names

edsl/conjure/InputDataMixinQuestionStats.py

@@ -1,182 +1,182 @@
-import functools
-from typing import List
-from edsl.conjure.utilities import Missing
-from collections import Counter
-
-
-class InputDataMixinQuestionStats:
-    def question_statistics(self, question_name: str) -> "QuestionStats":
-        """Return statistics for a question."""
-        return self.QuestionStats(**self._compute_question_statistics(question_name))
-
-    def _compute_question_statistics(self, question_name: str) -> dict:
-        """
-        Return a dictionary of statistics for a question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id._compute_question_statistics('morning')
-        {'num_responses': 2, 'num_unique_responses': 2, 'missing': 0, 'unique_responses': ..., 'frac_numerical': 0.0, 'top_5': [('1', 1), ('4', 1)], 'frac_obs_from_top_5': 1.0}
-        """
-        idx = self.question_names.index(question_name)
-        return {attr: getattr(self, attr)[idx] for attr in self.question_attributes}
-
-    @property
-    def num_responses(self) -> List[int]:
-        """
-        Return the number of responses for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id.num_responses
-        [2, 2]
-        """
-        return self.compute_num_responses()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_num_responses(self):
-        return [len(responses) for responses in self.raw_data]
-
-    @property
-    def num_unique_responses(self) -> List[int]:
-        """
-        The number of unique responses for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id.num_unique_responses
-        [2, 2]
-        """
-        return self.compute_num_unique_responses()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_num_unique_responses(self):
-        return [len(set(responses)) for responses in self.raw_data]
-
-    @property
-    def missing(self) -> List[int]:
-        """The number of observations that are missing.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> input_data = InputDataABC.example(raw_data = [[1,2,Missing().value()]], question_texts = ['A question'])
-        >>> input_data.missing
-        [1]
-
-        """
-        return self.compute_missing()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_missing(self):
-        return [sum([1 for x in v if x == Missing().value()]) for v in self.raw_data]
-
-    @property
-    def frac_numerical(self) -> List[float]:
-        """
-        The fraction of responses that are numerical for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> input_data = InputDataABC.example(raw_data = [[1,2,"Poop", 3]], question_texts = ['A question'])
-        >>> input_data.frac_numerical
-        [0.75]
-        """
-        return self.compute_frac_numerical()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_frac_numerical(self):
-        return [
-            sum([1 for x in v if isinstance(x, (int, float))]) / len(v)
-            for v in self.raw_data
-        ]
-
-    @functools.lru_cache(maxsize=1)
-    def top_k(self, k: int) -> List[List[tuple]]:
-        """
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> input_data = InputDataABC.example(raw_data = [[1,1,1,1,1,2]], question_texts = ['A question'])
-        >>> input_data.top_k(1)
-        [[(1, 5)]]
-        >>> input_data.top_k(2)
-        [[(1, 5), (2, 1)]]
-        """
-        return [Counter(value).most_common(k) for value in self.raw_data]
-
-    @functools.lru_cache(maxsize=1)
-    def frac_obs_from_top_k(self, k):
-        """
-        Return the fraction of observations that are in the top k for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> input_data = InputDataABC.example(raw_data = [[1,1,1,1,1,1,1,1,2, 3]], question_names = ['a'])
-        >>> input_data.frac_obs_from_top_k(1)
-        [0.8]
-        """
-        return [
-            round(
-                sum([x[1] for x in Counter(value).most_common(k) if x[0] != "missing"])
-                / len(value),
-                2,
-            )
-            for value in self.raw_data
-        ]
-
-    @property
-    def frac_obs_from_top_5(self):
-        """The fraction of observations that are in the top 5 for each question."""
-        return self.frac_obs_from_top_k(5)
-
-    @property
-    def top_5(self):
-        """The top 5 responses for each question."""
-        return self.top_k(5)
-
-    @property
-    def unique_responses(self) -> List[List[str]]:
-        """Return a list of unique responses for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id.unique_responses
-        [..., ...]
-        """
-        return self.compute_unique_responses()
-
-    @functools.lru_cache(maxsize=1)
-    def compute_unique_responses(self):
-        return [
-            list(set(self.filter_missing(responses))) for responses in self.raw_data
-        ]
-
-    @staticmethod
-    def filter_missing(responses) -> List[str]:
-        """Return a list of responses with missing values removed."""
-        return [
-            v
-            for v in responses
-            if v != Missing().value() and v != "missing" and v != ""
-        ]
-
-    def unique_responses_more_than_k(self, k, remove_missing=True) -> List[List[str]]:
-        """Return a list of unique responses that occur more than k times for each question.
-
-        >>> from edsl.conjure.InputData import InputDataABC
-        >>> id = InputDataABC.example()
-        >>> id.unique_responses_more_than_k(1)
-        [[...], [...]]
-
-        """
-        counters = [Counter(responses) for responses in self.raw_data]
-        new_counters = []
-        for question in counters:
-            top_options = []
-            for option, count in question.items():
-                if count > k and (option != "missing" or not remove_missing):
-                    top_options.append(option)
-            new_counters.append(top_options)
-        return new_counters
-
-
-if __name__ == "__main__":
-    from edsl.conjure.InputData import InputDataABC
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
+import functools
+from typing import List
+from edsl.conjure.utilities import Missing
+from collections import Counter
+
+
+class InputDataMixinQuestionStats:
+    def question_statistics(self, question_name: str) -> "QuestionStats":
+        """Return statistics for a question."""
+        return self.QuestionStats(**self._compute_question_statistics(question_name))
+
+    def _compute_question_statistics(self, question_name: str) -> dict:
+        """
+        Return a dictionary of statistics for a question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id._compute_question_statistics('morning')
+        {'num_responses': 2, 'num_unique_responses': 2, 'missing': 0, 'unique_responses': ..., 'frac_numerical': 0.0, 'top_5': [('1', 1), ('4', 1)], 'frac_obs_from_top_5': 1.0}
+        """
+        idx = self.question_names.index(question_name)
+        return {attr: getattr(self, attr)[idx] for attr in self.question_attributes}
+
+    @property
+    def num_responses(self) -> List[int]:
+        """
+        Return the number of responses for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id.num_responses
+        [2, 2]
+        """
+        return self.compute_num_responses()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_num_responses(self):
+        return [len(responses) for responses in self.raw_data]
+
+    @property
+    def num_unique_responses(self) -> List[int]:
+        """
+        The number of unique responses for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id.num_unique_responses
+        [2, 2]
+        """
+        return self.compute_num_unique_responses()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_num_unique_responses(self):
+        return [len(set(responses)) for responses in self.raw_data]
+
+    @property
+    def missing(self) -> List[int]:
+        """The number of observations that are missing.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> input_data = InputDataABC.example(raw_data = [[1,2,Missing().value()]], question_texts = ['A question'])
+        >>> input_data.missing
+        [1]
+
+        """
+        return self.compute_missing()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_missing(self):
+        return [sum([1 for x in v if x == Missing().value()]) for v in self.raw_data]
+
+    @property
+    def frac_numerical(self) -> List[float]:
+        """
+        The fraction of responses that are numerical for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> input_data = InputDataABC.example(raw_data = [[1,2,"Poop", 3]], question_texts = ['A question'])
+        >>> input_data.frac_numerical
+        [0.75]
+        """
+        return self.compute_frac_numerical()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_frac_numerical(self):
+        return [
+            sum([1 for x in v if isinstance(x, (int, float))]) / len(v)
+            for v in self.raw_data
+        ]
+
+    @functools.lru_cache(maxsize=1)
+    def top_k(self, k: int) -> List[List[tuple]]:
+        """
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> input_data = InputDataABC.example(raw_data = [[1,1,1,1,1,2]], question_texts = ['A question'])
+        >>> input_data.top_k(1)
+        [[(1, 5)]]
+        >>> input_data.top_k(2)
+        [[(1, 5), (2, 1)]]
+        """
+        return [Counter(value).most_common(k) for value in self.raw_data]
+
+    @functools.lru_cache(maxsize=1)
+    def frac_obs_from_top_k(self, k):
+        """
+        Return the fraction of observations that are in the top k for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> input_data = InputDataABC.example(raw_data = [[1,1,1,1,1,1,1,1,2, 3]], question_names = ['a'])
+        >>> input_data.frac_obs_from_top_k(1)
+        [0.8]
+        """
+        return [
+            round(
+                sum([x[1] for x in Counter(value).most_common(k) if x[0] != "missing"])
+                / len(value),
+                2,
+            )
+            for value in self.raw_data
+        ]
+
+    @property
+    def frac_obs_from_top_5(self):
+        """The fraction of observations that are in the top 5 for each question."""
+        return self.frac_obs_from_top_k(5)
+
+    @property
+    def top_5(self):
+        """The top 5 responses for each question."""
+        return self.top_k(5)
+
+    @property
+    def unique_responses(self) -> List[List[str]]:
+        """Return a list of unique responses for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id.unique_responses
+        [..., ...]
+        """
+        return self.compute_unique_responses()
+
+    @functools.lru_cache(maxsize=1)
+    def compute_unique_responses(self):
+        return [
+            list(set(self.filter_missing(responses))) for responses in self.raw_data
+        ]
+
+    @staticmethod
+    def filter_missing(responses) -> List[str]:
+        """Return a list of responses with missing values removed."""
+        return [
+            v
+            for v in responses
+            if v != Missing().value() and v != "missing" and v != ""
+        ]
+
+    def unique_responses_more_than_k(self, k, remove_missing=True) -> List[List[str]]:
+        """Return a list of unique responses that occur more than k times for each question.
+
+        >>> from edsl.conjure.InputData import InputDataABC
+        >>> id = InputDataABC.example()
+        >>> id.unique_responses_more_than_k(1)
+        [[...], [...]]
+
+        """
+        counters = [Counter(responses) for responses in self.raw_data]
+        new_counters = []
+        for question in counters:
+            top_options = []
+            for option, count in question.items():
+                if count > k and (option != "missing" or not remove_missing):
+                    top_options.append(option)
+            new_counters.append(top_options)
+        return new_counters
+
+
+if __name__ == "__main__":
+    from edsl.conjure.InputData import InputDataABC
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/conjure/InputDataPyRead.py

@@ -1,91 +1,91 @@
-import pandas as pd
-from typing import List
-
-from edsl.conjure.InputData import InputDataABC
-from edsl.conjure.utilities import convert_value
-from edsl.utilities.utilities import is_valid_variable_name
-
-try:
-    import pyreadstat
-except ImportError as e:
-    raise ImportError(
-        "The 'pyreadstat' package is required for this feature. Please install it by running:\n"
-        "pip install pyreadstat\n"
-    ) from e
-
-
-class InputDataPyRead(InputDataABC):
-    def pyread_function(self, datafile_name):
-        raise NotImplementedError
-
-    def _parse(self) -> None:
-        try:
-            df, meta = self.pyread_function(self.datafile_name)
-        except Exception as e:
-            raise ValueError(
-                f"An error occurred while reading the file {self.datafile_name}."
-            ) from e
-        float_columns = df.select_dtypes(include=["float64"]).columns
-        df[float_columns] = df[float_columns].astype(str)
-
-        df.fillna("", inplace=True)
-        df = df.astype(str)
-        self._df = df
-        self._meta = meta
-
-    def get_df(self) -> pd.DataFrame:
-        if not hasattr(self, "_df"):
-            self._parse()
-        return self._df
-
-    def get_answer_codebook(self):
-        if not hasattr(self, "_meta"):
-            self._parse()
-
-        question_name_to_label_name = self._meta.variable_to_label
-        label_name_to_labels = self._meta.value_labels
-        return {
-            qn: label_name_to_labels[label_name]
-            for qn, label_name in question_name_to_label_name.items()
-        }
-
-    def get_raw_data(self) -> List[List[str]]:
-        df = self.get_df()
-        data = [
-            [convert_value(obs) for obs in v]
-            for k, v in df.to_dict(orient="list").items()
-        ]
-        return data
-
-    @property
-    def question_names_to_question_texts(self):
-        """Return a dictionary of question names to question texts.
-        This will repair the question names if they are not valid Python identifiers using the
-        same question_name_repair_func that was passed in.
-        """
-        if not hasattr(self, "_meta"):
-            self._parse()
-        d = {}
-        for qn, label in self._meta.column_names_to_labels.items():
-            new_name = qn
-            if not is_valid_variable_name(qn):
-                new_name = self.question_name_repair_func(qn)
-                if not is_valid_variable_name(new_name):
-                    raise ValueError(
-                        f"""Question names must be valid Python identifiers. '{qn}' is not.""",
-                        """You can pass an entry in question_name_repair_dict to fix this.""",
-                    )
-            if label is not None:
-                d[new_name] = label
-        return d
-
-    def get_question_texts(self):
-        if not hasattr(self, "_meta"):
-            self._parse()
-        return [
-            self.question_names_to_question_texts.get(qn, qn)
-            for qn in self.question_names
-        ]
-
-    def get_question_names(self):
-        return self.get_df().columns.tolist()
+import pandas as pd
+from typing import List
+
+from edsl.conjure.InputData import InputDataABC
+from edsl.conjure.utilities import convert_value
+from edsl.utilities.utilities import is_valid_variable_name
+
+try:
+    import pyreadstat
+except ImportError as e:
+    raise ImportError(
+        "The 'pyreadstat' package is required for this feature. Please install it by running:\n"
+        "pip install pyreadstat\n"
+    ) from e
+
+
+class InputDataPyRead(InputDataABC):
+    def pyread_function(self, datafile_name):
+        raise NotImplementedError
+
+    def _parse(self) -> None:
+        try:
+            df, meta = self.pyread_function(self.datafile_name)
+        except Exception as e:
+            raise ValueError(
+                f"An error occurred while reading the file {self.datafile_name}."
+            ) from e
+        float_columns = df.select_dtypes(include=["float64"]).columns
+        df[float_columns] = df[float_columns].astype(str)
+
+        df.fillna("", inplace=True)
+        df = df.astype(str)
+        self._df = df
+        self._meta = meta
+
+    def get_df(self) -> pd.DataFrame:
+        if not hasattr(self, "_df"):
+            self._parse()
+        return self._df
+
+    def get_answer_codebook(self):
+        if not hasattr(self, "_meta"):
+            self._parse()
+
+        question_name_to_label_name = self._meta.variable_to_label
+        label_name_to_labels = self._meta.value_labels
+        return {
+            qn: label_name_to_labels[label_name]
+            for qn, label_name in question_name_to_label_name.items()
+        }
+
+    def get_raw_data(self) -> List[List[str]]:
+        df = self.get_df()
+        data = [
+            [convert_value(obs) for obs in v]
+            for k, v in df.to_dict(orient="list").items()
+        ]
+        return data
+
+    @property
+    def question_names_to_question_texts(self):
+        """Return a dictionary of question names to question texts.
+        This will repair the question names if they are not valid Python identifiers using the
+        same question_name_repair_func that was passed in.
+        """
+        if not hasattr(self, "_meta"):
+            self._parse()
+        d = {}
+        for qn, label in self._meta.column_names_to_labels.items():
+            new_name = qn
+            if not is_valid_variable_name(qn):
+                new_name = self.question_name_repair_func(qn)
+                if not is_valid_variable_name(new_name):
+                    raise ValueError(
+                        f"""Question names must be valid Python identifiers. '{qn}' is not.""",
+                        """You can pass an entry in question_name_repair_dict to fix this.""",
+                    )
+            if label is not None:
+                d[new_name] = label
+        return d
+
+    def get_question_texts(self):
+        if not hasattr(self, "_meta"):
+            self._parse()
+        return [
+            self.question_names_to_question_texts.get(qn, qn)
+            for qn in self.question_names
+        ]
+
+    def get_question_names(self):
+        return self.get_df().columns.tolist()

edsl/conjure/InputDataSPSS.py

@@ -1,8 +1,8 @@
-from edsl.conjure.InputDataPyRead import InputDataPyRead
-
-
-class InputDataSPSS(InputDataPyRead):
-    def pyread_function(self, datafile_name):
-        from pyreadstat import read_sav
-
-        return read_sav(datafile_name)
+from edsl.conjure.InputDataPyRead import InputDataPyRead
+
+
+class InputDataSPSS(InputDataPyRead):
+    def pyread_function(self, datafile_name):
+        from pyreadstat import read_sav
+
+        return read_sav(datafile_name)