edsl 0.1.27.dev2__py3-none-any.whl → 0.1.28__py3-none-any.whl

edsl/conjure/naming_utilities.py

@@ -0,0 +1,263 @@
+import re
+import keyword
+
+stop_words = {
+    "into",
+    "mustn't",
+    "there",
+    "you'll",
+    "don",
+    "have",
+    "at",
+    "if",
+    "on",
+    "some",
+    "with",
+    "in",
+    "can",
+    "mightn",
+    "off",
+    "few",
+    "not",
+    "d",
+    "hadn",
+    "shan't",
+    "t",
+    "re",
+    "where",
+    "s",
+    "won't",
+    "mustn",
+    "wasn't",
+    "didn't",
+    "has",
+    "same",
+    "too",
+    "will",
+    "you've",
+    "all",
+    "haven't",
+    "isn't",
+    "over",
+    "of",
+    "about",
+    "its",
+    "being",
+    "it",
+    "her",
+    "should",
+    "himself",
+    "wasn",
+    "out",
+    "theirs",
+    "aren",
+    "that",
+    "our",
+    "shouldn't",
+    "you'd",
+    "such",
+    "above",
+    "my",
+    "the",
+    "any",
+    "been",
+    "as",
+    "very",
+    "herself",
+    "o",
+    "weren",
+    "until",
+    "their",
+    "shouldn",
+    "up",
+    "wouldn",
+    "couldn't",
+    "hasn't",
+    "no",
+    "than",
+    "hadn't",
+    "had",
+    "you",
+    "here",
+    "yourself",
+    "yourselves",
+    "during",
+    "ain",
+    "once",
+    "aren't",
+    "what",
+    "so",
+    "hers",
+    "that'll",
+    "other",
+    "ours",
+    "yours",
+    "nor",
+    "him",
+    "doesn",
+    "doesn't",
+    "he",
+    "them",
+    "for",
+    "ll",
+    "isn",
+    "this",
+    "or",
+    "who",
+    "only",
+    "itself",
+    "they",
+    "between",
+    "against",
+    "under",
+    "me",
+    "now",
+    "mightn't",
+    "those",
+    "needn't",
+    "these",
+    "when",
+    "before",
+    "his",
+    "she's",
+    "having",
+    "be",
+    "don't",
+    "haven",
+    "won",
+    "while",
+    "both",
+    "didn",
+    "by",
+    "ourselves",
+    "m",
+    "your",
+    "then",
+    "myself",
+    "we",
+    "it's",
+    "should've",
+    "through",
+    "why",
+    "from",
+    "and",
+    "hasn",
+    "more",
+    "how",
+    "ve",
+    "most",
+    "because",
+    "did",
+    "y",
+    "i",
+    "an",
+    "but",
+    "whom",
+    "below",
+    "further",
+    "am",
+    "which",
+    "just",
+    "ma",
+    "you're",
+    "couldn",
+    "do",
+    "shan",
+    "own",
+    "again",
+    "are",
+    "weren't",
+    "down",
+    "is",
+    "were",
+    "each",
+    "needn",
+    "themselves",
+    "she",
+    "after",
+    "does",
+    "wouldn't",
+    "to",
+    "a",
+    "was",
+    "doing",
+}
+
+
+def sanitize_string(input_string, max_length=35):
+    """Return a sanitized version of the input string that can be used as a variable name.
+
+    >>> candidate_names = ["How are you doing this morning, Dave? What is your favorite kind of coffee?", "class", "def", "here_is_some_text"]
+    >>> [sanitize_string(name) for name in candidate_names]
+    ['morning_dave_favorite_kind_coffee', 'class_modified', 'def_modified', 'here_is_some_text']
+    """
+
+    # Ensure nltk stopwords are downloaded
+    # try:
+    #     from nltk.corpus import stopwords
+    # except ImportError or ModuleNotFoundError:
+    #     print(
+    #         "nltk is not installed. Please install it using 'pip install nltk' to use these features."
+    #     )
+    #     raise
+
+    # try:
+    #     stop_words = set(stopwords.words("english"))
+    # except LookupError:
+    #     nltk.download("stopwords")
+    #     stop_words = set(stopwords.words("english"))
+    #     # raise LookupError("Stopwords not found. Please download them using nltk.download('stopwords')")
+
+    # # Define the list of stopwords
+
+    # Replace special characters with spaces and split into words
+    words = re.sub(r"\W+", " ", input_string).split()
+
+    # Remove stopwords
+    important_words = [word for word in words if word.lower() not in stop_words]
+
+    # Join words with underscores
+    sanitized_string = "_".join(important_words)
+
+    # Ensure the length is less than 25 characters
+    if len(sanitized_string) > max_length:
+        # split off the last word and remove it
+        words = sanitized_string[:max_length].split("_")
+        if len(words) == 1:
+            sanitized_string = words[0]
+        else:
+            sanitized_string = "_".join(words[:-1])
+
+    # Remove leading and trailing underscores
+    sanitized_string = sanitized_string.strip("_")
+
+    # Check if the string is a Python keyword
+    if keyword.iskeyword(sanitized_string):
+        sanitized_string += "_modified"
+
+    result = sanitized_string.lower()
+    return result
+
+
+# Example usage
+# input_string = "This is a sample variable-name@123 for testing"
+# sanitized_string = sanitize_string(input_string)
+# print(sanitized_string)  # Output might be: sample_variable_name_123
+
+# if __name__ == "__main__":
+#     candidate_names = [
+#         "How are you doing this morning, Dave? What is your favorite kind of coffee?",
+#         "class",
+#         "def",
+#         "here_is_some_text",
+#     ]
+#     for name in candidate_names:
+#         print(f"Original: {name}")
+#         print(f"Sanitized: {sanitize_string(name)}")
+#         print()
+
+if __name__ == "__main__":
+    from edsl.conjure.InputData import InputDataABC
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/conjure/utilities.py

@@ -1,9 +1,59 @@
+import requests
 import subprocess
 from io import StringIO
-
+import os
 import pandas as pd
 
 
+class ValidFilename:
+    """A descriptor that checks if a file exists.
+
+
+    >>> f = ValidFilename()
+    >>> f = "hello"
+    """
+
+    def __set_name__(self, owner, name):
+        self.name = name
+
+    def __get__(self, instance, owner):
+        return instance.__dict__.get(self.name, None)
+
+    def __set__(self, instance, value):
+        if not isinstance(value, str):
+            raise ValueError(
+                f"The filename must be a string, not {type(value).__name__}"
+            )
+
+        if not os.path.exists(value):
+            raise ValueError(f"The file '{value}' does not exist.")
+
+        instance.__dict__[self.name] = value
+
+
+class DummyClassToTestDescriptor:
+    """
+
+    >>> d = DummyClassToTestDescriptor(1)
+    Traceback (most recent call last):
+    ...
+    ValueError: The filename must be a string, not int
+
+    >>> d = DummyClassToTestDescriptor("hello")
+    Traceback (most recent call last):
+    ...
+    ValueError: The file 'hello' does not exist.
+    """
+
+    filename = ValidFilename()
+
+    def __init__(self, filename):
+        self.filename = filename
+
+    def __repr__(self):
+        return f"DummyClassToTestDescriptor({self.filename})"
+
+
 class Missing:
     def __repr__(self):
         return "Missing()"
@@ -16,6 +66,18 @@ class Missing:
 
 
 def convert_value(x):
+    """Takes a string and tries to convert it.
+
+    >>> convert_value('1')
+    1
+    >>> convert_value('1.2')
+    1.2
+    >>> convert_value("how are you?")
+    'how are you?'
+    >>> convert_value("")
+    'missing'
+
+    """
     try:
         float_val = float(x)
         if float_val.is_integer():
@@ -29,36 +91,111 @@ def convert_value(x):
             return str(x)
 
 
-class RCodeSnippet:
-    def __init__(self, r_code):
-        self.r_code = r_code
+# class RCodeSnippet:
+#     def __init__(self, r_code):
+#         self.r_code = r_code
 
-    def __call__(self, data_file_name):
-        return self.run_R_stdin(self.r_code, data_file_name)
+#     def __call__(self, data_file_name):
+#         return self.run_R_stdin(self.r_code, data_file_name)
 
-    def __add__(self, other):
-        return RCodeSnippet(self.r_code + other.r_code)
+#     def __add__(self, other):
+#         return RCodeSnippet(self.r_code + other.r_code)
 
-    def write_to_file(self, filename) -> None:
-        """Writes the R code to a file; useful for debugging."""
-        if filename.endswith(".R") or filename.endswith(".r"):
-            pass
-        else:
-            filename += ".R"
+#     def write_to_file(self, filename) -> None:
+#         """Writes the R code to a file; useful for debugging."""
+#         if filename.endswith(".R") or filename.endswith(".r"):
+#             pass
+#         else:
+#             filename += ".R"
 
-        with open(filename, "w") as f:
-            f.write(self.r_code)
+#         with open(filename, "w") as f:
+#             f.write(self.r_code)
 
-    @staticmethod
-    def run_R_stdin(r_code, data_file_name, transform_func=lambda x: pd.read_csv(x)):
-        """Runs an R script and returns the stdout as a string."""
-        cmd = ["Rscript", "-e", r_code, data_file_name]
-        process = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+#     @staticmethod
+#     def run_R_stdin(r_code, data_file_name, transform_func=lambda x: pd.read_csv(x)):
+#         """Runs an R script and returns the stdout as a string."""
+#         cmd = ["Rscript", "-e", r_code, data_file_name]
+#         process = subprocess.Popen(
+#             cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+#         )
+#         stdout, stderr = process.communicate()
+#         if stderr != "":
+#             print("Warning: stderr is not empty.")
+#             print(f"Problem running: {r_code}")
+#             raise Exception(stderr)
+#         return transform_func(StringIO(stdout))
+
+
+def infer_question_type(question_text, responses, sample_size=15):
+    from edsl.questions import QuestionMultipleChoice
+
+    q = QuestionMultipleChoice(
+        question_text="""We have a survey question and we are trying to infer its type.
+                        The question text is: '{{question_text}}'.                                   
+                        The first {{ sample_size }} responses are: '{{responses}}'.
+                        There are {{ total }} responses in total.
+                        If a response is a command-separated list, it is likely a checkbox question.
+                        """,
+        question_name="infer_question_type",
+        question_options=[
+            "budget",
+            "checkbox",
+            "extract",
+            "free_text",
+            "likert_five",
+            "linear_scale",
+            "list",
+            "multiple_choice",
+            "numerical",
+            "rank",
+            "top_k",
+            "yes_no",
+        ],
+    )
+    response = (
+        q.to_survey()(
+            question_text=question_text,
+            sample_zize=sample_size,
+            responses=responses[:sample_size],
         )
-        stdout, stderr = process.communicate()
-        if stderr != "":
-            print("Warning: stderr is not empty.")
-            print(f"Problem running: {r_code}")
-            raise Exception(stderr)
-        return transform_func(StringIO(stdout))
+        .select("infer_question_type")
+        .first()
+    )
+    return response
+
+
+def download_file(url, filename):
+    """
+    Downloads a file from a given URL and saves it to the specified filename.
+
+    Parameters:
+    url (str): The URL of the file to download.
+    filename (str): The name of the file to save the downloaded content.
+
+    Returns:
+    str: The path to the saved file.
+    """
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    }
+
+    # Sending the GET request
+    response = requests.get(url, headers=headers)
+
+    # Checking if the request was successful
+    if response.status_code == 200:
+        # Writing the content to the specified file
+        with open(filename, "wb") as file:
+            file.write(response.content)
+        print(f"File downloaded successfully and saved to {filename}")
+        return filename
+    else:
+        print(f"Failed to download file: {response.status_code}")
+        return None
+
+
+# Example usage
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)