edsl 0.1.39.dev1__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl

edsl/scenarios/ScenarioListExportMixin.py

@@ -27,13 +27,6 @@ def decorate_methods_from_mixin(cls, mixin_cls):
     return cls
 
 
-# def decorate_all_methods(cls):
-#     for attr_name, attr_value in cls.__dict__.items():
-#         if callable(attr_value):
-#             setattr(cls, attr_name, to_dataset(attr_value))
-#     return cls
-
-
 # @decorate_all_methods
 class ScenarioListExportMixin(DatasetExportMixin):
     """Mixin class for exporting Results objects."""

edsl/scenarios/ScenarioListPdfMixin.py

@@ -1,22 +1,9 @@
-import fitz  # PyMuPDF
 import os
-import copy
-import subprocess
-import requests
-import tempfile
-import os
-
-# import urllib.parse as urlparse
-from urllib.parse import urlparse
-
-# from edsl import Scenario
-
-import requests
 import re
-import tempfile
-import os
+import copy
 import atexit
-from urllib.parse import urlparse, parse_qs
+import tempfile
+import subprocess
 
 
 class GoogleDriveDownloader:
@@ -25,6 +12,8 @@ class GoogleDriveDownloader:
 
     @classmethod
     def fetch_from_drive(cls, url, filename=None):
+        import requests
+
         # Extract file ID from the URL
         file_id = cls._extract_file_id(url)
         if not file_id:
@@ -67,6 +56,8 @@ class GoogleDriveDownloader:
 
     @staticmethod
     def _extract_file_id(url):
+        from urllib.parse import urlparse, parse_qs
+
         # Try to extract file ID from '/file/d/' format
         file_id_match = re.search(r"/d/([a-zA-Z0-9-_]+)", url)
         if file_id_match:
@@ -92,6 +83,8 @@ class GoogleDriveDownloader:
 
 def fetch_and_save_pdf(url, filename):
     # Send a GET request to the URL
+    import requests
+
     response = requests.get(url)
 
     # Check if the request was successful
@@ -114,11 +107,6 @@ def fetch_and_save_pdf(url, filename):
     return temp_file_path
 
 
-# Example usage:
-# url = "https://example.com/sample.pdf"
-# fetch_and_save_pdf(url, "sample.pdf")
-
-
 class ScenarioListPdfMixin:
     @classmethod
     def from_pdf(cls, filename_or_url, collapse_pages=False):
@@ -151,6 +139,8 @@ class ScenarioListPdfMixin:
 
     @staticmethod
     def is_url(string):
+        from urllib.parse import urlparse
+
         try:
             result = urlparse(string)
             return all([result.scheme, result.netloc])
@@ -189,7 +179,8 @@ class ScenarioListPdfMixin:
 
     @staticmethod
     def extract_text_from_pdf(pdf_path):
-        from edsl import Scenario
+        from edsl.scenarios.Scenario import Scenario
+        import fitz  # PyMuPDF
 
         # TODO: Add test case
         # Ensure the file exists
@@ -243,19 +234,6 @@ class ScenarioListPdfMixin:
 
 
 if __name__ == "__main__":
-    pass
-
-    # from edsl import ScenarioList
-
-    # class ScenarioListNew(ScenarioList, ScenaroListPdfMixin):
-    #     pass
-
-    # #ScenarioListNew.create_hello_world_pdf('hello_world')
-    # #scenarios = ScenarioListNew.from_pdf('hello_world.pdf')
-    # #print(scenarios)
+    import doctest
 
-    # from edsl import ScenarioList, QuestionFreeText
-    # homo_silicus = ScenarioList.from_pdf('w31122.pdf')
-    # q = QuestionFreeText(question_text = "What is the key point of the text in {{ text }}?", question_name = "key_point")
-    # results = q.by(homo_silicus).run(progress_bar = True)
-    # results.select('scenario.page', 'answer.key_point').order_by('page').print()
+    doctest.testmod()

edsl/scenarios/ScenarioSelector.py

@@ -0,0 +1,156 @@
+from typing import TYPE_CHECKING
+
+
+class ScenarioSelector:
+    """
+    A class for performing advanced field selection on ScenarioList objects,
+    including support for wildcard patterns.
+
+    Args:
+        scenario_list: The ScenarioList object to perform selections on
+
+    Examples:
+        >>> from edsl import Scenario, ScenarioList
+        >>> scenarios = ScenarioList([Scenario({'test_1': 1, 'test_2': 2, 'other': 3}), Scenario({'test_1': 4, 'test_2': 5, 'other': 6})])
+        >>> selector = ScenarioSelector(scenarios)
+        >>> selector.select('test*')
+        ScenarioList([Scenario({'test_1': 1, 'test_2': 2}), Scenario({'test_1': 4, 'test_2': 5})])
+    """
+
+    def __init__(self, scenario_list: "ScenarioList"):
+        """Initialize with a ScenarioList object."""
+        self.scenario_list = scenario_list
+        self.available_fields = (
+            list(scenario_list.data[0].keys()) if scenario_list.data else []
+        )
+
+    def _match_field_pattern(self, pattern: str, field: str) -> bool:
+        """
+        Checks if a field name matches a pattern with wildcards.
+        Supports '*' as wildcard at start or end of pattern.
+
+        Args:
+            pattern: The pattern to match against, may contain '*' at start or end
+            field: The field name to check
+
+        Examples:
+            >>> from edsl.scenarios import ScenarioList, Scenario
+            >>> selector = ScenarioSelector(ScenarioList([]))
+            >>> selector._match_field_pattern('test*', 'test_field')
+            True
+            >>> selector._match_field_pattern('*field', 'test_field')
+            True
+            >>> selector._match_field_pattern('test', 'test')
+            True
+            >>> selector._match_field_pattern('*test*', 'my_test_field')
+            True
+        """
+        if "*" not in pattern:
+            return pattern == field
+
+        if pattern.startswith("*") and pattern.endswith("*"):
+            return pattern[1:-1] in field
+        elif pattern.startswith("*"):
+            return field.endswith(pattern[1:])
+        elif pattern.endswith("*"):
+            return field.startswith(pattern[:-1])
+        return pattern == field
+
+    def _get_matching_fields(self, patterns: list[str]) -> list[str]:
+        """
+        Gets all fields that match any of the given patterns.
+
+        Args:
+            patterns: List of field patterns, may contain wildcards
+
+        Returns:
+            List of field names that match at least one pattern
+
+        Examples:
+            >>> from edsl import Scenario, ScenarioList
+            >>> scenarios = ScenarioList([
+            ...     Scenario({'test_1': 1, 'test_2': 2, 'other': 3})
+            ... ])
+            >>> selector = ScenarioSelector(scenarios)
+            >>> selector._get_matching_fields(['test*'])
+            ['test_1', 'test_2']
+        """
+        matching_fields = set()
+        for pattern in patterns:
+            matches = [
+                field
+                for field in self.available_fields
+                if self._match_field_pattern(pattern, field)
+            ]
+            matching_fields.update(matches)
+        return sorted(list(matching_fields))
+
+    def select(self, *fields) -> "ScenarioList":
+        """
+        Selects scenarios with only the referenced fields.
+        Supports wildcard patterns using '*' at the start or end of field names.
+
+        Args:
+            *fields: Field names or patterns to select. Patterns may include '*' for wildcards.
+
+        Returns:
+            A new ScenarioList containing only the matched fields.
+
+        Raises:
+            ValueError: If no fields match the given patterns.
+
+        Examples:
+            >>> from edsl import Scenario, ScenarioList
+            >>> scenarios = ScenarioList([
+            ...     Scenario({'test_1': 1, 'test_2': 2, 'other': 3}),
+            ...     Scenario({'test_1': 4, 'test_2': 5, 'other': 6})
+            ... ])
+            >>> selector = ScenarioSelector(scenarios)
+            >>> selector.select('test*')  # Selects all fields starting with 'test'
+            ScenarioList([Scenario({'test_1': 1, 'test_2': 2}), Scenario({'test_1': 4, 'test_2': 5})])
+            >>> selector.select('*_1')  # Selects all fields ending with '_1'
+            ScenarioList([Scenario({'test_1': 1}), Scenario({'test_1': 4})])
+            >>> selector.select('test_1', '*_2')  # Multiple patterns
+            ScenarioList([Scenario({'test_1': 1, 'test_2': 2}), Scenario({'test_1': 4, 'test_2': 5})])
+        """
+        if not self.scenario_list.data:
+            return self.scenario_list.__class__([])
+
+        # Convert single string to list for consistent processing
+        patterns = list(fields)
+
+        # Get all fields that match the patterns
+        fields_to_select = self._get_matching_fields(patterns)
+
+        # If no fields match, raise an informative error
+        if not fields_to_select:
+            raise ValueError(
+                f"No fields matched the given patterns: {patterns}. "
+                f"Available fields are: {self.available_fields}"
+            )
+
+        return self.scenario_list.__class__(
+            [scenario.select(fields_to_select) for scenario in self.scenario_list.data]
+        )
+
+    def get_available_fields(self) -> list[str]:
+        """
+        Returns a list of all available fields in the ScenarioList.
+
+        Returns:
+            List of field names available for selection.
+
+        Examples:
+            >>> from edsl import Scenario, ScenarioList
+            >>> scenarios = ScenarioList([Scenario({'test_1': 1, 'test_2': 2, 'other': 3})])
+            >>> selector = ScenarioSelector(scenarios)
+            >>> selector.get_available_fields()
+            ['other', 'test_1', 'test_2']
+        """
+        return sorted(self.available_fields)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/scenarios/__init__.py

@@ -1,4 +1,3 @@
 from edsl.scenarios.Scenario import Scenario
 from edsl.scenarios.ScenarioList import ScenarioList
-
-# from edsl.scenarios.FileStore import FileStore
+from edsl.scenarios.FileStore import FileStore

edsl/scenarios/file_methods.py

@@ -0,0 +1,85 @@
+from typing import Optional, Dict, Type
+from abc import ABC, abstractmethod
+import importlib.metadata
+import importlib.util
+
+from edsl.utilities.is_notebook import is_notebook
+
+
+class FileMethods(ABC):
+    _handlers: Dict[str, Type["FileMethods"]] = {}
+
+    def __init__(self, path: Optional[str] = None):
+        self.path = path
+
+    def __init_subclass__(cls) -> None:
+        """Register subclasses automatically when they're defined."""
+        super().__init_subclass__()
+        if hasattr(cls, "suffix"):
+            FileMethods._handlers[cls.suffix] = cls
+
+    @classmethod
+    def get_handler(cls, suffix: str) -> Optional[Type["FileMethods"]]:
+        """Get the appropriate handler class for a given suffix."""
+        # Load plugins if they haven't been loaded yet
+        if not cls._handlers:
+            cls.load_plugins()
+        return cls._handlers.get(suffix.lower())
+
+    @classmethod
+    def load_plugins(cls):
+        """Load all file handler plugins including built-ins and external plugins."""
+
+        from edsl.scenarios import handlers
+
+        # Then load any external plugins
+        try:
+            entries = importlib.metadata.entry_points(group="file_handlers")
+        except TypeError:  # some Python 3.9 bullshit
+            # entries = importlib.metadata.entry_points()
+            entries = []
+
+        for ep in entries:
+            try:
+                handler_class = ep.load()
+                # Registration happens automatically via __init_subclass__
+            except Exception as e:
+                print(f"Failed to load external handler {ep.name}: {e}")
+
+    @classmethod
+    def get_handler_for_path(cls, path: str) -> Optional[Type["FileMethods"]]:
+        """Get the appropriate handler class for a file path."""
+        suffix = path.split(".")[-1].lower() if "." in path else ""
+        return cls.get_handler(suffix)
+
+    @classmethod
+    def create(cls, path: str) -> Optional["FileMethods"]:
+        """Create an appropriate handler instance for the given path."""
+        handler_class = cls.get_handler_for_path(path)
+        if handler_class:
+            return handler_class(path)
+        return None
+
+    @classmethod
+    def supported_file_types(cls):
+        if not cls._handlers:
+            cls.load_plugins()
+        return list(cls._handlers.keys())
+
+    @abstractmethod
+    def view_system(self):
+        ...
+
+    @abstractmethod
+    def view_notebook(self):
+        ...
+
+    def view(self):
+        if is_notebook():
+            self.view_notebook()
+        else:
+            self.view_system()
+
+    @abstractmethod
+    def example(self):
+        ...

edsl/scenarios/handlers/__init__.py

@@ -0,0 +1,13 @@
+from .pdf import PdfMethods
+from .docx import DocxMethods
+from .png import PngMethods
+from .txt import TxtMethods
+from .html import HtmlMethods
+from .md import MarkdownMethods
+from .csv import CsvMethods
+from .json import JsonMethods
+from .sql import SqlMethods
+from .pptx import PptxMethods
+from .latex import LaTeXMethods
+from .py import PyMethods
+from .sqlite import SQLiteMethods

edsl/scenarios/handlers/csv.py

@@ -0,0 +1,38 @@
+import tempfile
+from edsl.scenarios.file_methods import FileMethods
+
+
+class CsvMethods(FileMethods):
+    suffix = "csv"
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening CSV: {e}")
+        else:
+            print("CSV file was not found.")
+
+    def view_notebook(self):
+        import pandas as pd
+        from IPython.display import display
+
+        df = pd.read_csv(self.path)
+        display(df)
+
+    def example(self):
+        import pandas as pd
+
+        df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as f:
+            df.to_csv(f.name, index=False)
+        return f.name

edsl/scenarios/handlers/docx.py

@@ -0,0 +1,76 @@
+from edsl.scenarios.file_methods import FileMethods
+import os
+import tempfile
+
+
+class DocxMethods(FileMethods):
+    suffix = "docx"
+
+    def extract_text(self):
+        from docx import Document
+
+        self.doc = Document(self.path)
+
+        # Extract all text
+        full_text = []
+        for para in self.doc.paragraphs:
+            full_text.append(para.text)
+
+        text = "\n".join(full_text)
+        return text
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening DOCX: {e}")
+        else:
+            print("DOCX file was not found.")
+
+    def view_notebook(self):
+        import mammoth
+        from IPython.display import HTML, display
+
+        with open(self.path, "rb") as docx_file:
+            result = mammoth.convert_to_html(docx_file)
+            html = f"""
+            <div style="width: 800px; height: 800px; padding: 20px; 
+                       border: 1px solid #ccc; overflow-y: auto;">
+                {result.value}
+            </div>
+            """
+            display(HTML(html))
+
+    def example(self):
+        from docx import Document
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.scenarios.ScenarioList import ScenarioList
+
+        os.makedirs("test_dir", exist_ok=True)
+        doc1 = Document()
+        _ = doc1.add_heading("First Survey")
+        doc1.save("test_dir/test1.docx")
+        doc2 = Document()
+        _ = doc2.add_heading("Second Survey")
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".docx") as tmp:
+            doc2.save(tmp.name)
+            tmp.close()
+
+        return tmp.name
+
+
+if __name__ == "__main__":
+    docx_temp = DocxMethods.example()
+    from edsl.scenarios.FileStore import FileStore
+
+    fs = FileStore(docx_temp)

edsl/scenarios/handlers/html.py

@@ -0,0 +1,37 @@
+from edsl.scenarios.file_methods import FileMethods
+import tempfile
+
+
+class HtmlMethods(FileMethods):
+    suffix = "html"
+
+    def view_system(self):
+        import webbrowser
+
+        # with open(self.path, "r") as f:
+        #     html_string = f.read()
+
+        # html_path = self.to_tempfile()
+        # webbrowser.open("file://" + html_path)
+        webbrowser.open("file://" + self.path)
+
+    def view_notebook(self):
+        from IPython.display import IFrame, display
+
+        display(IFrame(self.path, width=800, height=800))
+
+    def example(self):
+        html_string = b"""
+            <html>
+            <head>
+                <title>Test</title>
+            </head>
+            <body>
+                <h1>Hello, World!</h1>
+            </body>
+            </html>
+            """
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f:
+            f.write(html_string)
+        return f.name

edsl/scenarios/handlers/json.py

@@ -0,0 +1,111 @@
+from edsl.scenarios.file_methods import FileMethods
+import tempfile
+import json
+from typing import Optional, Dict, Any
+
+
+class JsonMethods(FileMethods):
+    suffix = "json"
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening JSON: {e}")
+        else:
+            print("JSON file was not found.")
+
+    def view_notebook(self):
+        from IPython.display import FileLink, JSON, display
+        import json
+
+        # Read and parse the JSON file
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+
+            # Display formatted JSON
+            display(JSON(content))
+
+            # Provide download link
+            display(FileLink(self.path))
+        except json.JSONDecodeError as e:
+            print(f"Error parsing JSON: {e}")
+        except Exception as e:
+            print(f"Error reading file: {e}")
+
+    def validate_json(self, schema: Optional[Dict[str, Any]] = None) -> bool:
+        """
+        Validate the JSON file against a schema if provided,
+        or check if it's valid JSON if no schema is provided.
+        """
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+
+            if schema is not None:
+                from jsonschema import validate
+
+                validate(instance=content, schema=schema)
+
+            return True
+        except json.JSONDecodeError as e:
+            print(f"Invalid JSON format: {e}")
+            return False
+        except Exception as e:
+            print(f"Validation error: {e}")
+            return False
+
+    def pretty_print(self):
+        """Pretty print the JSON content with proper indentation."""
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+
+            pretty_json = json.dumps(content, indent=2, sort_keys=True)
+            print(pretty_json)
+        except Exception as e:
+            print(f"Error pretty printing JSON: {e}")
+
+    def example(self):
+        sample_json = {
+            "person": {
+                "name": "John Doe",
+                "age": 30,
+                "contact": {"email": "john@example.com", "phone": "+1-555-555-5555"},
+                "interests": ["programming", "data science", "machine learning"],
+                "active": True,
+                "metadata": {"last_updated": "2024-01-01", "version": 1.0},
+            }
+        }
+
+        with tempfile.NamedTemporaryFile(
+            delete=False, suffix=".json", mode="w", encoding="utf-8"
+        ) as f:
+            json.dump(sample_json, f, indent=2)
+        return f.name
+
+    def format_file(self):
+        """Read, format, and write back the JSON with consistent formatting."""
+        try:
+            # Read the current content
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+
+            # Write back with consistent formatting
+            with open(self.path, "w", encoding="utf-8") as f:
+                json.dump(content, f, indent=2, sort_keys=True)
+
+            return True
+        except Exception as e:
+            print(f"Error formatting JSON file: {e}")
+            return False

edsl/scenarios/handlers/latex.py

@@ -0,0 +1,5 @@
+from edsl.scenarios.handlers.txt import TxtMethods
+
+
+class LaTeXMethods(TxtMethods):
+    pass