edsl 0.1.38.dev4__py3-none-any.whl → 0.1.39__py3-none-any.whl

edsl/scenarios/ScenarioListExportMixin.py

@@ -27,13 +27,6 @@ def decorate_methods_from_mixin(cls, mixin_cls):
     return cls
 
 
-# def decorate_all_methods(cls):
-#     for attr_name, attr_value in cls.__dict__.items():
-#         if callable(attr_value):
-#             setattr(cls, attr_name, to_dataset(attr_value))
-#     return cls
-
-
 # @decorate_all_methods
 class ScenarioListExportMixin(DatasetExportMixin):
     """Mixin class for exporting Results objects."""

edsl/scenarios/ScenarioListPdfMixin.py

@@ -1,22 +1,9 @@
-import fitz  # PyMuPDF
 import os
-import copy
-import subprocess
-import requests
-import tempfile
-import os
-
-# import urllib.parse as urlparse
-from urllib.parse import urlparse
-
-# from edsl import Scenario
-
-import requests
 import re
-import tempfile
-import os
+import copy
 import atexit
-from urllib.parse import urlparse, parse_qs
+import tempfile
+import subprocess
 
 
 class GoogleDriveDownloader:
@@ -25,6 +12,8 @@ class GoogleDriveDownloader:
 
     @classmethod
     def fetch_from_drive(cls, url, filename=None):
+        import requests
+
         # Extract file ID from the URL
         file_id = cls._extract_file_id(url)
         if not file_id:
@@ -67,6 +56,8 @@ class GoogleDriveDownloader:
 
     @staticmethod
     def _extract_file_id(url):
+        from urllib.parse import urlparse, parse_qs
+
         # Try to extract file ID from '/file/d/' format
         file_id_match = re.search(r"/d/([a-zA-Z0-9-_]+)", url)
         if file_id_match:
@@ -92,6 +83,8 @@ class GoogleDriveDownloader:
 
 def fetch_and_save_pdf(url, filename):
     # Send a GET request to the URL
+    import requests
+
     response = requests.get(url)
 
     # Check if the request was successful
@@ -114,11 +107,6 @@ def fetch_and_save_pdf(url, filename):
     return temp_file_path
 
 
-# Example usage:
-# url = "https://example.com/sample.pdf"
-# fetch_and_save_pdf(url, "sample.pdf")
-
-
 class ScenarioListPdfMixin:
     @classmethod
     def from_pdf(cls, filename_or_url, collapse_pages=False):
@@ -151,6 +139,8 @@ class ScenarioListPdfMixin:
 
     @staticmethod
     def is_url(string):
+        from urllib.parse import urlparse
+
         try:
             result = urlparse(string)
             return all([result.scheme, result.netloc])
@@ -189,7 +179,8 @@ class ScenarioListPdfMixin:
 
     @staticmethod
     def extract_text_from_pdf(pdf_path):
-        from edsl import Scenario
+        from edsl.scenarios.Scenario import Scenario
+        import fitz  # PyMuPDF
 
         # TODO: Add test case
         # Ensure the file exists
@@ -243,19 +234,6 @@ class ScenarioListPdfMixin:
 
 
 if __name__ == "__main__":
-    pass
-
-    # from edsl import ScenarioList
-
-    # class ScenarioListNew(ScenarioList, ScenaroListPdfMixin):
-    #     pass
-
-    # #ScenarioListNew.create_hello_world_pdf('hello_world')
-    # #scenarios = ScenarioListNew.from_pdf('hello_world.pdf')
-    # #print(scenarios)
+    import doctest
 
-    # from edsl import ScenarioList, QuestionFreeText
-    # homo_silicus = ScenarioList.from_pdf('w31122.pdf')
-    # q = QuestionFreeText(question_text = "What is the key point of the text in {{ text }}?", question_name = "key_point")
-    # results = q.by(homo_silicus).run(progress_bar = True)
-    # results.select('scenario.page', 'answer.key_point').order_by('page').print()
+    doctest.testmod()

edsl/scenarios/__init__.py

@@ -1,4 +1,3 @@
 from edsl.scenarios.Scenario import Scenario
 from edsl.scenarios.ScenarioList import ScenarioList
-
-# from edsl.scenarios.FileStore import FileStore
+from edsl.scenarios.FileStore import FileStore

edsl/scenarios/directory_scanner.py

@@ -0,0 +1,96 @@
+# directory_scanner.py
+from dataclasses import dataclass
+from typing import Optional, List, Iterator, TypeVar, Generic, Callable, Any
+import os
+
+T = TypeVar("T")
+
+
+@dataclass
+class DirectoryScanner:
+    """
+    Scanner for finding files in a directory based on various criteria.
+    """
+
+    directory_path: str
+
+    def scan(
+        self,
+        factory: Callable[[str], T],
+        recursive: bool = False,
+        suffix_allow_list: Optional[List[str]] = None,
+        suffix_exclude_list: Optional[List[str]] = None,
+        example_suffix: Optional[str] = None,
+        include_no_extension: bool = True,
+    ) -> List[T]:
+        """
+        Eagerly scan directory and return list of objects created by factory.
+
+        Args:
+            factory: Callable that creates objects from file paths
+            recursive: If True, recursively traverse subdirectories
+            suffix_allow_list: List of allowed file extensions (without dots)
+            suffix_exclude_list: List of excluded file extensions (takes precedence over allow list)
+            example_suffix: If provided, only include files with this example suffix
+            include_no_extension: Whether to include files without extensions
+        """
+        return list(
+            self.iter_scan(
+                factory,
+                recursive=recursive,
+                suffix_allow_list=suffix_allow_list,
+                suffix_exclude_list=suffix_exclude_list,
+                example_suffix=example_suffix,
+                include_no_extension=include_no_extension,
+            )
+        )
+
+    def iter_scan(
+        self,
+        factory: Callable[[str], T],
+        recursive: bool = False,
+        suffix_allow_list: Optional[List[str]] = None,
+        suffix_exclude_list: Optional[List[str]] = None,
+        example_suffix: Optional[str] = None,
+        include_no_extension: bool = True,
+    ) -> Iterator[T]:
+        """
+        Lazily scan directory and yield objects created by factory.
+        """
+
+        def should_include_file(filepath: str) -> bool:
+            _, ext = os.path.splitext(filepath)
+            ext = ext[1:] if ext else ""
+
+            # Handle no extension case
+            if not ext:
+                return include_no_extension
+
+            # Check exclusions first (they take precedence)
+            if suffix_exclude_list and ext in suffix_exclude_list:
+                return False
+
+            # Check example suffix if specified
+            if example_suffix and not filepath.endswith(example_suffix):
+                return False
+
+            # Check allowed suffixes if specified
+            if suffix_allow_list and ext not in suffix_allow_list:
+                return False
+
+            return True
+
+        def iter_files():
+            if recursive:
+                for root, _, files in os.walk(self.directory_path):
+                    for file in files:
+                        yield os.path.join(root, file)
+            else:
+                for file in os.listdir(self.directory_path):
+                    file_path = os.path.join(self.directory_path, file)
+                    if os.path.isfile(file_path):
+                        yield file_path
+
+        for file_path in iter_files():
+            if should_include_file(file_path):
+                yield factory(file_path)

edsl/scenarios/file_methods.py

@@ -0,0 +1,85 @@
+from typing import Optional, Dict, Type
+from abc import ABC, abstractmethod
+import importlib.metadata
+import importlib.util
+
+from edsl.utilities.is_notebook import is_notebook
+
+
+class FileMethods(ABC):
+    _handlers: Dict[str, Type["FileMethods"]] = {}
+
+    def __init__(self, path: Optional[str] = None):
+        self.path = path
+
+    def __init_subclass__(cls) -> None:
+        """Register subclasses automatically when they're defined."""
+        super().__init_subclass__()
+        if hasattr(cls, "suffix"):
+            FileMethods._handlers[cls.suffix] = cls
+
+    @classmethod
+    def get_handler(cls, suffix: str) -> Optional[Type["FileMethods"]]:
+        """Get the appropriate handler class for a given suffix."""
+        # Load plugins if they haven't been loaded yet
+        if not cls._handlers:
+            cls.load_plugins()
+        return cls._handlers.get(suffix.lower())
+
+    @classmethod
+    def load_plugins(cls):
+        """Load all file handler plugins including built-ins and external plugins."""
+
+        from edsl.scenarios import handlers
+
+        # Then load any external plugins
+        try:
+            entries = importlib.metadata.entry_points(group="file_handlers")
+        except TypeError:  # some Python 3.9 bullshit
+            # entries = importlib.metadata.entry_points()
+            entries = []
+
+        for ep in entries:
+            try:
+                handler_class = ep.load()
+                # Registration happens automatically via __init_subclass__
+            except Exception as e:
+                print(f"Failed to load external handler {ep.name}: {e}")
+
+    @classmethod
+    def get_handler_for_path(cls, path: str) -> Optional[Type["FileMethods"]]:
+        """Get the appropriate handler class for a file path."""
+        suffix = path.split(".")[-1].lower() if "." in path else ""
+        return cls.get_handler(suffix)
+
+    @classmethod
+    def create(cls, path: str) -> Optional["FileMethods"]:
+        """Create an appropriate handler instance for the given path."""
+        handler_class = cls.get_handler_for_path(path)
+        if handler_class:
+            return handler_class(path)
+        return None
+
+    @classmethod
+    def supported_file_types(cls):
+        if not cls._handlers:
+            cls.load_plugins()
+        return list(cls._handlers.keys())
+
+    @abstractmethod
+    def view_system(self):
+        ...
+
+    @abstractmethod
+    def view_notebook(self):
+        ...
+
+    def view(self):
+        if is_notebook():
+            self.view_notebook()
+        else:
+            self.view_system()
+
+    @abstractmethod
+    def example(self):
+        ...

edsl/scenarios/handlers/__init__.py

@@ -0,0 +1,13 @@
+from .pdf import PdfMethods
+from .docx import DocxMethods
+from .png import PngMethods
+from .txt import TxtMethods
+from .html import HtmlMethods
+from .md import MarkdownMethods
+from .csv import CsvMethods
+from .json import JsonMethods
+from .sql import SqlMethods
+from .pptx import PptxMethods
+from .latex import LaTeXMethods
+from .py import PyMethods
+from .sqlite import SQLiteMethods

edsl/scenarios/handlers/csv.py

@@ -0,0 +1,49 @@
+import tempfile
+from edsl.scenarios.file_methods import FileMethods
+
+
+class CsvMethods(FileMethods):
+    suffix = "csv"
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening CSV: {e}")
+        else:
+            print("CSV file was not found.")
+
+    def view_notebook(self):
+        import pandas as pd
+        from IPython.display import display
+
+        df = pd.read_csv(self.path)
+        display(df)
+
+    def example(self):
+        import pandas as pd
+
+        df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as f:
+            df.to_csv(f.name, index=False)
+        return f.name
+
+    def to_pandas(self):
+        """
+        Convert the CSV file to a pandas DataFrame.
+
+        Returns:
+            pandas.DataFrame: The data from the CSV as a DataFrame
+        """
+        import pandas as pd
+
+        return pd.read_csv(self.path)

edsl/scenarios/handlers/docx.py

@@ -0,0 +1,76 @@
+from edsl.scenarios.file_methods import FileMethods
+import os
+import tempfile
+
+
+class DocxMethods(FileMethods):
+    suffix = "docx"
+
+    def extract_text(self):
+        from docx import Document
+
+        self.doc = Document(self.path)
+
+        # Extract all text
+        full_text = []
+        for para in self.doc.paragraphs:
+            full_text.append(para.text)
+
+        text = "\n".join(full_text)
+        return text
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening DOCX: {e}")
+        else:
+            print("DOCX file was not found.")
+
+    def view_notebook(self):
+        import mammoth
+        from IPython.display import HTML, display
+
+        with open(self.path, "rb") as docx_file:
+            result = mammoth.convert_to_html(docx_file)
+            html = f"""
+            <div style="width: 800px; height: 800px; padding: 20px; 
+                       border: 1px solid #ccc; overflow-y: auto;">
+                {result.value}
+            </div>
+            """
+            display(HTML(html))
+
+    def example(self):
+        from docx import Document
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.scenarios.ScenarioList import ScenarioList
+
+        os.makedirs("test_dir", exist_ok=True)
+        doc1 = Document()
+        _ = doc1.add_heading("First Survey")
+        doc1.save("test_dir/test1.docx")
+        doc2 = Document()
+        _ = doc2.add_heading("Second Survey")
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".docx") as tmp:
+            doc2.save(tmp.name)
+            tmp.close()
+
+        return tmp.name
+
+
+if __name__ == "__main__":
+    docx_temp = DocxMethods.example()
+    from edsl.scenarios.FileStore import FileStore
+
+    fs = FileStore(docx_temp)

edsl/scenarios/handlers/html.py

@@ -0,0 +1,37 @@
+from edsl.scenarios.file_methods import FileMethods
+import tempfile
+
+
+class HtmlMethods(FileMethods):
+    suffix = "html"
+
+    def view_system(self):
+        import webbrowser
+
+        # with open(self.path, "r") as f:
+        #     html_string = f.read()
+
+        # html_path = self.to_tempfile()
+        # webbrowser.open("file://" + html_path)
+        webbrowser.open("file://" + self.path)
+
+    def view_notebook(self):
+        from IPython.display import IFrame, display
+
+        display(IFrame(self.path, width=800, height=800))
+
+    def example(self):
+        html_string = b"""
+            <html>
+            <head>
+                <title>Test</title>
+            </head>
+            <body>
+                <h1>Hello, World!</h1>
+            </body>
+            </html>
+            """
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f:
+            f.write(html_string)
+        return f.name

edsl/scenarios/handlers/json.py

@@ -0,0 +1,111 @@
+from edsl.scenarios.file_methods import FileMethods
+import tempfile
+import json
+from typing import Optional, Dict, Any
+
+
+class JsonMethods(FileMethods):
+    suffix = "json"
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening JSON: {e}")
+        else:
+            print("JSON file was not found.")
+
+    def view_notebook(self):
+        from IPython.display import FileLink, JSON, display
+        import json
+
+        # Read and parse the JSON file
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+
+            # Display formatted JSON
+            display(JSON(content))
+
+            # Provide download link
+            display(FileLink(self.path))
+        except json.JSONDecodeError as e:
+            print(f"Error parsing JSON: {e}")
+        except Exception as e:
+            print(f"Error reading file: {e}")
+
+    def validate_json(self, schema: Optional[Dict[str, Any]] = None) -> bool:
+        """
+        Validate the JSON file against a schema if provided,
+        or check if it's valid JSON if no schema is provided.
+        """
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+
+            if schema is not None:
+                from jsonschema import validate
+
+                validate(instance=content, schema=schema)
+
+            return True
+        except json.JSONDecodeError as e:
+            print(f"Invalid JSON format: {e}")
+            return False
+        except Exception as e:
+            print(f"Validation error: {e}")
+            return False
+
+    def pretty_print(self):
+        """Pretty print the JSON content with proper indentation."""
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+
+            pretty_json = json.dumps(content, indent=2, sort_keys=True)
+            print(pretty_json)
+        except Exception as e:
+            print(f"Error pretty printing JSON: {e}")
+
+    def example(self):
+        sample_json = {
+            "person": {
+                "name": "John Doe",
+                "age": 30,
+                "contact": {"email": "john@example.com", "phone": "+1-555-555-5555"},
+                "interests": ["programming", "data science", "machine learning"],
+                "active": True,
+                "metadata": {"last_updated": "2024-01-01", "version": 1.0},
+            }
+        }
+
+        with tempfile.NamedTemporaryFile(
+            delete=False, suffix=".json", mode="w", encoding="utf-8"
+        ) as f:
+            json.dump(sample_json, f, indent=2)
+        return f.name
+
+    def format_file(self):
+        """Read, format, and write back the JSON with consistent formatting."""
+        try:
+            # Read the current content
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+
+            # Write back with consistent formatting
+            with open(self.path, "w", encoding="utf-8") as f:
+                json.dump(content, f, indent=2, sort_keys=True)
+
+            return True
+        except Exception as e:
+            print(f"Error formatting JSON file: {e}")
+            return False

edsl/scenarios/handlers/latex.py

@@ -0,0 +1,5 @@
+from edsl.scenarios.handlers.txt import TxtMethods
+
+
+class LaTeXMethods(TxtMethods):
+    pass

edsl/scenarios/handlers/md.py

@@ -0,0 +1,51 @@
+from edsl.scenarios.file_methods import FileMethods
+import tempfile
+
+
+class MarkdownMethods(FileMethods):
+    suffix = "md"
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening Markdown: {e}")
+        else:
+            print("Markdown file was not found.")
+
+    def view_notebook(self):
+        from IPython.display import FileLink, Markdown, display
+
+        # First display the content of the markdown file
+        with open(self.path, "r", encoding="utf-8") as f:
+            content = f.read()
+            display(Markdown(content))
+
+        # Then provide a download link
+        display(FileLink(self.path))
+
+    def example(self):
+        markdown_content = """# Sample Markdown
+        
+## Features
+- **Bold text** demonstration
+- *Italic text* demonstration
+- Code block example:
+```python
+print("Hello, World!")
+```
+"""
+        with tempfile.NamedTemporaryFile(
+            delete=False, suffix=".md", mode="w", encoding="utf-8"
+        ) as f:
+            f.write(markdown_content)
+        return f.name