edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/scenarios/ScenarioListExportMixin.py

@@ -0,0 +1,45 @@
+"""Mixin class for exporting results."""
+
+from functools import wraps
+from edsl.results.DatasetExportMixin import DatasetExportMixin
+
+
+def to_dataset(func):
+    """Convert the object to a Dataset object before calling the function."""
+
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        """Return the function with the Results object converted to a Dataset object."""
+        if self.__class__.__name__ == "ScenarioList":
+            return func(self.to_dataset(), *args, **kwargs)
+        else:
+            raise Exception(
+                f"Class {self.__class__.__name__} not recognized as a Results or Dataset object."
+            )
+
+    return wrapper
+
+
+def decorate_methods_from_mixin(cls, mixin_cls):
+    for attr_name, attr_value in mixin_cls.__dict__.items():
+        if callable(attr_value) and not attr_name.startswith("__"):
+            setattr(cls, attr_name, to_dataset(attr_value))
+    return cls
+
+
+# @decorate_all_methods
+class ScenarioListExportMixin(DatasetExportMixin):
+    """Mixin class for exporting Results objects."""
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        decorate_methods_from_mixin(cls, DatasetExportMixin)
+
+    def to_docx(self, filename: str):
+        """Export the ScenarioList to a .docx file."""
+        dataset = self.to_dataset()
+        from edsl.results.DatasetTree import Tree
+
+        tree = Tree(dataset)
+        tree.construct_tree()
+        tree.to_docx(filename)

edsl/scenarios/ScenarioListPdfMixin.py

@@ -0,0 +1,239 @@
+import os
+import re
+import copy
+import atexit
+import tempfile
+import subprocess
+
+
+class GoogleDriveDownloader:
+    _temp_dir = None
+    _temp_file_path = None
+
+    @classmethod
+    def fetch_from_drive(cls, url, filename=None):
+        import requests
+
+        # Extract file ID from the URL
+        file_id = cls._extract_file_id(url)
+        if not file_id:
+            raise ValueError("Invalid Google Drive URL")
+
+        # Construct the download URL
+        download_url = f"https://drive.google.com/uc?export=download&id={file_id}"
+
+        # Send a GET request to the URL
+        session = requests.Session()
+        response = session.get(download_url, stream=True)
+        response.raise_for_status()
+
+        # Check for large file download prompt
+        for key, value in response.cookies.items():
+            if key.startswith("download_warning"):
+                params = {"id": file_id, "confirm": value}
+                response = session.get(download_url, params=params, stream=True)
+                break
+
+        # Create a temporary file to save the download
+        if not filename:
+            filename = "downloaded_file"
+
+        if cls._temp_dir is None:
+            cls._temp_dir = tempfile.TemporaryDirectory()
+            atexit.register(cls._cleanup)
+
+        cls._temp_file_path = os.path.join(cls._temp_dir.name, filename)
+
+        # Write the content to the temporary file
+        with open(cls._temp_file_path, "wb") as f:
+            for chunk in response.iter_content(32768):
+                if chunk:
+                    f.write(chunk)
+
+        print(f"File saved to: {cls._temp_file_path}")
+
+        return cls._temp_file_path
+
+    @staticmethod
+    def _extract_file_id(url):
+        from urllib.parse import urlparse, parse_qs
+
+        # Try to extract file ID from '/file/d/' format
+        file_id_match = re.search(r"/d/([a-zA-Z0-9-_]+)", url)
+        if file_id_match:
+            return file_id_match.group(1)
+
+        # If not found, try to extract from 'open?id=' format
+        parsed_url = urlparse(url)
+        query_params = parse_qs(parsed_url.query)
+        if "id" in query_params:
+            return query_params["id"][0]
+
+        return None
+
+    @classmethod
+    def _cleanup(cls):
+        if cls._temp_dir:
+            cls._temp_dir.cleanup()
+
+    @classmethod
+    def get_temp_file_path(cls):
+        return cls._temp_file_path
+
+
+def fetch_and_save_pdf(url, filename):
+    # Send a GET request to the URL
+    import requests
+
+    response = requests.get(url)
+
+    # Check if the request was successful
+    response.raise_for_status()
+
+    # Create a temporary directory
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Construct the full path for the file
+        temp_file_path = os.path.join(temp_dir, filename)
+
+        # Write the content to the temporary file
+        with open(temp_file_path, "wb") as file:
+            file.write(response.content)
+
+        print(f"PDF saved to: {temp_file_path}")
+
+        # Here you can perform operations with the file
+        # The file will be automatically deleted when you exit this block
+
+    return temp_file_path
+
+
+class ScenarioListPdfMixin:
+    @classmethod
+    def from_pdf(cls, filename_or_url, collapse_pages=False):
+        # Check if the input is a URL
+        if cls.is_url(filename_or_url):
+            # Check if it's a Google Drive URL
+            if "drive.google.com" in filename_or_url:
+                temp_filename = GoogleDriveDownloader.fetch_from_drive(
+                    filename_or_url, "temp_pdf.pdf"
+                )
+            else:
+                # For other URLs, use the previous fetch_and_save_pdf function
+                temp_filename = fetch_and_save_pdf(filename_or_url, "temp_pdf.pdf")
+
+            scenarios = list(cls.extract_text_from_pdf(temp_filename))
+        else:
+            # If it's not a URL, assume it's a local file path
+            scenarios = list(cls.extract_text_from_pdf(filename_or_url))
+        if not collapse_pages:
+            return cls(scenarios)
+        else:
+            txt = ""
+            for scenario in scenarios:
+                txt += scenario["text"]
+            from edsl.scenarios import Scenario
+
+            base_scenario = copy.copy(scenarios[0])
+            base_scenario["text"] = txt
+        return base_scenario
+
+    @staticmethod
+    def is_url(string):
+        from urllib.parse import urlparse
+
+        try:
+            result = urlparse(string)
+            return all([result.scheme, result.netloc])
+        except ValueError:
+            return False
+
+    @classmethod
+    def _from_pdf_to_image(cls, pdf_path, image_format="jpeg"):
+        """
+        Convert each page of a PDF into an image and create Scenario instances.
+
+        :param pdf_path: Path to the PDF file.
+        :param image_format: Format of the output images (default is 'jpeg').
+        :return: ScenarioList instance containing the Scenario instances.
+        """
+        import tempfile
+        from pdf2image import convert_from_path
+        from edsl.scenarios import Scenario
+
+        with tempfile.TemporaryDirectory() as output_folder:
+            # Convert PDF to images
+            images = convert_from_path(pdf_path)
+
+            scenarios = []
+
+            # Save each page as an image and create Scenario instances
+            for i, image in enumerate(images):
+                image_path = os.path.join(output_folder, f"page_{i+1}.{image_format}")
+                image.save(image_path, image_format.upper())
+
+                scenario = Scenario._from_filepath_image(image_path)
+                scenarios.append(scenario)
+
+            # print(f"Saved {len(images)} pages as images in {output_folder}")
+            return cls(scenarios)
+
+    @staticmethod
+    def extract_text_from_pdf(pdf_path):
+        from edsl.scenarios.Scenario import Scenario
+        import fitz  # PyMuPDF
+
+        # TODO: Add test case
+        # Ensure the file exists
+        if not os.path.exists(pdf_path):
+            raise FileNotFoundError(f"The file {pdf_path} does not exist.")
+
+        # Open the PDF file
+        document = fitz.open(pdf_path)
+
+        # Get the filename from the path
+        filename = os.path.basename(pdf_path)
+
+        # Iterate through each page and extract text
+        for page_num in range(len(document)):
+            page = document.load_page(page_num)
+            text = page.get_text()
+
+            # Create a dictionary for the current page
+            page_info = {"filename": filename, "page": page_num + 1, "text": text}
+            yield Scenario(page_info)
+
+    def create_hello_world_pdf(pdf_path):
+        # LaTeX content
+        latex_content = r"""
+        \documentclass{article}
+        \title{Hello World}
+        \author{John}
+        \date{\today}
+        \begin{document}
+        \maketitle
+        \section{Hello, World!}
+        This is a simple hello world example created with LaTeX and Python.
+        \end{document}
+        """
+
+        # Create a .tex file
+        tex_filename = pdf_path + ".tex"
+        with open(tex_filename, "w") as tex_file:
+            tex_file.write(latex_content)
+
+        # Compile the .tex file to PDF
+        subprocess.run(["pdflatex", tex_filename], check=True)
+
+        # Optionally, clean up auxiliary files generated by pdflatex
+        aux_files = [pdf_path + ext for ext in [".aux", ".log"]]
+        for aux_file in aux_files:
+            try:
+                os.remove(aux_file)
+            except FileNotFoundError:
+                pass
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

edsl/scenarios/__init__.py

@@ -1 +1,3 @@
 from edsl.scenarios.Scenario import Scenario
+from edsl.scenarios.ScenarioList import ScenarioList
+from edsl.scenarios.FileStore import FileStore

edsl/scenarios/directory_scanner.py

@@ -0,0 +1,96 @@
+# directory_scanner.py
+from dataclasses import dataclass
+from typing import Optional, List, Iterator, TypeVar, Generic, Callable, Any
+import os
+
+T = TypeVar("T")
+
+
+@dataclass
+class DirectoryScanner:
+    """
+    Scanner for finding files in a directory based on various criteria.
+    """
+
+    directory_path: str
+
+    def scan(
+        self,
+        factory: Callable[[str], T],
+        recursive: bool = False,
+        suffix_allow_list: Optional[List[str]] = None,
+        suffix_exclude_list: Optional[List[str]] = None,
+        example_suffix: Optional[str] = None,
+        include_no_extension: bool = True,
+    ) -> List[T]:
+        """
+        Eagerly scan directory and return list of objects created by factory.
+
+        Args:
+            factory: Callable that creates objects from file paths
+            recursive: If True, recursively traverse subdirectories
+            suffix_allow_list: List of allowed file extensions (without dots)
+            suffix_exclude_list: List of excluded file extensions (takes precedence over allow list)
+            example_suffix: If provided, only include files with this example suffix
+            include_no_extension: Whether to include files without extensions
+        """
+        return list(
+            self.iter_scan(
+                factory,
+                recursive=recursive,
+                suffix_allow_list=suffix_allow_list,
+                suffix_exclude_list=suffix_exclude_list,
+                example_suffix=example_suffix,
+                include_no_extension=include_no_extension,
+            )
+        )
+
+    def iter_scan(
+        self,
+        factory: Callable[[str], T],
+        recursive: bool = False,
+        suffix_allow_list: Optional[List[str]] = None,
+        suffix_exclude_list: Optional[List[str]] = None,
+        example_suffix: Optional[str] = None,
+        include_no_extension: bool = True,
+    ) -> Iterator[T]:
+        """
+        Lazily scan directory and yield objects created by factory.
+        """
+
+        def should_include_file(filepath: str) -> bool:
+            _, ext = os.path.splitext(filepath)
+            ext = ext[1:] if ext else ""
+
+            # Handle no extension case
+            if not ext:
+                return include_no_extension
+
+            # Check exclusions first (they take precedence)
+            if suffix_exclude_list and ext in suffix_exclude_list:
+                return False
+
+            # Check example suffix if specified
+            if example_suffix and not filepath.endswith(example_suffix):
+                return False
+
+            # Check allowed suffixes if specified
+            if suffix_allow_list and ext not in suffix_allow_list:
+                return False
+
+            return True
+
+        def iter_files():
+            if recursive:
+                for root, _, files in os.walk(self.directory_path):
+                    for file in files:
+                        yield os.path.join(root, file)
+            else:
+                for file in os.listdir(self.directory_path):
+                    file_path = os.path.join(self.directory_path, file)
+                    if os.path.isfile(file_path):
+                        yield file_path
+
+        for file_path in iter_files():
+            if should_include_file(file_path):
+                yield factory(file_path)

edsl/scenarios/file_methods.py

@@ -0,0 +1,85 @@
+from typing import Optional, Dict, Type
+from abc import ABC, abstractmethod
+import importlib.metadata
+import importlib.util
+
+from edsl.utilities.is_notebook import is_notebook
+
+
+class FileMethods(ABC):
+    _handlers: Dict[str, Type["FileMethods"]] = {}
+
+    def __init__(self, path: Optional[str] = None):
+        self.path = path
+
+    def __init_subclass__(cls) -> None:
+        """Register subclasses automatically when they're defined."""
+        super().__init_subclass__()
+        if hasattr(cls, "suffix"):
+            FileMethods._handlers[cls.suffix] = cls
+
+    @classmethod
+    def get_handler(cls, suffix: str) -> Optional[Type["FileMethods"]]:
+        """Get the appropriate handler class for a given suffix."""
+        # Load plugins if they haven't been loaded yet
+        if not cls._handlers:
+            cls.load_plugins()
+        return cls._handlers.get(suffix.lower())
+
+    @classmethod
+    def load_plugins(cls):
+        """Load all file handler plugins including built-ins and external plugins."""
+
+        from edsl.scenarios import handlers
+
+        # Then load any external plugins
+        try:
+            entries = importlib.metadata.entry_points(group="file_handlers")
+        except TypeError:  # some Python 3.9 bullshit
+            # entries = importlib.metadata.entry_points()
+            entries = []
+
+        for ep in entries:
+            try:
+                handler_class = ep.load()
+                # Registration happens automatically via __init_subclass__
+            except Exception as e:
+                print(f"Failed to load external handler {ep.name}: {e}")
+
+    @classmethod
+    def get_handler_for_path(cls, path: str) -> Optional[Type["FileMethods"]]:
+        """Get the appropriate handler class for a file path."""
+        suffix = path.split(".")[-1].lower() if "." in path else ""
+        return cls.get_handler(suffix)
+
+    @classmethod
+    def create(cls, path: str) -> Optional["FileMethods"]:
+        """Create an appropriate handler instance for the given path."""
+        handler_class = cls.get_handler_for_path(path)
+        if handler_class:
+            return handler_class(path)
+        return None
+
+    @classmethod
+    def supported_file_types(cls):
+        if not cls._handlers:
+            cls.load_plugins()
+        return list(cls._handlers.keys())
+
+    @abstractmethod
+    def view_system(self):
+        ...
+
+    @abstractmethod
+    def view_notebook(self):
+        ...
+
+    def view(self):
+        if is_notebook():
+            self.view_notebook()
+        else:
+            self.view_system()
+
+    @abstractmethod
+    def example(self):
+        ...

edsl/scenarios/handlers/__init__.py

@@ -0,0 +1,13 @@
+from .pdf import PdfMethods
+from .docx import DocxMethods
+from .png import PngMethods
+from .txt import TxtMethods
+from .html import HtmlMethods
+from .md import MarkdownMethods
+from .csv import CsvMethods
+from .json import JsonMethods
+from .sql import SqlMethods
+from .pptx import PptxMethods
+from .latex import LaTeXMethods
+from .py import PyMethods
+from .sqlite import SQLiteMethods

edsl/scenarios/handlers/csv.py

@@ -0,0 +1,49 @@
+import tempfile
+from edsl.scenarios.file_methods import FileMethods
+
+
+class CsvMethods(FileMethods):
+    suffix = "csv"
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening CSV: {e}")
+        else:
+            print("CSV file was not found.")
+
+    def view_notebook(self):
+        import pandas as pd
+        from IPython.display import display
+
+        df = pd.read_csv(self.path)
+        display(df)
+
+    def example(self):
+        import pandas as pd
+
+        df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as f:
+            df.to_csv(f.name, index=False)
+        return f.name
+
+    def to_pandas(self):
+        """
+        Convert the CSV file to a pandas DataFrame.
+
+        Returns:
+            pandas.DataFrame: The data from the CSV as a DataFrame
+        """
+        import pandas as pd
+
+        return pd.read_csv(self.path)

edsl/scenarios/handlers/docx.py

@@ -0,0 +1,76 @@
+from edsl.scenarios.file_methods import FileMethods
+import os
+import tempfile
+
+
+class DocxMethods(FileMethods):
+    suffix = "docx"
+
+    def extract_text(self):
+        from docx import Document
+
+        self.doc = Document(self.path)
+
+        # Extract all text
+        full_text = []
+        for para in self.doc.paragraphs:
+            full_text.append(para.text)
+
+        text = "\n".join(full_text)
+        return text
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening DOCX: {e}")
+        else:
+            print("DOCX file was not found.")
+
+    def view_notebook(self):
+        import mammoth
+        from IPython.display import HTML, display
+
+        with open(self.path, "rb") as docx_file:
+            result = mammoth.convert_to_html(docx_file)
+            html = f"""
+            <div style="width: 800px; height: 800px; padding: 20px; 
+                       border: 1px solid #ccc; overflow-y: auto;">
+                {result.value}
+            </div>
+            """
+            display(HTML(html))
+
+    def example(self):
+        from docx import Document
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.scenarios.ScenarioList import ScenarioList
+
+        os.makedirs("test_dir", exist_ok=True)
+        doc1 = Document()
+        _ = doc1.add_heading("First Survey")
+        doc1.save("test_dir/test1.docx")
+        doc2 = Document()
+        _ = doc2.add_heading("Second Survey")
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".docx") as tmp:
+            doc2.save(tmp.name)
+            tmp.close()
+
+        return tmp.name
+
+
+if __name__ == "__main__":
+    docx_temp = DocxMethods.example()
+    from edsl.scenarios.FileStore import FileStore
+
+    fs = FileStore(docx_temp)

edsl/scenarios/handlers/html.py

@@ -0,0 +1,37 @@
+from edsl.scenarios.file_methods import FileMethods
+import tempfile
+
+
+class HtmlMethods(FileMethods):
+    suffix = "html"
+
+    def view_system(self):
+        import webbrowser
+
+        # with open(self.path, "r") as f:
+        #     html_string = f.read()
+
+        # html_path = self.to_tempfile()
+        # webbrowser.open("file://" + html_path)
+        webbrowser.open("file://" + self.path)
+
+    def view_notebook(self):
+        from IPython.display import IFrame, display
+
+        display(IFrame(self.path, width=800, height=800))
+
+    def example(self):
+        html_string = b"""
+            <html>
+            <head>
+                <title>Test</title>
+            </head>
+            <body>
+                <h1>Hello, World!</h1>
+            </body>
+            </html>
+            """
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f:
+            f.write(html_string)
+        return f.name