edsl 0.1.39.dev1__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl

edsl/scenarios/DirectoryScanner.py

@@ -0,0 +1,96 @@
+# directory_scanner.py
+from dataclasses import dataclass
+from typing import Optional, List, Iterator, TypeVar, Generic, Callable, Any
+import os
+
+T = TypeVar("T")
+
+
+@dataclass
+class DirectoryScanner:
+    """
+    Scanner for finding files in a directory based on various criteria.
+    """
+
+    directory_path: str
+
+    def scan(
+        self,
+        factory: Callable[[str], T],
+        recursive: bool = False,
+        suffix_allow_list: Optional[List[str]] = None,
+        suffix_exclude_list: Optional[List[str]] = None,
+        example_suffix: Optional[str] = None,
+        include_no_extension: bool = True,
+    ) -> List[T]:
+        """
+        Eagerly scan directory and return list of objects created by factory.
+
+        Args:
+            factory: Callable that creates objects from file paths
+            recursive: If True, recursively traverse subdirectories
+            suffix_allow_list: List of allowed file extensions (without dots)
+            suffix_exclude_list: List of excluded file extensions (takes precedence over allow list)
+            example_suffix: If provided, only include files with this example suffix
+            include_no_extension: Whether to include files without extensions
+        """
+        return list(
+            self.iter_scan(
+                factory,
+                recursive=recursive,
+                suffix_allow_list=suffix_allow_list,
+                suffix_exclude_list=suffix_exclude_list,
+                example_suffix=example_suffix,
+                include_no_extension=include_no_extension,
+            )
+        )
+
+    def iter_scan(
+        self,
+        factory: Callable[[str], T],
+        recursive: bool = False,
+        suffix_allow_list: Optional[List[str]] = None,
+        suffix_exclude_list: Optional[List[str]] = None,
+        example_suffix: Optional[str] = None,
+        include_no_extension: bool = True,
+    ) -> Iterator[T]:
+        """
+        Lazily scan directory and yield objects created by factory.
+        """
+
+        def should_include_file(filepath: str) -> bool:
+            _, ext = os.path.splitext(filepath)
+            ext = ext[1:] if ext else ""
+
+            # Handle no extension case
+            if not ext:
+                return include_no_extension
+
+            # Check exclusions first (they take precedence)
+            if suffix_exclude_list and ext in suffix_exclude_list:
+                return False
+
+            # Check example suffix if specified
+            if example_suffix and not filepath.endswith(example_suffix):
+                return False
+
+            # Check allowed suffixes if specified
+            if suffix_allow_list and ext not in suffix_allow_list:
+                return False
+
+            return True
+
+        def iter_files():
+            if recursive:
+                for root, _, files in os.walk(self.directory_path):
+                    for file in files:
+                        yield os.path.join(root, file)
+            else:
+                for file in os.listdir(self.directory_path):
+                    file_path = os.path.join(self.directory_path, file)
+                    if os.path.isfile(file_path):
+                        yield file_path
+
+        for file_path in iter_files():
+            if should_include_file(file_path):
+                yield factory(file_path)

edsl/scenarios/DocumentChunker.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+from typing import Optional, Generator, TYPE_CHECKING
+import copy
+
+if TYPE_CHECKING:
+    from edsl.scenarios.Scenario import Scenario
+    from edsl.scenarios.ScenarioList import ScenarioList
+
+
+class DocumentChunker:
+    def __init__(self, scenario: "Scenario"):
+        self.scenario = scenario
+
+    @staticmethod
+    def _line_chunks(text, num_lines: int) -> Generator[str, None, None]:
+        """Split a text into chunks of a given size.
+
+        :param text: The text to split.
+        :param num_lines: The number of lines in each chunk.
+
+        Example:
+
+        >>> list(DocumentChunker._line_chunks("This is a test.\\nThis is a test. This is a test.", 1))
+        ['This is a test.', 'This is a test. This is a test.']
+        """
+        lines = text.split("\n")
+        for i in range(0, len(lines), num_lines):
+            chunk = "\n".join(lines[i : i + num_lines])
+            yield chunk
+
+    @staticmethod
+    def _word_chunks(text, num_words: int) -> Generator[str, None, None]:
+        """Split a text into chunks of a given size.
+
+        :param text: The text to split.
+        :param num_words: The number of words in each chunk.
+
+        Example:
+
+        >>> list(DocumentChunker._word_chunks("This is a test.", 2))
+        ['This is', 'a test.']
+        """
+        words = text.split()
+        for i in range(0, len(words), num_words):
+            chunk = " ".join(words[i : i + num_words])
+            yield chunk
+
+    def chunk(
+        self,
+        field,
+        num_words: Optional[int] = None,
+        num_lines: Optional[int] = None,
+        include_original=False,
+        hash_original=False,
+    ) -> ScenarioList:
+        """Split a field into chunks of a given size.
+
+        :param field: The field to split.
+        :param num_words: The number of words in each chunk.
+        :param num_lines: The number of lines in each chunk.
+        :param include_original: Whether to include the original field in the new scenarios.
+        :param hash_original: Whether to hash the original field in the new scenarios.
+
+        If you specify `include_original=True`, the original field will be included in the new scenarios with an "_original" suffix.
+        """
+        from edsl.scenarios.ScenarioList import ScenarioList
+        import hashlib
+
+        if num_words is not None:
+            chunks = list(self._word_chunks(self.scenario[field], num_words))
+
+        if num_lines is not None:
+            chunks = list(self._line_chunks(self.scenario[field], num_lines))
+
+        if num_words is None and num_lines is None:
+            raise ValueError("You must specify either num_words or num_lines.")
+
+        if num_words is not None and num_lines is not None:
+            raise ValueError(
+                "You must specify either num_words or num_lines, but not both."
+            )
+
+        scenarios = []
+        for i, chunk in enumerate(chunks):
+            new_scenario = copy.deepcopy(self.scenario)
+            new_scenario[field] = chunk
+            new_scenario[field + "_chunk"] = i
+            if include_original:
+                if hash_original:
+                    new_scenario[field + "_original"] = hashlib.md5(
+                        self.scenario[field].encode()
+                    ).hexdigest()
+                else:
+                    new_scenario[field + "_original"] = self.scenario[field]
+            scenarios.append(new_scenario)
+        return ScenarioList(scenarios)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

edsl/scenarios/DocxScenario.py

@@ -0,0 +1,16 @@
+class DocxScenario:
+    def __init__(self, docx_path: str):
+        from docx import Document
+
+        self.doc = Document(docx_path)
+        self.docx_path = docx_path
+
+    def get_scenario_dict(self) -> dict:
+        # Extract all text
+        full_text = []
+        for para in self.doc.paragraphs:
+            full_text.append(para.text)
+
+        # Join the text from all paragraphs
+        text = "\n".join(full_text)
+        return {"file_path": self.docx_path, "text": text}

edsl/scenarios/FileStore.py

@@ -4,111 +4,11 @@ import tempfile
 import mimetypes
 import os
 from typing import Dict, Any, IO, Optional
-import requests
-from urllib.parse import urlparse
 
-import google.generativeai as genai
+from edsl.scenarios.Scenario import Scenario
+from edsl.utilities.remove_edsl_version import remove_edsl_version
 
-from edsl import Scenario
-from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
-from edsl.utilities.utilities import is_notebook
-
-
-def view_csv(csv_path):
-    import pandas as pd
-
-    df = pd.read_csv(csv_path)
-    return df
-
-
-def view_html(html_path):
-    import os
-    import subprocess
-    from IPython.display import IFrame, display, HTML
-
-    if os.path.exists(html_path):
-        if is_notebook():
-            # Display the HTML inline in Jupyter Notebook
-            display(IFrame(src=html_path, width=700, height=600))
-            display(
-                HTML(
-                    f'<a href="{html_path}" target="_blank">Open HTML in a new tab</a>'
-                )
-            )
-        else:
-            try:
-                if (os_name := os.name) == "posix":
-                    # Open with the default browser on macOS
-                    subprocess.run(["open", html_path], check=True)
-                elif os_name == "nt":
-                    # Open with the default browser on Windows
-                    os.startfile(html_path)
-                else:
-                    # Open with the default browser on Linux
-                    subprocess.run(["xdg-open", html_path], check=True)
-            except Exception as e:
-                print(f"Error opening HTML file: {e}")
-    else:
-        print("HTML file was not found.")
-
-
-def view_html(html_path):
-    import os
-    from IPython.display import display, HTML
-
-    if is_notebook():
-        with open(html_path, "r") as f:
-            html_content = f.read()
-        display(HTML(html_content))
-    else:
-        if os.path.exists(html_path):
-            try:
-                if (os_name := os.name) == "posix":
-                    subprocess.run(["open", html_path], check=True)
-                elif os_name == "nt":
-                    os.startfile(html_path)
-                else:
-                    subprocess.run(["xdg-open", html_path], check=True)
-            except Exception as e:
-                print(f"Error opening file: {e}")
-        else:
-            print("File was not created successfully.")
-
-
-def view_pdf(pdf_path):
-    import os
-    import subprocess
-    import os
-    from IPython.display import HTML, display
-
-    if is_notebook():
-        # Convert to absolute path if needed
-        with open(pdf_path, "rb") as f:
-            base64_pdf = base64.b64encode(f.read()).decode("utf-8")
-
-        html = f"""
-        <iframe
-            src="data:application/pdf;base64,{base64_pdf}"
-            width="800px"
-            height="800px"
-            type="application/pdf"
-        ></iframe>
-        """
-        display(HTML(html))
-
-    if os.path.exists(pdf_path):
-        try:
-            if (os_name := os.name) == "posix":
-                # for cool kids
-                subprocess.run(["open", pdf_path], check=True)  # macOS
-            elif os_name == "nt":
-                os.startfile(pdf_path)  # Windows
-            else:
-                subprocess.run(["xdg-open", pdf_path], check=True)  # Linux
-        except Exception as e:
-            print(f"Error opening PDF: {e}")
-    else:
-        print("PDF file was not created successfully.")
+from edsl.scenarios.file_methods import FileMethods
 
 
 class FileStore(Scenario):
@@ -122,6 +22,7 @@ class FileStore(Scenario):
         suffix: Optional[str] = None,
         base64_string: Optional[str] = None,
         external_locations: Optional[Dict[str, str]] = None,
+        extracted_text: Optional[str] = None,
         **kwargs,
     ):
         if path is None and "filename" in kwargs:
@@ -137,6 +38,11 @@ class FileStore(Scenario):
         )
         self.base64_string = base64_string or self.encode_file_to_base64_string(path)
         self.external_locations = external_locations or {}
+
+        self.extracted_text = (
+            self.extract_text() if extracted_text is None else extracted_text
+        )
+
         super().__init__(
             {
                 "path": path,
@@ -145,6 +51,7 @@ class FileStore(Scenario):
                 "suffix": self.suffix,
                 "mime_type": self.mime_type,
                 "external_locations": self.external_locations,
+                "extracted_text": self.extracted_text,
             }
         )
 
@@ -170,88 +77,12 @@ class FileStore(Scenario):
         return "FileStore: self.path"
 
     @classmethod
-    def example(cls, example_type="text"):
-        import textwrap
-        import tempfile
-
-        if example_type == "png" or example_type == "image":
-            import importlib.resources
-            from pathlib import Path
-
-            # Get package root directory
-            package_root = Path(__file__).parent.parent.parent
-            logo_path = package_root / "static" / "logo.png"
-            return cls(str(logo_path))
-
-        if example_type == "text":
-            with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f:
-                f.write(b"Hello, World!")
-
-            return cls(path=f.name)
-
-        elif example_type == "csv":
-            from edsl.results.Results import Results
-
-            r = Results.example()
-
-            with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f:
-                r.to_csv(filename=f.name)
-            return cls(f.name)
-
-        elif example_type == "pdf":
-            pdf_string = textwrap.dedent(
-                """\
-            %PDF-1.4
-            1 0 obj
-            << /Type /Catalog /Pages 2 0 R >>
-            endobj
-            2 0 obj
-            << /Type /Pages /Kids [3 0 R] /Count 1 >>
-            endobj
-            3 0 obj
-            << /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>
-            endobj
-            4 0 obj
-            << /Length 44 >>
-            stream
-            BT
-            /F1 24 Tf
-            100 700 Td
-            (Hello, World!) Tj
-            ET
-            endstream
-            endobj
-            5 0 obj
-            << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
-            endobj
-            6 0 obj
-            << /ProcSet [/PDF /Text] /Font << /F1 5 0 R >> >>
-            endobj
-            xref
-            0 7
-            0000000000 65535 f 
-            0000000010 00000 n 
-            0000000053 00000 n 
-            0000000100 00000 n 
-            0000000173 00000 n 
-            0000000232 00000 n 
-            0000000272 00000 n 
-            trailer
-            << /Size 7 /Root 1 0 R >>
-            startxref
-            318
-            %%EOF"""
-            )
-            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
-                f.write(pdf_string.encode())
-
-            return cls(f.name)
-
-        elif example_type == "html":
-            with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as f:
-                f.write("<html><body><h1>Test</h1></body></html>".encode())
-
-            return cls(f.name)
+    def example(cls, example_type="txt"):
+        file_methods_class = FileMethods.get_handler(example_type)
+        if file_methods_class:
+            return cls(file_methods_class().example())
+        else:
+            print(f"Example for {example_type} is not supported.")
 
     @property
     def size(self) -> int:
@@ -260,6 +91,8 @@ class FileStore(Scenario):
         return os.path.getsize(self.path)
 
     def upload_google(self, refresh: bool = False) -> None:
+        import google.generativeai as genai
+
         genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
         google_info = genai.upload_file(self.path, mime_type=self.mime_type)
         self.external_locations["google"] = google_info.to_dict()
@@ -271,7 +104,21 @@ class FileStore(Scenario):
         return cls(**d)
 
     def __repr__(self):
-        return f"FileStore(path='{self.path}')"
+        import reprlib
+
+        r = reprlib.Repr()
+        r.maxstring = 20  # Limit strings to 20 chars
+        r.maxother = 30  # Limit other types to 30 chars
+
+        params = ", ".join(f"{key}={r.repr(value)}" for key, value in self.data.items())
+        return f"{self.__class__.__name__}({params})"
+
+    def _repr_html_(self):
+        parent_html = super()._repr_html_()
+        from edsl.scenarios.ConstructDownloadLink import ConstructDownloadLink
+
+        link = ConstructDownloadLink(self).html_create_link(self.path, style=None)
+        return f"{parent_html}<br>{link}"
 
     def encode_file_to_base64_string(self, file_path: str):
         try:
@@ -296,9 +143,44 @@ class FileStore(Scenario):
 
     def open(self) -> "IO":
         if self.binary:
-            return self.base64_to_file(self["base64_string"], is_binary=True)
+            return self.base64_to_file(self.base64_string, is_binary=True)
         else:
-            return self.base64_to_text_file(self["base64_string"])
+            return self.base64_to_text_file(self.base64_string)
+
+    def write(self, filename: Optional[str] = None) -> str:
+        """
+        Write the file content to disk, either to a specified filename or a temporary file.
+
+        Args:
+            filename (Optional[str]): The destination filename. If None, creates a temporary file.
+
+        Returns:
+            str: The path to the written file.
+        """
+        # Determine the mode based on binary flag
+        mode = "wb" if self.binary else "w"
+
+        # If no filename provided, create a temporary file
+        if filename is None:
+            from tempfile import NamedTemporaryFile
+
+            with NamedTemporaryFile(delete=False, suffix="." + self.suffix) as f:
+                filename = f.name
+
+        # Write the content using the appropriate mode
+        try:
+            with open(filename, mode) as f:
+                content = self.open().read()
+                # For text mode, ensure we're writing a string
+                if not self.binary and isinstance(content, bytes):
+                    content = content.decode("utf-8")
+                f.write(content)
+                print(f"File written to {filename}")
+        except Exception as e:
+            print(f"Error writing file: {e}")
+            raise
+
+        # return filename
 
     @staticmethod
     def base64_to_text_file(base64_string) -> "IO":
@@ -327,6 +209,15 @@ class FileStore(Scenario):
             # Create a StringIO object for text data
             return io.StringIO(text_data)
 
+    @property
+    def text(self):
+        if self.binary:
+            import warnings
+
+            warnings.warn("This is a binary file.")
+        else:
+            return self.base64_to_text_file(self.base64_string).read()
+
     def to_tempfile(self, suffix=None):
         if suffix is None:
             suffix = self.suffix
@@ -335,7 +226,7 @@ class FileStore(Scenario):
                 self["base64_string"], is_binary=True
             )
         else:
-            file_like_object = self.base64_to_text_file(self["base64_string"])
+            file_like_object = self.base64_to_text_file(self.base64_string)
 
         # Create a named temporary file
         mode = "wb" if self.binary else "w"
@@ -352,40 +243,23 @@ class FileStore(Scenario):
 
         return temp_file.name
 
-    def view(self, max_size: int = 300) -> None:
-        # with self.open() as f:
-        if self.suffix == "csv":
-            return view_csv(self.path)
-
-        if self.suffix == "pdf":
-            view_pdf(self.path)
-
-        if self.suffix == "html":
-            view_html(self.path)
-
-        if self.suffix == "png" or self.suffix == "jpg" or self.suffix == "jpeg":
-            if is_notebook():
-                from IPython.display import Image
-                from PIL import Image as PILImage
-
-                if max_size:
-                    # Open the image using Pillow
-                    with PILImage.open(self.path) as img:
-                        # Get original width and height
-                        original_width, original_height = img.size
+    def view(self) -> None:
+        handler = FileMethods.get_handler(self.suffix)
+        if handler:
+            handler(self.path).view()
+        else:
+            print(f"Viewing of {self.suffix} files is not supported.")
 
-                        # Calculate the scaling factor
-                        scale = min(
-                            max_size / original_width, max_size / original_height
-                        )
+    def extract_text(self) -> str:
+        handler = FileMethods.get_handler(self.suffix)
+        if handler and hasattr(handler, "extract_text"):
+            return handler(self.path).extract_text()
 
-                        # Calculate new dimensions
-                        new_width = int(original_width * scale)
-                        new_height = int(original_height * scale)
+        if not self.binary:
+            return self.text
 
-                        return Image(self.path, width=new_width, height=new_height)
-                else:
-                    return Image(self.path)
+        return None
+        # raise TypeError("No text method found for this file type.")
 
     def push(
         self, description: Optional[str] = None, visibility: str = "unlisted"
@@ -423,6 +297,8 @@ class FileStore(Scenario):
         :param download_path: The path to save the downloaded file.
         :param mime_type: The MIME type of the file. If None, it will be guessed from the file extension.
         """
+        import requests
+        from urllib.parse import urlparse
 
         response = requests.get(url, stream=True)
         response.raise_for_status()  # Raises an HTTPError for bad responses
@@ -446,6 +322,11 @@ class FileStore(Scenario):
         # Create and return a new File instance
         return cls(download_path, mime_type=mime_type)
 
+    def create_link(self, custom_filename=None, style=None):
+        from edsl.scenarios.ConstructDownloadLink import ConstructDownloadLink
+
+        return ConstructDownloadLink(self).create_link(custom_filename, style)
+
 
 class CSVFileStore(FileStore):
     @classmethod
@@ -606,27 +487,25 @@ class HTMLFileStore(FileStore):
 
 
 if __name__ == "__main__":
-    # file_path = "../conjure/examples/Ex11-2.sav"
-    # fs = FileStore(file_path)
-    # info = fs.push()
-    # print(info)
+    import doctest
 
-    # fs = CSVFileStore.example()
-    # fs.to_tempfile()
-    # print(fs.view())
+    doctest.testmod()
 
-    # fs = PDFFileStore.example()
+    # fs = FileStore.example("pdf")
     # fs.view()
 
-    # fs = PDFFileStore("paper.pdf")
-    # fs.view()
-    # from edsl import Conjure
-    pass
-    # fs = PNGFileStore("logo.png")
-    # fs.view()
-    # fs.upload_google()
+    formats = FileMethods.supported_file_types()
+    for file_type in formats:
+        print("Now testinging", file_type)
+        fs = FileStore.example(file_type)
+        fs.view()
+        input("Press Enter to continue...")
+
+    # pdf_example.view()
+    # FileStore(pdf_example).view()
+
+    # pdf_methods = methods.get("pdf")
+    # file = pdf_methods().example()
+    # pdf_methods(file).view()
 
-    # c = Conjure(datafile_name=fs.to_tempfile())
-    # f = PDFFileStore("paper.pdf")
-    # print(f.to_tempfile())
-    # f.push()
+    # print(FileMethods._handlers)