edsl 0.1.38.dev4__py3-none-any.whl → 0.1.39__py3-none-any.whl

edsl/scenarios/FileStore.py

@@ -4,111 +4,11 @@ import tempfile
 import mimetypes
 import os
 from typing import Dict, Any, IO, Optional
-import requests
-from urllib.parse import urlparse
 
-import google.generativeai as genai
+from edsl.scenarios.Scenario import Scenario
+from edsl.utilities.remove_edsl_version import remove_edsl_version
 
-from edsl import Scenario
-from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
-from edsl.utilities.utilities import is_notebook
-
-
-def view_csv(csv_path):
-    import pandas as pd
-
-    df = pd.read_csv(csv_path)
-    return df
-
-
-def view_html(html_path):
-    import os
-    import subprocess
-    from IPython.display import IFrame, display, HTML
-
-    if os.path.exists(html_path):
-        if is_notebook():
-            # Display the HTML inline in Jupyter Notebook
-            display(IFrame(src=html_path, width=700, height=600))
-            display(
-                HTML(
-                    f'<a href="{html_path}" target="_blank">Open HTML in a new tab</a>'
-                )
-            )
-        else:
-            try:
-                if (os_name := os.name) == "posix":
-                    # Open with the default browser on macOS
-                    subprocess.run(["open", html_path], check=True)
-                elif os_name == "nt":
-                    # Open with the default browser on Windows
-                    os.startfile(html_path)
-                else:
-                    # Open with the default browser on Linux
-                    subprocess.run(["xdg-open", html_path], check=True)
-            except Exception as e:
-                print(f"Error opening HTML file: {e}")
-    else:
-        print("HTML file was not found.")
-
-
-def view_html(html_path):
-    import os
-    from IPython.display import display, HTML
-
-    if is_notebook():
-        with open(html_path, "r") as f:
-            html_content = f.read()
-        display(HTML(html_content))
-    else:
-        if os.path.exists(html_path):
-            try:
-                if (os_name := os.name) == "posix":
-                    subprocess.run(["open", html_path], check=True)
-                elif os_name == "nt":
-                    os.startfile(html_path)
-                else:
-                    subprocess.run(["xdg-open", html_path], check=True)
-            except Exception as e:
-                print(f"Error opening file: {e}")
-        else:
-            print("File was not created successfully.")
-
-
-def view_pdf(pdf_path):
-    import os
-    import subprocess
-    import os
-    from IPython.display import HTML, display
-
-    if is_notebook():
-        # Convert to absolute path if needed
-        with open(pdf_path, "rb") as f:
-            base64_pdf = base64.b64encode(f.read()).decode("utf-8")
-
-        html = f"""
-        <iframe
-            src="data:application/pdf;base64,{base64_pdf}"
-            width="800px"
-            height="800px"
-            type="application/pdf"
-        ></iframe>
-        """
-        display(HTML(html))
-
-    if os.path.exists(pdf_path):
-        try:
-            if (os_name := os.name) == "posix":
-                # for cool kids
-                subprocess.run(["open", pdf_path], check=True)  # macOS
-            elif os_name == "nt":
-                os.startfile(pdf_path)  # Windows
-            else:
-                subprocess.run(["xdg-open", pdf_path], check=True)  # Linux
-        except Exception as e:
-            print(f"Error opening PDF: {e}")
-    else:
-        print("PDF file was not created successfully.")
+from edsl.scenarios.file_methods import FileMethods
 
 
 class FileStore(Scenario):
@@ -122,6 +22,7 @@ class FileStore(Scenario):
         suffix: Optional[str] = None,
         base64_string: Optional[str] = None,
         external_locations: Optional[Dict[str, str]] = None,
+        extracted_text: Optional[str] = None,
         **kwargs,
     ):
         if path is None and "filename" in kwargs:
@@ -137,6 +38,11 @@ class FileStore(Scenario):
         )
         self.base64_string = base64_string or self.encode_file_to_base64_string(path)
         self.external_locations = external_locations or {}
+
+        self.extracted_text = (
+            self.extract_text() if extracted_text is None else extracted_text
+        )
+
         super().__init__(
             {
                 "path": path,
@@ -145,6 +51,7 @@ class FileStore(Scenario):
                 "suffix": self.suffix,
                 "mime_type": self.mime_type,
                 "external_locations": self.external_locations,
+                "extracted_text": self.extracted_text,
             }
         )
 
@@ -170,88 +77,12 @@ class FileStore(Scenario):
         return "FileStore: self.path"
 
     @classmethod
-    def example(cls, example_type="text"):
-        import textwrap
-        import tempfile
-
-        if example_type == "png" or example_type == "image":
-            import importlib.resources
-            from pathlib import Path
-
-            # Get package root directory
-            package_root = Path(__file__).parent.parent.parent
-            logo_path = package_root / "static" / "logo.png"
-            return cls(str(logo_path))
-
-        if example_type == "text":
-            with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f:
-                f.write(b"Hello, World!")
-
-            return cls(path=f.name)
-
-        elif example_type == "csv":
-            from edsl.results.Results import Results
-
-            r = Results.example()
-
-            with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f:
-                r.to_csv(filename=f.name)
-            return cls(f.name)
-
-        elif example_type == "pdf":
-            pdf_string = textwrap.dedent(
-                """\
-            %PDF-1.4
-            1 0 obj
-            << /Type /Catalog /Pages 2 0 R >>
-            endobj
-            2 0 obj
-            << /Type /Pages /Kids [3 0 R] /Count 1 >>
-            endobj
-            3 0 obj
-            << /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>
-            endobj
-            4 0 obj
-            << /Length 44 >>
-            stream
-            BT
-            /F1 24 Tf
-            100 700 Td
-            (Hello, World!) Tj
-            ET
-            endstream
-            endobj
-            5 0 obj
-            << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
-            endobj
-            6 0 obj
-            << /ProcSet [/PDF /Text] /Font << /F1 5 0 R >> >>
-            endobj
-            xref
-            0 7
-            0000000000 65535 f 
-            0000000010 00000 n 
-            0000000053 00000 n 
-            0000000100 00000 n 
-            0000000173 00000 n 
-            0000000232 00000 n 
-            0000000272 00000 n 
-            trailer
-            << /Size 7 /Root 1 0 R >>
-            startxref
-            318
-            %%EOF"""
-            )
-            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
-                f.write(pdf_string.encode())
-
-            return cls(f.name)
-
-        elif example_type == "html":
-            with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as f:
-                f.write("<html><body><h1>Test</h1></body></html>".encode())
-
-            return cls(f.name)
+    def example(cls, example_type="txt"):
+        file_methods_class = FileMethods.get_handler(example_type)
+        if file_methods_class:
+            return cls(file_methods_class().example())
+        else:
+            print(f"Example for {example_type} is not supported.")
 
     @property
     def size(self) -> int:
@@ -260,6 +91,8 @@ class FileStore(Scenario):
         return os.path.getsize(self.path)
 
     def upload_google(self, refresh: bool = False) -> None:
+        import google.generativeai as genai
+
         genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
         google_info = genai.upload_file(self.path, mime_type=self.mime_type)
         self.external_locations["google"] = google_info.to_dict()
@@ -271,7 +104,21 @@ class FileStore(Scenario):
         return cls(**d)
 
     def __repr__(self):
-        return f"FileStore(path='{self.path}')"
+        import reprlib
+
+        r = reprlib.Repr()
+        r.maxstring = 20  # Limit strings to 20 chars
+        r.maxother = 30  # Limit other types to 30 chars
+
+        params = ", ".join(f"{key}={r.repr(value)}" for key, value in self.data.items())
+        return f"{self.__class__.__name__}({params})"
+
+    def _repr_html_(self):
+        parent_html = super()._repr_html_()
+        from edsl.scenarios.ConstructDownloadLink import ConstructDownloadLink
+
+        link = ConstructDownloadLink(self).html_create_link(self.path, style=None)
+        return f"{parent_html}<br>{link}"
 
     def encode_file_to_base64_string(self, file_path: str):
         try:
@@ -296,9 +143,44 @@ class FileStore(Scenario):
 
     def open(self) -> "IO":
         if self.binary:
-            return self.base64_to_file(self["base64_string"], is_binary=True)
+            return self.base64_to_file(self.base64_string, is_binary=True)
         else:
-            return self.base64_to_text_file(self["base64_string"])
+            return self.base64_to_text_file(self.base64_string)
+
+    def write(self, filename: Optional[str] = None) -> str:
+        """
+        Write the file content to disk, either to a specified filename or a temporary file.
+
+        Args:
+            filename (Optional[str]): The destination filename. If None, creates a temporary file.
+
+        Returns:
+            str: The path to the written file.
+        """
+        # Determine the mode based on binary flag
+        mode = "wb" if self.binary else "w"
+
+        # If no filename provided, create a temporary file
+        if filename is None:
+            from tempfile import NamedTemporaryFile
+
+            with NamedTemporaryFile(delete=False, suffix="." + self.suffix) as f:
+                filename = f.name
+
+        # Write the content using the appropriate mode
+        try:
+            with open(filename, mode) as f:
+                content = self.open().read()
+                # For text mode, ensure we're writing a string
+                if not self.binary and isinstance(content, bytes):
+                    content = content.decode("utf-8")
+                f.write(content)
+                print(f"File written to {filename}")
+        except Exception as e:
+            print(f"Error writing file: {e}")
+            raise
+
+        # return filename
 
     @staticmethod
     def base64_to_text_file(base64_string) -> "IO":
@@ -327,6 +209,15 @@ class FileStore(Scenario):
             # Create a StringIO object for text data
             return io.StringIO(text_data)
 
+    @property
+    def text(self):
+        if self.binary:
+            import warnings
+
+            warnings.warn("This is a binary file.")
+        else:
+            return self.base64_to_text_file(self.base64_string).read()
+
     def to_tempfile(self, suffix=None):
         if suffix is None:
             suffix = self.suffix
@@ -335,7 +226,7 @@ class FileStore(Scenario):
                 self["base64_string"], is_binary=True
             )
         else:
-            file_like_object = self.base64_to_text_file(self["base64_string"])
+            file_like_object = self.base64_to_text_file(self.base64_string)
 
         # Create a named temporary file
         mode = "wb" if self.binary else "w"
@@ -352,40 +243,23 @@ class FileStore(Scenario):
 
         return temp_file.name
 
-    def view(self, max_size: int = 300) -> None:
-        # with self.open() as f:
-        if self.suffix == "csv":
-            return view_csv(self.path)
-
-        if self.suffix == "pdf":
-            view_pdf(self.path)
-
-        if self.suffix == "html":
-            view_html(self.path)
-
-        if self.suffix == "png" or self.suffix == "jpg" or self.suffix == "jpeg":
-            if is_notebook():
-                from IPython.display import Image
-                from PIL import Image as PILImage
-
-                if max_size:
-                    # Open the image using Pillow
-                    with PILImage.open(self.path) as img:
-                        # Get original width and height
-                        original_width, original_height = img.size
+    def view(self) -> None:
+        handler = FileMethods.get_handler(self.suffix)
+        if handler:
+            handler(self.path).view()
+        else:
+            print(f"Viewing of {self.suffix} files is not supported.")
 
-                        # Calculate the scaling factor
-                        scale = min(
-                            max_size / original_width, max_size / original_height
-                        )
+    def extract_text(self) -> str:
+        handler = FileMethods.get_handler(self.suffix)
+        if handler and hasattr(handler, "extract_text"):
+            return handler(self.path).extract_text()
 
-                        # Calculate new dimensions
-                        new_width = int(original_width * scale)
-                        new_height = int(original_height * scale)
+        if not self.binary:
+            return self.text
 
-                        return Image(self.path, width=new_width, height=new_height)
-                else:
-                    return Image(self.path)
+        return None
+        # raise TypeError("No text method found for this file type.")
 
     def push(
         self, description: Optional[str] = None, visibility: str = "unlisted"
@@ -423,6 +297,8 @@ class FileStore(Scenario):
         :param download_path: The path to save the downloaded file.
         :param mime_type: The MIME type of the file. If None, it will be guessed from the file extension.
         """
+        import requests
+        from urllib.parse import urlparse
 
         response = requests.get(url, stream=True)
         response.raise_for_status()  # Raises an HTTPError for bad responses
@@ -446,6 +322,43 @@ class FileStore(Scenario):
         # Create and return a new File instance
         return cls(download_path, mime_type=mime_type)
 
+    def create_link(self, custom_filename=None, style=None):
+        from edsl.scenarios.ConstructDownloadLink import ConstructDownloadLink
+
+        return ConstructDownloadLink(self).create_link(custom_filename, style)
+
+    def to_pandas(self):
+        """
+        Convert the file content to a pandas DataFrame if supported by the file handler.
+
+        Returns:
+            pandas.DataFrame: The data from the file as a DataFrame
+
+        Raises:
+            AttributeError: If the file type's handler doesn't support pandas conversion
+        """
+        handler = FileMethods.get_handler(self.suffix)
+        if handler and hasattr(handler, "to_pandas"):
+            return handler(self.path).to_pandas()
+        raise AttributeError(
+            f"Converting {self.suffix} files to pandas DataFrame is not supported"
+        )
+
+    def __getattr__(self, name):
+        """
+        Delegate pandas DataFrame methods to the underlying DataFrame if this is a CSV file
+        """
+        if self.suffix == "csv":
+            # Get the pandas DataFrame
+            df = self.to_pandas()
+            # Check if the requested attribute exists in the DataFrame
+            if hasattr(df, name):
+                return getattr(df, name)
+        # If not a CSV or attribute doesn't exist in DataFrame, raise AttributeError
+        raise AttributeError(
+            f"'{self.__class__.__name__}' object has no attribute '{name}'"
+        )
+
 
 class CSVFileStore(FileStore):
     @classmethod
@@ -606,27 +519,25 @@ class HTMLFileStore(FileStore):
 
 
 if __name__ == "__main__":
-    # file_path = "../conjure/examples/Ex11-2.sav"
-    # fs = FileStore(file_path)
-    # info = fs.push()
-    # print(info)
+    import doctest
 
-    # fs = CSVFileStore.example()
-    # fs.to_tempfile()
-    # print(fs.view())
+    doctest.testmod()
 
-    # fs = PDFFileStore.example()
+    # fs = FileStore.example("pdf")
     # fs.view()
 
-    # fs = PDFFileStore("paper.pdf")
-    # fs.view()
-    # from edsl import Conjure
-    pass
-    # fs = PNGFileStore("logo.png")
-    # fs.view()
-    # fs.upload_google()
+    formats = FileMethods.supported_file_types()
+    for file_type in formats:
+        print("Now testinging", file_type)
+        fs = FileStore.example(file_type)
+        fs.view()
+        input("Press Enter to continue...")
+
+    # pdf_example.view()
+    # FileStore(pdf_example).view()
+
+    # pdf_methods = methods.get("pdf")
+    # file = pdf_methods().example()
+    # pdf_methods(file).view()
 
-    # c = Conjure(datafile_name=fs.to_tempfile())
-    # f = PDFFileStore("paper.pdf")
-    # print(f.to_tempfile())
-    # f.push()
+    # print(FileMethods._handlers)

edsl/scenarios/PdfExtractor.py

@@ -0,0 +1,40 @@
+import os
+
+
+class PdfExtractor:
+    def __init__(self, pdf_path: str, parent_object: object):
+        self.pdf_path = pdf_path
+        self.constructor = parent_object.__class__
+
+    def get_object(self) -> object:
+        return self.constructor(self._get_pdf_dict())
+
+    def _get_pdf_dict(self) -> dict:
+        # Ensure the file exists
+        import fitz
+
+        if not os.path.exists(self.pdf_path):
+            raise FileNotFoundError(f"The file {self.pdf_path} does not exist.")
+
+        # Open the PDF file
+        document = fitz.open(self.pdf_path)
+
+        # Get the filename from the path
+        filename = os.path.basename(self.pdf_path)
+
+        # Iterate through each page and extract text
+        text = ""
+        for page_num in range(len(document)):
+            page = document.load_page(page_num)
+            blocks = page.get_text("blocks")  # Extract text blocks
+
+            # Sort blocks by their vertical position (y0) to maintain reading order
+            blocks.sort(key=lambda b: (b[1], b[0]))  # Sort by y0 first, then x0
+
+            # Combine the text blocks in order
+            for block in blocks:
+                text += block[4] + "\n"
+
+        # Create a dictionary for the combined text
+        page_info = {"filename": filename, "text": text}
+        return page_info