pembot 0.1.2__tar.gz → 0.1.3__tar.gz

{pembot-0.1.2 → pembot-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pembot
-Version: 0.1.2
+Version: 0.1.3
 Summary: A Python Package to convert PEM blog content to usseful information by leveraging LLMs
 Author-email: cyto <aryan_sidhwani@protonmail.com>
 License-Expression: MIT

pembot-0.1.3/pembot/.git/COMMIT_EDITMSG

@@ -0,0 +1 @@
+cyto/fixed the excel to markdown conversion

{pembot-0.1.2 → pembot-0.1.3}/pembot/.git/logs/HEAD

@@ -13,3 +13,5 @@ af80ddb5890f062e364ea8ade2d602df4e12de8c 0d28f73897db0c9a9351ee9e64d2a0fe27db270
 a898d2c3947d30d8be64bd2bbcef68f956d5456b 784aa28d912b66e07748483efe0326c70d7541a5 cyto <silverstone965@gmail.com> 1752236415 +0530	commit: added prompt prefixing option to prime the llm with some more data; added the option for no-filter global search in a collection in search_within_document(s)
 784aa28d912b66e07748483efe0326c70d7541a5 f214d4d56726e2928479c5948bd88e038cf70b2e cyto <silverstone965@gmail.com> 1752858244 +0530	commit: added smolagent functionality with tool extension capability to query.py and added a caveat in search_within_documents by which you can just return no context if you set limit= 0, effectively doing a Non-contextual prompt
 f214d4d56726e2928479c5948bd88e038cf70b2e 8fc00bf69f4ad3e50c13acc4a0988b6c0fe72b5a cyto <silverstone965@gmail.com> 1752859643 +0530	commit: minor oopsie
+8fc00bf69f4ad3e50c13acc4a0988b6c0fe72b5a 203b390ad0aeb3bc5a8540840b004e6a42e5ce7a cyto <silverstone965@gmail.com> 1752865439 +0530	commit: added requirements
+203b390ad0aeb3bc5a8540840b004e6a42e5ce7a c3cc0da3d955ecec0f865c46c030a0c073697495 cyto <silverstone965@gmail.com> 1758877816 +0530	commit: cyto/fixed the excel to markdown conversion

{pembot-0.1.2 → pembot-0.1.3}/pembot/.git/logs/refs/heads/main

@@ -13,3 +13,5 @@ af80ddb5890f062e364ea8ade2d602df4e12de8c 0d28f73897db0c9a9351ee9e64d2a0fe27db270
 a898d2c3947d30d8be64bd2bbcef68f956d5456b 784aa28d912b66e07748483efe0326c70d7541a5 cyto <silverstone965@gmail.com> 1752236415 +0530	commit: added prompt prefixing option to prime the llm with some more data; added the option for no-filter global search in a collection in search_within_document(s)
 784aa28d912b66e07748483efe0326c70d7541a5 f214d4d56726e2928479c5948bd88e038cf70b2e cyto <silverstone965@gmail.com> 1752858244 +0530	commit: added smolagent functionality with tool extension capability to query.py and added a caveat in search_within_documents by which you can just return no context if you set limit= 0, effectively doing a Non-contextual prompt
 f214d4d56726e2928479c5948bd88e038cf70b2e 8fc00bf69f4ad3e50c13acc4a0988b6c0fe72b5a cyto <silverstone965@gmail.com> 1752859643 +0530	commit: minor oopsie
+8fc00bf69f4ad3e50c13acc4a0988b6c0fe72b5a 203b390ad0aeb3bc5a8540840b004e6a42e5ce7a cyto <silverstone965@gmail.com> 1752865439 +0530	commit: added requirements
+203b390ad0aeb3bc5a8540840b004e6a42e5ce7a c3cc0da3d955ecec0f865c46c030a0c073697495 cyto <silverstone965@gmail.com> 1758877816 +0530	commit: cyto/fixed the excel to markdown conversion

{pembot-0.1.2 → pembot-0.1.3}/pembot/.git/logs/refs/remotes/origin/main

@@ -12,3 +12,5 @@ af80ddb5890f062e364ea8ade2d602df4e12de8c 0d28f73897db0c9a9351ee9e64d2a0fe27db270
 a898d2c3947d30d8be64bd2bbcef68f956d5456b 784aa28d912b66e07748483efe0326c70d7541a5 cyto <silverstone965@gmail.com> 1752236436 +0530	update by push
 784aa28d912b66e07748483efe0326c70d7541a5 f214d4d56726e2928479c5948bd88e038cf70b2e cyto <silverstone965@gmail.com> 1752858280 +0530	update by push
 f214d4d56726e2928479c5948bd88e038cf70b2e 8fc00bf69f4ad3e50c13acc4a0988b6c0fe72b5a cyto <silverstone965@gmail.com> 1752859659 +0530	update by push
+8fc00bf69f4ad3e50c13acc4a0988b6c0fe72b5a 203b390ad0aeb3bc5a8540840b004e6a42e5ce7a cyto <silverstone965@gmail.com> 1752865472 +0530	update by push
+203b390ad0aeb3bc5a8540840b004e6a42e5ce7a c3cc0da3d955ecec0f865c46c030a0c073697495 cyto <silverstone965@gmail.com> 1758877832 +0530	update by push

pembot-0.1.3/pembot/.git/refs/heads/main

@@ -0,0 +1 @@
+c3cc0da3d955ecec0f865c46c030a0c073697495

pembot-0.1.3/pembot/.git/refs/remotes/origin/main

@@ -0,0 +1 @@
+c3cc0da3d955ecec0f865c46c030a0c073697495

{pembot-0.1.2 → pembot-0.1.3}/pembot/.gitignore

@@ -1,4 +1,3 @@
-*.txt
 TextEmbedder/__pycache__/
 pdf2markdown/
 __pycache__/

pembot-0.1.3/pembot/AnyToText/convertor.py

@@ -0,0 +1,201 @@
+from tempfile import TemporaryDirectory
+import mimetypes
+from pathlib import Path
+from pembot.pdf2markdown.extract import MarkdownPDFExtractor
+import os
+import pandas as pd
+from typing import Literal, Union
+import tempfile
+from datetime import datetime, date
+from tabulate import tabulate
+
+
+PandasReadEngineType = Literal['xlrd', 'openpyxl', 'odf', 'pyxlsb', 'calamine', None]
+
+EXCEL_FILE_TYPES= [
+        'text/csv',
+        'application/vnd.ms-excel',
+        'application/msexcel',
+        'application/x-msexcel',
+        'application/x-ms-excel',
+        'application/x-excel',
+        'application/x-dos_ms_excel',
+        'application/x-dos_ms_excel',
+        'application/xls',
+        'application/x-xls',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        'application/vnd.oasis.opendocument.spreadsheet',
+]
+
+
+class Convertor():
+
+
+    def __init__(self, myfile: Path | None= None, output_dir: Path | None= None, file_bytes: bytes | None= None, suffix: str | None= None, file_type: str | None= None, model_name: str | None = None):
+
+        self.output= ""
+
+        if model_name is None:
+            # model_name=  "gemini-2.5-flash"
+            model_name=  "Nanonets-OCR-s"
+
+        # file_type can be pdf, excel, etc.
+        if output_dir is None and myfile is None and file_bytes is not None and suffix is not None:
+            with tempfile.TemporaryDirectory() as dp:
+                with tempfile.NamedTemporaryFile(suffix= suffix, mode= 'wb') as fp:
+                    fp.write(file_bytes)
+                    myfile= Path(fp.name)
+                    output_dir= Path(dp)
+                    if file_type == 'pdf':
+                        extractor= MarkdownPDFExtractor(str(myfile), output_path= str(output_dir), page_delimiter= "-- NEXT PAGE --", model_name= model_name)
+                        extractor.extract()
+                        with open(output_dir / (myfile.stem + '.md')) as output_file:
+                            self.output= output_file.read()
+                    elif file_type == 'excel':
+                        self.input_filepath= myfile
+                        self.output= self.convert_excel_to_markdown()
+
+        elif output_dir is not None and myfile is not None:
+            print("got output path for conversion: ", output_dir)
+            mt= mimetypes.guess_file_type(str(myfile))[0]
+
+            self.output_dir= output_dir
+            self.input_filepath= myfile
+
+            if mt == 'application/json':
+                print("the file was json")
+            elif mt == 'application/pdf':
+                print("the file was pdf, outputting in: ", output_dir)
+                extractor= MarkdownPDFExtractor(str(myfile), output_path= str(self.output_dir), page_delimiter= "-- NEXT PAGE --", model_name= model_name)
+                extractor.extract()
+                with open(self.output_dir / (myfile.stem + '.md')) as output_file:
+                    self.output= output_file.read()
+
+            elif mt in EXCEL_FILE_TYPES:
+                self.output = self.convert_excel_to_markdown()
+
+            else:
+                print(mt)
+
+    def convert_excel_to_markdown(self, excel_ods_engine: PandasReadEngineType = None) -> str:
+        """
+        Converts all sheets from an Excel or ODS file into a single Markdown string.
+        Each sheet is converted to a Markdown table, prefixed with the sheet's name.
+
+        Args:
+            excel_ods_engine (str | None, optional): Pandas engine for reading Excel or ODS files.
+                - For Excel: 'openpyxl' (for .xlsx), 'xlrd' (for .xls).
+                - For ODS: 'odf' (requires 'odfpy' library).
+                If None, pandas auto-detects based on file extension and installed libraries.
+
+        Returns:
+            str: A string containing the Markdown tables for all sheets, or an error message.
+        """
+        input_filepath = self.input_filepath
+        markdown_output = []
+
+        file_suffix= ''
+        try:
+            if not input_filepath.exists():
+                return f"Error: Input file not found at {input_filepath}"
+
+            file_suffix = input_filepath.suffix.lower()
+            current_engine: PandasReadEngineType = excel_ods_engine
+
+            if file_suffix in ['.xls', '.xlsx', '.ods']:
+                if file_suffix == '.ods':
+                    if current_engine is None:
+                        current_engine = 'odf'
+                    elif current_engine != 'odf':
+                        print(f"Warning: Specified engine '{current_engine}' may not be optimal for ODS. Forcing 'odf'.")
+                        current_engine = 'odf'
+
+                excel_file = pd.ExcelFile(input_filepath, engine=current_engine)
+                if not excel_file.sheet_names:
+                    return f"Warning: File '{input_filepath.name}' contains no sheets."
+
+                for sheet_name in excel_file.sheet_names:
+                    df = excel_file.parse(sheet_name)
+                    markdown_output.append(f"## {sheet_name}\n")
+                    markdown_table = tabulate(df, headers='keys', tablefmt='pipe')
+                    markdown_output.append(markdown_table)
+                    markdown_output.append("\n")
+
+                return "\n".join(markdown_output)
+
+            elif file_suffix == '.csv':
+                df = pd.read_csv(input_filepath)
+                markdown_table = tabulate(df, headers='keys', tablefmt='pipe')
+                return markdown_table
+
+            else:
+                return f"Error: Unsupported file type: '{file_suffix}'. Please provide a CSV, XLS, XLSX, or ODS file."
+
+        except ImportError as ie:
+            if 'odfpy' in str(ie).lower() and file_suffix == '.ods':
+                return f"Error reading ODS file '{input_filepath.name}': The 'odfpy' library is required. Please install it using 'pip install odfpy'."
+            elif 'xlrd' in str(ie).lower() and file_suffix == '.xls':
+                return f"Error reading .xls file '{input_filepath.name}': The 'xlrd' library might be required. Please install it using 'pip install xlrd'."
+            elif 'openpyxl' in str(ie).lower() and file_suffix == '.xlsx':
+                return f"Error reading .xlsx file '{input_filepath.name}': The 'openpyxl' library might be required. Please install it using 'pip install openpyxl'."
+            else:
+                return f"ImportError reading file '{input_filepath.name}': {ie}"
+        except Exception as e:
+            return f"An unexpected error occurred during conversion of '{input_filepath.name}': {e}"
+
+
+def chunk_text(text, chunk_size=500, overlap_size=50):
+    """
+    Chunks a given text into smaller pieces with optional overlap.
+
+    Args:
+        text (str): The input text to be chunked.
+        chunk_size (int): The maximum size of each chunk (in characters).
+        overlap_size (int): The number of characters to overlap between consecutive chunks.
+
+    Returns:
+        list: A list of text chunks.
+    """
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        chunk = text[start:end]
+        chunks.append(chunk)
+        start += (chunk_size - overlap_size)
+        if start < 0:  # Handle cases where overlap_size is greater than chunk_size
+            start = 0
+    return chunks
+
+if __name__ == '__main__':
+    print("Test Run Start:")
+    try:
+        # print("Test 1: scaned pdf page, bytes")
+        # with open("/home/cyto/Documents/scanned.pdf", "rb") as imgpdf:
+        #     conv= Convertor(file_bytes= imgpdf.read(), suffix= ".pdf", file_type= "pdf")
+        #     print(conv.output)
+
+        # print("Test 2: JD pdf, bytes")
+        # with open("/home/cyto/dev/pembotdir/jds/PM Trainee.pdf", "rb") as imgpdf:
+        #     conv= Convertor(file_bytes= imgpdf.read(), suffix= ".pdf", file_type= "pdf")
+        #     print(conv.output)
+
+        print("Test 3: excel schedule, bytes")
+        with open("/home/cyto/Downloads/Assignment schedule.xlsx", "rb") as imgpdf:
+            conv= Convertor(file_bytes= imgpdf.read(), suffix= ".xlsx", file_type= "excel")
+            print(conv.output)
+
+        # without bytes example:
+        print("Test 4: scanned pdf, path")
+        conv= Convertor(myfile= Path('/home/cyto/Documents/scanned.pdf'), output_dir= Path('/home/cyto/Documents'))
+        print(conv.output)
+
+        print("Test 5: schedule excel, path")
+        conv= Convertor(myfile= Path('/home/cyto/Downloads/Assignment schedule.xlsx'), output_dir= Path('/home/cyto/Downloads'))
+        print(conv.output)
+    except FileNotFoundError as fe:
+        print("file not found, modify the driver code to get sample files to test:\n\n", fe)
+    except Exception as e:
+        print("unhandled: ", e)
+
+    print("Test Run End.")

{pembot-0.1.2 → pembot-0.1.3}/pembot/__init__.py

@@ -1,6 +1,6 @@
 """
 A Python Package to convert PEM blog content to usseful information by leveraging LLMs
 """
-__version__ = '0.1.2'
+__version__ = '0.1.3'
 from .main import save_to_json_file, make_query
 __all__ = ["save_to_json_file", "make_query"]

{pembot-0.1.2 → pembot-0.1.3}/pembot/config/config.yaml

@@ -2,4 +2,4 @@ OUTPUT_DIR: /home/cyto/dev/pembotdir
 PAGE_DELIMITER: ___________________________ NEXT PAGE ___________________________
 app:
   name: pembot
-  version: 0.1.2
+  version: 0.1.3

{pembot-0.1.2 → pembot-0.1.3}/pembot/requirements.txt

@@ -9,6 +9,7 @@ cffi==1.17.1
 charset-normalizer==3.4.2
 click==8.2.1
 cryptography==45.0.5
+defusedxml==0.7.1
 dnspython==2.7.0
 duckduckgo_search==8.1.1
 et_xmlfile==2.0.0
@@ -36,6 +37,7 @@ MarkupSafe==3.0.2
 mdurl==0.1.2
 msgpack==1.1.1
 numpy==2.3.1
+odfpy==1.4.1
 ollama==0.5.1
 openpyxl==3.1.5
 orjson==3.10.18
@@ -44,7 +46,7 @@ pandas==2.3.0
 pathlib==1.0.1
 pdfminer.six==20250506
 pdfplumber==0.11.7
-pembot==0.1.2
+pembot==0.1.3
 pillow==11.3.0
 primp==0.15.0
 pyasn1==0.6.1
@@ -76,6 +78,7 @@ smolagents==1.20.0
 sniffio==1.3.1
 soupsieve==2.7
 starlette==0.46.2
+tabulate==0.9.0
 tenacity==8.5.0
 tomlkit==0.13.3
 tqdm==4.67.1
@@ -86,3 +89,4 @@ tzdata==2025.2
 urllib3==2.5.0
 uvicorn==0.35.0
 websockets==15.0.1
+xlrd==2.0.2

pembot-0.1.2/pembot/.git/COMMIT_EDITMSG

@@ -1 +0,0 @@
-minor oopsie

pembot-0.1.2/pembot/.git/refs/heads/main

@@ -1 +0,0 @@
-8fc00bf69f4ad3e50c13acc4a0988b6c0fe72b5a

pembot-0.1.2/pembot/.git/refs/remotes/origin/main

@@ -1 +0,0 @@
-8fc00bf69f4ad3e50c13acc4a0988b6c0fe72b5a