sunholo 0.111.0__py3-none-any.whl → 0.112.0__py3-none-any.whl

sunholo/chunker/loaders.py

@@ -11,9 +11,10 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from langchain_community.document_loaders import UnstructuredFileLoader
-from langchain_community.document_loaders import UnstructuredAPIFileLoader
-from langchain_community.document_loaders import UnstructuredURLLoader
+try:
+    from langchain_unstructured import UnstructuredLoader
+except ImportError:
+    UnstructuredLoader = None
 
 from langchain_community.document_loaders import GitLoader
 from langchain_community.document_loaders import GoogleDriveLoader
@@ -159,10 +160,12 @@ def read_gdrive_to_document(url: str, metadata: dict = None):
 
 def read_url_to_document(url: str, metadata: dict = None):
 
+    if not  UnstructuredLoader:
+        raise ImportError("UnstructuredLoader requires 'langchain_unstructured' to be installed")
     unstructured_kwargs = {"pdf_infer_table_structure": True,
                             "extract_image_block_types":  ["Image", "Table"]
                             }
-    loader = UnstructuredURLLoader(urls=[url], mode="elements", unstructured_kwargs=unstructured_kwargs)
+    loader = UnstructuredLoader(web_url=url, mode="elements", unstructured_kwargs=unstructured_kwargs)
     docs = loader.load()
     if metadata is not None:
         for doc in docs:
@@ -170,7 +173,7 @@ def read_url_to_document(url: str, metadata: dict = None):
             if not doc.metadata.get("source") and doc.metadata.get("url"):
                 doc.metadata["source"] = doc.metadata["url"]
     
-    log.info(f"UnstructuredURLLoader docs: {docs}")
+    log.info(f"UnstructuredLoader docs: {docs}")
     
     return docs
 
@@ -184,18 +187,21 @@ def read_file_to_documents(gs_file: pathlib.Path, metadata: dict = None):
             log.info(f"Already uploaded to bucket, skipping {pdf_path}")
             return []
 
-    log.info(f"Sending {pdf_path} to UnstructuredAPIFileLoader")
+    log.info(f"Sending {pdf_path} to UnstructuredLoader")
     UNSTRUCTURED_URL = os.getenv("UNSTRUCTURED_URL")
     unstructured_kwargs = {"pdf_infer_table_structure": True,
                             "extract_image_block_types":  ["Image", "Table"]
                             }
     
+    if not  UnstructuredLoader:
+        raise ImportError("UnstructuredLoader requires 'langchain_unstructured' to be installed")
+    
     if UNSTRUCTURED_URL:
         log.debug(f"Found UNSTRUCTURED_URL: {UNSTRUCTURED_URL}")
         the_endpoint = f"{UNSTRUCTURED_URL}/general/v0/general"
         try:
-            loader = UnstructuredAPIFileLoader(
-                pdf_path,
+            loader = UnstructuredLoader(
+                file_path=pdf_path,
                 url=the_endpoint,
                 mode="elements",
                 **unstructured_kwargs)
@@ -206,8 +212,8 @@ def read_file_to_documents(gs_file: pathlib.Path, metadata: dict = None):
             else:
                 raise err
     else:
-        loader = UnstructuredAPIFileLoader(
-            pdf_path,
+        loader = UnstructuredLoader(
+            file_path=pdf_path,
             api_key=UNSTRUCTURED_KEY,
             mode="elements",
             **unstructured_kwargs)
@@ -216,7 +222,7 @@ def read_file_to_documents(gs_file: pathlib.Path, metadata: dict = None):
     try:
         docs = loader.load() # this takes a long time 30m+ for big PDF files
     except ValueError as e:
-        log.info(f"Error for {gs_file} from UnstructuredAPIFileLoader: {str(e)}")
+        log.info(f"Error for {gs_file} from UnstructuredLoader: {str(e)}")
         pdf_path = pathlib.Path(gs_file)
         if pdf_path.suffix == ".pdf":
             local_doc = read_pdf_file(pdf_path, metadata=metadata)
@@ -262,13 +268,16 @@ def read_file_to_documents(gs_file: pathlib.Path, metadata: dict = None):
 
 def convert_to_txt_and_extract(gs_file, split=False):
 
+    if not  UnstructuredLoader:
+        raise ImportError("UnstructuredLoader requires 'langchain_unstructured' to be installed")
+    
     log.info("trying file parsing locally via .txt conversion")
     txt_file = None
     docs = []
     try:
         # Convert the file to .txt and try again
         txt_file = convert_to_txt(gs_file)
-        loader = UnstructuredFileLoader(
+        loader = UnstructuredLoader(
             txt_file, 
             mode="elements")
 

{sunholo-0.111.0.dist-info → sunholo-0.112.0.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.111.0
+Version: 0.112.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.111.0.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.112.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -59,6 +59,7 @@ Requires-Dist: langchain-google-genai==1.0.10; extra == "all"
 Requires-Dist: langchain_google_alloydb_pg; extra == "all"
 Requires-Dist: langchain-anthropic==0.1.23; extra == "all"
 Requires-Dist: langchain-google-vertexai; extra == "all"
+Requires-Dist: langchain-unstructured; extra == "all"
 Requires-Dist: langfuse; extra == "all"
 Requires-Dist: numpy; extra == "all"
 Requires-Dist: pg8000; extra == "all"
@@ -78,7 +79,7 @@ Requires-Dist: tabulate; extra == "all"
 Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tenacity; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
-Requires-Dist: unstructured[local-inference]==0.14.9; extra == "all"
+Requires-Dist: unstructured[all-docs,local-inference]; extra == "all"
 Requires-Dist: xlwings; extra == "all"
 Provides-Extra: azure
 Requires-Dist: azure-identity; extra == "azure"
@@ -98,11 +99,12 @@ Requires-Dist: tantivy; extra == "database"
 Provides-Extra: pipeline
 Requires-Dist: GitPython; extra == "pipeline"
 Requires-Dist: lark; extra == "pipeline"
+Requires-Dist: langchain-unstructured; extra == "pipeline"
 Requires-Dist: psutil; extra == "pipeline"
 Requires-Dist: pypdf; extra == "pipeline"
 Requires-Dist: pytesseract; extra == "pipeline"
 Requires-Dist: tabulate; extra == "pipeline"
-Requires-Dist: unstructured[local-inference]==0.14.9; extra == "pipeline"
+Requires-Dist: unstructured[all-docs,local-inference]; extra == "pipeline"
 Provides-Extra: gcp
 Requires-Dist: anthropic[vertex]; extra == "gcp"
 Requires-Dist: google-api-python-client; extra == "gcp"

{sunholo-0.111.0.dist-info → sunholo-0.112.0.dist-info}/RECORD

@@ -34,7 +34,7 @@ sunholo/chunker/azure.py,sha256=MVF9_-QdKUoJqlpEJ49pv2sdjMDxEiMNxzmO7w5nWDQ,3270
 sunholo/chunker/doc_handling.py,sha256=UAf9BmUMpKCKRlAMl1qNZK6xDNYWk1z3ARoftWoa_54,8734
 sunholo/chunker/encode_metadata.py,sha256=hxxd9KU35Xi0Z_EL8kt_oD66pKfBLhEjBImC16ew-Eo,1919
 sunholo/chunker/images.py,sha256=id2PBu6XyGEOtgafq2v0c9_O6kxaC_pYFMnbsIitkSg,1868
-sunholo/chunker/loaders.py,sha256=n64UgXDB8ZkApnqJwb3VKc8lGTVo8TC1AXmARiaCsQY,10269
+sunholo/chunker/loaders.py,sha256=QaM-M1wmbA2iLIDvBKpC5-TPKMlQIxS01gMKj5n9RyM,10547
 sunholo/chunker/message_data.py,sha256=EaiY7_HClpcfPUAYaAm6Zk5ReeZ9s9F_jBVd0kDgI-4,10836
 sunholo/chunker/pdfs.py,sha256=njDPop751GMHi3cOwIKd2Yct-_lWR2gqcB7WykfHphs,2480
 sunholo/chunker/process_chunker_data.py,sha256=uO-YOEHIjAOy0ZMJ0vea9OMNsQBISHfhbtgoyuHiP6s,3598
@@ -150,9 +150,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.111.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.111.0.dist-info/METADATA,sha256=DC7Aa6AgAANAsvuyg3h0DLz-d0auWyaZu5oviSIgwpU,8570
-sunholo-0.111.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-sunholo-0.111.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.111.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.111.0.dist-info/RECORD,,
+sunholo-0.112.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.112.0.dist-info/METADATA,sha256=kiphk-fAQurwpMyZVdgAqVdhS4yUr8AJPLjUAZze2_I,8685
+sunholo-0.112.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+sunholo-0.112.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.112.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.112.0.dist-info/RECORD,,