sunholo 0.111.0__tar.gz → 0.112.3__tar.gz

{sunholo-0.111.0 → sunholo-0.112.3}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.111.0
+Version: 0.112.3
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.111.0.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.112.3.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -59,6 +59,7 @@ Requires-Dist: langchain-google-genai==1.0.10; extra == "all"
 Requires-Dist: langchain_google_alloydb_pg; extra == "all"
 Requires-Dist: langchain-anthropic==0.1.23; extra == "all"
 Requires-Dist: langchain-google-vertexai; extra == "all"
+Requires-Dist: langchain-unstructured; extra == "all"
 Requires-Dist: langfuse; extra == "all"
 Requires-Dist: numpy; extra == "all"
 Requires-Dist: pg8000; extra == "all"
@@ -78,7 +79,7 @@ Requires-Dist: tabulate; extra == "all"
 Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tenacity; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
-Requires-Dist: unstructured[local-inference]==0.14.9; extra == "all"
+Requires-Dist: unstructured[all-docs,local-inference]; extra == "all"
 Requires-Dist: xlwings; extra == "all"
 Provides-Extra: azure
 Requires-Dist: azure-identity; extra == "azure"
@@ -98,11 +99,12 @@ Requires-Dist: tantivy; extra == "database"
 Provides-Extra: pipeline
 Requires-Dist: GitPython; extra == "pipeline"
 Requires-Dist: lark; extra == "pipeline"
+Requires-Dist: langchain-unstructured; extra == "pipeline"
 Requires-Dist: psutil; extra == "pipeline"
 Requires-Dist: pypdf; extra == "pipeline"
 Requires-Dist: pytesseract; extra == "pipeline"
 Requires-Dist: tabulate; extra == "pipeline"
-Requires-Dist: unstructured[local-inference]==0.14.9; extra == "pipeline"
+Requires-Dist: unstructured[all-docs,local-inference]; extra == "pipeline"
 Provides-Extra: gcp
 Requires-Dist: anthropic[vertex]; extra == "gcp"
 Requires-Dist: google-api-python-client; extra == "gcp"

{sunholo-0.111.0 → sunholo-0.112.3}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-version = '0.111.0'
+version = '0.112.3'
 
 setup(
     name='sunholo',
@@ -71,6 +71,7 @@ setup(
             "langchain_google_alloydb_pg",
             "langchain-anthropic==0.1.23",
             "langchain-google-vertexai",
+            "langchain-unstructured",
             "langfuse",
             "numpy",
             "pg8000",
@@ -90,7 +91,7 @@ setup(
             "tantivy",
             "tenacity",
             "tiktoken",
-            "unstructured[local-inference]==0.14.9",
+            "unstructured[local-inference,all-docs]",
             "xlwings"
         ],
         'azure': [
@@ -114,11 +115,12 @@ setup(
         'pipeline': [
             "GitPython",
             "lark",
+            "langchain-unstructured",
             "psutil",
             "pypdf",
             "pytesseract",
             "tabulate",
-            "unstructured[local-inference]==0.14.9",
+            "unstructured[local-inference,all-docs]"
         ],
         'gcp': [
             "anthropic[vertex]",

{sunholo-0.111.0 → sunholo-0.112.3}/sunholo/chunker/loaders.py

@@ -11,9 +11,10 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from langchain_community.document_loaders import UnstructuredFileLoader
-from langchain_community.document_loaders import UnstructuredAPIFileLoader
-from langchain_community.document_loaders import UnstructuredURLLoader
+try:
+    from langchain_unstructured import UnstructuredLoader
+except ImportError:
+    UnstructuredLoader = None
 
 from langchain_community.document_loaders import GitLoader
 from langchain_community.document_loaders import GoogleDriveLoader
@@ -159,10 +160,12 @@ def read_gdrive_to_document(url: str, metadata: dict = None):
 
 def read_url_to_document(url: str, metadata: dict = None):
 
+    if not  UnstructuredLoader:
+        raise ImportError("UnstructuredLoader requires 'langchain_unstructured' to be installed")
     unstructured_kwargs = {"pdf_infer_table_structure": True,
                             "extract_image_block_types":  ["Image", "Table"]
                             }
-    loader = UnstructuredURLLoader(urls=[url], mode="elements", unstructured_kwargs=unstructured_kwargs)
+    loader = UnstructuredLoader(web_url=url, mode="elements", unstructured_kwargs=unstructured_kwargs)
     docs = loader.load()
     if metadata is not None:
         for doc in docs:
@@ -170,7 +173,7 @@ def read_url_to_document(url: str, metadata: dict = None):
             if not doc.metadata.get("source") and doc.metadata.get("url"):
                 doc.metadata["source"] = doc.metadata["url"]
     
-    log.info(f"UnstructuredURLLoader docs: {docs}")
+    log.info(f"UnstructuredLoader docs: {docs}")
     
     return docs
 
@@ -184,18 +187,21 @@ def read_file_to_documents(gs_file: pathlib.Path, metadata: dict = None):
             log.info(f"Already uploaded to bucket, skipping {pdf_path}")
             return []
 
-    log.info(f"Sending {pdf_path} to UnstructuredAPIFileLoader")
+    log.info(f"Sending {pdf_path} to UnstructuredLoader")
     UNSTRUCTURED_URL = os.getenv("UNSTRUCTURED_URL")
     unstructured_kwargs = {"pdf_infer_table_structure": True,
                             "extract_image_block_types":  ["Image", "Table"]
                             }
     
+    if not  UnstructuredLoader:
+        raise ImportError("UnstructuredLoader requires 'langchain_unstructured' to be installed")
+    
     if UNSTRUCTURED_URL:
         log.debug(f"Found UNSTRUCTURED_URL: {UNSTRUCTURED_URL}")
         the_endpoint = f"{UNSTRUCTURED_URL}/general/v0/general"
         try:
-            loader = UnstructuredAPIFileLoader(
-                pdf_path,
+            loader = UnstructuredLoader(
+                file_path=pdf_path,
                 url=the_endpoint,
                 mode="elements",
                 **unstructured_kwargs)
@@ -206,8 +212,8 @@ def read_file_to_documents(gs_file: pathlib.Path, metadata: dict = None):
             else:
                 raise err
     else:
-        loader = UnstructuredAPIFileLoader(
-            pdf_path,
+        loader = UnstructuredLoader(
+            file_path=pdf_path,
             api_key=UNSTRUCTURED_KEY,
             mode="elements",
             **unstructured_kwargs)
@@ -216,7 +222,7 @@ def read_file_to_documents(gs_file: pathlib.Path, metadata: dict = None):
     try:
         docs = loader.load() # this takes a long time 30m+ for big PDF files
     except ValueError as e:
-        log.info(f"Error for {gs_file} from UnstructuredAPIFileLoader: {str(e)}")
+        log.info(f"Error for {gs_file} from UnstructuredLoader: {str(e)}")
         pdf_path = pathlib.Path(gs_file)
         if pdf_path.suffix == ".pdf":
             local_doc = read_pdf_file(pdf_path, metadata=metadata)
@@ -262,13 +268,16 @@ def read_file_to_documents(gs_file: pathlib.Path, metadata: dict = None):
 
 def convert_to_txt_and_extract(gs_file, split=False):
 
+    if not  UnstructuredLoader:
+        raise ImportError("UnstructuredLoader requires 'langchain_unstructured' to be installed")
+    
     log.info("trying file parsing locally via .txt conversion")
     txt_file = None
     docs = []
     try:
         # Convert the file to .txt and try again
         txt_file = convert_to_txt(gs_file)
-        loader = UnstructuredFileLoader(
+        loader = UnstructuredLoader(
             txt_file, 
             mode="elements")
 

{sunholo-0.111.0 → sunholo-0.112.3}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.111.0
+Version: 0.112.3
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.111.0.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.112.3.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -59,6 +59,7 @@ Requires-Dist: langchain-google-genai==1.0.10; extra == "all"
 Requires-Dist: langchain_google_alloydb_pg; extra == "all"
 Requires-Dist: langchain-anthropic==0.1.23; extra == "all"
 Requires-Dist: langchain-google-vertexai; extra == "all"
+Requires-Dist: langchain-unstructured; extra == "all"
 Requires-Dist: langfuse; extra == "all"
 Requires-Dist: numpy; extra == "all"
 Requires-Dist: pg8000; extra == "all"
@@ -78,7 +79,7 @@ Requires-Dist: tabulate; extra == "all"
 Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tenacity; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
-Requires-Dist: unstructured[local-inference]==0.14.9; extra == "all"
+Requires-Dist: unstructured[all-docs,local-inference]; extra == "all"
 Requires-Dist: xlwings; extra == "all"
 Provides-Extra: azure
 Requires-Dist: azure-identity; extra == "azure"
@@ -98,11 +99,12 @@ Requires-Dist: tantivy; extra == "database"
 Provides-Extra: pipeline
 Requires-Dist: GitPython; extra == "pipeline"
 Requires-Dist: lark; extra == "pipeline"
+Requires-Dist: langchain-unstructured; extra == "pipeline"
 Requires-Dist: psutil; extra == "pipeline"
 Requires-Dist: pypdf; extra == "pipeline"
 Requires-Dist: pytesseract; extra == "pipeline"
 Requires-Dist: tabulate; extra == "pipeline"
-Requires-Dist: unstructured[local-inference]==0.14.9; extra == "pipeline"
+Requires-Dist: unstructured[all-docs,local-inference]; extra == "pipeline"
 Provides-Extra: gcp
 Requires-Dist: anthropic[vertex]; extra == "gcp"
 Requires-Dist: google-api-python-client; extra == "gcp"

{sunholo-0.111.0 → sunholo-0.112.3}/sunholo.egg-info/SOURCES.txt

@@ -163,4 +163,5 @@ sunholo/vertex/safety.py
 sunholo/vertex/type_dict_to_json.py
 tests/test_async.py
 tests/test_chat_history.py
-tests/test_config.py
+tests/test_config.py
+tests/test_unstructured.py

{sunholo-0.111.0 → sunholo-0.112.3}/sunholo.egg-info/requires.txt

@@ -40,6 +40,7 @@ langchain-google-genai==1.0.10
 langchain_google_alloydb_pg
 langchain-anthropic==0.1.23
 langchain-google-vertexai
+langchain-unstructured
 langfuse
 numpy
 pg8000
@@ -59,7 +60,7 @@ tabulate
 tantivy
 tenacity
 tiktoken
-unstructured[local-inference]==0.14.9
+unstructured[all-docs,local-inference]
 xlwings
 
 [anthropic]
@@ -130,11 +131,12 @@ tiktoken
 [pipeline]
 GitPython
 lark
+langchain-unstructured
 psutil
 pypdf
 pytesseract
 tabulate
-unstructured[local-inference]==0.14.9
+unstructured[all-docs,local-inference]
 
 [tools]
 openapi-spec-validator

sunholo-0.112.3/tests/test_unstructured.py

@@ -0,0 +1,9 @@
+def main():
+    from sunholo.chunker.loaders import read_file_to_documents
+
+    result = read_file_to_documents("README.md")
+    print(result)
+
+
+if __name__ == "__main__":
+    main()