sunholo 0.113.3__tar.gz → 0.114.1__tar.gz

{sunholo-0.113.3 → sunholo-0.114.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.113.3
+Version: 0.114.1
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.113.3.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.114.1.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -81,6 +81,11 @@ Requires-Dist: tenacity; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
 Requires-Dist: unstructured[all-docs,local-inference]; extra == "all"
 Requires-Dist: xlwings; extra == "all"
+Provides-Extra: langchain
+Requires-Dist: langchain==0.2.16; extra == "langchain"
+Requires-Dist: langchain_experimental==0.0.65; extra == "langchain"
+Requires-Dist: langchain-community==0.2.17; extra == "langchain"
+Requires-Dist: langsmith==0.1.143; extra == "langchain"
 Provides-Extra: azure
 Requires-Dist: azure-identity; extra == "azure"
 Requires-Dist: azure-storage-blob; extra == "azure"

{sunholo-0.113.3 → sunholo-0.114.1}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-version = '0.113.3'
+version = '0.114.1'
 
 setup(
     name='sunholo',
@@ -94,6 +94,12 @@ setup(
             "unstructured[local-inference,all-docs]",
             "xlwings"
         ],
+        'langchain': [
+            "langchain==0.2.16",
+            "langchain_experimental==0.0.65",
+            "langchain-community==0.2.17",
+            "langsmith==0.1.143",
+        ],
         'azure': [
             "azure-identity",
             "azure-storage-blob"

{sunholo-0.113.3 → sunholo-0.114.1}/sunholo/chunker/doc_handling.py

@@ -12,10 +12,14 @@ import tempfile
 import traceback
 import json
 import os
-from langchain.docstore.document import Document
-
-from langchain.prompts import PromptTemplate
-from langchain_core.output_parsers import StrOutputParser
+try:
+    from langchain.docstore.document import Document
+    from langchain.prompts import PromptTemplate
+    from langchain_core.output_parsers import StrOutputParser
+except ImportError:
+    Document = None
+    PromptTemplate = None
+    StrOutputParser = None
 
 def send_doc_to_docstore(docs, vector_name):
 
@@ -53,6 +57,9 @@ def send_doc_to_docstore(docs, vector_name):
 
 def create_big_doc(docs):
 
+    if not Document:
+        raise ImportError("Document object requires langchain installed")
+
     if not docs:
         return None, None, None
     
@@ -107,6 +114,9 @@ def create_big_doc(docs):
 
 def summarise_docs(docs, vector_name, summary_threshold_default=10000, model_limit_default=25000):
 
+    if not PromptTemplate or not StrOutputParser:
+        raise ImportError("PromptTemplate and StrOutputParser requires langchain installed")
+    
     if not docs:
         return None
     

{sunholo-0.113.3 → sunholo-0.114.1}/sunholo/chunker/loaders.py

@@ -13,11 +13,12 @@
 #   limitations under the License.
 try:
     from langchain_unstructured import UnstructuredLoader
+    from langchain_community.document_loaders import GitLoader
+    from langchain_community.document_loaders import GoogleDriveLoader
 except ImportError:
     UnstructuredLoader = None
-
-from langchain_community.document_loaders import GitLoader
-from langchain_community.document_loaders import GoogleDriveLoader
+    GitLoader=None
+    GoogleDriveLoader=None
 
 from ..custom_logging import log
 from .pdfs import read_pdf_file
@@ -43,56 +44,56 @@ def convert_to_txt(file_path):
     shutil.copyfile(file_path, txt_file)
     return txt_file
 
+if GoogleDriveLoader is not None:
+    class MyGoogleDriveLoader(GoogleDriveLoader):
+        url: Optional[str] = Field(None)
 
-class MyGoogleDriveLoader(GoogleDriveLoader):
-    url: Optional[str] = Field(None)
-
-    def __init__(self, url, *args, **kwargs):
-        super().__init__(*args, **kwargs, file_ids=['dummy']) # Pass dummy value
-        self.url = url
-
-    def _extract_id(self, url):
-        parsed_url = urlparse(unquote(url))
-        path_parts = parsed_url.path.split('/')
-        
-        # Iterate over the parts
-        for part in path_parts:
-            # IDs are typically alphanumeric and at least a few characters long
-            # So let's say that to be an ID, a part has to be at least 15 characters long
-            if all(char.isalnum() or char in ['_', '-'] for char in part) and len(part) >= 15:
-                return part
-        
-        # Return None if no ID was found
-        return None
+        def __init__(self, url, *args, **kwargs):
+            super().__init__(*args, **kwargs, file_ids=['dummy']) # Pass dummy value
+            self.url = url
 
-    def load_from_url(self, url: str):
-        id = self._extract_id(url)
-        from googleapiclient.errors import HttpError
-        from googleapiclient.discovery import build
-
-        # Identify type of URL
-        try:
-            service = build("drive", "v3", credentials=self._load_credentials())
-            file = service.files().get(fileId=id).execute()
-        except HttpError as err:
-            log.error(f"Error loading file {url}: {str(err)}")
-            raise
-
-        mime_type = file["mimeType"]
-
-        if "folder" in mime_type:
-            # If it's a folder, load documents from the folder
-            return self._load_documents_from_folder(id)
-        else:
-            # If it's not a folder, treat it as a single file
-            if mime_type == "application/vnd.google-apps.document":
-                return [self._load_document_from_id(id)]
-            elif mime_type == "application/vnd.google-apps.spreadsheet":
-                return self._load_sheet_from_id(id)
-            elif mime_type == "application/pdf":
-                return [self._load_file_from_id(id)]
+        def _extract_id(self, url):
+            parsed_url = urlparse(unquote(url))
+            path_parts = parsed_url.path.split('/')
+            
+            # Iterate over the parts
+            for part in path_parts:
+                # IDs are typically alphanumeric and at least a few characters long
+                # So let's say that to be an ID, a part has to be at least 15 characters long
+                if all(char.isalnum() or char in ['_', '-'] for char in part) and len(part) >= 15:
+                    return part
+            
+            # Return None if no ID was found
+            return None
+
+        def load_from_url(self, url: str):
+            id = self._extract_id(url)
+            from googleapiclient.errors import HttpError
+            from googleapiclient.discovery import build
+
+            # Identify type of URL
+            try:
+                service = build("drive", "v3", credentials=self._load_credentials())
+                file = service.files().get(fileId=id).execute()
+            except HttpError as err:
+                log.error(f"Error loading file {url}: {str(err)}")
+                raise
+
+            mime_type = file["mimeType"]
+
+            if "folder" in mime_type:
+                # If it's a folder, load documents from the folder
+                return self._load_documents_from_folder(id)
             else:
-                return []
+                # If it's not a folder, treat it as a single file
+                if mime_type == "application/vnd.google-apps.document":
+                    return [self._load_document_from_id(id)]
+                elif mime_type == "application/vnd.google-apps.spreadsheet":
+                    return self._load_sheet_from_id(id)
+                elif mime_type == "application/pdf":
+                    return [self._load_file_from_id(id)]
+                else:
+                    return []
 
 def ignore_files(filepath):
     """Returns True if the given path's file extension is found within 

{sunholo-0.113.3 → sunholo-0.114.1}/sunholo/chunker/message_data.py

@@ -29,8 +29,10 @@ try:
 except ImportError:
     BlobServiceClient = None
 
-from langchain.schema import Document
-
+try:
+    from langchain.schema import Document
+except ImportError:
+    Document = None
 
 from .splitter import chunk_doc_to_docs
 from .pdfs import split_pdf_to_pages

{sunholo-0.113.3 → sunholo-0.114.1}/sunholo/chunker/publish.py

@@ -3,8 +3,11 @@ from ..pubsub import PubSubManager
 from ..utils.parsers import contains_url, extract_urls
 from ..utils.gcp_project import get_gcp_project
 
-from langchain.schema import Document
-
+try:
+    from langchain.schema import Document
+except ImportError:
+    Document=None
+    
 def publish_if_urls(the_content, vector_name):
     """
     Extracts URLs and puts them in a queue for processing on PubSub

{sunholo-0.113.3 → sunholo-0.114.1}/sunholo/chunker/splitter.py

@@ -13,12 +13,17 @@
 #   limitations under the License.
 from ..custom_logging import log
 from ..utils.parsers import remove_whitespace
-from langchain.schema import Document
-import langchain.text_splitter as text_splitter
+
 from .images import upload_doc_images
 from .doc_handling import send_doc_to_docstore, summarise_docs
 from ..database.uuid import generate_uuid_from_object_id
 
+try:
+    from langchain.schema import Document
+    import langchain.text_splitter as text_splitter
+except ImportError:
+    Document=None
+    text_splitter=None
 
 def chunk_doc_to_docs(documents: list, extension: str = ".md", min_size: int = 800, vector_name=None, **kwargs):
     """Turns a Document object into a list of many Document chunks.

{sunholo-0.113.3 → sunholo-0.114.1}/sunholo/components/retriever.py

@@ -17,13 +17,17 @@ from ..utils import ConfigManager
 from .llm import get_embeddings
 from ..utils.gcp_project import get_gcp_project
 
-from langchain.retrievers import MergerRetriever
-from langchain_community.retrievers import GoogleCloudEnterpriseSearchRetriever
-# https://python.langchain.com/docs/integrations/retrievers/merger_retriever
-from langchain_community.document_transformers import EmbeddingsRedundantFilter
-from langchain.retrievers.document_compressors import DocumentCompressorPipeline
-from langchain.retrievers import ContextualCompressionRetriever
-
+try:
+    from langchain.retrievers import MergerRetriever
+    # https://python.langchain.com/docs/integrations/retrievers/merger_retriever
+    from langchain_community.document_transformers import EmbeddingsRedundantFilter
+    from langchain.retrievers.document_compressors import DocumentCompressorPipeline
+    from langchain.retrievers import ContextualCompressionRetriever
+except ImportError:
+    MergerRetriever=None
+    EmbeddingsRedundantFilter=None
+    DocumentCompressorPipeline=None
+    ContextualCompressionRetriever=None
 
 
 def load_memories(vector_name:str=None, config:ConfigManager=None):

{sunholo-0.113.3 → sunholo-0.114.1}/sunholo/embedder/embed_chunk.py

@@ -17,8 +17,11 @@ import json
 import datetime
 import uuid
 
-from langchain.schema import Document
-
+try:
+    from langchain.schema import Document
+except ImportError:
+    Document = None
+    
 from ..components import get_embeddings, pick_vectorstore, load_memories, pick_embedding
 from ..custom_logging import log
 from ..database.uuid import generate_uuid_from_object_id