sunholo 0.69.0__tar.gz → 0.69.3__tar.gz

{sunholo-0.69.0 → sunholo-0.69.3}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.0
+Version: 0.69.3
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.0.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.3.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.0 → sunholo-0.69.3}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.69.0'
+version = '0.69.3'
 
 setup(
     name='sunholo',

{sunholo-0.69.0 → sunholo-0.69.3}/sunholo/chunker/data_to_embed_pubsub.py

@@ -20,6 +20,8 @@ from .publish import process_docs_chunks_vector_name
 from .splitter import chunk_doc_to_docs
 
 from ..llamaindex.import_files import llamaindex_chunker_check
+from ..discovery_engine.chunker_handler import discovery_engine_chunker_check
+
 from . import loaders
 
 def direct_file_to_embed(file_name: pathlib.Path, metadata: dict, vector_name: str):
@@ -58,17 +60,20 @@ def process_chunker_data(message_data, metadata, vector_name):
         metadata["vector_name"] = vector_name
 
     if message_data is None:
-        log.error("No message_data was found in data: {data}")
+        log.error(f"No message_data was found in data: {message_data}")
         return
 
     log.debug(f"Found metadata in pubsub: {metadata}")
 
     # checks if only a llamaindex chunking/embedder, return early as no other processing needed
     llamacheck = llamaindex_chunker_check(message_data, metadata, vector_name)
-    
     if llamacheck:
-
         return llamacheck
+    
+    # if only a discovery engine memory, return early as no other processing needed
+    discovery_check = discovery_engine_chunker_check(message_data, metadata, vector_name)
+    if discovery_check:
+        return discovery_check
 
     chunks = []
 

sunholo-0.69.3/sunholo/discovery_engine/chunker_handler.py

@@ -0,0 +1,109 @@
+from ..logging import log
+from ..utils.config import load_config_key
+from ..components import load_memories
+
+from .discovery_engine_client import DiscoveryEngineClient
+    
+
+def do_discovery_engine(message_data, metadata, vector_name):
+    """
+
+    Example:
+    ```python
+    message_data = "gs://bucket_name/path_to_file.txt"
+    metadata = {"user": "admin"}
+    vector_name = "example_vector"
+    response = do_discovery_engine(message_data, metadata, vector_name)
+    print(response)
+    # Imported file to corpus: {'status': 'success'}
+    ```
+    """
+
+    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
+    if not gcp_config:
+        raise ValueError(f"Need config.{vector_name}.gcp_config to configure discovery engine")
+
+    global_project_id = gcp_config.get('project_id')
+    #global_location = gcp_config.get('location')
+    global_data_store_id = gcp_config.get('data_store_id')
+
+    memories = load_memories(vector_name)
+    tools = []
+
+    if not memories:
+        return tools
+    
+    corpuses = []
+    for memory in memories:
+        for key, value in memory.items():  # Now iterate over the dictionary
+            log.info(f"Found memory {key}")
+            vectorstore = value.get('vectorstore')
+            if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
+                log.info(f"Found vectorstore {vectorstore}")
+                data_store_id = value.get('data_store_id')
+                project_id = gcp_config.get('project_id')
+                #location = gcp_config.get('location') 
+                corpus = DiscoveryEngineClient(
+                    data_store_id=data_store_id or global_data_store_id, 
+                    project_id=project_id or global_project_id,
+                    # location needs to be 'eu' or 'us' which doesn't work with other configurations
+                    #location=location or global_location
+                    )
+
+                corpuses.append(corpus)
+    if not corpuses:
+        log.error("Could not find any Discovery Engine corpus to import data to")
+        return None
+
+    log.info(f"Found Discovery Engine / Vertex AI Search {corpuses=}")
+
+    if message_data.startswith("gs://"):
+        log.info(f"DiscoveryEngineClient.import_files for {message_data}")
+        for corp in corpuses:
+            try:
+                response = corp.import_documents(
+                    gcs_uri=message_data
+                )
+                log.info(f"Imported file to corpus: {response} with metadata: {metadata}")
+            except Exception as err:
+                log.error(f"Error importing {message_data} - {corp=} - {str(err)}")
+                continue
+
+        metadata["source"] = message_data
+        return metadata
+    
+    else:
+        log.warning("Only gs:// data is supported for Discovery Engine")
+
+
+def check_discovery_engine_in_memory(vector_name):
+    memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")
+    for memory in memories:  # Iterate over the list
+        for key, value in memory.items():  # Now iterate over the dictionary
+            log.info(f"Found memory {key}")
+            vectorstore = value.get('vectorstore')
+            if vectorstore:
+                if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
+                    log.info(f"Found vectorstore {vectorstore}")
+                    return True
+    
+    return False
+
+def discovery_engine_chunker_check(message_data, metadata, vector_name):
+    # discovery engine handles its own chunking/embedding
+    memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")
+    total_memories = len(memories)
+    llama = None
+    if check_discovery_engine_in_memory(vector_name):
+        llama = do_discovery_engine(message_data, metadata, vector_name)
+        log.info(f"Processed discovery engine: {llama}")
+
+    # If discovery engine is the only entry, return
+    if llama and total_memories == 1:
+
+        return llama
+    
+    elif llama:
+        log.info("Discovery Engine found but not the only memory, continuing with other processes.")
+
+        return None

sunholo-0.69.3/sunholo/discovery_engine/create_new.py

@@ -0,0 +1,28 @@
+from .discovery_engine_client import DiscoveryEngineClient
+from ..utils.config import load_config_key
+
+def create_new_discovery_engine(vector_name):
+    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
+
+    chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
+
+    if chunker_config:
+        chunk_size = chunker_config.get("chunk_size")
+    
+    if not chunk_size:
+        chunk_size = 500
+
+
+    project_id = gcp_config.get('project_id')
+    #location = gcp_config.get('location')
+
+    de = DiscoveryEngineClient(
+                    data_store_id=vector_name, 
+                    project_id=project_id,
+                    # location needs to be 'eu' or 'us' which doesn't work with other configurations
+                    #location=location
+                    )
+
+    new_store = de.create_data_store(chunk_size=chunk_size)
+
+    return new_store

sunholo-0.69.0/sunholo/database/discovery_engine.py → sunholo-0.69.3/sunholo/discovery_engine/discovery_engine_client.py

@@ -6,6 +6,7 @@ except ImportError:
     discoveryengine = None
 
 from ..logging import log
+from typing import Optional
 
 class DiscoveryEngineClient:
     """
@@ -19,7 +20,7 @@ class DiscoveryEngineClient:
     Example:
         ```python
         client = DiscoveryEngineClient(project_id='your-project-id', data_store_id='your-data-store-id')
-        
+
         # Create a collection
         collection_name = client.create_collection("my_new_collection")
 
@@ -61,39 +62,13 @@ class DiscoveryEngineClient:
             else None
         )
         self.client = discoveryengine.DataStoreServiceClient(client_options=client_options)
-        
-
-    def create_collection(self, collection_id: str) -> str:
-        """
-        Creates a new collection within the specified data store.
-
-        Args:
-            collection_id (str): The ID of the collection to create.
-
-        Returns:
-            str: The resource name of the created collection.
-
-        Example:
-            ```python
-            collection_name = client.create_collection('my_new_collection')
-            `
-        """
-
-        parent = self.client.data_store_path(
-            project=self.project_id, location=self.location, data_store=self.data_store_id
-        )
-
-        collection = discoveryengine.Collection(display_name=collection_id)
-        request = discoveryengine.CreateCollectionRequest(
-            parent=parent, collection_id=collection_id, collection=collection
+        self.parent = self.client.branch_path(
+            project=project_id,
+            location=location,
+            data_store=data_store_id,
+            branch="default_branch",
         )
 
-        operation = self.client.create_collection(request=request)
-        log.info(f"Waiting for operation to complete: {operation.operation.name}")
-        response = operation.result()
-
-        return response.name
-
     def create_data_store(
         self, chunk_size: int = 500
     ) -> str:
@@ -106,7 +81,6 @@ class DiscoveryEngineClient:
         Returns:
             str: The name of the long-running operation for data store creation.
         """
-        parent = self.client.common_location_path(project=self.project_id, location=self.location)
 
         # https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1alpha.types.DocumentProcessingConfig
         doc_config = discoveryengine.DocumentProcessingConfig(
@@ -137,7 +111,7 @@ class DiscoveryEngineClient:
 
         # https://cloud.google.com/python/docs/reference/discoveryengine/0.11.4/google.cloud.discoveryengine_v1alpha.types.CreateDataStoreRequest
         request = discoveryengine.CreateDataStoreRequest(
-            parent=parent,
+            parent=self.parent,
             data_store_id=self.data_store_id,
             data_store=data_store,
             # Optional: For Advanced Site Search Only
@@ -162,7 +136,6 @@ class DiscoveryEngineClient:
     def get_chunks(
         self,
         query: str,
-        collection_id: str,
         num_previous_chunks: int = 3,
         num_next_chunks: int = 3,
         page_size: int = 10,
@@ -196,13 +169,10 @@ class DiscoveryEngineClient:
                 serving_config="default_serving_config")
                 ).name
         
-        filter = f'content_search=true AND collection_id="{collection_id}"'
-
         search_request = discoveryengine.SearchRequest(
             serving_config=serving_config,
             query=query,
             page_size=page_size, 
-            filter=filter,
             content_search_spec=discoveryengine.SearchRequest.ContentSearchSpec(
                 #snippet_spec=discoveryengine.SearchRequest.ContentSearchSpec.SnippetSpec(
                 #    return_snippet=True
@@ -219,3 +189,55 @@ class DiscoveryEngineClient:
 
         return search_response
 
+    def import_documents(self,
+        gcs_uri: Optional[str] = None,
+        data_schema="content",
+        bigquery_dataset: Optional[str] = None,
+        bigquery_table: Optional[str] = None,
+        bigquery_project_id: Optional[str] = None,
+    ) -> str:
+        """
+        Args:
+        - gcs_uri: Required. List of Cloud Storage URIs to input files. Each URI can be up to 2000 characters long. URIs can match the full object path (for example, gs://bucket/directory/object.json) or a pattern matching one or more files, such as gs://bucket/directory/*.json. A request can contain at most 100 files (or 100,000 files if data_schema is content). Each file can be up to 2 GB (or 100 MB if data_schema is content). 
+        - data_schema: Must be one of 'user_event', 'custom' or 'document' if using BigQuery. Default 'content' only for GCS. The schema to use when parsing the data from the source. Supported values for document imports: - document (default): One JSON Document per line. Each document must have a valid Document.id. - content: Unstructured data (e.g. PDF, HTML). Each file matched by input_uris becomes a document, with the ID set to the first 128 bits of SHA256(URI) encoded as a hex string. - custom: One custom data JSON per row in arbitrary format that conforms to the defined Schema of the data store. This can only be used by the GENERIC Data Store vertical. - csv: A CSV file with header conforming to the defined Schema of the data store. Each entry after the header is imported as a Document. This can only be used by the GENERIC Data Store vertical. Supported values for user event imports: - user_event (default): One JSON UserEvent per line. 
+        
+        """
+
+        if gcs_uri:
+            request = discoveryengine.ImportDocumentsRequest(
+                parent=self.parent,
+                # https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1alpha.types.GcsSource
+                gcs_source=discoveryengine.GcsSource(
+                    input_uris=[gcs_uri], data_schema=data_schema,
+                ),
+                # Options: `FULL`, `INCREMENTAL`
+                reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,
+            )
+        else:
+            request = discoveryengine.ImportDocumentsRequest(
+                parent=self.parent,
+                bigquery_source=discoveryengine.BigQuerySource(
+                    project_id=bigquery_project_id or self.project_id,
+                    dataset_id=bigquery_dataset,
+                    table_id=bigquery_table,
+                    data_schema=data_schema,
+                ),
+                # Options: `FULL`, `INCREMENTAL`
+                reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,
+            )
+
+        # Make the request
+        operation = self.client.import_documents(request=request)
+
+        log.info(f"Waiting for operation to complete: {operation.operation.name}")
+        response = operation.result()
+
+        # Once the operation is complete,
+        # get information from operation metadata
+        metadata = discoveryengine.ImportDocumentsMetadata(operation.metadata)
+
+        # Handle the response
+        log.info(f"{response=} {metadata=}")
+
+        return operation.operation.name
+

{sunholo-0.69.0 → sunholo-0.69.3}/sunholo/llamaindex/import_files.py

@@ -105,28 +105,8 @@ def do_llamaindex(message_data, metadata, vector_name):
         return metadata
     
     else:
-        raise NotImplementedError("Only gs:// and https://drive data is supported")
-        # write text to file and upload it
-        # TODO(developer): Update and un-comment below lines
-        # project_id = "PROJECT_ID"
-        # corpus_name = "projects/{project_id}/locations/us-central1/ragCorpora/{rag_corpus_id}"
-        # path = "path/to/local/file.txt"
-        # display_name = "file_display_name"
-        # description = "file description"
-
-        # Initialize Vertex AI API once per session
-        #path = 'path/to/local/file.txt'
-
-        # Write the message_data to a file
-        #with open(path, 'w') as file:
-        #    file.write(message_data)
-
-        #rag_file = rag.upload_file(
-        #    corpus_name=corpus_name,
-        #    path=path,
-        #    display_name=display_name,
-        #    description=description,
-        #)
+        log.warning("Only gs:// and https://drive data is supported for llamaindex")
+        
 
 def check_llamaindex_in_memory(vector_name):
     memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")

{sunholo-0.69.0 → sunholo-0.69.3}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.0
+Version: 0.69.3
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.0.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.3.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.0 → sunholo-0.69.3}/sunholo.egg-info/SOURCES.txt

@@ -62,7 +62,6 @@ sunholo/database/__init__.py
 sunholo/database/alloydb.py
 sunholo/database/alloydb_client.py
 sunholo/database/database.py
-sunholo/database/discovery_engine.py
 sunholo/database/lancedb.py
 sunholo/database/static_dbs.py
 sunholo/database/uuid.py
@@ -72,6 +71,10 @@ sunholo/database/sql/sb/create_table.sql
 sunholo/database/sql/sb/delete_source_row.sql
 sunholo/database/sql/sb/return_sources.sql
 sunholo/database/sql/sb/setup.sql
+sunholo/discovery_engine/__init__.py
+sunholo/discovery_engine/chunker_handler.py
+sunholo/discovery_engine/create_new.py
+sunholo/discovery_engine/discovery_engine_client.py
 sunholo/embedder/__init__.py
 sunholo/embedder/embed_chunk.py
 sunholo/gcs/__init__.py