sunholo 0.68.1__tar.gz → 0.69.2__tar.gz

{sunholo-0.68.1 → sunholo-0.69.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.68.1
+Version: 0.69.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.68.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.68.1 → sunholo-0.69.2}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.68.1'
+version = '0.69.2'
 
 setup(
     name='sunholo',

{sunholo-0.68.1 → sunholo-0.69.2}/sunholo/chunker/data_to_embed_pubsub.py

@@ -20,6 +20,8 @@ from .publish import process_docs_chunks_vector_name
 from .splitter import chunk_doc_to_docs
 
 from ..llamaindex.import_files import llamaindex_chunker_check
+from ..discovery_engine.chunker_handler import discovery_engine_chunker_check
+
 from . import loaders
 
 def direct_file_to_embed(file_name: pathlib.Path, metadata: dict, vector_name: str):
@@ -65,10 +67,13 @@ def process_chunker_data(message_data, metadata, vector_name):
 
     # checks if only a llamaindex chunking/embedder, return early as no other processing needed
     llamacheck = llamaindex_chunker_check(message_data, metadata, vector_name)
-    
     if llamacheck:
-
         return llamacheck
+    
+    # if only a discovery engine memory, return early as no other processing needed
+    discovery_check = discovery_engine_chunker_check(message_data, metadata, vector_name)
+    if discovery_check:
+        return discovery_check
 
     chunks = []
 

sunholo-0.69.2/sunholo/discovery_engine/chunker_handler.py

@@ -0,0 +1,109 @@
+from ..logging import log
+from ..utils.config import load_config_key
+from ..components import load_memories
+
+from .discovery_engine_client import DiscoveryEngineClient
+    
+
+def do_discovery_engine(message_data, metadata, vector_name):
+    """
+
+    Example:
+    ```python
+    message_data = "gs://bucket_name/path_to_file.txt"
+    metadata = {"user": "admin"}
+    vector_name = "example_vector"
+    response = do_discovery_engine(message_data, metadata, vector_name)
+    print(response)
+    # Imported file to corpus: {'status': 'success'}
+    ```
+    """
+
+    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
+    if not gcp_config:
+        raise ValueError(f"Need config.{vector_name}.gcp_config to configure discovery engine")
+
+    global_project_id = gcp_config.get('project_id')
+    #global_location = gcp_config.get('location')
+    global_data_store_id = gcp_config.get('data_store_id')
+
+    memories = load_memories(vector_name)
+    tools = []
+
+    if not memories:
+        return tools
+    
+    corpuses = []
+    for memory in memories:
+        for key, value in memory.items():  # Now iterate over the dictionary
+            log.info(f"Found memory {key}")
+            vectorstore = value.get('vectorstore')
+            if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
+                log.info(f"Found vectorstore {vectorstore}")
+                data_store_id = value.get('data_store_id')
+                project_id = gcp_config.get('project_id')
+                #location = gcp_config.get('location') 
+                corpus = DiscoveryEngineClient(
+                    data_store_id=data_store_id or global_data_store_id, 
+                    project_id=project_id or global_project_id,
+                    # location needs to be 'eu' or 'us' which doesn't work with other configurations
+                    #location=location or global_location
+                    )
+
+                corpuses.append(corpus)
+    if not corpuses:
+        log.error("Could not find any Discovery Engine corpus to import data to")
+        return None
+
+    log.info(f"Found Discovery Engine / Vertex AI Search {corpuses=}")
+
+    if message_data.startswith("gs://"):
+        log.info(f"DiscoveryEngineClient.import_files for {message_data}")
+        for corp in corpuses:
+            try:
+                response = corp.import_documents(
+                    gcs_uri=message_data
+                )
+                log.info(f"Imported file to corpus: {response} with metadata: {metadata}")
+            except Exception as err:
+                log.error(f"Error importing {message_data} - {corp=} - {str(err)}")
+                continue
+
+        metadata["source"] = message_data
+        return metadata
+    
+    else:
+        log.warning("Only gs:// data is supported for Discovery Engine")
+
+
+def check_discovery_engine_in_memory(vector_name):
+    memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")
+    for memory in memories:  # Iterate over the list
+        for key, value in memory.items():  # Now iterate over the dictionary
+            log.info(f"Found memory {key}")
+            vectorstore = value.get('vectorstore')
+            if vectorstore:
+                if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
+                    log.info(f"Found vectorstore {vectorstore}")
+                    return True
+    
+    return False
+
+def discovery_engine_chunker_check(message_data, metadata, vector_name):
+    # discovery engine handles its own chunking/embedding
+    memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")
+    total_memories = len(memories)
+    llama = None
+    if check_discovery_engine_in_memory(vector_name):
+        llama = do_discovery_engine(message_data, metadata, vector_name)
+        log.info(f"Processed discovery engine: {llama}")
+
+    # If discovery engine is the only entry, return
+    if llama and total_memories == 1:
+
+        return llama
+    
+    elif llama:
+        log.info("Discovery Engine found but not the only memory, continuing with other processes.")
+
+        return None

sunholo-0.69.2/sunholo/discovery_engine/create_new.py

@@ -0,0 +1,21 @@
+from .discovery_engine_client import DiscoveryEngineClient
+from ..utils.config import load_config_key
+
+def create_new_discovery_engine(vector_name):
+    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
+
+    chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
+
+    project_id = gcp_config.get('project_id')
+    #location = gcp_config.get('location')
+
+    de = DiscoveryEngineClient(
+                    data_store_id=vector_name, 
+                    project_id=project_id,
+                    # location needs to be 'eu' or 'us' which doesn't work with other configurations
+                    #location=location
+                    )
+
+    new_store = de.create_data_store(chunk_size=chunker_config.get("chunk_size"))
+
+    return new_store

sunholo-0.69.2/sunholo/discovery_engine/discovery_engine_client.py

@@ -0,0 +1,243 @@
+try:
+    from google.api_core.client_options import ClientOptions
+    from google.cloud import discoveryengine_v1alpha as discoveryengine
+except ImportError:
+    ClientOptions = None
+    discoveryengine = None
+
+from ..logging import log
+from typing import Optional
+
+class DiscoveryEngineClient:
+    """
+    Client for interacting with Google Cloud Discovery Engine.
+
+    Args:
+        project_id (str): Your Google Cloud project ID.
+        data_store_id (str): The ID of your Discovery Engine data store.
+        location (str, optional): The location of the data store (default is 'eu').
+
+    Example:
+        ```python
+        client = DiscoveryEngineClient(project_id='your-project-id', data_store_id='your-data-store-id')
+
+        # Create a collection
+        collection_name = client.create_collection("my_new_collection")
+
+        # Perform a search
+        search_response = client.get_chunks("your query", "your_collection_id")
+
+        ```
+
+        Parsing:
+        ```python
+        # Perform a search
+        search_response = client.get_chunks("your query", "your_collection_id")
+
+        # Iterate through the search results
+        for result in search_response.results:
+            # Get the document (which contains the chunks)
+            document = result.document
+
+            # Iterate through the chunks within the document
+            for chunk in document.chunks:
+                chunk_text = chunk.snippet  # Extract the text content of the chunk
+                chunk_document_name = chunk.document_name  # Get the name of the document the chunk belongs to
+                
+                # Do something with the chunk_text and chunk_document_name (e.g., print, store, etc.)
+                print(f"Chunk Text: {chunk_text}")
+                print(f"Document Name: {chunk_document_name}")
+        ```
+    """
+    def __init__(self, data_store_id, project_id, location="eu"):
+        if not discoveryengine:
+            raise ImportError("Google Cloud Discovery Engine not available, install via `pip install sunholo[gcp]`")
+
+        self.project_id = project_id
+        self.data_store_id = data_store_id
+        self.location = location
+        client_options = (
+            ClientOptions(api_endpoint=f"{location}-discoveryengine.googleapis.com")
+            if location != "global"
+            else None
+        )
+        self.client = discoveryengine.DataStoreServiceClient(client_options=client_options)
+        self.parent = self.client.branch_path(
+            project=project_id,
+            location=location,
+            data_store=data_store_id,
+            branch="default_branch",
+        )
+
+    def create_data_store(
+        self, chunk_size: int = 500
+    ) -> str:
+        """
+        Creates a new data store with default configuration.
+
+        Args:
+            chunk_size (int, optional): The size of the chunks to create for documents (default is 500).
+
+        Returns:
+            str: The name of the long-running operation for data store creation.
+        """
+
+        # https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1alpha.types.DocumentProcessingConfig
+        doc_config = discoveryengine.DocumentProcessingConfig(
+            chunking_config=discoveryengine.DocumentProcessingConfig.ChunkingConfig(
+                layout_based_chunking_config=discoveryengine.DocumentProcessingConfig.ChunkingConfig.LayoutBasedChunkingConfig(
+                    chunk_size=chunk_size,
+                    include_ancestor_headings=True
+                )
+            ),
+            default_parsing_config=discoveryengine.DocumentProcessingConfig.ParsingConfig(
+                layout_parsing_config=discoveryengine.DocumentProcessingConfig.ParsingConfig.LayoutParsingConfig()
+            )
+        )
+
+        # https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1.services.data_store_service.DataStoreServiceClient
+        # https://cloud.google.com/python/docs/reference/discoveryengine/0.11.4/google.cloud.discoveryengine_v1alpha.types.DataStore
+        data_store = discoveryengine.DataStore(
+            display_name=self.data_store_id,
+            # Options: GENERIC, MEDIA, HEALTHCARE_FHIR
+            industry_vertical=discoveryengine.IndustryVertical.GENERIC,
+            # Options: SOLUTION_TYPE_RECOMMENDATION, SOLUTION_TYPE_SEARCH, SOLUTION_TYPE_CHAT, SOLUTION_TYPE_GENERATIVE_CHAT
+            solution_types=[discoveryengine.SolutionType.SOLUTION_TYPE_SEARCH],
+            # Options: NO_CONTENT, CONTENT_REQUIRED, PUBLIC_WEBSITE
+            content_config=discoveryengine.DataStore.ContentConfig.CONTENT_REQUIRED,
+            # https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1.types.DocumentProcessingConfig
+            document_processing_config=doc_config
+        )
+
+        # https://cloud.google.com/python/docs/reference/discoveryengine/0.11.4/google.cloud.discoveryengine_v1alpha.types.CreateDataStoreRequest
+        request = discoveryengine.CreateDataStoreRequest(
+            parent=self.parent,
+            data_store_id=self.data_store_id,
+            data_store=data_store,
+            # Optional: For Advanced Site Search Only
+            # create_advanced_site_search=True,
+        )
+
+        # Make the request
+        operation = self.client.create_data_store(request=request)
+
+        log.info(f"Waiting for operation to complete: {operation.operation.name}")
+        response = operation.result()
+
+        # Once the operation is complete,
+        # get information from operation metadata
+        metadata = discoveryengine.CreateDataStoreMetadata(operation.metadata)
+
+        # Handle the response
+        log.info(f"{response=} {metadata=}")
+
+        return operation.operation.name
+
+    def get_chunks(
+        self,
+        query: str,
+        num_previous_chunks: int = 3,
+        num_next_chunks: int = 3,
+        page_size: int = 10,
+        doc_or_chunks: str = "CHUNKS",  # or DOCUMENTS
+    ):
+        """Retrieves chunks or documents based on a query.
+
+        Args:
+            query (str): The search query.
+            collection_id (str): The ID of the collection to search.
+            num_previous_chunks (int, optional): Number of previous chunks to return for context (default is 3).
+            num_next_chunks (int, optional): Number of next chunks to return for context (default is 3).
+            page_size (int, optional): The maximum number of results to return per page (default is 10).
+
+        Returns:
+            discoveryengine.SearchResponse: The search response object containing the search results.
+
+        Example:
+            ```python
+            search_response = client.get_chunks('your query', 'your_collection_id')
+            for result in search_response.results:
+                for chunk in result.document.chunks:
+                    print(f"Chunk: {chunk.snippet}, document name: {chunk.document_name}")
+            ```
+        """
+        serving_config = self.client.get_default_serving_config(
+            name=self.client.serving_config_path(
+                project=self.project_id, 
+                location=self.location, 
+                data_store=self.data_store_id, 
+                serving_config="default_serving_config")
+                ).name
+        
+        search_request = discoveryengine.SearchRequest(
+            serving_config=serving_config,
+            query=query,
+            page_size=page_size, 
+            content_search_spec=discoveryengine.SearchRequest.ContentSearchSpec(
+                #snippet_spec=discoveryengine.SearchRequest.ContentSearchSpec.SnippetSpec(
+                #    return_snippet=True
+                #),
+                search_result_mode=doc_or_chunks,  # CHUNKS or DOCUMENTS
+                chunk_spec=discoveryengine.SearchRequest.ContentSearchSpec.ChunkSpec(
+                    num_previous_chunks=num_previous_chunks,
+                    num_next_chunks=num_next_chunks,
+                ),
+            ),
+        )
+
+        search_response = self.client.search(search_request)
+
+        return search_response
+
+    def import_documents(self,
+        gcs_uri: Optional[str] = None,
+        data_schema="content",
+        bigquery_dataset: Optional[str] = None,
+        bigquery_table: Optional[str] = None,
+        bigquery_project_id: Optional[str] = None,
+    ) -> str:
+        """
+        Args:
+        - gcs_uri: Required. List of Cloud Storage URIs to input files. Each URI can be up to 2000 characters long. URIs can match the full object path (for example, gs://bucket/directory/object.json) or a pattern matching one or more files, such as gs://bucket/directory/*.json. A request can contain at most 100 files (or 100,000 files if data_schema is content). Each file can be up to 2 GB (or 100 MB if data_schema is content). 
+        - data_schema: Must be one of 'user_event', 'custom' or 'document' if using BigQuery. Default 'content' only for GCS. The schema to use when parsing the data from the source. Supported values for document imports: - document (default): One JSON Document per line. Each document must have a valid Document.id. - content: Unstructured data (e.g. PDF, HTML). Each file matched by input_uris becomes a document, with the ID set to the first 128 bits of SHA256(URI) encoded as a hex string. - custom: One custom data JSON per row in arbitrary format that conforms to the defined Schema of the data store. This can only be used by the GENERIC Data Store vertical. - csv: A CSV file with header conforming to the defined Schema of the data store. Each entry after the header is imported as a Document. This can only be used by the GENERIC Data Store vertical. Supported values for user event imports: - user_event (default): One JSON UserEvent per line. 
+        
+        """
+
+        if gcs_uri:
+            request = discoveryengine.ImportDocumentsRequest(
+                parent=self.parent,
+                # https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1alpha.types.GcsSource
+                gcs_source=discoveryengine.GcsSource(
+                    input_uris=[gcs_uri], data_schema=data_schema,
+                ),
+                # Options: `FULL`, `INCREMENTAL`
+                reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,
+            )
+        else:
+            request = discoveryengine.ImportDocumentsRequest(
+                parent=self.parent,
+                bigquery_source=discoveryengine.BigQuerySource(
+                    project_id=bigquery_project_id or self.project_id,
+                    dataset_id=bigquery_dataset,
+                    table_id=bigquery_table,
+                    data_schema=data_schema,
+                ),
+                # Options: `FULL`, `INCREMENTAL`
+                reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,
+            )
+
+        # Make the request
+        operation = self.client.import_documents(request=request)
+
+        log.info(f"Waiting for operation to complete: {operation.operation.name}")
+        response = operation.result()
+
+        # Once the operation is complete,
+        # get information from operation metadata
+        metadata = discoveryengine.ImportDocumentsMetadata(operation.metadata)
+
+        # Handle the response
+        log.info(f"{response=} {metadata=}")
+
+        return operation.operation.name
+

{sunholo-0.68.1 → sunholo-0.69.2}/sunholo/llamaindex/import_files.py

@@ -105,28 +105,8 @@ def do_llamaindex(message_data, metadata, vector_name):
         return metadata
     
     else:
-        raise NotImplementedError("Only gs:// and https://drive data is supported")
-        # write text to file and upload it
-        # TODO(developer): Update and un-comment below lines
-        # project_id = "PROJECT_ID"
-        # corpus_name = "projects/{project_id}/locations/us-central1/ragCorpora/{rag_corpus_id}"
-        # path = "path/to/local/file.txt"
-        # display_name = "file_display_name"
-        # description = "file description"
-
-        # Initialize Vertex AI API once per session
-        #path = 'path/to/local/file.txt'
-
-        # Write the message_data to a file
-        #with open(path, 'w') as file:
-        #    file.write(message_data)
-
-        #rag_file = rag.upload_file(
-        #    corpus_name=corpus_name,
-        #    path=path,
-        #    display_name=display_name,
-        #    description=description,
-        #)
+        log.warning("Only gs:// and https://drive data is supported for llamaindex")
+        
 
 def check_llamaindex_in_memory(vector_name):
     memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")

{sunholo-0.68.1 → sunholo-0.69.2}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.68.1
+Version: 0.69.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.68.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.68.1 → sunholo-0.69.2}/sunholo.egg-info/SOURCES.txt

@@ -71,6 +71,10 @@ sunholo/database/sql/sb/create_table.sql
 sunholo/database/sql/sb/delete_source_row.sql
 sunholo/database/sql/sb/return_sources.sql
 sunholo/database/sql/sb/setup.sql
+sunholo/discovery_engine/__init__.py
+sunholo/discovery_engine/chunker_handler.py
+sunholo/discovery_engine/create_new.py
+sunholo/discovery_engine/discovery_engine_client.py
 sunholo/embedder/__init__.py
 sunholo/embedder/embed_chunk.py
 sunholo/gcs/__init__.py