sunholo 0.119.8__tar.gz → 0.119.10__tar.gz

{sunholo-0.119.8/src/sunholo.egg-info → sunholo-0.119.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.119.8
+Version: 0.119.10
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.119.8 → sunholo-0.119.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sunholo"
-version = "0.119.8"
+version = "0.119.10"
 description = "Large Language Model DevOps - a package to help deploy LLMs to the Cloud."
 readme = "README.md"
 requires-python = ">=3.10"

{sunholo-0.119.8 → sunholo-0.119.10}/src/sunholo/chunker/process_chunker_data.py

@@ -38,7 +38,7 @@ def process_chunker_data(message_data, metadata, vector_name):
         return llamacheck
     
     # if only a discovery engine memory, return early as no other processing needed
-    discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config)
+    discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config, process=False)
     if discovery_check:
         return discovery_check
 

{sunholo-0.119.8 → sunholo-0.119.10}/src/sunholo/discovery_engine/chunker_handler.py

@@ -125,7 +125,11 @@ def check_write_memories(config:ConfigManager):
     
     return write_mem
 
-def discovery_engine_chunker_check(message_data, metadata, vector_name:str=None, config:ConfigManager=None):
+def discovery_engine_chunker_check(message_data, 
+                                   metadata, 
+                                   vector_name:str=None, 
+                                   config:ConfigManager=None,
+                                   process:bool=True):
 
     if config is None:
         if vector_name is None:
@@ -139,7 +143,7 @@ def discovery_engine_chunker_check(message_data, metadata, vector_name:str=None,
     
     total_memories = len(check_write_memories(config))
     llama = None
-    if check_discovery_engine_in_memory(config):
+    if check_discovery_engine_in_memory(config) and process:
         llama = do_discovery_engine(message_data, metadata, config=config)
         log.info(f"Processed discovery engine: {llama}")
 

{sunholo-0.119.8 → sunholo-0.119.10}/src/sunholo/discovery_engine/cli.py

@@ -223,12 +223,11 @@ def setup_discovery_engine_subparser(subparsers):
     # Search subcommand
     search_parser = discovery_engine_subparsers.add_parser('search', help='Search a Discovery Engine datastore')
     search_parser.add_argument('--query', required=True, help='The search query')
-    search_parser.add_argument('--num-previous-chunks', type=int, default=3, help='Number of previous chunks to return for context')
-    search_parser.add_argument('--num-next-chunks', type=int, default=3, help='Number of next chunks to return for context')
+    search_parser.add_argument('--data-store-id', required=True, help='Data store ID to search')
     search_parser.add_argument('--page-size', type=int, default=10, help='The maximum number of results to return per page')
     search_parser.add_argument('--parse-chunks-to-string', action='store_true', help='Combine chunks into a single string')
     search_parser.add_argument('--serving-config', default='default_serving_config', help='The serving configuration to use')
-    search_parser.add_argument('--data-store-ids', nargs='+', help='List of data store IDs to search (optional)')
+
     search_parser.set_defaults(func=discovery_engine_command)
 
     # Search by ID and/or Date subcommand

{sunholo-0.119.8 → sunholo-0.119.10}/src/sunholo/discovery_engine/get_ai_search_chunks.py

@@ -5,12 +5,13 @@ from .discovery_engine_client import DiscoveryEngineClient
 from ..components import load_memories
 import traceback
 
-def get_all_chunks(question:str, config:ConfigManager):
+def get_all_chunks(question:str, config:ConfigManager, filter_str=None):
     """
     Look through a config memory key and find all Vertex AI Search retrievers, call them and return a joined string of chunks
 
         args: question - question to search similarity for
         config: A ConfigManager object
+        filter_str: A filter that will restrict ai search via its metadata. See https://cloud.google.com/generative-ai-app-builder/docs/filter-search-metadata
 
         returns: a big string of chunks
     """
@@ -39,7 +40,7 @@ def get_all_chunks(question:str, config:ConfigManager):
                 project_id = gcp_config.get('project_id')
                 serving_config = value.get('serving_config')
 
-                chunk = get_chunks(question, vector_name, num_chunks, project_id=project_id, serving_config=serving_config)
+                chunk = get_chunks(question, vector_name, num_chunks, filter_str=filter_str, project_id=project_id, serving_config=serving_config)
                 if chunk:
                     chunks.append(chunk)
     if chunks:
@@ -48,23 +49,30 @@ def get_all_chunks(question:str, config:ConfigManager):
     log.warning(f"No chunks found for {vector_name}")
     return None
 
-def get_chunks(question, vector_name, num_chunks, project_id=None, serving_config=None):
+def get_chunks(question, vector_name, num_chunks, filter_str=None, project_id=None, serving_config=None):
     if serving_config is None:
         serving_config = "default_serving_config"
     de = DiscoveryEngineClient(vector_name, project_id=project_id or get_gcp_project(include_config=True))
     try:
-        return de.get_chunks(question, num_previous_chunks=num_chunks, num_next_chunks=num_chunks, serving_config=serving_config)
+        if filter_str:
+            return de.search_with_filters(query=question, 
+                                         filter_str=filter_str,
+                                         num_previous_chunks=num_chunks, 
+                                         num_next_chunks=num_chunks)
+        else:
+            return de.get_chunks(question, num_previous_chunks=num_chunks, num_next_chunks=num_chunks, serving_config=serving_config)
     except Exception as err:
-        log.error(f"No discovery engine chunks found: {str(err)}")
+        log.error(f"No discovery engine chunks found: {str(err)} {traceback.format_exc()}")
     
 
 
-async def async_get_all_chunks(question:str, config:ConfigManager):
+async def async_get_all_chunks(question:str, config:ConfigManager, filter_str=None):
     """
     Look through a config memory key and find all Vertex AI Search retrievers, call them and return a joined string of chunks
 
         args: question - question to search similarity for
         config: A ConfigManager object
+        filter_str: A filter that will restrict ai search via its metadata. See https://cloud.google.com/generative-ai-app-builder/docs/filter-search-metadata
 
         returns: a big string of chunks
     """
@@ -87,10 +95,16 @@ async def async_get_all_chunks(question:str, config:ConfigManager):
                         continue
                     else:
                         vector_name = new_vector_name
-                
+
+                project_id = value.get('project_id') or get_gcp_project(include_config=True)
                 num_chunks = value.get('num_chunks') or 3
 
-                chunk = await async_get_chunks(question, vector_name, num_chunks)
+                chunk = await async_get_chunks(question, 
+                                               vector_name=vector_name, 
+                                               num_chunks=num_chunks, 
+                                               filter_str=filter_str, 
+                                               project_id=project_id)
+
                 if chunk:
                     chunks.append(chunk)
     if chunks:
@@ -99,9 +113,12 @@ async def async_get_all_chunks(question:str, config:ConfigManager):
     log.warning(f"No chunks found for {vector_name}")
     return None
 
-async def async_get_chunks(question, vector_name, num_chunks):
-    de = DiscoveryEngineClient(vector_name, project_id=get_gcp_project(include_config=True))
+async def async_get_chunks(question, vector_name, num_chunks, filter_str, project_id=None):
+    de = DiscoveryEngineClient(vector_name, project_id=project_id)
     try:
-        return await de.async_get_chunks(question, num_previous_chunks=num_chunks, num_next_chunks=num_chunks)
+        return await de.search_with_filters(query=question, 
+                                            filter_str=filter_str,
+                                            num_previous_chunks=num_chunks, 
+                                            num_next_chunks=num_chunks)
     except Exception as err:
         log.error(f"No discovery engine chunks found: {str(err)} {traceback.format_exc()}")

{sunholo-0.119.8 → sunholo-0.119.10/src/sunholo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.119.8
+Version: 0.119.10
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0