sunholo 0.125.2__tar.gz → 0.126.1__tar.gz

{sunholo-0.125.2/src/sunholo.egg-info → sunholo-0.126.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.125.2
+Version: 0.126.1
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.125.2 → sunholo-0.126.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sunholo"
-version = "0.125.2"
+version = "0.126.1"
 description = "Large Language Model DevOps - a package to help deploy LLMs to the Cloud."
 readme = "README.md"
 requires-python = ">=3.10"

{sunholo-0.125.2 → sunholo-0.126.1}/src/sunholo/chunker/process_chunker_data.py

@@ -35,13 +35,16 @@ def process_chunker_data(message_data, metadata, vector_name):
     # checks if only a llamaindex chunking/embedder, return early as no other processing needed
     llamacheck = llamaindex_chunker_check(message_data, metadata, vector_name)
     if llamacheck:
-        return llamacheck
+        log.info(f"No further chunker required for {message_data}")
+        return None
     
     # if only a discovery engine memory, return early as no other processing needed
     discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config, process=False)
     if discovery_check:
-        return discovery_check
+        log.info(f"No further chunker required for {message_data}")
+        return None
 
+    log.info(f"Discovery engine and llamaindex checks passed - continuing to other memory types for {metadata}")
     chunks = []
 
     if message_data.startswith("gs://"):

{sunholo-0.125.2 → sunholo-0.126.1}/src/sunholo/discovery_engine/chunker_handler.py

@@ -36,20 +36,24 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
                 log.info(f"Found vectorstore {vectorstore}")
                 if value.get('read_only'):
                     continue
-                gcp_config = config.vacConfig("gcp_config")
-                if not gcp_config:
-                    project_id = get_gcp_project()
-                else:
-                    project_id = gcp_config.get("project_id")
+
+                project_id = value.get("project_id")
+                location = value.get("location", "eu")
+                if not project_id:
+                    gcp_config = config.vacConfig("gcp_config")
+                    if not gcp_config:
+                        project_id = get_gcp_project()
+                    else:
+                        project_id = gcp_config.get("project_id")
 
                 if not project_id:
                     raise ValueError("Couldn't retrieve project_id for vertex_ai_search")        
-                #location = gcp_config.get('location') 
+
                 corpus = DiscoveryEngineClient(
                     data_store_id=config.vector_name, 
-                    project_id=project_id
+                    project_id=project_id,
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
-                    #location=location or global_location
+                    location=location
                     )
 
                 corpuses.append(corpus)
@@ -100,9 +104,10 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
         log.warning("Only gs:// data is supported for Discovery Engine")
 
 
-def check_discovery_engine_in_memory(config:ConfigManager):
+def check_discovery_engine_in_memory(config:ConfigManager) -> int:
     memories = config.vacConfig("memory")
 
+    discovery_engine_memories = 0
     for memory in memories:  # Iterate over the list
         for key, value in memory.items():  # Now iterate over the dictionary
             log.info(f"Found memory {key}")
@@ -110,16 +115,16 @@ def check_discovery_engine_in_memory(config:ConfigManager):
             if vectorstore:
                 if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                     log.info(f"Found vectorstore {vectorstore}")
-                    return True
+                    discovery_engine_memories += 1
     
-    return False
+    return discovery_engine_memories
 
 def check_write_memories(config:ConfigManager):
     write_mem = []
     memories = config.vacConfig("memory")
     for memory in memories:
         for key, value in memory.items():
-            if value.get('read_only'):
+            if value and value.get('read_only'):
                 continue
             write_mem.append(memory)
     
@@ -142,21 +147,25 @@ def discovery_engine_chunker_check(message_data,
         return None
     
     total_memories = len(check_write_memories(config))
-    llama = None
-    if check_discovery_engine_in_memory(config) and process:
-        llama = do_discovery_engine(message_data, metadata, config=config)
-        log.info(f"Processed discovery engine: {llama}")
+    total_discovery_memories = check_discovery_engine_in_memory(config)
+
+    if not process and total_memories == total_discovery_memories:
+        log.info(f"Do not process discovery engine, and only memory found is discovery engine for {metadata} - stopping")
+        
+        return metadata
+
+    if total_discovery_memories > 0:
+        log.info(f"Process discovery engine for {metadata}")
+        disc_meta = do_discovery_engine(message_data, metadata, config=config)
+        log.info(f"Processed discovery engine: {disc_meta}")
 
     # If discovery engine is the only entry, return
-    if llama and total_memories == 1:
+    if total_discovery_memories == total_memories:
+        log.info(f"Process discovery engine was only type found in {metadata} - stopping")
 
-        return llama
-    
-    # If not processing and only memory, do not process further
-    if not process and total_memories == 1:
-        return metadata
+        return disc_meta
     
-    elif llama:
-        log.info("Discovery Engine found but not the only memory, continuing with other processes.")
+    elif disc_meta:
+        log.info("Discovery Engine found but not the only memory, continuing with other processes - returning None")
 
         return None

{sunholo-0.125.2 → sunholo-0.126.1/src/sunholo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.125.2
+Version: 0.126.1
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0