sunholo 0.125.2__py3-none-any.whl → 0.126.1__py3-none-any.whl

sunholo/chunker/process_chunker_data.py

@@ -35,13 +35,16 @@ def process_chunker_data(message_data, metadata, vector_name):
     # checks if only a llamaindex chunking/embedder, return early as no other processing needed
     llamacheck = llamaindex_chunker_check(message_data, metadata, vector_name)
     if llamacheck:
-        return llamacheck
+        log.info(f"No further chunker required for {message_data}")
+        return None
     
     # if only a discovery engine memory, return early as no other processing needed
     discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config, process=False)
     if discovery_check:
-        return discovery_check
+        log.info(f"No further chunker required for {message_data}")
+        return None
 
+    log.info(f"Discovery engine and llamaindex checks passed - continuing to other memory types for {metadata}")
     chunks = []
 
     if message_data.startswith("gs://"):

sunholo/discovery_engine/chunker_handler.py

@@ -36,20 +36,24 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
                 log.info(f"Found vectorstore {vectorstore}")
                 if value.get('read_only'):
                     continue
-                gcp_config = config.vacConfig("gcp_config")
-                if not gcp_config:
-                    project_id = get_gcp_project()
-                else:
-                    project_id = gcp_config.get("project_id")
+
+                project_id = value.get("project_id")
+                location = value.get("location", "eu")
+                if not project_id:
+                    gcp_config = config.vacConfig("gcp_config")
+                    if not gcp_config:
+                        project_id = get_gcp_project()
+                    else:
+                        project_id = gcp_config.get("project_id")
 
                 if not project_id:
                     raise ValueError("Couldn't retrieve project_id for vertex_ai_search")        
-                #location = gcp_config.get('location') 
+
                 corpus = DiscoveryEngineClient(
                     data_store_id=config.vector_name, 
-                    project_id=project_id
+                    project_id=project_id,
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
-                    #location=location or global_location
+                    location=location
                     )
 
                 corpuses.append(corpus)
@@ -100,9 +104,10 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
         log.warning("Only gs:// data is supported for Discovery Engine")
 
 
-def check_discovery_engine_in_memory(config:ConfigManager):
+def check_discovery_engine_in_memory(config:ConfigManager) -> int:
     memories = config.vacConfig("memory")
 
+    discovery_engine_memories = 0
     for memory in memories:  # Iterate over the list
         for key, value in memory.items():  # Now iterate over the dictionary
             log.info(f"Found memory {key}")
@@ -110,16 +115,16 @@ def check_discovery_engine_in_memory(config:ConfigManager):
             if vectorstore:
                 if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                     log.info(f"Found vectorstore {vectorstore}")
-                    return True
+                    discovery_engine_memories += 1
     
-    return False
+    return discovery_engine_memories
 
 def check_write_memories(config:ConfigManager):
     write_mem = []
     memories = config.vacConfig("memory")
     for memory in memories:
         for key, value in memory.items():
-            if value.get('read_only'):
+            if value and value.get('read_only'):
                 continue
             write_mem.append(memory)
     
@@ -142,21 +147,25 @@ def discovery_engine_chunker_check(message_data,
         return None
     
     total_memories = len(check_write_memories(config))
-    llama = None
-    if check_discovery_engine_in_memory(config) and process:
-        llama = do_discovery_engine(message_data, metadata, config=config)
-        log.info(f"Processed discovery engine: {llama}")
+    total_discovery_memories = check_discovery_engine_in_memory(config)
+
+    if not process and total_memories == total_discovery_memories:
+        log.info(f"Do not process discovery engine, and only memory found is discovery engine for {metadata} - stopping")
+        
+        return metadata
+
+    if total_discovery_memories > 0:
+        log.info(f"Process discovery engine for {metadata}")
+        disc_meta = do_discovery_engine(message_data, metadata, config=config)
+        log.info(f"Processed discovery engine: {disc_meta}")
 
     # If discovery engine is the only entry, return
-    if llama and total_memories == 1:
+    if total_discovery_memories == total_memories:
+        log.info(f"Process discovery engine was only type found in {metadata} - stopping")
 
-        return llama
-    
-    # If not processing and only memory, do not process further
-    if not process and total_memories == 1:
-        return metadata
+        return disc_meta
     
-    elif llama:
-        log.info("Discovery Engine found but not the only memory, continuing with other processes.")
+    elif disc_meta:
+        log.info("Discovery Engine found but not the only memory, continuing with other processes - returning None")
 
         return None

{sunholo-0.125.2.dist-info → sunholo-0.126.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.125.2
+Version: 0.126.1
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.125.2.dist-info → sunholo-0.126.1.dist-info}/RECORD

@@ -38,7 +38,7 @@ sunholo/chunker/images.py,sha256=id2PBu6XyGEOtgafq2v0c9_O6kxaC_pYFMnbsIitkSg,186
 sunholo/chunker/loaders.py,sha256=KEFPHBr32DPJnRUNjOecLiZKsFD-Gk3BFIKyXSVHgbY,11143
 sunholo/chunker/message_data.py,sha256=bpb8QWQttqazm5lr7fTFJ5JDwf-P0SQ5cOIf6NikNyI,10836
 sunholo/chunker/pdfs.py,sha256=xwbuMJrbypcyPXfZ8tiUidWeMr80C2NhfTC1mwa8SHY,2477
-sunholo/chunker/process_chunker_data.py,sha256=xjOAf1FvHDwQaBm7kgDzLQUnwm6AW8qf4fTrwDnwmtc,3613
+sunholo/chunker/process_chunker_data.py,sha256=kdFkNYxt0HwVCCNzhFR37prfMm-2bU8Zxnl0XcRbyd0,3845
 sunholo/chunker/publish.py,sha256=IDud-NhRcEZFv9GkyWJFRKwfptIU052kSPKEx8AYW68,2943
 sunholo/chunker/pubsub.py,sha256=48bhuAcszN7LGe3-ksPSLHHhq0uKxiXOrizck5qpcP0,1012
 sunholo/chunker/splitter.py,sha256=RfekLPkjhCcNd1PFXIj_FxusJMJ8_3cyWl7bsYvtQ0g,7068
@@ -72,7 +72,7 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=hLgqRDJ22Aov9o2QjAEfsVgnL3kMdM-g5p8RJ9OyKdQ,130
-sunholo/discovery_engine/chunker_handler.py,sha256=2775W5wHzdkYXqkRMlwh8MRbas20wcMnnnNngo0ljms,6160
+sunholo/discovery_engine/chunker_handler.py,sha256=ZplseIFT1sW48DtKtujB9e8cOD2gWwb-yi-K0CvLkEU,6655
 sunholo/discovery_engine/cli.py,sha256=KGVle5rkLL49oF9TQhrGI--8017IvvLOEoYur545Qb0,12790
 sunholo/discovery_engine/create_new.py,sha256=WUi4_xh_dFaGX3xA9jkNKZhaR6LCELjMPeRb0hyj4FU,1226
 sunholo/discovery_engine/discovery_engine_client.py,sha256=NjIcP10I2-8yj6QZKrxGzNbh3SqQ5vGYsq9OwxCpWas,36935
@@ -168,9 +168,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.125.2.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.125.2.dist-info/METADATA,sha256=McMROrwPXX3t0IDFLSBXYIj84-oGpkcQCDGWsVMpB8M,10001
-sunholo-0.125.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-sunholo-0.125.2.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.125.2.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.125.2.dist-info/RECORD,,
+sunholo-0.126.1.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.126.1.dist-info/METADATA,sha256=HULIOhd32X0GAYPLPHV92eWFYJEe-vpwLzDNsy-LIDY,10001
+sunholo-0.126.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+sunholo-0.126.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.126.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.126.1.dist-info/RECORD,,