sunholo 0.125.1__py3-none-any.whl → 0.126.0__py3-none-any.whl

sunholo/chunker/process_chunker_data.py

@@ -35,13 +35,16 @@ def process_chunker_data(message_data, metadata, vector_name):
     # checks if only a llamaindex chunking/embedder, return early as no other processing needed
     llamacheck = llamaindex_chunker_check(message_data, metadata, vector_name)
     if llamacheck:
+
         return llamacheck
     
     # if only a discovery engine memory, return early as no other processing needed
     discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config, process=False)
     if discovery_check:
+
         return discovery_check
 
+    log.info(f"Discovery engine and llamaindex checks passed - continuing to other memory types for {metadata}")
     chunks = []
 
     if message_data.startswith("gs://"):

sunholo/database/alloydb_client.py

@@ -610,19 +610,20 @@ class AlloyDBClient:
                 if items and isinstance(items[0], dict):
                     # If the first item is a dictionary, we need to create columns for all its keys
                     sample_item = items[0]
-                    # Flatten the dictionary with the key as prefix
-                    flattened_item = self._flatten_dict_for_schema(sample_item, key, "_")
                     
-                    # Add columns for all keys in the flattened dictionary
-                    for item_key, item_value in flattened_item.items():
-                        item_column = self._get_column_definition(item_key, item_value)
-                        columns.append(item_column)
+                    # Create columns for all keys in the sample item
+                    for item_key, item_value in sample_item.items():
+                        column_key = f"{key}_{item_key}"
+                        column_type = self._get_sql_type(item_value)
+                        columns.append(f'"{column_key}" {column_type}')
                 else:
                     # If items are simple values, just add a column for the list key itself
-                    columns.append(self._get_column_definition(key, items[0] if items else None))
+                    column_type = self._get_sql_type(items[0] if items else None)
+                    columns.append(f'"{key}" {column_type}')
             else:
                 # Regular handling for non-list fields
-                columns.append(self._get_column_definition(key, value))
+                column_type = self._get_sql_type(value)
+                columns.append(f'"{key}" {column_type}')
         
         # Add metadata columns
         columns.extend([
@@ -656,41 +657,40 @@ class AlloyDBClient:
             for user in users:
                 grant_sql = f'GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE "{table_name}" TO "{user}";'
                 if self.engine_type == "pg8000":
-                    self.execute_sql(grant_sql)
+                    self._execute_sql_pg8000(grant_sql)
                 else:
                     await self._execute_sql_async_langchain(grant_sql)
         
         return result
 
-    def _get_column_definition(self, key, value):
+    def _get_sql_type(self, value):
         """
-        Helper method to get SQL column definition from a key and value.
+        Helper method to determine SQL type from a Python value.
         
         Args:
-            key (str): The column name
             value: The value to determine the column type
             
         Returns:
-            str: SQL column definition
+            str: SQL type
         """
         if value is None:
             # For unknown types (None), default to TEXT
-            return f'"{key}" TEXT'
+            return "TEXT"
         elif isinstance(value, dict):
             # For nested objects, store as JSONB
-            return f'"{key}" JSONB'
+            return "JSONB"
         elif isinstance(value, list):
             # For arrays, store as JSONB
-            return f'"{key}" JSONB'
+            return "JSONB"
         elif isinstance(value, int):
-            return f'"{key}" INTEGER'
+            return "INTEGER"
         elif isinstance(value, float):
-            return f'"{key}" NUMERIC'
+            return "NUMERIC"
         elif isinstance(value, bool):
-            return f'"{key}" BOOLEAN'
+            return "BOOLEAN"
         else:
             # Default to TEXT for strings and other types
-            return f'"{key}" TEXT'
+            return "TEXT"
 
     def _flatten_dict_for_schema(self, nested_dict, parent_key='', separator='.'):
         """
@@ -789,6 +789,8 @@ class AlloyDBClient:
             primary_list_key = next(iter(expandable_lists))
             primary_list = expandable_lists[primary_list_key]
             
+            log.info(f"Expanding list '{primary_list_key}' with {len(primary_list)} items into separate rows")
+            
             # For each item in the primary list, create a new row
             for item_idx, item in enumerate(primary_list):
                 # Create a copy of the regular data
@@ -797,7 +799,12 @@ class AlloyDBClient:
                 # Add the current item from the primary list
                 if isinstance(item, dict):
                     # If it's a dictionary, flatten it with the primary key as prefix
-                    flattened_item = self._flatten_dict(item, primary_list_key, "_")
+                    flattened_item = {}
+                    for k, v in item.items():
+                        flattened_key = f"{primary_list_key}_{k}"
+                        flattened_item[flattened_key] = v
+                    
+                    # Update row data with flattened item
                     row_data.update(flattened_item)
                 else:
                     # If it's a simple value, just add it with the list key
@@ -809,8 +816,12 @@ class AlloyDBClient:
                 # Insert this row
                 result = await self._insert_single_row(table_name, row_data, metadata)
                 results.append(result)
+            
+            return results
         
-        return results
+        # If we somehow get here (shouldn't happen), fall back to single row insert
+        return await self._insert_single_row(table_name, regular_data, metadata)
+
 
     async def _insert_single_row(self, table_name: str, data: dict, metadata: dict = None):
         """

sunholo/discovery_engine/chunker_handler.py

@@ -100,9 +100,10 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
         log.warning("Only gs:// data is supported for Discovery Engine")
 
 
-def check_discovery_engine_in_memory(config:ConfigManager):
+def check_discovery_engine_in_memory(config:ConfigManager) -> int:
     memories = config.vacConfig("memory")
 
+    discovery_engine_memories = 0
     for memory in memories:  # Iterate over the list
         for key, value in memory.items():  # Now iterate over the dictionary
             log.info(f"Found memory {key}")
@@ -110,16 +111,16 @@ def check_discovery_engine_in_memory(config:ConfigManager):
             if vectorstore:
                 if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                     log.info(f"Found vectorstore {vectorstore}")
-                    return True
+                    discovery_engine_memories += 1
     
-    return False
+    return discovery_engine_memories
 
 def check_write_memories(config:ConfigManager):
     write_mem = []
     memories = config.vacConfig("memory")
     for memory in memories:
         for key, value in memory.items():
-            if value.get('read_only'):
+            if value and value.get('read_only'):
                 continue
             write_mem.append(memory)
     
@@ -142,21 +143,25 @@ def discovery_engine_chunker_check(message_data,
         return None
     
     total_memories = len(check_write_memories(config))
-    llama = None
-    if check_discovery_engine_in_memory(config) and process:
-        llama = do_discovery_engine(message_data, metadata, config=config)
-        log.info(f"Processed discovery engine: {llama}")
+    total_discovery_memories = check_discovery_engine_in_memory(config)
 
-    # If discovery engine is the only entry, return
-    if llama and total_memories == 1:
-
-        return llama
-    
-    # If not processing and only memory, do not process further
-    if not process and total_memories == 1:
+    if not process and total_memories == total_discovery_memories:
+        log.info(f"Do not process discovery engine, and only memory found is discovery engine for {metadata} - stopping")
+        
         return metadata
+
+    if total_discovery_memories > 0:
+        log.info(f"Process discovery engine for {metadata}")
+        disc_meta = do_discovery_engine(message_data, metadata, config=config)
+        log.info(f"Processed discovery engine: {disc_meta}")
+
+    # If discovery engine is the only entry, return
+    if total_discovery_memories == total_memories:
+        log.info(f"Process discovery engine was only type found in {metadata} - stopping")
+        
+        return disc_meta
     
-    elif llama:
-        log.info("Discovery Engine found but not the only memory, continuing with other processes.")
+    elif disc_meta:
+        log.info("Discovery Engine found but not the only memory, continuing with other processes - returning None")
 
         return None

{sunholo-0.125.1.dist-info → sunholo-0.126.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.125.1
+Version: 0.126.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.125.1.dist-info → sunholo-0.126.0.dist-info}/RECORD

@@ -38,7 +38,7 @@ sunholo/chunker/images.py,sha256=id2PBu6XyGEOtgafq2v0c9_O6kxaC_pYFMnbsIitkSg,186
 sunholo/chunker/loaders.py,sha256=KEFPHBr32DPJnRUNjOecLiZKsFD-Gk3BFIKyXSVHgbY,11143
 sunholo/chunker/message_data.py,sha256=bpb8QWQttqazm5lr7fTFJ5JDwf-P0SQ5cOIf6NikNyI,10836
 sunholo/chunker/pdfs.py,sha256=xwbuMJrbypcyPXfZ8tiUidWeMr80C2NhfTC1mwa8SHY,2477
-sunholo/chunker/process_chunker_data.py,sha256=xjOAf1FvHDwQaBm7kgDzLQUnwm6AW8qf4fTrwDnwmtc,3613
+sunholo/chunker/process_chunker_data.py,sha256=soFl5blyHBryiAkeoalBsPnkU0wDg6bu8rZZrb3LGFc,3728
 sunholo/chunker/publish.py,sha256=IDud-NhRcEZFv9GkyWJFRKwfptIU052kSPKEx8AYW68,2943
 sunholo/chunker/pubsub.py,sha256=48bhuAcszN7LGe3-ksPSLHHhq0uKxiXOrizck5qpcP0,1012
 sunholo/chunker/splitter.py,sha256=RfekLPkjhCcNd1PFXIj_FxusJMJ8_3cyWl7bsYvtQ0g,7068
@@ -60,7 +60,7 @@ sunholo/components/retriever.py,sha256=Wmchv3huAM4w7DIS-a5Lp9Hi7M8pE6vZdxgseiT9S
 sunholo/components/vectorstore.py,sha256=k7GS1Y5c6ZGXSDAJvyCes6dTjhDAi0fjGbVLqpyfzBc,5918
 sunholo/database/__init__.py,sha256=bpB5Nk21kwqYj-qdVnvNgXjLsbflnH4g-San7OHMqR4,283
 sunholo/database/alloydb.py,sha256=x1zUMB-EVWbE2Zvp4nAs2Z-tB_kOZmS45H2lwVHdYnk,11678
-sunholo/database/alloydb_client.py,sha256=IszkoMgL_OBBg6YCWBIq2v5blIX9TApdlxPgNREDt1o,34429
+sunholo/database/alloydb_client.py,sha256=Ih_9nd_5fXzki02fNhssn5Grg5-GcQrfmNjqv1wLK7A,34788
 sunholo/database/database.py,sha256=VqhZdkXUNdvWn8sUcUV3YNby1JDVf7IykPVXWBtxo9U,7361
 sunholo/database/lancedb.py,sha256=DyfZntiFKBlVPaFooNN1Z6Pl-LAs4nxWKKuq8GBqN58,715
 sunholo/database/static_dbs.py,sha256=8cvcMwUK6c32AS2e_WguKXWMkFf5iN3g9WHzsh0C07Q,442
@@ -72,7 +72,7 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=hLgqRDJ22Aov9o2QjAEfsVgnL3kMdM-g5p8RJ9OyKdQ,130
-sunholo/discovery_engine/chunker_handler.py,sha256=2775W5wHzdkYXqkRMlwh8MRbas20wcMnnnNngo0ljms,6160
+sunholo/discovery_engine/chunker_handler.py,sha256=qznc5lUkYv-Au18Nl4dJSY0qF0RpNqJMigzWdVrmOdM,6573
 sunholo/discovery_engine/cli.py,sha256=KGVle5rkLL49oF9TQhrGI--8017IvvLOEoYur545Qb0,12790
 sunholo/discovery_engine/create_new.py,sha256=WUi4_xh_dFaGX3xA9jkNKZhaR6LCELjMPeRb0hyj4FU,1226
 sunholo/discovery_engine/discovery_engine_client.py,sha256=NjIcP10I2-8yj6QZKrxGzNbh3SqQ5vGYsq9OwxCpWas,36935
@@ -168,9 +168,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.125.1.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.125.1.dist-info/METADATA,sha256=KFpMxsW_gucPhBoSwzZU8UHg9tnFEMe3Lv254hgsKqE,10001
-sunholo-0.125.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-sunholo-0.125.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.125.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.125.1.dist-info/RECORD,,
+sunholo-0.126.0.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.126.0.dist-info/METADATA,sha256=8qEuySuUqgo6zCfDjUB0aThQRZci-Mx5vEQPpfS2-wE,10001
+sunholo-0.126.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+sunholo-0.126.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.126.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.126.0.dist-info/RECORD,,