sunholo 0.58.0__py3-none-any.whl → 0.58.3__py3-none-any.whl

sunholo/components/retriever.py

@@ -27,7 +27,7 @@ from langchain.retrievers import ContextualCompressionRetriever
 
 
 def load_memories(vector_name):
-    memories = load_config_key("memory", vector_name, filename="config/llm_config.yaml")
+    memories = load_config_key("memory", vector_name, type="vacConfig")
     log.info(f"Found memory settings for {vector_name}: {memories}")
     if len(memories) == 0:
         log.info(f"No memory settings found for {vector_name}")
@@ -49,7 +49,8 @@ def pick_retriever(vector_name, embeddings=None):
                 if embeddings is None:
                     embeddings = get_embeddings(vector_name)
                 vectorstore = pick_vectorstore(vectorstore, vector_name=vector_name, embeddings=embeddings)
-                vs_retriever = vectorstore.as_retriever(search_kwargs=dict(k=3))
+                k_override = value.get('k', 3)
+                vs_retriever = vectorstore.as_retriever(search_kwargs=dict(k=k_override))
                 retriever_list.append(vs_retriever)
             
             if value.get('provider') == "GoogleCloudEnterpriseSearchRetriever":
@@ -68,6 +69,10 @@ def pick_retriever(vector_name, embeddings=None):
     if len(retriever_list) == 0:
         log.info(f"No retrievers were created for {memories}")
         return None
+    
+    k_override = load_config_key("memory_k", vector_name, type="vacConfig")
+    if not k_override:
+        k_override = 3
         
     lotr = MergerRetriever(retrievers=retriever_list)
 
@@ -76,6 +81,6 @@ def pick_retriever(vector_name, embeddings=None):
     pipeline = DocumentCompressorPipeline(transformers=[filter])
     retriever = ContextualCompressionRetriever(
         base_compressor=pipeline, base_retriever=lotr, 
-        k=3)
+        k=k_override)
 
     return retriever

sunholo/embedder/embed_chunk.py

@@ -115,6 +115,15 @@ def embed_pubsub_chunk(data: dict):
                 embed_llm = value.get('llm')
                 if embed_llm:
                     embeddings = pick_embedding(embed_llm)
+                # check if read only
+                read_only = value.get('readonly')
+                if read_only:
+                    continue
+                # read from a different vector_name
+                vector_name_other = value.get('vector_name')
+                if vector_name_other:
+                    log.warning(f"Using different vector_name for vectorstore: {vector_name_other} overriding {vector_name}")
+                    vector_name = vector_name_other
                 vectorstore_obj = pick_vectorstore(vectorstore, vector_name=vector_name, embeddings=embeddings)
                 vs_retriever = vectorstore_obj.as_retriever(search_kwargs=dict(k=3))
                 vectorstore_list.append(vs_retriever)

sunholo/gcs/download_url.py

@@ -1,9 +1,11 @@
 import os
 from urllib.parse import quote
 from datetime import datetime, timedelta
+
+import google.auth # needs to be in minimal to check gcp
+
 try:
-    from google.cloud import storage 
-    import google.auth
+    from google.cloud import storage     
     from google.auth.transport import requests
     from google.auth.exceptions import RefreshError
 except ImportError:

sunholo/llamaindex/import_files.py

@@ -159,12 +159,8 @@ def do_llamaindex(message_data, metadata, vector_name):
         #    description=description,
         #)
 
-
-def llamaindex_chunker_check(message_data, metadata, vector_name):
-    # llamaindex handles its own chunking/embedding
+def check_llamaindex_in_memory(vector_name):
     memories = load_config_key("memory", vector_name=vector_name, type="vacConfig")
-    total_memories = len(memories)
-    llama = None
     for memory in memories:  # Iterate over the list
         for key, value in memory.items():  # Now iterate over the dictionary
             log.info(f"Found memory {key}")
@@ -172,10 +168,19 @@ def llamaindex_chunker_check(message_data, metadata, vector_name):
             if vectorstore:
                 log.info(f"Found vectorstore {vectorstore}")
                 if vectorstore == "llamaindex":
-                    # https://cloud.google.com/vertex-ai/generative-ai/docs/llamaindex-on-vertexai
-                    log.info(f"llamaindex on vertex indexing for {vector_name}")
-                    llama = do_llamaindex(message_data, metadata, vector_name)
-                    log.info(f"Processed llamaindex: {llama}")
+
+                    return True
+    
+    return False
+
+def llamaindex_chunker_check(message_data, metadata, vector_name):
+    # llamaindex handles its own chunking/embedding
+    memories = load_config_key("memory", vector_name=vector_name, type="vacConfig")
+    total_memories = len(memories)
+    llama = None
+    if check_llamaindex_in_memory(vector_name):
+        llama = do_llamaindex(message_data, metadata, vector_name)
+        log.info(f"Processed llamaindex: {llama}")
 
     # If llamaindex is the only entry, return
     if llama and total_memories == 1:

{sunholo-0.58.0.dist-info → sunholo-0.58.3.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.58.0
+Version: 0.58.3
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.58.0.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.58.3.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -21,6 +21,7 @@ License-File: LICENSE.txt
 Requires-Dist: langchain
 Requires-Dist: langchain-experimental
 Requires-Dist: langchain-community
+Requires-Dist: google-auth
 Provides-Extra: all
 Requires-Dist: asyncpg ; extra == 'all'
 Requires-Dist: flask ; extra == 'all'
@@ -68,7 +69,6 @@ Requires-Dist: pgvector ; extra == 'database'
 Requires-Dist: psycopg2-binary ; extra == 'database'
 Requires-Dist: lancedb ; extra == 'database'
 Provides-Extra: gcp
-Requires-Dist: google-auth ; extra == 'gcp'
 Requires-Dist: google-auth-httplib2 ; extra == 'gcp'
 Requires-Dist: google-auth-oauthlib ; extra == 'gcp'
 Requires-Dist: google-cloud-aiplatform ; extra == 'gcp'

{sunholo-0.58.0.dist-info → sunholo-0.58.3.dist-info}/RECORD

@@ -38,7 +38,7 @@ sunholo/cli/deploy.py,sha256=zxdwUsRTRMC8U5vyRv0JiKBLFn84Ug_Tc88-_h9hJSs,1609
 sunholo/components/__init__.py,sha256=RJGNEihwvRIiDScKis04RHJv4yZGI1UpXlOmuCptNZI,208
 sunholo/components/llm.py,sha256=T4we3tGmqUj4tPwxQr9M6AXv_BALqZV_dRSvINan-oU,10374
 sunholo/components/prompt.py,sha256=eZSghXkIlRzXiSrzgkG7e5ytUYq6R6LV-qjHU8jStig,6353
-sunholo/components/retriever.py,sha256=TiM-axCeaZ6CZ8rGKGx-io16JKDe8z0pnMccBi1yqHw,3509
+sunholo/components/retriever.py,sha256=QA_l7HXwd4g6IFuT2A5mICzUcV80K0YvDbCtRxefq8o,3684
 sunholo/components/vectorstore.py,sha256=RB_Dgc9234G_TE3w3abCfBw1pqm2br2RrLP0UqshkvA,5172
 sunholo/database/__init__.py,sha256=Zz0Shcq-CtStf9rJGIYB_Ybzb8rY_Q9mfSj-nviM490,241
 sunholo/database/alloydb.py,sha256=18Q4AG_W-Sz8udIj3gWMkGrMWmiEelqgOwJ7VKHElV0,14877
@@ -53,17 +53,17 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
-sunholo/embedder/embed_chunk.py,sha256=8kZI2XB1uB6joCqzVFjelqV3cXuqc6iog_SuFKR99x8,5376
+sunholo/embedder/embed_chunk.py,sha256=XV1kdDUWw2QO-am5_Yl7GrYP9V_4i1XRNNFPhqUSnZQ,5851
 sunholo/gcs/__init__.py,sha256=DtVw_AZwQn-IguR5BJuIi2XJeF_FQXizhJikzRNrXiE,50
 sunholo/gcs/add_file.py,sha256=JmJIuz5Z1h7-eJ6s2eE3wc8Y4IAv3Jridq1xfQbD9_E,4711
-sunholo/gcs/download_url.py,sha256=PAwYShV-sRd9sNvuJrEOvfF1V34ovVP0omWbuwDkRrA,4751
+sunholo/gcs/download_url.py,sha256=WnRF5H0IX9tyiuD4b_UfV2InsR8kDTi1VYfXpx5_Vc0,4791
 sunholo/gcs/metadata.py,sha256=C9sMPsHsq1ETetdQCqB3EBs3Kws8b8QHS9L7ei_v5aw,891
 sunholo/langfuse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/langfuse/callback.py,sha256=G9xcZHpLvyzolU57ycItLaooMCtRuM37QJSWjiwQEd0,1776
 sunholo/langfuse/prompts.py,sha256=HO4Zy9usn5tKooBPCKksuw4Lff3c03Ny5wqn4ce_xZM,1217
 sunholo/llamaindex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/llamaindex/generate.py,sha256=l1Picr-hVwkmAUD7XmTCa63qY9ERliFHQXwyX3BqB2Q,686
-sunholo/llamaindex/import_files.py,sha256=6VE4TvZP3oLaskzD9VSswFtIKJJbnhWXKkAvss191Rk,7758
+sunholo/llamaindex/import_files.py,sha256=j71_rGIpZg1Zbsy-PnNIUgpNJznnsDiYxR6sulBlQ_8,7786
 sunholo/lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/lookup/model_lookup.yaml,sha256=O7o-jP53MLA06C8pI-ILwERShO-xf6z_258wtpZBv6A,739
 sunholo/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -89,9 +89,9 @@ sunholo/utils/gcp.py,sha256=B2G1YKjeD7X9dqO86Jrp2vPuFwZ223Xl5Tg09Ndw-oc,5760
 sunholo/utils/parsers.py,sha256=OrHmASqIbI45atVOhiGodgLvnfrzkvVzyHnSvAXD89I,3841
 sunholo/vertex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/vertex/init_vertex.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
-sunholo-0.58.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.58.0.dist-info/METADATA,sha256=SjGmg3X9BCN6olNH4NWI4uBnPexRuCNw5tMztrLCmhc,7894
-sunholo-0.58.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-sunholo-0.58.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.58.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.58.0.dist-info/RECORD,,
+sunholo-0.58.3.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.58.3.dist-info/METADATA,sha256=b6EyBKaGDUvY355FAT2ZbVZgnnrFxiowbkhiAcQHe3Q,7877
+sunholo-0.58.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+sunholo-0.58.3.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.58.3.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.58.3.dist-info/RECORD,,