sunholo 0.78.5__py3-none-any.whl → 0.79.0__py3-none-any.whl

sunholo/chunker/doc_handling.py

@@ -175,6 +175,10 @@ Be careful not to add any speculation or any details that are not covered in the
                 bucket_name = os.getenv("DOC_BUCKET")
                 if not bucket_name:
                     raise ValueError("No DOC_BUCKET configured for summary")
+                
+                if bucket_name.startswith("gs://"):
+                    bucket_name = bucket_name[len("gs://"):]
+                    
                 with tempfile.NamedTemporaryFile(mode='w+', delete=False) as temp_file:
                     temp_file.write(summary)
                     temp_file.flush()

sunholo/database/alloydb.py

@@ -232,7 +232,7 @@ async def load_alloydb_sql_async(sql, vector_name):
     return documents
 
 def and_or_ilike(sources, search_type="OR", operator="ILIKE"):
-    unique_sources = set(sources)
+    unique_sources = set(sources.split())
     # Choose the delimiter based on the search_type argument
     delimiter = ' AND ' if search_type.upper() == "AND" else ' OR '
 
@@ -240,14 +240,14 @@ def and_or_ilike(sources, search_type="OR", operator="ILIKE"):
     conditions = delimiter.join(f"TRIM(source) {operator} '%{source}%'" for source in unique_sources)
     if not conditions:
         log.warning("Alloydb doc query found no like_patterns")
-        return []
+        return ""
     
     return conditions
 
 def _get_sources_from_docstore(sources, vector_name, search_type="OR"):
     if not sources:
         log.warning("No sources found for alloydb fetch")
-        return []
+        return ""
 
     table_name = f"{vector_name}_docstore"
 
@@ -263,10 +263,37 @@ def _get_sources_from_docstore(sources, vector_name, search_type="OR"):
 
     return query
 
+def _list_sources_from_docstore(sources, vector_name, search_type="OR"):
+    table_name = f"{vector_name}_docstore"
+    
+
+    if sources:
+        conditions = and_or_ilike(sources, search_type=search_type)
+        query = f"""
+            SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+            FROM {table_name}
+            WHERE {conditions}
+            ORDER BY langchain_metadata->>'objectId' ASC
+            LIMIT 500;
+        """
+    else:
+        query = f"""
+            SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+            FROM {table_name}
+            ORDER BY langchain_metadata->>'objectId' ASC
+            LIMIT 500;
+        """
+
+    return query
 
-async def get_sources_from_docstore_async(sources, vector_name, search_type="OR"):
+
+async def get_sources_from_docstore_async(sources, vector_name, search_type="OR", just_source_name=False):
     
-    query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    if just_source_name:
+        query = _list_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    else:
+        query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+
     if not query:
         return []
     
@@ -274,9 +301,13 @@ async def get_sources_from_docstore_async(sources, vector_name, search_type="OR"
     
     return documents
 
-def get_sources_from_docstore(sources, vector_name, search_type="OR"):
+def get_sources_from_docstore(sources, vector_name, search_type="OR", just_source_name=False):
     
-    query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    if just_source_name:
+        query = _list_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    else:
+        query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+
     if not query:
         return []
     
@@ -303,3 +334,5 @@ def delete_sources_from_alloydb(sources, vector_name):
         DELETE FROM {vector_name}_vectorstore_{vector_length}
         WHERE {conditions}
     """
+
+    return query

sunholo/utils/config_class.py

@@ -37,6 +37,10 @@ class ConfigManager:
         self.local_config_folder = local_config_folder
         self.configs_by_kind = self.load_all_configs()
 
+        test_agent = self.vacConfig("agent")
+        if not test_agent:
+            print(f"WARNING: No vacConfig.agent found for {self.vector_name} - are you in right folder? {local_config_folder=} {self.config_folder=}")
+
     def load_all_configs(self):
         """
         Load all configuration files from the specified directories into a dictionary.
@@ -121,7 +125,7 @@ class ConfigManager:
         self.config_cache[filename] = (config, datetime.now())
         log.debug(f"Loaded and cached {config_file}")
         if is_local:
-            log.warning(f"Local configuration override for {filename}")
+            log.warning(f"Local configuration override for {filename} via {self.local_config_folder}")
         return config
 
     def _check_and_reload_configs(self):

{sunholo-0.78.5.dist-info → sunholo-0.79.0.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.78.5
+Version: 0.79.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.78.5.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.78.5.dist-info → sunholo-0.79.0.dist-info}/RECORD

@@ -31,7 +31,7 @@ sunholo/bots/github_webhook.py,sha256=5pQPRLM_wxxcILVaIzUDV8Kt7Arcm2dL1r1kMMHA52
 sunholo/bots/webapp.py,sha256=EIMxdAJ_xtufwJmvnn7N_Fb_1hZ9DjhJ0Kf_hp02vEU,1926
 sunholo/chunker/__init__.py,sha256=A5canS0XPgisHu0OZ7sVdILgEHGzgH9kpkDi4oBwLZk,135
 sunholo/chunker/azure.py,sha256=iZ0mXjei0cILsLuSUnZK0mmUUsQNiC3ZQr1iX8q5IeY,3263
-sunholo/chunker/doc_handling.py,sha256=rIyknpzDyj5A0u_DqSQVD_CXLRNZPOU6TCL4bhCdjOI,8563
+sunholo/chunker/doc_handling.py,sha256=AV-HU4FePKsk1mPASc3XOhJrqwdxnvEKc0GSpPTswMA,8714
 sunholo/chunker/encode_metadata.py,sha256=SYHaqKcr4lCzwmrzUGhgX4_l4pzDv7wAeNCw7a461MA,1912
 sunholo/chunker/images.py,sha256=Xmh1vwHrVhoXm5iH2dhCc52O8YgdzE8KrDSdL-pGnp8,1861
 sunholo/chunker/loaders.py,sha256=xiToUVgPz2ZzcqpUAq7aNP3PTenb_rBUAFzu0JPycIg,10268
@@ -58,7 +58,7 @@ sunholo/components/llm.py,sha256=XhSFuvthK35LDirX-zUbeLrLU8ccLSGxdJOOQovBGEM,114
 sunholo/components/retriever.py,sha256=F-wgZMpGJ8mGxJMAHA7HNgDwEhnvq1Pd6EGnTuBFlY8,6719
 sunholo/components/vectorstore.py,sha256=zUJ90L1S4IyxLB0JUWopeuwVjcsSqdhj1QreEfsJhsE,5548
 sunholo/database/__init__.py,sha256=Zz0Shcq-CtStf9rJGIYB_Ybzb8rY_Q9mfSj-nviM490,241
-sunholo/database/alloydb.py,sha256=d9W0pbZB0jTVIGF5OVaQ6kXHo-X3-6e9NpWNmV5e9UY,10464
+sunholo/database/alloydb.py,sha256=ZZGDA6DBSoWouDFi69LvTT1DgiiBz3aSR6u1hFO-IZY,11520
 sunholo/database/alloydb_client.py,sha256=AYA0SSaBy-1XEfeZI97sMGehfrwnfbwZ8sE0exzI2E0,7254
 sunholo/database/database.py,sha256=UDHkceiEvJmS3esQX2LYEjEMrHcogN_JHuJXoVWCH3M,7354
 sunholo/database/lancedb.py,sha256=2rAbJVusMrm5TPtVTsUtmwn0z1iZ_wvbKhc6eyT6ClE,708
@@ -116,7 +116,7 @@ sunholo/utils/__init__.py,sha256=Hv02T5L2zYWvCso5hzzwm8FQogwBq0OgtUbN_7Quzqc,89
 sunholo/utils/api_key.py,sha256=Ct4bIAQZxzPEw14hP586LpVxBAVi_W9Serpy0BK-7KI,244
 sunholo/utils/big_context.py,sha256=gJIP7_ZL-YSLhOMq8jmFTMqH1wq8eB1NK7oKPeZAq2s,5578
 sunholo/utils/config.py,sha256=XOH2pIvHs6QLnCwVAn7RuyRyV10TfbCEXabSjuEhKdo,8947
-sunholo/utils/config_class.py,sha256=4fm2Bwn_zFhVJBiUnMBzfCA5LKhTcBMU3mzhf5seXrw,8553
+sunholo/utils/config_class.py,sha256=GP58SfYYn32dSTUnyixKGujgF2DZRctc23ZhFRvDTZ8,8808
 sunholo/utils/config_schema.py,sha256=Wv-ncitzljOhgbDaq9qnFqH5LCuxNv59dTGDWgd1qdk,4189
 sunholo/utils/gcp.py,sha256=uueODEpA-P6O15-t0hmcGC9dONLO_hLfzSsSoQnkUss,4854
 sunholo/utils/gcp_project.py,sha256=0ozs6tzI4qEvEeXb8MxLnCdEVoWKxlM6OH05htj7_tc,1325
@@ -132,9 +132,9 @@ sunholo/vertex/init.py,sha256=uyg76EqS39jWJ2gxMqXOLWP6MQ2hc81wFdwgG86ZoCM,2868
 sunholo/vertex/memory_tools.py,sha256=pomHrDKqvY8MZxfUqoEwhdlpCvSGP6KmFJMVKOimXjs,6842
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.78.5.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.78.5.dist-info/METADATA,sha256=seWt-IjHlh20kmVCIeYAftXTj7OPKwqVvdOZhZ4J7KM,7348
-sunholo-0.78.5.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-sunholo-0.78.5.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.78.5.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.78.5.dist-info/RECORD,,
+sunholo-0.79.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.79.0.dist-info/METADATA,sha256=-sMhBNEnku2Fz06P5Ihk8N6KP-VlQ37zCidev8zEMjU,7348
+sunholo-0.79.0.dist-info/WHEEL,sha256=Rp8gFpivVLXx-k3U95ozHnQw8yDcPxmhOpn_Gx8d5nc,91
+sunholo-0.79.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.79.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.79.0.dist-info/RECORD,,

{sunholo-0.78.5.dist-info → sunholo-0.79.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.3.0)
+Generator: setuptools (72.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any