sunholo 0.69.2__py3-none-any.whl → 0.69.4__py3-none-any.whl

sunholo/chunker/data_to_embed_pubsub.py

@@ -60,7 +60,7 @@ def process_chunker_data(message_data, metadata, vector_name):
         metadata["vector_name"] = vector_name
 
     if message_data is None:
-        log.error("No message_data was found in data: {data}")
+        log.error(f"No message_data was found in data: {message_data}")
         return
 
     log.debug(f"Found metadata in pubsub: {metadata}")

sunholo/discovery_engine/create_new.py

@@ -6,6 +6,13 @@ def create_new_discovery_engine(vector_name):
 
     chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
 
+    if chunker_config:
+        chunk_size = chunker_config.get("chunk_size")
+    
+    if not chunk_size:
+        chunk_size = 500
+
+
     project_id = gcp_config.get('project_id')
     #location = gcp_config.get('location')
 
@@ -16,6 +23,6 @@ def create_new_discovery_engine(vector_name):
                     #location=location
                     )
 
-    new_store = de.create_data_store(chunk_size=chunker_config.get("chunk_size"))
+    new_store = de.create_data_store(chunk_size=chunk_size)
 
     return new_store

sunholo/discovery_engine/discovery_engine_client.py

@@ -61,16 +61,13 @@ class DiscoveryEngineClient:
             if location != "global"
             else None
         )
-        self.client = discoveryengine.DataStoreServiceClient(client_options=client_options)
-        self.parent = self.client.branch_path(
-            project=project_id,
-            location=location,
-            data_store=data_store_id,
-            branch="default_branch",
-        )
+        self.store_client  = discoveryengine.DataStoreServiceClient(client_options=client_options)
+        self.doc_client    = discoveryengine.DocumentServiceClient(client_options=client_options)
+        self.search_client = discoveryengine.SearchServiceClient(client_options=client_options)
 
     def create_data_store(
-        self, chunk_size: int = 500
+        self, chunk_size: int = 500,
+        collection: str = "default_collection"
     ) -> str:
         """
         Creates a new data store with default configuration.
@@ -109,9 +106,15 @@ class DiscoveryEngineClient:
             document_processing_config=doc_config
         )
 
+        parent = self.store_client.collection_path(
+            project=self.project_id,
+            location=self.location,
+            collection=collection,
+        )
+
         # https://cloud.google.com/python/docs/reference/discoveryengine/0.11.4/google.cloud.discoveryengine_v1alpha.types.CreateDataStoreRequest
         request = discoveryengine.CreateDataStoreRequest(
-            parent=self.parent,
+            parent=parent,
             data_store_id=self.data_store_id,
             data_store=data_store,
             # Optional: For Advanced Site Search Only
@@ -119,7 +122,7 @@ class DiscoveryEngineClient:
         )
 
         # Make the request
-        operation = self.client.create_data_store(request=request)
+        operation = self.store_client.create_data_store(request=request)
 
         log.info(f"Waiting for operation to complete: {operation.operation.name}")
         response = operation.result()
@@ -140,6 +143,7 @@ class DiscoveryEngineClient:
         num_next_chunks: int = 3,
         page_size: int = 10,
         doc_or_chunks: str = "CHUNKS",  # or DOCUMENTS
+        serving_config: str = "default_serving_config",
     ):
         """Retrieves chunks or documents based on a query.
 
@@ -161,22 +165,19 @@ class DiscoveryEngineClient:
                     print(f"Chunk: {chunk.snippet}, document name: {chunk.document_name}")
             ```
         """
-        serving_config = self.client.get_default_serving_config(
-            name=self.client.serving_config_path(
-                project=self.project_id, 
-                location=self.location, 
-                data_store=self.data_store_id, 
-                serving_config="default_serving_config")
-                ).name
-        
+
+        serving_config_path = self.search_client.serving_config_path(
+            self.project_id,
+            self.location,
+            self.data_store_id,
+            serving_config
+        )
+
         search_request = discoveryengine.SearchRequest(
-            serving_config=serving_config,
+            serving_config=serving_config_path,
             query=query,
             page_size=page_size, 
             content_search_spec=discoveryengine.SearchRequest.ContentSearchSpec(
-                #snippet_spec=discoveryengine.SearchRequest.ContentSearchSpec.SnippetSpec(
-                #    return_snippet=True
-                #),
                 search_result_mode=doc_or_chunks,  # CHUNKS or DOCUMENTS
                 chunk_spec=discoveryengine.SearchRequest.ContentSearchSpec.ChunkSpec(
                     num_previous_chunks=num_previous_chunks,
@@ -185,13 +186,14 @@ class DiscoveryEngineClient:
             ),
         )
 
-        search_response = self.client.search(search_request)
+        search_response = self.search_client.search(search_request)
 
         return search_response
 
     def import_documents(self,
         gcs_uri: Optional[str] = None,
         data_schema="content",
+        branch="default_branch",
         bigquery_dataset: Optional[str] = None,
         bigquery_table: Optional[str] = None,
         bigquery_project_id: Optional[str] = None,
@@ -203,9 +205,16 @@ class DiscoveryEngineClient:
         
         """
 
+        parent = self.doc_client.branch_path(
+            self.project_id, 
+            self.location, 
+            self.data_store_id, 
+            branch
+        )
+
         if gcs_uri:
             request = discoveryengine.ImportDocumentsRequest(
-                parent=self.parent,
+                parent=parent,
                 # https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1alpha.types.GcsSource
                 gcs_source=discoveryengine.GcsSource(
                     input_uris=[gcs_uri], data_schema=data_schema,
@@ -215,7 +224,7 @@ class DiscoveryEngineClient:
             )
         else:
             request = discoveryengine.ImportDocumentsRequest(
-                parent=self.parent,
+                parent=parent,
                 bigquery_source=discoveryengine.BigQuerySource(
                     project_id=bigquery_project_id or self.project_id,
                     dataset_id=bigquery_dataset,
@@ -227,7 +236,7 @@ class DiscoveryEngineClient:
             )
 
         # Make the request
-        operation = self.client.import_documents(request=request)
+        operation = self.doc_client.import_documents(request=request)
 
         log.info(f"Waiting for operation to complete: {operation.operation.name}")
         response = operation.result()

{sunholo-0.69.2.dist-info → sunholo-0.69.4.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.2
+Version: 0.69.4
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.2.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.4.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.2.dist-info → sunholo-0.69.4.dist-info}/RECORD

@@ -24,7 +24,7 @@ sunholo/bots/discord.py,sha256=cCFae5K1BCa6JVkWGLh_iZ9qFO1JpXb6K4eJrlDfEro,2442
 sunholo/bots/github_webhook.py,sha256=5pQPRLM_wxxcILVaIzUDV8Kt7Arcm2dL1r1kMMHA524,9629
 sunholo/bots/webapp.py,sha256=EIMxdAJ_xtufwJmvnn7N_Fb_1hZ9DjhJ0Kf_hp02vEU,1926
 sunholo/chunker/__init__.py,sha256=yWYwpejyYxDpZv1joTrFMsh2SWAkd0z7a1VKtmOfMhA,77
-sunholo/chunker/data_to_embed_pubsub.py,sha256=GWAAAhnJse6mztwjj9-EF0G2Oun0GBuYV2-m52CBAas,4194
+sunholo/chunker/data_to_embed_pubsub.py,sha256=tL9J7s-F4szmpbZYU4dpvLr5R-LgiuZLpRvhQnrTD1E,4203
 sunholo/chunker/doc_handling.py,sha256=rIyknpzDyj5A0u_DqSQVD_CXLRNZPOU6TCL4bhCdjOI,8563
 sunholo/chunker/images.py,sha256=Xmh1vwHrVhoXm5iH2dhCc52O8YgdzE8KrDSdL-pGnp8,1861
 sunholo/chunker/loaders.py,sha256=xiToUVgPz2ZzcqpUAq7aNP3PTenb_rBUAFzu0JPycIg,10268
@@ -62,8 +62,8 @@ sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUt
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/discovery_engine/chunker_handler.py,sha256=4NT4fRAWY-kKM4X3imZ2RISYLMvpIVsPDpsAgUOhdlA,4149
-sunholo/discovery_engine/create_new.py,sha256=tO8xf52vysXWbPuBchr12gPXBnjvTeNiARvZh9Jf3Ok,811
-sunholo/discovery_engine/discovery_engine_client.py,sha256=62xesszWmDeg37TCPVc-w7QfuiOiBxUdL7k8pu-tRTk,11495
+sunholo/discovery_engine/create_new.py,sha256=6sowAvblLDBZq6skYwdc6n99YlyHvNLaIs6kYMnkdO8,921
+sunholo/discovery_engine/discovery_engine_client.py,sha256=yjMu_MXYzXzQMSxp_PATDcM7PXlB8VTgJT56fRbD0xs,11693
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=P744zUQJgqrjILunzaqtTerB9AwoXFU6tXBtz4rjWgQ,6673
 sunholo/gcs/__init__.py,sha256=DtVw_AZwQn-IguR5BJuIi2XJeF_FQXizhJikzRNrXiE,50
@@ -111,9 +111,9 @@ sunholo/vertex/__init__.py,sha256=JvHcGFuv6R_nAhY2AdoqqhMpJ5ugeWPZ_svGhWrObBk,13
 sunholo/vertex/init.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
 sunholo/vertex/memory_tools.py,sha256=8F1iTWnqEK9mX4W5RzCVKIjydIcNp6OFxjn_dtQ3GXo,5379
 sunholo/vertex/safety.py,sha256=3meAX0HyGZYrH7rXPUAHxtI_3w_zoy_RX7Shtkoa660,1275
-sunholo-0.69.2.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.69.2.dist-info/METADATA,sha256=mXk3Xa9hWlZadUN_zaENjfpmyI3TyVQc7-2kTUkVIR0,6155
-sunholo-0.69.2.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-sunholo-0.69.2.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.69.2.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.69.2.dist-info/RECORD,,
+sunholo-0.69.4.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.69.4.dist-info/METADATA,sha256=gaLbxVufOz5-8Juy1fMBHvw_BXCv5MiUPoSMI1hJotQ,6155
+sunholo-0.69.4.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+sunholo-0.69.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.69.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.69.4.dist-info/RECORD,,