sunholo 0.119.7__py3-none-any.whl → 0.119.9__py3-none-any.whl

sunholo/chunker/process_chunker_data.py

@@ -38,7 +38,7 @@ def process_chunker_data(message_data, metadata, vector_name):
         return llamacheck
     
     # if only a discovery engine memory, return early as no other processing needed
-    discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config)
+    discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config, process=False)
     if discovery_check:
         return discovery_check
 

sunholo/discovery_engine/chunker_handler.py

@@ -125,7 +125,11 @@ def check_write_memories(config:ConfigManager):
     
     return write_mem
 
-def discovery_engine_chunker_check(message_data, metadata, vector_name:str=None, config:ConfigManager=None):
+def discovery_engine_chunker_check(message_data, 
+                                   metadata, 
+                                   vector_name:str=None, 
+                                   config:ConfigManager=None,
+                                   process:bool=True):
 
     if config is None:
         if vector_name is None:
@@ -139,7 +143,7 @@ def discovery_engine_chunker_check(message_data, metadata, vector_name:str=None,
     
     total_memories = len(check_write_memories(config))
     llama = None
-    if check_discovery_engine_in_memory(config):
+    if check_discovery_engine_in_memory(config) and process:
         llama = do_discovery_engine(message_data, metadata, config=config)
         log.info(f"Processed discovery engine: {llama}")
 

sunholo/discovery_engine/cli.py

@@ -223,12 +223,11 @@ def setup_discovery_engine_subparser(subparsers):
     # Search subcommand
     search_parser = discovery_engine_subparsers.add_parser('search', help='Search a Discovery Engine datastore')
     search_parser.add_argument('--query', required=True, help='The search query')
-    search_parser.add_argument('--num-previous-chunks', type=int, default=3, help='Number of previous chunks to return for context')
-    search_parser.add_argument('--num-next-chunks', type=int, default=3, help='Number of next chunks to return for context')
+    search_parser.add_argument('--data-store-id', required=True, help='Data store ID to search')
     search_parser.add_argument('--page-size', type=int, default=10, help='The maximum number of results to return per page')
     search_parser.add_argument('--parse-chunks-to-string', action='store_true', help='Combine chunks into a single string')
     search_parser.add_argument('--serving-config', default='default_serving_config', help='The serving configuration to use')
-    search_parser.add_argument('--data-store-ids', nargs='+', help='List of data store IDs to search (optional)')
+
     search_parser.set_defaults(func=discovery_engine_command)
 
     # Search by ID and/or Date subcommand

sunholo/discovery_engine/discovery_engine_client.py

@@ -607,6 +607,12 @@ class DiscoveryEngineClient:
 
         return self._import_document_request(request)
 
+    def _create_unique_gsuri_docid(self, gcs_uri:str):
+        import hashlib
+        # Create SHA-256 hash of the URI
+        hash_object = hashlib.sha256(gcs_uri.encode())
+        # Take first 16 bytes (128 bits) and encode as hex
+        return hash_object.hexdigest()[:32]
     
     def import_document_with_metadata(self, gcs_uri: str, metadata: dict, branch="default_branch"):
         """
@@ -622,7 +628,7 @@ class DiscoveryEngineClient:
         """
         try:
             # 1. Generate a unique document ID
-            document_id = str(uuid.uuid4())
+            document_id = self._create_unique_gsuri_docid(gcs_uri)
 
             # 2. Create a Document object
             parent = self.doc_client.branch_path(

{sunholo-0.119.7.dist-info → sunholo-0.119.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.119.7
+Version: 0.119.9
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.119.7.dist-info → sunholo-0.119.9.dist-info}/RECORD

@@ -38,7 +38,7 @@ sunholo/chunker/images.py,sha256=id2PBu6XyGEOtgafq2v0c9_O6kxaC_pYFMnbsIitkSg,186
 sunholo/chunker/loaders.py,sha256=5NXrMxV-WdbFpxeLhFzccw0_zhf1UQ7yKFFeaMkc9Bc,11105
 sunholo/chunker/message_data.py,sha256=bpb8QWQttqazm5lr7fTFJ5JDwf-P0SQ5cOIf6NikNyI,10836
 sunholo/chunker/pdfs.py,sha256=xwbuMJrbypcyPXfZ8tiUidWeMr80C2NhfTC1mwa8SHY,2477
-sunholo/chunker/process_chunker_data.py,sha256=uO-YOEHIjAOy0ZMJ0vea9OMNsQBISHfhbtgoyuHiP6s,3598
+sunholo/chunker/process_chunker_data.py,sha256=xjOAf1FvHDwQaBm7kgDzLQUnwm6AW8qf4fTrwDnwmtc,3613
 sunholo/chunker/publish.py,sha256=IDud-NhRcEZFv9GkyWJFRKwfptIU052kSPKEx8AYW68,2943
 sunholo/chunker/pubsub.py,sha256=48bhuAcszN7LGe3-ksPSLHHhq0uKxiXOrizck5qpcP0,1012
 sunholo/chunker/splitter.py,sha256=RfekLPkjhCcNd1PFXIj_FxusJMJ8_3cyWl7bsYvtQ0g,7068
@@ -72,10 +72,10 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=hLgqRDJ22Aov9o2QjAEfsVgnL3kMdM-g5p8RJ9OyKdQ,130
-sunholo/discovery_engine/chunker_handler.py,sha256=1kwVhy9hAxwXjOLpeF_Zc1uFK3uJIwB54gvTnwyjcv0,5849
-sunholo/discovery_engine/cli.py,sha256=5zebiZbrGtUnG1woAINNDF6ZF7hYM_Y6H1ekZXpCZPQ,13064
+sunholo/discovery_engine/chunker_handler.py,sha256=CYsyZLz_VZ0S_TcVlYmPxYz3JiAiPtw3tbcjvE3IFYA,6023
+sunholo/discovery_engine/cli.py,sha256=KGVle5rkLL49oF9TQhrGI--8017IvvLOEoYur545Qb0,12790
 sunholo/discovery_engine/create_new.py,sha256=WUi4_xh_dFaGX3xA9jkNKZhaR6LCELjMPeRb0hyj4FU,1226
-sunholo/discovery_engine/discovery_engine_client.py,sha256=PyX8CUalgqQG5H-V6KsBW7cSaPU-zmDlpFIEvGBzVPE,32753
+sunholo/discovery_engine/discovery_engine_client.py,sha256=0h3h_Vy8vCcvfSBhGl8qSgVzDakOmsaPPFqHIGh3mv8,33053
 sunholo/discovery_engine/get_ai_search_chunks.py,sha256=hsFGOQugSeTMPEaQ16XTs_D45F8NABBm2IsAEdTk7kQ,4316
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=did2pKkWM2o0KkRcb0H9l2x_WjCq6OyuHDxGbITFKPM,6530
@@ -166,9 +166,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.119.7.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.119.7.dist-info/METADATA,sha256=tPM66q3QUs977ICkghprD3VE9MUgOf7SBDJuVYo_IH0,9654
-sunholo-0.119.7.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-sunholo-0.119.7.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.119.7.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.119.7.dist-info/RECORD,,
+sunholo-0.119.9.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.119.9.dist-info/METADATA,sha256=XDQ0g73OJrI5qnlk8SuhbYxaUPF7yuAZQyMUUqkZ0_4,9654
+sunholo-0.119.9.dist-info/WHEEL,sha256=nn6H5-ilmfVryoAQl3ZQ2l8SH5imPWFpm1A5FgEuFV4,91
+sunholo-0.119.9.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.119.9.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.119.9.dist-info/RECORD,,

{sunholo-0.119.7.dist-info → sunholo-0.119.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.1)
 Root-Is-Purelib: true
 Tag: py3-none-any