sunholo 0.119.4__py3-none-any.whl → 0.119.6__py3-none-any.whl

sunholo/discovery_engine/chunker_handler.py

@@ -5,7 +5,7 @@ from ..components import load_memories
 
 from .discovery_engine_client import DiscoveryEngineClient
 from .create_new import create_new_discovery_engine
-    
+from ..embedder.embed_metadata import audit_metadata
 
 def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=None):
     """
@@ -65,6 +65,7 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
             return None
         for corp in corpuses:
             try:
+                metadata = audit_metadata(metadata, chunk_length=500)
                 response = corp.import_document_with_metadata(
                     gcs_uri=message_data,
                     metadata=metadata

sunholo/embedder/embed_chunk.py

@@ -26,6 +26,7 @@ from ..components import get_embeddings, pick_vectorstore, load_memories, pick_e
 from ..custom_logging import log
 from ..database.uuid import generate_uuid_from_object_id
 from ..utils import ConfigManager
+from .embed_metadata import audit_metadata
 
 def embed_pubsub_chunk(data: dict):
     """Triggered from a message on a Cloud Pub/Sub topic "embed_chunk" topic
@@ -75,25 +76,7 @@ def embed_pubsub_chunk(data: dict):
     
     log.info(f"Embedding: {vector_name} page_content: {page_content[:30]}...[{len(page_content)}] - {metadata}")
 
-    if 'eventTime' not in metadata:
-        metadata['eventTime'] = datetime.datetime.now().isoformat(timespec='microseconds') + "Z"
-    metadata['eventtime'] = metadata['eventTime']
-
-    if 'source' not in metadata:
-        if 'objectId' in metadata:
-            metadata['source'] = metadata['objectId']
-        elif 'url' in metadata:
-            metadata['source'] = metadata['url']
-        else:
-            log.warning(f"No source found in metadata: {metadata}")
-    
-    if 'original_source' not in metadata:
-        metadata['original_source'] = metadata.get('source')
-    else:
-        metadata['source'] = metadata['original_source']
-    
-    if 'chunk_length' not in metadata:
-        metadata['chunk_length'] = len(page_content)
+    metadata = audit_metadata(metadata, chunk_length=len(page_content))
     
     if 'doc_id' not in metadata:
         log.warning(f"No doc_id found in metadata for {metadata['source']}- creating one")

sunholo/embedder/embed_metadata.py

@@ -0,0 +1,27 @@
+
+import datetime
+from ..custom_logging import log
+
+def audit_metadata(metadata, chunk_length=None):
+    
+    if 'eventTime' not in metadata:
+        metadata['eventTime'] = datetime.datetime.now().isoformat(timespec='microseconds') + "Z"
+    metadata['eventtime'] = metadata['eventTime']
+
+    if 'source' not in metadata:
+        if 'objectId' in metadata:
+            metadata['source'] = metadata['objectId']
+        elif 'url' in metadata:
+            metadata['source'] = metadata['url']
+        else:
+            log.warning(f"No source found in metadata: {metadata}")
+    
+    if 'original_source' not in metadata:
+        metadata['original_source'] = metadata.get('source')
+    else:
+        metadata['source'] = metadata['original_source']
+    
+    if 'chunk_length' not in metadata:
+        metadata['chunk_length'] = chunk_length
+    
+    return metadata

{sunholo-0.119.4.dist-info → sunholo-0.119.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.119.4
+Version: 0.119.6
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.119.4.dist-info → sunholo-0.119.6.dist-info}/RECORD

@@ -72,12 +72,13 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=hLgqRDJ22Aov9o2QjAEfsVgnL3kMdM-g5p8RJ9OyKdQ,130
-sunholo/discovery_engine/chunker_handler.py,sha256=dmdXpUIYIkmXUf3M6V-vuWqDXNthLPsKoSyzQmQOu4c,5731
+sunholo/discovery_engine/chunker_handler.py,sha256=1kwVhy9hAxwXjOLpeF_Zc1uFK3uJIwB54gvTnwyjcv0,5849
 sunholo/discovery_engine/create_new.py,sha256=WUi4_xh_dFaGX3xA9jkNKZhaR6LCELjMPeRb0hyj4FU,1226
 sunholo/discovery_engine/discovery_engine_client.py,sha256=b6UT7s-1zqrLnIfoP9bDs64hk0SUsqaA40ghbs2A7go,27552
 sunholo/discovery_engine/get_ai_search_chunks.py,sha256=hsFGOQugSeTMPEaQ16XTs_D45F8NABBm2IsAEdTk7kQ,4316
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
-sunholo/embedder/embed_chunk.py,sha256=Vvvj3-H4pSb1a2sLik3-X3X459j2jrUq1dBNAsOcQLo,7156
+sunholo/embedder/embed_chunk.py,sha256=did2pKkWM2o0KkRcb0H9l2x_WjCq6OyuHDxGbITFKPM,6530
+sunholo/embedder/embed_metadata.py,sha256=2ziUIdVwnbCUU8gOwQWEvkrRcyp-7IeyZfSsWNkMquA,866
 sunholo/excel/__init__.py,sha256=AqTMN9K4qJYi4maEgoORc5oxDVGO_eqmwzDaVP37JgY,56
 sunholo/excel/plugin.py,sha256=rl3FoECZ6Ts8KKExPrbPwr3u3CegZfsevmcjgUXAlhE,4033
 sunholo/gcs/__init__.py,sha256=SZvbsMFDko40sIRHTHppA37IijvJTae54vrhooEF5-4,90
@@ -164,9 +165,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.119.4.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.119.4.dist-info/METADATA,sha256=cYBqEOOkHvurx9b3-ipXsEMDpa6JIARsvEpVIqWE2ZQ,9654
-sunholo-0.119.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-sunholo-0.119.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.119.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.119.4.dist-info/RECORD,,
+sunholo-0.119.6.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.119.6.dist-info/METADATA,sha256=z8wn78_BYKX8y6CyyxcMbKPA4hkJNPO3kwgVivrJSKQ,9654
+sunholo-0.119.6.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+sunholo-0.119.6.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.119.6.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.119.6.dist-info/RECORD,,