sunholo 0.119.3__tar.gz → 0.119.6__tar.gz

{sunholo-0.119.3/src/sunholo.egg-info → sunholo-0.119.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.119.3
+Version: 0.119.6
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.119.3 → sunholo-0.119.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sunholo"
-version = "0.119.3"
+version = "0.119.6"
 description = "Large Language Model DevOps - a package to help deploy LLMs to the Cloud."
 readme = "README.md"
 requires-python = ">=3.10"

{sunholo-0.119.3 → sunholo-0.119.6}/src/sunholo/discovery_engine/chunker_handler.py

@@ -5,7 +5,7 @@ from ..components import load_memories
 
 from .discovery_engine_client import DiscoveryEngineClient
 from .create_new import create_new_discovery_engine
-    
+from ..embedder.embed_metadata import audit_metadata
 
 def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=None):
     """
@@ -65,11 +65,15 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
             return None
         for corp in corpuses:
             try:
+                metadata = audit_metadata(metadata, chunk_length=500)
                 response = corp.import_document_with_metadata(
                     gcs_uri=message_data,
                     metadata=metadata
                 )
-                log.info(f"Imported file to corpus: {response} with metadata: {metadata}")
+                if response:
+                    log.info(f"Imported file to corpus: {response} with metadata: {metadata}")
+                else:
+                    log.warning(f"Could not import {message_data} got not response")
             except Exception as err:
                 log.error(f"Error importing {message_data} - {corp=} - {str(err)}")
 

{sunholo-0.119.3 → sunholo-0.119.6}/src/sunholo/discovery_engine/discovery_engine_client.py

@@ -558,6 +558,9 @@ class DiscoveryEngineClient:
                 except json.JSONDecodeError as e:
                     log.error(f"Error decoding JSON in line: {line.strip()}. Error: {e}")
                     continue  # Skip to the next line if there's an error
+                except Exception as e:
+                    log.error(f"Unknown error: {str(e)}")
+                    raise e
 
         # 2. Use InlineSource to import:
         request = discoveryengine.ImportDocumentsRequest(
@@ -615,6 +618,7 @@ class DiscoveryEngineClient:
 
         except Exception as e:
             log.error(f"Error importing document with metadata: {e}")
+            raise e
     
     def get_mime_type(self, uri:str):
         return guess_mime_type(uri)

{sunholo-0.119.3 → sunholo-0.119.6}/src/sunholo/embedder/embed_chunk.py

@@ -26,6 +26,7 @@ from ..components import get_embeddings, pick_vectorstore, load_memories, pick_e
 from ..custom_logging import log
 from ..database.uuid import generate_uuid_from_object_id
 from ..utils import ConfigManager
+from .embed_metadata import audit_metadata
 
 def embed_pubsub_chunk(data: dict):
     """Triggered from a message on a Cloud Pub/Sub topic "embed_chunk" topic
@@ -75,25 +76,7 @@ def embed_pubsub_chunk(data: dict):
     
     log.info(f"Embedding: {vector_name} page_content: {page_content[:30]}...[{len(page_content)}] - {metadata}")
 
-    if 'eventTime' not in metadata:
-        metadata['eventTime'] = datetime.datetime.now().isoformat(timespec='microseconds') + "Z"
-    metadata['eventtime'] = metadata['eventTime']
-
-    if 'source' not in metadata:
-        if 'objectId' in metadata:
-            metadata['source'] = metadata['objectId']
-        elif 'url' in metadata:
-            metadata['source'] = metadata['url']
-        else:
-            log.warning(f"No source found in metadata: {metadata}")
-    
-    if 'original_source' not in metadata:
-        metadata['original_source'] = metadata.get('source')
-    else:
-        metadata['source'] = metadata['original_source']
-    
-    if 'chunk_length' not in metadata:
-        metadata['chunk_length'] = len(page_content)
+    metadata = audit_metadata(metadata, chunk_length=len(page_content))
     
     if 'doc_id' not in metadata:
         log.warning(f"No doc_id found in metadata for {metadata['source']}- creating one")

sunholo-0.119.6/src/sunholo/embedder/embed_metadata.py

@@ -0,0 +1,27 @@
+
+import datetime
+from ..custom_logging import log
+
+def audit_metadata(metadata, chunk_length=None):
+    
+    if 'eventTime' not in metadata:
+        metadata['eventTime'] = datetime.datetime.now().isoformat(timespec='microseconds') + "Z"
+    metadata['eventtime'] = metadata['eventTime']
+
+    if 'source' not in metadata:
+        if 'objectId' in metadata:
+            metadata['source'] = metadata['objectId']
+        elif 'url' in metadata:
+            metadata['source'] = metadata['url']
+        else:
+            log.warning(f"No source found in metadata: {metadata}")
+    
+    if 'original_source' not in metadata:
+        metadata['original_source'] = metadata.get('source')
+    else:
+        metadata['source'] = metadata['original_source']
+    
+    if 'chunk_length' not in metadata:
+        metadata['chunk_length'] = chunk_length
+    
+    return metadata

{sunholo-0.119.3 → sunholo-0.119.6/src/sunholo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.119.3
+Version: 0.119.6
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.119.3 → sunholo-0.119.6}/src/sunholo.egg-info/SOURCES.txt

@@ -88,6 +88,7 @@ src/sunholo/discovery_engine/discovery_engine_client.py
 src/sunholo/discovery_engine/get_ai_search_chunks.py
 src/sunholo/embedder/__init__.py
 src/sunholo/embedder/embed_chunk.py
+src/sunholo/embedder/embed_metadata.py
 src/sunholo/excel/__init__.py
 src/sunholo/excel/plugin.py
 src/sunholo/gcs/__init__.py