sunholo 0.126.2__tar.gz → 0.126.4__tar.gz

{sunholo-0.126.2/src/sunholo.egg-info → sunholo-0.126.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.126.2
+Version: 0.126.4
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.126.2 → sunholo-0.126.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sunholo"
-version = "0.126.2"
+version = "0.126.4"
 description = "Large Language Model DevOps - a package to help deploy LLMs to the Cloud."
 readme = "README.md"
 requires-python = ">=3.10"

{sunholo-0.126.2 → sunholo-0.126.4}/src/sunholo/discovery_engine/chunker_handler.py

@@ -6,6 +6,7 @@ from ..components import load_memories
 from .discovery_engine_client import DiscoveryEngineClient
 from .create_new import create_new_discovery_engine
 from ..embedder.embed_metadata import audit_metadata
+import traceback
 
 def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=None):
     """
@@ -35,6 +36,7 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
             if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                 log.info(f"Found vectorstore {vectorstore}")
                 if value.get('read_only'):
+                    log.info(f"{vectorstore} is read only, skipping")
                     continue
 
                 project_id = value.get("project_id")
@@ -49,6 +51,7 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
                 if not project_id:
                     raise ValueError("Couldn't retrieve project_id for vertex_ai_search")        
 
+                log.info(f"Using {project_id} and {location} for DiscoveryEngineClient")
                 corpus = DiscoveryEngineClient(
                     data_store_id=config.vector_name, 
                     project_id=project_id,
@@ -66,10 +69,13 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
     if message_data.startswith("gs://"):
         log.info(f"DiscoveryEngineClient.import_files for {message_data}")
         if "/pdf_parts/" in message_data:
+            log.info(f"Not processing files with /pdf_parts/ - {message_data}")
             return None
         for corp in corpuses:
             try:
+                
                 metadata = audit_metadata(metadata, chunk_length=500)
+                log.info(f"Importing {message_data} {metadata=} to {corp}")
                 response = corp.import_document_with_metadata(
                     gcs_uri=message_data,
                     metadata=metadata
@@ -79,7 +85,7 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
                 else:
                     log.warning(f"Could not import {message_data} got not response")
             except Exception as err:
-                log.error(f"Error importing {message_data} - {corp=} - {str(err)}")
+                log.error(f"Error importing {message_data} - {corp=} - {str(err)} {traceback.format_exc()}")
 
                 if str(err).startswith("404"):
                     log.info(f"Attempting to create a new DiscoveryEngine corpus: {config.vector_name}")
@@ -94,8 +100,8 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
                             gcs_uri=message_data,
                             metadata=metadata
                         )
-                    
-                continue
+                else:
+                    raise Exception(f"Error importing {message_data} - {corp=} - {str(err)}")
 
         metadata["source"] = message_data
         return metadata
@@ -156,9 +162,18 @@ def discovery_engine_chunker_check(message_data,
         return metadata
 
     if total_discovery_memories > 0:
-        log.info(f"Process discovery engine for {metadata}")
-        disc_meta = do_discovery_engine(message_data, metadata, config=config)
-        log.info(f"Processed discovery engine: {disc_meta}")
+        try:
+            log.info(f"Process discovery engine for {metadata}")
+            disc_meta = do_discovery_engine(message_data, metadata, config=config)
+            if disc_meta is None:
+                log.error(f"No disc_meta found for {metadata}")
+            else:
+                log.info(f"Processed discovery engine: {disc_meta}")
+        except Exception as err:
+            log.error(f"Error processing discovery engine: {str(err)} {traceback.format_exc()}")
+            disc_meta = None
+            
+        return disc_meta
 
     # If discovery engine is the only entry, return
     if total_discovery_memories == total_memories:

{sunholo-0.126.2 → sunholo-0.126.4}/src/sunholo/discovery_engine/discovery_engine_client.py

@@ -16,6 +16,7 @@ import asyncio
 import json
 import uuid
 from ..utils.mime import guess_mime_type
+import traceback
 
 class DiscoveryEngineClient:
     """
@@ -80,8 +81,10 @@ class DiscoveryEngineClient:
             self.async_search_client = discoveryengine.SearchServiceAsyncClient(client_options=client_options)
         except RuntimeError:
             # No event loop in non-async environment, set async client to None
-            log.info("No event loop detected; skipping async client initialization")
+            log.info("No event loop detected; skipping Discoveryengine async client initialization")
             self.async_search_client = None
+        
+        log.info(f"Discovery Engine client initialized with {self.project_id=}, {self.data_store_id=}, {self.location=}")
 
     @classmethod
     def my_retry(cls):
@@ -490,6 +493,7 @@ class DiscoveryEngineClient:
             return doc_client.import_documents(request=request)
         
         try:
+            log.debug(f"Requesting import of documents: {request=}")
             operation = import_documents_with_retry(self.doc_client, request)
         except ResourceExhausted as e:
             log.error(f"DiscoveryEngine Operation failed after retries due to quota exceeded: {e}")
@@ -632,6 +636,7 @@ class DiscoveryEngineClient:
             str: The operation name.
         """
         try:
+            log.info(f"Importing doc with metadata: {gcs_uri=}, {metadata=}")
             # 1. Generate a unique document ID
             document_id = self._create_unique_gsuri_docid(gcs_uri)
 
@@ -662,7 +667,7 @@ class DiscoveryEngineClient:
             return self._import_document_request(request)
 
         except Exception as e:
-            log.error(f"Error importing document with metadata: {e}")
+            log.error(f"Error importing document with metadata: {e} {traceback.format_exc()}")
             raise e
     
     def get_mime_type(self, uri:str):

{sunholo-0.126.2 → sunholo-0.126.4}/src/sunholo/embedder/embed_metadata.py

@@ -6,7 +6,7 @@ from ..utils.mime import guess_mime_type
 
 from ..custom_logging import log
 
-def audit_metadata(metadata, chunk_length=None):
+def audit_metadata(metadata:dict, chunk_length:int=None) -> dict:
     
     if 'eventTime' not in metadata:
         metadata['eventTime'] = datetime.datetime.now().isoformat(timespec='microseconds') + "Z"

{sunholo-0.126.2 → sunholo-0.126.4/src/sunholo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.126.2
+Version: 0.126.4
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0