sunholo 0.69.10__tar.gz → 0.69.13__tar.gz

{sunholo-0.69.10 → sunholo-0.69.13}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.10
+Version: 0.69.13
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.10.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.13.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -61,6 +61,7 @@ Requires-Dist: pypdf; extra == "all"
 Requires-Dist: python-socketio; extra == "all"
 Requires-Dist: rich; extra == "all"
 Requires-Dist: supabase; extra == "all"
+Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
 Provides-Extra: cli
 Requires-Dist: jsonschema>=4.21.1; extra == "cli"
@@ -73,6 +74,7 @@ Requires-Dist: pg8000; extra == "database"
 Requires-Dist: pgvector; extra == "database"
 Requires-Dist: psycopg2-binary; extra == "database"
 Requires-Dist: lancedb; extra == "database"
+Requires-Dist: tantivy; extra == "database"
 Provides-Extra: pipeline
 Requires-Dist: GitPython; extra == "pipeline"
 Requires-Dist: lark; extra == "pipeline"

{sunholo-0.69.10 → sunholo-0.69.13}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.69.10'
+version = '0.69.13'
 
 setup(
     name='sunholo',
@@ -76,6 +76,7 @@ setup(
             "python-socketio",
             "rich",
             "supabase",
+            "tantivy",
             "tiktoken",
 
         ],
@@ -91,6 +92,7 @@ setup(
             "pgvector",
             "psycopg2-binary",
             "lancedb",
+            "tantivy"
         ],
         'pipeline': [
             "GitPython",

{sunholo-0.69.10 → sunholo-0.69.13}/sunholo/discovery_engine/chunker_handler.py

@@ -51,6 +51,8 @@ def do_discovery_engine(message_data, metadata, vector_name):
 
     if message_data.startswith("gs://"):
         log.info(f"DiscoveryEngineClient.import_files for {message_data}")
+        if "/pdf_parts/" in message_data:
+            return None
         for corp in corpuses:
             try:
                 response = corp.import_documents(

{sunholo-0.69.10 → sunholo-0.69.13}/sunholo/discovery_engine/discovery_engine_client.py

@@ -201,55 +201,63 @@ class DiscoveryEngineClient:
             ),
         )
 
+        log.info(f"Discovery engine request: {search_request=}")
         search_response = self.search_client.search(search_request)
+        
 
         if parse_chunks_to_string:
-            
-            return self.process_chunks(search_response)
+
+            big_string = self.process_chunks(search_response)
+            log.info(f"Discovery engine chunks string sample: {big_string[:100]}")
+
+            return big_string
         
+        log.info("Discovery engine response object")
         return search_response
 
     def process_chunks(self, response):
         all_chunks = []
 
-        if 'results' not in response:
-            raise ValueError('No results found in response')
+        # Check if the response contains results
+        if not hasattr(response, 'results') or not response.results:
+            raise ValueError(f'No results found in response: {response=}')
         
-        for result in response['results']:
-            chunk = result['chunk']
-            chunk_metadata = chunk['chunkMetadata']
+        # Iterate through each result in the response
+        for result in response.results:
+            chunk = result.chunk
+            chunk_metadata = chunk.chunkMetadata
 
-            if 'previousChunks' in chunk_metadata:
+            if hasattr(chunk_metadata, 'previousChunks'):
                 # Process previous chunks
-                for prev_chunk in chunk['chunkMetadata']['previousChunks']:
+                for prev_chunk in chunk_metadata.previousChunks:
                     prev_chunk_string = (
-                        f"# {prev_chunk['id']}\n"
-                        f"{prev_chunk['content']}\n"
+                        f"# {prev_chunk.id}\n"
+                        f"{prev_chunk.content}\n"
                         f"## metadata\n"
-                        f"Document URI: {prev_chunk['documentMetadata']['uri']}\n"
-                        f"Document Title: {prev_chunk['documentMetadata']['title']}\n"
+                        f"Document URI: {prev_chunk.documentMetadata.uri}\n"
+                        f"Document Title: {prev_chunk.documentMetadata.title}\n"
                     )
                     all_chunks.append(prev_chunk_string)
 
             # Process fetched chunk
             fetched_chunk_string = (
-                f"# {chunk['id']}\n"
-                f"{chunk['content']}\n"
+                f"# {chunk.id}\n"
+                f"{chunk.content}\n"
                 f"## metadata\n"
-                f"Document URI: {chunk['documentMetadata']['uri']}\n"
-                f"Document Title: {chunk['documentMetadata']['title']}\n"
+                f"Document URI: {chunk.documentMetadata.uri}\n"
+                f"Document Title: {chunk.documentMetadata.title}\n"
             )
             all_chunks.append(fetched_chunk_string)
 
             # Process next chunks
-            if 'nextChunks' in chunk_metadata:
-                for next_chunk in chunk_metadata['nextChunks']:
+            if hasattr(chunk_metadata, 'nextChunks'):
+                for next_chunk in chunk_metadata.nextChunks:
                     next_chunk_string = (
-                        f"# {next_chunk['id']}\n"
-                        f"{next_chunk['content']}\n"
+                        f"# {next_chunk.id}\n"
+                        f"{next_chunk.content}\n"
                         f"## metadata\n"
-                        f"Document URI: {next_chunk['documentMetadata']['uri']}\n"
-                        f"Document Title: {next_chunk['documentMetadata']['title']}\n"
+                        f"Document URI: {next_chunk.documentMetadata.uri}\n"
+                        f"Document Title: {next_chunk.documentMetadata.title}\n"
                     )
                     all_chunks.append(next_chunk_string)
 
@@ -311,9 +319,9 @@ class DiscoveryEngineClient:
         try:
             operation = import_documents_with_retry(self.doc_client, request)
         except ResourceExhausted as e:
-            print(f"Operation failed after retries due to quota exceeded: {e}")
+            log.error(f"Operation failed after retries due to quota exceeded: {e}")
         except Exception as e:
-            print(f"An unexpected error occurred: {e}")
+            log.error(f"An unexpected error occurred: {e}")
 
         return operation.operation.name
 

{sunholo-0.69.10 → sunholo-0.69.13}/sunholo/logging.py

@@ -119,8 +119,14 @@ class GoogleCloudLogging:
         if log_text:
             if isinstance(log_struct, dict):
                 logger.log_struct(log_struct, severity=severity, source_location=caller_info)
-            else:
+            elif isinstance(log_struct, str):
                 logger.log_text(log_text, severity=severity, source_location=caller_info)
+            else:
+                try:
+                    turn_to_text = str(log_text)
+                    logger.log_text(turn_to_text, severity=severity, source_location=caller_info)
+                except Exception as err:
+                    print(f"Could not log this: {log_text=} - {str(err)}")
 
         elif log_struct:
             if not isinstance(log_struct, dict):

{sunholo-0.69.10 → sunholo-0.69.13}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.10
+Version: 0.69.13
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.10.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.13.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -61,6 +61,7 @@ Requires-Dist: pypdf; extra == "all"
 Requires-Dist: python-socketio; extra == "all"
 Requires-Dist: rich; extra == "all"
 Requires-Dist: supabase; extra == "all"
+Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
 Provides-Extra: cli
 Requires-Dist: jsonschema>=4.21.1; extra == "cli"
@@ -73,6 +74,7 @@ Requires-Dist: pg8000; extra == "database"
 Requires-Dist: pgvector; extra == "database"
 Requires-Dist: psycopg2-binary; extra == "database"
 Requires-Dist: lancedb; extra == "database"
+Requires-Dist: tantivy; extra == "database"
 Provides-Extra: pipeline
 Requires-Dist: GitPython; extra == "pipeline"
 Requires-Dist: lark; extra == "pipeline"

{sunholo-0.69.10 → sunholo-0.69.13}/sunholo.egg-info/requires.txt

@@ -42,6 +42,7 @@ pypdf
 python-socketio
 rich
 supabase
+tantivy
 tiktoken
 
 [anthropic]
@@ -59,6 +60,7 @@ pg8000
 pgvector
 psycopg2-binary
 lancedb
+tantivy
 
 [gcp]
 google-auth-httplib2