sunholo 0.134.8__py3-none-any.whl → 0.136.0__py3-none-any.whl

sunholo/custom_logging.py

@@ -22,6 +22,8 @@ from .utils.version import sunholo_version
 import logging
 import inspect
 import os
+import json
+from functools import wraps
 
 class GoogleCloudLogging:
     
@@ -270,6 +272,91 @@ class GoogleCloudLogging:
         """
         self.structured_log(log_text=log_text, log_struct=log_struct, severity="CRITICAL")
 
+class StandardLoggerWrapper:
+    """
+    A wrapper for standard Python logger that mimics the interface of GoogleCloudLogging.
+    """
+    def __init__(self, logger):
+        self.logger = logger
+        self.trace_id = None
+        self.version = None
+
+    def _format_message(self, log_text=None, log_struct=None, severity=None):
+        """Format message to include structured data as JSON"""
+        parts = []
+        
+        # Add severity if provided
+        if severity:
+            parts.append(f"[{severity}]")
+            
+        # Add trace ID if available
+        if self.trace_id:
+            parts.append(f"[trace_id:{self.trace_id}]")
+            
+        # Add version if available
+        if self.version:
+            parts.append(f"[version:{self.version}]")
+            
+        # Add the log text if provided
+        if log_text:
+            parts.append(log_text)
+            
+        # Add structured data as JSON if provided
+        if log_struct:
+            if self.trace_id and isinstance(log_struct, dict):
+                log_struct["trace_id"] = self.trace_id
+            if self.version and isinstance(log_struct, dict):
+                log_struct["version"] = self.version
+            parts.append(f"STRUCT: {json.dumps(log_struct, default=str)}")
+            
+        return " ".join(parts)
+
+    def update_trace_id(self, trace_id):
+        """Set trace ID to be included in all logs."""
+        self.trace_id = trace_id
+        
+    def set_version(self, version):
+        """Set version to be included in all logs."""
+        self.version = version
+        
+    def structured_log(self, log_text=None, log_struct=None, logger_name=None, severity="INFO"):
+        """
+        Emulates Google Cloud's structured_log method using standard logging.
+        """
+        message = self._format_message(log_text, log_struct, severity)
+        
+        if severity == "DEBUG":
+            self.logger.debug(message)
+        elif severity == "INFO":
+            self.logger.info(message)
+        elif severity == "WARNING":
+            self.logger.warning(message)
+        elif severity == "ERROR":
+            self.logger.error(message)
+        elif severity == "CRITICAL":
+            self.logger.critical(message)
+        else:
+            self.logger.info(message)  # Default to info
+            
+    def debug(self, log_text=None, log_struct=None):
+        self.structured_log(log_text=log_text, log_struct=log_struct, severity="DEBUG")
+        
+    def info(self, log_text=None, log_struct=None):
+        self.structured_log(log_text=log_text, log_struct=log_struct, severity="INFO")
+        
+    def warning(self, log_text=None, log_struct=None):
+        self.structured_log(log_text=log_text, log_struct=log_struct, severity="WARNING")
+        
+    def error(self, log_text=None, log_struct=None):
+        self.structured_log(log_text=log_text, log_struct=log_struct, severity="ERROR")
+        
+    def exception(self, log_text=None, log_struct=None):
+        self.structured_log(log_text=log_text, log_struct=log_struct, severity="CRITICAL")
+        
+    # Forward any other standard logging methods to the underlying logger
+    def __getattr__(self, name):
+        return getattr(self.logger, name)
+    
 def is_logging_setup(logger=None):
     logger = logger or logging.getLogger()  # If no logger is specified, use the root logger
     return bool(logger.handlers)
@@ -352,7 +439,14 @@ def setup_logging(logger_name=None, log_level=logging.INFO, project_id=None):
 
     logger = logging.getLogger(logger_name)
     if is_logging_setup(logger):
-        return logger
+        # Check if it's already our wrapper
+        if hasattr(logger, 'structured_log'):
+            return logger
+        else:
+            # Wrap the existing logger
+            wrapper = StandardLoggerWrapper(logger)
+            wrapper.set_version(sunholo_version())
+            return wrapper
     
     if logger_name is None:
         logger_name = "sunholo"
@@ -370,10 +464,13 @@ def setup_logging(logger_name=None, log_level=logging.INFO, project_id=None):
         return gc_logger.setup_logging()
     else:
         if not logging.getLogger().hasHandlers():
-            logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+            logging.basicConfig(level=log_level, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
         log = logging.getLogger(logger_name)
-
-        return log
+        
+        # Wrap the standard logger to support log_struct parameter
+        wrapper = StandardLoggerWrapper(log)
+        wrapper.set_version(sunholo_version())
+        return wrapper
     
 # for debugging
 def safe_log_struct(log, severity, message, struct):

sunholo/discovery_engine/discovery_engine_client.py

@@ -374,7 +374,8 @@ class DiscoveryEngineClient:
         parse_documents_to_string: bool = True,
         serving_config: str = "default_serving_config",
         data_store_ids: Optional[List[str]] = None,
-        filter_str:str=None
+        filter_str:str=None,
+        max_limit:int=None
     ):
         """Retrieves entire documents based on a query.
 
@@ -405,7 +406,8 @@ class DiscoveryEngineClient:
             parse_chunks_to_string=parse_documents_to_string,
             serving_config=serving_config,
             data_store_ids=data_store_ids,
-            content_search_spec_type="documents"
+            content_search_spec_type="documents",
+            max_limit=max_limit
         )
 
     async def async_get_documents(
@@ -415,7 +417,9 @@ class DiscoveryEngineClient:
         parse_documents_to_string: bool = True,
         serving_config: str = "default_serving_config",
         data_store_ids: Optional[List[str]] = None,
-        filter_str:str=None
+        filter_str:str=None,
+        max_limit:int=None
+
     ):
         """Asynchronously retrieves entire documents based on a query.
 
@@ -439,7 +443,9 @@ class DiscoveryEngineClient:
             parse_chunks_to_string=parse_documents_to_string, 
             serving_config=serving_config,
             data_store_ids=data_store_ids,
-            content_search_spec_type="documents"
+            content_search_spec_type="documents",
+            max_limit=max_limit
+
         )
 
     def document_format(self, document):
@@ -476,20 +482,26 @@ class DiscoveryEngineClient:
             f"{derived_data}"
         )
 
-    def process_documents(self, response):
+    def process_documents(self, response, max_limit:int=None):
         """Process a search response containing documents into a formatted string."""
         all_documents = []
-
+        result_count = 0
         # Check if the response contains results
         if not hasattr(response, 'results') or not response.results:
             log.info(f'No results found in response: {response=}')
             return []
         
-        # Iterate through each result in the response
+        # Iterate through each result in the page
         for result in response.results:
             if hasattr(result, 'document'):
                 document = result.document
                 all_documents.append(self.document_format(document))
+                result_count += 1
+                
+                # Check if we've reached max_limit
+                if max_limit is not None and result_count >= max_limit:
+                    log.info(f"Reached max_limit of {max_limit} results, stopping processing")
+                    break
             else:
                 log.warning("No document found in result")
         
@@ -498,20 +510,27 @@ class DiscoveryEngineClient:
         
         return result_string
 
-    async def async_process_documents(self, response):
+    async def async_process_documents(self, response, max_limit:int=None):
         """Process a search response containing documents into a formatted string asynchronously."""
         all_documents = []
+        result_count = 0
 
         # Check if the response contains results
         if not hasattr(response, 'results') or not response.results:
             log.info(f'No results found in response: {response=}')
             return []
         
-        # Iterate through each result in the response
+        # Iterate through each result in the page
         for result in response.results:
             if hasattr(result, 'document'):
                 document = result.document
                 all_documents.append(self.document_format(document))
+                result_count += 1
+                
+                # Check if we've reached max_limit
+                if max_limit is not None and result_count >= max_limit:
+                    log.info(f"Reached max_limit of {max_limit} results, stopping processing")
+                    break
             else:
                 log.warning("No document found in result")
         
@@ -781,7 +800,8 @@ class DiscoveryEngineClient:
                         page_size=10, parse_chunks_to_string=True, 
                         serving_config="default_serving_config",
                         data_store_ids: Optional[List[str]] = None,
-                        content_search_spec_type="chunks"):
+                        content_search_spec_type="chunks",
+                        max_limit=None):
         """
         Searches with a generic filter string.
 
@@ -793,6 +813,8 @@ class DiscoveryEngineClient:
         Returns:
             discoveryengine.SearchResponse or str: The search response object or string of chunks.
         """
+        if max_limit is not None and max_limit < page_size:
+            page_size = max_limit
 
         serving_config_path = self.search_client.serving_config_path(
             self.project_id,
@@ -840,6 +862,33 @@ class DiscoveryEngineClient:
         except Exception as e:
             log.info(f"No results {search_request.data_store_specs=}: {str(e)}")
             return None
+
+        # Apply max_limit if needed
+        if content_search_spec_type=="documents" and max_limit is not None:
+            # For raw response objects (when parse_chunks_to_string=False)
+            if not parse_chunks_to_string:
+                # We need to limit the pager results before returning
+                limited_response = search_response
+                # Store the original pages iterator method
+                original_pages = limited_response.pages
+                
+                # Override the pages property with a custom iterator that respects max_limit
+                def limited_pages_iterator():
+                    results_count = 0
+                    for page in original_pages:
+                        yield page
+                        
+                        # Count results in this page
+                        if hasattr(page, 'results'):
+                            results_count += len(page.results)
+                        
+                        # Stop if we've reached max_limit
+                        if results_count >= max_limit:
+                            break
+                
+                # Replace the pages property with our custom iterator
+                limited_response.pages = limited_pages_iterator()
+                return limited_response
         
         if parse_chunks_to_string:
             if content_search_spec_type=="chunks":
@@ -850,7 +899,7 @@ class DiscoveryEngineClient:
                     return big_string
                 
             elif content_search_spec_type=="documents":
-                big_string = self.process_documents(search_response)
+                big_string = self.process_documents(search_response, max_limit=max_limit)
                 log.info(f"Discovery engine documents string sample: {big_string[:100]}")
 
                 return big_string
@@ -864,7 +913,8 @@ class DiscoveryEngineClient:
                             page_size=10, parse_chunks_to_string=True, 
                             serving_config="default_serving_config",
                             data_store_ids: Optional[List[str]] = None,
-                            content_search_spec_type="chunks"):
+                            content_search_spec_type="chunks",
+                            max_limit=None):
         """
         Searches with a generic filter string asynchronously.
 
@@ -876,6 +926,8 @@ class DiscoveryEngineClient:
         Returns:
             discoveryengine.SearchResponse or str: The search response object or string of chunks.
         """
+        if max_limit is not None and max_limit < page_size:
+            page_size = max_limit
 
         serving_config_path = self.async_search_client.serving_config_path(
             self.project_id,
@@ -922,6 +974,33 @@ class DiscoveryEngineClient:
             log.info(f"No results {search_request.data_store_specs=}: {str(e)}")
             return None
         
+        # Apply max_limit if needed
+        if content_search_spec_type=="documents" and max_limit is not None:
+            # For raw response objects (when parse_chunks_to_string=False)
+            if not parse_chunks_to_string:
+                # We need to limit the pager results before returning
+                limited_response = search_response
+                # Store the original pages iterator method
+                original_pages = limited_response.pages
+                
+                # Override the pages property with a custom iterator that respects max_limit
+                async def limited_pages_iterator():
+                    results_count = 0
+                    async for page in original_pages:
+                        yield page
+                        
+                        # Count results in this page
+                        if hasattr(page, 'results'):
+                            results_count += len(page.results)
+                        
+                        # Stop if we've reached max_limit
+                        if results_count >= max_limit:
+                            break
+                
+                # Replace the pages property with our custom iterator
+                limited_response.pages = limited_pages_iterator()
+                return limited_response
+            
         if parse_chunks_to_string:
             if content_search_spec_type=="chunks":
                 if parse_chunks_to_string:
@@ -931,7 +1010,7 @@ class DiscoveryEngineClient:
                     return big_string
                 
             elif content_search_spec_type=="documents":
-                big_string = await self.async_process_documents(search_response)
+                big_string = await self.async_process_documents(search_response, max_limit=max_limit)
                 log.info(f"Discovery engine documents string sample: {big_string[:100]}")
 
                 return big_string

{sunholo-0.134.8.dist-info → sunholo-0.136.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.134.8
+Version: 0.136.0
 Summary: AI DevOps - a package to help deploy GenAI to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.134.8.dist-info → sunholo-0.136.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sunholo/__init__.py,sha256=InRbX4V0-qdNHo9zYH3GEye7ASLR6LX8-SMvPV4Jsaw,1212
-sunholo/custom_logging.py,sha256=PrUXreBF0nZJMfa8A8cbFhTWoaOWWB_i6qwZfnjvRlo,18296
+sunholo/custom_logging.py,sha256=JXZTnXp_DixP3jwYfKw4LYRDS9IuTq7ctCgfZbI2rxA,22023
 sunholo/langchain_types.py,sha256=uZ4zvgej_f7pLqjtu4YP7qMC_eZD5ym_5x4pyvA1Ih4,1834
 sunholo/agents/__init__.py,sha256=X2I3pPkGeKWjc3d0QgSpkTyqD8J8JtrEWqwrumf1MMc,391
 sunholo/agents/chat_history.py,sha256=Gph_CdlP2otYnNdR1q1Umyyyvcad2F6K3LxU5yBQ9l0,5387
@@ -75,7 +75,7 @@ sunholo/discovery_engine/__init__.py,sha256=hLgqRDJ22Aov9o2QjAEfsVgnL3kMdM-g5p8R
 sunholo/discovery_engine/chunker_handler.py,sha256=wkvXl4rFtYfN6AZUKdW9_QD49Whf77BukDbO82UwlAg,7480
 sunholo/discovery_engine/cli.py,sha256=tsKqNSDCEsDTz5-wuNwjttb3Xt35D97-KyyEiaqolMQ,35628
 sunholo/discovery_engine/create_new.py,sha256=WUi4_xh_dFaGX3xA9jkNKZhaR6LCELjMPeRb0hyj4FU,1226
-sunholo/discovery_engine/discovery_engine_client.py,sha256=Sf7Sr6FYKA_jn19Ba2ENShrB1jnZ4HgPScuytDIuK9c,58705
+sunholo/discovery_engine/discovery_engine_client.py,sha256=xOcOj7D49bh7aKvDM1KujrFxu8dhgbFugSJv-fUPjt0,62444
 sunholo/discovery_engine/get_ai_search_chunks.py,sha256=I6Dt1CznqEvE7XIZ2PkLqopmjpO96iVEWJJqL5cJjOU,5554
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=did2pKkWM2o0KkRcb0H9l2x_WjCq6OyuHDxGbITFKPM,6530
@@ -168,9 +168,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.134.8.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.134.8.dist-info/METADATA,sha256=TQgU4vjziCS7pXHPGlAbdPeM0bmYYYFs3lgHmNFBjHg,10067
-sunholo-0.134.8.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-sunholo-0.134.8.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.134.8.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.134.8.dist-info/RECORD,,
+sunholo-0.136.0.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.136.0.dist-info/METADATA,sha256=VeoffpbDPqtye_pewu_LASUZo-79qT6Rpb4UjrG1TQ8,10067
+sunholo-0.136.0.dist-info/WHEEL,sha256=ooBFpIzZCPdw3uqIQsOo4qqbA4ZRPxHnOH7peeONza0,91
+sunholo-0.136.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.136.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.136.0.dist-info/RECORD,,

{sunholo-0.134.8.dist-info → sunholo-0.136.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.0)
+Generator: setuptools (80.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any