sunholo 0.126.4__py3-none-any.whl → 0.127.1__py3-none-any.whl

sunholo/agents/dispatch_to_qa.py

@@ -21,11 +21,6 @@ import traceback
 from .route import route_endpoint
 import os
 
-try:
-    from langfuse import Langfuse
-    langfuse = Langfuse()
-except ImportError:
-    langfuse = None
 
 def prep_request_payload(user_input, chat_history, vector_name, stream, **kwargs):
     """
@@ -93,7 +88,11 @@ def prep_request_payload(user_input, chat_history, vector_name, stream, **kwargs
     return qna_endpoint, qna_data
 
 def add_langfuse_trace(qna_endpoint):
-    if not langfuse:
+    try:
+        from langfuse import Langfuse
+        langfuse = Langfuse()
+    except Exception as err:
+        log.error(err)
         return None
     
     trace = langfuse.trace(name = f'dispatch/{os.path.basename(qna_endpoint)}')

sunholo/database/alloydb_client.py

@@ -882,4 +882,402 @@ class AlloyDBClient:
         
         log.info(f"Inserted data into table {table_name}")
         
-        return result
+        return result
+    
+    async def get_table_columns(self, table_name, schema="public"):
+        """
+        Fetch column information for an existing table.
+        
+        Args:
+            table_name (str): The table name to get columns for
+            schema (str): Database schema, defaults to "public"
+            
+        Returns:
+            List[dict]: List of column information dictionaries with keys:
+                - name: column name
+                - type: PostgreSQL data type
+                - is_nullable: whether the column allows NULL values
+                - default: default value if any
+        """
+        try:
+            query = f"""
+            SELECT 
+                column_name, 
+                data_type, 
+                is_nullable, 
+                column_default,
+                character_maximum_length
+            FROM 
+                information_schema.columns 
+            WHERE 
+                table_name = '{table_name}'
+                AND table_schema = '{schema}'
+            ORDER BY 
+                ordinal_position;
+            """
+            
+            if self.engine_type == "pg8000":
+                result = self._execute_sql_pg8000(query)
+                rows = result.fetchall() if hasattr(result, 'fetchall') else result
+            else:
+                rows = await self._execute_sql_async_langchain(query)
+            
+            columns = []
+            for row in rows:
+                column_info = {
+                    "name": row[0],
+                    "type": row[1],
+                    "is_nullable": row[2] == "YES",
+                    "default": row[3],
+                    "max_length": row[4]
+                }
+                columns.append(column_info)
+            
+            log.info(f"Retrieved {len(columns)} columns for table '{table_name}'")
+            return columns
+        
+        except Exception as e:
+            log.error(f"Error getting table columns: {e}")
+            return []
+
+    def map_data_to_columns(self, data, column_info, case_sensitive=False):
+        """
+        Map data dictionary to available table columns, handling case sensitivity.
+        
+        Args:
+            data (dict): Dictionary of data to map
+            column_info (list): List of column information dictionaries from get_table_columns
+            case_sensitive (bool): Whether to match column names case-sensitively
+            
+        Returns:
+            dict: Filtered data dictionary with only columns that exist in the table
+        """
+        if not column_info:
+            return data  # No column info, return original data
+        
+        # Create lookup dictionaries for columns
+        columns = {}
+        columns_lower = {}
+        
+        for col in column_info:
+            col_name = col["name"]
+            columns[col_name] = col
+            columns_lower[col_name.lower()] = col_name
+        
+        # Filter and map the data
+        filtered_data = {}
+        for key, value in data.items():
+            if case_sensitive:
+                # Case-sensitive matching
+                if key in columns:
+                    filtered_data[key] = value
+            else:
+                # Case-insensitive matching
+                key_lower = key.lower()
+                if key_lower in columns_lower:
+                    # Use the original column name from the database
+                    original_key = columns_lower[key_lower]
+                    filtered_data[original_key] = value
+        
+        return filtered_data
+
+    def safe_convert_value(self, value, target_type):
+        """
+        Safely convert a value to the target PostgreSQL type.
+        Handles various formats and placeholder values.
+        
+        Args:
+            value: The value to convert
+            target_type (str): PostgreSQL data type name
+            
+        Returns:
+            The converted value appropriate for the target type, or None if conversion fails
+        """
+        if value is None:
+            return None
+        
+        # Handle placeholder values
+        if isinstance(value, str):
+            if value.startswith("No ") or value.lower() in ("none", "n/a", "null", ""):
+                # Special placeholders are converted to None for most types
+                return None
+        
+        try:
+            # Handle different target types
+            if target_type in ("integer", "bigint", "smallint"):
+                if isinstance(value, (int, float)):
+                    return int(value)
+                elif isinstance(value, str) and value.strip():
+                    # Try to extract a number from the string
+                    cleaned = value.replace(',', '')
+                    # Extract the first number if there's text
+                    import re
+                    match = re.search(r'[-+]?\d+', cleaned)
+                    if match:
+                        return int(match.group())
+                return None
+                
+            elif target_type in ("numeric", "decimal", "real", "double precision"):
+                if isinstance(value, (int, float)):
+                    return float(value)
+                elif isinstance(value, str) and value.strip():
+                    # Remove currency symbols and try to convert
+                    cleaned = value.replace('$', '').replace('€', '').replace('£', '')
+                    cleaned = cleaned.replace(',', '.')
+                    # Extract the first number if there's text
+                    import re
+                    match = re.search(r'[-+]?\d+(\.\d+)?', cleaned)
+                    if match:
+                        return float(match.group())
+                return None
+                
+            elif target_type == "boolean":
+                if isinstance(value, bool):
+                    return value
+                elif isinstance(value, (int, float)):
+                    return bool(value)
+                elif isinstance(value, str):
+                    value_lower = value.lower()
+                    if value_lower in ("true", "t", "yes", "y", "1"):
+                        return True
+                    elif value_lower in ("false", "f", "no", "n", "0"):
+                        return False
+                return None
+                
+            elif target_type.startswith("timestamp"):
+                if isinstance(value, str):
+                    # For dates, keep the string format - DB driver will handle conversion
+                    return value
+                # Other types, just return as is
+                return value
+                
+            elif target_type == "jsonb" or target_type == "json":
+                if isinstance(value, (dict, list)):
+                    return json.dumps(value)
+                elif isinstance(value, str):
+                    # Validate it's valid JSON
+                    try:
+                        json.loads(value)
+                        return value
+                    except:
+                        return None
+                return None
+                
+            else:
+                # For text and other types, convert to string
+                if isinstance(value, (dict, list)):
+                    return json.dumps(value)
+                elif value is not None:
+                    return str(value)
+                return None
+                    
+        except Exception as e:
+            log.debug(f"Conversion error for value '{value}' to {target_type}: {e}")
+            return None
+
+    async def insert_rows_safely(self, table_name, rows, metadata=None, continue_on_error=False):
+        """
+        Insert multiple rows into a table with error handling for individual rows.
+        
+        Args:
+            table_name (str): The table to insert into
+            rows (list): List of dictionaries containing row data
+            metadata (dict, optional): Additional metadata to include in each row
+            continue_on_error (bool): Whether to continue if some rows fail
+            
+        Returns:
+            dict: {
+                'success': bool,
+                'total_rows': int,
+                'inserted_rows': int,
+                'failed_rows': int,
+                'errors': list of errors with row data
+            }
+        """
+        if not rows:
+            return {'success': True, 'total_rows': 0, 'inserted_rows': 0, 'failed_rows': 0, 'errors': []}
+        
+        # Get table columns for mapping and type conversion
+        columns = await self.get_table_columns(table_name)
+        column_map = {col['name']: col for col in columns}
+        column_map_lower = {col['name'].lower(): col for col in columns}
+        
+        results = {
+            'success': True,
+            'total_rows': len(rows),
+            'inserted_rows': 0,
+            'failed_rows': 0,
+            'errors': []
+        }
+        
+        for i, row in enumerate(rows):
+            try:
+                # Map row data to actual table columns
+                filtered_row = {}
+                
+                # First, do case-insensitive mapping
+                for key, value in row.items():
+                    key_lower = key.lower()
+                    if key_lower in column_map_lower:
+                        col_info = column_map_lower[key_lower]
+                        col_name = col_info['name']  # Use the correct case from DB
+                        col_type = col_info['type']
+                        
+                        # Try to convert value to the appropriate type
+                        converted_value = self.safe_convert_value(value, col_type)
+                        filtered_row[col_name] = converted_value
+                
+                # Add metadata if provided
+                if metadata:
+                    for key, value in metadata.items():
+                        key_lower = key.lower()
+                        if key_lower in column_map_lower:
+                            col_name = column_map_lower[key_lower]['name']
+                            filtered_row[col_name] = value
+                
+                # Insert the row
+                result = await self._insert_single_row(table_name, filtered_row)
+                results['inserted_rows'] += 1
+                
+            except Exception as e:
+                error_info = {
+                    'row_index': i,
+                    'error': str(e),
+                    'row_data': row
+                }
+                results['errors'].append(error_info)
+                results['failed_rows'] += 1
+                
+                log.error(f"Error inserting row {i}: {e}")
+                
+                if not continue_on_error:
+                    results['success'] = False
+                    return results
+        
+        # Overall success is true if any rows were inserted successfully
+        results['success'] = results['inserted_rows'] > 0
+        return results
+
+    async def create_table_with_columns(self, table_name, column_definitions, if_not_exists=True):
+        """
+        Create a table with explicit column definitions.
+        
+        Args:
+            table_name (str): The name of the table to create
+            column_definitions (list): List of column definition dictionaries:
+                - name: Column name
+                - type: PostgreSQL data type
+                - nullable: Whether column allows NULL (default True)
+                - default: Default value expression (optional)
+                - primary_key: Whether this is a primary key (default False)
+            if_not_exists (bool): Whether to use IF NOT EXISTS clause
+            
+        Returns:
+            Result of the execution
+        """
+        if not column_definitions:
+            raise ValueError("No column definitions provided")
+        
+        # Generate column definition strings
+        column_strs = []
+        
+        # Check if we need to add a serial primary key
+        has_primary_key = any(col.get('primary_key', False) for col in column_definitions)
+        
+        if not has_primary_key:
+            # Add an ID column as primary key
+            column_strs.append("id SERIAL PRIMARY KEY")
+        
+        for col in column_definitions:
+            col_name = col.get('name')
+            col_type = col.get('type', 'TEXT')
+            nullable = col.get('nullable', True)
+            default = col.get('default')
+            primary_key = col.get('primary_key', False)
+            
+            if not col_name:
+                continue
+                
+            # Build the column definition
+            col_def = f'"{col_name}" {col_type}'
+            
+            if primary_key:
+                col_def += " PRIMARY KEY"
+                
+            if not nullable:
+                col_def += " NOT NULL"
+                
+            if default is not None:
+                col_def += f" DEFAULT {default}"
+                
+            column_strs.append(col_def)
+        
+        # Create the SQL statement
+        exists_clause = "IF NOT EXISTS " if if_not_exists else ""
+        columns_sql = ",\n    ".join(column_strs)
+        
+        create_table_sql = f"""
+        CREATE TABLE {exists_clause}"{table_name}" (
+        {columns_sql}
+        )
+        """
+        
+        # Execute the SQL based on engine type
+        log.info(f"Creating table '{table_name}' with explicit column definitions")
+        try:
+            if self.engine_type == "pg8000":
+                result = self._execute_sql_pg8000(create_table_sql)
+            else:
+                result = await self._execute_sql_async_langchain(create_table_sql)
+                
+            log.info(f"Table '{table_name}' created successfully")
+            return result
+        except Exception as e:
+            log.error(f"Error creating table: {e}")
+            raise
+
+    def _get_sql_type_safe(self, value):
+        """
+        Enhanced version of _get_sql_type with better type detection.
+        Handles placeholder values and common patterns.
+        
+        Args:
+            value: The value to determine the column type
+            
+        Returns:
+            str: SQL type
+        """
+        if value is None:
+            return "TEXT"
+        
+        # Handle placeholder values
+        if isinstance(value, str) and (value.startswith("No ") or value.lower() in ("none", "n/a", "null", "")):
+            return "TEXT"  # Always use TEXT for placeholder values
+        
+        if isinstance(value, dict):
+            return "JSONB"
+        elif isinstance(value, list):
+            return "JSONB"
+        elif isinstance(value, bool):
+            return "BOOLEAN"
+        elif isinstance(value, int):
+            return "INTEGER"
+        elif isinstance(value, float):
+            return "NUMERIC"
+        else:
+            # Check if it's a date string
+            if isinstance(value, str):
+                # Try to detect date formats
+                value_lower = value.lower()
+                if len(value) in (8, 10) and ('-' in value or '/' in value):
+                    # Likely a date (YYYY-MM-DD or MM/DD/YYYY)
+                    return "DATE"
+                elif 'date' in value_lower or 'time' in value_lower:
+                    # Column name hint suggests it's a date
+                    return "TIMESTAMP"
+                elif any(currency in value for currency in ('$', '€', '£')):
+                    # Likely a monetary value
+                    return "NUMERIC"
+                
+            # Default to TEXT
+            return "TEXT"

sunholo/discovery_engine/discovery_engine_client.py

@@ -614,6 +614,7 @@ class DiscoveryEngineClient:
             reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,
         )
 
+        log.debug(f"Making import_document_request: {request}")
         return self._import_document_request(request)
 
     def _create_unique_gsuri_docid(self, gcs_uri:str):

sunholo/utils/config_class.py

@@ -112,18 +112,6 @@ class ConfigManager:
         """
         Helper function to load a config file and update the cache.
 
-        Args:
-            config_file (str): The path to the configuration file.
-            filename (str): The name of the configuration file.
-            is_local (bool): Indicates if the config file is from the local folder.
-
-        Returns:
-            dict: The loaded configuration.
-        """
-    def _reload_config_file(self, config_file, filename, is_local=False):
-        """
-        Helper function to load a config file and update the cache.
-
         Args:
             config_file (str): The path to the configuration file.
             filename (str): The name of the configuration file.
@@ -244,4 +232,23 @@ class ConfigManager:
             return agents[key]
         else:
             return agents.get("default")
+    
+    def permissionConfig(self, key: str):
+        """
+        Fetch a key from 'permissionConfig' kind configuration.
 
+        Args:
+            key (str): The key to fetch from the configuration.
+
+        Returns:
+            str: The value associated with the specified key.
+        """        
+        self._check_and_reload_configs()
+        config = self.configs_by_kind.get('permissionConfig')
+        if not config:
+            return None
+        agents = config.get('permissions')
+        if key in agents:
+            return agents[key]
+        else:
+            return None

{sunholo-0.126.4.dist-info → sunholo-0.127.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.126.4
+Version: 0.127.1
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0
@@ -142,6 +142,7 @@ Requires-Dist: google-cloud-discoveryengine>=0.13.4; extra == "gcp"
 Requires-Dist: google-cloud-texttospeech; extra == "gcp"
 Requires-Dist: google-genai>=0.2.2; extra == "gcp"
 Requires-Dist: google-generativeai>=0.8.3; extra == "gcp"
+Requires-Dist: langchain; extra == "gcp"
 Requires-Dist: langchain-google-genai>=2.0.0; extra == "gcp"
 Requires-Dist: langchain_google_alloydb_pg>=0.2.2; extra == "gcp"
 Requires-Dist: langchain-google-vertexai; extra == "gcp"
@@ -164,6 +165,7 @@ Requires-Dist: flask; extra == "http"
 Requires-Dist: gunicorn; extra == "http"
 Requires-Dist: httpcore; extra == "http"
 Requires-Dist: httpx; extra == "http"
+Requires-Dist: langchain; extra == "http"
 Requires-Dist: langfuse; extra == "http"
 Requires-Dist: python-socketio; extra == "http"
 Requires-Dist: requests; extra == "http"

{sunholo-0.126.4.dist-info → sunholo-0.127.1.dist-info}/RECORD

@@ -3,7 +3,7 @@ sunholo/custom_logging.py,sha256=YfIN1oP3dOEkkYkyRBU8BGS3uJFGwUDsFCl8mIVbwvE,122
 sunholo/langchain_types.py,sha256=uZ4zvgej_f7pLqjtu4YP7qMC_eZD5ym_5x4pyvA1Ih4,1834
 sunholo/agents/__init__.py,sha256=X2I3pPkGeKWjc3d0QgSpkTyqD8J8JtrEWqwrumf1MMc,391
 sunholo/agents/chat_history.py,sha256=Gph_CdlP2otYnNdR1q1Umyyyvcad2F6K3LxU5yBQ9l0,5387
-sunholo/agents/dispatch_to_qa.py,sha256=AwLS41oK6iS8xre-HuWjS4jj9dvU-evWI58EYfG65fg,8879
+sunholo/agents/dispatch_to_qa.py,sha256=NHihwAoCJ5_Lk11e_jZnucVUGQyZHCB-YpkfMHBCpQk,8882
 sunholo/agents/langserve.py,sha256=C46ph2mnygr6bdHijYWYyfQDI9ylAF0_9Kx2PfcCJpU,4414
 sunholo/agents/pubsub.py,sha256=TscZN_6am6DfaQkC-Yl18ZIBOoLE-0nDSiil6GpQEh4,1344
 sunholo/agents/route.py,sha256=mV8tGABbSqcg3PQL02MgQOs41gKEHLMyIJJJcTuFdbE,2988
@@ -60,7 +60,7 @@ sunholo/components/retriever.py,sha256=Wmchv3huAM4w7DIS-a5Lp9Hi7M8pE6vZdxgseiT9S
 sunholo/components/vectorstore.py,sha256=k7GS1Y5c6ZGXSDAJvyCes6dTjhDAi0fjGbVLqpyfzBc,5918
 sunholo/database/__init__.py,sha256=bpB5Nk21kwqYj-qdVnvNgXjLsbflnH4g-San7OHMqR4,283
 sunholo/database/alloydb.py,sha256=x1zUMB-EVWbE2Zvp4nAs2Z-tB_kOZmS45H2lwVHdYnk,11678
-sunholo/database/alloydb_client.py,sha256=Ih_9nd_5fXzki02fNhssn5Grg5-GcQrfmNjqv1wLK7A,34788
+sunholo/database/alloydb_client.py,sha256=B_vCN9d2wQj77TGoyHAMryCNKljKt0ehtXNTdASqTIk,50297
 sunholo/database/database.py,sha256=VqhZdkXUNdvWn8sUcUV3YNby1JDVf7IykPVXWBtxo9U,7361
 sunholo/database/lancedb.py,sha256=DyfZntiFKBlVPaFooNN1Z6Pl-LAs4nxWKKuq8GBqN58,715
 sunholo/database/static_dbs.py,sha256=8cvcMwUK6c32AS2e_WguKXWMkFf5iN3g9WHzsh0C07Q,442
@@ -75,7 +75,7 @@ sunholo/discovery_engine/__init__.py,sha256=hLgqRDJ22Aov9o2QjAEfsVgnL3kMdM-g5p8R
 sunholo/discovery_engine/chunker_handler.py,sha256=44qlTpdtz2GKzrhoQrxVMk-RPVFp7vQDPJoe9KmCcsw,7517
 sunholo/discovery_engine/cli.py,sha256=KGVle5rkLL49oF9TQhrGI--8017IvvLOEoYur545Qb0,12790
 sunholo/discovery_engine/create_new.py,sha256=WUi4_xh_dFaGX3xA9jkNKZhaR6LCELjMPeRb0hyj4FU,1226
-sunholo/discovery_engine/discovery_engine_client.py,sha256=8jebH3cccdGxl1XO5txjj0cA1JPgzEZmYSfv9z86UdA,37271
+sunholo/discovery_engine/discovery_engine_client.py,sha256=0KhKRFKCvqvtkUOIrCXNk5353t9duuEtUQDhQnN2B24,37335
 sunholo/discovery_engine/get_ai_search_chunks.py,sha256=I6Dt1CznqEvE7XIZ2PkLqopmjpO96iVEWJJqL5cJjOU,5554
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=did2pKkWM2o0KkRcb0H9l2x_WjCq6OyuHDxGbITFKPM,6530
@@ -151,7 +151,7 @@ sunholo/utils/__init__.py,sha256=Hv02T5L2zYWvCso5hzzwm8FQogwBq0OgtUbN_7Quzqc,89
 sunholo/utils/api_key.py,sha256=Ct4bIAQZxzPEw14hP586LpVxBAVi_W9Serpy0BK-7KI,244
 sunholo/utils/big_context.py,sha256=HuP9_r_Nx1jvZHxjMEihgoZAXmnCh80zzsj1fq3mIOg,6021
 sunholo/utils/config.py,sha256=bz0ODJyqnoHQIsk4pmNpVxxq5WvwS0SfOq4cnCjQPJk,9105
-sunholo/utils/config_class.py,sha256=Z4sGzEkuxlCAJ8b-65_yzLmybnunywwOD9eXL8an5Wg,9619
+sunholo/utils/config_class.py,sha256=zhp71gNZb-t1cbkN2N3zwCAEiX0FSvKaKlpBxosgI4U,9750
 sunholo/utils/config_schema.py,sha256=Wv-ncitzljOhgbDaq9qnFqH5LCuxNv59dTGDWgd1qdk,4189
 sunholo/utils/gcp.py,sha256=lus1HH8YhFInw6QRKwfvKZq-Lz-2KQg4ips9v1I_3zE,4783
 sunholo/utils/gcp_project.py,sha256=Fa0IhCX12bZ1ctF_PKN8PNYd7hihEUfb90kilBfUDjg,1411
@@ -168,9 +168,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.126.4.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.126.4.dist-info/METADATA,sha256=RInwd4S0z69DV7OLnEaqcJV3CiQXzoJ1OOJl_AZTXBQ,10001
-sunholo-0.126.4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-sunholo-0.126.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.126.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.126.4.dist-info/RECORD,,
+sunholo-0.127.1.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.127.1.dist-info/METADATA,sha256=kKvoiijhfyGLL7CfJOEYZGYiCuIAZ6m2hGagaNjfCAQ,10084
+sunholo-0.127.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+sunholo-0.127.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.127.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.127.1.dist-info/RECORD,,