npcpy 1.2.20__py3-none-any.whl → 1.2.22__py3-none-any.whl

npcpy/data/load.py

@@ -132,7 +132,7 @@ def load_file_contents(file_path, chunk_size=None):
         elif file_ext in ['XLS', 'XLSX']:
             df = load_excel(file_path)
             full_content = df.to_string()
-        elif file_ext in ['TXT', 'MD', 'PY', 'JSX', 'TSX', 'TS', 'JS', 'JSON', 'SQL', 'NPC', 'JINX', 'LINE', 'YAML']:
+        elif file_ext in ['TXT', 'MD', 'PY', 'JSX', 'TSX', 'TS', 'JS', 'JSON', 'SQL', 'NPC', 'JINX', 'LINE', 'YAML', 'DART', 'JAVA']:
             full_content = load_txt(file_path)
         elif file_ext == 'JSON':
             data = load_json(file_path)

npcpy/gen/response.py

@@ -475,9 +475,9 @@ def get_litellm_response(
                         pdf_data = load_pdf(attachment)
                         if pdf_data is not None:
                             if prompt:
-                                prompt += f"\n\nContent from PDF: {os.path.basename(attachment)}\n{pdf_data[:5000]}..."
+                                prompt += f"\n\nContent from PDF: {os.path.basename(attachment)}\n{pdf_data}..."
                             else:
-                                prompt = f"Content from PDF: {os.path.basename(attachment)}\n{pdf_data[:5000]}..."
+                                prompt = f"Content from PDF: {os.path.basename(attachment)}\n{pdf_data}..."
 
                     except Exception:
                         pass

npcpy/npc_compiler.py

@@ -1908,7 +1908,8 @@ class Team:
             "dataframes": {},
             "memories": {},          
             "execution_history": [],   
-            "npc_messages": {}                 
+            "npc_messages": {},
+            "context":''       
             }
                 
         if team_path:
@@ -2017,11 +2018,12 @@ class Team:
                         self.databases = ctx_data['databases']
                     else:
                         self.databases = []
-                    if 'context' in ctx_data:
-                        self.context = ctx_data['context']
-                    else:
-                        self.context = ''
-
+                    
+                    base_context = ctx_data.get('context', '')
+                    self.shared_context['context'] = base_context
+                    if 'file_patterns' in ctx_data:
+                        file_cache = self._parse_file_patterns(ctx_data['file_patterns'])
+                        self.shared_context['files'] = file_cache
                     if 'preferences' in ctx_data:
                         self.preferences = ctx_data['preferences']
                     else:
@@ -2031,7 +2033,7 @@ class Team:
                     else:
                         self.forenpc = self.npcs[list(self.npcs.keys())[0]] if self.npcs else None
                     for key, item in ctx_data.items():
-                        if key not in ['name', 'mcp_servers', 'databases', 'context']:
+                        if key not in ['name', 'mcp_servers', 'databases', 'context', 'file_patterns']:
                             self.shared_context[key] = item
                 return ctx_data
         return {}
@@ -2288,6 +2290,85 @@ class Team:
             team.save(team_dir)
             
         return True
+    def _parse_file_patterns(self, patterns_config):
+        """Parse file patterns configuration and load matching files into KV cache"""
+        if not patterns_config:
+            return {}
+        
+        file_cache = {}
+        
+        for pattern_entry in patterns_config:
+            if isinstance(pattern_entry, str):
+                pattern_entry = {"pattern": pattern_entry}
+            
+            pattern = pattern_entry.get("pattern", "")
+            recursive = pattern_entry.get("recursive", False)
+            base_path = pattern_entry.get("base_path", ".")
+            
+            if not pattern:
+                continue
+                
+            base_path = os.path.expanduser(base_path)
+            if not os.path.isabs(base_path):
+                base_path = os.path.join(self.team_path or os.getcwd(), base_path)
+            
+            matching_files = self._find_matching_files(pattern, base_path, recursive)
+            
+            for file_path in matching_files:
+                file_content = self._load_file_content(file_path)
+                if file_content:
+                    relative_path = os.path.relpath(file_path, base_path)
+                    file_cache[relative_path] = file_content
+        
+        return file_cache
+
+    def _find_matching_files(self, pattern, base_path, recursive=False):
+        """Find files matching the given pattern"""
+        matching_files = []
+        
+        if not os.path.exists(base_path):
+            return matching_files
+        
+        if recursive:
+            for root, dirs, files in os.walk(base_path):
+                for filename in files:
+                    if fnmatch.fnmatch(filename, pattern):
+                        matching_files.append(os.path.join(root, filename))
+        else:
+            try:
+                for item in os.listdir(base_path):
+                    item_path = os.path.join(base_path, item)
+                    if os.path.isfile(item_path) and fnmatch.fnmatch(item, pattern):
+                        matching_files.append(item_path)
+            except PermissionError:
+                print(f"Permission denied accessing {base_path}")
+        
+        return matching_files
+
+    def _load_file_content(self, file_path):
+        """Load content from a file with error handling"""
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                return f.read()
+        except Exception as e:
+            print(f"Error reading {file_path}: {e}")
+            return None
+
+
+    def _format_parsed_files_context(self, parsed_files):
+        """Format parsed files into context string"""
+        if not parsed_files:
+            return ""
+        
+        context_parts = ["Additional context from files:"]
+        
+        for file_path, content in parsed_files.items():
+            context_parts.append(f"\n--- {file_path} ---")
+            context_parts.append(content)
+            context_parts.append("")
+        
+        return "\n".join(context_parts)
+
 class Pipeline:
     def __init__(self, pipeline_data=None, pipeline_path=None, npc_team=None):
         """Initialize a pipeline from data or file path"""

npcpy/serve.py

@@ -443,7 +443,7 @@ def get_global_settings():
             "embedding_model": "nomic-embed-text",
             "embedding_provider": "ollama",
             "search_provider": "perplexity",
-            "NPCSH_LICENSE_KEY": "",
+            "NPC_STUDIO_LICENSE_KEY": "",
             "default_folder": os.path.expanduser("~/.npcsh/"),
         }
         global_vars = {}
@@ -479,7 +479,7 @@ def get_global_settings():
                         "NPCSH_EMBEDDING_MODEL": "embedding_model",
                         "NPCSH_EMBEDDING_PROVIDER": "embedding_provider",
                         "NPCSH_SEARCH_PROVIDER": "search_provider",
-                        "NPCSH_LICENSE_KEY": "NPCSH_LICENSE_KEY",
+                        "NPC_STUDIO_LICENSE_KEY": "NPC_STUDIO_LICENSE_KEY",
                         "NPCSH_STREAM_OUTPUT": "NPCSH_STREAM_OUTPUT",
                         "NPC_STUDIO_DEFAULT_FOLDER": "default_folder",
                     }
@@ -521,7 +521,7 @@ def save_global_settings():
             "embedding_model": "NPCSH_EMBEDDING_MODEL",
             "embedding_provider": "NPCSH_EMBEDDING_PROVIDER",
             "search_provider": "NPCSH_SEARCH_PROVIDER",
-            "NPCSH_LICENSE_KEY": "NPCSH_LICENSE_KEY",
+            "NPC_STUDIO_LICENSE_KEY": "NPC_STUDIO_LICENSE_KEY",
             "NPCSH_STREAM_OUTPUT": "NPCSH_STREAM_OUTPUT",
             "default_folder": "NPC_STUDIO_DEFAULT_FOLDER",
         }

npcpy/sql/ai_function_tools.py

@@ -0,0 +1,257 @@
+import json
+from typing import Dict, Any, List, Optional
+
+class SQLToolCallResponse:
+    """
+    Represents a structured response with tool calling capabilities 
+    that can be generated and processed within SQL
+    """
+    @staticmethod
+    def generate_tool_call_prompt(
+        prompt: str, 
+        tools: List[Dict[str, Any]], 
+        model: str = 'snowflake-arctic'
+    ) -> str:
+        """
+        Generate a prompt that instructs the model to use tools
+        
+        :param prompt: Original user prompt
+        :param tools: List of available tools/functions
+        :param model: AI model to use
+        :return: Formatted prompt for tool-aware generation
+        """
+        tool_descriptions = [
+            f"Tool: {tool.get('name', 'unnamed')}\n"
+            f"Description: {tool.get('description', 'No description')}\n"
+            f"Parameters: {json.dumps(tool.get('parameters', {}))}"
+            for tool in tools
+        ]
+        
+        return f"""
+        You are an AI assistant capable of using the following tools:
+
+        {"\n\n".join(tool_descriptions)}
+
+        User Prompt: {prompt}
+
+        IMPORTANT INSTRUCTIONS:
+        1. Carefully analyze the user's request
+        2. Determine which tool(s) are most appropriate
+        3. Generate a structured JSON response with:
+           - tool_calls: List of tool invocations
+           - final_response: Your overall response to the user
+        4. ONLY use tools that are directly relevant
+        5. Format the output as a valid JSON object
+
+        Output Format:
+        {{
+            "tool_calls": [
+                {{
+                    "tool_name": "tool_name",
+                    "parameters": {{...}}
+                }}
+            ],
+            "final_response": "Optional explanation or summary"
+        }}
+        """
+
+    @staticmethod
+    def parse_tool_calls_sql(tool_call_json: str) -> Dict[str, Any]:
+        """
+        Parse tool calls within SQL, with error handling
+        
+        :param tool_call_json: JSON string of tool calls
+        :return: Parsed tool call dictionary
+        """
+        try:
+            parsed = json.loads(tool_call_json)
+            return {
+                'tool_calls': parsed.get('tool_calls', []),
+                'final_response': parsed.get('final_response', '')
+            }
+        except json.JSONDecodeError:
+            return {
+                'tool_calls': [],
+                'final_response': 'Error parsing tool calls'
+            }
+
+class SnowflakeSQLToolCaller:
+    """
+    Snowflake-specific tool calling implementation
+    """
+    @staticmethod
+    def generate_tool_call_sql(
+        prompt: str, 
+        tools: List[Dict[str, Any]], 
+        model: str = 'snowflake-arctic'
+    ) -> str:
+        """
+        Generate a SQL function that performs tool calling
+        
+        :param prompt: User prompt
+        :param tools: Available tools
+        :param model: AI model to use
+        :return: SQL function definition
+        """
+        tool_call_prompt = SQLToolCallResponse.generate_tool_call_prompt(
+            prompt, tools, model
+        )
+        
+        return f"""
+        WITH ai_response AS (
+            SELECT SNOWFLAKE.CORTEX.COMPLETE(
+                model => '{model}',
+                prompt => '{tool_call_prompt}'
+            ) AS response_json
+        ),
+        parsed_response AS (
+            SELECT 
+                response_json,
+                PARSE_JSON(response_json) AS parsed_json
+            FROM ai_response
+        ),
+        tool_calls AS (
+            SELECT 
+                elem.tool_name,
+                elem.parameters
+            FROM parsed_response,
+            LATERAL FLATTEN(input => parsed_json:tool_calls) elem
+        )
+        SELECT 
+            response_json,
+            tool_calls.tool_name,
+            tool_calls.parameters
+        FROM parsed_response
+        LEFT JOIN tool_calls ON 1=1
+        """
+
+class BigQuerySQLToolCaller:
+    """
+    BigQuery-specific tool calling implementation
+    """
+    @staticmethod
+    def generate_tool_call_sql(
+        prompt: str, 
+        tools: List[Dict[str, Any]], 
+        model: str = 'text-bison'
+    ) -> str:
+        """
+        Generate a BigQuery ML function for tool calling
+        
+        :param prompt: User prompt
+        :param tools: Available tools
+        :param model: AI model to use
+        :return: SQL function definition
+        """
+        tool_call_prompt = SQLToolCallResponse.generate_tool_call_prompt(
+            prompt, tools, model
+        )
+        
+        return f"""
+        ML.PREDICT(
+            MODEL `{model}`,
+            (
+                SELECT '{tool_call_prompt}' AS prompt
+            )
+        )
+        """
+
+class SQLToolCallOrchestrator:
+    """
+    Orchestrates tool calling across different SQL databases
+    """
+    @staticmethod
+    def generate_tool_calls(
+        engine_type: str,
+        prompt: str, 
+        tools: List[Dict[str, Any]], 
+        model: Optional[str] = None
+    ) -> str:
+        """
+        Generate appropriate SQL for tool calling
+        
+        :param engine_type: Type of SQL database
+        :param prompt: User prompt
+        :param tools: Available tools
+        :param model: Optional model override
+        :return: SQL for tool calling
+        """
+        model_map = {
+            'snowflake': 'snowflake-arctic',
+            'bigquery': 'text-bison'
+        }
+        
+        model = model or model_map.get(engine_type.lower(), 'snowflake-arctic')
+        
+        if engine_type.lower() == 'snowflake':
+            return SnowflakeSQLToolCaller.generate_tool_call_sql(
+                prompt, tools, model
+            )
+        elif engine_type.lower() == 'bigquery':
+            return BigQuerySQLToolCaller.generate_tool_call_sql(
+                prompt, tools, model
+            )
+        else:
+            raise ValueError(f"Unsupported engine type: {engine_type}")
+
+# Example integration with ModelCompiler
+def _execute_ai_agent_sql(
+    self, 
+    prompt: str, 
+    tools: List[Dict[str, Any]]
+) -> Dict[str, Any]:
+    """
+    Execute an AI agent entirely within SQL
+    
+    :param prompt: User prompt
+    :param tools: Available tools
+    :return: Tool call results
+    """
+    engine_type = self.engine.dialect.name.lower()
+    
+    try:
+        tool_call_sql = SQLToolCallOrchestrator.generate_tool_calls(
+            engine_type, prompt, tools
+        )
+        
+        # Execute the SQL and process results
+        df = pd.read_sql(tool_call_sql, self.engine)
+        
+        # Process tool calls and generate final response
+        tool_calls = self._process_sql_tool_calls(df)
+        
+        return {
+            'tool_calls': tool_calls,
+            'final_response': df['final_response'].iloc[0] if not df.empty else ''
+        }
+    
+    except Exception as e:
+        return {
+            'tool_calls': [],
+            'final_response': f"Error in SQL tool calling: {str(e)}"
+        }
+
+def _process_sql_tool_calls(self, df: pd.DataFrame) -> List[Dict[str, Any]]:
+    """
+    Process tool calls from SQL result DataFrame
+    
+    :param df: DataFrame containing tool call results
+    :return: List of processed tool calls
+    """
+    processed_calls = []
+    
+    for _, row in df.iterrows():
+        tool_name = row.get('tool_name')
+        parameters = row.get('parameters')
+        
+        if tool_name and parameters:
+            # Execute the tool using existing tool calling mechanism
+            tool_result = self._execute_tool(tool_name, parameters)
+            
+            processed_calls.append({
+                'tool_name': tool_name,
+                'parameters': parameters,
+                'result': tool_result
+            })
+    
+    return processed_calls

npcpy/sql/database_ai_adapters.py

@@ -0,0 +1,186 @@
+import sqlalchemy
+from typing import Dict, Any, Optional, Callable
+import textwrap
+
+class DatabaseAIAdapter:
+    """
+    Base class for database-specific AI function adapters
+    """
+    def __init__(self, engine: sqlalchemy.engine.base.Engine):
+        self.engine = engine
+        self.dialect = self._get_dialect()
+
+    def _get_dialect(self) -> str:
+        """Determine the specific database dialect"""
+        dialect_map = {
+            'postgresql': 'postgresql',
+            'mysql': 'mysql',
+            'mssql': 'mssql',
+            'sqlite': 'sqlite',
+            'snowflake': 'snowflake'
+        }
+        return dialect_map.get(self.engine.dialect.name.lower(), 'unknown')
+
+    def generate_ai_function(self, function_type: str, prompt: str, **kwargs) -> str:
+        """
+        Generate AI function implementation based on database type
+        
+        :param function_type: Type of AI function (generate_text, summarize, etc.)
+        :param prompt: Input prompt
+        :param kwargs: Additional parameters
+        :return: SQL implementation of AI function
+        """
+        adapter_method = getattr(self, f'_{self.dialect}_{function_type}', None)
+        
+        if adapter_method:
+            return adapter_method(prompt, **kwargs)
+        
+        # Fallback to generic implementation
+        return self._generic_ai_function(function_type, prompt, **kwargs)
+
+    def _generic_ai_function(self, function_type: str, prompt: str, **kwargs) -> str:
+        """
+        Generic fallback implementation using Python-based AI processing
+        """
+        # Create a temporary table-based approach for AI function simulation
+        return textwrap.dedent(f'''
+        WITH ai_input AS (
+            SELECT '{prompt}' AS input_text
+        )
+        SELECT 
+            CASE 
+                WHEN '{function_type}' = 'generate_text' THEN 
+                    'Generated text based on: ' || input_text
+                WHEN '{function_type}' = 'summarize' THEN 
+                    'Summary of: ' || input_text
+                WHEN '{function_type}' = 'analyze_sentiment' THEN 
+                    CASE 
+                        WHEN input_text LIKE '%good%' OR input_text LIKE '%great%' THEN 'positive'
+                        WHEN input_text LIKE '%bad%' OR input_text LIKE '%terrible%' THEN 'negative'
+                        ELSE 'neutral'
+                    END
+                ELSE input_text
+            END AS ai_result
+        FROM ai_input
+        ''')
+
+    def _postgresql_generate_text(self, prompt: str, **kwargs) -> str:
+        """
+        PostgreSQL-specific text generation using pgai extension
+        Requires: CREATE EXTENSION IF NOT EXISTS pgai;
+        """
+        return textwrap.dedent(f'''
+        SELECT pgai.generate_text(
+            model => 'openai-gpt-3.5-turbo', 
+            prompt => '{prompt}'
+        ) AS generated_text
+        ''')
+
+    def _mysql_generate_text(self, prompt: str, **kwargs) -> str:
+        """
+        MySQL-specific text generation 
+        Uses a custom table-based approach with external AI call simulation
+        """
+        return textwrap.dedent(f'''
+        WITH ai_input AS (
+            SELECT '{prompt}' AS input_text
+        )
+        SELECT 
+            CONCAT('Generated text based on: ', input_text) AS generated_text
+        FROM ai_input
+        ''')
+
+    def _mssql_generate_text(self, prompt: str, **kwargs) -> str:
+        """
+        MSSQL-specific text generation
+        Uses a CLR integration or external call simulation
+        """
+        return textwrap.dedent(f'''
+        WITH ai_input AS (
+            SELECT '{prompt}' AS input_text
+        )
+        SELECT 
+            CONCAT('Generated text based on: ', input_text) AS generated_text
+        FROM ai_input
+        ''')
+
+    def _postgresql_summarize(self, text: str, **kwargs) -> str:
+        """PostgreSQL summarization implementation"""
+        return textwrap.dedent(f'''
+        SELECT pgai.summarize(
+            text => '{text}',
+            max_length => 100
+        ) AS summary
+        ''')
+
+    def _postgresql_analyze_sentiment(self, text: str, **kwargs) -> str:
+        """PostgreSQL sentiment analysis implementation"""
+        return textwrap.dedent(f'''
+        SELECT 
+            CASE 
+                WHEN pgai.sentiment_score('{text}') > 0 THEN 'positive'
+                WHEN pgai.sentiment_score('{text}') < 0 THEN 'negative'
+                ELSE 'neutral'
+            END AS sentiment
+        ''')
+
+class AIFunctionRouter:
+    """
+    Routes AI function calls to appropriate database-specific adapters
+    """
+    @staticmethod
+    def route_ai_function(engine: sqlalchemy.engine.base.Engine, 
+                           function_type: str, 
+                           prompt: str, 
+                           **kwargs) -> str:
+        """
+        Route AI function to appropriate database adapter
+        
+        :param engine: SQLAlchemy database engine
+        :param function_type: Type of AI function
+        :param prompt: Input prompt
+        :param kwargs: Additional parameters
+        :return: SQL implementation of AI function
+        """
+        adapter = DatabaseAIAdapter(engine)
+        return adapter.generate_ai_function(function_type, prompt, **kwargs)
+
+# Example integration with existing ModelCompiler
+def _execute_ai_model(self, sql: str, model: SQLModel) -> pd.DataFrame:
+    """
+    Enhanced method to use AI function adapters
+    """
+    from npcpy.sql.database_ai_adapters import AIFunctionRouter
+    
+    # Existing code to determine source and engine
+    source_pattern = r'FROM\s+(\\w+)\\.(\\w+)'
+    matches = re.findall(source_pattern, sql)
+    
+    if matches:
+        source_name, table_name = matches[0]
+        engine = self._get_engine(source_name)
+        
+        # Modify SQL to use database-specific AI functions
+        for func_name, params in model.ai_functions.items():
+            try:
+                # Route AI function through adapter
+                native_func_call = AIFunctionRouter.route_ai_function(
+                    engine,
+                    func_name, 
+                    text=params.get('column', ''),
+                    **{k: v for k, v in params.items() if k != 'column'}
+                )
+                
+                # Replace the NQL function with native/adapted function
+                sql = sql.replace(
+                    f"nql.{func_name}({params.get('column', '')})", 
+                    native_func_call
+                )
+            except Exception as e:
+                # Fallback to original method if transformation fails
+                print(f"Warning: AI function adaptation failed: {e}. Falling back to default.")
+        
+        return pd.read_sql(sql.replace(f"{source_name}.", ""), engine)
+    
+    # Fallback to existing AI model execution
+    return super()._execute_ai_model(sql, model)

npcpy/sql/database_ai_functions.py

@@ -0,0 +1,163 @@
+from typing import Dict, Callable, Any
+
+class DatabaseAIFunctionMapper:
+    @staticmethod
+    def get_snowflake_cortex_mapping() -> Dict[str, Dict[str, Any]]:
+        """
+        Map NPC AI functions to Snowflake Cortex functions
+        
+        Structure:
+        {
+            'npc_function_name': {
+                'cortex_function': 'snowflake_cortex_function_name',
+                'transformer': optional_transformation_function
+            }
+        }
+        """
+        return {
+            # Text Generation Mapping
+            'generate_text': {
+                'cortex_function': 'COMPLETE',
+                'transformer': lambda prompt, **kwargs: f"SNOWFLAKE.CORTEX.COMPLETE('{prompt}')"
+            },
+            
+            # Summarization Mapping
+            'summarize': {
+                'cortex_function': 'SUMMARIZE',
+                'transformer': lambda text, **kwargs: f"SNOWFLAKE.CORTEX.SUMMARIZE('{text}')"
+            },
+            
+            # Sentiment Analysis Mapping
+            'analyze_sentiment': {
+                'cortex_function': 'SENTIMENT',
+                'transformer': lambda text, **kwargs: f"SNOWFLAKE.CORTEX.SENTIMENT('{text}')"
+            },
+            
+            # Translation Mapping
+            'translate': {
+                'cortex_function': 'TRANSLATE',
+                'transformer': lambda text, source_lang='auto', target_lang='en', **kwargs: 
+                    f"SNOWFLAKE.CORTEX.TRANSLATE('{text}', '{source_lang}', '{target_lang}')"
+            },
+            
+            # Named Entity Recognition
+            'extract_entities': {
+                'cortex_function': 'EXTRACT_ENTITIES',
+                'transformer': lambda text, **kwargs: f"SNOWFLAKE.CORTEX.EXTRACT_ENTITIES('{text}')"
+            },
+            
+            # Embedding Generation
+            'generate_embedding': {
+                'cortex_function': 'EMBED_TEXT',
+                'transformer': lambda text, model='snowflake-arctic', **kwargs: 
+                    f"SNOWFLAKE.CORTEX.EMBED_TEXT('{model}', '{text}')"
+            }
+        }
+    
+    @staticmethod
+    def get_databricks_ai_mapping() -> Dict[str, Dict[str, Any]]:
+        """
+        Map NPC AI functions to Databricks AI functions
+        """
+        return {
+            # Databricks uses different function names and approaches
+            'generate_text': {
+                'databricks_function': 'serving.predict',
+                'transformer': lambda prompt, model='databricks-dolly', **kwargs: 
+                    f"serving.predict('{model}', '{prompt}')"
+            },
+            # Add more Databricks-specific mappings
+        }
+    
+    @staticmethod
+    def get_bigquery_ai_mapping() -> Dict[str, Dict[str, Any]]:
+        """
+        Map NPC AI functions to BigQuery AI functions
+        """
+        return {
+            'generate_text': {
+                'bigquery_function': 'ML.GENERATE_TEXT',
+                'transformer': lambda prompt, model='text-bison', **kwargs:
+                    f"ML.GENERATE_TEXT(MODEL `{model}`, '{prompt}')"
+            },
+            # Add more BigQuery-specific mappings
+        }
+
+class NativeDatabaseAITransformer:
+    def __init__(self, database_type: str):
+        self.database_type = database_type
+        self.function_mappings = self._get_database_mappings()
+    
+    def _get_database_mappings(self) -> Dict[str, Dict[str, Any]]:
+        """
+        Get the appropriate AI function mapping based on database type
+        """
+        mappings = {
+            'snowflake': DatabaseAIFunctionMapper.get_snowflake_cortex_mapping(),
+            'databricks': DatabaseAIFunctionMapper.get_databricks_ai_mapping(),
+            'bigquery': DatabaseAIFunctionMapper.get_bigquery_ai_mapping()
+        }
+        return mappings.get(self.database_type.lower(), {})
+    
+    def transform_ai_function(self, function_name: str, **kwargs) -> str:
+        """
+        Transform an NPC AI function to a native database AI function
+        """
+        mapping = self.function_mappings.get(function_name)
+        if not mapping:
+            raise ValueError(f"No native mapping found for function: {function_name}")
+        
+        transformer = mapping.get('transformer')
+        if not transformer:
+            raise ValueError(f"No transformer found for function: {function_name}")
+        
+        return transformer(**kwargs)
+
+# Example usage in ModelCompiler
+def _has_native_ai_functions(self, source_name: str) -> bool:
+    """Enhanced method to check native AI function support"""
+    ai_enabled = {
+        'snowflake': True,
+        'databricks': True,
+        'bigquery': True
+    }
+    return ai_enabled.get(source_name.lower(), False)
+
+def _execute_ai_model(self, sql: str, model: SQLModel) -> pd.DataFrame:
+    """
+    Enhanced method to use native AI functions when available
+    """
+    source_pattern = r'FROM\s+(\\w+)\\.(\\w+)'
+    matches = re.findall(source_pattern, sql)
+    
+    if matches:
+        source_name, table_name = matches[0]
+        engine = self._get_engine(source_name)
+        
+        # Check for native AI function support
+        if self._has_native_ai_functions(source_name):
+            # Use native transformer
+            transformer = NativeDatabaseAITransformer(source_name)
+            
+            # Modify SQL to use native AI functions
+            for func_name, params in model.ai_functions.items():
+                try:
+                    native_func_call = transformer.transform_ai_function(
+                        func_name, 
+                        text=params.get('column', ''),
+                        **{k: v for k, v in params.items() if k != 'column'}
+                    )
+                    
+                    # Replace the NQL function with native function
+                    sql = sql.replace(
+                        f"nql.{func_name}({params.get('column', '')})", 
+                        native_func_call
+                    )
+                except ValueError as e:
+                    # Fallback to original method if transformation fails
+                    print(f"Warning: {e}. Falling back to default AI function.")
+            
+            return pd.read_sql(sql.replace(f"{source_name}.", ""), engine)
+    
+    # Fallback to existing AI model execution
+    return super()._execute_ai_model(sql, model)

npcpy/sql/sql_model_compiler.py

@@ -0,0 +1,156 @@
+import os
+import yaml
+import sqlalchemy
+import pandas as pd
+from typing import Dict, Any, Optional
+
+class SQLModelCompiler:
+    """
+    Compile and execute SQL models across different database engines
+    """
+    def __init__(
+        self, 
+        models_dir: str, 
+        engine: Optional[sqlalchemy.engine.base.Engine] = None,
+        engine_type: str = 'sqlite'
+    ):
+        """
+        Initialize SQL Model Compiler
+        
+        :param models_dir: Directory containing SQL model files
+        :param engine: SQLAlchemy database engine
+        :param engine_type: Type of database engine (sqlite, snowflake, bigquery, etc.)
+        """
+        self.models_dir = models_dir
+        self.engine = engine
+        self.engine_type = engine_type.lower()
+        self.models = {}
+        
+        # Discover models
+        self._discover_models()
+    
+    def _discover_models(self):
+        """
+        Discover and load SQL model files
+        """
+        for filename in os.listdir(self.models_dir):
+            if filename.endswith('.sql'):
+                model_name = os.path.splitext(filename)[0]
+                model_path = os.path.join(self.models_dir, filename)
+                
+                with open(model_path, 'r') as f:
+                    model_content = f.read()
+                
+                self.models[model_name] = {
+                    'name': model_name,
+                    'content': model_content,
+                    'path': model_path
+                }
+    
+    def _compile_model(self, model_name: str) -> str:
+        """
+        Compile a SQL model for the specific engine
+        
+        :param model_name: Name of the model to compile
+        :return: Compiled SQL query
+        """
+        model = self.models[model_name]
+        content = model['content']
+        
+        # Engine-specific compilation
+        if self.engine_type == 'snowflake':
+            # Snowflake-specific transformations
+            content = content.replace('{{', 'SNOWFLAKE.').replace('}}', '')
+        elif self.engine_type == 'bigquery':
+            # BigQuery-specific transformations
+            content = content.replace('{{', 'ML.').replace('}}', '')
+        
+        return content
+    
+    def execute_model(
+        self, 
+        model_name: str, 
+        seed_data: Optional[Dict[str, pd.DataFrame]] = None
+    ) -> pd.DataFrame:
+        """
+        Execute a SQL model
+        
+        :param model_name: Name of the model to execute
+        :param seed_data: Optional seed data for the model
+        :return: Result DataFrame
+        """
+        if model_name not in self.models:
+            raise ValueError(f"Model {model_name} not found")
+        
+        # Compile model for specific engine
+        compiled_sql = self._compile_model(model_name)
+        
+        # If seed data is provided, prepare the database
+        if seed_data and self.engine:
+            for table_name, df in seed_data.items():
+                df.to_sql(table_name, self.engine, if_exists='replace', index=False)
+        
+        # Execute the model
+        if self.engine:
+            return pd.read_sql(compiled_sql, self.engine)
+        else:
+            # Fallback to pandas evaluation
+            import sqlite3
+            
+            # Create an in-memory SQLite database for evaluation
+            conn = sqlite3.connect(':memory:')
+            
+            # Load seed data if available
+            if seed_data:
+                for table_name, df in seed_data.items():
+                    df.to_sql(table_name, conn, if_exists='replace', index=False)
+            
+            return pd.read_sql(compiled_sql, conn)
+    
+    def run_all_models(self, seed_data: Optional[Dict[str, pd.DataFrame]] = None):
+        """
+        Run all discovered models
+        
+        :param seed_data: Optional seed data for models
+        :return: Dictionary of model results
+        """
+        results = {}
+        for model_name in self.models:
+            results[model_name] = self.execute_model(model_name, seed_data)
+        return results
+
+# Example usage in a pipeline
+def create_model_compiler(
+    models_dir: str, 
+    engine_type: str = 'sqlite', 
+    connection_params: Optional[Dict[str, Any]] = None
+) -> SQLModelCompiler:
+    """
+    Create a SQL Model Compiler with the specified engine
+    
+    :param models_dir: Directory containing SQL model files
+    :param engine_type: Type of database engine
+    :param connection_params: Connection parameters for the database
+    :return: SQLModelCompiler instance
+    """
+    if engine_type == 'snowflake':
+        from sqlalchemy.dialects.snowflake import base
+        engine = sqlalchemy.create_engine(
+            f"snowflake://{connection_params['username']}:{connection_params['password']}@"
+            f"{connection_params['account']}/{connection_params['database']}/{connection_params['schema']}"
+        )
+    elif engine_type == 'bigquery':
+        from google.cloud import bigquery
+        from sqlalchemy.dialects.bigquery import base
+        engine = sqlalchemy.create_engine(
+            f"bigquery://{connection_params['project_id']}"
+        )
+    else:
+        # Default to SQLite
+        engine = sqlalchemy.create_engine('sqlite:///models.db')
+    
+    return SQLModelCompiler(
+        models_dir=models_dir, 
+        engine=engine, 
+        engine_type=engine_type
+    )

{npcpy-1.2.20.dist-info → npcpy-1.2.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: npcpy
-Version: 1.2.20
+Version: 1.2.22
 Summary: npcpy is the premier open-source library for integrating LLMs and Agents into python systems.
 Home-page: https://github.com/NPC-Worldwide/npcpy
 Author: Christopher Agostino
@@ -55,6 +55,7 @@ Requires-Dist: kuzu; extra == "local"
 Requires-Dist: chromadb; extra == "local"
 Requires-Dist: diffusers; extra == "local"
 Requires-Dist: torch; extra == "local"
+Requires-Dist: datasets; extra == "local"
 Provides-Extra: yap
 Requires-Dist: pyaudio; extra == "yap"
 Requires-Dist: gtts; extra == "yap"
@@ -74,6 +75,7 @@ Requires-Dist: kuzu; extra == "all"
 Requires-Dist: chromadb; extra == "all"
 Requires-Dist: diffusers; extra == "all"
 Requires-Dist: torch; extra == "all"
+Requires-Dist: datasets; extra == "all"
 Requires-Dist: pyaudio; extra == "all"
 Requires-Dist: gtts; extra == "all"
 Requires-Dist: playsound==1.2.2; extra == "all"
@@ -93,19 +95,15 @@ Dynamic: requires-python
 Dynamic: summary
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy.png" alt="npcpy logo of a solarpunk sign">
+  <a href= "https://github.com/cagostino/npcpy/blob/main/docs/npcpy.md"> 
+  <img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy/npc-python.png" alt="npc-python logo" width=250></a>
 </p>
 
-
 # npcpy
 
 Welcome to `npcpy`, the core library of the NPC Toolkit that supercharges natural language processing pipelines and agent tooling. `npcpy` is a flexible framework for building state-of-the-art applications and conducting novel research with LLMs.
 
 
-<p align="center">
-  <a href= "https://github.com/cagostino/npcpy/blob/main/docs/npcpy.md"> 
-  <img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy/npc-python.png" alt="npc-python logo" width=250></a>
-</p>
 
 
 Here is an example for getting responses for a particular agent:

{npcpy-1.2.20.dist-info → npcpy-1.2.22.dist-info}/RECORD

@@ -1,16 +1,16 @@
 npcpy/__init__.py,sha256=9imxFtK74_6Rw9rz0kyMnZYl_voPb569tkTlYLt0Urg,131
 npcpy/llm_funcs.py,sha256=tvcZuQEcIUJClwEJQXBF6ArEVjSuXt1jAcZOcnYWsVQ,85101
 npcpy/main.py,sha256=RWoRIj6VQLxKdOKvdVyaq2kwG35oRpeXPvp1CAAoG-w,81
-npcpy/npc_compiler.py,sha256=BpNlrjwkxhERTrFeFtvv9CUqzULoD2JQuEwRtqwQHLY,92107
+npcpy/npc_compiler.py,sha256=10vu-9WUmlVzaFM_hMJH28iNS1IJXQP3Rb5RT1rZmpA,95326
 npcpy/npc_sysenv.py,sha256=lPYlKM_TeR4l4-Jcgiqq3CCge8b2oFHdfISD4L_G7eo,30308
 npcpy/npcs.py,sha256=eExuVsbTfrRobTRRptRpDm46jCLWUgbvy4_U7IUQo-c,744
-npcpy/serve.py,sha256=RPYT3ZMu-OmO6dg3Ss04wZQNlZQive5cm2UPUbtToV0,100271
+npcpy/serve.py,sha256=O1dxISi0nQ6jsSOSxBXsULgkltnIcyBS6Z0AjfWmuXA,100296
 npcpy/tools.py,sha256=A5_oVmZkzGnI3BI-NmneuxeXQq-r29PbpAZP4nV4jrc,5303
 npcpy/data/__init__.py,sha256=1tcoChR-Hjn905JDLqaW9ElRmcISCTJdE7BGXPlym2Q,642
 npcpy/data/audio.py,sha256=goon4HfsYgx0bI-n1lhkrzWPrJoejJlycXcB0P62pyk,11280
 npcpy/data/data_models.py,sha256=q7xpI4_nK5HvlOE1XB5u5nFQs4SE5zcgt0kIZJF2dhs,682
 npcpy/data/image.py,sha256=UQcioNPDd5HYMLL_KStf45SuiIPXDcUY-dEFHwSWUeE,6564
-npcpy/data/load.py,sha256=f3-bgKUq_pnfUhqjZdXwfIEZmMbGJpJfGTBjuiYJos8,4258
+npcpy/data/load.py,sha256=7Ay-TYNhCvjJLwdQ5qAgxXSrGwow9ZrazHFVPqMw_cI,4274
 npcpy/data/text.py,sha256=jP0a1qZZaSJdK-LdZTn2Jjdxqmkd3efxDLEoxflJQeY,5010
 npcpy/data/video.py,sha256=aPUgj0fA_lFQ7Jf94-PutggCF4695FVCh3q5mnVthvI,574
 npcpy/data/web.py,sha256=ARGoVKUlQmaiX0zJbSvvFmRCwOv_Z7Pcan9c5GxYObQ,5117
@@ -24,7 +24,7 @@ npcpy/gen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 npcpy/gen/audio_gen.py,sha256=w4toESu7nmli1T5FOwRRCGC_QK9W-SMWknYYkbRv9jE,635
 npcpy/gen/embeddings.py,sha256=QStTJ2ELiC379OEZsLEgGGIIFD267Y8zQchs7HRn2Zg,2089
 npcpy/gen/image_gen.py,sha256=ln71jmLoJHekbZYDJpTe5DtOamVte9gjr2BPQ1DzjMQ,14955
-npcpy/gen/response.py,sha256=Rrk3rrSSGode94JJkKyL4n9Dpw9CH7R5l9xyjQSDFxE,28954
+npcpy/gen/response.py,sha256=dK0Ux1_0GHo4gOfSHrrp34Ub4YJ-88NjFZfaG3kSrB0,28940
 npcpy/gen/video_gen.py,sha256=JMp2s2qMp5uy0rOgv6BRZ7nkQI4vdT1hbJ2nSu4s-KA,3243
 npcpy/memory/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 npcpy/memory/command_history.py,sha256=Ww7vZTSjQDuElQXuOjsvu7NTljOLAg07QIFrfKARpVg,45562
@@ -35,14 +35,18 @@ npcpy/memory/search.py,sha256=glN6WYzaixcoDphTEHAXSMX3vKZGjR12Jx9YVL_gYfE,18433
 npcpy/mix/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 npcpy/mix/debate.py,sha256=lQXxC7nl6Rwyf7HIYrsVQILMUmYYx55Tjt2pkTg56qY,9019
 npcpy/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+npcpy/sql/ai_function_tools.py,sha256=ZCpjVHtaMRdL2dXxbQy5NhhjtPrVViGT1wyEl8ADrks,7755
+npcpy/sql/database_ai_adapters.py,sha256=CMlNGOhmJZhGB47RPvLIMqB61m_eYPVg1lwx42_b0jQ,6865
+npcpy/sql/database_ai_functions.py,sha256=XQCmaFOE1lNCnwrLTNpotYOlv6sx41bb8hxZI_sqpy8,6335
 npcpy/sql/model_runner.py,sha256=hJZ7hx2mwI-8DAh47Q6BwOsRjx30-HzebL4ajEUO4HA,5734
 npcpy/sql/npcsql.py,sha256=AOffqGK1Jwjf_tgE8qo4KY5KXE21ylm1dhRSKS0Cesw,13587
+npcpy/sql/sql_model_compiler.py,sha256=G-0dpTlgzc-dXy9YEsdWGjO8xaQ3jFNbc6oUja1Ef4M,5364
 npcpy/work/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 npcpy/work/desktop.py,sha256=F3I8mUtJp6LAkXodsh8hGZIncoads6c_2Utty-0EdDA,2986
 npcpy/work/plan.py,sha256=QyUwg8vElWiHuoS-xK4jXTxxHvkMD3VkaCEsCmrEPQk,8300
 npcpy/work/trigger.py,sha256=P1Y8u1wQRsS2WACims_2IdkBEar-iBQix-2TDWoW0OM,9948
-npcpy-1.2.20.dist-info/licenses/LICENSE,sha256=j0YPvce7Ng9e32zYOu0EmXjXeJ0Nwawd0RA3uSGGH4E,1070
-npcpy-1.2.20.dist-info/METADATA,sha256=P5knjysgVTcYCKlMkbJT-iiw_t9cy1SlskD2YuyknHE,26084
-npcpy-1.2.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-npcpy-1.2.20.dist-info/top_level.txt,sha256=g1pbSvrOOncB74Bg5-J0Olg4V0A5VzDw-Xz5YObq8BU,6
-npcpy-1.2.20.dist-info/RECORD,,
+npcpy-1.2.22.dist-info/licenses/LICENSE,sha256=j0YPvce7Ng9e32zYOu0EmXjXeJ0Nwawd0RA3uSGGH4E,1070
+npcpy-1.2.22.dist-info/METADATA,sha256=wCegeQ_fbKicQoqXvR4E08dHC_cf2yl_0711U_rCeHk,26025
+npcpy-1.2.22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+npcpy-1.2.22.dist-info/top_level.txt,sha256=g1pbSvrOOncB74Bg5-J0Olg4V0A5VzDw-Xz5YObq8BU,6
+npcpy-1.2.22.dist-info/RECORD,,