npcpy 1.2.20__tar.gz → 1.2.22__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {npcpy-1.2.20/npcpy.egg-info → npcpy-1.2.22}/PKG-INFO +5 -7
- {npcpy-1.2.20 → npcpy-1.2.22}/README.md +2 -6
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/data/load.py +1 -1
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/gen/response.py +2 -2
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/npc_compiler.py +88 -7
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/serve.py +3 -3
- npcpy-1.2.22/npcpy/sql/ai_function_tools.py +257 -0
- npcpy-1.2.22/npcpy/sql/database_ai_adapters.py +186 -0
- npcpy-1.2.22/npcpy/sql/database_ai_functions.py +163 -0
- npcpy-1.2.22/npcpy/sql/sql_model_compiler.py +156 -0
- {npcpy-1.2.20 → npcpy-1.2.22/npcpy.egg-info}/PKG-INFO +5 -7
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy.egg-info/SOURCES.txt +4 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy.egg-info/requires.txt +2 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/setup.py +2 -1
- {npcpy-1.2.20 → npcpy-1.2.22}/LICENSE +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/MANIFEST.in +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/__init__.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/data/__init__.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/data/audio.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/data/data_models.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/data/image.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/data/text.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/data/video.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/data/web.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/ft/__init__.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/ft/diff.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/ft/ge.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/ft/memory_trainer.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/ft/rl.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/ft/sft.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/gen/__init__.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/gen/audio_gen.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/gen/embeddings.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/gen/image_gen.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/gen/video_gen.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/llm_funcs.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/main.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/memory/__init__.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/memory/command_history.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/memory/kg_vis.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/memory/knowledge_graph.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/memory/memory_processor.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/memory/search.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/mix/__init__.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/mix/debate.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/npc_sysenv.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/npcs.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/sql/__init__.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/sql/model_runner.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/sql/npcsql.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/tools.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/work/__init__.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/work/desktop.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/work/plan.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy/work/trigger.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy.egg-info/dependency_links.txt +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/npcpy.egg-info/top_level.txt +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/setup.cfg +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_audio.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_command_history.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_image.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_llm_funcs.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_load.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_npc_compiler.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_npcsql.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_response.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_serve.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_text.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_tools.py +0 -0
- {npcpy-1.2.20 → npcpy-1.2.22}/tests/test_web.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: npcpy
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.22
|
|
4
4
|
Summary: npcpy is the premier open-source library for integrating LLMs and Agents into python systems.
|
|
5
5
|
Home-page: https://github.com/NPC-Worldwide/npcpy
|
|
6
6
|
Author: Christopher Agostino
|
|
@@ -55,6 +55,7 @@ Requires-Dist: kuzu; extra == "local"
|
|
|
55
55
|
Requires-Dist: chromadb; extra == "local"
|
|
56
56
|
Requires-Dist: diffusers; extra == "local"
|
|
57
57
|
Requires-Dist: torch; extra == "local"
|
|
58
|
+
Requires-Dist: datasets; extra == "local"
|
|
58
59
|
Provides-Extra: yap
|
|
59
60
|
Requires-Dist: pyaudio; extra == "yap"
|
|
60
61
|
Requires-Dist: gtts; extra == "yap"
|
|
@@ -74,6 +75,7 @@ Requires-Dist: kuzu; extra == "all"
|
|
|
74
75
|
Requires-Dist: chromadb; extra == "all"
|
|
75
76
|
Requires-Dist: diffusers; extra == "all"
|
|
76
77
|
Requires-Dist: torch; extra == "all"
|
|
78
|
+
Requires-Dist: datasets; extra == "all"
|
|
77
79
|
Requires-Dist: pyaudio; extra == "all"
|
|
78
80
|
Requires-Dist: gtts; extra == "all"
|
|
79
81
|
Requires-Dist: playsound==1.2.2; extra == "all"
|
|
@@ -93,19 +95,15 @@ Dynamic: requires-python
|
|
|
93
95
|
Dynamic: summary
|
|
94
96
|
|
|
95
97
|
<p align="center">
|
|
96
|
-
<
|
|
98
|
+
<a href= "https://github.com/cagostino/npcpy/blob/main/docs/npcpy.md">
|
|
99
|
+
<img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy/npc-python.png" alt="npc-python logo" width=250></a>
|
|
97
100
|
</p>
|
|
98
101
|
|
|
99
|
-
|
|
100
102
|
# npcpy
|
|
101
103
|
|
|
102
104
|
Welcome to `npcpy`, the core library of the NPC Toolkit that supercharges natural language processing pipelines and agent tooling. `npcpy` is a flexible framework for building state-of-the-art applications and conducting novel research with LLMs.
|
|
103
105
|
|
|
104
106
|
|
|
105
|
-
<p align="center">
|
|
106
|
-
<a href= "https://github.com/cagostino/npcpy/blob/main/docs/npcpy.md">
|
|
107
|
-
<img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy/npc-python.png" alt="npc-python logo" width=250></a>
|
|
108
|
-
</p>
|
|
109
107
|
|
|
110
108
|
|
|
111
109
|
Here is an example for getting responses for a particular agent:
|
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<
|
|
2
|
+
<a href= "https://github.com/cagostino/npcpy/blob/main/docs/npcpy.md">
|
|
3
|
+
<img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy/npc-python.png" alt="npc-python logo" width=250></a>
|
|
3
4
|
</p>
|
|
4
5
|
|
|
5
|
-
|
|
6
6
|
# npcpy
|
|
7
7
|
|
|
8
8
|
Welcome to `npcpy`, the core library of the NPC Toolkit that supercharges natural language processing pipelines and agent tooling. `npcpy` is a flexible framework for building state-of-the-art applications and conducting novel research with LLMs.
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
<p align="center">
|
|
12
|
-
<a href= "https://github.com/cagostino/npcpy/blob/main/docs/npcpy.md">
|
|
13
|
-
<img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy/npc-python.png" alt="npc-python logo" width=250></a>
|
|
14
|
-
</p>
|
|
15
11
|
|
|
16
12
|
|
|
17
13
|
Here is an example for getting responses for a particular agent:
|
|
@@ -132,7 +132,7 @@ def load_file_contents(file_path, chunk_size=None):
|
|
|
132
132
|
elif file_ext in ['XLS', 'XLSX']:
|
|
133
133
|
df = load_excel(file_path)
|
|
134
134
|
full_content = df.to_string()
|
|
135
|
-
elif file_ext in ['TXT', 'MD', 'PY', 'JSX', 'TSX', 'TS', 'JS', 'JSON', 'SQL', 'NPC', 'JINX', 'LINE', 'YAML']:
|
|
135
|
+
elif file_ext in ['TXT', 'MD', 'PY', 'JSX', 'TSX', 'TS', 'JS', 'JSON', 'SQL', 'NPC', 'JINX', 'LINE', 'YAML', 'DART', 'JAVA']:
|
|
136
136
|
full_content = load_txt(file_path)
|
|
137
137
|
elif file_ext == 'JSON':
|
|
138
138
|
data = load_json(file_path)
|
|
@@ -475,9 +475,9 @@ def get_litellm_response(
|
|
|
475
475
|
pdf_data = load_pdf(attachment)
|
|
476
476
|
if pdf_data is not None:
|
|
477
477
|
if prompt:
|
|
478
|
-
prompt += f"\n\nContent from PDF: {os.path.basename(attachment)}\n{pdf_data
|
|
478
|
+
prompt += f"\n\nContent from PDF: {os.path.basename(attachment)}\n{pdf_data}..."
|
|
479
479
|
else:
|
|
480
|
-
prompt = f"Content from PDF: {os.path.basename(attachment)}\n{pdf_data
|
|
480
|
+
prompt = f"Content from PDF: {os.path.basename(attachment)}\n{pdf_data}..."
|
|
481
481
|
|
|
482
482
|
except Exception:
|
|
483
483
|
pass
|
|
@@ -1908,7 +1908,8 @@ class Team:
|
|
|
1908
1908
|
"dataframes": {},
|
|
1909
1909
|
"memories": {},
|
|
1910
1910
|
"execution_history": [],
|
|
1911
|
-
"npc_messages": {}
|
|
1911
|
+
"npc_messages": {},
|
|
1912
|
+
"context":''
|
|
1912
1913
|
}
|
|
1913
1914
|
|
|
1914
1915
|
if team_path:
|
|
@@ -2017,11 +2018,12 @@ class Team:
|
|
|
2017
2018
|
self.databases = ctx_data['databases']
|
|
2018
2019
|
else:
|
|
2019
2020
|
self.databases = []
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2021
|
+
|
|
2022
|
+
base_context = ctx_data.get('context', '')
|
|
2023
|
+
self.shared_context['context'] = base_context
|
|
2024
|
+
if 'file_patterns' in ctx_data:
|
|
2025
|
+
file_cache = self._parse_file_patterns(ctx_data['file_patterns'])
|
|
2026
|
+
self.shared_context['files'] = file_cache
|
|
2025
2027
|
if 'preferences' in ctx_data:
|
|
2026
2028
|
self.preferences = ctx_data['preferences']
|
|
2027
2029
|
else:
|
|
@@ -2031,7 +2033,7 @@ class Team:
|
|
|
2031
2033
|
else:
|
|
2032
2034
|
self.forenpc = self.npcs[list(self.npcs.keys())[0]] if self.npcs else None
|
|
2033
2035
|
for key, item in ctx_data.items():
|
|
2034
|
-
if key not in ['name', 'mcp_servers', 'databases', 'context']:
|
|
2036
|
+
if key not in ['name', 'mcp_servers', 'databases', 'context', 'file_patterns']:
|
|
2035
2037
|
self.shared_context[key] = item
|
|
2036
2038
|
return ctx_data
|
|
2037
2039
|
return {}
|
|
@@ -2288,6 +2290,85 @@ class Team:
|
|
|
2288
2290
|
team.save(team_dir)
|
|
2289
2291
|
|
|
2290
2292
|
return True
|
|
2293
|
+
def _parse_file_patterns(self, patterns_config):
|
|
2294
|
+
"""Parse file patterns configuration and load matching files into KV cache"""
|
|
2295
|
+
if not patterns_config:
|
|
2296
|
+
return {}
|
|
2297
|
+
|
|
2298
|
+
file_cache = {}
|
|
2299
|
+
|
|
2300
|
+
for pattern_entry in patterns_config:
|
|
2301
|
+
if isinstance(pattern_entry, str):
|
|
2302
|
+
pattern_entry = {"pattern": pattern_entry}
|
|
2303
|
+
|
|
2304
|
+
pattern = pattern_entry.get("pattern", "")
|
|
2305
|
+
recursive = pattern_entry.get("recursive", False)
|
|
2306
|
+
base_path = pattern_entry.get("base_path", ".")
|
|
2307
|
+
|
|
2308
|
+
if not pattern:
|
|
2309
|
+
continue
|
|
2310
|
+
|
|
2311
|
+
base_path = os.path.expanduser(base_path)
|
|
2312
|
+
if not os.path.isabs(base_path):
|
|
2313
|
+
base_path = os.path.join(self.team_path or os.getcwd(), base_path)
|
|
2314
|
+
|
|
2315
|
+
matching_files = self._find_matching_files(pattern, base_path, recursive)
|
|
2316
|
+
|
|
2317
|
+
for file_path in matching_files:
|
|
2318
|
+
file_content = self._load_file_content(file_path)
|
|
2319
|
+
if file_content:
|
|
2320
|
+
relative_path = os.path.relpath(file_path, base_path)
|
|
2321
|
+
file_cache[relative_path] = file_content
|
|
2322
|
+
|
|
2323
|
+
return file_cache
|
|
2324
|
+
|
|
2325
|
+
def _find_matching_files(self, pattern, base_path, recursive=False):
|
|
2326
|
+
"""Find files matching the given pattern"""
|
|
2327
|
+
matching_files = []
|
|
2328
|
+
|
|
2329
|
+
if not os.path.exists(base_path):
|
|
2330
|
+
return matching_files
|
|
2331
|
+
|
|
2332
|
+
if recursive:
|
|
2333
|
+
for root, dirs, files in os.walk(base_path):
|
|
2334
|
+
for filename in files:
|
|
2335
|
+
if fnmatch.fnmatch(filename, pattern):
|
|
2336
|
+
matching_files.append(os.path.join(root, filename))
|
|
2337
|
+
else:
|
|
2338
|
+
try:
|
|
2339
|
+
for item in os.listdir(base_path):
|
|
2340
|
+
item_path = os.path.join(base_path, item)
|
|
2341
|
+
if os.path.isfile(item_path) and fnmatch.fnmatch(item, pattern):
|
|
2342
|
+
matching_files.append(item_path)
|
|
2343
|
+
except PermissionError:
|
|
2344
|
+
print(f"Permission denied accessing {base_path}")
|
|
2345
|
+
|
|
2346
|
+
return matching_files
|
|
2347
|
+
|
|
2348
|
+
def _load_file_content(self, file_path):
|
|
2349
|
+
"""Load content from a file with error handling"""
|
|
2350
|
+
try:
|
|
2351
|
+
with open(file_path, 'r', encoding='utf-8') as f:
|
|
2352
|
+
return f.read()
|
|
2353
|
+
except Exception as e:
|
|
2354
|
+
print(f"Error reading {file_path}: {e}")
|
|
2355
|
+
return None
|
|
2356
|
+
|
|
2357
|
+
|
|
2358
|
+
def _format_parsed_files_context(self, parsed_files):
|
|
2359
|
+
"""Format parsed files into context string"""
|
|
2360
|
+
if not parsed_files:
|
|
2361
|
+
return ""
|
|
2362
|
+
|
|
2363
|
+
context_parts = ["Additional context from files:"]
|
|
2364
|
+
|
|
2365
|
+
for file_path, content in parsed_files.items():
|
|
2366
|
+
context_parts.append(f"\n--- {file_path} ---")
|
|
2367
|
+
context_parts.append(content)
|
|
2368
|
+
context_parts.append("")
|
|
2369
|
+
|
|
2370
|
+
return "\n".join(context_parts)
|
|
2371
|
+
|
|
2291
2372
|
class Pipeline:
|
|
2292
2373
|
def __init__(self, pipeline_data=None, pipeline_path=None, npc_team=None):
|
|
2293
2374
|
"""Initialize a pipeline from data or file path"""
|
|
@@ -443,7 +443,7 @@ def get_global_settings():
|
|
|
443
443
|
"embedding_model": "nomic-embed-text",
|
|
444
444
|
"embedding_provider": "ollama",
|
|
445
445
|
"search_provider": "perplexity",
|
|
446
|
-
"
|
|
446
|
+
"NPC_STUDIO_LICENSE_KEY": "",
|
|
447
447
|
"default_folder": os.path.expanduser("~/.npcsh/"),
|
|
448
448
|
}
|
|
449
449
|
global_vars = {}
|
|
@@ -479,7 +479,7 @@ def get_global_settings():
|
|
|
479
479
|
"NPCSH_EMBEDDING_MODEL": "embedding_model",
|
|
480
480
|
"NPCSH_EMBEDDING_PROVIDER": "embedding_provider",
|
|
481
481
|
"NPCSH_SEARCH_PROVIDER": "search_provider",
|
|
482
|
-
"
|
|
482
|
+
"NPC_STUDIO_LICENSE_KEY": "NPC_STUDIO_LICENSE_KEY",
|
|
483
483
|
"NPCSH_STREAM_OUTPUT": "NPCSH_STREAM_OUTPUT",
|
|
484
484
|
"NPC_STUDIO_DEFAULT_FOLDER": "default_folder",
|
|
485
485
|
}
|
|
@@ -521,7 +521,7 @@ def save_global_settings():
|
|
|
521
521
|
"embedding_model": "NPCSH_EMBEDDING_MODEL",
|
|
522
522
|
"embedding_provider": "NPCSH_EMBEDDING_PROVIDER",
|
|
523
523
|
"search_provider": "NPCSH_SEARCH_PROVIDER",
|
|
524
|
-
"
|
|
524
|
+
"NPC_STUDIO_LICENSE_KEY": "NPC_STUDIO_LICENSE_KEY",
|
|
525
525
|
"NPCSH_STREAM_OUTPUT": "NPCSH_STREAM_OUTPUT",
|
|
526
526
|
"default_folder": "NPC_STUDIO_DEFAULT_FOLDER",
|
|
527
527
|
}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import Dict, Any, List, Optional
|
|
3
|
+
|
|
4
|
+
class SQLToolCallResponse:
|
|
5
|
+
"""
|
|
6
|
+
Represents a structured response with tool calling capabilities
|
|
7
|
+
that can be generated and processed within SQL
|
|
8
|
+
"""
|
|
9
|
+
@staticmethod
|
|
10
|
+
def generate_tool_call_prompt(
|
|
11
|
+
prompt: str,
|
|
12
|
+
tools: List[Dict[str, Any]],
|
|
13
|
+
model: str = 'snowflake-arctic'
|
|
14
|
+
) -> str:
|
|
15
|
+
"""
|
|
16
|
+
Generate a prompt that instructs the model to use tools
|
|
17
|
+
|
|
18
|
+
:param prompt: Original user prompt
|
|
19
|
+
:param tools: List of available tools/functions
|
|
20
|
+
:param model: AI model to use
|
|
21
|
+
:return: Formatted prompt for tool-aware generation
|
|
22
|
+
"""
|
|
23
|
+
tool_descriptions = [
|
|
24
|
+
f"Tool: {tool.get('name', 'unnamed')}\n"
|
|
25
|
+
f"Description: {tool.get('description', 'No description')}\n"
|
|
26
|
+
f"Parameters: {json.dumps(tool.get('parameters', {}))}"
|
|
27
|
+
for tool in tools
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
return f"""
|
|
31
|
+
You are an AI assistant capable of using the following tools:
|
|
32
|
+
|
|
33
|
+
{"\n\n".join(tool_descriptions)}
|
|
34
|
+
|
|
35
|
+
User Prompt: {prompt}
|
|
36
|
+
|
|
37
|
+
IMPORTANT INSTRUCTIONS:
|
|
38
|
+
1. Carefully analyze the user's request
|
|
39
|
+
2. Determine which tool(s) are most appropriate
|
|
40
|
+
3. Generate a structured JSON response with:
|
|
41
|
+
- tool_calls: List of tool invocations
|
|
42
|
+
- final_response: Your overall response to the user
|
|
43
|
+
4. ONLY use tools that are directly relevant
|
|
44
|
+
5. Format the output as a valid JSON object
|
|
45
|
+
|
|
46
|
+
Output Format:
|
|
47
|
+
{{
|
|
48
|
+
"tool_calls": [
|
|
49
|
+
{{
|
|
50
|
+
"tool_name": "tool_name",
|
|
51
|
+
"parameters": {{...}}
|
|
52
|
+
}}
|
|
53
|
+
],
|
|
54
|
+
"final_response": "Optional explanation or summary"
|
|
55
|
+
}}
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
@staticmethod
|
|
59
|
+
def parse_tool_calls_sql(tool_call_json: str) -> Dict[str, Any]:
|
|
60
|
+
"""
|
|
61
|
+
Parse tool calls within SQL, with error handling
|
|
62
|
+
|
|
63
|
+
:param tool_call_json: JSON string of tool calls
|
|
64
|
+
:return: Parsed tool call dictionary
|
|
65
|
+
"""
|
|
66
|
+
try:
|
|
67
|
+
parsed = json.loads(tool_call_json)
|
|
68
|
+
return {
|
|
69
|
+
'tool_calls': parsed.get('tool_calls', []),
|
|
70
|
+
'final_response': parsed.get('final_response', '')
|
|
71
|
+
}
|
|
72
|
+
except json.JSONDecodeError:
|
|
73
|
+
return {
|
|
74
|
+
'tool_calls': [],
|
|
75
|
+
'final_response': 'Error parsing tool calls'
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
class SnowflakeSQLToolCaller:
|
|
79
|
+
"""
|
|
80
|
+
Snowflake-specific tool calling implementation
|
|
81
|
+
"""
|
|
82
|
+
@staticmethod
|
|
83
|
+
def generate_tool_call_sql(
|
|
84
|
+
prompt: str,
|
|
85
|
+
tools: List[Dict[str, Any]],
|
|
86
|
+
model: str = 'snowflake-arctic'
|
|
87
|
+
) -> str:
|
|
88
|
+
"""
|
|
89
|
+
Generate a SQL function that performs tool calling
|
|
90
|
+
|
|
91
|
+
:param prompt: User prompt
|
|
92
|
+
:param tools: Available tools
|
|
93
|
+
:param model: AI model to use
|
|
94
|
+
:return: SQL function definition
|
|
95
|
+
"""
|
|
96
|
+
tool_call_prompt = SQLToolCallResponse.generate_tool_call_prompt(
|
|
97
|
+
prompt, tools, model
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
return f"""
|
|
101
|
+
WITH ai_response AS (
|
|
102
|
+
SELECT SNOWFLAKE.CORTEX.COMPLETE(
|
|
103
|
+
model => '{model}',
|
|
104
|
+
prompt => '{tool_call_prompt}'
|
|
105
|
+
) AS response_json
|
|
106
|
+
),
|
|
107
|
+
parsed_response AS (
|
|
108
|
+
SELECT
|
|
109
|
+
response_json,
|
|
110
|
+
PARSE_JSON(response_json) AS parsed_json
|
|
111
|
+
FROM ai_response
|
|
112
|
+
),
|
|
113
|
+
tool_calls AS (
|
|
114
|
+
SELECT
|
|
115
|
+
elem.tool_name,
|
|
116
|
+
elem.parameters
|
|
117
|
+
FROM parsed_response,
|
|
118
|
+
LATERAL FLATTEN(input => parsed_json:tool_calls) elem
|
|
119
|
+
)
|
|
120
|
+
SELECT
|
|
121
|
+
response_json,
|
|
122
|
+
tool_calls.tool_name,
|
|
123
|
+
tool_calls.parameters
|
|
124
|
+
FROM parsed_response
|
|
125
|
+
LEFT JOIN tool_calls ON 1=1
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
class BigQuerySQLToolCaller:
|
|
129
|
+
"""
|
|
130
|
+
BigQuery-specific tool calling implementation
|
|
131
|
+
"""
|
|
132
|
+
@staticmethod
|
|
133
|
+
def generate_tool_call_sql(
|
|
134
|
+
prompt: str,
|
|
135
|
+
tools: List[Dict[str, Any]],
|
|
136
|
+
model: str = 'text-bison'
|
|
137
|
+
) -> str:
|
|
138
|
+
"""
|
|
139
|
+
Generate a BigQuery ML function for tool calling
|
|
140
|
+
|
|
141
|
+
:param prompt: User prompt
|
|
142
|
+
:param tools: Available tools
|
|
143
|
+
:param model: AI model to use
|
|
144
|
+
:return: SQL function definition
|
|
145
|
+
"""
|
|
146
|
+
tool_call_prompt = SQLToolCallResponse.generate_tool_call_prompt(
|
|
147
|
+
prompt, tools, model
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
return f"""
|
|
151
|
+
ML.PREDICT(
|
|
152
|
+
MODEL `{model}`,
|
|
153
|
+
(
|
|
154
|
+
SELECT '{tool_call_prompt}' AS prompt
|
|
155
|
+
)
|
|
156
|
+
)
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
class SQLToolCallOrchestrator:
|
|
160
|
+
"""
|
|
161
|
+
Orchestrates tool calling across different SQL databases
|
|
162
|
+
"""
|
|
163
|
+
@staticmethod
|
|
164
|
+
def generate_tool_calls(
|
|
165
|
+
engine_type: str,
|
|
166
|
+
prompt: str,
|
|
167
|
+
tools: List[Dict[str, Any]],
|
|
168
|
+
model: Optional[str] = None
|
|
169
|
+
) -> str:
|
|
170
|
+
"""
|
|
171
|
+
Generate appropriate SQL for tool calling
|
|
172
|
+
|
|
173
|
+
:param engine_type: Type of SQL database
|
|
174
|
+
:param prompt: User prompt
|
|
175
|
+
:param tools: Available tools
|
|
176
|
+
:param model: Optional model override
|
|
177
|
+
:return: SQL for tool calling
|
|
178
|
+
"""
|
|
179
|
+
model_map = {
|
|
180
|
+
'snowflake': 'snowflake-arctic',
|
|
181
|
+
'bigquery': 'text-bison'
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
model = model or model_map.get(engine_type.lower(), 'snowflake-arctic')
|
|
185
|
+
|
|
186
|
+
if engine_type.lower() == 'snowflake':
|
|
187
|
+
return SnowflakeSQLToolCaller.generate_tool_call_sql(
|
|
188
|
+
prompt, tools, model
|
|
189
|
+
)
|
|
190
|
+
elif engine_type.lower() == 'bigquery':
|
|
191
|
+
return BigQuerySQLToolCaller.generate_tool_call_sql(
|
|
192
|
+
prompt, tools, model
|
|
193
|
+
)
|
|
194
|
+
else:
|
|
195
|
+
raise ValueError(f"Unsupported engine type: {engine_type}")
|
|
196
|
+
|
|
197
|
+
# Example integration with ModelCompiler
|
|
198
|
+
def _execute_ai_agent_sql(
|
|
199
|
+
self,
|
|
200
|
+
prompt: str,
|
|
201
|
+
tools: List[Dict[str, Any]]
|
|
202
|
+
) -> Dict[str, Any]:
|
|
203
|
+
"""
|
|
204
|
+
Execute an AI agent entirely within SQL
|
|
205
|
+
|
|
206
|
+
:param prompt: User prompt
|
|
207
|
+
:param tools: Available tools
|
|
208
|
+
:return: Tool call results
|
|
209
|
+
"""
|
|
210
|
+
engine_type = self.engine.dialect.name.lower()
|
|
211
|
+
|
|
212
|
+
try:
|
|
213
|
+
tool_call_sql = SQLToolCallOrchestrator.generate_tool_calls(
|
|
214
|
+
engine_type, prompt, tools
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
# Execute the SQL and process results
|
|
218
|
+
df = pd.read_sql(tool_call_sql, self.engine)
|
|
219
|
+
|
|
220
|
+
# Process tool calls and generate final response
|
|
221
|
+
tool_calls = self._process_sql_tool_calls(df)
|
|
222
|
+
|
|
223
|
+
return {
|
|
224
|
+
'tool_calls': tool_calls,
|
|
225
|
+
'final_response': df['final_response'].iloc[0] if not df.empty else ''
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
except Exception as e:
|
|
229
|
+
return {
|
|
230
|
+
'tool_calls': [],
|
|
231
|
+
'final_response': f"Error in SQL tool calling: {str(e)}"
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
def _process_sql_tool_calls(self, df: pd.DataFrame) -> List[Dict[str, Any]]:
|
|
235
|
+
"""
|
|
236
|
+
Process tool calls from SQL result DataFrame
|
|
237
|
+
|
|
238
|
+
:param df: DataFrame containing tool call results
|
|
239
|
+
:return: List of processed tool calls
|
|
240
|
+
"""
|
|
241
|
+
processed_calls = []
|
|
242
|
+
|
|
243
|
+
for _, row in df.iterrows():
|
|
244
|
+
tool_name = row.get('tool_name')
|
|
245
|
+
parameters = row.get('parameters')
|
|
246
|
+
|
|
247
|
+
if tool_name and parameters:
|
|
248
|
+
# Execute the tool using existing tool calling mechanism
|
|
249
|
+
tool_result = self._execute_tool(tool_name, parameters)
|
|
250
|
+
|
|
251
|
+
processed_calls.append({
|
|
252
|
+
'tool_name': tool_name,
|
|
253
|
+
'parameters': parameters,
|
|
254
|
+
'result': tool_result
|
|
255
|
+
})
|
|
256
|
+
|
|
257
|
+
return processed_calls
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import sqlalchemy
|
|
2
|
+
from typing import Dict, Any, Optional, Callable
|
|
3
|
+
import textwrap
|
|
4
|
+
|
|
5
|
+
class DatabaseAIAdapter:
|
|
6
|
+
"""
|
|
7
|
+
Base class for database-specific AI function adapters
|
|
8
|
+
"""
|
|
9
|
+
def __init__(self, engine: sqlalchemy.engine.base.Engine):
|
|
10
|
+
self.engine = engine
|
|
11
|
+
self.dialect = self._get_dialect()
|
|
12
|
+
|
|
13
|
+
def _get_dialect(self) -> str:
|
|
14
|
+
"""Determine the specific database dialect"""
|
|
15
|
+
dialect_map = {
|
|
16
|
+
'postgresql': 'postgresql',
|
|
17
|
+
'mysql': 'mysql',
|
|
18
|
+
'mssql': 'mssql',
|
|
19
|
+
'sqlite': 'sqlite',
|
|
20
|
+
'snowflake': 'snowflake'
|
|
21
|
+
}
|
|
22
|
+
return dialect_map.get(self.engine.dialect.name.lower(), 'unknown')
|
|
23
|
+
|
|
24
|
+
def generate_ai_function(self, function_type: str, prompt: str, **kwargs) -> str:
|
|
25
|
+
"""
|
|
26
|
+
Generate AI function implementation based on database type
|
|
27
|
+
|
|
28
|
+
:param function_type: Type of AI function (generate_text, summarize, etc.)
|
|
29
|
+
:param prompt: Input prompt
|
|
30
|
+
:param kwargs: Additional parameters
|
|
31
|
+
:return: SQL implementation of AI function
|
|
32
|
+
"""
|
|
33
|
+
adapter_method = getattr(self, f'_{self.dialect}_{function_type}', None)
|
|
34
|
+
|
|
35
|
+
if adapter_method:
|
|
36
|
+
return adapter_method(prompt, **kwargs)
|
|
37
|
+
|
|
38
|
+
# Fallback to generic implementation
|
|
39
|
+
return self._generic_ai_function(function_type, prompt, **kwargs)
|
|
40
|
+
|
|
41
|
+
def _generic_ai_function(self, function_type: str, prompt: str, **kwargs) -> str:
|
|
42
|
+
"""
|
|
43
|
+
Generic fallback implementation using Python-based AI processing
|
|
44
|
+
"""
|
|
45
|
+
# Create a temporary table-based approach for AI function simulation
|
|
46
|
+
return textwrap.dedent(f'''
|
|
47
|
+
WITH ai_input AS (
|
|
48
|
+
SELECT '{prompt}' AS input_text
|
|
49
|
+
)
|
|
50
|
+
SELECT
|
|
51
|
+
CASE
|
|
52
|
+
WHEN '{function_type}' = 'generate_text' THEN
|
|
53
|
+
'Generated text based on: ' || input_text
|
|
54
|
+
WHEN '{function_type}' = 'summarize' THEN
|
|
55
|
+
'Summary of: ' || input_text
|
|
56
|
+
WHEN '{function_type}' = 'analyze_sentiment' THEN
|
|
57
|
+
CASE
|
|
58
|
+
WHEN input_text LIKE '%good%' OR input_text LIKE '%great%' THEN 'positive'
|
|
59
|
+
WHEN input_text LIKE '%bad%' OR input_text LIKE '%terrible%' THEN 'negative'
|
|
60
|
+
ELSE 'neutral'
|
|
61
|
+
END
|
|
62
|
+
ELSE input_text
|
|
63
|
+
END AS ai_result
|
|
64
|
+
FROM ai_input
|
|
65
|
+
''')
|
|
66
|
+
|
|
67
|
+
def _postgresql_generate_text(self, prompt: str, **kwargs) -> str:
|
|
68
|
+
"""
|
|
69
|
+
PostgreSQL-specific text generation using pgai extension
|
|
70
|
+
Requires: CREATE EXTENSION IF NOT EXISTS pgai;
|
|
71
|
+
"""
|
|
72
|
+
return textwrap.dedent(f'''
|
|
73
|
+
SELECT pgai.generate_text(
|
|
74
|
+
model => 'openai-gpt-3.5-turbo',
|
|
75
|
+
prompt => '{prompt}'
|
|
76
|
+
) AS generated_text
|
|
77
|
+
''')
|
|
78
|
+
|
|
79
|
+
def _mysql_generate_text(self, prompt: str, **kwargs) -> str:
|
|
80
|
+
"""
|
|
81
|
+
MySQL-specific text generation
|
|
82
|
+
Uses a custom table-based approach with external AI call simulation
|
|
83
|
+
"""
|
|
84
|
+
return textwrap.dedent(f'''
|
|
85
|
+
WITH ai_input AS (
|
|
86
|
+
SELECT '{prompt}' AS input_text
|
|
87
|
+
)
|
|
88
|
+
SELECT
|
|
89
|
+
CONCAT('Generated text based on: ', input_text) AS generated_text
|
|
90
|
+
FROM ai_input
|
|
91
|
+
''')
|
|
92
|
+
|
|
93
|
+
def _mssql_generate_text(self, prompt: str, **kwargs) -> str:
|
|
94
|
+
"""
|
|
95
|
+
MSSQL-specific text generation
|
|
96
|
+
Uses a CLR integration or external call simulation
|
|
97
|
+
"""
|
|
98
|
+
return textwrap.dedent(f'''
|
|
99
|
+
WITH ai_input AS (
|
|
100
|
+
SELECT '{prompt}' AS input_text
|
|
101
|
+
)
|
|
102
|
+
SELECT
|
|
103
|
+
CONCAT('Generated text based on: ', input_text) AS generated_text
|
|
104
|
+
FROM ai_input
|
|
105
|
+
''')
|
|
106
|
+
|
|
107
|
+
def _postgresql_summarize(self, text: str, **kwargs) -> str:
|
|
108
|
+
"""PostgreSQL summarization implementation"""
|
|
109
|
+
return textwrap.dedent(f'''
|
|
110
|
+
SELECT pgai.summarize(
|
|
111
|
+
text => '{text}',
|
|
112
|
+
max_length => 100
|
|
113
|
+
) AS summary
|
|
114
|
+
''')
|
|
115
|
+
|
|
116
|
+
def _postgresql_analyze_sentiment(self, text: str, **kwargs) -> str:
|
|
117
|
+
"""PostgreSQL sentiment analysis implementation"""
|
|
118
|
+
return textwrap.dedent(f'''
|
|
119
|
+
SELECT
|
|
120
|
+
CASE
|
|
121
|
+
WHEN pgai.sentiment_score('{text}') > 0 THEN 'positive'
|
|
122
|
+
WHEN pgai.sentiment_score('{text}') < 0 THEN 'negative'
|
|
123
|
+
ELSE 'neutral'
|
|
124
|
+
END AS sentiment
|
|
125
|
+
''')
|
|
126
|
+
|
|
127
|
+
class AIFunctionRouter:
|
|
128
|
+
"""
|
|
129
|
+
Routes AI function calls to appropriate database-specific adapters
|
|
130
|
+
"""
|
|
131
|
+
@staticmethod
|
|
132
|
+
def route_ai_function(engine: sqlalchemy.engine.base.Engine,
|
|
133
|
+
function_type: str,
|
|
134
|
+
prompt: str,
|
|
135
|
+
**kwargs) -> str:
|
|
136
|
+
"""
|
|
137
|
+
Route AI function to appropriate database adapter
|
|
138
|
+
|
|
139
|
+
:param engine: SQLAlchemy database engine
|
|
140
|
+
:param function_type: Type of AI function
|
|
141
|
+
:param prompt: Input prompt
|
|
142
|
+
:param kwargs: Additional parameters
|
|
143
|
+
:return: SQL implementation of AI function
|
|
144
|
+
"""
|
|
145
|
+
adapter = DatabaseAIAdapter(engine)
|
|
146
|
+
return adapter.generate_ai_function(function_type, prompt, **kwargs)
|
|
147
|
+
|
|
148
|
+
# Example integration with existing ModelCompiler
|
|
149
|
+
def _execute_ai_model(self, sql: str, model: SQLModel) -> pd.DataFrame:
|
|
150
|
+
"""
|
|
151
|
+
Enhanced method to use AI function adapters
|
|
152
|
+
"""
|
|
153
|
+
from npcpy.sql.database_ai_adapters import AIFunctionRouter
|
|
154
|
+
|
|
155
|
+
# Existing code to determine source and engine
|
|
156
|
+
source_pattern = r'FROM\s+(\\w+)\\.(\\w+)'
|
|
157
|
+
matches = re.findall(source_pattern, sql)
|
|
158
|
+
|
|
159
|
+
if matches:
|
|
160
|
+
source_name, table_name = matches[0]
|
|
161
|
+
engine = self._get_engine(source_name)
|
|
162
|
+
|
|
163
|
+
# Modify SQL to use database-specific AI functions
|
|
164
|
+
for func_name, params in model.ai_functions.items():
|
|
165
|
+
try:
|
|
166
|
+
# Route AI function through adapter
|
|
167
|
+
native_func_call = AIFunctionRouter.route_ai_function(
|
|
168
|
+
engine,
|
|
169
|
+
func_name,
|
|
170
|
+
text=params.get('column', ''),
|
|
171
|
+
**{k: v for k, v in params.items() if k != 'column'}
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
# Replace the NQL function with native/adapted function
|
|
175
|
+
sql = sql.replace(
|
|
176
|
+
f"nql.{func_name}({params.get('column', '')})",
|
|
177
|
+
native_func_call
|
|
178
|
+
)
|
|
179
|
+
except Exception as e:
|
|
180
|
+
# Fallback to original method if transformation fails
|
|
181
|
+
print(f"Warning: AI function adaptation failed: {e}. Falling back to default.")
|
|
182
|
+
|
|
183
|
+
return pd.read_sql(sql.replace(f"{source_name}.", ""), engine)
|
|
184
|
+
|
|
185
|
+
# Fallback to existing AI model execution
|
|
186
|
+
return super()._execute_ai_model(sql, model)
|