npcpy 1.0.26__py3-none-any.whl → 1.2.32__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- npcpy/__init__.py +0 -7
- npcpy/data/audio.py +16 -99
- npcpy/data/image.py +43 -42
- npcpy/data/load.py +83 -124
- npcpy/data/text.py +28 -28
- npcpy/data/video.py +8 -32
- npcpy/data/web.py +51 -23
- npcpy/ft/diff.py +110 -0
- npcpy/ft/ge.py +115 -0
- npcpy/ft/memory_trainer.py +171 -0
- npcpy/ft/model_ensembler.py +357 -0
- npcpy/ft/rl.py +360 -0
- npcpy/ft/sft.py +248 -0
- npcpy/ft/usft.py +128 -0
- npcpy/gen/audio_gen.py +24 -0
- npcpy/gen/embeddings.py +13 -13
- npcpy/gen/image_gen.py +262 -117
- npcpy/gen/response.py +615 -415
- npcpy/gen/video_gen.py +53 -7
- npcpy/llm_funcs.py +1869 -437
- npcpy/main.py +1 -1
- npcpy/memory/command_history.py +844 -510
- npcpy/memory/kg_vis.py +833 -0
- npcpy/memory/knowledge_graph.py +892 -1845
- npcpy/memory/memory_processor.py +81 -0
- npcpy/memory/search.py +188 -90
- npcpy/mix/debate.py +192 -3
- npcpy/npc_compiler.py +1672 -801
- npcpy/npc_sysenv.py +593 -1266
- npcpy/serve.py +3120 -0
- npcpy/sql/ai_function_tools.py +257 -0
- npcpy/sql/database_ai_adapters.py +186 -0
- npcpy/sql/database_ai_functions.py +163 -0
- npcpy/sql/model_runner.py +19 -19
- npcpy/sql/npcsql.py +706 -507
- npcpy/sql/sql_model_compiler.py +156 -0
- npcpy/tools.py +183 -0
- npcpy/work/plan.py +13 -279
- npcpy/work/trigger.py +3 -3
- npcpy-1.2.32.dist-info/METADATA +803 -0
- npcpy-1.2.32.dist-info/RECORD +54 -0
- npcpy/data/dataframes.py +0 -171
- npcpy/memory/deep_research.py +0 -125
- npcpy/memory/sleep.py +0 -557
- npcpy/modes/_state.py +0 -78
- npcpy/modes/alicanto.py +0 -1075
- npcpy/modes/guac.py +0 -785
- npcpy/modes/mcp_npcsh.py +0 -822
- npcpy/modes/npc.py +0 -213
- npcpy/modes/npcsh.py +0 -1158
- npcpy/modes/plonk.py +0 -409
- npcpy/modes/pti.py +0 -234
- npcpy/modes/serve.py +0 -1637
- npcpy/modes/spool.py +0 -312
- npcpy/modes/wander.py +0 -549
- npcpy/modes/yap.py +0 -572
- npcpy/npc_team/alicanto.npc +0 -2
- npcpy/npc_team/alicanto.png +0 -0
- npcpy/npc_team/assembly_lines/test_pipeline.py +0 -181
- npcpy/npc_team/corca.npc +0 -13
- npcpy/npc_team/foreman.npc +0 -7
- npcpy/npc_team/frederic.npc +0 -6
- npcpy/npc_team/frederic4.png +0 -0
- npcpy/npc_team/guac.png +0 -0
- npcpy/npc_team/jinxs/automator.jinx +0 -18
- npcpy/npc_team/jinxs/bash_executer.jinx +0 -31
- npcpy/npc_team/jinxs/calculator.jinx +0 -11
- npcpy/npc_team/jinxs/edit_file.jinx +0 -96
- npcpy/npc_team/jinxs/file_chat.jinx +0 -14
- npcpy/npc_team/jinxs/gui_controller.jinx +0 -28
- npcpy/npc_team/jinxs/image_generation.jinx +0 -29
- npcpy/npc_team/jinxs/internet_search.jinx +0 -30
- npcpy/npc_team/jinxs/local_search.jinx +0 -152
- npcpy/npc_team/jinxs/npcsh_executor.jinx +0 -31
- npcpy/npc_team/jinxs/python_executor.jinx +0 -8
- npcpy/npc_team/jinxs/screen_cap.jinx +0 -25
- npcpy/npc_team/jinxs/sql_executor.jinx +0 -33
- npcpy/npc_team/kadiefa.npc +0 -3
- npcpy/npc_team/kadiefa.png +0 -0
- npcpy/npc_team/npcsh.ctx +0 -9
- npcpy/npc_team/npcsh_sibiji.png +0 -0
- npcpy/npc_team/plonk.npc +0 -2
- npcpy/npc_team/plonk.png +0 -0
- npcpy/npc_team/plonkjr.npc +0 -2
- npcpy/npc_team/plonkjr.png +0 -0
- npcpy/npc_team/sibiji.npc +0 -5
- npcpy/npc_team/sibiji.png +0 -0
- npcpy/npc_team/spool.png +0 -0
- npcpy/npc_team/templates/analytics/celona.npc +0 -0
- npcpy/npc_team/templates/hr_support/raone.npc +0 -0
- npcpy/npc_team/templates/humanities/eriane.npc +0 -4
- npcpy/npc_team/templates/it_support/lineru.npc +0 -0
- npcpy/npc_team/templates/marketing/slean.npc +0 -4
- npcpy/npc_team/templates/philosophy/maurawa.npc +0 -0
- npcpy/npc_team/templates/sales/turnic.npc +0 -4
- npcpy/npc_team/templates/software/welxor.npc +0 -0
- npcpy/npc_team/yap.png +0 -0
- npcpy/routes.py +0 -958
- npcpy/work/mcp_helpers.py +0 -357
- npcpy/work/mcp_server.py +0 -194
- npcpy-1.0.26.data/data/npcpy/npc_team/alicanto.npc +0 -2
- npcpy-1.0.26.data/data/npcpy/npc_team/alicanto.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/automator.jinx +0 -18
- npcpy-1.0.26.data/data/npcpy/npc_team/bash_executer.jinx +0 -31
- npcpy-1.0.26.data/data/npcpy/npc_team/calculator.jinx +0 -11
- npcpy-1.0.26.data/data/npcpy/npc_team/celona.npc +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/corca.npc +0 -13
- npcpy-1.0.26.data/data/npcpy/npc_team/edit_file.jinx +0 -96
- npcpy-1.0.26.data/data/npcpy/npc_team/eriane.npc +0 -4
- npcpy-1.0.26.data/data/npcpy/npc_team/file_chat.jinx +0 -14
- npcpy-1.0.26.data/data/npcpy/npc_team/foreman.npc +0 -7
- npcpy-1.0.26.data/data/npcpy/npc_team/frederic.npc +0 -6
- npcpy-1.0.26.data/data/npcpy/npc_team/frederic4.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/guac.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/gui_controller.jinx +0 -28
- npcpy-1.0.26.data/data/npcpy/npc_team/image_generation.jinx +0 -29
- npcpy-1.0.26.data/data/npcpy/npc_team/internet_search.jinx +0 -30
- npcpy-1.0.26.data/data/npcpy/npc_team/kadiefa.npc +0 -3
- npcpy-1.0.26.data/data/npcpy/npc_team/kadiefa.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/lineru.npc +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/local_search.jinx +0 -152
- npcpy-1.0.26.data/data/npcpy/npc_team/maurawa.npc +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/npcsh.ctx +0 -9
- npcpy-1.0.26.data/data/npcpy/npc_team/npcsh_executor.jinx +0 -31
- npcpy-1.0.26.data/data/npcpy/npc_team/npcsh_sibiji.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/plonk.npc +0 -2
- npcpy-1.0.26.data/data/npcpy/npc_team/plonk.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/plonkjr.npc +0 -2
- npcpy-1.0.26.data/data/npcpy/npc_team/plonkjr.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/python_executor.jinx +0 -8
- npcpy-1.0.26.data/data/npcpy/npc_team/raone.npc +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/screen_cap.jinx +0 -25
- npcpy-1.0.26.data/data/npcpy/npc_team/sibiji.npc +0 -5
- npcpy-1.0.26.data/data/npcpy/npc_team/sibiji.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/slean.npc +0 -4
- npcpy-1.0.26.data/data/npcpy/npc_team/spool.png +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/sql_executor.jinx +0 -33
- npcpy-1.0.26.data/data/npcpy/npc_team/test_pipeline.py +0 -181
- npcpy-1.0.26.data/data/npcpy/npc_team/turnic.npc +0 -4
- npcpy-1.0.26.data/data/npcpy/npc_team/welxor.npc +0 -0
- npcpy-1.0.26.data/data/npcpy/npc_team/yap.png +0 -0
- npcpy-1.0.26.dist-info/METADATA +0 -827
- npcpy-1.0.26.dist-info/RECORD +0 -139
- npcpy-1.0.26.dist-info/entry_points.txt +0 -11
- /npcpy/{modes → ft}/__init__.py +0 -0
- {npcpy-1.0.26.dist-info → npcpy-1.2.32.dist-info}/WHEEL +0 -0
- {npcpy-1.0.26.dist-info → npcpy-1.2.32.dist-info}/licenses/LICENSE +0 -0
- {npcpy-1.0.26.dist-info → npcpy-1.2.32.dist-info}/top_level.txt +0 -0
npcpy/modes/yap.py
DELETED
|
@@ -1,572 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
try:
|
|
3
|
-
from faster_whisper import WhisperModel
|
|
4
|
-
from gtts import gTTS
|
|
5
|
-
import torch
|
|
6
|
-
import pyaudio
|
|
7
|
-
import wave
|
|
8
|
-
import queue
|
|
9
|
-
|
|
10
|
-
from npcpy.data.audio import (
|
|
11
|
-
cleanup_temp_files,
|
|
12
|
-
FORMAT,
|
|
13
|
-
CHANNELS,
|
|
14
|
-
RATE,
|
|
15
|
-
CHUNK,
|
|
16
|
-
transcribe_recording,
|
|
17
|
-
convert_mp3_to_wav,
|
|
18
|
-
)
|
|
19
|
-
import threading
|
|
20
|
-
import tempfile
|
|
21
|
-
import os
|
|
22
|
-
import re
|
|
23
|
-
import time
|
|
24
|
-
import numpy as np
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
except Exception as e:
|
|
28
|
-
print(
|
|
29
|
-
"Exception: "
|
|
30
|
-
+ str(e)
|
|
31
|
-
+ "\n"
|
|
32
|
-
+ "Could not load the whisper package. If you want to use tts/stt features, please run `pip install npcsh[audio]` and follow the instructions in the npcsh github readme to ensure your OS can handle the audio dependencies."
|
|
33
|
-
)
|
|
34
|
-
from npcpy.data.load import load_csv, load_pdf
|
|
35
|
-
from npcpy.npc_sysenv import (
|
|
36
|
-
NPCSH_CHAT_MODEL,
|
|
37
|
-
NPCSH_CHAT_PROVIDER,
|
|
38
|
-
NPCSH_DB_PATH,
|
|
39
|
-
NPCSH_API_URL,
|
|
40
|
-
NPCSH_STREAM_OUTPUT,
|
|
41
|
-
get_system_message,
|
|
42
|
-
print_and_process_stream_with_markdown,
|
|
43
|
-
render_markdown
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
)
|
|
47
|
-
from sqlalchemy import create_engine
|
|
48
|
-
from npcpy.llm_funcs import check_llm_command
|
|
49
|
-
from npcpy.data.text import rag_search
|
|
50
|
-
from npcpy.npc_compiler import (
|
|
51
|
-
NPC, Team
|
|
52
|
-
)
|
|
53
|
-
from npcpy.memory.command_history import CommandHistory, save_conversation_message,start_new_conversation
|
|
54
|
-
from typing import Dict, Any, List
|
|
55
|
-
def enter_yap_mode(
|
|
56
|
-
|
|
57
|
-
model: str ,
|
|
58
|
-
provider: str ,
|
|
59
|
-
messages: list = None,
|
|
60
|
-
npc = None,
|
|
61
|
-
team= None,
|
|
62
|
-
tts_model="kokoro",
|
|
63
|
-
voice="af_heart",
|
|
64
|
-
files: List[str] = None,
|
|
65
|
-
rag_similarity_threshold: float = 0.3,
|
|
66
|
-
stream: bool = NPCSH_STREAM_OUTPUT,
|
|
67
|
-
conversation_id = None,
|
|
68
|
-
) -> Dict[str, Any]:
|
|
69
|
-
running = True
|
|
70
|
-
is_recording = False
|
|
71
|
-
recording_data = []
|
|
72
|
-
buffer_data = []
|
|
73
|
-
last_speech_time = 0
|
|
74
|
-
vad_model, _ = torch.hub.load(
|
|
75
|
-
repo_or_dir="snakers4/silero-vad",
|
|
76
|
-
model="silero_vad",
|
|
77
|
-
force_reload=False,
|
|
78
|
-
onnx=False,
|
|
79
|
-
verbose=False,
|
|
80
|
-
|
|
81
|
-
)
|
|
82
|
-
device = 'cpu'
|
|
83
|
-
vad_model.to(device)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
print("Entering yap mode. Initializing...")
|
|
87
|
-
|
|
88
|
-
concise_instruction = "Please provide brief responses of 1-2 sentences unless the user specifically asks for more detailed information. Keep responses clear and concise."
|
|
89
|
-
|
|
90
|
-
provider = (
|
|
91
|
-
NPCSH_CHAT_PROVIDER if npc is None else npc.provider or NPCSH_CHAT_PROVIDER
|
|
92
|
-
)
|
|
93
|
-
api_url = NPCSH_API_URL if npc is None else npc.api_url or NPCSH_API_URL
|
|
94
|
-
|
|
95
|
-
print(f"\nUsing model: {model} with provider: {provider}")
|
|
96
|
-
|
|
97
|
-
system_message = get_system_message(npc) if npc else "You are a helpful assistant."
|
|
98
|
-
|
|
99
|
-
# Add conciseness instruction to the system message
|
|
100
|
-
system_message = system_message + " " + concise_instruction
|
|
101
|
-
|
|
102
|
-
if messages is None:
|
|
103
|
-
messages = [{"role": "system", "content": system_message}]
|
|
104
|
-
elif messages is not None and messages[0]['role'] != 'system':
|
|
105
|
-
messages.insert(0, {"role": "system", "content": system_message})
|
|
106
|
-
|
|
107
|
-
kokoro_pipeline = None
|
|
108
|
-
if tts_model == "kokoro":
|
|
109
|
-
try:
|
|
110
|
-
from kokoro import KPipeline
|
|
111
|
-
import soundfile as sf
|
|
112
|
-
|
|
113
|
-
kokoro_pipeline = KPipeline(lang_code="a")
|
|
114
|
-
print("Kokoro TTS model initialized")
|
|
115
|
-
except ImportError:
|
|
116
|
-
print("Kokoro not installed, falling back to gTTS")
|
|
117
|
-
tts_model = "gtts"
|
|
118
|
-
|
|
119
|
-
# Initialize PyAudio
|
|
120
|
-
pyaudio_instance = pyaudio.PyAudio()
|
|
121
|
-
audio_stream = None # We'll open and close as needed
|
|
122
|
-
transcription_queue = queue.Queue()
|
|
123
|
-
|
|
124
|
-
# Create and properly use the is_speaking event
|
|
125
|
-
is_speaking = threading.Event()
|
|
126
|
-
is_speaking.clear() # Not speaking initially
|
|
127
|
-
|
|
128
|
-
speech_queue = queue.Queue(maxsize=20)
|
|
129
|
-
speech_thread_active = threading.Event()
|
|
130
|
-
speech_thread_active.set()
|
|
131
|
-
|
|
132
|
-
def speech_playback_thread():
|
|
133
|
-
nonlocal running, audio_stream
|
|
134
|
-
|
|
135
|
-
while running and speech_thread_active.is_set():
|
|
136
|
-
try:
|
|
137
|
-
# Get next speech item from queue
|
|
138
|
-
if not speech_queue.empty():
|
|
139
|
-
text_to_speak = speech_queue.get(timeout=0.1)
|
|
140
|
-
|
|
141
|
-
# Only process if there's text to speak
|
|
142
|
-
if text_to_speak.strip():
|
|
143
|
-
# IMPORTANT: Set is_speaking flag BEFORE starting audio output
|
|
144
|
-
is_speaking.set()
|
|
145
|
-
|
|
146
|
-
# Safely close the audio input stream before speaking
|
|
147
|
-
current_audio_stream = audio_stream
|
|
148
|
-
audio_stream = (
|
|
149
|
-
None # Set to None to prevent capture thread from using it
|
|
150
|
-
)
|
|
151
|
-
|
|
152
|
-
if current_audio_stream and current_audio_stream.is_active():
|
|
153
|
-
current_audio_stream.stop_stream()
|
|
154
|
-
current_audio_stream.close()
|
|
155
|
-
|
|
156
|
-
print(f"Speaking full response...")
|
|
157
|
-
|
|
158
|
-
# Generate and play speech
|
|
159
|
-
generate_and_play_speech(text_to_speak)
|
|
160
|
-
|
|
161
|
-
# Delay after speech to prevent echo
|
|
162
|
-
time.sleep(0.005 * len(text_to_speak))
|
|
163
|
-
print(len(text_to_speak))
|
|
164
|
-
|
|
165
|
-
# Clear the speaking flag to allow listening again
|
|
166
|
-
is_speaking.clear()
|
|
167
|
-
else:
|
|
168
|
-
time.sleep(0.5)
|
|
169
|
-
except Exception as e:
|
|
170
|
-
print(f"Error in speech thread: {e}")
|
|
171
|
-
is_speaking.clear() # Make sure to clear the flag if there's an error
|
|
172
|
-
time.sleep(0.1)
|
|
173
|
-
|
|
174
|
-
def safely_close_audio_stream(stream):
|
|
175
|
-
"""Safely close an audio stream with error handling"""
|
|
176
|
-
if stream:
|
|
177
|
-
try:
|
|
178
|
-
if stream.is_active():
|
|
179
|
-
stream.stop_stream()
|
|
180
|
-
stream.close()
|
|
181
|
-
except Exception as e:
|
|
182
|
-
print(f"Error closing audio stream: {e}")
|
|
183
|
-
|
|
184
|
-
# Start speech thread
|
|
185
|
-
speech_thread = threading.Thread(target=speech_playback_thread)
|
|
186
|
-
speech_thread.daemon = True
|
|
187
|
-
speech_thread.start()
|
|
188
|
-
|
|
189
|
-
def generate_and_play_speech(text):
|
|
190
|
-
try:
|
|
191
|
-
# Create a temporary file for audio
|
|
192
|
-
unique_id = str(time.time()).replace(".", "")
|
|
193
|
-
temp_dir = tempfile.gettempdir()
|
|
194
|
-
wav_file = os.path.join(temp_dir, f"temp_{unique_id}.wav")
|
|
195
|
-
|
|
196
|
-
# Generate speech based on selected TTS model
|
|
197
|
-
if tts_model == "kokoro" and kokoro_pipeline:
|
|
198
|
-
# Use Kokoro for generation
|
|
199
|
-
generator = kokoro_pipeline(text, voice=voice)
|
|
200
|
-
|
|
201
|
-
# Get the audio from the generator
|
|
202
|
-
for _, _, audio in generator:
|
|
203
|
-
# Save audio to WAV file
|
|
204
|
-
import soundfile as sf
|
|
205
|
-
|
|
206
|
-
sf.write(wav_file, audio, 24000)
|
|
207
|
-
break # Just use the first chunk for now
|
|
208
|
-
else:
|
|
209
|
-
# Fall back to gTTS
|
|
210
|
-
mp3_file = os.path.join(temp_dir, f"temp_{unique_id}.mp3")
|
|
211
|
-
tts = gTTS(text=text, lang="en", slow=False)
|
|
212
|
-
tts.save(mp3_file)
|
|
213
|
-
convert_mp3_to_wav(mp3_file, wav_file)
|
|
214
|
-
|
|
215
|
-
# Play the audio
|
|
216
|
-
wf = wave.open(wav_file, "rb")
|
|
217
|
-
p = pyaudio.PyAudio()
|
|
218
|
-
|
|
219
|
-
stream = p.open(
|
|
220
|
-
format=p.get_format_from_width(wf.getsampwidth()),
|
|
221
|
-
channels=wf.getnchannels(),
|
|
222
|
-
rate=wf.getframerate(),
|
|
223
|
-
output=True,
|
|
224
|
-
)
|
|
225
|
-
|
|
226
|
-
data = wf.readframes(4096)
|
|
227
|
-
while data and running:
|
|
228
|
-
stream.write(data)
|
|
229
|
-
data = wf.readframes(4096)
|
|
230
|
-
|
|
231
|
-
stream.stop_stream()
|
|
232
|
-
stream.close()
|
|
233
|
-
p.terminate()
|
|
234
|
-
|
|
235
|
-
# Cleanup temp files
|
|
236
|
-
try:
|
|
237
|
-
if os.path.exists(wav_file):
|
|
238
|
-
os.remove(wav_file)
|
|
239
|
-
if tts_model == "gtts" and "mp3_file" in locals():
|
|
240
|
-
if os.path.exists(mp3_file):
|
|
241
|
-
os.remove(mp3_file)
|
|
242
|
-
except Exception as e:
|
|
243
|
-
print(f"Error removing temp file: {e}")
|
|
244
|
-
|
|
245
|
-
except Exception as e:
|
|
246
|
-
print(f"Error in TTS process: {e}")
|
|
247
|
-
|
|
248
|
-
# Modified speak_text function that just queues text
|
|
249
|
-
def speak_text(text):
|
|
250
|
-
speech_queue.put(text)
|
|
251
|
-
|
|
252
|
-
def process_input(user_input, messages):
|
|
253
|
-
#try:
|
|
254
|
-
full_response = ""
|
|
255
|
-
|
|
256
|
-
# Use get_stream for streaming response
|
|
257
|
-
check = check_llm_command(
|
|
258
|
-
user_input,
|
|
259
|
-
npc=npc,
|
|
260
|
-
team=team,
|
|
261
|
-
messages=messages,
|
|
262
|
-
model=model,
|
|
263
|
-
provider=provider,
|
|
264
|
-
stream=False,
|
|
265
|
-
)
|
|
266
|
-
#mport pdb
|
|
267
|
-
#pdb.set_trace()
|
|
268
|
-
assistant_reply = check["output"]
|
|
269
|
-
messages = check['messages']
|
|
270
|
-
#print(messages)
|
|
271
|
-
#import pdb
|
|
272
|
-
#pdb.set_trace()
|
|
273
|
-
if stream and not isinstance(assistant_reply,str) and not isinstance(assistant_reply, dict):
|
|
274
|
-
assistant_reply = print_and_process_stream_with_markdown(assistant_reply, model, provider)
|
|
275
|
-
elif isinstance(assistant_reply,dict):
|
|
276
|
-
# assume its a jinx output, to fix later
|
|
277
|
-
assistant_reply = assistant_reply.get('output')
|
|
278
|
-
render_markdown(assistant_reply)
|
|
279
|
-
full_response += assistant_reply
|
|
280
|
-
|
|
281
|
-
print("\n") # End the progress display
|
|
282
|
-
|
|
283
|
-
# Process and speak the entire response at once
|
|
284
|
-
if full_response.strip():
|
|
285
|
-
processed_text = process_text_for_tts(full_response)
|
|
286
|
-
speak_text(processed_text)
|
|
287
|
-
|
|
288
|
-
# Add assistant's response to messages
|
|
289
|
-
messages.append({"role": "assistant", "content": full_response})
|
|
290
|
-
return messages
|
|
291
|
-
#except Exception as e:
|
|
292
|
-
# print(f"Error in LLM response: {e}")
|
|
293
|
-
# speak_text("I'm sorry, there was an error processing your request.")
|
|
294
|
-
|
|
295
|
-
# Function to capture and process audio
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
def capture_audio():
|
|
299
|
-
nonlocal is_recording, recording_data, buffer_data, last_speech_time, running, is_speaking
|
|
300
|
-
nonlocal audio_stream, transcription_queue
|
|
301
|
-
|
|
302
|
-
# Don't try to record if we're speaking
|
|
303
|
-
if is_speaking.is_set():
|
|
304
|
-
return False
|
|
305
|
-
|
|
306
|
-
try:
|
|
307
|
-
# Only create a new audio stream if we don't have one
|
|
308
|
-
if audio_stream is None and not is_speaking.is_set():
|
|
309
|
-
audio_stream = pyaudio_instance.open(
|
|
310
|
-
format=FORMAT,
|
|
311
|
-
channels=CHANNELS,
|
|
312
|
-
rate=RATE,
|
|
313
|
-
input=True,
|
|
314
|
-
frames_per_buffer=CHUNK,
|
|
315
|
-
)
|
|
316
|
-
|
|
317
|
-
# Initialize or reset the recording variables
|
|
318
|
-
is_recording = False
|
|
319
|
-
recording_data = []
|
|
320
|
-
buffer_data = []
|
|
321
|
-
|
|
322
|
-
print("\nListening for speech...")
|
|
323
|
-
|
|
324
|
-
while (
|
|
325
|
-
running
|
|
326
|
-
and audio_stream
|
|
327
|
-
and audio_stream.is_active()
|
|
328
|
-
and not is_speaking.is_set()
|
|
329
|
-
):
|
|
330
|
-
try:
|
|
331
|
-
data = audio_stream.read(CHUNK, exception_on_overflow=False)
|
|
332
|
-
if data:
|
|
333
|
-
audio_array = np.frombuffer(data, dtype=np.int16)
|
|
334
|
-
audio_float = audio_array.astype(np.float32) / 32768.0
|
|
335
|
-
|
|
336
|
-
tensor = torch.from_numpy(audio_float).to(device)
|
|
337
|
-
speech_prob = vad_model(tensor, RATE).item()
|
|
338
|
-
current_time = time.time()
|
|
339
|
-
|
|
340
|
-
if speech_prob > 0.5: # VAD threshold
|
|
341
|
-
last_speech_time = current_time
|
|
342
|
-
if not is_recording:
|
|
343
|
-
is_recording = True
|
|
344
|
-
print("\nSpeech detected, listening...")
|
|
345
|
-
recording_data.extend(buffer_data)
|
|
346
|
-
buffer_data = []
|
|
347
|
-
recording_data.append(data)
|
|
348
|
-
else:
|
|
349
|
-
if is_recording:
|
|
350
|
-
if (
|
|
351
|
-
current_time - last_speech_time > 1
|
|
352
|
-
): # silence duration
|
|
353
|
-
is_recording = False
|
|
354
|
-
print("Speech ended, transcribing...")
|
|
355
|
-
|
|
356
|
-
# Stop stream before transcribing
|
|
357
|
-
safely_close_audio_stream(audio_stream)
|
|
358
|
-
audio_stream = None
|
|
359
|
-
|
|
360
|
-
# Transcribe in this thread to avoid race conditions
|
|
361
|
-
transcription = transcribe_recording(recording_data)
|
|
362
|
-
if transcription:
|
|
363
|
-
transcription_queue.put(transcription)
|
|
364
|
-
recording_data = []
|
|
365
|
-
return True # Got speech
|
|
366
|
-
else:
|
|
367
|
-
buffer_data.append(data)
|
|
368
|
-
if len(buffer_data) > int(
|
|
369
|
-
0.65 * RATE / CHUNK
|
|
370
|
-
): # buffer duration
|
|
371
|
-
buffer_data.pop(0)
|
|
372
|
-
|
|
373
|
-
# Check frequently if we need to stop capturing
|
|
374
|
-
if is_speaking.is_set():
|
|
375
|
-
safely_close_audio_stream(audio_stream)
|
|
376
|
-
audio_stream = None
|
|
377
|
-
return False
|
|
378
|
-
|
|
379
|
-
except Exception as e:
|
|
380
|
-
print(f"Error processing audio frame: {e}")
|
|
381
|
-
time.sleep(0.1)
|
|
382
|
-
|
|
383
|
-
except Exception as e:
|
|
384
|
-
print(f"Error in audio capture: {e}")
|
|
385
|
-
|
|
386
|
-
# Close stream if we exit without finding speech
|
|
387
|
-
safely_close_audio_stream(audio_stream)
|
|
388
|
-
audio_stream = None
|
|
389
|
-
|
|
390
|
-
return False
|
|
391
|
-
|
|
392
|
-
def process_text_for_tts(text):
|
|
393
|
-
# Remove special characters that might cause issues in TTS
|
|
394
|
-
text = re.sub(r"[*<>{}()\[\]&%#@^_=+~]", "", text)
|
|
395
|
-
text = text.strip()
|
|
396
|
-
# Add spaces after periods that are followed by words (for better pronunciation)
|
|
397
|
-
text = re.sub(r"(\w)\.(\w)\.", r"\1 \2 ", text)
|
|
398
|
-
text = re.sub(r"([.!?])(\w)", r"\1 \2", text)
|
|
399
|
-
return text
|
|
400
|
-
|
|
401
|
-
# Now that functions are defined, play welcome messages
|
|
402
|
-
speak_text("Entering yap mode. Please wait.")
|
|
403
|
-
|
|
404
|
-
try:
|
|
405
|
-
loaded_content = {} # New dictionary to hold loaded content
|
|
406
|
-
if not conversation_id:
|
|
407
|
-
conversation_id = start_new_conversation()
|
|
408
|
-
command_history = CommandHistory()
|
|
409
|
-
# Load specified files if any
|
|
410
|
-
if files:
|
|
411
|
-
for file in files:
|
|
412
|
-
extension = os.path.splitext(file)[1].lower()
|
|
413
|
-
try:
|
|
414
|
-
if extension == ".pdf":
|
|
415
|
-
content = load_pdf(file)["texts"].iloc[0]
|
|
416
|
-
elif extension == ".csv":
|
|
417
|
-
content = load_csv(file)
|
|
418
|
-
else:
|
|
419
|
-
print(f"Unsupported file type: {file}")
|
|
420
|
-
continue
|
|
421
|
-
loaded_content[file] = content
|
|
422
|
-
print(f"Loaded content from: {file}")
|
|
423
|
-
except Exception as e:
|
|
424
|
-
print(f"Error loading {file}: {str(e)}")
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
while running:
|
|
429
|
-
|
|
430
|
-
# First check for typed input (non-blocking)
|
|
431
|
-
import select
|
|
432
|
-
import sys
|
|
433
|
-
|
|
434
|
-
# Don't spam the console with prompts when speaking
|
|
435
|
-
if not is_speaking.is_set():
|
|
436
|
-
print(
|
|
437
|
-
"🎤🎤🎤🎤\n Speak or type your message (or 'exit' to quit): ",
|
|
438
|
-
end="",
|
|
439
|
-
flush=True,
|
|
440
|
-
)
|
|
441
|
-
|
|
442
|
-
rlist, _, _ = select.select([sys.stdin], [], [], 0.1)
|
|
443
|
-
if rlist:
|
|
444
|
-
user_input = sys.stdin.readline().strip()
|
|
445
|
-
if user_input.lower() in ("exit", "quit", "goodbye"):
|
|
446
|
-
print("\nExiting yap mode.")
|
|
447
|
-
break
|
|
448
|
-
if user_input:
|
|
449
|
-
print(f"\nYou (typed): {user_input}")
|
|
450
|
-
# Handle RAG context
|
|
451
|
-
if loaded_content:
|
|
452
|
-
context_content = ""
|
|
453
|
-
for filename, content in loaded_content.items():
|
|
454
|
-
retrieved_docs = rag_search(
|
|
455
|
-
user_input,
|
|
456
|
-
content,
|
|
457
|
-
similarity_threshold=rag_similarity_threshold,
|
|
458
|
-
)
|
|
459
|
-
if retrieved_docs:
|
|
460
|
-
context_content += (
|
|
461
|
-
f"\n\nLoaded content from: {filename}\n{content}\n\n"
|
|
462
|
-
)
|
|
463
|
-
if len(context_content) > 0:
|
|
464
|
-
user_input += f"""
|
|
465
|
-
Here is the loaded content that may be relevant to your query:
|
|
466
|
-
{context_content}
|
|
467
|
-
Please reference it explicitly in your response and use it for answering.
|
|
468
|
-
"""
|
|
469
|
-
message_id = save_conversation_message(
|
|
470
|
-
command_history,
|
|
471
|
-
conversation_id,
|
|
472
|
-
"user",
|
|
473
|
-
user_input,
|
|
474
|
-
wd=os.getcwd(),
|
|
475
|
-
model=model,
|
|
476
|
-
provider=provider,
|
|
477
|
-
npc=npc.name if npc else None,
|
|
478
|
-
)
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
messages= process_input(user_input, messages)
|
|
482
|
-
|
|
483
|
-
message_id = save_conversation_message(
|
|
484
|
-
command_history,
|
|
485
|
-
conversation_id,
|
|
486
|
-
"assistant",
|
|
487
|
-
messages[-1]["content"],
|
|
488
|
-
wd=os.getcwd(),
|
|
489
|
-
model=model,
|
|
490
|
-
provider=provider,
|
|
491
|
-
npc=npc.name if npc else None,
|
|
492
|
-
)
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
continue # Skip audio capture this cycle
|
|
496
|
-
|
|
497
|
-
# Then try to capture some audio (if no typed input)
|
|
498
|
-
if not is_speaking.is_set(): # Only capture if not currently speaking
|
|
499
|
-
got_speech = capture_audio()
|
|
500
|
-
|
|
501
|
-
# If we got speech, process it
|
|
502
|
-
if got_speech:
|
|
503
|
-
try:
|
|
504
|
-
transcription = transcription_queue.get_nowait()
|
|
505
|
-
print(f"\nYou (spoke): {transcription}")
|
|
506
|
-
messages = process_input(transcription, messages)
|
|
507
|
-
except queue.Empty:
|
|
508
|
-
pass
|
|
509
|
-
else:
|
|
510
|
-
# If we're speaking, just wait a bit without spamming the console
|
|
511
|
-
time.sleep(0.1)
|
|
512
|
-
|
|
513
|
-
except KeyboardInterrupt:
|
|
514
|
-
print("\nInterrupted by user.")
|
|
515
|
-
|
|
516
|
-
finally:
|
|
517
|
-
# Set running to False to signal threads to exit
|
|
518
|
-
running = False
|
|
519
|
-
speech_thread_active.clear()
|
|
520
|
-
|
|
521
|
-
# Clean up audio resources
|
|
522
|
-
safely_close_audio_stream(audio_stream)
|
|
523
|
-
|
|
524
|
-
if pyaudio_instance:
|
|
525
|
-
pyaudio_instance.terminate()
|
|
526
|
-
|
|
527
|
-
print("\nExiting yap mode.")
|
|
528
|
-
speak_text("Exiting yap mode. Goodbye!")
|
|
529
|
-
time.sleep(1)
|
|
530
|
-
cleanup_temp_files()
|
|
531
|
-
|
|
532
|
-
return {"messages": messages, "output": "yap mode session ended."}
|
|
533
|
-
|
|
534
|
-
def main():
|
|
535
|
-
# Example usage
|
|
536
|
-
import argparse
|
|
537
|
-
parser = argparse.ArgumentParser(description="Enter yap mode for chatting with an NPC")
|
|
538
|
-
parser.add_argument("--model", default=NPCSH_CHAT_MODEL, help="Model to use")
|
|
539
|
-
parser.add_argument("--provider", default=NPCSH_CHAT_PROVIDER, help="Provider to use")
|
|
540
|
-
parser.add_argument("--files", nargs="*", help="Files to load into context")
|
|
541
|
-
parser.add_argument("--stream", default="true", help="Use streaming mode")
|
|
542
|
-
parser.add_argument("--npc", type=str, default=os.path.expanduser('~/.npcsh/npc_team/sibiji.npc'), help="Path to NPC file")
|
|
543
|
-
args = parser.parse_args()
|
|
544
|
-
npc_db_conn = create_engine(
|
|
545
|
-
f"sqlite:///{NPCSH_DB_PATH}")
|
|
546
|
-
|
|
547
|
-
sibiji = NPC(file=args.npc, db_conn=npc_db_conn)
|
|
548
|
-
|
|
549
|
-
team = Team(team_path = '~/.npcsh/npc_team/', db_conn=npc_db_conn, forenpc= sibiji)
|
|
550
|
-
if sibiji.model is None:
|
|
551
|
-
sibiji.model = args.model
|
|
552
|
-
model = args.model
|
|
553
|
-
else:
|
|
554
|
-
model = sibiji.model
|
|
555
|
-
if sibiji.provider is None:
|
|
556
|
-
sibiji.provider = args.provider
|
|
557
|
-
provider = args.provider
|
|
558
|
-
else:
|
|
559
|
-
provider = sibiji.provider
|
|
560
|
-
# Enter spool mode
|
|
561
|
-
enter_yap_mode(
|
|
562
|
-
model,
|
|
563
|
-
provider,
|
|
564
|
-
messages=None,
|
|
565
|
-
npc=sibiji,
|
|
566
|
-
team = team,
|
|
567
|
-
files=args.files,
|
|
568
|
-
stream= args.stream.lower() == "true",
|
|
569
|
-
)
|
|
570
|
-
|
|
571
|
-
if __name__ == "__main__":
|
|
572
|
-
main()
|
npcpy/npc_team/alicanto.npc
DELETED
npcpy/npc_team/alicanto.png
DELETED
|
Binary file
|