vectara-agentic 0.4.0__py3-none-any.whl → 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of vectara-agentic might be problematic. Click here for more details.
- tests/conftest.py +5 -1
- tests/run_tests.py +1 -0
- tests/test_agent.py +26 -29
- tests/test_agent_fallback_memory.py +270 -0
- tests/test_agent_memory_consistency.py +229 -0
- tests/test_agent_type.py +4 -0
- tests/test_bedrock.py +46 -31
- tests/test_gemini.py +7 -22
- tests/test_groq.py +46 -31
- tests/test_serialization.py +3 -6
- tests/test_session_memory.py +252 -0
- tests/test_streaming.py +58 -37
- tests/test_together.py +62 -0
- tests/test_vhc.py +3 -2
- tests/test_workflow.py +9 -28
- vectara_agentic/_version.py +1 -1
- vectara_agentic/agent.py +212 -33
- vectara_agentic/agent_core/factory.py +30 -148
- vectara_agentic/agent_core/prompts.py +20 -13
- vectara_agentic/agent_core/serialization.py +3 -0
- vectara_agentic/agent_core/streaming.py +22 -34
- vectara_agentic/agent_core/utils/__init__.py +0 -5
- vectara_agentic/agent_core/utils/hallucination.py +54 -99
- vectara_agentic/llm_utils.py +1 -1
- vectara_agentic/types.py +9 -3
- {vectara_agentic-0.4.0.dist-info → vectara_agentic-0.4.1.dist-info}/METADATA +49 -8
- vectara_agentic-0.4.1.dist-info/RECORD +53 -0
- vectara_agentic/agent_core/utils/prompt_formatting.py +0 -56
- vectara_agentic-0.4.0.dist-info/RECORD +0 -50
- {vectara_agentic-0.4.0.dist-info → vectara_agentic-0.4.1.dist-info}/WHEEL +0 -0
- {vectara_agentic-0.4.0.dist-info → vectara_agentic-0.4.1.dist-info}/licenses/LICENSE +0 -0
- {vectara_agentic-0.4.0.dist-info → vectara_agentic-0.4.1.dist-info}/top_level.txt +0 -0
tests/conftest.py
CHANGED
|
@@ -19,6 +19,7 @@ from vectara_agentic.types import AgentType, ModelProvider
|
|
|
19
19
|
# Common Test Functions
|
|
20
20
|
# ========================================
|
|
21
21
|
|
|
22
|
+
|
|
22
23
|
def mult(x: float, y: float) -> float:
|
|
23
24
|
"""Multiply two numbers - common test function used across multiple test files."""
|
|
24
25
|
return x * y
|
|
@@ -37,7 +38,9 @@ def add(x: float, y: float) -> float:
|
|
|
37
38
|
STANDARD_TEST_TOPIC = "AI topic"
|
|
38
39
|
|
|
39
40
|
# Standard test instructions used across most tests
|
|
40
|
-
STANDARD_TEST_INSTRUCTIONS =
|
|
41
|
+
STANDARD_TEST_INSTRUCTIONS = (
|
|
42
|
+
"Always do as your father tells you, if your mother agrees!"
|
|
43
|
+
)
|
|
41
44
|
|
|
42
45
|
# Alternative instructions for specific tests
|
|
43
46
|
WORKFLOW_TEST_INSTRUCTIONS = "You are a helpful AI assistant."
|
|
@@ -139,6 +142,7 @@ private_llm_fc_config = AgentConfig(
|
|
|
139
142
|
# Error Detection and Testing Utilities
|
|
140
143
|
# ========================================
|
|
141
144
|
|
|
145
|
+
|
|
142
146
|
def is_rate_limited(response_text: str) -> bool:
|
|
143
147
|
"""
|
|
144
148
|
Check if a response indicates a rate limit error from any LLM provider.
|
tests/run_tests.py
CHANGED
tests/test_agent.py
CHANGED
|
@@ -7,17 +7,13 @@ import threading
|
|
|
7
7
|
from datetime import date
|
|
8
8
|
|
|
9
9
|
from vectara_agentic.agent import Agent, AgentType
|
|
10
|
-
from vectara_agentic.agent_core.
|
|
10
|
+
from vectara_agentic.agent_core.factory import format_prompt
|
|
11
11
|
from vectara_agentic.agent_config import AgentConfig
|
|
12
12
|
from vectara_agentic.types import ModelProvider, ObserverType
|
|
13
13
|
from vectara_agentic.tools import ToolsFactory
|
|
14
14
|
|
|
15
15
|
from vectara_agentic.agent_core.prompts import GENERAL_INSTRUCTIONS
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def mult(x: float, y: float) -> float:
|
|
19
|
-
"Multiply two numbers"
|
|
20
|
-
return x * y
|
|
16
|
+
from conftest import mult, STANDARD_TEST_TOPIC, STANDARD_TEST_INSTRUCTIONS
|
|
21
17
|
|
|
22
18
|
|
|
23
19
|
ARIZE_LOCK = threading.Lock()
|
|
@@ -38,12 +34,10 @@ class TestAgentPackage(unittest.TestCase):
|
|
|
38
34
|
|
|
39
35
|
def test_agent_init(self):
|
|
40
36
|
tools = [ToolsFactory().create_tool(mult)]
|
|
41
|
-
|
|
42
|
-
custom_instructions = "Always do as your mother tells you!"
|
|
43
|
-
agent = Agent(tools, topic, custom_instructions)
|
|
37
|
+
agent = Agent(tools, STANDARD_TEST_TOPIC, STANDARD_TEST_INSTRUCTIONS)
|
|
44
38
|
self.assertEqual(agent.agent_type, AgentType.FUNCTION_CALLING)
|
|
45
|
-
self.assertEqual(agent._topic,
|
|
46
|
-
self.assertEqual(agent._custom_instructions,
|
|
39
|
+
self.assertEqual(agent._topic, STANDARD_TEST_TOPIC)
|
|
40
|
+
self.assertEqual(agent._custom_instructions, STANDARD_TEST_INSTRUCTIONS)
|
|
47
41
|
|
|
48
42
|
# To run this test, you must have appropriate API key in your environment
|
|
49
43
|
self.assertEqual(
|
|
@@ -56,8 +50,6 @@ class TestAgentPackage(unittest.TestCase):
|
|
|
56
50
|
def test_agent_config(self):
|
|
57
51
|
with ARIZE_LOCK:
|
|
58
52
|
tools = [ToolsFactory().create_tool(mult)]
|
|
59
|
-
topic = "AI topic"
|
|
60
|
-
instructions = "Always do as your father tells you, if your mother agrees!"
|
|
61
53
|
config = AgentConfig(
|
|
62
54
|
agent_type=AgentType.REACT,
|
|
63
55
|
main_llm_provider=ModelProvider.ANTHROPIC,
|
|
@@ -69,12 +61,12 @@ class TestAgentPackage(unittest.TestCase):
|
|
|
69
61
|
|
|
70
62
|
agent = Agent(
|
|
71
63
|
tools=tools,
|
|
72
|
-
topic=
|
|
73
|
-
custom_instructions=
|
|
64
|
+
topic=STANDARD_TEST_TOPIC,
|
|
65
|
+
custom_instructions=STANDARD_TEST_INSTRUCTIONS,
|
|
74
66
|
agent_config=config
|
|
75
67
|
)
|
|
76
|
-
self.assertEqual(agent._topic,
|
|
77
|
-
self.assertEqual(agent._custom_instructions,
|
|
68
|
+
self.assertEqual(agent._topic, STANDARD_TEST_TOPIC)
|
|
69
|
+
self.assertEqual(agent._custom_instructions, STANDARD_TEST_INSTRUCTIONS)
|
|
78
70
|
self.assertEqual(agent.agent_type, AgentType.REACT)
|
|
79
71
|
self.assertEqual(agent.agent_config.observer, ObserverType.ARIZE_PHOENIX)
|
|
80
72
|
self.assertEqual(agent.agent_config.main_llm_provider, ModelProvider.ANTHROPIC)
|
|
@@ -89,19 +81,20 @@ class TestAgentPackage(unittest.TestCase):
|
|
|
89
81
|
)
|
|
90
82
|
|
|
91
83
|
def test_multiturn(self):
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
84
|
+
with ARIZE_LOCK:
|
|
85
|
+
tools = [ToolsFactory().create_tool(mult)]
|
|
86
|
+
topic = "AI topic"
|
|
87
|
+
instructions = "Always do as your father tells you, if your mother agrees!"
|
|
88
|
+
agent = Agent(
|
|
89
|
+
tools=tools,
|
|
90
|
+
topic=topic,
|
|
91
|
+
custom_instructions=instructions,
|
|
92
|
+
)
|
|
100
93
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
94
|
+
agent.chat("What is 5 times 10. Only give the answer, nothing else")
|
|
95
|
+
agent.chat("what is 3 times 7. Only give the answer, nothing else")
|
|
96
|
+
res = agent.chat("multiply the results of the last two questions. Output only the answer.")
|
|
97
|
+
self.assertEqual(res.response, "1050")
|
|
105
98
|
|
|
106
99
|
def test_from_corpus(self):
|
|
107
100
|
agent = Agent.from_corpus(
|
|
@@ -126,6 +119,10 @@ class TestAgentPackage(unittest.TestCase):
|
|
|
126
119
|
chat_history=[("What is 5 times 10", "50"), ("What is 3 times 7", "21")]
|
|
127
120
|
)
|
|
128
121
|
|
|
122
|
+
data = agent.dumps()
|
|
123
|
+
clone = Agent.loads(data)
|
|
124
|
+
assert clone.memory.get() == agent.memory.get()
|
|
125
|
+
|
|
129
126
|
res = agent.chat("multiply the results of the last two questions. Output only the answer.")
|
|
130
127
|
self.assertEqual(res.response, "1050")
|
|
131
128
|
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# Suppress external dependency warnings before any other imports
|
|
2
|
+
import warnings
|
|
3
|
+
|
|
4
|
+
warnings.simplefilter("ignore", DeprecationWarning)
|
|
5
|
+
|
|
6
|
+
import unittest
|
|
7
|
+
import threading
|
|
8
|
+
|
|
9
|
+
from vectara_agentic.agent import Agent, AgentType
|
|
10
|
+
from vectara_agentic.agent_config import AgentConfig
|
|
11
|
+
from vectara_agentic.types import ModelProvider, AgentConfigType
|
|
12
|
+
from vectara_agentic.tools import ToolsFactory
|
|
13
|
+
|
|
14
|
+
from llama_index.core.llms import ChatMessage, MessageRole
|
|
15
|
+
from conftest import mult, add
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
ARIZE_LOCK = threading.Lock()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TestAgentFallbackMemoryConsistency(unittest.TestCase):
|
|
22
|
+
"""Test memory consistency between main and fallback agents"""
|
|
23
|
+
|
|
24
|
+
def setUp(self):
|
|
25
|
+
"""Set up test fixtures"""
|
|
26
|
+
self.tools = [ToolsFactory().create_tool(mult), ToolsFactory().create_tool(add)]
|
|
27
|
+
self.topic = "Mathematics"
|
|
28
|
+
self.custom_instructions = "You are a helpful math assistant."
|
|
29
|
+
|
|
30
|
+
# Main agent config
|
|
31
|
+
self.main_config = AgentConfig(
|
|
32
|
+
agent_type=AgentType.FUNCTION_CALLING,
|
|
33
|
+
main_llm_provider=ModelProvider.ANTHROPIC,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
# Fallback agent config
|
|
37
|
+
self.fallback_config = AgentConfig(
|
|
38
|
+
agent_type=AgentType.REACT, main_llm_provider=ModelProvider.ANTHROPIC
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
self.session_id = "test-fallback-session-123"
|
|
42
|
+
|
|
43
|
+
def test_memory_consistency_on_agent_creation(self):
|
|
44
|
+
"""Test that main and fallback agents are created with the same memory content"""
|
|
45
|
+
agent = Agent(
|
|
46
|
+
tools=self.tools,
|
|
47
|
+
topic=self.topic,
|
|
48
|
+
custom_instructions=self.custom_instructions,
|
|
49
|
+
agent_config=self.main_config,
|
|
50
|
+
fallback_agent_config=self.fallback_config,
|
|
51
|
+
session_id=self.session_id,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
# Add some memory before creating the agents
|
|
55
|
+
test_messages = [
|
|
56
|
+
ChatMessage(role=MessageRole.USER, content="What is 2*3?"),
|
|
57
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="2*3 = 6"),
|
|
58
|
+
]
|
|
59
|
+
agent.memory.put_messages(test_messages)
|
|
60
|
+
|
|
61
|
+
# Verify both agents have memory with the same content
|
|
62
|
+
# Memory is managed by the main Agent class, not individual agent instances
|
|
63
|
+
main_memory = agent.memory.get()
|
|
64
|
+
fallback_memory = agent.memory.get() # Both access the same memory
|
|
65
|
+
|
|
66
|
+
self.assertEqual(len(main_memory), 2)
|
|
67
|
+
self.assertEqual(len(fallback_memory), 2)
|
|
68
|
+
self.assertEqual(main_memory[0].content, "What is 2*3?")
|
|
69
|
+
self.assertEqual(fallback_memory[0].content, "What is 2*3?")
|
|
70
|
+
|
|
71
|
+
# Verify session_id consistency
|
|
72
|
+
# Memory is managed by the main Agent class
|
|
73
|
+
self.assertEqual(agent.memory.session_id, self.session_id)
|
|
74
|
+
|
|
75
|
+
def test_memory_sync_during_agent_switching(self):
|
|
76
|
+
"""Test that memory remains consistent when switching between main and fallback agents"""
|
|
77
|
+
agent = Agent(
|
|
78
|
+
tools=self.tools,
|
|
79
|
+
topic=self.topic,
|
|
80
|
+
custom_instructions=self.custom_instructions,
|
|
81
|
+
agent_config=self.main_config,
|
|
82
|
+
fallback_agent_config=self.fallback_config,
|
|
83
|
+
session_id=self.session_id,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
# Start with main agent
|
|
87
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.DEFAULT)
|
|
88
|
+
|
|
89
|
+
# Add initial memory
|
|
90
|
+
initial_messages = [
|
|
91
|
+
ChatMessage(role=MessageRole.USER, content="Initial question"),
|
|
92
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="Initial response"),
|
|
93
|
+
]
|
|
94
|
+
agent.memory.put_messages(initial_messages)
|
|
95
|
+
|
|
96
|
+
# Access main agent to ensure it's loaded
|
|
97
|
+
main_memory_before = agent.memory.get() # Memory managed by main Agent class
|
|
98
|
+
self.assertEqual(len(main_memory_before), 2)
|
|
99
|
+
|
|
100
|
+
# Switch to fallback agent (this should clear the fallback agent instance)
|
|
101
|
+
agent._switch_agent_config()
|
|
102
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.FALLBACK)
|
|
103
|
+
|
|
104
|
+
# Access fallback agent (should be recreated with current memory)
|
|
105
|
+
fallback_memory = agent.memory.get() # Memory managed by main Agent class
|
|
106
|
+
|
|
107
|
+
# Verify fallback agent has the same memory content
|
|
108
|
+
self.assertEqual(len(fallback_memory), 2)
|
|
109
|
+
self.assertEqual(fallback_memory[0].content, "Initial question")
|
|
110
|
+
self.assertEqual(fallback_memory[1].content, "Initial response")
|
|
111
|
+
|
|
112
|
+
# Add more memory while using fallback agent
|
|
113
|
+
additional_messages = [
|
|
114
|
+
ChatMessage(role=MessageRole.USER, content="Fallback question"),
|
|
115
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="Fallback response"),
|
|
116
|
+
]
|
|
117
|
+
agent.memory.put_messages(additional_messages)
|
|
118
|
+
|
|
119
|
+
# Switch back to main agent (this should clear the main agent instance)
|
|
120
|
+
agent._switch_agent_config()
|
|
121
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.DEFAULT)
|
|
122
|
+
|
|
123
|
+
# Verify recreated main agent now has all the memory including what was added during fallback
|
|
124
|
+
main_memory_after = agent.memory.get() # Memory managed by main Agent class
|
|
125
|
+
self.assertEqual(len(main_memory_after), 4)
|
|
126
|
+
self.assertEqual(main_memory_after[2].content, "Fallback question")
|
|
127
|
+
self.assertEqual(main_memory_after[3].content, "Fallback response")
|
|
128
|
+
|
|
129
|
+
def test_memory_sync_on_clear_memory(self):
|
|
130
|
+
"""Test that memory clearing resets agent instances for consistency"""
|
|
131
|
+
agent = Agent(
|
|
132
|
+
tools=self.tools,
|
|
133
|
+
topic=self.topic,
|
|
134
|
+
custom_instructions=self.custom_instructions,
|
|
135
|
+
agent_config=self.main_config,
|
|
136
|
+
fallback_agent_config=self.fallback_config,
|
|
137
|
+
session_id=self.session_id,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# Add memory
|
|
141
|
+
test_messages = [
|
|
142
|
+
ChatMessage(role=MessageRole.USER, content="Test question"),
|
|
143
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="Test response"),
|
|
144
|
+
]
|
|
145
|
+
agent.memory.put_messages(test_messages)
|
|
146
|
+
|
|
147
|
+
# Verify memory exists
|
|
148
|
+
# Memory is managed by the main Agent class
|
|
149
|
+
self.assertEqual(len(agent.memory.get()), 2)
|
|
150
|
+
self.assertEqual(len(agent.memory.get()), 2) # Both access same memory
|
|
151
|
+
|
|
152
|
+
# Clear memory (should reset agent instances)
|
|
153
|
+
agent.clear_memory()
|
|
154
|
+
|
|
155
|
+
# Verify core memory is cleared
|
|
156
|
+
self.assertEqual(len(agent.memory.get()), 0)
|
|
157
|
+
|
|
158
|
+
# Verify agent instances were reset (None)
|
|
159
|
+
self.assertIsNone(agent._agent)
|
|
160
|
+
self.assertIsNone(agent._fallback_agent)
|
|
161
|
+
|
|
162
|
+
# Verify new agents have cleared memory
|
|
163
|
+
# Memory is managed by the main Agent class
|
|
164
|
+
self.assertEqual(len(agent.memory.get()), 0)
|
|
165
|
+
self.assertEqual(len(agent.memory.get()), 0) # Both access same memory
|
|
166
|
+
|
|
167
|
+
def test_memory_consistency_after_serialization(self):
|
|
168
|
+
"""Test that memory consistency is maintained after serialization/deserialization"""
|
|
169
|
+
agent = Agent(
|
|
170
|
+
tools=self.tools,
|
|
171
|
+
topic=self.topic,
|
|
172
|
+
custom_instructions=self.custom_instructions,
|
|
173
|
+
agent_config=self.main_config,
|
|
174
|
+
fallback_agent_config=self.fallback_config,
|
|
175
|
+
session_id=self.session_id,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
# Add memory and load both agents
|
|
179
|
+
test_messages = [
|
|
180
|
+
ChatMessage(role=MessageRole.USER, content="Serialization test"),
|
|
181
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="Serialization response"),
|
|
182
|
+
]
|
|
183
|
+
agent.memory.put_messages(test_messages)
|
|
184
|
+
|
|
185
|
+
# Access both agents
|
|
186
|
+
_ = agent.agent
|
|
187
|
+
_ = agent.fallback_agent
|
|
188
|
+
|
|
189
|
+
# Serialize and deserialize
|
|
190
|
+
serialized_data = agent.dumps()
|
|
191
|
+
restored_agent = Agent.loads(serialized_data)
|
|
192
|
+
|
|
193
|
+
# Verify memory is preserved and consistent
|
|
194
|
+
self.assertEqual(restored_agent.session_id, self.session_id)
|
|
195
|
+
self.assertEqual(len(restored_agent.memory.get()), 2)
|
|
196
|
+
|
|
197
|
+
# Verify memory consistency
|
|
198
|
+
# Individual agent instances don't have .memory attribute - memory is managed by main Agent class
|
|
199
|
+
# Both agent instances should use the same memory from the main Agent
|
|
200
|
+
|
|
201
|
+
main_memory = restored_agent.memory.get()
|
|
202
|
+
fallback_memory = restored_agent.memory.get() # Both access same memory
|
|
203
|
+
|
|
204
|
+
self.assertEqual(len(main_memory), 2)
|
|
205
|
+
self.assertEqual(len(fallback_memory), 2)
|
|
206
|
+
self.assertEqual(main_memory[0].content, "Serialization test")
|
|
207
|
+
self.assertEqual(fallback_memory[0].content, "Serialization test")
|
|
208
|
+
|
|
209
|
+
def test_session_id_consistency_across_agents(self):
|
|
210
|
+
"""Test that session_id is consistent between main and fallback agents"""
|
|
211
|
+
agent = Agent(
|
|
212
|
+
tools=self.tools,
|
|
213
|
+
topic=self.topic,
|
|
214
|
+
custom_instructions=self.custom_instructions,
|
|
215
|
+
agent_config=self.main_config,
|
|
216
|
+
fallback_agent_config=self.fallback_config,
|
|
217
|
+
session_id=self.session_id,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
# Verify main agent session_id consistency
|
|
221
|
+
self.assertEqual(agent.session_id, self.session_id)
|
|
222
|
+
self.assertEqual(agent.memory.session_id, self.session_id)
|
|
223
|
+
|
|
224
|
+
# Verify session_id consistency across all agents
|
|
225
|
+
# Memory is managed by the main Agent class
|
|
226
|
+
self.assertEqual(agent.memory.session_id, self.session_id)
|
|
227
|
+
self.assertEqual(
|
|
228
|
+
agent.memory.session_id, self.session_id
|
|
229
|
+
) # Both access same memory
|
|
230
|
+
|
|
231
|
+
def test_agent_recreation_on_switch(self):
|
|
232
|
+
"""Test that agents are properly recreated when switching configurations"""
|
|
233
|
+
agent = Agent(
|
|
234
|
+
tools=self.tools,
|
|
235
|
+
topic=self.topic,
|
|
236
|
+
custom_instructions=self.custom_instructions,
|
|
237
|
+
agent_config=self.main_config,
|
|
238
|
+
fallback_agent_config=self.fallback_config,
|
|
239
|
+
session_id=self.session_id,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# Load main agent
|
|
243
|
+
original_main_agent = agent.agent
|
|
244
|
+
self.assertIsNotNone(original_main_agent)
|
|
245
|
+
|
|
246
|
+
# Load fallback agent
|
|
247
|
+
original_fallback_agent = agent.fallback_agent
|
|
248
|
+
self.assertIsNotNone(original_fallback_agent)
|
|
249
|
+
|
|
250
|
+
# Switch to fallback - should clear the fallback agent instance
|
|
251
|
+
agent._switch_agent_config()
|
|
252
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.FALLBACK)
|
|
253
|
+
self.assertIsNone(agent._fallback_agent) # Should be cleared
|
|
254
|
+
|
|
255
|
+
# Access fallback agent again - should be a new instance
|
|
256
|
+
new_fallback_agent = agent.fallback_agent
|
|
257
|
+
self.assertIsNot(new_fallback_agent, original_fallback_agent)
|
|
258
|
+
|
|
259
|
+
# Switch back to main - should clear the main agent instance
|
|
260
|
+
agent._switch_agent_config()
|
|
261
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.DEFAULT)
|
|
262
|
+
self.assertIsNone(agent._agent) # Should be cleared
|
|
263
|
+
|
|
264
|
+
# Access main agent again - should be a new instance
|
|
265
|
+
new_main_agent = agent.agent
|
|
266
|
+
self.assertIsNot(new_main_agent, original_main_agent)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
if __name__ == "__main__":
|
|
270
|
+
unittest.main()
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# Suppress external dependency warnings before any other imports
|
|
2
|
+
import warnings
|
|
3
|
+
|
|
4
|
+
warnings.simplefilter("ignore", DeprecationWarning)
|
|
5
|
+
|
|
6
|
+
import unittest
|
|
7
|
+
import threading
|
|
8
|
+
|
|
9
|
+
from vectara_agentic.agent import Agent, AgentType
|
|
10
|
+
from vectara_agentic.agent_config import AgentConfig
|
|
11
|
+
from vectara_agentic.types import ModelProvider, AgentConfigType
|
|
12
|
+
from vectara_agentic.tools import ToolsFactory
|
|
13
|
+
from llama_index.core.llms import ChatMessage, MessageRole
|
|
14
|
+
from conftest import mult, STANDARD_TEST_TOPIC, STANDARD_TEST_INSTRUCTIONS
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
ARIZE_LOCK = threading.Lock()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TestAgentMemoryConsistency(unittest.TestCase):
|
|
21
|
+
"""Test memory consistency behavior for main/fallback agent switching"""
|
|
22
|
+
|
|
23
|
+
def setUp(self):
|
|
24
|
+
"""Set up test fixtures"""
|
|
25
|
+
self.tools = [ToolsFactory().create_tool(mult)]
|
|
26
|
+
self.topic = STANDARD_TEST_TOPIC
|
|
27
|
+
self.custom_instructions = STANDARD_TEST_INSTRUCTIONS
|
|
28
|
+
|
|
29
|
+
# Main agent config
|
|
30
|
+
self.main_config = AgentConfig(
|
|
31
|
+
agent_type=AgentType.FUNCTION_CALLING,
|
|
32
|
+
main_llm_provider=ModelProvider.ANTHROPIC,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
# Fallback agent config
|
|
36
|
+
self.fallback_config = AgentConfig(
|
|
37
|
+
agent_type=AgentType.REACT, main_llm_provider=ModelProvider.ANTHROPIC
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
self.session_id = "test-memory-consistency-123"
|
|
41
|
+
|
|
42
|
+
def test_agent_recreation_on_config_switch(self):
|
|
43
|
+
"""Test that agent instances are properly recreated when switching configurations"""
|
|
44
|
+
agent = Agent(
|
|
45
|
+
tools=self.tools,
|
|
46
|
+
topic=self.topic,
|
|
47
|
+
custom_instructions=self.custom_instructions,
|
|
48
|
+
agent_config=self.main_config,
|
|
49
|
+
fallback_agent_config=self.fallback_config,
|
|
50
|
+
session_id=self.session_id,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
# Load main agent first
|
|
54
|
+
original_main_agent = agent.agent
|
|
55
|
+
self.assertIsNotNone(original_main_agent)
|
|
56
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.DEFAULT)
|
|
57
|
+
|
|
58
|
+
# Switch to fallback - should clear fallback agent instance for recreation
|
|
59
|
+
agent._switch_agent_config()
|
|
60
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.FALLBACK)
|
|
61
|
+
self.assertIsNone(agent._fallback_agent) # Should be cleared for recreation
|
|
62
|
+
|
|
63
|
+
# Load fallback agent - should be new instance
|
|
64
|
+
new_fallback_agent = agent.fallback_agent
|
|
65
|
+
self.assertIsNotNone(new_fallback_agent)
|
|
66
|
+
|
|
67
|
+
# Switch back to main - should clear main agent instance for recreation
|
|
68
|
+
agent._switch_agent_config()
|
|
69
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.DEFAULT)
|
|
70
|
+
self.assertIsNone(agent._agent) # Should be cleared for recreation
|
|
71
|
+
|
|
72
|
+
# Load main agent again - should be new instance
|
|
73
|
+
recreated_main_agent = agent.agent
|
|
74
|
+
self.assertIsNotNone(recreated_main_agent)
|
|
75
|
+
self.assertIsNot(recreated_main_agent, original_main_agent)
|
|
76
|
+
|
|
77
|
+
def test_memory_persistence_across_config_switches(self):
|
|
78
|
+
"""Test that Agent memory persists correctly when switching configurations"""
|
|
79
|
+
agent = Agent(
|
|
80
|
+
tools=self.tools,
|
|
81
|
+
topic=self.topic,
|
|
82
|
+
custom_instructions=self.custom_instructions,
|
|
83
|
+
agent_config=self.main_config,
|
|
84
|
+
fallback_agent_config=self.fallback_config,
|
|
85
|
+
session_id=self.session_id,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
# Add initial memory
|
|
89
|
+
initial_messages = [
|
|
90
|
+
ChatMessage(role=MessageRole.USER, content="Initial question"),
|
|
91
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="Initial response"),
|
|
92
|
+
]
|
|
93
|
+
agent.memory.put_messages(initial_messages)
|
|
94
|
+
|
|
95
|
+
# Verify initial memory
|
|
96
|
+
self.assertEqual(len(agent.memory.get()), 2)
|
|
97
|
+
self.assertEqual(agent.memory.get()[0].content, "Initial question")
|
|
98
|
+
|
|
99
|
+
# Switch to fallback configuration
|
|
100
|
+
agent._switch_agent_config()
|
|
101
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.FALLBACK)
|
|
102
|
+
|
|
103
|
+
# Memory should persist at the Agent level
|
|
104
|
+
self.assertEqual(len(agent.memory.get()), 2)
|
|
105
|
+
self.assertEqual(agent.memory.get()[0].content, "Initial question")
|
|
106
|
+
|
|
107
|
+
# Add more memory while in fallback mode
|
|
108
|
+
fallback_messages = [
|
|
109
|
+
ChatMessage(role=MessageRole.USER, content="Fallback question"),
|
|
110
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="Fallback response"),
|
|
111
|
+
]
|
|
112
|
+
agent.memory.put_messages(fallback_messages)
|
|
113
|
+
|
|
114
|
+
# Verify combined memory
|
|
115
|
+
self.assertEqual(len(agent.memory.get()), 4)
|
|
116
|
+
self.assertEqual(agent.memory.get()[2].content, "Fallback question")
|
|
117
|
+
|
|
118
|
+
# Switch back to main configuration
|
|
119
|
+
agent._switch_agent_config()
|
|
120
|
+
self.assertEqual(agent.agent_config_type, AgentConfigType.DEFAULT)
|
|
121
|
+
|
|
122
|
+
# All memory should still be present
|
|
123
|
+
self.assertEqual(len(agent.memory.get()), 4)
|
|
124
|
+
self.assertEqual(agent.memory.get()[0].content, "Initial question")
|
|
125
|
+
self.assertEqual(agent.memory.get()[2].content, "Fallback question")
|
|
126
|
+
|
|
127
|
+
def test_clear_memory_resets_agent_instances(self):
|
|
128
|
+
"""Test that clearing memory properly resets agent instances"""
|
|
129
|
+
agent = Agent(
|
|
130
|
+
tools=self.tools,
|
|
131
|
+
topic=self.topic,
|
|
132
|
+
custom_instructions=self.custom_instructions,
|
|
133
|
+
agent_config=self.main_config,
|
|
134
|
+
fallback_agent_config=self.fallback_config,
|
|
135
|
+
session_id=self.session_id,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# Add memory and load both agents
|
|
139
|
+
test_messages = [
|
|
140
|
+
ChatMessage(role=MessageRole.USER, content="Test question"),
|
|
141
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="Test response"),
|
|
142
|
+
]
|
|
143
|
+
agent.memory.put_messages(test_messages)
|
|
144
|
+
|
|
145
|
+
# Load both agents
|
|
146
|
+
_ = agent.agent
|
|
147
|
+
_ = agent.fallback_agent
|
|
148
|
+
|
|
149
|
+
# Verify memory exists
|
|
150
|
+
self.assertEqual(len(agent.memory.get()), 2)
|
|
151
|
+
|
|
152
|
+
# Clear memory
|
|
153
|
+
agent.clear_memory()
|
|
154
|
+
|
|
155
|
+
# Verify memory is cleared
|
|
156
|
+
self.assertEqual(len(agent.memory.get()), 0)
|
|
157
|
+
|
|
158
|
+
# Verify agent instances were reset
|
|
159
|
+
self.assertIsNone(agent._agent)
|
|
160
|
+
self.assertIsNone(agent._fallback_agent)
|
|
161
|
+
|
|
162
|
+
def test_session_id_consistency(self):
|
|
163
|
+
"""Test that session_id remains consistent throughout agent lifecycle"""
|
|
164
|
+
agent = Agent(
|
|
165
|
+
tools=self.tools,
|
|
166
|
+
topic=self.topic,
|
|
167
|
+
custom_instructions=self.custom_instructions,
|
|
168
|
+
agent_config=self.main_config,
|
|
169
|
+
fallback_agent_config=self.fallback_config,
|
|
170
|
+
session_id=self.session_id,
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
# Verify initial session_id
|
|
174
|
+
self.assertEqual(agent.session_id, self.session_id)
|
|
175
|
+
self.assertEqual(agent.memory.session_id, self.session_id)
|
|
176
|
+
|
|
177
|
+
# Switch configurations multiple times
|
|
178
|
+
agent._switch_agent_config()
|
|
179
|
+
self.assertEqual(agent.session_id, self.session_id)
|
|
180
|
+
self.assertEqual(agent.memory.session_id, self.session_id)
|
|
181
|
+
|
|
182
|
+
agent._switch_agent_config()
|
|
183
|
+
self.assertEqual(agent.session_id, self.session_id)
|
|
184
|
+
self.assertEqual(agent.memory.session_id, self.session_id)
|
|
185
|
+
|
|
186
|
+
# Clear memory
|
|
187
|
+
agent.clear_memory()
|
|
188
|
+
self.assertEqual(agent.session_id, self.session_id)
|
|
189
|
+
self.assertEqual(agent.memory.session_id, self.session_id)
|
|
190
|
+
|
|
191
|
+
def test_serialization_preserves_consistency(self):
|
|
192
|
+
"""Test that serialization/deserialization preserves memory consistency behavior"""
|
|
193
|
+
agent = Agent(
|
|
194
|
+
tools=self.tools,
|
|
195
|
+
topic=self.topic,
|
|
196
|
+
custom_instructions=self.custom_instructions,
|
|
197
|
+
agent_config=self.main_config,
|
|
198
|
+
fallback_agent_config=self.fallback_config,
|
|
199
|
+
session_id=self.session_id,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
# Add memory and switch configurations
|
|
203
|
+
test_messages = [
|
|
204
|
+
ChatMessage(role=MessageRole.USER, content="Serialization test"),
|
|
205
|
+
ChatMessage(role=MessageRole.ASSISTANT, content="Serialization response"),
|
|
206
|
+
]
|
|
207
|
+
agent.memory.put_messages(test_messages)
|
|
208
|
+
agent._switch_agent_config() # Switch to fallback
|
|
209
|
+
|
|
210
|
+
# Serialize and deserialize
|
|
211
|
+
serialized_data = agent.dumps()
|
|
212
|
+
restored_agent = Agent.loads(serialized_data)
|
|
213
|
+
|
|
214
|
+
# Verify restored agent has same memory (config type resets to DEFAULT on deserialization)
|
|
215
|
+
self.assertEqual(restored_agent.session_id, self.session_id)
|
|
216
|
+
self.assertEqual(len(restored_agent.memory.get()), 2)
|
|
217
|
+
self.assertEqual(restored_agent.memory.get()[0].content, "Serialization test")
|
|
218
|
+
self.assertEqual(
|
|
219
|
+
restored_agent.agent_config_type, AgentConfigType.DEFAULT
|
|
220
|
+
) # Resets to default
|
|
221
|
+
|
|
222
|
+
# Verify memory consistency behavior is preserved
|
|
223
|
+
restored_agent._switch_agent_config() # Switch to fallback
|
|
224
|
+
self.assertEqual(restored_agent.agent_config_type, AgentConfigType.FALLBACK)
|
|
225
|
+
self.assertEqual(len(restored_agent.memory.get()), 2) # Memory should persist
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
if __name__ == "__main__":
|
|
229
|
+
unittest.main()
|