jaf-py 2.5.9__py3-none-any.whl → 2.5.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jaf/__init__.py +154 -57
- jaf/a2a/__init__.py +42 -21
- jaf/a2a/agent.py +79 -126
- jaf/a2a/agent_card.py +87 -78
- jaf/a2a/client.py +30 -66
- jaf/a2a/examples/client_example.py +12 -12
- jaf/a2a/examples/integration_example.py +38 -47
- jaf/a2a/examples/server_example.py +56 -53
- jaf/a2a/memory/__init__.py +0 -4
- jaf/a2a/memory/cleanup.py +28 -21
- jaf/a2a/memory/factory.py +155 -133
- jaf/a2a/memory/providers/composite.py +21 -26
- jaf/a2a/memory/providers/in_memory.py +89 -83
- jaf/a2a/memory/providers/postgres.py +117 -115
- jaf/a2a/memory/providers/redis.py +128 -121
- jaf/a2a/memory/serialization.py +77 -87
- jaf/a2a/memory/tests/run_comprehensive_tests.py +112 -83
- jaf/a2a/memory/tests/test_cleanup.py +211 -94
- jaf/a2a/memory/tests/test_serialization.py +73 -68
- jaf/a2a/memory/tests/test_stress_concurrency.py +186 -133
- jaf/a2a/memory/tests/test_task_lifecycle.py +138 -120
- jaf/a2a/memory/types.py +91 -53
- jaf/a2a/protocol.py +95 -125
- jaf/a2a/server.py +90 -118
- jaf/a2a/standalone_client.py +30 -43
- jaf/a2a/tests/__init__.py +16 -33
- jaf/a2a/tests/run_tests.py +17 -53
- jaf/a2a/tests/test_agent.py +40 -140
- jaf/a2a/tests/test_client.py +54 -117
- jaf/a2a/tests/test_integration.py +28 -82
- jaf/a2a/tests/test_protocol.py +54 -139
- jaf/a2a/tests/test_types.py +50 -136
- jaf/a2a/types.py +58 -34
- jaf/cli.py +21 -41
- jaf/core/__init__.py +7 -1
- jaf/core/agent_tool.py +93 -72
- jaf/core/analytics.py +257 -207
- jaf/core/checkpoint.py +223 -0
- jaf/core/composition.py +249 -235
- jaf/core/engine.py +817 -519
- jaf/core/errors.py +55 -42
- jaf/core/guardrails.py +276 -202
- jaf/core/handoff.py +47 -31
- jaf/core/parallel_agents.py +69 -75
- jaf/core/performance.py +75 -73
- jaf/core/proxy.py +43 -44
- jaf/core/proxy_helpers.py +24 -27
- jaf/core/regeneration.py +220 -129
- jaf/core/state.py +68 -66
- jaf/core/streaming.py +115 -108
- jaf/core/tool_results.py +111 -101
- jaf/core/tools.py +114 -116
- jaf/core/tracing.py +269 -210
- jaf/core/types.py +371 -151
- jaf/core/workflows.py +209 -168
- jaf/exceptions.py +46 -38
- jaf/memory/__init__.py +1 -6
- jaf/memory/approval_storage.py +54 -77
- jaf/memory/factory.py +4 -4
- jaf/memory/providers/in_memory.py +216 -180
- jaf/memory/providers/postgres.py +216 -146
- jaf/memory/providers/redis.py +173 -116
- jaf/memory/types.py +70 -51
- jaf/memory/utils.py +36 -34
- jaf/plugins/__init__.py +12 -12
- jaf/plugins/base.py +105 -96
- jaf/policies/__init__.py +0 -1
- jaf/policies/handoff.py +37 -46
- jaf/policies/validation.py +76 -52
- jaf/providers/__init__.py +6 -3
- jaf/providers/mcp.py +97 -51
- jaf/providers/model.py +361 -280
- jaf/server/__init__.py +1 -1
- jaf/server/main.py +7 -11
- jaf/server/server.py +514 -359
- jaf/server/types.py +208 -52
- jaf/utils/__init__.py +17 -18
- jaf/utils/attachments.py +111 -116
- jaf/utils/document_processor.py +175 -174
- jaf/visualization/__init__.py +1 -1
- jaf/visualization/example.py +111 -110
- jaf/visualization/functional_core.py +46 -71
- jaf/visualization/graphviz.py +154 -189
- jaf/visualization/imperative_shell.py +7 -16
- jaf/visualization/types.py +8 -4
- {jaf_py-2.5.9.dist-info → jaf_py-2.5.11.dist-info}/METADATA +2 -2
- jaf_py-2.5.11.dist-info/RECORD +97 -0
- jaf_py-2.5.9.dist-info/RECORD +0 -96
- {jaf_py-2.5.9.dist-info → jaf_py-2.5.11.dist-info}/WHEEL +0 -0
- {jaf_py-2.5.9.dist-info → jaf_py-2.5.11.dist-info}/entry_points.txt +0 -0
- {jaf_py-2.5.9.dist-info → jaf_py-2.5.11.dist-info}/licenses/LICENSE +0 -0
- {jaf_py-2.5.9.dist-info → jaf_py-2.5.11.dist-info}/top_level.txt +0 -0
jaf/core/analytics.py
CHANGED
|
@@ -19,6 +19,7 @@ from .performance import PerformanceMetrics
|
|
|
19
19
|
@dataclass(frozen=True)
|
|
20
20
|
class ConversationAnalytics:
|
|
21
21
|
"""Analytics for conversation patterns and quality."""
|
|
22
|
+
|
|
22
23
|
total_messages: int
|
|
23
24
|
user_messages: int
|
|
24
25
|
assistant_messages: int
|
|
@@ -34,6 +35,7 @@ class ConversationAnalytics:
|
|
|
34
35
|
@dataclass(frozen=True)
|
|
35
36
|
class AgentAnalytics:
|
|
36
37
|
"""Analytics for individual agent performance."""
|
|
38
|
+
|
|
37
39
|
agent_name: str
|
|
38
40
|
total_invocations: int
|
|
39
41
|
success_rate: float
|
|
@@ -48,6 +50,7 @@ class AgentAnalytics:
|
|
|
48
50
|
@dataclass(frozen=True)
|
|
49
51
|
class SystemAnalytics:
|
|
50
52
|
"""System-wide analytics and insights."""
|
|
53
|
+
|
|
51
54
|
total_conversations: int
|
|
52
55
|
active_agents: int
|
|
53
56
|
peak_concurrent_sessions: int
|
|
@@ -59,37 +62,39 @@ class SystemAnalytics:
|
|
|
59
62
|
|
|
60
63
|
class ConversationAnalyzer:
|
|
61
64
|
"""Analyzes conversation patterns and extracts insights."""
|
|
62
|
-
|
|
65
|
+
|
|
63
66
|
def __init__(self):
|
|
64
67
|
self.keyword_extractors = {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
68
|
+
"technical": ["error", "bug", "issue", "problem", "fix", "debug"],
|
|
69
|
+
"business": ["revenue", "cost", "profit", "customer", "sales", "market"],
|
|
70
|
+
"support": ["help", "assistance", "question", "how to", "tutorial"],
|
|
71
|
+
"creative": ["design", "create", "generate", "brainstorm", "idea"],
|
|
69
72
|
}
|
|
70
|
-
|
|
71
|
-
def analyze_conversation(
|
|
73
|
+
|
|
74
|
+
def analyze_conversation(
|
|
75
|
+
self, messages: List[Message], start_time: float, end_time: float
|
|
76
|
+
) -> ConversationAnalytics:
|
|
72
77
|
"""Analyze a complete conversation."""
|
|
73
78
|
user_msgs = [m for m in messages if m.role == ContentRole.USER]
|
|
74
79
|
assistant_msgs = [m for m in messages if m.role == ContentRole.ASSISTANT]
|
|
75
80
|
tool_msgs = [m for m in messages if m.role == ContentRole.TOOL]
|
|
76
|
-
|
|
81
|
+
|
|
77
82
|
# Calculate basic metrics
|
|
78
|
-
total_length = sum(len(m.content or
|
|
83
|
+
total_length = sum(len(m.content or "") for m in messages)
|
|
79
84
|
avg_length = total_length / len(messages) if messages else 0
|
|
80
85
|
duration = (end_time - start_time) / 60 # Convert to minutes
|
|
81
|
-
|
|
86
|
+
|
|
82
87
|
# Extract topics and keywords
|
|
83
|
-
all_text =
|
|
88
|
+
all_text = " ".join(m.content or "" for m in messages)
|
|
84
89
|
keywords = self._extract_keywords(all_text)
|
|
85
|
-
|
|
90
|
+
|
|
86
91
|
# Calculate sentiment and engagement
|
|
87
92
|
sentiment = self._calculate_sentiment(user_msgs + assistant_msgs)
|
|
88
93
|
engagement = self._calculate_engagement(messages)
|
|
89
|
-
|
|
94
|
+
|
|
90
95
|
# Determine resolution status
|
|
91
96
|
resolution = self._determine_resolution_status(messages)
|
|
92
|
-
|
|
97
|
+
|
|
93
98
|
return ConversationAnalytics(
|
|
94
99
|
total_messages=len(messages),
|
|
95
100
|
user_messages=len(user_msgs),
|
|
@@ -100,190 +105,198 @@ class ConversationAnalyzer:
|
|
|
100
105
|
topic_keywords=keywords,
|
|
101
106
|
sentiment_score=sentiment,
|
|
102
107
|
engagement_score=engagement,
|
|
103
|
-
resolution_status=resolution
|
|
108
|
+
resolution_status=resolution,
|
|
104
109
|
)
|
|
105
|
-
|
|
110
|
+
|
|
106
111
|
def _extract_keywords(self, text: str) -> List[str]:
|
|
107
112
|
"""Extract relevant keywords from conversation text."""
|
|
108
113
|
text_lower = text.lower()
|
|
109
114
|
found_keywords = []
|
|
110
|
-
|
|
115
|
+
|
|
111
116
|
for category, keywords in self.keyword_extractors.items():
|
|
112
117
|
for keyword in keywords:
|
|
113
118
|
if keyword in text_lower:
|
|
114
119
|
found_keywords.append(f"{category}:{keyword}")
|
|
115
|
-
|
|
120
|
+
|
|
116
121
|
# Add simple word frequency analysis
|
|
117
122
|
words = text_lower.split()
|
|
118
123
|
word_freq = Counter(word for word in words if len(word) > 4)
|
|
119
124
|
top_words = [word for word, count in word_freq.most_common(5) if count > 1]
|
|
120
|
-
|
|
125
|
+
|
|
121
126
|
return found_keywords + top_words
|
|
122
|
-
|
|
127
|
+
|
|
123
128
|
def _calculate_sentiment(self, messages: List[Message]) -> float:
|
|
124
129
|
"""Calculate sentiment score (simplified implementation)."""
|
|
125
|
-
positive_words = [
|
|
126
|
-
negative_words = [
|
|
127
|
-
|
|
130
|
+
positive_words = ["good", "great", "excellent", "perfect", "amazing", "helpful", "thanks"]
|
|
131
|
+
negative_words = ["bad", "terrible", "awful", "wrong", "error", "problem", "issue"]
|
|
132
|
+
|
|
128
133
|
total_score = 0
|
|
129
134
|
total_words = 0
|
|
130
|
-
|
|
135
|
+
|
|
131
136
|
for message in messages:
|
|
132
137
|
content_text = get_text_content(message.content)
|
|
133
138
|
if not content_text:
|
|
134
139
|
continue
|
|
135
|
-
|
|
140
|
+
|
|
136
141
|
words = content_text.lower().split()
|
|
137
142
|
total_words += len(words)
|
|
138
|
-
|
|
143
|
+
|
|
139
144
|
for word in words:
|
|
140
145
|
if word in positive_words:
|
|
141
146
|
total_score += 1
|
|
142
147
|
elif word in negative_words:
|
|
143
148
|
total_score -= 1
|
|
144
|
-
|
|
149
|
+
|
|
145
150
|
return total_score / max(total_words, 1) * 100 # Normalize to percentage
|
|
146
|
-
|
|
151
|
+
|
|
147
152
|
def _calculate_engagement(self, messages: List[Message]) -> float:
|
|
148
153
|
"""Calculate engagement score based on conversation patterns."""
|
|
149
154
|
if len(messages) < 2:
|
|
150
155
|
return 0.0
|
|
151
|
-
|
|
156
|
+
|
|
152
157
|
# Factors that indicate engagement
|
|
153
158
|
user_messages = [m for m in messages if m.role == ContentRole.USER]
|
|
154
159
|
assistant_messages = [m for m in messages if m.role == ContentRole.ASSISTANT]
|
|
155
|
-
|
|
160
|
+
|
|
156
161
|
# Message frequency
|
|
157
162
|
message_score = min(len(messages) / 10, 1.0) * 30
|
|
158
|
-
|
|
163
|
+
|
|
159
164
|
# Response balance
|
|
160
165
|
if user_messages and assistant_messages:
|
|
161
|
-
balance = min(len(user_messages), len(assistant_messages)) / max(
|
|
166
|
+
balance = min(len(user_messages), len(assistant_messages)) / max(
|
|
167
|
+
len(user_messages), len(assistant_messages)
|
|
168
|
+
)
|
|
162
169
|
balance_score = balance * 40
|
|
163
170
|
else:
|
|
164
171
|
balance_score = 0
|
|
165
|
-
|
|
172
|
+
|
|
166
173
|
# Message length variety (indicates thoughtful responses)
|
|
167
|
-
lengths = [len(m.content or
|
|
174
|
+
lengths = [len(m.content or "") for m in messages]
|
|
168
175
|
if lengths:
|
|
169
176
|
length_variety = (max(lengths) - min(lengths)) / max(max(lengths), 1)
|
|
170
177
|
variety_score = min(length_variety, 1.0) * 30
|
|
171
178
|
else:
|
|
172
179
|
variety_score = 0
|
|
173
|
-
|
|
180
|
+
|
|
174
181
|
return message_score + balance_score + variety_score
|
|
175
|
-
|
|
182
|
+
|
|
176
183
|
def _determine_resolution_status(self, messages: List[Message]) -> str:
|
|
177
184
|
"""Determine if the conversation was resolved."""
|
|
178
185
|
if not messages:
|
|
179
|
-
return
|
|
180
|
-
|
|
186
|
+
return "ongoing"
|
|
187
|
+
|
|
181
188
|
last_messages = messages[-3:] if len(messages) >= 3 else messages
|
|
182
|
-
last_text =
|
|
183
|
-
|
|
184
|
-
resolution_indicators = [
|
|
185
|
-
escalation_indicators = [
|
|
186
|
-
|
|
189
|
+
last_text = " ".join(m.content or "" for m in last_messages).lower()
|
|
190
|
+
|
|
191
|
+
resolution_indicators = ["thank you", "thanks", "solved", "resolved", "perfect", "exactly"]
|
|
192
|
+
escalation_indicators = ["escalate", "manager", "supervisor", "complaint"]
|
|
193
|
+
|
|
187
194
|
if any(indicator in last_text for indicator in resolution_indicators):
|
|
188
|
-
return
|
|
195
|
+
return "resolved"
|
|
189
196
|
elif any(indicator in last_text for indicator in escalation_indicators):
|
|
190
|
-
return
|
|
197
|
+
return "escalated"
|
|
191
198
|
else:
|
|
192
|
-
return
|
|
199
|
+
return "ongoing"
|
|
193
200
|
|
|
194
201
|
|
|
195
202
|
class AgentPerformanceAnalyzer:
|
|
196
203
|
"""Analyzes individual agent performance and behavior."""
|
|
197
|
-
|
|
204
|
+
|
|
198
205
|
def __init__(self):
|
|
199
|
-
self.agent_metrics: Dict[str, Dict[str, Any]] = defaultdict(
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
206
|
+
self.agent_metrics: Dict[str, Dict[str, Any]] = defaultdict(
|
|
207
|
+
lambda: {
|
|
208
|
+
"invocations": 0,
|
|
209
|
+
"successes": 0,
|
|
210
|
+
"response_times": [],
|
|
211
|
+
"tool_usage": defaultdict(int),
|
|
212
|
+
"handoffs": defaultdict(int),
|
|
213
|
+
"errors": defaultdict(int),
|
|
214
|
+
"satisfaction_scores": [],
|
|
215
|
+
}
|
|
216
|
+
)
|
|
217
|
+
|
|
209
218
|
def record_agent_invocation(self, agent_name: str, success: bool, response_time_ms: float):
|
|
210
219
|
"""Record an agent invocation."""
|
|
211
220
|
metrics = self.agent_metrics[agent_name]
|
|
212
|
-
metrics[
|
|
221
|
+
metrics["invocations"] += 1
|
|
213
222
|
if success:
|
|
214
|
-
metrics[
|
|
215
|
-
metrics[
|
|
216
|
-
|
|
223
|
+
metrics["successes"] += 1
|
|
224
|
+
metrics["response_times"].append(response_time_ms)
|
|
225
|
+
|
|
217
226
|
def record_tool_usage(self, agent_name: str, tool_name: str):
|
|
218
227
|
"""Record tool usage by an agent."""
|
|
219
|
-
self.agent_metrics[agent_name][
|
|
220
|
-
|
|
228
|
+
self.agent_metrics[agent_name]["tool_usage"][tool_name] += 1
|
|
229
|
+
|
|
221
230
|
def record_handoff(self, from_agent: str, to_agent: str):
|
|
222
231
|
"""Record an agent handoff."""
|
|
223
|
-
self.agent_metrics[from_agent][
|
|
224
|
-
|
|
232
|
+
self.agent_metrics[from_agent]["handoffs"][to_agent] += 1
|
|
233
|
+
|
|
225
234
|
def record_error(self, agent_name: str, error_type: str):
|
|
226
235
|
"""Record an error for an agent."""
|
|
227
|
-
self.agent_metrics[agent_name][
|
|
228
|
-
|
|
236
|
+
self.agent_metrics[agent_name]["errors"][error_type] += 1
|
|
237
|
+
|
|
229
238
|
def record_satisfaction(self, agent_name: str, score: float):
|
|
230
239
|
"""Record user satisfaction score for an agent."""
|
|
231
|
-
self.agent_metrics[agent_name][
|
|
232
|
-
|
|
240
|
+
self.agent_metrics[agent_name]["satisfaction_scores"].append(score)
|
|
241
|
+
|
|
233
242
|
def get_agent_analytics(self, agent_name: str) -> Optional[AgentAnalytics]:
|
|
234
243
|
"""Get comprehensive analytics for a specific agent."""
|
|
235
244
|
if agent_name not in self.agent_metrics:
|
|
236
245
|
return None
|
|
237
|
-
|
|
246
|
+
|
|
238
247
|
metrics = self.agent_metrics[agent_name]
|
|
239
|
-
|
|
248
|
+
|
|
240
249
|
# Calculate success rate
|
|
241
|
-
success_rate = (metrics[
|
|
242
|
-
|
|
250
|
+
success_rate = (metrics["successes"] / max(metrics["invocations"], 1)) * 100
|
|
251
|
+
|
|
243
252
|
# Calculate average response time
|
|
244
|
-
avg_response_time = sum(metrics[
|
|
245
|
-
|
|
253
|
+
avg_response_time = sum(metrics["response_times"]) / max(len(metrics["response_times"]), 1)
|
|
254
|
+
|
|
246
255
|
# Calculate satisfaction score
|
|
247
|
-
satisfaction_scores = metrics[
|
|
248
|
-
avg_satisfaction =
|
|
249
|
-
|
|
256
|
+
satisfaction_scores = metrics["satisfaction_scores"]
|
|
257
|
+
avg_satisfaction = (
|
|
258
|
+
sum(satisfaction_scores) / max(len(satisfaction_scores), 1)
|
|
259
|
+
if satisfaction_scores
|
|
260
|
+
else 0
|
|
261
|
+
)
|
|
262
|
+
|
|
250
263
|
# Determine specialization areas
|
|
251
|
-
specializations = self._determine_specializations(metrics[
|
|
252
|
-
|
|
264
|
+
specializations = self._determine_specializations(metrics["tool_usage"])
|
|
265
|
+
|
|
253
266
|
return AgentAnalytics(
|
|
254
267
|
agent_name=agent_name,
|
|
255
|
-
total_invocations=metrics[
|
|
268
|
+
total_invocations=metrics["invocations"],
|
|
256
269
|
success_rate=success_rate,
|
|
257
270
|
average_response_time_ms=avg_response_time,
|
|
258
|
-
tool_usage_frequency=dict(metrics[
|
|
259
|
-
handoff_patterns=dict(metrics[
|
|
260
|
-
error_patterns=dict(metrics[
|
|
271
|
+
tool_usage_frequency=dict(metrics["tool_usage"]),
|
|
272
|
+
handoff_patterns=dict(metrics["handoffs"]),
|
|
273
|
+
error_patterns=dict(metrics["errors"]),
|
|
261
274
|
user_satisfaction_score=avg_satisfaction,
|
|
262
|
-
specialization_areas=specializations
|
|
275
|
+
specialization_areas=specializations,
|
|
263
276
|
)
|
|
264
|
-
|
|
277
|
+
|
|
265
278
|
def _determine_specializations(self, tool_usage: Dict[str, int]) -> List[str]:
|
|
266
279
|
"""Determine agent specialization areas based on tool usage."""
|
|
267
280
|
if not tool_usage:
|
|
268
281
|
return []
|
|
269
|
-
|
|
282
|
+
|
|
270
283
|
# Define tool categories
|
|
271
284
|
tool_categories = {
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
285
|
+
"math": ["calculate", "compute", "math", "formula"],
|
|
286
|
+
"search": ["search", "find", "lookup", "query"],
|
|
287
|
+
"file": ["read", "write", "file", "document"],
|
|
288
|
+
"communication": ["email", "message", "notify", "send"],
|
|
289
|
+
"data": ["analyze", "process", "transform", "export"],
|
|
277
290
|
}
|
|
278
|
-
|
|
291
|
+
|
|
279
292
|
category_scores = defaultdict(int)
|
|
280
|
-
|
|
293
|
+
|
|
281
294
|
for tool, count in tool_usage.items():
|
|
282
295
|
tool_lower = tool.lower()
|
|
283
296
|
for category, keywords in tool_categories.items():
|
|
284
297
|
if any(keyword in tool_lower for keyword in keywords):
|
|
285
298
|
category_scores[category] += count
|
|
286
|
-
|
|
299
|
+
|
|
287
300
|
# Return top specializations
|
|
288
301
|
sorted_categories = sorted(category_scores.items(), key=lambda x: x[1], reverse=True)
|
|
289
302
|
return [category for category, score in sorted_categories[:3] if score > 0]
|
|
@@ -291,7 +304,7 @@ class AgentPerformanceAnalyzer:
|
|
|
291
304
|
|
|
292
305
|
class SystemAnalyzer:
|
|
293
306
|
"""Analyzes system-wide performance and provides optimization insights."""
|
|
294
|
-
|
|
307
|
+
|
|
295
308
|
def __init__(self):
|
|
296
309
|
self.conversation_count = 0
|
|
297
310
|
self.active_agents: Set[str] = set()
|
|
@@ -299,27 +312,27 @@ class SystemAnalyzer:
|
|
|
299
312
|
self.peak_concurrent = 0
|
|
300
313
|
self.performance_history: List[PerformanceMetrics] = []
|
|
301
314
|
self.resource_usage: Dict[str, List[float]] = defaultdict(list)
|
|
302
|
-
|
|
315
|
+
|
|
303
316
|
def record_conversation_start(self, agent_name: str):
|
|
304
317
|
"""Record the start of a new conversation."""
|
|
305
318
|
self.conversation_count += 1
|
|
306
319
|
self.active_agents.add(agent_name)
|
|
307
320
|
self.concurrent_sessions += 1
|
|
308
321
|
self.peak_concurrent = max(self.peak_concurrent, self.concurrent_sessions)
|
|
309
|
-
|
|
322
|
+
|
|
310
323
|
def record_conversation_end(self):
|
|
311
324
|
"""Record the end of a conversation."""
|
|
312
325
|
self.concurrent_sessions = max(0, self.concurrent_sessions - 1)
|
|
313
|
-
|
|
326
|
+
|
|
314
327
|
def record_performance_metrics(self, metrics: PerformanceMetrics):
|
|
315
328
|
"""Record system performance metrics."""
|
|
316
329
|
self.performance_history.append(metrics)
|
|
317
|
-
|
|
330
|
+
|
|
318
331
|
# Track resource usage trends
|
|
319
|
-
self.resource_usage[
|
|
320
|
-
self.resource_usage[
|
|
321
|
-
self.resource_usage[
|
|
322
|
-
|
|
332
|
+
self.resource_usage["memory"].append(metrics.memory_usage_mb)
|
|
333
|
+
self.resource_usage["execution_time"].append(metrics.execution_time_ms)
|
|
334
|
+
self.resource_usage["token_usage"].append(metrics.token_count)
|
|
335
|
+
|
|
323
336
|
def get_system_analytics(self) -> SystemAnalytics:
|
|
324
337
|
"""Get comprehensive system analytics."""
|
|
325
338
|
# Calculate resource utilization
|
|
@@ -328,26 +341,26 @@ class SystemAnalyzer:
|
|
|
328
341
|
if values:
|
|
329
342
|
recent_values = values[-10:] # Last 10 measurements
|
|
330
343
|
resource_util[resource] = {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
344
|
+
"current": recent_values[-1] if recent_values else 0,
|
|
345
|
+
"average": sum(recent_values) / len(recent_values),
|
|
346
|
+
"peak": max(values),
|
|
347
|
+
"trend": self._calculate_trend(recent_values),
|
|
335
348
|
}
|
|
336
|
-
|
|
349
|
+
|
|
337
350
|
# Calculate performance trends
|
|
338
351
|
trends = {}
|
|
339
352
|
if len(self.performance_history) >= 5:
|
|
340
353
|
recent_metrics = self.performance_history[-5:]
|
|
341
354
|
trends = {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
355
|
+
"response_time": [m.execution_time_ms for m in recent_metrics],
|
|
356
|
+
"memory_usage": [m.memory_usage_mb for m in recent_metrics],
|
|
357
|
+
"error_rate": [m.error_count for m in recent_metrics],
|
|
345
358
|
}
|
|
346
|
-
|
|
359
|
+
|
|
347
360
|
# Identify bottlenecks and recommendations
|
|
348
361
|
bottlenecks = self._identify_bottlenecks()
|
|
349
362
|
recommendations = self._generate_recommendations(bottlenecks, resource_util)
|
|
350
|
-
|
|
363
|
+
|
|
351
364
|
return SystemAnalytics(
|
|
352
365
|
total_conversations=self.conversation_count,
|
|
353
366
|
active_agents=len(self.active_agents),
|
|
@@ -355,188 +368,223 @@ class SystemAnalyzer:
|
|
|
355
368
|
resource_utilization=resource_util,
|
|
356
369
|
performance_trends=trends,
|
|
357
370
|
bottlenecks=bottlenecks,
|
|
358
|
-
optimization_recommendations=recommendations
|
|
371
|
+
optimization_recommendations=recommendations,
|
|
359
372
|
)
|
|
360
|
-
|
|
373
|
+
|
|
361
374
|
def _calculate_trend(self, values: List[float]) -> str:
|
|
362
375
|
"""Calculate trend direction for a series of values."""
|
|
363
376
|
if len(values) < 2:
|
|
364
|
-
return
|
|
365
|
-
|
|
366
|
-
first_half = values[:len(values)//2]
|
|
367
|
-
second_half = values[len(values)//2:]
|
|
368
|
-
|
|
377
|
+
return "stable"
|
|
378
|
+
|
|
379
|
+
first_half = values[: len(values) // 2]
|
|
380
|
+
second_half = values[len(values) // 2 :]
|
|
381
|
+
|
|
369
382
|
first_avg = sum(first_half) / len(first_half)
|
|
370
383
|
second_avg = sum(second_half) / len(second_half)
|
|
371
|
-
|
|
384
|
+
|
|
372
385
|
if second_avg > first_avg * 1.1:
|
|
373
|
-
return
|
|
386
|
+
return "increasing"
|
|
374
387
|
elif second_avg < first_avg * 0.9:
|
|
375
|
-
return
|
|
388
|
+
return "decreasing"
|
|
376
389
|
else:
|
|
377
|
-
return
|
|
378
|
-
|
|
390
|
+
return "stable"
|
|
391
|
+
|
|
379
392
|
def _identify_bottlenecks(self) -> List[str]:
|
|
380
393
|
"""Identify system bottlenecks."""
|
|
381
394
|
bottlenecks = []
|
|
382
|
-
|
|
395
|
+
|
|
383
396
|
if not self.performance_history:
|
|
384
397
|
return bottlenecks
|
|
385
|
-
|
|
386
|
-
recent_metrics =
|
|
387
|
-
|
|
398
|
+
|
|
399
|
+
recent_metrics = (
|
|
400
|
+
self.performance_history[-5:]
|
|
401
|
+
if len(self.performance_history) >= 5
|
|
402
|
+
else self.performance_history
|
|
403
|
+
)
|
|
404
|
+
|
|
388
405
|
# Check for high response times
|
|
389
406
|
avg_response_time = sum(m.execution_time_ms for m in recent_metrics) / len(recent_metrics)
|
|
390
407
|
if avg_response_time > 5000: # 5 seconds
|
|
391
|
-
bottlenecks.append(
|
|
392
|
-
|
|
408
|
+
bottlenecks.append("high_response_time")
|
|
409
|
+
|
|
393
410
|
# Check for high memory usage
|
|
394
411
|
avg_memory = sum(m.memory_usage_mb for m in recent_metrics) / len(recent_metrics)
|
|
395
412
|
if avg_memory > 500: # 500 MB
|
|
396
|
-
bottlenecks.append(
|
|
397
|
-
|
|
413
|
+
bottlenecks.append("high_memory_usage")
|
|
414
|
+
|
|
398
415
|
# Check for high error rates
|
|
399
416
|
avg_errors = sum(m.error_count for m in recent_metrics) / len(recent_metrics)
|
|
400
417
|
if avg_errors > 0.1: # More than 10% error rate
|
|
401
|
-
bottlenecks.append(
|
|
402
|
-
|
|
418
|
+
bottlenecks.append("high_error_rate")
|
|
419
|
+
|
|
403
420
|
# Check for excessive retries
|
|
404
421
|
avg_retries = sum(m.retry_count for m in recent_metrics) / len(recent_metrics)
|
|
405
422
|
if avg_retries > 1:
|
|
406
|
-
bottlenecks.append(
|
|
407
|
-
|
|
423
|
+
bottlenecks.append("excessive_retries")
|
|
424
|
+
|
|
408
425
|
return bottlenecks
|
|
409
|
-
|
|
410
|
-
def _generate_recommendations(
|
|
426
|
+
|
|
427
|
+
def _generate_recommendations(
|
|
428
|
+
self, bottlenecks: List[str], resource_util: Dict[str, Any]
|
|
429
|
+
) -> List[str]:
|
|
411
430
|
"""Generate optimization recommendations."""
|
|
412
431
|
recommendations = []
|
|
413
|
-
|
|
414
|
-
if
|
|
415
|
-
recommendations.append(
|
|
416
|
-
|
|
417
|
-
if
|
|
418
|
-
recommendations.append(
|
|
419
|
-
|
|
420
|
-
if
|
|
421
|
-
recommendations.append(
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
432
|
+
|
|
433
|
+
if "high_response_time" in bottlenecks:
|
|
434
|
+
recommendations.append("Consider implementing response caching or optimizing LLM calls")
|
|
435
|
+
|
|
436
|
+
if "high_memory_usage" in bottlenecks:
|
|
437
|
+
recommendations.append("Implement conversation compression or increase memory limits")
|
|
438
|
+
|
|
439
|
+
if "high_error_rate" in bottlenecks:
|
|
440
|
+
recommendations.append(
|
|
441
|
+
"Review error handling strategies and implement circuit breakers"
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
if "excessive_retries" in bottlenecks:
|
|
445
|
+
recommendations.append("Optimize retry policies and implement exponential backoff")
|
|
446
|
+
|
|
426
447
|
# Check resource trends
|
|
427
448
|
for resource, data in resource_util.items():
|
|
428
|
-
if isinstance(data, dict) and data.get(
|
|
429
|
-
recommendations.append(f
|
|
430
|
-
|
|
449
|
+
if isinstance(data, dict) and data.get("trend") == "increasing":
|
|
450
|
+
recommendations.append(f"Monitor {resource} usage - showing increasing trend")
|
|
451
|
+
|
|
431
452
|
if not recommendations:
|
|
432
|
-
recommendations.append(
|
|
433
|
-
|
|
453
|
+
recommendations.append(
|
|
454
|
+
"System performance is optimal - no immediate optimizations needed"
|
|
455
|
+
)
|
|
456
|
+
|
|
434
457
|
return recommendations
|
|
435
458
|
|
|
436
459
|
|
|
437
460
|
class AnalyticsEngine:
|
|
438
461
|
"""Main analytics engine that coordinates all analysis components."""
|
|
439
|
-
|
|
462
|
+
|
|
440
463
|
def __init__(self):
|
|
441
464
|
self.conversation_analyzer = ConversationAnalyzer()
|
|
442
465
|
self.agent_analyzer = AgentPerformanceAnalyzer()
|
|
443
466
|
self.system_analyzer = SystemAnalyzer()
|
|
444
467
|
self.analytics_history: List[Dict[str, Any]] = []
|
|
445
|
-
|
|
446
|
-
def analyze_conversation(
|
|
468
|
+
|
|
469
|
+
def analyze_conversation(
|
|
470
|
+
self, messages: List[Message], start_time: float, end_time: float
|
|
471
|
+
) -> ConversationAnalytics:
|
|
447
472
|
"""Analyze a conversation and return insights."""
|
|
448
473
|
return self.conversation_analyzer.analyze_conversation(messages, start_time, end_time)
|
|
449
|
-
|
|
450
|
-
def record_agent_performance(
|
|
451
|
-
|
|
474
|
+
|
|
475
|
+
def record_agent_performance(
|
|
476
|
+
self,
|
|
477
|
+
agent_name: str,
|
|
478
|
+
success: bool,
|
|
479
|
+
response_time_ms: float,
|
|
480
|
+
tool_name: Optional[str] = None,
|
|
481
|
+
error_type: Optional[str] = None,
|
|
482
|
+
):
|
|
452
483
|
"""Record agent performance data."""
|
|
453
484
|
self.agent_analyzer.record_agent_invocation(agent_name, success, response_time_ms)
|
|
454
|
-
|
|
485
|
+
|
|
455
486
|
if tool_name:
|
|
456
487
|
self.agent_analyzer.record_tool_usage(agent_name, tool_name)
|
|
457
|
-
|
|
488
|
+
|
|
458
489
|
if error_type:
|
|
459
490
|
self.agent_analyzer.record_error(agent_name, error_type)
|
|
460
|
-
|
|
491
|
+
|
|
461
492
|
def record_system_metrics(self, metrics: PerformanceMetrics, agent_name: str):
|
|
462
493
|
"""Record system-wide metrics."""
|
|
463
494
|
self.system_analyzer.record_performance_metrics(metrics)
|
|
464
495
|
self.system_analyzer.record_conversation_start(agent_name)
|
|
465
|
-
|
|
496
|
+
|
|
466
497
|
def get_comprehensive_analytics(self) -> Dict[str, Any]:
|
|
467
498
|
"""Get comprehensive analytics across all dimensions."""
|
|
468
499
|
system_analytics = self.system_analyzer.get_system_analytics()
|
|
469
|
-
|
|
500
|
+
|
|
470
501
|
# Get analytics for all agents
|
|
471
502
|
agent_analytics = {}
|
|
472
503
|
for agent_name in self.system_analyzer.active_agents:
|
|
473
504
|
agent_data = self.agent_analyzer.get_agent_analytics(agent_name)
|
|
474
505
|
if agent_data:
|
|
475
506
|
agent_analytics[agent_name] = agent_data
|
|
476
|
-
|
|
507
|
+
|
|
477
508
|
analytics_report = {
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
}
|
|
509
|
+
"timestamp": datetime.now().isoformat(),
|
|
510
|
+
"system": system_analytics,
|
|
511
|
+
"agents": agent_analytics,
|
|
512
|
+
"summary": {
|
|
513
|
+
"total_conversations": system_analytics.total_conversations,
|
|
514
|
+
"active_agents": system_analytics.active_agents,
|
|
515
|
+
"top_performing_agents": self._get_top_performing_agents(agent_analytics),
|
|
516
|
+
"key_insights": self._generate_key_insights(system_analytics, agent_analytics),
|
|
517
|
+
},
|
|
487
518
|
}
|
|
488
|
-
|
|
519
|
+
|
|
489
520
|
self.analytics_history.append(analytics_report)
|
|
490
521
|
return analytics_report
|
|
491
|
-
|
|
492
|
-
def _get_top_performing_agents(
|
|
522
|
+
|
|
523
|
+
def _get_top_performing_agents(
|
|
524
|
+
self, agent_analytics: Dict[str, AgentAnalytics]
|
|
525
|
+
) -> List[Dict[str, Any]]:
|
|
493
526
|
"""Get top performing agents by success rate and satisfaction."""
|
|
494
527
|
if not agent_analytics:
|
|
495
528
|
return []
|
|
496
|
-
|
|
529
|
+
|
|
497
530
|
agents_with_scores = []
|
|
498
531
|
for name, analytics in agent_analytics.items():
|
|
499
532
|
# Combined score: success rate + satisfaction
|
|
500
533
|
combined_score = (analytics.success_rate + analytics.user_satisfaction_score) / 2
|
|
501
|
-
agents_with_scores.append(
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
534
|
+
agents_with_scores.append(
|
|
535
|
+
{
|
|
536
|
+
"name": name,
|
|
537
|
+
"success_rate": analytics.success_rate,
|
|
538
|
+
"satisfaction_score": analytics.user_satisfaction_score,
|
|
539
|
+
"combined_score": combined_score,
|
|
540
|
+
}
|
|
541
|
+
)
|
|
542
|
+
|
|
508
543
|
# Sort by combined score and return top 3
|
|
509
|
-
agents_with_scores.sort(key=lambda x: x[
|
|
544
|
+
agents_with_scores.sort(key=lambda x: x["combined_score"], reverse=True)
|
|
510
545
|
return agents_with_scores[:3]
|
|
511
|
-
|
|
512
|
-
def _generate_key_insights(
|
|
513
|
-
|
|
546
|
+
|
|
547
|
+
def _generate_key_insights(
|
|
548
|
+
self, system_analytics: SystemAnalytics, agent_analytics: Dict[str, AgentAnalytics]
|
|
549
|
+
) -> List[str]:
|
|
514
550
|
"""Generate key insights from the analytics data."""
|
|
515
551
|
insights = []
|
|
516
|
-
|
|
552
|
+
|
|
517
553
|
# System insights
|
|
518
554
|
if system_analytics.peak_concurrent_sessions > 10:
|
|
519
|
-
insights.append(
|
|
520
|
-
|
|
555
|
+
insights.append(
|
|
556
|
+
f"High concurrent usage detected: {system_analytics.peak_concurrent_sessions} peak sessions"
|
|
557
|
+
)
|
|
558
|
+
|
|
521
559
|
if system_analytics.bottlenecks:
|
|
522
|
-
insights.append(
|
|
523
|
-
|
|
560
|
+
insights.append(
|
|
561
|
+
f"Performance bottlenecks identified: {', '.join(system_analytics.bottlenecks)}"
|
|
562
|
+
)
|
|
563
|
+
|
|
524
564
|
# Agent insights
|
|
525
565
|
if agent_analytics:
|
|
526
|
-
avg_success_rate = sum(a.success_rate for a in agent_analytics.values()) / len(
|
|
566
|
+
avg_success_rate = sum(a.success_rate for a in agent_analytics.values()) / len(
|
|
567
|
+
agent_analytics
|
|
568
|
+
)
|
|
527
569
|
if avg_success_rate > 90:
|
|
528
|
-
insights.append(
|
|
570
|
+
insights.append(
|
|
571
|
+
f"Excellent agent performance: {avg_success_rate:.1f}% average success rate"
|
|
572
|
+
)
|
|
529
573
|
elif avg_success_rate < 70:
|
|
530
|
-
insights.append(
|
|
531
|
-
|
|
574
|
+
insights.append(
|
|
575
|
+
f"Agent performance needs attention: {avg_success_rate:.1f}% average success rate"
|
|
576
|
+
)
|
|
577
|
+
|
|
532
578
|
# Usage patterns
|
|
533
579
|
total_conversations = system_analytics.total_conversations
|
|
534
580
|
if total_conversations > 100:
|
|
535
|
-
insights.append(
|
|
536
|
-
|
|
581
|
+
insights.append(
|
|
582
|
+
f"High system usage: {total_conversations} total conversations processed"
|
|
583
|
+
)
|
|
584
|
+
|
|
537
585
|
if not insights:
|
|
538
586
|
insights.append("System operating normally with good performance metrics")
|
|
539
|
-
|
|
587
|
+
|
|
540
588
|
return insights
|
|
541
589
|
|
|
542
590
|
|
|
@@ -549,6 +597,8 @@ def get_analytics_report() -> Dict[str, Any]:
|
|
|
549
597
|
return global_analytics_engine.get_comprehensive_analytics()
|
|
550
598
|
|
|
551
599
|
|
|
552
|
-
def analyze_conversation_quality(
|
|
600
|
+
def analyze_conversation_quality(
|
|
601
|
+
messages: List[Message], start_time: float, end_time: float
|
|
602
|
+
) -> ConversationAnalytics:
|
|
553
603
|
"""Analyze conversation quality and patterns."""
|
|
554
604
|
return global_analytics_engine.analyze_conversation(messages, start_time, end_time)
|