agno 2.3.20__py3-none-any.whl → 2.3.21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agno/eval/agent_as_judge.py +24 -14
- agno/knowledge/embedder/mistral.py +1 -1
- agno/os/routers/evals/evals.py +0 -9
- agno/os/routers/evals/utils.py +6 -6
- agno/run/agent.py +19 -19
- agno/run/team.py +19 -19
- {agno-2.3.20.dist-info → agno-2.3.21.dist-info}/METADATA +59 -129
- {agno-2.3.20.dist-info → agno-2.3.21.dist-info}/RECORD +11 -11
- {agno-2.3.20.dist-info → agno-2.3.21.dist-info}/WHEEL +0 -0
- {agno-2.3.20.dist-info → agno-2.3.21.dist-info}/licenses/LICENSE +0 -0
- {agno-2.3.20.dist-info → agno-2.3.21.dist-info}/top_level.txt +0 -0
agno/eval/agent_as_judge.py
CHANGED
|
@@ -518,8 +518,11 @@ class AgentAsJudgeEval(BaseEval):
|
|
|
518
518
|
if self.print_summary or print_summary:
|
|
519
519
|
result.print_summary(console)
|
|
520
520
|
|
|
521
|
+
# evaluator model info
|
|
522
|
+
model_id = self.model.id if self.model is not None else None
|
|
523
|
+
model_provider = self.model.provider if self.model is not None else None
|
|
521
524
|
# Log to DB
|
|
522
|
-
self._log_eval_to_db(run_id=run_id, result=result)
|
|
525
|
+
self._log_eval_to_db(run_id=run_id, result=result, model_id=model_id, model_provider=model_provider)
|
|
523
526
|
|
|
524
527
|
if self.telemetry:
|
|
525
528
|
from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
|
|
@@ -613,8 +616,11 @@ class AgentAsJudgeEval(BaseEval):
|
|
|
613
616
|
if self.print_summary or print_summary:
|
|
614
617
|
result.print_summary(console)
|
|
615
618
|
|
|
619
|
+
# evaluator model info
|
|
620
|
+
model_id = self.model.id if self.model is not None else None
|
|
621
|
+
model_provider = self.model.provider if self.model is not None else None
|
|
616
622
|
# Log to DB
|
|
617
|
-
await self._async_log_eval_to_db(run_id=run_id, result=result)
|
|
623
|
+
await self._async_log_eval_to_db(run_id=run_id, result=result, model_id=model_id, model_provider=model_provider)
|
|
618
624
|
|
|
619
625
|
if self.telemetry:
|
|
620
626
|
from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
|
|
@@ -681,8 +687,11 @@ class AgentAsJudgeEval(BaseEval):
|
|
|
681
687
|
if self.print_summary or print_summary:
|
|
682
688
|
result.print_summary(console)
|
|
683
689
|
|
|
690
|
+
# evaluator model info
|
|
691
|
+
model_id = self.model.id if self.model is not None else None
|
|
692
|
+
model_provider = self.model.provider if self.model is not None else None
|
|
684
693
|
# Log to DB
|
|
685
|
-
self._log_eval_to_db(run_id=run_id, result=result)
|
|
694
|
+
self._log_eval_to_db(run_id=run_id, result=result, model_id=model_id, model_provider=model_provider)
|
|
686
695
|
|
|
687
696
|
if self.telemetry:
|
|
688
697
|
from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
|
|
@@ -748,8 +757,11 @@ class AgentAsJudgeEval(BaseEval):
|
|
|
748
757
|
if self.print_summary or print_summary:
|
|
749
758
|
result.print_summary(console)
|
|
750
759
|
|
|
760
|
+
# evaluator model info
|
|
761
|
+
model_id = self.model.id if self.model is not None else None
|
|
762
|
+
model_provider = self.model.provider if self.model is not None else None
|
|
751
763
|
# Log to DB
|
|
752
|
-
await self._async_log_eval_to_db(run_id=run_id, result=result)
|
|
764
|
+
await self._async_log_eval_to_db(run_id=run_id, result=result, model_id=model_id, model_provider=model_provider)
|
|
753
765
|
|
|
754
766
|
if self.telemetry:
|
|
755
767
|
from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
|
|
@@ -801,15 +813,14 @@ class AgentAsJudgeEval(BaseEval):
|
|
|
801
813
|
if isinstance(run_output, RunOutput):
|
|
802
814
|
agent_id = run_output.agent_id
|
|
803
815
|
team_id = None
|
|
804
|
-
model_id = run_output.model
|
|
805
|
-
model_provider = run_output.model_provider
|
|
806
816
|
elif isinstance(run_output, TeamRunOutput):
|
|
807
817
|
agent_id = None
|
|
808
818
|
team_id = run_output.team_id
|
|
809
|
-
model_id = run_output.model
|
|
810
|
-
model_provider = run_output.model_provider
|
|
811
819
|
|
|
812
|
-
#
|
|
820
|
+
# evaluator model info
|
|
821
|
+
model_id = self.model.id if self.model is not None else None
|
|
822
|
+
model_provider = self.model.provider if self.model is not None else None
|
|
823
|
+
# Log to DB if we have a valid result
|
|
813
824
|
if result:
|
|
814
825
|
self._log_eval_to_db(
|
|
815
826
|
run_id=result.run_id,
|
|
@@ -841,15 +852,14 @@ class AgentAsJudgeEval(BaseEval):
|
|
|
841
852
|
if isinstance(run_output, RunOutput):
|
|
842
853
|
agent_id = run_output.agent_id
|
|
843
854
|
team_id = None
|
|
844
|
-
model_id = run_output.model
|
|
845
|
-
model_provider = run_output.model_provider
|
|
846
855
|
elif isinstance(run_output, TeamRunOutput):
|
|
847
856
|
agent_id = None
|
|
848
857
|
team_id = run_output.team_id
|
|
849
|
-
model_id = run_output.model
|
|
850
|
-
model_provider = run_output.model_provider
|
|
851
858
|
|
|
852
|
-
#
|
|
859
|
+
# evaluator model info
|
|
860
|
+
model_id = self.model.id if self.model is not None else None
|
|
861
|
+
model_provider = self.model.provider if self.model is not None else None
|
|
862
|
+
# Log to DB if we have a valid result
|
|
853
863
|
if result:
|
|
854
864
|
await self._async_log_eval_to_db(
|
|
855
865
|
run_id=result.run_id,
|
|
@@ -37,7 +37,7 @@ class MistralEmbedder(Embedder):
|
|
|
37
37
|
"api_key": self.api_key,
|
|
38
38
|
"endpoint": self.endpoint,
|
|
39
39
|
"max_retries": self.max_retries,
|
|
40
|
-
"
|
|
40
|
+
"timeout_ms": self.timeout * 1000 if self.timeout else None,
|
|
41
41
|
}
|
|
42
42
|
_client_params = {k: v for k, v in _client_params.items() if v is not None}
|
|
43
43
|
|
agno/os/routers/evals/evals.py
CHANGED
|
@@ -144,15 +144,6 @@ def attach_routes(
|
|
|
144
144
|
headers=headers,
|
|
145
145
|
)
|
|
146
146
|
|
|
147
|
-
# TODO: Delete me:
|
|
148
|
-
# Filtering out agent-as-judge by default for now,
|
|
149
|
-
# as they are not supported yet in the AgentOS UI.
|
|
150
|
-
eval_types = eval_types or [
|
|
151
|
-
EvalType.ACCURACY,
|
|
152
|
-
EvalType.PERFORMANCE,
|
|
153
|
-
EvalType.RELIABILITY,
|
|
154
|
-
]
|
|
155
|
-
|
|
156
147
|
if isinstance(db, AsyncBaseDb):
|
|
157
148
|
db = cast(AsyncBaseDb, db)
|
|
158
149
|
eval_runs, total_count = await db.get_eval_runs(
|
agno/os/routers/evals/utils.py
CHANGED
|
@@ -68,15 +68,11 @@ async def run_agent_as_judge_eval(
|
|
|
68
68
|
if agent:
|
|
69
69
|
agent_response = await agent.arun(eval_run_input.input, stream=False)
|
|
70
70
|
output = str(agent_response.content) if agent_response.content else ""
|
|
71
|
-
model_id = agent.model.id if agent and agent.model else None
|
|
72
|
-
model_provider = agent.model.provider if agent and agent.model else None
|
|
73
71
|
agent_id = agent.id
|
|
74
72
|
team_id = None
|
|
75
73
|
elif team:
|
|
76
74
|
team_response = await team.arun(eval_run_input.input, stream=False)
|
|
77
75
|
output = str(team_response.content) if team_response.content else ""
|
|
78
|
-
model_id = team.model.id if team and team.model else None
|
|
79
|
-
model_provider = team.model.provider if team and team.model else None
|
|
80
76
|
agent_id = None
|
|
81
77
|
team_id = team.id
|
|
82
78
|
else:
|
|
@@ -98,13 +94,17 @@ async def run_agent_as_judge_eval(
|
|
|
98
94
|
if not result:
|
|
99
95
|
raise HTTPException(status_code=500, detail="Failed to run agent as judge evaluation")
|
|
100
96
|
|
|
97
|
+
# Use evaluator's model
|
|
98
|
+
eval_model_id = agent_as_judge_eval.model.id if agent_as_judge_eval.model is not None else None
|
|
99
|
+
eval_model_provider = agent_as_judge_eval.model.provider if agent_as_judge_eval.model is not None else None
|
|
100
|
+
|
|
101
101
|
eval_run = EvalSchema.from_agent_as_judge_eval(
|
|
102
102
|
agent_as_judge_eval=agent_as_judge_eval,
|
|
103
103
|
result=result,
|
|
104
104
|
agent_id=agent_id,
|
|
105
105
|
team_id=team_id,
|
|
106
|
-
model_id=
|
|
107
|
-
model_provider=
|
|
106
|
+
model_id=eval_model_id,
|
|
107
|
+
model_provider=eval_model_provider,
|
|
108
108
|
)
|
|
109
109
|
|
|
110
110
|
# Restore original model after eval
|
agno/run/agent.py
CHANGED
|
@@ -55,8 +55,11 @@ class RunInput:
|
|
|
55
55
|
return self.input_content.model_dump_json(exclude_none=True)
|
|
56
56
|
elif isinstance(self.input_content, Message):
|
|
57
57
|
return json.dumps(self.input_content.to_dict())
|
|
58
|
-
elif isinstance(self.input_content, list)
|
|
59
|
-
|
|
58
|
+
elif isinstance(self.input_content, list):
|
|
59
|
+
try:
|
|
60
|
+
return json.dumps(self.to_dict().get("input_content"))
|
|
61
|
+
except Exception:
|
|
62
|
+
return str(self.input_content)
|
|
60
63
|
else:
|
|
61
64
|
return str(self.input_content)
|
|
62
65
|
|
|
@@ -71,22 +74,15 @@ class RunInput:
|
|
|
71
74
|
result["input_content"] = self.input_content.model_dump(exclude_none=True)
|
|
72
75
|
elif isinstance(self.input_content, Message):
|
|
73
76
|
result["input_content"] = self.input_content.to_dict()
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
# Handle input_content provided as a list of dicts
|
|
84
|
-
elif (
|
|
85
|
-
isinstance(self.input_content, list) and self.input_content and isinstance(self.input_content[0], dict)
|
|
86
|
-
):
|
|
87
|
-
for content in self.input_content:
|
|
88
|
-
# Handle media input
|
|
89
|
-
if isinstance(content, dict):
|
|
77
|
+
elif isinstance(self.input_content, list):
|
|
78
|
+
serialized_items: List[Any] = []
|
|
79
|
+
for item in self.input_content:
|
|
80
|
+
if isinstance(item, Message):
|
|
81
|
+
serialized_items.append(item.to_dict())
|
|
82
|
+
elif isinstance(item, BaseModel):
|
|
83
|
+
serialized_items.append(item.model_dump(exclude_none=True))
|
|
84
|
+
elif isinstance(item, dict):
|
|
85
|
+
content = dict(item)
|
|
90
86
|
if content.get("images"):
|
|
91
87
|
content["images"] = [
|
|
92
88
|
img.to_dict() if isinstance(img, Image) else img for img in content["images"]
|
|
@@ -103,7 +99,11 @@ class RunInput:
|
|
|
103
99
|
content["files"] = [
|
|
104
100
|
file.to_dict() if isinstance(file, File) else file for file in content["files"]
|
|
105
101
|
]
|
|
106
|
-
|
|
102
|
+
serialized_items.append(content)
|
|
103
|
+
else:
|
|
104
|
+
serialized_items.append(item)
|
|
105
|
+
|
|
106
|
+
result["input_content"] = serialized_items
|
|
107
107
|
else:
|
|
108
108
|
result["input_content"] = self.input_content
|
|
109
109
|
|
agno/run/team.py
CHANGED
|
@@ -51,8 +51,11 @@ class TeamRunInput:
|
|
|
51
51
|
return self.input_content.model_dump_json(exclude_none=True)
|
|
52
52
|
elif isinstance(self.input_content, Message):
|
|
53
53
|
return json.dumps(self.input_content.to_dict())
|
|
54
|
-
elif isinstance(self.input_content, list)
|
|
55
|
-
|
|
54
|
+
elif isinstance(self.input_content, list):
|
|
55
|
+
try:
|
|
56
|
+
return json.dumps(self.to_dict().get("input_content"))
|
|
57
|
+
except Exception:
|
|
58
|
+
return str(self.input_content)
|
|
56
59
|
else:
|
|
57
60
|
return str(self.input_content)
|
|
58
61
|
|
|
@@ -67,22 +70,15 @@ class TeamRunInput:
|
|
|
67
70
|
result["input_content"] = self.input_content.model_dump(exclude_none=True)
|
|
68
71
|
elif isinstance(self.input_content, Message):
|
|
69
72
|
result["input_content"] = self.input_content.to_dict()
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
# Handle input_content provided as a list of dicts
|
|
80
|
-
elif (
|
|
81
|
-
isinstance(self.input_content, list) and self.input_content and isinstance(self.input_content[0], dict)
|
|
82
|
-
):
|
|
83
|
-
for content in self.input_content:
|
|
84
|
-
# Handle media input
|
|
85
|
-
if isinstance(content, dict):
|
|
73
|
+
elif isinstance(self.input_content, list):
|
|
74
|
+
serialized_items: List[Any] = []
|
|
75
|
+
for item in self.input_content:
|
|
76
|
+
if isinstance(item, Message):
|
|
77
|
+
serialized_items.append(item.to_dict())
|
|
78
|
+
elif isinstance(item, BaseModel):
|
|
79
|
+
serialized_items.append(item.model_dump(exclude_none=True))
|
|
80
|
+
elif isinstance(item, dict):
|
|
81
|
+
content = dict(item)
|
|
86
82
|
if content.get("images"):
|
|
87
83
|
content["images"] = [
|
|
88
84
|
img.to_dict() if isinstance(img, Image) else img for img in content["images"]
|
|
@@ -99,7 +95,11 @@ class TeamRunInput:
|
|
|
99
95
|
content["files"] = [
|
|
100
96
|
file.to_dict() if isinstance(file, File) else file for file in content["files"]
|
|
101
97
|
]
|
|
102
|
-
|
|
98
|
+
serialized_items.append(content)
|
|
99
|
+
else:
|
|
100
|
+
serialized_items.append(item)
|
|
101
|
+
|
|
102
|
+
result["input_content"] = serialized_items
|
|
103
103
|
else:
|
|
104
104
|
result["input_content"] = self.input_content
|
|
105
105
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agno
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.21
|
|
4
4
|
Summary: Agno: a lightweight library for building Multi-Agent Systems
|
|
5
5
|
Author-email: Ashpreet Bedi <ashpreet@agno.com>
|
|
6
6
|
Project-URL: homepage, https://agno.com
|
|
@@ -415,7 +415,7 @@ Dynamic: license-file
|
|
|
415
415
|
<div align="center">
|
|
416
416
|
<a href="https://docs.agno.com">Documentation</a>
|
|
417
417
|
<span> • </span>
|
|
418
|
-
<a href="https://
|
|
418
|
+
<a href="https://github.com/agno-agi/agno/tree/main/cookbook">Cookbook</a>
|
|
419
419
|
<span> • </span>
|
|
420
420
|
<a href="https://www.agno.com/?utm_source=github&utm_medium=readme&utm_campaign=agno-github">Website</a>
|
|
421
421
|
<br />
|
|
@@ -423,25 +423,27 @@ Dynamic: license-file
|
|
|
423
423
|
|
|
424
424
|
## What is Agno?
|
|
425
425
|
|
|
426
|
-
Agno is
|
|
426
|
+
Agno is a multi-agent framework, runtime, and control plane. Use it to build private and secure AI products that run in your cloud.
|
|
427
427
|
|
|
428
|
-
|
|
428
|
+
- **Build** agents, teams, and workflows with memory, knowledge, guardrails, and 100+ toolkits.
|
|
429
|
+
- **Run** in production with a stateless FastAPI runtime. Horizontally scalable.
|
|
430
|
+
- **Manage** with a control plane that connects directly to your runtime — no data leaves your environment.
|
|
429
431
|
|
|
430
|
-
|
|
432
|
+
## Why Agno?
|
|
431
433
|
|
|
432
|
-
- **
|
|
433
|
-
- **
|
|
434
|
-
- **
|
|
434
|
+
- **Your cloud, your data:** AgentOS runs entirely in your infrastructure. Zero data leaves your environment.
|
|
435
|
+
- **Production-ready from day one:** Pre-built FastAPI runtime with SSE endpoints, ready to deploy.
|
|
436
|
+
- **Actually fast:** 529× faster than LangGraph, 24× lower memory. Matters at scale.
|
|
435
437
|
|
|
436
|
-
|
|
438
|
+
## Getting Started
|
|
437
439
|
|
|
438
|
-
|
|
440
|
+
New to Agno? Start with the [getting started guide](https://github.com/agno-agi/agno/tree/main/cookbook/00_getting_started).
|
|
439
441
|
|
|
440
|
-
|
|
442
|
+
Then:
|
|
443
|
+
- Browse the [cookbooks](https://github.com/agno-agi/agno/tree/main/cookbook) for real-world examples
|
|
444
|
+
- Read the [docs](https://docs.agno.com) to learn more.
|
|
441
445
|
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
## Documentation, Community & More Examples
|
|
446
|
+
## Resources
|
|
445
447
|
|
|
446
448
|
- Docs: <a href="https://docs.agno.com" target="_blank" rel="noopener noreferrer">docs.agno.com</a>
|
|
447
449
|
- Cookbook: <a href="https://github.com/agno-agi/agno/tree/main/cookbook" target="_blank" rel="noopener noreferrer">Cookbook</a>
|
|
@@ -497,144 +499,72 @@ When you run the example script shared above, you get a FastAPI app that you can
|
|
|
497
499
|
|
|
498
500
|
https://github.com/user-attachments/assets/feb23db8-15cc-4e88-be7c-01a21a03ebf6
|
|
499
501
|
|
|
500
|
-
##
|
|
502
|
+
## Private by Design
|
|
501
503
|
|
|
502
|
-
|
|
504
|
+
This is the part we care most about.
|
|
503
505
|
|
|
504
|
-
|
|
505
|
-
- A ready-to-use FastAPI app that gets you building AI products on day one.
|
|
506
|
-
- A control plane for testing, monitoring and managing your system.
|
|
506
|
+
AgentOS runs in **your** cloud. The control plane UI connects directly to your runtime from your browser. Your data never touches our servers. No retention costs, no vendor lock-in, no compliance headaches.
|
|
507
507
|
|
|
508
|
-
|
|
508
|
+
This isn't a privacy mode or enterprise add-on. It's how Agno works.
|
|
509
509
|
|
|
510
510
|
## Features
|
|
511
511
|
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
| | **Data Governance** | Your data lives securely in your Agent database, no external data sharing or vendor lock-in. |
|
|
538
|
-
| | **Access Control** | Role-based access (RBAC) and per-agent permissions to protect sensitive contexts and tools. |
|
|
539
|
-
|
|
540
|
-
Every part of Agno is built for real-world deployment — where developer experience meets production performance.
|
|
541
|
-
|
|
542
|
-
## Setup Your Coding Agent to Use Agno
|
|
543
|
-
|
|
544
|
-
For LLMs and AI assistants to understand and navigate Agno's documentation, we provide an [llms.txt](https://docs.agno.com/llms.txt) or [llms-full.txt](https://docs.agno.com/llms-full.txt) file. This file is built for AI systems to efficiently parse and reference our documentation.
|
|
545
|
-
|
|
546
|
-
### IDE Integration
|
|
547
|
-
|
|
548
|
-
When building Agno agents, using Agno documentation as a source in your IDE is a great way to speed up your development. Here's how to integrate with Cursor:
|
|
549
|
-
|
|
550
|
-
1. In Cursor, go to the "Cursor Settings" menu.
|
|
551
|
-
2. Find the "Indexing & Docs" section.
|
|
552
|
-
3. Add `https://docs.agno.com/llms-full.txt` to the list of documentation URLs.
|
|
553
|
-
4. Save the changes.
|
|
554
|
-
|
|
555
|
-
Now, Cursor will have access to the Agno documentation. You can do the same with other IDEs like VSCode, Windsurf etc.
|
|
512
|
+
**Core**
|
|
513
|
+
- Model agnostic — works with OpenAI, Anthropic, Google, local models, whatever
|
|
514
|
+
- Type-safe I/O with `input_schema` and `output_schema`
|
|
515
|
+
- Async-first, built for long-running tasks
|
|
516
|
+
- Natively multimodal (text, images, audio, video, files)
|
|
517
|
+
|
|
518
|
+
**Memory & Knowledge**
|
|
519
|
+
- Persistent storage for session history and state
|
|
520
|
+
- User memory that persists across sessions
|
|
521
|
+
- Agentic RAG with 20+ vector stores, hybrid search, reranking
|
|
522
|
+
- Culture — shared long-term memory across agents
|
|
523
|
+
|
|
524
|
+
**Execution**
|
|
525
|
+
- Human-in-the-loop (confirmations, approvals, overrides)
|
|
526
|
+
- Guardrails for validation and security
|
|
527
|
+
- Pre/post hooks for the agent lifecycle
|
|
528
|
+
- First-class MCP and A2A support
|
|
529
|
+
- 100+ built-in toolkits
|
|
530
|
+
|
|
531
|
+
**Production**
|
|
532
|
+
- Ready-to-use FastAPI runtime
|
|
533
|
+
- Integrated control plane UI
|
|
534
|
+
- Evals for accuracy, performance, latency
|
|
535
|
+
- Durable execution for resumable workflows
|
|
536
|
+
- RBAC and per-agent permissions
|
|
556
537
|
|
|
557
538
|
## Performance
|
|
558
539
|
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
At Agno, we optimize performance across 3 dimensions:
|
|
562
|
-
|
|
563
|
-
1. **Agent performance:** We optimize static operations (instantiation, memory footprint) and runtime operations (tool calls, memory updates, history management).
|
|
564
|
-
2. **System performance:** The AgentOS API is async by default and has a minimal memory footprint. The system is stateless and horizontally scalable, with a focus on preventing memory leaks. It handles parallel and batch embedding generation during knowledge ingestion, metrics collection in background tasks, and other system-level optimizations.
|
|
565
|
-
3. **Agent reliability and accuracy:** Monitored through evals, which we'll explore later.
|
|
566
|
-
|
|
567
|
-
### Agent Performance
|
|
568
|
-
|
|
569
|
-
Let's measure the time it takes to instantiate an Agent and the memory footprint of an Agent. Here are the numbers (last measured in Oct 2025, on an Apple M4 MacBook Pro):
|
|
570
|
-
|
|
571
|
-
- **Agent instantiation:** ~3μs on average
|
|
572
|
-
- **Memory footprint:** ~6.6Kib on average
|
|
573
|
-
|
|
574
|
-
We'll show below that Agno Agents instantiate **529× faster than Langgraph**, **57× faster than PydanticAI**, and **70× faster than CrewAI**. Agno Agents also use **24× lower memory than Langgraph**, **4× lower than PydanticAI**, and **10× lower than CrewAI**.
|
|
575
|
-
|
|
576
|
-
> [!NOTE]
|
|
577
|
-
> Run time performance is bottlenecked by inference and hard to benchmark accurately, so we focus on minimizing overhead, reducing memory usage, and parallelizing tool calls.
|
|
578
|
-
|
|
579
|
-
### Instantiation Time
|
|
540
|
+
We're obsessive about performance because agent workloads spawn hundreds of instances and run long tasks. Stateless, horizontal scalability isn't optional.
|
|
580
541
|
|
|
581
|
-
|
|
542
|
+
**Benchmarks** (Apple M4 MacBook Pro, Oct 2025):
|
|
582
543
|
|
|
583
|
-
|
|
584
|
-
|
|
544
|
+
| Metric | Agno | LangGraph | PydanticAI | CrewAI |
|
|
545
|
+
|--------|------|-----------|------------|--------|
|
|
546
|
+
| Instantiation | **3μs** | 1,587μs (529× slower) | 170μs (57× slower) | 210μs (70× slower) |
|
|
547
|
+
| Memory | **6.6 KiB** | 161 KiB (24× higher) | 29 KiB (4× higher) | 66 KiB (10× higher) |
|
|
585
548
|
|
|
586
|
-
|
|
587
|
-
# Setup virtual environment
|
|
588
|
-
./scripts/perf_setup.sh
|
|
589
|
-
source .venvs/perfenv/bin/activate
|
|
590
|
-
|
|
591
|
-
# Agno
|
|
592
|
-
python cookbook/evals/performance/instantiate_agent_with_tool.py
|
|
593
|
-
|
|
594
|
-
# LangGraph
|
|
595
|
-
python cookbook/evals/performance/comparison/langgraph_instantiation.py
|
|
596
|
-
# CrewAI
|
|
597
|
-
python cookbook/evals/performance/comparison/crewai_instantiation.py
|
|
598
|
-
# Pydantic AI
|
|
599
|
-
python cookbook/evals/performance/comparison/pydantic_ai_instantiation.py
|
|
600
|
-
```
|
|
601
|
-
|
|
602
|
-
LangGraph is on the right, **let's start it first and give it a head start**. Then CrewAI and Pydantic AI follow, and finally Agno. Agno obviously finishes first, but let's see by how much.
|
|
549
|
+
Run the benchmarks yourself: [`cookbook/evals/performance`](https://github.com/agno-agi/agno/tree/main/cookbook/evals/performance)
|
|
603
550
|
|
|
604
551
|
https://github.com/user-attachments/assets/54b98576-1859-4880-9f2d-15e1a426719d
|
|
605
552
|
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
To measure memory usage, we use the `tracemalloc` library. We first calculate a baseline memory usage by running an empty function, then run the Agent 1000x times and calculate the difference. This gives a (reasonably) isolated measurement of the memory usage of the Agent.
|
|
609
|
-
|
|
610
|
-
We recommend running the evaluation yourself on your own machine, and digging into the code to see how it works. If we've made a mistake, please let us know.
|
|
611
|
-
|
|
612
|
-
### Results
|
|
613
|
-
|
|
614
|
-
Taking Agno as the baseline, we can see that:
|
|
615
|
-
|
|
616
|
-
| Metric | Agno | Langgraph | PydanticAI | CrewAI |
|
|
617
|
-
| ------------------ | ---- | ----------- | ---------- | ---------- |
|
|
618
|
-
| **Time (seconds)** | 1× | 529× slower | 57× slower | 70× slower |
|
|
619
|
-
| **Memory (MiB)** | 1× | 24× higher | 4× higher | 10× higher |
|
|
553
|
+
## IDE Integration
|
|
620
554
|
|
|
621
|
-
|
|
555
|
+
For AI-assisted development, add our docs to your IDE:
|
|
622
556
|
|
|
623
|
-
|
|
624
|
-
| ------------------ | -------- | --------- | ---------- | -------- |
|
|
625
|
-
| **Time (seconds)** | 0.000003 | 0.001587 | 0.000170 | 0.000210 |
|
|
626
|
-
| **Memory (MiB)** | 0.006642 | 0.161435 | 0.028712 | 0.065652 |
|
|
557
|
+
**Cursor:** Settings → Indexing & Docs → Add `https://docs.agno.com/llms-full.txt`
|
|
627
558
|
|
|
628
|
-
|
|
629
|
-
> Agno agents are designed for performance and while we share benchmarks against other frameworks, we should be mindful that accuracy and reliability are more important than speed.
|
|
559
|
+
Works with VSCode, Windsurf, and other AI-enabled editors too.
|
|
630
560
|
|
|
631
|
-
##
|
|
561
|
+
## Contributing
|
|
632
562
|
|
|
633
|
-
We welcome contributions
|
|
563
|
+
We welcome contributions. See the [contributing guide](https://github.com/agno-agi/agno/blob/v2.0/CONTRIBUTING.md).
|
|
634
564
|
|
|
635
565
|
## Telemetry
|
|
636
566
|
|
|
637
|
-
Agno logs which model
|
|
567
|
+
Agno logs which model providers are used so we can prioritize updates. Disable with `AGNO_TELEMETRY=false`.
|
|
638
568
|
|
|
639
569
|
<p align="left">
|
|
640
570
|
<a href="#top">⬆️ Back to Top</a>
|
|
@@ -104,7 +104,7 @@ agno/db/surrealdb/surrealdb.py,sha256=h9nU9LbAqAq7xk84Rhkh-75mLZFf-YKhrbsT6nyazm
|
|
|
104
104
|
agno/db/surrealdb/utils.py,sha256=PcZo_cTy-jI59I-XhzAomRLdV9-m0irtO4C-AYGSghs,5405
|
|
105
105
|
agno/eval/__init__.py,sha256=RmiGpnwGm1dL9DpPMvzrXzFc1jRr111B361rTICVvXE,1173
|
|
106
106
|
agno/eval/accuracy.py,sha256=20rxNKjH3_Y6lMPRlrnKiIBlJLfKMsCN8JiJZ9jGofk,33775
|
|
107
|
-
agno/eval/agent_as_judge.py,sha256=
|
|
107
|
+
agno/eval/agent_as_judge.py,sha256=4Vo5lqMJeBroP3vFfCZ9SmQ3WdXjFaQ49bRkZCujiyQ,32859
|
|
108
108
|
agno/eval/base.py,sha256=esF8GyeuA9Gk0GM2MQXzPAEVwX8j5HROUp0_A-3bJ6Q,861
|
|
109
109
|
agno/eval/performance.py,sha256=9FLilFKrq_auCboE_xBG42V-_IPPpm8zEvqswZ83Pm0,30764
|
|
110
110
|
agno/eval/reliability.py,sha256=2PAIO8g-9ukqMg_iazorFY1uxzu9gk6eFPBPFE3NC4A,12873
|
|
@@ -146,7 +146,7 @@ agno/knowledge/embedder/google.py,sha256=K2C0Q-rGssuftDv_D6fvJxFvduDnvMfvXCQbL85
|
|
|
146
146
|
agno/knowledge/embedder/huggingface.py,sha256=eQEVHb6jh_OiR985JR88lZvtrtilI-NcQRgCliVucCo,3572
|
|
147
147
|
agno/knowledge/embedder/jina.py,sha256=M2M2ki0XyylyitJs_ZL2JoepC1y70IJzYntPsCVUxr8,7121
|
|
148
148
|
agno/knowledge/embedder/langdb.py,sha256=_vbE1Jbj16WGP4aEkh54D7ESTu0LlI1_lp5WiXPIABA,731
|
|
149
|
-
agno/knowledge/embedder/mistral.py,sha256=
|
|
149
|
+
agno/knowledge/embedder/mistral.py,sha256=7I3xnnPjE-8f5JiQ4easVKj5Sg3VHUE2ab8LNIomXzo,8644
|
|
150
150
|
agno/knowledge/embedder/nebius.py,sha256=Bp7DOyF22VxTZ2Nw6At4ywz7tzMYiE-NNGGc8Un3vtA,369
|
|
151
151
|
agno/knowledge/embedder/ollama.py,sha256=faLkOvPt2dWs8h87anAgeeG3LJpMH4jVMWP7_3f7JHs,6313
|
|
152
152
|
agno/knowledge/embedder/openai.py,sha256=KFNVJqxJQGvM1nEkHA4qTDIRHslQMjVWKgIGn_fvUPU,7355
|
|
@@ -316,9 +316,9 @@ agno/os/routers/agents/__init__.py,sha256=nr1H0Mp7NlWPnvu0ccaHVSPHz-lXg43TRMApkU
|
|
|
316
316
|
agno/os/routers/agents/router.py,sha256=PCeJwpCy2TEpRzIUMuPlEC3c2hwIDM3JLPbq4f6ouCI,25670
|
|
317
317
|
agno/os/routers/agents/schema.py,sha256=dtnFw5e0MFzLmfApHvBwXp0ByN1w10F-swYeZyNkN7c,12776
|
|
318
318
|
agno/os/routers/evals/__init__.py,sha256=3s0M-Ftg5A3rFyRfTATs-0aNA6wcbj_5tCvtwH9gORQ,87
|
|
319
|
-
agno/os/routers/evals/evals.py,sha256=
|
|
319
|
+
agno/os/routers/evals/evals.py,sha256=Oh_ysbeQDevx1Wx9kMpGRajINz8peHUPupYWa94jEi0,23466
|
|
320
320
|
agno/os/routers/evals/schemas.py,sha256=xkbOSOX1scmvN782CVqv_MrsiXlxAqXA_esjmy5KEvY,7775
|
|
321
|
-
agno/os/routers/evals/utils.py,sha256=
|
|
321
|
+
agno/os/routers/evals/utils.py,sha256=MjOPY0xNdJZY04_4YQxsv3FPCsKTMDix8jyKDWwn6kc,8367
|
|
322
322
|
agno/os/routers/knowledge/__init__.py,sha256=ZSqMQ8X7C_oYn8xt7NaYlriarWUpHgaWDyHXOWooMaU,105
|
|
323
323
|
agno/os/routers/knowledge/knowledge.py,sha256=6mv9lUs2xhdvqVV5_dDlAeamA22JAUMW2O8NoJE0d4o,49859
|
|
324
324
|
agno/os/routers/knowledge/schemas.py,sha256=Uh7v-vGT8lMOoFc2Zh0PFMsPNenHIdDVF1cyKIp-uIU,8877
|
|
@@ -355,12 +355,12 @@ agno/reasoning/vertexai.py,sha256=CCdY-ygLEdQLQuo2D-dYC0aDr4FJ9BqtkCkk2U2Ftfs,61
|
|
|
355
355
|
agno/remote/__init__.py,sha256=zgDS3cO_6VavICojsmG8opJ49wLwydftGE6i6_yPDTo,66
|
|
356
356
|
agno/remote/base.py,sha256=059JJXC9TcxnkgqGjeHLnodjeIMEqyZ6UyKxJh-ye0c,19545
|
|
357
357
|
agno/run/__init__.py,sha256=GZwloCe48rEAjkb_xOJ7piOjICmHawiR1d4SqBtUd-k,222
|
|
358
|
-
agno/run/agent.py,sha256=
|
|
358
|
+
agno/run/agent.py,sha256=ISzr_ivLyXTAW87i8Sgssk4nINeiP8yuREj9S7U9z0I,29720
|
|
359
359
|
agno/run/base.py,sha256=SqiprPcwBbjVRxSi9zlMqPsN6QDh0UHMvodnRBZnH5c,9814
|
|
360
360
|
agno/run/cancel.py,sha256=Tyxg4lxwSvO2wSZDG4OF82ORQU6MYBu94YLqrEnK09Y,3019
|
|
361
361
|
agno/run/messages.py,sha256=rAC4CLW-xBA6qFS1BOvcjJ9j_qYf0a7sX1mcdY04zMU,1126
|
|
362
362
|
agno/run/requirement.py,sha256=5M_L-3nKLPDS0tUxTbFNvgXgWfWUyEohQz_3bxNc39M,6817
|
|
363
|
-
agno/run/team.py,sha256=
|
|
363
|
+
agno/run/team.py,sha256=y5E2WzTNb1AdTIKG3LhgWK4o8mhNHbuPAmYZJmX-C1g,28775
|
|
364
364
|
agno/run/workflow.py,sha256=PMpOP7i2jl47kIMtw_rxY4U2S-Egxlvxe7QegGFihxA,25134
|
|
365
365
|
agno/run/cancellation_management/__init__.py,sha256=tA_u4MeG2dgm8cWJRXy0X6HF4mv6Kc55gykoSnuMPf0,406
|
|
366
366
|
agno/run/cancellation_management/base.py,sha256=nNh5yqfjp4Bpn03RE7T2jzLbzvY007ckzGhd-XKmZpk,2408
|
|
@@ -622,8 +622,8 @@ agno/workflow/step.py,sha256=voTmWWihLKDAMeLgYoie96KLCiVpxPFrZoFHEMhA6QM,75046
|
|
|
622
622
|
agno/workflow/steps.py,sha256=rbfue2M4qYEkgHueojCY1-cB4i1MFjO-jX6uTxyoKwk,27118
|
|
623
623
|
agno/workflow/types.py,sha256=t4304WCKB19QFdV3ixXZICcU8wtBza4EBCIz5Ve6MSQ,18035
|
|
624
624
|
agno/workflow/workflow.py,sha256=MPBPGuh-9snFx_d83zJXkbMaUtEMqhwJQYNpKOp4p1Q,195871
|
|
625
|
-
agno-2.3.
|
|
626
|
-
agno-2.3.
|
|
627
|
-
agno-2.3.
|
|
628
|
-
agno-2.3.
|
|
629
|
-
agno-2.3.
|
|
625
|
+
agno-2.3.21.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
|
|
626
|
+
agno-2.3.21.dist-info/METADATA,sha256=cws3bvDC8a1xbzLferi571kLY6m0S5RqBigbHpwlt-Q,23279
|
|
627
|
+
agno-2.3.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
628
|
+
agno-2.3.21.dist-info/top_level.txt,sha256=MKyeuVesTyOKIXUhc-d_tPa2Hrh0oTA4LM0izowpx70,5
|
|
629
|
+
agno-2.3.21.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|