agno 2.0.3__py3-none-any.whl → 2.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agno/agent/agent.py +162 -86
- agno/db/dynamo/dynamo.py +8 -0
- agno/db/firestore/firestore.py +8 -1
- agno/db/gcs_json/gcs_json_db.py +9 -0
- agno/db/json/json_db.py +8 -0
- agno/db/mongo/mongo.py +10 -1
- agno/db/mysql/mysql.py +10 -0
- agno/db/postgres/postgres.py +16 -8
- agno/db/redis/redis.py +6 -0
- agno/db/singlestore/schemas.py +1 -1
- agno/db/singlestore/singlestore.py +8 -1
- agno/db/sqlite/sqlite.py +9 -1
- agno/db/utils.py +14 -0
- agno/knowledge/knowledge.py +91 -65
- agno/models/base.py +2 -2
- agno/models/openai/chat.py +3 -0
- agno/models/openai/responses.py +6 -0
- agno/models/response.py +5 -0
- agno/models/siliconflow/__init__.py +5 -0
- agno/models/siliconflow/siliconflow.py +25 -0
- agno/os/app.py +4 -1
- agno/os/auth.py +24 -14
- agno/os/router.py +128 -55
- agno/os/routers/evals/utils.py +9 -9
- agno/os/routers/health.py +26 -0
- agno/os/routers/knowledge/knowledge.py +11 -11
- agno/os/routers/session/session.py +24 -8
- agno/os/schema.py +8 -2
- agno/run/workflow.py +64 -10
- agno/session/team.py +1 -0
- agno/team/team.py +192 -92
- agno/tools/mem0.py +11 -17
- agno/tools/memory.py +34 -6
- agno/utils/common.py +90 -1
- agno/utils/streamlit.py +14 -8
- agno/vectordb/chroma/chromadb.py +8 -2
- agno/workflow/step.py +111 -13
- agno/workflow/workflow.py +16 -13
- {agno-2.0.3.dist-info → agno-2.0.4.dist-info}/METADATA +1 -1
- {agno-2.0.3.dist-info → agno-2.0.4.dist-info}/RECORD +43 -40
- {agno-2.0.3.dist-info → agno-2.0.4.dist-info}/WHEEL +0 -0
- {agno-2.0.3.dist-info → agno-2.0.4.dist-info}/licenses/LICENSE +0 -0
- {agno-2.0.3.dist-info → agno-2.0.4.dist-info}/top_level.txt +0 -0
agno/db/mongo/mongo.py
CHANGED
|
@@ -16,7 +16,7 @@ from agno.db.mongo.utils import (
|
|
|
16
16
|
from agno.db.schemas.evals import EvalFilterType, EvalRunRecord, EvalType
|
|
17
17
|
from agno.db.schemas.knowledge import KnowledgeRow
|
|
18
18
|
from agno.db.schemas.memory import UserMemory
|
|
19
|
-
from agno.db.utils import deserialize_session_json_fields, serialize_session_json_fields
|
|
19
|
+
from agno.db.utils import deserialize_session_json_fields, generate_deterministic_id, serialize_session_json_fields
|
|
20
20
|
from agno.session import AgentSession, Session, TeamSession, WorkflowSession
|
|
21
21
|
from agno.utils.log import log_debug, log_error, log_info
|
|
22
22
|
|
|
@@ -40,6 +40,7 @@ class MongoDb(BaseDb):
|
|
|
40
40
|
metrics_collection: Optional[str] = None,
|
|
41
41
|
eval_collection: Optional[str] = None,
|
|
42
42
|
knowledge_collection: Optional[str] = None,
|
|
43
|
+
id: Optional[str] = None,
|
|
43
44
|
):
|
|
44
45
|
"""
|
|
45
46
|
Interface for interacting with a MongoDB database.
|
|
@@ -53,11 +54,19 @@ class MongoDb(BaseDb):
|
|
|
53
54
|
metrics_collection (Optional[str]): Name of the collection to store metrics.
|
|
54
55
|
eval_collection (Optional[str]): Name of the collection to store evaluation runs.
|
|
55
56
|
knowledge_collection (Optional[str]): Name of the collection to store knowledge documents.
|
|
57
|
+
id (Optional[str]): ID of the database.
|
|
56
58
|
|
|
57
59
|
Raises:
|
|
58
60
|
ValueError: If neither db_url nor db_client is provided.
|
|
59
61
|
"""
|
|
62
|
+
if id is None:
|
|
63
|
+
base_seed = db_url or str(db_client)
|
|
64
|
+
db_name_suffix = db_name if db_name is not None else "agno"
|
|
65
|
+
seed = f"{base_seed}#{db_name_suffix}"
|
|
66
|
+
id = generate_deterministic_id(seed)
|
|
67
|
+
|
|
60
68
|
super().__init__(
|
|
69
|
+
id=id,
|
|
61
70
|
session_table=session_collection,
|
|
62
71
|
memory_table=memory_collection,
|
|
63
72
|
metrics_table=metrics_collection,
|
agno/db/mysql/mysql.py
CHANGED
|
@@ -20,6 +20,7 @@ from agno.db.mysql.utils import (
|
|
|
20
20
|
from agno.db.schemas.evals import EvalFilterType, EvalRunRecord, EvalType
|
|
21
21
|
from agno.db.schemas.knowledge import KnowledgeRow
|
|
22
22
|
from agno.db.schemas.memory import UserMemory
|
|
23
|
+
from agno.db.utils import generate_deterministic_id
|
|
23
24
|
from agno.session import AgentSession, Session, TeamSession, WorkflowSession
|
|
24
25
|
from agno.utils.log import log_debug, log_error, log_info
|
|
25
26
|
|
|
@@ -45,6 +46,7 @@ class MySQLDb(BaseDb):
|
|
|
45
46
|
metrics_table: Optional[str] = None,
|
|
46
47
|
eval_table: Optional[str] = None,
|
|
47
48
|
knowledge_table: Optional[str] = None,
|
|
49
|
+
id: Optional[str] = None,
|
|
48
50
|
):
|
|
49
51
|
"""
|
|
50
52
|
Interface for interacting with a MySQL database.
|
|
@@ -63,12 +65,20 @@ class MySQLDb(BaseDb):
|
|
|
63
65
|
metrics_table (Optional[str]): Name of the table to store metrics.
|
|
64
66
|
eval_table (Optional[str]): Name of the table to store evaluation runs data.
|
|
65
67
|
knowledge_table (Optional[str]): Name of the table to store knowledge content.
|
|
68
|
+
id (Optional[str]): ID of the database.
|
|
66
69
|
|
|
67
70
|
Raises:
|
|
68
71
|
ValueError: If neither db_url nor db_engine is provided.
|
|
69
72
|
ValueError: If none of the tables are provided.
|
|
70
73
|
"""
|
|
74
|
+
if id is None:
|
|
75
|
+
base_seed = db_url or str(db_engine.url) # type: ignore
|
|
76
|
+
schema_suffix = db_schema if db_schema is not None else "ai"
|
|
77
|
+
seed = f"{base_seed}#{schema_suffix}"
|
|
78
|
+
id = generate_deterministic_id(seed)
|
|
79
|
+
|
|
71
80
|
super().__init__(
|
|
81
|
+
id=id,
|
|
72
82
|
session_table=session_table,
|
|
73
83
|
memory_table=memory_table,
|
|
74
84
|
metrics_table=metrics_table,
|
agno/db/postgres/postgres.py
CHANGED
|
@@ -18,6 +18,7 @@ from agno.db.postgres.utils import (
|
|
|
18
18
|
from agno.db.schemas.evals import EvalFilterType, EvalRunRecord, EvalType
|
|
19
19
|
from agno.db.schemas.knowledge import KnowledgeRow
|
|
20
20
|
from agno.db.schemas.memory import UserMemory
|
|
21
|
+
from agno.db.utils import generate_deterministic_id
|
|
21
22
|
from agno.session import AgentSession, Session, TeamSession, WorkflowSession
|
|
22
23
|
from agno.utils.log import log_debug, log_error, log_info, log_warning
|
|
23
24
|
|
|
@@ -68,6 +69,21 @@ class PostgresDb(BaseDb):
|
|
|
68
69
|
ValueError: If neither db_url nor db_engine is provided.
|
|
69
70
|
ValueError: If none of the tables are provided.
|
|
70
71
|
"""
|
|
72
|
+
_engine: Optional[Engine] = db_engine
|
|
73
|
+
if _engine is None and db_url is not None:
|
|
74
|
+
_engine = create_engine(db_url)
|
|
75
|
+
if _engine is None:
|
|
76
|
+
raise ValueError("One of db_url or db_engine must be provided")
|
|
77
|
+
|
|
78
|
+
self.db_url: Optional[str] = db_url
|
|
79
|
+
self.db_engine: Engine = _engine
|
|
80
|
+
|
|
81
|
+
if id is None:
|
|
82
|
+
base_seed = db_url or str(db_engine.url) # type: ignore
|
|
83
|
+
schema_suffix = db_schema if db_schema is not None else "ai"
|
|
84
|
+
seed = f"{base_seed}#{schema_suffix}"
|
|
85
|
+
id = generate_deterministic_id(seed)
|
|
86
|
+
|
|
71
87
|
super().__init__(
|
|
72
88
|
id=id,
|
|
73
89
|
session_table=session_table,
|
|
@@ -77,14 +93,6 @@ class PostgresDb(BaseDb):
|
|
|
77
93
|
knowledge_table=knowledge_table,
|
|
78
94
|
)
|
|
79
95
|
|
|
80
|
-
_engine: Optional[Engine] = db_engine
|
|
81
|
-
if _engine is None and db_url is not None:
|
|
82
|
-
_engine = create_engine(db_url)
|
|
83
|
-
if _engine is None:
|
|
84
|
-
raise ValueError("One of db_url or db_engine must be provided")
|
|
85
|
-
|
|
86
|
-
self.db_url: Optional[str] = db_url
|
|
87
|
-
self.db_engine: Engine = _engine
|
|
88
96
|
self.db_schema: str = db_schema if db_schema is not None else "ai"
|
|
89
97
|
self.metadata: MetaData = MetaData()
|
|
90
98
|
|
agno/db/redis/redis.py
CHANGED
|
@@ -21,6 +21,7 @@ from agno.db.redis.utils import (
|
|
|
21
21
|
from agno.db.schemas.evals import EvalFilterType, EvalRunRecord, EvalType
|
|
22
22
|
from agno.db.schemas.knowledge import KnowledgeRow
|
|
23
23
|
from agno.db.schemas.memory import UserMemory
|
|
24
|
+
from agno.db.utils import generate_deterministic_id
|
|
24
25
|
from agno.session import AgentSession, Session, TeamSession, WorkflowSession
|
|
25
26
|
from agno.utils.log import log_debug, log_error, log_info
|
|
26
27
|
|
|
@@ -67,6 +68,11 @@ class RedisDb(BaseDb):
|
|
|
67
68
|
Raises:
|
|
68
69
|
ValueError: If neither redis_client nor db_url is provided.
|
|
69
70
|
"""
|
|
71
|
+
if id is None:
|
|
72
|
+
base_seed = db_url or str(redis_client)
|
|
73
|
+
seed = f"{base_seed}#{db_prefix}"
|
|
74
|
+
id = generate_deterministic_id(seed)
|
|
75
|
+
|
|
70
76
|
super().__init__(
|
|
71
77
|
id=id,
|
|
72
78
|
session_table=session_table,
|
agno/db/singlestore/schemas.py
CHANGED
|
@@ -106,7 +106,7 @@ def get_table_schema_definition(table_type: str) -> dict[str, Any]:
|
|
|
106
106
|
"evals": EVAL_TABLE_SCHEMA,
|
|
107
107
|
"metrics": METRICS_TABLE_SCHEMA,
|
|
108
108
|
"memories": USER_MEMORY_TABLE_SCHEMA,
|
|
109
|
-
"
|
|
109
|
+
"knowledge": KNOWLEDGE_TABLE_SCHEMA,
|
|
110
110
|
}
|
|
111
111
|
schema = schemas.get(table_type, {})
|
|
112
112
|
|
|
@@ -19,6 +19,7 @@ from agno.db.singlestore.utils import (
|
|
|
19
19
|
is_table_available,
|
|
20
20
|
is_valid_table,
|
|
21
21
|
)
|
|
22
|
+
from agno.db.utils import generate_deterministic_id
|
|
22
23
|
from agno.session import AgentSession, Session, TeamSession, WorkflowSession
|
|
23
24
|
from agno.utils.log import log_debug, log_error, log_info, log_warning
|
|
24
25
|
|
|
@@ -69,6 +70,12 @@ class SingleStoreDb(BaseDb):
|
|
|
69
70
|
ValueError: If neither db_url nor db_engine is provided.
|
|
70
71
|
ValueError: If none of the tables are provided.
|
|
71
72
|
"""
|
|
73
|
+
if id is None:
|
|
74
|
+
base_seed = db_url or str(db_engine.url) if db_engine else "singlestore" # type: ignore
|
|
75
|
+
schema_suffix = db_schema if db_schema is not None else "ai"
|
|
76
|
+
seed = f"{base_seed}#{schema_suffix}"
|
|
77
|
+
id = generate_deterministic_id(seed)
|
|
78
|
+
|
|
72
79
|
super().__init__(
|
|
73
80
|
id=id,
|
|
74
81
|
session_table=session_table,
|
|
@@ -156,10 +163,10 @@ class SingleStoreDb(BaseDb):
|
|
|
156
163
|
Returns:
|
|
157
164
|
Table: SQLAlchemy Table object
|
|
158
165
|
"""
|
|
166
|
+
table_ref = f"{db_schema}.{table_name}" if db_schema else table_name
|
|
159
167
|
try:
|
|
160
168
|
table_schema = get_table_schema_definition(table_type)
|
|
161
169
|
|
|
162
|
-
table_ref = f"{db_schema}.{table_name}" if db_schema else table_name
|
|
163
170
|
log_debug(f"Creating table {table_ref} with schema: {table_schema}")
|
|
164
171
|
|
|
165
172
|
columns: List[Column] = []
|
agno/db/sqlite/sqlite.py
CHANGED
|
@@ -18,7 +18,7 @@ from agno.db.sqlite.utils import (
|
|
|
18
18
|
is_table_available,
|
|
19
19
|
is_valid_table,
|
|
20
20
|
)
|
|
21
|
-
from agno.db.utils import deserialize_session_json_fields, serialize_session_json_fields
|
|
21
|
+
from agno.db.utils import deserialize_session_json_fields, generate_deterministic_id, serialize_session_json_fields
|
|
22
22
|
from agno.session import AgentSession, Session, TeamSession, WorkflowSession
|
|
23
23
|
from agno.utils.log import log_debug, log_error, log_info, log_warning
|
|
24
24
|
|
|
@@ -43,6 +43,7 @@ class SqliteDb(BaseDb):
|
|
|
43
43
|
metrics_table: Optional[str] = None,
|
|
44
44
|
eval_table: Optional[str] = None,
|
|
45
45
|
knowledge_table: Optional[str] = None,
|
|
46
|
+
id: Optional[str] = None,
|
|
46
47
|
):
|
|
47
48
|
"""
|
|
48
49
|
Interface for interacting with a SQLite database.
|
|
@@ -62,11 +63,17 @@ class SqliteDb(BaseDb):
|
|
|
62
63
|
metrics_table (Optional[str]): Name of the table to store metrics.
|
|
63
64
|
eval_table (Optional[str]): Name of the table to store evaluation runs data.
|
|
64
65
|
knowledge_table (Optional[str]): Name of the table to store knowledge documents data.
|
|
66
|
+
id (Optional[str]): ID of the database.
|
|
65
67
|
|
|
66
68
|
Raises:
|
|
67
69
|
ValueError: If none of the tables are provided.
|
|
68
70
|
"""
|
|
71
|
+
if id is None:
|
|
72
|
+
seed = db_url or db_file or str(db_engine.url) if db_engine else "sqlite:///agno.db"
|
|
73
|
+
id = generate_deterministic_id(seed)
|
|
74
|
+
|
|
69
75
|
super().__init__(
|
|
76
|
+
id=id,
|
|
70
77
|
session_table=session_table,
|
|
71
78
|
memory_table=memory_table,
|
|
72
79
|
metrics_table=metrics_table,
|
|
@@ -1341,6 +1348,7 @@ class SqliteDb(BaseDb):
|
|
|
1341
1348
|
"linked_to": knowledge_row.linked_to,
|
|
1342
1349
|
"access_count": knowledge_row.access_count,
|
|
1343
1350
|
"status": knowledge_row.status,
|
|
1351
|
+
"status_message": knowledge_row.status_message,
|
|
1344
1352
|
"created_at": knowledge_row.created_at,
|
|
1345
1353
|
"updated_at": knowledge_row.updated_at,
|
|
1346
1354
|
"external_id": knowledge_row.external_id,
|
agno/db/utils.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Logic shared across different database implementations"""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
+
import uuid
|
|
4
5
|
from datetime import date, datetime
|
|
5
6
|
from uuid import UUID
|
|
6
7
|
|
|
@@ -86,3 +87,16 @@ def deserialize_session_json_fields(session: dict) -> dict:
|
|
|
86
87
|
session["runs"] = json.loads(session["runs"])
|
|
87
88
|
|
|
88
89
|
return session
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def generate_deterministic_id(seed: str) -> str:
|
|
93
|
+
"""
|
|
94
|
+
Generate a deterministic UUID5 based on a seed string.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
seed (str): The seed string to generate the UUID from.
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
str: A deterministic UUID5 string.
|
|
101
|
+
"""
|
|
102
|
+
return str(uuid.uuid5(uuid.NAMESPACE_DNS, seed))
|
agno/knowledge/knowledge.py
CHANGED
|
@@ -9,12 +9,12 @@ from io import BytesIO
|
|
|
9
9
|
from os.path import basename
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload
|
|
12
|
-
from uuid import uuid4
|
|
13
12
|
|
|
14
13
|
from httpx import AsyncClient
|
|
15
14
|
|
|
16
15
|
from agno.db.base import BaseDb
|
|
17
16
|
from agno.db.schemas.knowledge import KnowledgeRow
|
|
17
|
+
from agno.db.utils import generate_deterministic_id
|
|
18
18
|
from agno.knowledge.content import Content, ContentAuth, ContentStatus, FileData
|
|
19
19
|
from agno.knowledge.document import Document
|
|
20
20
|
from agno.knowledge.reader import Reader, ReaderFactory
|
|
@@ -241,7 +241,6 @@ class Knowledge:
|
|
|
241
241
|
file_data = FileData(content=text_content, type="Text")
|
|
242
242
|
|
|
243
243
|
content = Content(
|
|
244
|
-
id=str(uuid4()),
|
|
245
244
|
name=name,
|
|
246
245
|
description=description,
|
|
247
246
|
path=path,
|
|
@@ -253,6 +252,8 @@ class Knowledge:
|
|
|
253
252
|
reader=reader,
|
|
254
253
|
auth=auth,
|
|
255
254
|
)
|
|
255
|
+
content.content_hash = self._build_content_hash(content)
|
|
256
|
+
content.id = generate_deterministic_id(content.content_hash)
|
|
256
257
|
|
|
257
258
|
await self._load_content(content, upsert, skip_if_exists, include, exclude)
|
|
258
259
|
|
|
@@ -329,6 +330,22 @@ class Knowledge:
|
|
|
329
330
|
)
|
|
330
331
|
)
|
|
331
332
|
|
|
333
|
+
def _should_skip(self, content_hash: str, skip_if_exists: bool) -> bool:
|
|
334
|
+
"""
|
|
335
|
+
Handle the skip_if_exists logic for content that already exists in the vector database.
|
|
336
|
+
|
|
337
|
+
Args:
|
|
338
|
+
content_hash: The content hash string to check for existence
|
|
339
|
+
skip_if_exists: Whether to skip if content already exists
|
|
340
|
+
|
|
341
|
+
Returns:
|
|
342
|
+
bool: True if should skip processing, False if should continue
|
|
343
|
+
"""
|
|
344
|
+
if self.vector_db and self.vector_db.content_hash_exists(content_hash) and skip_if_exists:
|
|
345
|
+
return True
|
|
346
|
+
|
|
347
|
+
return False
|
|
348
|
+
|
|
332
349
|
async def _load_from_path(
|
|
333
350
|
self,
|
|
334
351
|
content: Content,
|
|
@@ -344,18 +361,17 @@ class Knowledge:
|
|
|
344
361
|
if self._should_include_file(str(path), include, exclude):
|
|
345
362
|
log_info(f"Adding file {path} due to include/exclude filters")
|
|
346
363
|
|
|
364
|
+
self._add_to_contents_db(content)
|
|
365
|
+
if self._should_skip(content.content_hash, skip_if_exists): # type: ignore[arg-type]
|
|
366
|
+
content.status = ContentStatus.COMPLETED
|
|
367
|
+
self._update_content(content)
|
|
368
|
+
return
|
|
369
|
+
|
|
347
370
|
# Handle LightRAG special case - read file and upload directly
|
|
348
371
|
if self.vector_db.__class__.__name__ == "LightRag":
|
|
349
372
|
await self._process_lightrag_content(content, KnowledgeContentOrigin.PATH)
|
|
350
373
|
return
|
|
351
374
|
|
|
352
|
-
content.content_hash = self._build_content_hash(content)
|
|
353
|
-
if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
|
|
354
|
-
log_info(f"Content {content.content_hash} already exists, skipping")
|
|
355
|
-
return
|
|
356
|
-
|
|
357
|
-
self._add_to_contents_db(content)
|
|
358
|
-
|
|
359
375
|
if content.reader:
|
|
360
376
|
# TODO: We will refactor this to eventually pass authorization to all readers
|
|
361
377
|
import inspect
|
|
@@ -407,15 +423,16 @@ class Knowledge:
|
|
|
407
423
|
log_debug(f"Skipping file {file_path} due to include/exclude filters")
|
|
408
424
|
continue
|
|
409
425
|
|
|
410
|
-
id = str(uuid4())
|
|
411
426
|
file_content = Content(
|
|
412
|
-
id=id,
|
|
413
427
|
name=content.name,
|
|
414
428
|
path=str(file_path),
|
|
415
429
|
metadata=content.metadata,
|
|
416
430
|
description=content.description,
|
|
417
431
|
reader=content.reader,
|
|
418
432
|
)
|
|
433
|
+
file_content.content_hash = self._build_content_hash(file_content)
|
|
434
|
+
file_content.id = generate_deterministic_id(file_content.content_hash)
|
|
435
|
+
|
|
419
436
|
await self._load_from_path(file_content, upsert, skip_if_exists, include, exclude)
|
|
420
437
|
else:
|
|
421
438
|
log_warning(f"Invalid path: {path}")
|
|
@@ -439,16 +456,16 @@ class Knowledge:
|
|
|
439
456
|
if not content.url:
|
|
440
457
|
raise ValueError("No url provided")
|
|
441
458
|
|
|
442
|
-
|
|
443
|
-
|
|
459
|
+
# 1. Add content to contents database
|
|
460
|
+
self._add_to_contents_db(content)
|
|
461
|
+
if self._should_skip(content.content_hash, skip_if_exists): # type: ignore[arg-type]
|
|
462
|
+
content.status = ContentStatus.COMPLETED
|
|
463
|
+
self._update_content(content)
|
|
444
464
|
return
|
|
445
465
|
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
|
|
449
|
-
log_info(f"Content {content.content_hash} already exists, skipping")
|
|
466
|
+
if self.vector_db.__class__.__name__ == "LightRag":
|
|
467
|
+
await self._process_lightrag_content(content, KnowledgeContentOrigin.URL)
|
|
450
468
|
return
|
|
451
|
-
self._add_to_contents_db(content)
|
|
452
469
|
|
|
453
470
|
# 2. Validate URL
|
|
454
471
|
try:
|
|
@@ -466,19 +483,23 @@ class Knowledge:
|
|
|
466
483
|
self._update_content(content)
|
|
467
484
|
log_warning(f"Invalid URL: {content.url} - {str(e)}")
|
|
468
485
|
|
|
469
|
-
# 3. Fetch and load content
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
486
|
+
# 3. Fetch and load content if file has an extension
|
|
487
|
+
url_path = Path(parsed_url.path)
|
|
488
|
+
file_extension = url_path.suffix.lower()
|
|
489
|
+
|
|
490
|
+
bytes_content = None
|
|
491
|
+
if file_extension:
|
|
492
|
+
async with AsyncClient() as client:
|
|
493
|
+
response = await async_fetch_with_retry(content.url, client=client)
|
|
494
|
+
bytes_content = BytesIO(response.content)
|
|
473
495
|
|
|
474
496
|
# 4. Select reader
|
|
475
497
|
# If a reader was provided by the user, use it
|
|
476
498
|
reader = content.reader
|
|
477
499
|
name = content.name if content.name else content.url
|
|
478
500
|
# Else select based on file extension
|
|
501
|
+
|
|
479
502
|
if reader is None:
|
|
480
|
-
url_path = Path(parsed_url.path)
|
|
481
|
-
file_extension = url_path.suffix.lower()
|
|
482
503
|
if file_extension == ".csv":
|
|
483
504
|
name = basename(parsed_url.path) or "data.csv"
|
|
484
505
|
reader = self.csv_reader
|
|
@@ -504,9 +525,15 @@ class Knowledge:
|
|
|
504
525
|
if reader.__class__.__name__ == "YouTubeReader":
|
|
505
526
|
read_documents = reader.read(content.url, name=name)
|
|
506
527
|
elif "password" in read_signature.parameters and content.auth and content.auth.password:
|
|
507
|
-
|
|
528
|
+
if bytes_content:
|
|
529
|
+
read_documents = reader.read(bytes_content, name=name, password=content.auth.password)
|
|
530
|
+
else:
|
|
531
|
+
read_documents = reader.read(content.url, name=name, password=content.auth.password)
|
|
508
532
|
else:
|
|
509
|
-
|
|
533
|
+
if bytes_content:
|
|
534
|
+
read_documents = reader.read(bytes_content, name=name)
|
|
535
|
+
else:
|
|
536
|
+
read_documents = reader.read(content.url, name=name)
|
|
510
537
|
except Exception as e:
|
|
511
538
|
log_error(f"Error reading URL: {content.url} - {str(e)}")
|
|
512
539
|
content.status = ContentStatus.FAILED
|
|
@@ -554,16 +581,15 @@ class Knowledge:
|
|
|
554
581
|
|
|
555
582
|
log_info(f"Adding content from {content.name}")
|
|
556
583
|
|
|
557
|
-
|
|
558
|
-
|
|
584
|
+
self._add_to_contents_db(content)
|
|
585
|
+
if self._should_skip(content.content_hash, skip_if_exists): # type: ignore[arg-type]
|
|
586
|
+
content.status = ContentStatus.COMPLETED
|
|
587
|
+
self._update_content(content)
|
|
559
588
|
return
|
|
560
589
|
|
|
561
|
-
content.
|
|
562
|
-
|
|
563
|
-
log_info(f"Content {content.content_hash} already exists, skipping")
|
|
564
|
-
|
|
590
|
+
if content.file_data and self.vector_db.__class__.__name__ == "LightRag":
|
|
591
|
+
await self._process_lightrag_content(content, KnowledgeContentOrigin.CONTENT)
|
|
565
592
|
return
|
|
566
|
-
self._add_to_contents_db(content)
|
|
567
593
|
|
|
568
594
|
read_documents = []
|
|
569
595
|
|
|
@@ -612,7 +638,6 @@ class Knowledge:
|
|
|
612
638
|
reader = self._select_reader(content.file_data.type)
|
|
613
639
|
name = content.name if content.name else f"content_{content.file_data.type}"
|
|
614
640
|
read_documents = reader.read(content_io, name=name)
|
|
615
|
-
|
|
616
641
|
for read_document in read_documents:
|
|
617
642
|
if content.metadata:
|
|
618
643
|
read_document.meta_data.update(content.metadata)
|
|
@@ -644,9 +669,7 @@ class Knowledge:
|
|
|
644
669
|
return
|
|
645
670
|
|
|
646
671
|
for topic in content.topics:
|
|
647
|
-
id = str(uuid4())
|
|
648
672
|
content = Content(
|
|
649
|
-
id=id,
|
|
650
673
|
name=topic,
|
|
651
674
|
metadata=content.metadata,
|
|
652
675
|
reader=content.reader,
|
|
@@ -656,30 +679,37 @@ class Knowledge:
|
|
|
656
679
|
),
|
|
657
680
|
topics=[topic],
|
|
658
681
|
)
|
|
682
|
+
content.content_hash = self._build_content_hash(content)
|
|
683
|
+
content.id = generate_deterministic_id(content.content_hash)
|
|
684
|
+
|
|
685
|
+
self._add_to_contents_db(content)
|
|
686
|
+
if self._should_skip(content.content_hash, skip_if_exists):
|
|
687
|
+
content.status = ContentStatus.COMPLETED
|
|
688
|
+
self._update_content(content)
|
|
689
|
+
return
|
|
659
690
|
|
|
660
691
|
if self.vector_db.__class__.__name__ == "LightRag":
|
|
661
692
|
await self._process_lightrag_content(content, KnowledgeContentOrigin.TOPIC)
|
|
662
693
|
return
|
|
663
694
|
|
|
664
|
-
content.content_hash = self._build_content_hash(content)
|
|
665
|
-
if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
|
|
666
|
-
log_info(f"Content {content.content_hash} already exists, skipping")
|
|
667
|
-
continue
|
|
668
|
-
|
|
669
|
-
self._add_to_contents_db(content)
|
|
670
695
|
if content.reader is None:
|
|
671
696
|
log_error(f"No reader available for topic: {topic}")
|
|
697
|
+
content.status = ContentStatus.FAILED
|
|
698
|
+
content.status_message = "No reader available for topic"
|
|
699
|
+
self._update_content(content)
|
|
672
700
|
continue
|
|
701
|
+
|
|
673
702
|
read_documents = content.reader.read(topic)
|
|
674
703
|
if len(read_documents) > 0:
|
|
675
704
|
for read_document in read_documents:
|
|
676
|
-
read_document.content_id = id
|
|
705
|
+
read_document.content_id = content.id
|
|
677
706
|
if read_document.content:
|
|
678
707
|
read_document.size = len(read_document.content.encode("utf-8"))
|
|
679
708
|
else:
|
|
680
709
|
content.status = ContentStatus.FAILED
|
|
681
710
|
content.status_message = "No content found for topic"
|
|
682
711
|
self._update_content(content)
|
|
712
|
+
continue
|
|
683
713
|
|
|
684
714
|
await self._handle_vector_db_insert(content, read_documents, upsert)
|
|
685
715
|
|
|
@@ -735,11 +765,9 @@ class Knowledge:
|
|
|
735
765
|
|
|
736
766
|
for s3_object in objects_to_read:
|
|
737
767
|
# 2. Setup Content object
|
|
738
|
-
id = str(uuid4())
|
|
739
768
|
content_name = content.name or ""
|
|
740
769
|
content_name += "_" + (s3_object.name or "")
|
|
741
770
|
content_entry = Content(
|
|
742
|
-
id=id,
|
|
743
771
|
name=content_name,
|
|
744
772
|
description=content.description,
|
|
745
773
|
status=ContentStatus.PROCESSING,
|
|
@@ -748,11 +776,13 @@ class Knowledge:
|
|
|
748
776
|
)
|
|
749
777
|
|
|
750
778
|
# 3. Hash content and add it to the contents database
|
|
751
|
-
content_hash = self._build_content_hash(content_entry)
|
|
752
|
-
|
|
753
|
-
log_info(f"Content {content_hash} already exists, skipping")
|
|
754
|
-
continue
|
|
779
|
+
content_entry.content_hash = self._build_content_hash(content_entry)
|
|
780
|
+
content_entry.id = generate_deterministic_id(content_entry.content_hash)
|
|
755
781
|
self._add_to_contents_db(content_entry)
|
|
782
|
+
if self._should_skip(content_entry.content_hash, skip_if_exists):
|
|
783
|
+
content_entry.status = ContentStatus.COMPLETED
|
|
784
|
+
self._update_content(content_entry)
|
|
785
|
+
return
|
|
756
786
|
|
|
757
787
|
# 4. Select reader
|
|
758
788
|
reader = content.reader
|
|
@@ -818,10 +848,8 @@ class Knowledge:
|
|
|
818
848
|
|
|
819
849
|
for gcs_object in objects_to_read:
|
|
820
850
|
# 2. Setup Content object
|
|
821
|
-
id = str(uuid4())
|
|
822
851
|
name = (content.name or "content") + "_" + gcs_object.name
|
|
823
852
|
content_entry = Content(
|
|
824
|
-
id=id,
|
|
825
853
|
name=name,
|
|
826
854
|
description=content.description,
|
|
827
855
|
status=ContentStatus.PROCESSING,
|
|
@@ -830,15 +858,15 @@ class Knowledge:
|
|
|
830
858
|
)
|
|
831
859
|
|
|
832
860
|
# 3. Hash content and add it to the contents database
|
|
833
|
-
content_hash = self._build_content_hash(content_entry)
|
|
834
|
-
|
|
835
|
-
log_info(f"Content {content_hash} already exists, skipping")
|
|
836
|
-
continue
|
|
837
|
-
|
|
838
|
-
# 4. Add it to the contents database
|
|
861
|
+
content_entry.content_hash = self._build_content_hash(content_entry)
|
|
862
|
+
content_entry.id = generate_deterministic_id(content_entry.content_hash)
|
|
839
863
|
self._add_to_contents_db(content_entry)
|
|
864
|
+
if self._should_skip(content_entry.content_hash, skip_if_exists):
|
|
865
|
+
content_entry.status = ContentStatus.COMPLETED
|
|
866
|
+
self._update_content(content_entry)
|
|
867
|
+
return
|
|
840
868
|
|
|
841
|
-
#
|
|
869
|
+
# 4. Select reader
|
|
842
870
|
reader = content.reader
|
|
843
871
|
if reader is None:
|
|
844
872
|
if gcs_object.name.endswith(".pdf"):
|
|
@@ -866,7 +894,7 @@ class Knowledge:
|
|
|
866
894
|
read_document.content_id = content.id
|
|
867
895
|
await self._handle_vector_db_insert(content_entry, read_documents, upsert)
|
|
868
896
|
|
|
869
|
-
async def _handle_vector_db_insert(self, content, read_documents, upsert):
|
|
897
|
+
async def _handle_vector_db_insert(self, content: Content, read_documents, upsert):
|
|
870
898
|
if not self.vector_db:
|
|
871
899
|
log_error("No vector database configured")
|
|
872
900
|
content.status = ContentStatus.FAILED
|
|
@@ -876,7 +904,7 @@ class Knowledge:
|
|
|
876
904
|
|
|
877
905
|
if self.vector_db.upsert_available() and upsert:
|
|
878
906
|
try:
|
|
879
|
-
await self.vector_db.async_upsert(content.content_hash, read_documents, content.metadata)
|
|
907
|
+
await self.vector_db.async_upsert(content.content_hash, read_documents, content.metadata) # type: ignore[arg-type]
|
|
880
908
|
except Exception as e:
|
|
881
909
|
log_error(f"Error upserting document: {e}")
|
|
882
910
|
content.status = ContentStatus.FAILED
|
|
@@ -886,7 +914,9 @@ class Knowledge:
|
|
|
886
914
|
else:
|
|
887
915
|
try:
|
|
888
916
|
await self.vector_db.async_insert(
|
|
889
|
-
content.content_hash,
|
|
917
|
+
content.content_hash, # type: ignore[arg-type]
|
|
918
|
+
documents=read_documents,
|
|
919
|
+
filters=content.metadata, # type: ignore[arg-type]
|
|
890
920
|
)
|
|
891
921
|
except Exception as e:
|
|
892
922
|
log_error(f"Error inserting document: {e}")
|
|
@@ -1010,7 +1040,6 @@ class Knowledge:
|
|
|
1010
1040
|
content_row.status_message = content.status_message if content.status_message else ""
|
|
1011
1041
|
if content.external_id is not None:
|
|
1012
1042
|
content_row.external_id = content.external_id
|
|
1013
|
-
|
|
1014
1043
|
content_row.updated_at = int(time.time())
|
|
1015
1044
|
self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
|
|
1016
1045
|
|
|
@@ -1161,9 +1190,6 @@ class Knowledge:
|
|
|
1161
1190
|
|
|
1162
1191
|
read_documents = content.reader.read(content.topics)
|
|
1163
1192
|
if len(read_documents) > 0:
|
|
1164
|
-
print("READ DOCUMENTS: ", len(read_documents))
|
|
1165
|
-
print("READ DOCUMENTS: ", read_documents[0])
|
|
1166
|
-
|
|
1167
1193
|
if self.vector_db and hasattr(self.vector_db, "insert_text"):
|
|
1168
1194
|
result = await self.vector_db.insert_text(
|
|
1169
1195
|
file_source=content.topics[0],
|
agno/models/base.py
CHANGED
|
@@ -1228,7 +1228,7 @@ class Model(ABC):
|
|
|
1228
1228
|
function_execution_result=function_execution_result,
|
|
1229
1229
|
)
|
|
1230
1230
|
yield ModelResponse(
|
|
1231
|
-
content=f"{function_call.get_call_str()} completed in {function_call_timer.elapsed:.4f}s.",
|
|
1231
|
+
content=f"{function_call.get_call_str()} completed in {function_call_timer.elapsed:.4f}s. ",
|
|
1232
1232
|
tool_executions=[
|
|
1233
1233
|
ToolExecution(
|
|
1234
1234
|
tool_call_id=function_call_result.tool_call_id,
|
|
@@ -1632,7 +1632,7 @@ class Model(ABC):
|
|
|
1632
1632
|
function_execution_result=function_execution_result,
|
|
1633
1633
|
)
|
|
1634
1634
|
yield ModelResponse(
|
|
1635
|
-
content=f"{function_call.get_call_str()} completed in {function_call_timer.elapsed:.4f}s.",
|
|
1635
|
+
content=f"{function_call.get_call_str()} completed in {function_call_timer.elapsed:.4f}s. ",
|
|
1636
1636
|
tool_executions=[
|
|
1637
1637
|
ToolExecution(
|
|
1638
1638
|
tool_call_id=function_call_result.tool_call_id,
|
agno/models/openai/chat.py
CHANGED
|
@@ -70,6 +70,7 @@ class OpenAIChat(Model):
|
|
|
70
70
|
service_tier: Optional[str] = None # "auto" | "default" | "flex" | "priority", defaults to "auto" when not set
|
|
71
71
|
extra_headers: Optional[Any] = None
|
|
72
72
|
extra_query: Optional[Any] = None
|
|
73
|
+
extra_body: Optional[Any] = None
|
|
73
74
|
request_params: Optional[Dict[str, Any]] = None
|
|
74
75
|
role_map: Optional[Dict[str, str]] = None
|
|
75
76
|
|
|
@@ -191,6 +192,7 @@ class OpenAIChat(Model):
|
|
|
191
192
|
"top_p": self.top_p,
|
|
192
193
|
"extra_headers": self.extra_headers,
|
|
193
194
|
"extra_query": self.extra_query,
|
|
195
|
+
"extra_body": self.extra_body,
|
|
194
196
|
"metadata": self.metadata,
|
|
195
197
|
"service_tier": self.service_tier,
|
|
196
198
|
}
|
|
@@ -270,6 +272,7 @@ class OpenAIChat(Model):
|
|
|
270
272
|
"user": self.user,
|
|
271
273
|
"extra_headers": self.extra_headers,
|
|
272
274
|
"extra_query": self.extra_query,
|
|
275
|
+
"extra_body": self.extra_body,
|
|
273
276
|
"service_tier": self.service_tier,
|
|
274
277
|
}
|
|
275
278
|
)
|
agno/models/openai/responses.py
CHANGED
|
@@ -56,6 +56,9 @@ class OpenAIResponses(Model):
|
|
|
56
56
|
truncation: Optional[Literal["auto", "disabled"]] = None
|
|
57
57
|
user: Optional[str] = None
|
|
58
58
|
service_tier: Optional[Literal["auto", "default", "flex", "priority"]] = None
|
|
59
|
+
extra_headers: Optional[Any] = None
|
|
60
|
+
extra_query: Optional[Any] = None
|
|
61
|
+
extra_body: Optional[Any] = None
|
|
59
62
|
request_params: Optional[Dict[str, Any]] = None
|
|
60
63
|
|
|
61
64
|
# Client parameters
|
|
@@ -202,6 +205,9 @@ class OpenAIResponses(Model):
|
|
|
202
205
|
"truncation": self.truncation,
|
|
203
206
|
"user": self.user,
|
|
204
207
|
"service_tier": self.service_tier,
|
|
208
|
+
"extra_headers": self.extra_headers,
|
|
209
|
+
"extra_query": self.extra_query,
|
|
210
|
+
"extra_body": self.extra_body,
|
|
205
211
|
}
|
|
206
212
|
# Populate the reasoning parameter
|
|
207
213
|
base_params = self._set_reasoning_request_param(base_params)
|