agno 2.2.13__py3-none-any.whl → 2.4.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agno/agent/__init__.py +6 -0
- agno/agent/agent.py +5252 -3145
- agno/agent/remote.py +525 -0
- agno/api/api.py +2 -0
- agno/client/__init__.py +3 -0
- agno/client/a2a/__init__.py +10 -0
- agno/client/a2a/client.py +554 -0
- agno/client/a2a/schemas.py +112 -0
- agno/client/a2a/utils.py +369 -0
- agno/client/os.py +2669 -0
- agno/compression/__init__.py +3 -0
- agno/compression/manager.py +247 -0
- agno/culture/manager.py +2 -2
- agno/db/base.py +927 -6
- agno/db/dynamo/dynamo.py +788 -2
- agno/db/dynamo/schemas.py +128 -0
- agno/db/dynamo/utils.py +26 -3
- agno/db/firestore/firestore.py +674 -50
- agno/db/firestore/schemas.py +41 -0
- agno/db/firestore/utils.py +25 -10
- agno/db/gcs_json/gcs_json_db.py +506 -3
- agno/db/gcs_json/utils.py +14 -2
- agno/db/in_memory/in_memory_db.py +203 -4
- agno/db/in_memory/utils.py +14 -2
- agno/db/json/json_db.py +498 -2
- agno/db/json/utils.py +14 -2
- agno/db/migrations/manager.py +199 -0
- agno/db/migrations/utils.py +19 -0
- agno/db/migrations/v1_to_v2.py +54 -16
- agno/db/migrations/versions/__init__.py +0 -0
- agno/db/migrations/versions/v2_3_0.py +977 -0
- agno/db/mongo/async_mongo.py +1013 -39
- agno/db/mongo/mongo.py +684 -4
- agno/db/mongo/schemas.py +48 -0
- agno/db/mongo/utils.py +17 -0
- agno/db/mysql/__init__.py +2 -1
- agno/db/mysql/async_mysql.py +2958 -0
- agno/db/mysql/mysql.py +722 -53
- agno/db/mysql/schemas.py +77 -11
- agno/db/mysql/utils.py +151 -8
- agno/db/postgres/async_postgres.py +1254 -137
- agno/db/postgres/postgres.py +2316 -93
- agno/db/postgres/schemas.py +153 -21
- agno/db/postgres/utils.py +22 -7
- agno/db/redis/redis.py +531 -3
- agno/db/redis/schemas.py +36 -0
- agno/db/redis/utils.py +31 -15
- agno/db/schemas/evals.py +1 -0
- agno/db/schemas/memory.py +20 -9
- agno/db/singlestore/schemas.py +70 -1
- agno/db/singlestore/singlestore.py +737 -74
- agno/db/singlestore/utils.py +13 -3
- agno/db/sqlite/async_sqlite.py +1069 -89
- agno/db/sqlite/schemas.py +133 -1
- agno/db/sqlite/sqlite.py +2203 -165
- agno/db/sqlite/utils.py +21 -11
- agno/db/surrealdb/models.py +25 -0
- agno/db/surrealdb/surrealdb.py +603 -1
- agno/db/utils.py +60 -0
- agno/eval/__init__.py +26 -3
- agno/eval/accuracy.py +25 -12
- agno/eval/agent_as_judge.py +871 -0
- agno/eval/base.py +29 -0
- agno/eval/performance.py +10 -4
- agno/eval/reliability.py +22 -13
- agno/eval/utils.py +2 -1
- agno/exceptions.py +42 -0
- agno/hooks/__init__.py +3 -0
- agno/hooks/decorator.py +164 -0
- agno/integrations/discord/client.py +13 -2
- agno/knowledge/__init__.py +4 -0
- agno/knowledge/chunking/code.py +90 -0
- agno/knowledge/chunking/document.py +65 -4
- agno/knowledge/chunking/fixed.py +4 -1
- agno/knowledge/chunking/markdown.py +102 -11
- agno/knowledge/chunking/recursive.py +2 -2
- agno/knowledge/chunking/semantic.py +130 -48
- agno/knowledge/chunking/strategy.py +18 -0
- agno/knowledge/embedder/azure_openai.py +0 -1
- agno/knowledge/embedder/google.py +1 -1
- agno/knowledge/embedder/mistral.py +1 -1
- agno/knowledge/embedder/nebius.py +1 -1
- agno/knowledge/embedder/openai.py +16 -12
- agno/knowledge/filesystem.py +412 -0
- agno/knowledge/knowledge.py +4261 -1199
- agno/knowledge/protocol.py +134 -0
- agno/knowledge/reader/arxiv_reader.py +3 -2
- agno/knowledge/reader/base.py +9 -7
- agno/knowledge/reader/csv_reader.py +91 -42
- agno/knowledge/reader/docx_reader.py +9 -10
- agno/knowledge/reader/excel_reader.py +225 -0
- agno/knowledge/reader/field_labeled_csv_reader.py +38 -48
- agno/knowledge/reader/firecrawl_reader.py +3 -2
- agno/knowledge/reader/json_reader.py +16 -22
- agno/knowledge/reader/markdown_reader.py +15 -14
- agno/knowledge/reader/pdf_reader.py +33 -28
- agno/knowledge/reader/pptx_reader.py +9 -10
- agno/knowledge/reader/reader_factory.py +135 -1
- agno/knowledge/reader/s3_reader.py +8 -16
- agno/knowledge/reader/tavily_reader.py +3 -3
- agno/knowledge/reader/text_reader.py +15 -14
- agno/knowledge/reader/utils/__init__.py +17 -0
- agno/knowledge/reader/utils/spreadsheet.py +114 -0
- agno/knowledge/reader/web_search_reader.py +8 -65
- agno/knowledge/reader/website_reader.py +16 -13
- agno/knowledge/reader/wikipedia_reader.py +36 -3
- agno/knowledge/reader/youtube_reader.py +3 -2
- agno/knowledge/remote_content/__init__.py +33 -0
- agno/knowledge/remote_content/config.py +266 -0
- agno/knowledge/remote_content/remote_content.py +105 -17
- agno/knowledge/utils.py +76 -22
- agno/learn/__init__.py +71 -0
- agno/learn/config.py +463 -0
- agno/learn/curate.py +185 -0
- agno/learn/machine.py +725 -0
- agno/learn/schemas.py +1114 -0
- agno/learn/stores/__init__.py +38 -0
- agno/learn/stores/decision_log.py +1156 -0
- agno/learn/stores/entity_memory.py +3275 -0
- agno/learn/stores/learned_knowledge.py +1583 -0
- agno/learn/stores/protocol.py +117 -0
- agno/learn/stores/session_context.py +1217 -0
- agno/learn/stores/user_memory.py +1495 -0
- agno/learn/stores/user_profile.py +1220 -0
- agno/learn/utils.py +209 -0
- agno/media.py +22 -6
- agno/memory/__init__.py +14 -1
- agno/memory/manager.py +223 -8
- agno/memory/strategies/__init__.py +15 -0
- agno/memory/strategies/base.py +66 -0
- agno/memory/strategies/summarize.py +196 -0
- agno/memory/strategies/types.py +37 -0
- agno/models/aimlapi/aimlapi.py +17 -0
- agno/models/anthropic/claude.py +434 -59
- agno/models/aws/bedrock.py +121 -20
- agno/models/aws/claude.py +131 -274
- agno/models/azure/ai_foundry.py +10 -6
- agno/models/azure/openai_chat.py +33 -10
- agno/models/base.py +1162 -561
- agno/models/cerebras/cerebras.py +120 -24
- agno/models/cerebras/cerebras_openai.py +21 -2
- agno/models/cohere/chat.py +65 -6
- agno/models/cometapi/cometapi.py +18 -1
- agno/models/dashscope/dashscope.py +2 -3
- agno/models/deepinfra/deepinfra.py +18 -1
- agno/models/deepseek/deepseek.py +69 -3
- agno/models/fireworks/fireworks.py +18 -1
- agno/models/google/gemini.py +959 -89
- agno/models/google/utils.py +22 -0
- agno/models/groq/groq.py +48 -18
- agno/models/huggingface/huggingface.py +17 -6
- agno/models/ibm/watsonx.py +16 -6
- agno/models/internlm/internlm.py +18 -1
- agno/models/langdb/langdb.py +13 -1
- agno/models/litellm/chat.py +88 -9
- agno/models/litellm/litellm_openai.py +18 -1
- agno/models/message.py +24 -5
- agno/models/meta/llama.py +40 -13
- agno/models/meta/llama_openai.py +22 -21
- agno/models/metrics.py +12 -0
- agno/models/mistral/mistral.py +8 -4
- agno/models/n1n/__init__.py +3 -0
- agno/models/n1n/n1n.py +57 -0
- agno/models/nebius/nebius.py +6 -7
- agno/models/nvidia/nvidia.py +20 -3
- agno/models/ollama/__init__.py +2 -0
- agno/models/ollama/chat.py +17 -6
- agno/models/ollama/responses.py +100 -0
- agno/models/openai/__init__.py +2 -0
- agno/models/openai/chat.py +117 -26
- agno/models/openai/open_responses.py +46 -0
- agno/models/openai/responses.py +110 -32
- agno/models/openrouter/__init__.py +2 -0
- agno/models/openrouter/openrouter.py +67 -2
- agno/models/openrouter/responses.py +146 -0
- agno/models/perplexity/perplexity.py +19 -1
- agno/models/portkey/portkey.py +7 -6
- agno/models/requesty/requesty.py +19 -2
- agno/models/response.py +20 -2
- agno/models/sambanova/sambanova.py +20 -3
- agno/models/siliconflow/siliconflow.py +19 -2
- agno/models/together/together.py +20 -3
- agno/models/vercel/v0.py +20 -3
- agno/models/vertexai/claude.py +124 -4
- agno/models/vllm/vllm.py +19 -14
- agno/models/xai/xai.py +19 -2
- agno/os/app.py +467 -137
- agno/os/auth.py +253 -5
- agno/os/config.py +22 -0
- agno/os/interfaces/a2a/a2a.py +7 -6
- agno/os/interfaces/a2a/router.py +635 -26
- agno/os/interfaces/a2a/utils.py +32 -33
- agno/os/interfaces/agui/agui.py +5 -3
- agno/os/interfaces/agui/router.py +26 -16
- agno/os/interfaces/agui/utils.py +97 -57
- agno/os/interfaces/base.py +7 -7
- agno/os/interfaces/slack/router.py +16 -7
- agno/os/interfaces/slack/slack.py +7 -7
- agno/os/interfaces/whatsapp/router.py +35 -7
- agno/os/interfaces/whatsapp/security.py +3 -1
- agno/os/interfaces/whatsapp/whatsapp.py +11 -8
- agno/os/managers.py +326 -0
- agno/os/mcp.py +652 -79
- agno/os/middleware/__init__.py +4 -0
- agno/os/middleware/jwt.py +718 -115
- agno/os/middleware/trailing_slash.py +27 -0
- agno/os/router.py +105 -1558
- agno/os/routers/agents/__init__.py +3 -0
- agno/os/routers/agents/router.py +655 -0
- agno/os/routers/agents/schema.py +288 -0
- agno/os/routers/components/__init__.py +3 -0
- agno/os/routers/components/components.py +475 -0
- agno/os/routers/database.py +155 -0
- agno/os/routers/evals/evals.py +111 -18
- agno/os/routers/evals/schemas.py +38 -5
- agno/os/routers/evals/utils.py +80 -11
- agno/os/routers/health.py +3 -3
- agno/os/routers/knowledge/knowledge.py +284 -35
- agno/os/routers/knowledge/schemas.py +14 -2
- agno/os/routers/memory/memory.py +274 -11
- agno/os/routers/memory/schemas.py +44 -3
- agno/os/routers/metrics/metrics.py +30 -15
- agno/os/routers/metrics/schemas.py +10 -6
- agno/os/routers/registry/__init__.py +3 -0
- agno/os/routers/registry/registry.py +337 -0
- agno/os/routers/session/session.py +143 -14
- agno/os/routers/teams/__init__.py +3 -0
- agno/os/routers/teams/router.py +550 -0
- agno/os/routers/teams/schema.py +280 -0
- agno/os/routers/traces/__init__.py +3 -0
- agno/os/routers/traces/schemas.py +414 -0
- agno/os/routers/traces/traces.py +549 -0
- agno/os/routers/workflows/__init__.py +3 -0
- agno/os/routers/workflows/router.py +757 -0
- agno/os/routers/workflows/schema.py +139 -0
- agno/os/schema.py +157 -584
- agno/os/scopes.py +469 -0
- agno/os/settings.py +3 -0
- agno/os/utils.py +574 -185
- agno/reasoning/anthropic.py +85 -1
- agno/reasoning/azure_ai_foundry.py +93 -1
- agno/reasoning/deepseek.py +102 -2
- agno/reasoning/default.py +6 -7
- agno/reasoning/gemini.py +87 -3
- agno/reasoning/groq.py +109 -2
- agno/reasoning/helpers.py +6 -7
- agno/reasoning/manager.py +1238 -0
- agno/reasoning/ollama.py +93 -1
- agno/reasoning/openai.py +115 -1
- agno/reasoning/vertexai.py +85 -1
- agno/registry/__init__.py +3 -0
- agno/registry/registry.py +68 -0
- agno/remote/__init__.py +3 -0
- agno/remote/base.py +581 -0
- agno/run/__init__.py +2 -4
- agno/run/agent.py +134 -19
- agno/run/base.py +49 -1
- agno/run/cancel.py +65 -52
- agno/run/cancellation_management/__init__.py +9 -0
- agno/run/cancellation_management/base.py +78 -0
- agno/run/cancellation_management/in_memory_cancellation_manager.py +100 -0
- agno/run/cancellation_management/redis_cancellation_manager.py +236 -0
- agno/run/requirement.py +181 -0
- agno/run/team.py +111 -19
- agno/run/workflow.py +2 -1
- agno/session/agent.py +57 -92
- agno/session/summary.py +1 -1
- agno/session/team.py +62 -115
- agno/session/workflow.py +353 -57
- agno/skills/__init__.py +17 -0
- agno/skills/agent_skills.py +377 -0
- agno/skills/errors.py +32 -0
- agno/skills/loaders/__init__.py +4 -0
- agno/skills/loaders/base.py +27 -0
- agno/skills/loaders/local.py +216 -0
- agno/skills/skill.py +65 -0
- agno/skills/utils.py +107 -0
- agno/skills/validator.py +277 -0
- agno/table.py +10 -0
- agno/team/__init__.py +5 -1
- agno/team/remote.py +447 -0
- agno/team/team.py +3769 -2202
- agno/tools/brandfetch.py +27 -18
- agno/tools/browserbase.py +225 -16
- agno/tools/crawl4ai.py +3 -0
- agno/tools/duckduckgo.py +25 -71
- agno/tools/exa.py +0 -21
- agno/tools/file.py +14 -13
- agno/tools/file_generation.py +12 -6
- agno/tools/firecrawl.py +15 -7
- agno/tools/function.py +94 -113
- agno/tools/google_bigquery.py +11 -2
- agno/tools/google_drive.py +4 -3
- agno/tools/knowledge.py +9 -4
- agno/tools/mcp/mcp.py +301 -18
- agno/tools/mcp/multi_mcp.py +269 -14
- agno/tools/mem0.py +11 -10
- agno/tools/memory.py +47 -46
- agno/tools/mlx_transcribe.py +10 -7
- agno/tools/models/nebius.py +5 -5
- agno/tools/models_labs.py +20 -10
- agno/tools/nano_banana.py +151 -0
- agno/tools/parallel.py +0 -7
- agno/tools/postgres.py +76 -36
- agno/tools/python.py +14 -6
- agno/tools/reasoning.py +30 -23
- agno/tools/redshift.py +406 -0
- agno/tools/shopify.py +1519 -0
- agno/tools/spotify.py +919 -0
- agno/tools/tavily.py +4 -1
- agno/tools/toolkit.py +253 -18
- agno/tools/websearch.py +93 -0
- agno/tools/website.py +1 -1
- agno/tools/wikipedia.py +1 -1
- agno/tools/workflow.py +56 -48
- agno/tools/yfinance.py +12 -11
- agno/tracing/__init__.py +12 -0
- agno/tracing/exporter.py +161 -0
- agno/tracing/schemas.py +276 -0
- agno/tracing/setup.py +112 -0
- agno/utils/agent.py +251 -10
- agno/utils/cryptography.py +22 -0
- agno/utils/dttm.py +33 -0
- agno/utils/events.py +264 -7
- agno/utils/hooks.py +111 -3
- agno/utils/http.py +161 -2
- agno/utils/mcp.py +49 -8
- agno/utils/media.py +22 -1
- agno/utils/models/ai_foundry.py +9 -2
- agno/utils/models/claude.py +20 -5
- agno/utils/models/cohere.py +9 -2
- agno/utils/models/llama.py +9 -2
- agno/utils/models/mistral.py +4 -2
- agno/utils/os.py +0 -0
- agno/utils/print_response/agent.py +99 -16
- agno/utils/print_response/team.py +223 -24
- agno/utils/print_response/workflow.py +0 -2
- agno/utils/prompts.py +8 -6
- agno/utils/remote.py +23 -0
- agno/utils/response.py +1 -13
- agno/utils/string.py +91 -2
- agno/utils/team.py +62 -12
- agno/utils/tokens.py +657 -0
- agno/vectordb/base.py +15 -2
- agno/vectordb/cassandra/cassandra.py +1 -1
- agno/vectordb/chroma/__init__.py +2 -1
- agno/vectordb/chroma/chromadb.py +468 -23
- agno/vectordb/clickhouse/clickhousedb.py +1 -1
- agno/vectordb/couchbase/couchbase.py +6 -2
- agno/vectordb/lancedb/lance_db.py +7 -38
- agno/vectordb/lightrag/lightrag.py +7 -6
- agno/vectordb/milvus/milvus.py +118 -84
- agno/vectordb/mongodb/__init__.py +2 -1
- agno/vectordb/mongodb/mongodb.py +14 -31
- agno/vectordb/pgvector/pgvector.py +120 -66
- agno/vectordb/pineconedb/pineconedb.py +2 -19
- agno/vectordb/qdrant/__init__.py +2 -1
- agno/vectordb/qdrant/qdrant.py +33 -56
- agno/vectordb/redis/__init__.py +2 -1
- agno/vectordb/redis/redisdb.py +19 -31
- agno/vectordb/singlestore/singlestore.py +17 -9
- agno/vectordb/surrealdb/surrealdb.py +2 -38
- agno/vectordb/weaviate/__init__.py +2 -1
- agno/vectordb/weaviate/weaviate.py +7 -3
- agno/workflow/__init__.py +5 -1
- agno/workflow/agent.py +2 -2
- agno/workflow/condition.py +12 -10
- agno/workflow/loop.py +28 -9
- agno/workflow/parallel.py +21 -13
- agno/workflow/remote.py +362 -0
- agno/workflow/router.py +12 -9
- agno/workflow/step.py +261 -36
- agno/workflow/steps.py +12 -8
- agno/workflow/types.py +40 -77
- agno/workflow/workflow.py +939 -213
- {agno-2.2.13.dist-info → agno-2.4.3.dist-info}/METADATA +134 -181
- agno-2.4.3.dist-info/RECORD +677 -0
- {agno-2.2.13.dist-info → agno-2.4.3.dist-info}/WHEEL +1 -1
- agno/tools/googlesearch.py +0 -98
- agno/tools/memori.py +0 -339
- agno-2.2.13.dist-info/RECORD +0 -575
- {agno-2.2.13.dist-info → agno-2.4.3.dist-info}/licenses/LICENSE +0 -0
- {agno-2.2.13.dist-info → agno-2.4.3.dist-info}/top_level.txt +0 -0
agno/db/postgres/postgres.py
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import time
|
|
2
2
|
from datetime import date, datetime, timedelta, timezone
|
|
3
|
-
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Set, Tuple, Union, cast
|
|
4
4
|
from uuid import uuid4
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
from agno.tracing.schemas import Span, Trace
|
|
8
|
+
|
|
9
|
+
from agno.db.base import BaseDb, ComponentType, SessionType
|
|
10
|
+
from agno.db.migrations.manager import MigrationManager
|
|
7
11
|
from agno.db.postgres.schemas import get_table_schema_definition
|
|
8
12
|
from agno.db.postgres.utils import (
|
|
9
13
|
apply_sorting,
|
|
@@ -23,15 +27,30 @@ from agno.db.schemas.knowledge import KnowledgeRow
|
|
|
23
27
|
from agno.db.schemas.memory import UserMemory
|
|
24
28
|
from agno.session import AgentSession, Session, TeamSession, WorkflowSession
|
|
25
29
|
from agno.utils.log import log_debug, log_error, log_info, log_warning
|
|
26
|
-
from agno.utils.string import generate_id
|
|
30
|
+
from agno.utils.string import generate_id, sanitize_postgres_string, sanitize_postgres_strings
|
|
27
31
|
|
|
28
32
|
try:
|
|
29
|
-
from sqlalchemy import
|
|
33
|
+
from sqlalchemy import (
|
|
34
|
+
ForeignKey,
|
|
35
|
+
ForeignKeyConstraint,
|
|
36
|
+
Index,
|
|
37
|
+
PrimaryKeyConstraint,
|
|
38
|
+
String,
|
|
39
|
+
UniqueConstraint,
|
|
40
|
+
and_,
|
|
41
|
+
case,
|
|
42
|
+
func,
|
|
43
|
+
or_,
|
|
44
|
+
select,
|
|
45
|
+
update,
|
|
46
|
+
)
|
|
30
47
|
from sqlalchemy.dialects import postgresql
|
|
48
|
+
from sqlalchemy.dialects.postgresql import TIMESTAMP
|
|
31
49
|
from sqlalchemy.engine import Engine, create_engine
|
|
50
|
+
from sqlalchemy.exc import ProgrammingError
|
|
32
51
|
from sqlalchemy.orm import scoped_session, sessionmaker
|
|
33
52
|
from sqlalchemy.schema import Column, MetaData, Table
|
|
34
|
-
from sqlalchemy.sql.expression import
|
|
53
|
+
from sqlalchemy.sql.expression import text
|
|
35
54
|
except ImportError:
|
|
36
55
|
raise ImportError("`sqlalchemy` not installed. Please install it using `pip install sqlalchemy`")
|
|
37
56
|
|
|
@@ -48,7 +67,15 @@ class PostgresDb(BaseDb):
|
|
|
48
67
|
metrics_table: Optional[str] = None,
|
|
49
68
|
eval_table: Optional[str] = None,
|
|
50
69
|
knowledge_table: Optional[str] = None,
|
|
70
|
+
traces_table: Optional[str] = None,
|
|
71
|
+
spans_table: Optional[str] = None,
|
|
72
|
+
versions_table: Optional[str] = None,
|
|
73
|
+
components_table: Optional[str] = None,
|
|
74
|
+
component_configs_table: Optional[str] = None,
|
|
75
|
+
component_links_table: Optional[str] = None,
|
|
76
|
+
learnings_table: Optional[str] = None,
|
|
51
77
|
id: Optional[str] = None,
|
|
78
|
+
create_schema: bool = True,
|
|
52
79
|
):
|
|
53
80
|
"""
|
|
54
81
|
Interface for interacting with a PostgreSQL database.
|
|
@@ -68,7 +95,16 @@ class PostgresDb(BaseDb):
|
|
|
68
95
|
eval_table (Optional[str]): Name of the table to store evaluation runs data.
|
|
69
96
|
knowledge_table (Optional[str]): Name of the table to store knowledge content.
|
|
70
97
|
culture_table (Optional[str]): Name of the table to store cultural knowledge.
|
|
98
|
+
traces_table (Optional[str]): Name of the table to store run traces.
|
|
99
|
+
spans_table (Optional[str]): Name of the table to store span events.
|
|
100
|
+
versions_table (Optional[str]): Name of the table to store schema versions.
|
|
101
|
+
components_table (Optional[str]): Name of the table to store components.
|
|
102
|
+
component_configs_table (Optional[str]): Name of the table to store component configurations.
|
|
103
|
+
component_links_table (Optional[str]): Name of the table to store component references.
|
|
104
|
+
learnings_table (Optional[str]): Name of the table to store learnings.
|
|
71
105
|
id (Optional[str]): ID of the database.
|
|
106
|
+
create_schema (bool): Whether to automatically create the database schema if it doesn't exist.
|
|
107
|
+
Set to False if schema is managed externally (e.g., via migrations). Defaults to True.
|
|
72
108
|
|
|
73
109
|
Raises:
|
|
74
110
|
ValueError: If neither db_url nor db_engine is provided.
|
|
@@ -76,7 +112,11 @@ class PostgresDb(BaseDb):
|
|
|
76
112
|
"""
|
|
77
113
|
_engine: Optional[Engine] = db_engine
|
|
78
114
|
if _engine is None and db_url is not None:
|
|
79
|
-
_engine = create_engine(
|
|
115
|
+
_engine = create_engine(
|
|
116
|
+
db_url,
|
|
117
|
+
pool_pre_ping=True,
|
|
118
|
+
pool_recycle=3600,
|
|
119
|
+
)
|
|
80
120
|
if _engine is None:
|
|
81
121
|
raise ValueError("One of db_url or db_engine must be provided")
|
|
82
122
|
|
|
@@ -97,13 +137,62 @@ class PostgresDb(BaseDb):
|
|
|
97
137
|
eval_table=eval_table,
|
|
98
138
|
knowledge_table=knowledge_table,
|
|
99
139
|
culture_table=culture_table,
|
|
140
|
+
traces_table=traces_table,
|
|
141
|
+
spans_table=spans_table,
|
|
142
|
+
versions_table=versions_table,
|
|
143
|
+
components_table=components_table,
|
|
144
|
+
component_configs_table=component_configs_table,
|
|
145
|
+
component_links_table=component_links_table,
|
|
146
|
+
learnings_table=learnings_table,
|
|
100
147
|
)
|
|
101
148
|
|
|
102
149
|
self.db_schema: str = db_schema if db_schema is not None else "ai"
|
|
103
|
-
self.metadata: MetaData = MetaData()
|
|
150
|
+
self.metadata: MetaData = MetaData(schema=self.db_schema)
|
|
151
|
+
self.create_schema: bool = create_schema
|
|
104
152
|
|
|
105
153
|
# Initialize database session
|
|
106
|
-
self.Session: scoped_session = scoped_session(sessionmaker(bind=self.db_engine))
|
|
154
|
+
self.Session: scoped_session = scoped_session(sessionmaker(bind=self.db_engine, expire_on_commit=False))
|
|
155
|
+
|
|
156
|
+
# -- Serialization methods --
|
|
157
|
+
def to_dict(self):
|
|
158
|
+
base = super().to_dict()
|
|
159
|
+
base.update(
|
|
160
|
+
{
|
|
161
|
+
"db_url": self.db_url,
|
|
162
|
+
"db_schema": self.db_schema,
|
|
163
|
+
"type": "postgres",
|
|
164
|
+
}
|
|
165
|
+
)
|
|
166
|
+
return base
|
|
167
|
+
|
|
168
|
+
@classmethod
|
|
169
|
+
def from_dict(cls, data):
|
|
170
|
+
return cls(
|
|
171
|
+
db_url=data.get("db_url"),
|
|
172
|
+
db_schema=data.get("db_schema"),
|
|
173
|
+
session_table=data.get("session_table"),
|
|
174
|
+
culture_table=data.get("culture_table"),
|
|
175
|
+
memory_table=data.get("memory_table"),
|
|
176
|
+
metrics_table=data.get("metrics_table"),
|
|
177
|
+
eval_table=data.get("eval_table"),
|
|
178
|
+
knowledge_table=data.get("knowledge_table"),
|
|
179
|
+
traces_table=data.get("traces_table"),
|
|
180
|
+
spans_table=data.get("spans_table"),
|
|
181
|
+
versions_table=data.get("versions_table"),
|
|
182
|
+
components_table=data.get("components_table"),
|
|
183
|
+
component_configs_table=data.get("component_configs_table"),
|
|
184
|
+
component_links_table=data.get("component_links_table"),
|
|
185
|
+
id=data.get("id"),
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
def close(self) -> None:
|
|
189
|
+
"""Close database connections and dispose of the connection pool.
|
|
190
|
+
|
|
191
|
+
Should be called during application shutdown to properly release
|
|
192
|
+
all database connections.
|
|
193
|
+
"""
|
|
194
|
+
if self.db_engine is not None:
|
|
195
|
+
self.db_engine.dispose()
|
|
107
196
|
|
|
108
197
|
# -- DB methods --
|
|
109
198
|
def table_exists(self, table_name: str) -> bool:
|
|
@@ -126,102 +215,214 @@ class PostgresDb(BaseDb):
|
|
|
126
215
|
(self.metrics_table_name, "metrics"),
|
|
127
216
|
(self.eval_table_name, "evals"),
|
|
128
217
|
(self.knowledge_table_name, "knowledge"),
|
|
218
|
+
(self.versions_table_name, "versions"),
|
|
219
|
+
(self.components_table_name, "components"),
|
|
220
|
+
(self.component_configs_table_name, "component_configs"),
|
|
221
|
+
(self.component_links_table_name, "component_links"),
|
|
222
|
+
(self.learnings_table_name, "learnings"),
|
|
129
223
|
]
|
|
130
224
|
|
|
131
225
|
for table_name, table_type in tables_to_create:
|
|
132
|
-
self.
|
|
226
|
+
self._get_or_create_table(table_name=table_name, table_type=table_type, create_table_if_not_found=True)
|
|
133
227
|
|
|
134
|
-
def _create_table(self, table_name: str, table_type: str
|
|
228
|
+
def _create_table(self, table_name: str, table_type: str) -> Table:
|
|
135
229
|
"""
|
|
136
230
|
Create a table with the appropriate schema based on the table type.
|
|
137
231
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
Returns:
|
|
144
|
-
Table: SQLAlchemy Table object
|
|
232
|
+
Supports:
|
|
233
|
+
- _unique_constraints: [{"name": "...", "columns": [...]}]
|
|
234
|
+
- __primary_key__: ["col1", "col2", ...]
|
|
235
|
+
- __foreign_keys__: [{"columns":[...], "ref_table":"...", "ref_columns":[...]}]
|
|
236
|
+
- column-level foreign_key: "logical_table.column" (resolved via _resolve_* helpers)
|
|
145
237
|
"""
|
|
146
238
|
try:
|
|
147
|
-
|
|
239
|
+
# Pass traces_table_name and db_schema for spans table foreign key resolution
|
|
240
|
+
table_schema = get_table_schema_definition(
|
|
241
|
+
table_type, traces_table_name=self.trace_table_name, db_schema=self.db_schema
|
|
242
|
+
).copy()
|
|
148
243
|
|
|
149
244
|
columns: List[Column] = []
|
|
150
245
|
indexes: List[str] = []
|
|
151
|
-
|
|
246
|
+
|
|
247
|
+
# Extract special schema keys before iterating columns
|
|
152
248
|
schema_unique_constraints = table_schema.pop("_unique_constraints", [])
|
|
249
|
+
schema_primary_key = table_schema.pop("__primary_key__", None)
|
|
250
|
+
schema_foreign_keys = table_schema.pop("__foreign_keys__", [])
|
|
153
251
|
|
|
154
|
-
#
|
|
252
|
+
# Build columns
|
|
155
253
|
for col_name, col_config in table_schema.items():
|
|
156
254
|
column_args = [col_name, col_config["type"]()]
|
|
157
|
-
column_kwargs = {}
|
|
158
|
-
|
|
255
|
+
column_kwargs: Dict[str, Any] = {}
|
|
256
|
+
|
|
257
|
+
# Column-level PK only if no composite PK is defined
|
|
258
|
+
if col_config.get("primary_key", False) and schema_primary_key is None:
|
|
159
259
|
column_kwargs["primary_key"] = True
|
|
260
|
+
|
|
160
261
|
if "nullable" in col_config:
|
|
161
262
|
column_kwargs["nullable"] = col_config["nullable"]
|
|
263
|
+
|
|
264
|
+
if "default" in col_config:
|
|
265
|
+
column_kwargs["default"] = col_config["default"]
|
|
266
|
+
|
|
162
267
|
if col_config.get("index", False):
|
|
163
268
|
indexes.append(col_name)
|
|
269
|
+
|
|
164
270
|
if col_config.get("unique", False):
|
|
165
271
|
column_kwargs["unique"] = True
|
|
166
|
-
|
|
167
|
-
|
|
272
|
+
|
|
273
|
+
# Single-column FK
|
|
274
|
+
if "foreign_key" in col_config:
|
|
275
|
+
fk_ref = self._resolve_fk_reference(col_config["foreign_key"])
|
|
276
|
+
column_args.append(ForeignKey(fk_ref))
|
|
277
|
+
|
|
278
|
+
columns.append(Column(*column_args, **column_kwargs))
|
|
168
279
|
|
|
169
280
|
# Create the table object
|
|
170
|
-
|
|
171
|
-
|
|
281
|
+
table = Table(table_name, self.metadata, *columns, schema=self.db_schema)
|
|
282
|
+
|
|
283
|
+
# Composite PK
|
|
284
|
+
if schema_primary_key is not None:
|
|
285
|
+
missing = [c for c in schema_primary_key if c not in table.c]
|
|
286
|
+
if missing:
|
|
287
|
+
raise ValueError(f"Composite PK references missing columns in {table_name}: {missing}")
|
|
288
|
+
|
|
289
|
+
pk_constraint_name = f"{table_name}_pkey"
|
|
290
|
+
table.append_constraint(PrimaryKeyConstraint(*schema_primary_key, name=pk_constraint_name))
|
|
291
|
+
|
|
292
|
+
# Composite FKs
|
|
293
|
+
for fk_config in schema_foreign_keys:
|
|
294
|
+
fk_columns = fk_config["columns"]
|
|
295
|
+
ref_table_logical = fk_config["ref_table"]
|
|
296
|
+
ref_columns = fk_config["ref_columns"]
|
|
297
|
+
|
|
298
|
+
if len(fk_columns) != len(ref_columns):
|
|
299
|
+
raise ValueError(
|
|
300
|
+
f"Composite FK in {table_name} has mismatched columns/ref_columns: {fk_columns} vs {ref_columns}"
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
missing = [c for c in fk_columns if c not in table.c]
|
|
304
|
+
if missing:
|
|
305
|
+
raise ValueError(f"Composite FK references missing columns in {table_name}: {missing}")
|
|
306
|
+
|
|
307
|
+
resolved_ref_table = self._resolve_table_name(ref_table_logical)
|
|
308
|
+
fk_constraint_name = f"{table_name}_{'_'.join(fk_columns)}_fkey"
|
|
309
|
+
|
|
310
|
+
# IMPORTANT: since Table(schema=self.db_schema) is used, do NOT schema-qualify these targets.
|
|
311
|
+
ref_column_strings = [f"{resolved_ref_table}.{col}" for col in ref_columns]
|
|
312
|
+
|
|
313
|
+
table.append_constraint(
|
|
314
|
+
ForeignKeyConstraint(
|
|
315
|
+
fk_columns,
|
|
316
|
+
ref_column_strings,
|
|
317
|
+
name=fk_constraint_name,
|
|
318
|
+
)
|
|
319
|
+
)
|
|
172
320
|
|
|
173
|
-
#
|
|
321
|
+
# Multi-column unique constraints
|
|
174
322
|
for constraint in schema_unique_constraints:
|
|
175
323
|
constraint_name = f"{table_name}_{constraint['name']}"
|
|
176
324
|
constraint_columns = constraint["columns"]
|
|
325
|
+
|
|
326
|
+
missing = [c for c in constraint_columns if c not in table.c]
|
|
327
|
+
if missing:
|
|
328
|
+
raise ValueError(f"Unique constraint references missing columns in {table_name}: {missing}")
|
|
329
|
+
|
|
177
330
|
table.append_constraint(UniqueConstraint(*constraint_columns, name=constraint_name))
|
|
178
331
|
|
|
179
|
-
#
|
|
332
|
+
# Indexes
|
|
180
333
|
for idx_col in indexes:
|
|
334
|
+
if idx_col not in table.c:
|
|
335
|
+
raise ValueError(f"Index references missing column in {table_name}: {idx_col}")
|
|
181
336
|
idx_name = f"idx_{table_name}_{idx_col}"
|
|
182
|
-
|
|
337
|
+
Index(idx_name, table.c[idx_col]) # Correct way; do NOT append as constraint
|
|
183
338
|
|
|
184
|
-
|
|
185
|
-
|
|
339
|
+
# Create schema if requested
|
|
340
|
+
if self.create_schema:
|
|
341
|
+
with self.Session() as sess, sess.begin():
|
|
342
|
+
create_schema(session=sess, db_schema=self.db_schema)
|
|
186
343
|
|
|
187
344
|
# Create table
|
|
188
|
-
|
|
345
|
+
table_created = False
|
|
346
|
+
if not self.table_exists(table_name):
|
|
347
|
+
table.create(self.db_engine, checkfirst=True)
|
|
348
|
+
log_debug(f"Successfully created table '{self.db_schema}.{table_name}'")
|
|
349
|
+
table_created = True
|
|
350
|
+
else:
|
|
351
|
+
log_debug(f"Table {self.db_schema}.{table_name} already exists, skipping creation")
|
|
189
352
|
|
|
190
|
-
# Create indexes
|
|
353
|
+
# Create indexes (Postgres)
|
|
191
354
|
for idx in table.indexes:
|
|
192
355
|
try:
|
|
193
|
-
# Check if index already exists
|
|
194
356
|
with self.Session() as sess:
|
|
195
357
|
exists_query = text(
|
|
196
358
|
"SELECT 1 FROM pg_indexes WHERE schemaname = :schema AND indexname = :index_name"
|
|
197
359
|
)
|
|
198
360
|
exists = (
|
|
199
|
-
sess.execute(exists_query, {"schema": db_schema, "index_name": idx.name}).scalar()
|
|
361
|
+
sess.execute(exists_query, {"schema": self.db_schema, "index_name": idx.name}).scalar()
|
|
200
362
|
is not None
|
|
201
363
|
)
|
|
202
364
|
if exists:
|
|
203
|
-
log_debug(
|
|
365
|
+
log_debug(
|
|
366
|
+
f"Index {idx.name} already exists in {self.db_schema}.{table_name}, skipping creation"
|
|
367
|
+
)
|
|
204
368
|
continue
|
|
205
369
|
|
|
206
370
|
idx.create(self.db_engine)
|
|
207
|
-
log_debug(f"Created index: {idx.name} for table {db_schema}.{table_name}")
|
|
371
|
+
log_debug(f"Created index: {idx.name} for table {self.db_schema}.{table_name}")
|
|
208
372
|
|
|
209
373
|
except Exception as e:
|
|
210
374
|
log_error(f"Error creating index {idx.name}: {e}")
|
|
211
375
|
|
|
212
|
-
|
|
376
|
+
# Store the schema version for the created table
|
|
377
|
+
if table_name != self.versions_table_name and table_created:
|
|
378
|
+
latest_schema_version = MigrationManager(self).latest_schema_version
|
|
379
|
+
self.upsert_schema_version(table_name=table_name, version=latest_schema_version.public)
|
|
380
|
+
|
|
213
381
|
return table
|
|
214
382
|
|
|
215
383
|
except Exception as e:
|
|
216
|
-
log_error(f"Could not create table {db_schema}.{table_name}: {e}")
|
|
384
|
+
log_error(f"Could not create table {self.db_schema}.{table_name}: {e}")
|
|
217
385
|
raise
|
|
218
386
|
|
|
387
|
+
def _resolve_fk_reference(self, fk_ref: str) -> str:
|
|
388
|
+
"""
|
|
389
|
+
Resolve a simple foreign key reference to fully qualified name.
|
|
390
|
+
|
|
391
|
+
Accepts:
|
|
392
|
+
- "logical_table.column" -> "{schema}.{resolved_table}.{column}"
|
|
393
|
+
- already-qualified refs -> returned as-is
|
|
394
|
+
"""
|
|
395
|
+
parts = fk_ref.split(".")
|
|
396
|
+
if len(parts) == 2:
|
|
397
|
+
table, column = parts
|
|
398
|
+
resolved_table = self._resolve_table_name(table)
|
|
399
|
+
return f"{self.db_schema}.{resolved_table}.{column}"
|
|
400
|
+
return fk_ref
|
|
401
|
+
|
|
402
|
+
def _resolve_table_name(self, logical_name: str) -> str:
|
|
403
|
+
"""
|
|
404
|
+
Resolve logical table name to configured table name.
|
|
405
|
+
"""
|
|
406
|
+
table_map = {
|
|
407
|
+
"traces": self.trace_table_name,
|
|
408
|
+
"spans": self.span_table_name,
|
|
409
|
+
"sessions": self.session_table_name,
|
|
410
|
+
"memories": self.memory_table_name,
|
|
411
|
+
"metrics": self.metrics_table_name,
|
|
412
|
+
"evals": self.eval_table_name,
|
|
413
|
+
"knowledge": self.knowledge_table_name,
|
|
414
|
+
"versions": self.versions_table_name,
|
|
415
|
+
"components": self.components_table_name,
|
|
416
|
+
"component_configs": self.component_configs_table_name,
|
|
417
|
+
"component_links": self.component_links_table_name,
|
|
418
|
+
}
|
|
419
|
+
return table_map.get(logical_name, logical_name)
|
|
420
|
+
|
|
219
421
|
def _get_table(self, table_type: str, create_table_if_not_found: Optional[bool] = False) -> Optional[Table]:
|
|
220
422
|
if table_type == "sessions":
|
|
221
423
|
self.session_table = self._get_or_create_table(
|
|
222
424
|
table_name=self.session_table_name,
|
|
223
425
|
table_type="sessions",
|
|
224
|
-
db_schema=self.db_schema,
|
|
225
426
|
create_table_if_not_found=create_table_if_not_found,
|
|
226
427
|
)
|
|
227
428
|
return self.session_table
|
|
@@ -230,7 +431,6 @@ class PostgresDb(BaseDb):
|
|
|
230
431
|
self.memory_table = self._get_or_create_table(
|
|
231
432
|
table_name=self.memory_table_name,
|
|
232
433
|
table_type="memories",
|
|
233
|
-
db_schema=self.db_schema,
|
|
234
434
|
create_table_if_not_found=create_table_if_not_found,
|
|
235
435
|
)
|
|
236
436
|
return self.memory_table
|
|
@@ -239,7 +439,6 @@ class PostgresDb(BaseDb):
|
|
|
239
439
|
self.metrics_table = self._get_or_create_table(
|
|
240
440
|
table_name=self.metrics_table_name,
|
|
241
441
|
table_type="metrics",
|
|
242
|
-
db_schema=self.db_schema,
|
|
243
442
|
create_table_if_not_found=create_table_if_not_found,
|
|
244
443
|
)
|
|
245
444
|
return self.metrics_table
|
|
@@ -248,7 +447,6 @@ class PostgresDb(BaseDb):
|
|
|
248
447
|
self.eval_table = self._get_or_create_table(
|
|
249
448
|
table_name=self.eval_table_name,
|
|
250
449
|
table_type="evals",
|
|
251
|
-
db_schema=self.db_schema,
|
|
252
450
|
create_table_if_not_found=create_table_if_not_found,
|
|
253
451
|
)
|
|
254
452
|
return self.eval_table
|
|
@@ -257,7 +455,6 @@ class PostgresDb(BaseDb):
|
|
|
257
455
|
self.knowledge_table = self._get_or_create_table(
|
|
258
456
|
table_name=self.knowledge_table_name,
|
|
259
457
|
table_type="knowledge",
|
|
260
|
-
db_schema=self.db_schema,
|
|
261
458
|
create_table_if_not_found=create_table_if_not_found,
|
|
262
459
|
)
|
|
263
460
|
return self.knowledge_table
|
|
@@ -266,15 +463,73 @@ class PostgresDb(BaseDb):
|
|
|
266
463
|
self.culture_table = self._get_or_create_table(
|
|
267
464
|
table_name=self.culture_table_name,
|
|
268
465
|
table_type="culture",
|
|
269
|
-
db_schema=self.db_schema,
|
|
270
466
|
create_table_if_not_found=create_table_if_not_found,
|
|
271
467
|
)
|
|
272
468
|
return self.culture_table
|
|
273
469
|
|
|
470
|
+
if table_type == "versions":
|
|
471
|
+
self.versions_table = self._get_or_create_table(
|
|
472
|
+
table_name=self.versions_table_name,
|
|
473
|
+
table_type="versions",
|
|
474
|
+
create_table_if_not_found=create_table_if_not_found,
|
|
475
|
+
)
|
|
476
|
+
return self.versions_table
|
|
477
|
+
|
|
478
|
+
if table_type == "traces":
|
|
479
|
+
self.traces_table = self._get_or_create_table(
|
|
480
|
+
table_name=self.trace_table_name,
|
|
481
|
+
table_type="traces",
|
|
482
|
+
create_table_if_not_found=create_table_if_not_found,
|
|
483
|
+
)
|
|
484
|
+
return self.traces_table
|
|
485
|
+
|
|
486
|
+
if table_type == "spans":
|
|
487
|
+
# Ensure traces table exists first (spans has FK to traces)
|
|
488
|
+
if create_table_if_not_found:
|
|
489
|
+
self._get_table(table_type="traces", create_table_if_not_found=True)
|
|
490
|
+
|
|
491
|
+
self.spans_table = self._get_or_create_table(
|
|
492
|
+
table_name=self.span_table_name,
|
|
493
|
+
table_type="spans",
|
|
494
|
+
create_table_if_not_found=create_table_if_not_found,
|
|
495
|
+
)
|
|
496
|
+
return self.spans_table
|
|
497
|
+
|
|
498
|
+
if table_type == "components":
|
|
499
|
+
self.component_table = self._get_or_create_table(
|
|
500
|
+
table_name=self.components_table_name,
|
|
501
|
+
table_type="components",
|
|
502
|
+
create_table_if_not_found=create_table_if_not_found,
|
|
503
|
+
)
|
|
504
|
+
return self.component_table
|
|
505
|
+
|
|
506
|
+
if table_type == "component_configs":
|
|
507
|
+
self.component_configs_table = self._get_or_create_table(
|
|
508
|
+
table_name=self.component_configs_table_name,
|
|
509
|
+
table_type="component_configs",
|
|
510
|
+
create_table_if_not_found=create_table_if_not_found,
|
|
511
|
+
)
|
|
512
|
+
return self.component_configs_table
|
|
513
|
+
|
|
514
|
+
if table_type == "component_links":
|
|
515
|
+
self.component_links_table = self._get_or_create_table(
|
|
516
|
+
table_name=self.component_links_table_name,
|
|
517
|
+
table_type="component_links",
|
|
518
|
+
create_table_if_not_found=create_table_if_not_found,
|
|
519
|
+
)
|
|
520
|
+
return self.component_links_table
|
|
521
|
+
if table_type == "learnings":
|
|
522
|
+
self.learnings_table = self._get_or_create_table(
|
|
523
|
+
table_name=self.learnings_table_name,
|
|
524
|
+
table_type="learnings",
|
|
525
|
+
create_table_if_not_found=create_table_if_not_found,
|
|
526
|
+
)
|
|
527
|
+
return self.learnings_table
|
|
528
|
+
|
|
274
529
|
raise ValueError(f"Unknown table type: {table_type}")
|
|
275
530
|
|
|
276
531
|
def _get_or_create_table(
|
|
277
|
-
self, table_name: str, table_type: str,
|
|
532
|
+
self, table_name: str, table_type: str, create_table_if_not_found: Optional[bool] = False
|
|
278
533
|
) -> Optional[Table]:
|
|
279
534
|
"""
|
|
280
535
|
Check if the table exists and is valid, else create it.
|
|
@@ -282,39 +537,72 @@ class PostgresDb(BaseDb):
|
|
|
282
537
|
Args:
|
|
283
538
|
table_name (str): Name of the table to get or create
|
|
284
539
|
table_type (str): Type of table (used to get schema definition)
|
|
285
|
-
db_schema (str): Database schema name
|
|
286
540
|
|
|
287
541
|
Returns:
|
|
288
542
|
Optional[Table]: SQLAlchemy Table object representing the schema.
|
|
289
543
|
"""
|
|
290
544
|
|
|
291
545
|
with self.Session() as sess, sess.begin():
|
|
292
|
-
table_is_available = is_table_available(session=sess, table_name=table_name, db_schema=db_schema)
|
|
546
|
+
table_is_available = is_table_available(session=sess, table_name=table_name, db_schema=self.db_schema)
|
|
293
547
|
|
|
294
548
|
if not table_is_available:
|
|
295
549
|
if not create_table_if_not_found:
|
|
296
550
|
return None
|
|
297
|
-
|
|
298
|
-
return self._create_table(table_name=table_name, table_type=table_type, db_schema=db_schema)
|
|
551
|
+
return self._create_table(table_name=table_name, table_type=table_type)
|
|
299
552
|
|
|
300
553
|
if not is_valid_table(
|
|
301
554
|
db_engine=self.db_engine,
|
|
302
555
|
table_name=table_name,
|
|
303
556
|
table_type=table_type,
|
|
304
|
-
db_schema=db_schema,
|
|
557
|
+
db_schema=self.db_schema,
|
|
305
558
|
):
|
|
306
|
-
raise ValueError(f"Table {db_schema}.{table_name} has an invalid schema")
|
|
559
|
+
raise ValueError(f"Table {self.db_schema}.{table_name} has an invalid schema")
|
|
307
560
|
|
|
308
561
|
try:
|
|
309
|
-
table = Table(table_name, self.metadata, schema=db_schema, autoload_with=self.db_engine)
|
|
562
|
+
table = Table(table_name, self.metadata, schema=self.db_schema, autoload_with=self.db_engine)
|
|
310
563
|
return table
|
|
311
564
|
|
|
312
565
|
except Exception as e:
|
|
313
|
-
log_error(f"Error loading existing table {db_schema}.{table_name}: {e}")
|
|
566
|
+
log_error(f"Error loading existing table {self.db_schema}.{table_name}: {e}")
|
|
314
567
|
raise
|
|
315
568
|
|
|
316
|
-
|
|
569
|
+
def get_latest_schema_version(self, table_name: str):
|
|
570
|
+
"""Get the latest version of the database schema."""
|
|
571
|
+
table = self._get_table(table_type="versions", create_table_if_not_found=True)
|
|
572
|
+
if table is None:
|
|
573
|
+
return "2.0.0"
|
|
574
|
+
with self.Session() as sess:
|
|
575
|
+
stmt = select(table)
|
|
576
|
+
# Latest version for the given table
|
|
577
|
+
stmt = stmt.where(table.c.table_name == table_name)
|
|
578
|
+
stmt = stmt.order_by(table.c.version.desc()).limit(1)
|
|
579
|
+
result = sess.execute(stmt).fetchone()
|
|
580
|
+
if result is None:
|
|
581
|
+
return "2.0.0"
|
|
582
|
+
version_dict = dict(result._mapping)
|
|
583
|
+
return version_dict.get("version") or "2.0.0"
|
|
584
|
+
|
|
585
|
+
def upsert_schema_version(self, table_name: str, version: str) -> None:
|
|
586
|
+
"""Upsert the schema version into the database."""
|
|
587
|
+
table = self._get_table(table_type="versions", create_table_if_not_found=True)
|
|
588
|
+
if table is None:
|
|
589
|
+
return
|
|
590
|
+
current_datetime = datetime.now().isoformat()
|
|
591
|
+
with self.Session() as sess, sess.begin():
|
|
592
|
+
stmt = postgresql.insert(table).values(
|
|
593
|
+
table_name=table_name,
|
|
594
|
+
version=version,
|
|
595
|
+
created_at=current_datetime, # Store as ISO format string
|
|
596
|
+
updated_at=current_datetime,
|
|
597
|
+
)
|
|
598
|
+
# Update version if table_name already exists
|
|
599
|
+
stmt = stmt.on_conflict_do_update(
|
|
600
|
+
index_elements=["table_name"],
|
|
601
|
+
set_=dict(version=version, updated_at=current_datetime),
|
|
602
|
+
)
|
|
603
|
+
sess.execute(stmt)
|
|
317
604
|
|
|
605
|
+
# -- Session methods --
|
|
318
606
|
def delete_session(self, session_id: str) -> bool:
|
|
319
607
|
"""
|
|
320
608
|
Delete a session from the database.
|
|
@@ -408,6 +696,11 @@ class PostgresDb(BaseDb):
|
|
|
408
696
|
|
|
409
697
|
if user_id is not None:
|
|
410
698
|
stmt = stmt.where(table.c.user_id == user_id)
|
|
699
|
+
|
|
700
|
+
# Filter by session_type to ensure we get the correct session type
|
|
701
|
+
session_type_value = session_type.value if isinstance(session_type, SessionType) else session_type
|
|
702
|
+
stmt = stmt.where(table.c.session_type == session_type_value)
|
|
703
|
+
|
|
411
704
|
result = sess.execute(stmt).fetchone()
|
|
412
705
|
if result is None:
|
|
413
706
|
return None
|
|
@@ -445,12 +738,12 @@ class PostgresDb(BaseDb):
|
|
|
445
738
|
deserialize: Optional[bool] = True,
|
|
446
739
|
) -> Union[List[Session], Tuple[List[Dict[str, Any]], int]]:
|
|
447
740
|
"""
|
|
448
|
-
Get all sessions in the given table. Can filter by user_id and
|
|
741
|
+
Get all sessions in the given table. Can filter by user_id and component_id.
|
|
449
742
|
|
|
450
743
|
Args:
|
|
451
744
|
session_type (Optional[SessionType]): The type of session to get.
|
|
452
745
|
user_id (Optional[str]): The ID of the user to filter by.
|
|
453
|
-
|
|
746
|
+
component_id (Optional[str]): The ID of the agent / workflow to filter by.
|
|
454
747
|
start_timestamp (Optional[int]): The start timestamp to filter by.
|
|
455
748
|
end_timestamp (Optional[int]): The end timestamp to filter by.
|
|
456
749
|
session_name (Optional[str]): The name of the session to filter by.
|
|
@@ -492,9 +785,7 @@ class PostgresDb(BaseDb):
|
|
|
492
785
|
stmt = stmt.where(table.c.created_at <= end_timestamp)
|
|
493
786
|
if session_name is not None:
|
|
494
787
|
stmt = stmt.where(
|
|
495
|
-
func.coalesce(
|
|
496
|
-
f"%{session_name}%"
|
|
497
|
-
)
|
|
788
|
+
func.coalesce(table.c.session_data["session_name"].astext, "").ilike(f"%{session_name}%")
|
|
498
789
|
)
|
|
499
790
|
if session_type is not None:
|
|
500
791
|
session_type_value = session_type.value if isinstance(session_type, SessionType) else session_type
|
|
@@ -559,6 +850,8 @@ class PostgresDb(BaseDb):
|
|
|
559
850
|
return None
|
|
560
851
|
|
|
561
852
|
with self.Session() as sess, sess.begin():
|
|
853
|
+
# Sanitize session_name to remove null bytes
|
|
854
|
+
sanitized_session_name = sanitize_postgres_string(session_name)
|
|
562
855
|
stmt = (
|
|
563
856
|
update(table)
|
|
564
857
|
.where(table.c.session_id == session_id)
|
|
@@ -568,7 +861,7 @@ class PostgresDb(BaseDb):
|
|
|
568
861
|
func.jsonb_set(
|
|
569
862
|
func.cast(table.c.session_data, postgresql.JSONB),
|
|
570
863
|
text("'{session_name}'"),
|
|
571
|
-
func.to_jsonb(
|
|
864
|
+
func.to_jsonb(sanitized_session_name),
|
|
572
865
|
),
|
|
573
866
|
postgresql.JSON,
|
|
574
867
|
)
|
|
@@ -624,6 +917,21 @@ class PostgresDb(BaseDb):
|
|
|
624
917
|
return None
|
|
625
918
|
|
|
626
919
|
session_dict = session.to_dict()
|
|
920
|
+
# Sanitize JSON/dict fields to remove null bytes from nested strings
|
|
921
|
+
if session_dict.get("agent_data"):
|
|
922
|
+
session_dict["agent_data"] = sanitize_postgres_strings(session_dict["agent_data"])
|
|
923
|
+
if session_dict.get("team_data"):
|
|
924
|
+
session_dict["team_data"] = sanitize_postgres_strings(session_dict["team_data"])
|
|
925
|
+
if session_dict.get("workflow_data"):
|
|
926
|
+
session_dict["workflow_data"] = sanitize_postgres_strings(session_dict["workflow_data"])
|
|
927
|
+
if session_dict.get("session_data"):
|
|
928
|
+
session_dict["session_data"] = sanitize_postgres_strings(session_dict["session_data"])
|
|
929
|
+
if session_dict.get("summary"):
|
|
930
|
+
session_dict["summary"] = sanitize_postgres_strings(session_dict["summary"])
|
|
931
|
+
if session_dict.get("metadata"):
|
|
932
|
+
session_dict["metadata"] = sanitize_postgres_strings(session_dict["metadata"])
|
|
933
|
+
if session_dict.get("runs"):
|
|
934
|
+
session_dict["runs"] = sanitize_postgres_strings(session_dict["runs"])
|
|
627
935
|
|
|
628
936
|
if isinstance(session, AgentSession):
|
|
629
937
|
with self.Session() as sess, sess.begin():
|
|
@@ -777,6 +1085,18 @@ class PostgresDb(BaseDb):
|
|
|
777
1085
|
session_records = []
|
|
778
1086
|
for agent_session in agent_sessions:
|
|
779
1087
|
session_dict = agent_session.to_dict()
|
|
1088
|
+
# Sanitize JSON/dict fields to remove null bytes from nested strings
|
|
1089
|
+
if session_dict.get("agent_data"):
|
|
1090
|
+
session_dict["agent_data"] = sanitize_postgres_strings(session_dict["agent_data"])
|
|
1091
|
+
if session_dict.get("session_data"):
|
|
1092
|
+
session_dict["session_data"] = sanitize_postgres_strings(session_dict["session_data"])
|
|
1093
|
+
if session_dict.get("summary"):
|
|
1094
|
+
session_dict["summary"] = sanitize_postgres_strings(session_dict["summary"])
|
|
1095
|
+
if session_dict.get("metadata"):
|
|
1096
|
+
session_dict["metadata"] = sanitize_postgres_strings(session_dict["metadata"])
|
|
1097
|
+
if session_dict.get("runs"):
|
|
1098
|
+
session_dict["runs"] = sanitize_postgres_strings(session_dict["runs"])
|
|
1099
|
+
|
|
780
1100
|
# Use preserved updated_at if flag is set (even if None), otherwise use current time
|
|
781
1101
|
updated_at = session_dict.get("updated_at") if preserve_updated_at else int(time.time())
|
|
782
1102
|
session_records.append(
|
|
@@ -822,6 +1142,18 @@ class PostgresDb(BaseDb):
|
|
|
822
1142
|
session_records = []
|
|
823
1143
|
for team_session in team_sessions:
|
|
824
1144
|
session_dict = team_session.to_dict()
|
|
1145
|
+
# Sanitize JSON/dict fields to remove null bytes from nested strings
|
|
1146
|
+
if session_dict.get("team_data"):
|
|
1147
|
+
session_dict["team_data"] = sanitize_postgres_strings(session_dict["team_data"])
|
|
1148
|
+
if session_dict.get("session_data"):
|
|
1149
|
+
session_dict["session_data"] = sanitize_postgres_strings(session_dict["session_data"])
|
|
1150
|
+
if session_dict.get("summary"):
|
|
1151
|
+
session_dict["summary"] = sanitize_postgres_strings(session_dict["summary"])
|
|
1152
|
+
if session_dict.get("metadata"):
|
|
1153
|
+
session_dict["metadata"] = sanitize_postgres_strings(session_dict["metadata"])
|
|
1154
|
+
if session_dict.get("runs"):
|
|
1155
|
+
session_dict["runs"] = sanitize_postgres_strings(session_dict["runs"])
|
|
1156
|
+
|
|
825
1157
|
# Use preserved updated_at if flag is set (even if None), otherwise use current time
|
|
826
1158
|
updated_at = session_dict.get("updated_at") if preserve_updated_at else int(time.time())
|
|
827
1159
|
session_records.append(
|
|
@@ -867,6 +1199,18 @@ class PostgresDb(BaseDb):
|
|
|
867
1199
|
session_records = []
|
|
868
1200
|
for workflow_session in workflow_sessions:
|
|
869
1201
|
session_dict = workflow_session.to_dict()
|
|
1202
|
+
# Sanitize JSON/dict fields to remove null bytes from nested strings
|
|
1203
|
+
if session_dict.get("workflow_data"):
|
|
1204
|
+
session_dict["workflow_data"] = sanitize_postgres_strings(session_dict["workflow_data"])
|
|
1205
|
+
if session_dict.get("session_data"):
|
|
1206
|
+
session_dict["session_data"] = sanitize_postgres_strings(session_dict["session_data"])
|
|
1207
|
+
if session_dict.get("summary"):
|
|
1208
|
+
session_dict["summary"] = sanitize_postgres_strings(session_dict["summary"])
|
|
1209
|
+
if session_dict.get("metadata"):
|
|
1210
|
+
session_dict["metadata"] = sanitize_postgres_strings(session_dict["metadata"])
|
|
1211
|
+
if session_dict.get("runs"):
|
|
1212
|
+
session_dict["runs"] = sanitize_postgres_strings(session_dict["runs"])
|
|
1213
|
+
|
|
870
1214
|
# Use preserved updated_at if flag is set (even if None), otherwise use current time
|
|
871
1215
|
updated_at = session_dict.get("updated_at") if preserve_updated_at else int(time.time())
|
|
872
1216
|
session_records.append(
|
|
@@ -994,11 +1338,35 @@ class PostgresDb(BaseDb):
|
|
|
994
1338
|
return []
|
|
995
1339
|
|
|
996
1340
|
with self.Session() as sess, sess.begin():
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1341
|
+
# Filter out NULL topics and ensure topics is an array before extracting elements
|
|
1342
|
+
# jsonb_typeof returns 'array' for JSONB arrays
|
|
1343
|
+
conditions = [
|
|
1344
|
+
table.c.topics.is_not(None),
|
|
1345
|
+
func.jsonb_typeof(table.c.topics) == "array",
|
|
1346
|
+
]
|
|
1000
1347
|
|
|
1001
|
-
|
|
1348
|
+
try:
|
|
1349
|
+
# jsonb_array_elements_text is a set-returning function that must be used with select_from
|
|
1350
|
+
stmt = select(func.jsonb_array_elements_text(table.c.topics).label("topic"))
|
|
1351
|
+
stmt = stmt.select_from(table)
|
|
1352
|
+
stmt = stmt.where(and_(*conditions))
|
|
1353
|
+
result = sess.execute(stmt).fetchall()
|
|
1354
|
+
except ProgrammingError:
|
|
1355
|
+
# Retrying with json_array_elements_text. This works in older versions,
|
|
1356
|
+
# where the topics column was of type JSON instead of JSONB
|
|
1357
|
+
# For JSON (not JSONB), we use json_typeof
|
|
1358
|
+
json_conditions = [
|
|
1359
|
+
table.c.topics.is_not(None),
|
|
1360
|
+
func.json_typeof(table.c.topics) == "array",
|
|
1361
|
+
]
|
|
1362
|
+
stmt = select(func.json_array_elements_text(table.c.topics).label("topic"))
|
|
1363
|
+
stmt = stmt.select_from(table)
|
|
1364
|
+
stmt = stmt.where(and_(*json_conditions))
|
|
1365
|
+
result = sess.execute(stmt).fetchall()
|
|
1366
|
+
|
|
1367
|
+
# Extract topics from records - each record is a Row with a 'topic' attribute
|
|
1368
|
+
topics = [record.topic for record in result if record.topic is not None]
|
|
1369
|
+
return list(set(topics))
|
|
1002
1370
|
|
|
1003
1371
|
except Exception as e:
|
|
1004
1372
|
log_error(f"Exception reading from memory table: {e}")
|
|
@@ -1149,13 +1517,14 @@ class PostgresDb(BaseDb):
|
|
|
1149
1517
|
raise e
|
|
1150
1518
|
|
|
1151
1519
|
def get_user_memory_stats(
|
|
1152
|
-
self, limit: Optional[int] = None, page: Optional[int] = None
|
|
1520
|
+
self, limit: Optional[int] = None, page: Optional[int] = None, user_id: Optional[str] = None
|
|
1153
1521
|
) -> Tuple[List[Dict[str, Any]], int]:
|
|
1154
1522
|
"""Get user memories stats.
|
|
1155
1523
|
|
|
1156
1524
|
Args:
|
|
1157
1525
|
limit (Optional[int]): The maximum number of user stats to return.
|
|
1158
1526
|
page (Optional[int]): The page number.
|
|
1527
|
+
user_id (Optional[str]): User ID for filtering.
|
|
1159
1528
|
|
|
1160
1529
|
Returns:
|
|
1161
1530
|
Tuple[List[Dict[str, Any]], int]: A list of dictionaries containing user stats and total count.
|
|
@@ -1178,16 +1547,17 @@ class PostgresDb(BaseDb):
|
|
|
1178
1547
|
return [], 0
|
|
1179
1548
|
|
|
1180
1549
|
with self.Session() as sess, sess.begin():
|
|
1181
|
-
stmt = (
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
func.max(table.c.updated_at).label("last_memory_updated_at"),
|
|
1186
|
-
)
|
|
1187
|
-
.where(table.c.user_id.is_not(None))
|
|
1188
|
-
.group_by(table.c.user_id)
|
|
1189
|
-
.order_by(func.max(table.c.updated_at).desc())
|
|
1550
|
+
stmt = select(
|
|
1551
|
+
table.c.user_id,
|
|
1552
|
+
func.count(table.c.memory_id).label("total_memories"),
|
|
1553
|
+
func.max(table.c.updated_at).label("last_memory_updated_at"),
|
|
1190
1554
|
)
|
|
1555
|
+
if user_id is not None:
|
|
1556
|
+
stmt = stmt.where(table.c.user_id == user_id)
|
|
1557
|
+
else:
|
|
1558
|
+
stmt = stmt.where(table.c.user_id.is_not(None))
|
|
1559
|
+
stmt = stmt.group_by(table.c.user_id)
|
|
1560
|
+
stmt = stmt.order_by(func.max(table.c.updated_at).desc())
|
|
1191
1561
|
|
|
1192
1562
|
count_stmt = select(func.count()).select_from(stmt.alias())
|
|
1193
1563
|
total_count = sess.execute(count_stmt).scalar()
|
|
@@ -1237,29 +1607,42 @@ class PostgresDb(BaseDb):
|
|
|
1237
1607
|
if table is None:
|
|
1238
1608
|
return None
|
|
1239
1609
|
|
|
1610
|
+
# Sanitize string fields to remove null bytes (PostgreSQL doesn't allow them)
|
|
1611
|
+
sanitized_input = sanitize_postgres_string(memory.input)
|
|
1612
|
+
sanitized_feedback = sanitize_postgres_string(memory.feedback)
|
|
1613
|
+
|
|
1240
1614
|
with self.Session() as sess, sess.begin():
|
|
1241
1615
|
if memory.memory_id is None:
|
|
1242
1616
|
memory.memory_id = str(uuid4())
|
|
1243
1617
|
|
|
1618
|
+
current_time = int(time.time())
|
|
1619
|
+
|
|
1244
1620
|
stmt = postgresql.insert(table).values(
|
|
1245
1621
|
memory_id=memory.memory_id,
|
|
1246
1622
|
memory=memory.memory,
|
|
1247
|
-
input=
|
|
1623
|
+
input=sanitized_input,
|
|
1248
1624
|
user_id=memory.user_id,
|
|
1249
1625
|
agent_id=memory.agent_id,
|
|
1250
1626
|
team_id=memory.team_id,
|
|
1251
1627
|
topics=memory.topics,
|
|
1252
|
-
|
|
1628
|
+
feedback=sanitized_feedback,
|
|
1629
|
+
created_at=memory.created_at,
|
|
1630
|
+
updated_at=memory.updated_at
|
|
1631
|
+
if memory.updated_at is not None
|
|
1632
|
+
else (memory.created_at if memory.created_at is not None else current_time),
|
|
1253
1633
|
)
|
|
1254
1634
|
stmt = stmt.on_conflict_do_update( # type: ignore
|
|
1255
1635
|
index_elements=["memory_id"],
|
|
1256
1636
|
set_=dict(
|
|
1257
1637
|
memory=memory.memory,
|
|
1258
1638
|
topics=memory.topics,
|
|
1259
|
-
input=
|
|
1639
|
+
input=sanitized_input,
|
|
1260
1640
|
agent_id=memory.agent_id,
|
|
1261
1641
|
team_id=memory.team_id,
|
|
1262
|
-
|
|
1642
|
+
feedback=sanitized_feedback,
|
|
1643
|
+
updated_at=current_time,
|
|
1644
|
+
# Preserve created_at on update - don't overwrite existing value
|
|
1645
|
+
created_at=table.c.created_at,
|
|
1263
1646
|
),
|
|
1264
1647
|
).returning(table)
|
|
1265
1648
|
|
|
@@ -1313,15 +1696,22 @@ class PostgresDb(BaseDb):
|
|
|
1313
1696
|
|
|
1314
1697
|
# Use preserved updated_at if flag is set (even if None), otherwise use current time
|
|
1315
1698
|
updated_at = memory.updated_at if preserve_updated_at else current_time
|
|
1699
|
+
|
|
1700
|
+
# Sanitize string fields to remove null bytes (PostgreSQL doesn't allow them)
|
|
1701
|
+
sanitized_input = sanitize_postgres_string(memory.input)
|
|
1702
|
+
sanitized_feedback = sanitize_postgres_string(memory.feedback)
|
|
1703
|
+
|
|
1316
1704
|
memory_records.append(
|
|
1317
1705
|
{
|
|
1318
1706
|
"memory_id": memory.memory_id,
|
|
1319
1707
|
"memory": memory.memory,
|
|
1320
|
-
"input":
|
|
1708
|
+
"input": sanitized_input,
|
|
1321
1709
|
"user_id": memory.user_id,
|
|
1322
1710
|
"agent_id": memory.agent_id,
|
|
1323
1711
|
"team_id": memory.team_id,
|
|
1324
1712
|
"topics": memory.topics,
|
|
1713
|
+
"feedback": sanitized_feedback,
|
|
1714
|
+
"created_at": memory.created_at,
|
|
1325
1715
|
"updated_at": updated_at,
|
|
1326
1716
|
}
|
|
1327
1717
|
)
|
|
@@ -1333,7 +1723,7 @@ class PostgresDb(BaseDb):
|
|
|
1333
1723
|
update_columns = {
|
|
1334
1724
|
col.name: insert_stmt.excluded[col.name]
|
|
1335
1725
|
for col in table.columns
|
|
1336
|
-
if col.name not in ["memory_id"] # Don't update primary key
|
|
1726
|
+
if col.name not in ["memory_id", "created_at"] # Don't update primary key or created_at
|
|
1337
1727
|
}
|
|
1338
1728
|
stmt = insert_stmt.on_conflict_do_update(index_elements=["memory_id"], set_=update_columns).returning(
|
|
1339
1729
|
table
|
|
@@ -1626,8 +2016,7 @@ class PostgresDb(BaseDb):
|
|
|
1626
2016
|
stmt = select(table)
|
|
1627
2017
|
|
|
1628
2018
|
# Apply sorting
|
|
1629
|
-
|
|
1630
|
-
stmt = stmt.order_by(getattr(table.c, sort_by) * (1 if sort_order == "asc" else -1))
|
|
2019
|
+
stmt = apply_sorting(stmt, table, sort_by, sort_order)
|
|
1631
2020
|
|
|
1632
2021
|
# Get total count before applying limit and pagination
|
|
1633
2022
|
count_stmt = select(func.count()).select_from(stmt.alias())
|
|
@@ -1686,10 +2075,19 @@ class PostgresDb(BaseDb):
|
|
|
1686
2075
|
}
|
|
1687
2076
|
|
|
1688
2077
|
# Build insert and update data only for fields that exist in the table
|
|
2078
|
+
# String fields that need sanitization
|
|
2079
|
+
string_fields = {"name", "description", "type", "status", "status_message", "external_id", "linked_to"}
|
|
2080
|
+
|
|
1689
2081
|
for model_field, table_column in field_mapping.items():
|
|
1690
2082
|
if table_column in table_columns:
|
|
1691
2083
|
value = getattr(knowledge_row, model_field, None)
|
|
1692
2084
|
if value is not None:
|
|
2085
|
+
# Sanitize string fields to remove null bytes
|
|
2086
|
+
if table_column in string_fields and isinstance(value, str):
|
|
2087
|
+
value = sanitize_postgres_string(value)
|
|
2088
|
+
# Sanitize metadata dict if present
|
|
2089
|
+
elif table_column == "metadata" and isinstance(value, dict):
|
|
2090
|
+
value = sanitize_postgres_strings(value)
|
|
1693
2091
|
insert_data[table_column] = value
|
|
1694
2092
|
# Don't include ID in update_fields since it's the primary key
|
|
1695
2093
|
if table_column != "id":
|
|
@@ -1744,8 +2142,22 @@ class PostgresDb(BaseDb):
|
|
|
1744
2142
|
|
|
1745
2143
|
with self.Session() as sess, sess.begin():
|
|
1746
2144
|
current_time = int(time.time())
|
|
2145
|
+
eval_data = eval_run.model_dump()
|
|
2146
|
+
# Sanitize string fields in eval_run
|
|
2147
|
+
if eval_data.get("name"):
|
|
2148
|
+
eval_data["name"] = sanitize_postgres_string(eval_data["name"])
|
|
2149
|
+
if eval_data.get("evaluated_component_name"):
|
|
2150
|
+
eval_data["evaluated_component_name"] = sanitize_postgres_string(
|
|
2151
|
+
eval_data["evaluated_component_name"]
|
|
2152
|
+
)
|
|
2153
|
+
# Sanitize nested dicts/JSON fields
|
|
2154
|
+
if eval_data.get("eval_data"):
|
|
2155
|
+
eval_data["eval_data"] = sanitize_postgres_strings(eval_data["eval_data"])
|
|
2156
|
+
if eval_data.get("eval_input"):
|
|
2157
|
+
eval_data["eval_input"] = sanitize_postgres_strings(eval_data["eval_input"])
|
|
2158
|
+
|
|
1747
2159
|
stmt = postgresql.insert(table).values(
|
|
1748
|
-
{"created_at": current_time, "updated_at": current_time, **
|
|
2160
|
+
{"created_at": current_time, "updated_at": current_time, **eval_data}
|
|
1749
2161
|
)
|
|
1750
2162
|
sess.execute(stmt)
|
|
1751
2163
|
|
|
@@ -1959,8 +2371,12 @@ class PostgresDb(BaseDb):
|
|
|
1959
2371
|
return None
|
|
1960
2372
|
|
|
1961
2373
|
with self.Session() as sess, sess.begin():
|
|
2374
|
+
# Sanitize string field to remove null bytes
|
|
2375
|
+
sanitized_name = sanitize_postgres_string(name)
|
|
1962
2376
|
stmt = (
|
|
1963
|
-
table.update()
|
|
2377
|
+
table.update()
|
|
2378
|
+
.where(table.c.run_id == eval_run_id)
|
|
2379
|
+
.values(name=sanitized_name, updated_at=int(time.time()))
|
|
1964
2380
|
)
|
|
1965
2381
|
sess.execute(stmt)
|
|
1966
2382
|
|
|
@@ -2157,15 +2573,25 @@ class PostgresDb(BaseDb):
|
|
|
2157
2573
|
|
|
2158
2574
|
# Serialize content, categories, and notes into a JSON dict for DB storage
|
|
2159
2575
|
content_dict = serialize_cultural_knowledge(cultural_knowledge)
|
|
2576
|
+
# Sanitize content_dict to remove null bytes from nested strings
|
|
2577
|
+
if content_dict:
|
|
2578
|
+
content_dict = cast(Dict[str, Any], sanitize_postgres_strings(content_dict))
|
|
2579
|
+
|
|
2580
|
+
# Sanitize string fields to remove null bytes (PostgreSQL doesn't allow them)
|
|
2581
|
+
sanitized_name = sanitize_postgres_string(cultural_knowledge.name)
|
|
2582
|
+
sanitized_summary = sanitize_postgres_string(cultural_knowledge.summary)
|
|
2583
|
+
sanitized_input = sanitize_postgres_string(cultural_knowledge.input)
|
|
2160
2584
|
|
|
2161
2585
|
with self.Session() as sess, sess.begin():
|
|
2162
2586
|
stmt = postgresql.insert(table).values(
|
|
2163
2587
|
id=cultural_knowledge.id,
|
|
2164
|
-
name=
|
|
2165
|
-
summary=
|
|
2588
|
+
name=sanitized_name,
|
|
2589
|
+
summary=sanitized_summary,
|
|
2166
2590
|
content=content_dict if content_dict else None,
|
|
2167
|
-
metadata=cultural_knowledge.metadata
|
|
2168
|
-
|
|
2591
|
+
metadata=sanitize_postgres_strings(cultural_knowledge.metadata)
|
|
2592
|
+
if cultural_knowledge.metadata
|
|
2593
|
+
else None,
|
|
2594
|
+
input=sanitized_input,
|
|
2169
2595
|
created_at=cultural_knowledge.created_at,
|
|
2170
2596
|
updated_at=int(time.time()),
|
|
2171
2597
|
agent_id=cultural_knowledge.agent_id,
|
|
@@ -2174,11 +2600,13 @@ class PostgresDb(BaseDb):
|
|
|
2174
2600
|
stmt = stmt.on_conflict_do_update( # type: ignore
|
|
2175
2601
|
index_elements=["id"],
|
|
2176
2602
|
set_=dict(
|
|
2177
|
-
name=
|
|
2178
|
-
summary=
|
|
2603
|
+
name=sanitized_name,
|
|
2604
|
+
summary=sanitized_summary,
|
|
2179
2605
|
content=content_dict if content_dict else None,
|
|
2180
|
-
metadata=cultural_knowledge.metadata
|
|
2181
|
-
|
|
2606
|
+
metadata=sanitize_postgres_strings(cultural_knowledge.metadata)
|
|
2607
|
+
if cultural_knowledge.metadata
|
|
2608
|
+
else None,
|
|
2609
|
+
input=sanitized_input,
|
|
2182
2610
|
updated_at=int(time.time()),
|
|
2183
2611
|
agent_id=cultural_knowledge.agent_id,
|
|
2184
2612
|
team_id=cultural_knowledge.team_id,
|
|
@@ -2258,3 +2686,1798 @@ class PostgresDb(BaseDb):
|
|
|
2258
2686
|
for memory in memories:
|
|
2259
2687
|
self.upsert_user_memory(memory)
|
|
2260
2688
|
log_info(f"Migrated {len(memories)} memories to table: {self.memory_table}")
|
|
2689
|
+
|
|
2690
|
+
# --- Traces ---
|
|
2691
|
+
def _get_traces_base_query(self, table: Table, spans_table: Optional[Table] = None):
|
|
2692
|
+
"""Build base query for traces with aggregated span counts.
|
|
2693
|
+
|
|
2694
|
+
Args:
|
|
2695
|
+
table: The traces table.
|
|
2696
|
+
spans_table: The spans table (optional).
|
|
2697
|
+
|
|
2698
|
+
Returns:
|
|
2699
|
+
SQLAlchemy select statement with total_spans and error_count calculated dynamically.
|
|
2700
|
+
"""
|
|
2701
|
+
from sqlalchemy import case, literal
|
|
2702
|
+
|
|
2703
|
+
if spans_table is not None:
|
|
2704
|
+
# JOIN with spans table to calculate total_spans and error_count
|
|
2705
|
+
return (
|
|
2706
|
+
select(
|
|
2707
|
+
table,
|
|
2708
|
+
func.coalesce(func.count(spans_table.c.span_id), 0).label("total_spans"),
|
|
2709
|
+
func.coalesce(func.sum(case((spans_table.c.status_code == "ERROR", 1), else_=0)), 0).label(
|
|
2710
|
+
"error_count"
|
|
2711
|
+
),
|
|
2712
|
+
)
|
|
2713
|
+
.select_from(table.outerjoin(spans_table, table.c.trace_id == spans_table.c.trace_id))
|
|
2714
|
+
.group_by(table.c.trace_id)
|
|
2715
|
+
)
|
|
2716
|
+
else:
|
|
2717
|
+
# Fallback if spans table doesn't exist
|
|
2718
|
+
return select(table, literal(0).label("total_spans"), literal(0).label("error_count"))
|
|
2719
|
+
|
|
2720
|
+
def _get_trace_component_level_expr(self, workflow_id_col, team_id_col, agent_id_col, name_col):
|
|
2721
|
+
"""Build a SQL CASE expression that returns the component level for a trace.
|
|
2722
|
+
|
|
2723
|
+
Component levels (higher = more important):
|
|
2724
|
+
- 3: Workflow root (.run or .arun with workflow_id)
|
|
2725
|
+
- 2: Team root (.run or .arun with team_id)
|
|
2726
|
+
- 1: Agent root (.run or .arun with agent_id)
|
|
2727
|
+
- 0: Child span (not a root)
|
|
2728
|
+
|
|
2729
|
+
Args:
|
|
2730
|
+
workflow_id_col: SQL column/expression for workflow_id
|
|
2731
|
+
team_id_col: SQL column/expression for team_id
|
|
2732
|
+
agent_id_col: SQL column/expression for agent_id
|
|
2733
|
+
name_col: SQL column/expression for name
|
|
2734
|
+
|
|
2735
|
+
Returns:
|
|
2736
|
+
SQLAlchemy CASE expression returning the component level as an integer.
|
|
2737
|
+
"""
|
|
2738
|
+
is_root_name = or_(name_col.contains(".run"), name_col.contains(".arun"))
|
|
2739
|
+
|
|
2740
|
+
return case(
|
|
2741
|
+
# Workflow root (level 3)
|
|
2742
|
+
(and_(workflow_id_col.isnot(None), is_root_name), 3),
|
|
2743
|
+
# Team root (level 2)
|
|
2744
|
+
(and_(team_id_col.isnot(None), is_root_name), 2),
|
|
2745
|
+
# Agent root (level 1)
|
|
2746
|
+
(and_(agent_id_col.isnot(None), is_root_name), 1),
|
|
2747
|
+
# Child span or unknown (level 0)
|
|
2748
|
+
else_=0,
|
|
2749
|
+
)
|
|
2750
|
+
|
|
2751
|
+
def upsert_trace(self, trace: "Trace") -> None:
|
|
2752
|
+
"""Create or update a single trace record in the database.
|
|
2753
|
+
|
|
2754
|
+
Uses INSERT ... ON CONFLICT DO UPDATE (upsert) to handle concurrent inserts
|
|
2755
|
+
atomically and avoid race conditions.
|
|
2756
|
+
|
|
2757
|
+
Args:
|
|
2758
|
+
trace: The Trace object to store (one per trace_id).
|
|
2759
|
+
"""
|
|
2760
|
+
try:
|
|
2761
|
+
table = self._get_table(table_type="traces", create_table_if_not_found=True)
|
|
2762
|
+
if table is None:
|
|
2763
|
+
return
|
|
2764
|
+
|
|
2765
|
+
trace_dict = trace.to_dict()
|
|
2766
|
+
trace_dict.pop("total_spans", None)
|
|
2767
|
+
trace_dict.pop("error_count", None)
|
|
2768
|
+
# Sanitize string fields and nested JSON structures
|
|
2769
|
+
if trace_dict.get("name"):
|
|
2770
|
+
trace_dict["name"] = sanitize_postgres_string(trace_dict["name"])
|
|
2771
|
+
if trace_dict.get("status"):
|
|
2772
|
+
trace_dict["status"] = sanitize_postgres_string(trace_dict["status"])
|
|
2773
|
+
# Sanitize any nested dict/JSON fields
|
|
2774
|
+
trace_dict = cast(Dict[str, Any], sanitize_postgres_strings(trace_dict))
|
|
2775
|
+
|
|
2776
|
+
with self.Session() as sess, sess.begin():
|
|
2777
|
+
# Use upsert to handle concurrent inserts atomically
|
|
2778
|
+
# On conflict, update fields while preserving existing non-null context values
|
|
2779
|
+
# and keeping the earliest start_time
|
|
2780
|
+
insert_stmt = postgresql.insert(table).values(trace_dict)
|
|
2781
|
+
|
|
2782
|
+
# Build component level expressions for comparing trace priority
|
|
2783
|
+
new_level = self._get_trace_component_level_expr(
|
|
2784
|
+
insert_stmt.excluded.workflow_id,
|
|
2785
|
+
insert_stmt.excluded.team_id,
|
|
2786
|
+
insert_stmt.excluded.agent_id,
|
|
2787
|
+
insert_stmt.excluded.name,
|
|
2788
|
+
)
|
|
2789
|
+
existing_level = self._get_trace_component_level_expr(
|
|
2790
|
+
table.c.workflow_id,
|
|
2791
|
+
table.c.team_id,
|
|
2792
|
+
table.c.agent_id,
|
|
2793
|
+
table.c.name,
|
|
2794
|
+
)
|
|
2795
|
+
|
|
2796
|
+
# Build the ON CONFLICT DO UPDATE clause
|
|
2797
|
+
# Use LEAST for start_time, GREATEST for end_time to capture full trace duration
|
|
2798
|
+
# Use COALESCE to preserve existing non-null context values
|
|
2799
|
+
upsert_stmt = insert_stmt.on_conflict_do_update(
|
|
2800
|
+
index_elements=["trace_id"],
|
|
2801
|
+
set_={
|
|
2802
|
+
"end_time": func.greatest(table.c.end_time, insert_stmt.excluded.end_time),
|
|
2803
|
+
"start_time": func.least(table.c.start_time, insert_stmt.excluded.start_time),
|
|
2804
|
+
"duration_ms": func.extract(
|
|
2805
|
+
"epoch",
|
|
2806
|
+
func.cast(
|
|
2807
|
+
func.greatest(table.c.end_time, insert_stmt.excluded.end_time),
|
|
2808
|
+
TIMESTAMP(timezone=True),
|
|
2809
|
+
)
|
|
2810
|
+
- func.cast(
|
|
2811
|
+
func.least(table.c.start_time, insert_stmt.excluded.start_time),
|
|
2812
|
+
TIMESTAMP(timezone=True),
|
|
2813
|
+
),
|
|
2814
|
+
)
|
|
2815
|
+
* 1000,
|
|
2816
|
+
"status": insert_stmt.excluded.status,
|
|
2817
|
+
# Update name only if new trace is from a higher-level component
|
|
2818
|
+
# Priority: workflow (3) > team (2) > agent (1) > child spans (0)
|
|
2819
|
+
"name": case(
|
|
2820
|
+
(new_level > existing_level, insert_stmt.excluded.name),
|
|
2821
|
+
else_=table.c.name,
|
|
2822
|
+
),
|
|
2823
|
+
# Preserve existing non-null context values using COALESCE
|
|
2824
|
+
"run_id": func.coalesce(insert_stmt.excluded.run_id, table.c.run_id),
|
|
2825
|
+
"session_id": func.coalesce(insert_stmt.excluded.session_id, table.c.session_id),
|
|
2826
|
+
"user_id": func.coalesce(insert_stmt.excluded.user_id, table.c.user_id),
|
|
2827
|
+
"agent_id": func.coalesce(insert_stmt.excluded.agent_id, table.c.agent_id),
|
|
2828
|
+
"team_id": func.coalesce(insert_stmt.excluded.team_id, table.c.team_id),
|
|
2829
|
+
"workflow_id": func.coalesce(insert_stmt.excluded.workflow_id, table.c.workflow_id),
|
|
2830
|
+
},
|
|
2831
|
+
)
|
|
2832
|
+
sess.execute(upsert_stmt)
|
|
2833
|
+
|
|
2834
|
+
except Exception as e:
|
|
2835
|
+
log_error(f"Error creating trace: {e}")
|
|
2836
|
+
# Don't raise - tracing should not break the main application flow
|
|
2837
|
+
|
|
2838
|
+
def get_trace(
|
|
2839
|
+
self,
|
|
2840
|
+
trace_id: Optional[str] = None,
|
|
2841
|
+
run_id: Optional[str] = None,
|
|
2842
|
+
):
|
|
2843
|
+
"""Get a single trace by trace_id or other filters.
|
|
2844
|
+
|
|
2845
|
+
Args:
|
|
2846
|
+
trace_id: The unique trace identifier.
|
|
2847
|
+
run_id: Filter by run ID (returns first match).
|
|
2848
|
+
|
|
2849
|
+
Returns:
|
|
2850
|
+
Optional[Trace]: The trace if found, None otherwise.
|
|
2851
|
+
|
|
2852
|
+
Note:
|
|
2853
|
+
If multiple filters are provided, trace_id takes precedence.
|
|
2854
|
+
For other filters, the most recent trace is returned.
|
|
2855
|
+
"""
|
|
2856
|
+
try:
|
|
2857
|
+
from agno.tracing.schemas import Trace
|
|
2858
|
+
|
|
2859
|
+
table = self._get_table(table_type="traces")
|
|
2860
|
+
if table is None:
|
|
2861
|
+
return None
|
|
2862
|
+
|
|
2863
|
+
# Get spans table for JOIN
|
|
2864
|
+
spans_table = self._get_table(table_type="spans")
|
|
2865
|
+
|
|
2866
|
+
with self.Session() as sess:
|
|
2867
|
+
# Build query with aggregated span counts
|
|
2868
|
+
stmt = self._get_traces_base_query(table, spans_table)
|
|
2869
|
+
|
|
2870
|
+
if trace_id:
|
|
2871
|
+
stmt = stmt.where(table.c.trace_id == trace_id)
|
|
2872
|
+
elif run_id:
|
|
2873
|
+
stmt = stmt.where(table.c.run_id == run_id)
|
|
2874
|
+
else:
|
|
2875
|
+
log_debug("get_trace called without any filter parameters")
|
|
2876
|
+
return None
|
|
2877
|
+
|
|
2878
|
+
# Order by most recent and get first result
|
|
2879
|
+
stmt = stmt.order_by(table.c.start_time.desc()).limit(1)
|
|
2880
|
+
result = sess.execute(stmt).fetchone()
|
|
2881
|
+
|
|
2882
|
+
if result:
|
|
2883
|
+
return Trace.from_dict(dict(result._mapping))
|
|
2884
|
+
return None
|
|
2885
|
+
|
|
2886
|
+
except Exception as e:
|
|
2887
|
+
log_error(f"Error getting trace: {e}")
|
|
2888
|
+
return None
|
|
2889
|
+
|
|
2890
|
+
def get_traces(
|
|
2891
|
+
self,
|
|
2892
|
+
run_id: Optional[str] = None,
|
|
2893
|
+
session_id: Optional[str] = None,
|
|
2894
|
+
user_id: Optional[str] = None,
|
|
2895
|
+
agent_id: Optional[str] = None,
|
|
2896
|
+
team_id: Optional[str] = None,
|
|
2897
|
+
workflow_id: Optional[str] = None,
|
|
2898
|
+
status: Optional[str] = None,
|
|
2899
|
+
start_time: Optional[datetime] = None,
|
|
2900
|
+
end_time: Optional[datetime] = None,
|
|
2901
|
+
limit: Optional[int] = 20,
|
|
2902
|
+
page: Optional[int] = 1,
|
|
2903
|
+
) -> tuple[List, int]:
|
|
2904
|
+
"""Get traces matching the provided filters with pagination.
|
|
2905
|
+
|
|
2906
|
+
Args:
|
|
2907
|
+
run_id: Filter by run ID.
|
|
2908
|
+
session_id: Filter by session ID.
|
|
2909
|
+
user_id: Filter by user ID.
|
|
2910
|
+
agent_id: Filter by agent ID.
|
|
2911
|
+
team_id: Filter by team ID.
|
|
2912
|
+
workflow_id: Filter by workflow ID.
|
|
2913
|
+
status: Filter by status (OK, ERROR, UNSET).
|
|
2914
|
+
start_time: Filter traces starting after this datetime.
|
|
2915
|
+
end_time: Filter traces ending before this datetime.
|
|
2916
|
+
limit: Maximum number of traces to return per page.
|
|
2917
|
+
page: Page number (1-indexed).
|
|
2918
|
+
|
|
2919
|
+
Returns:
|
|
2920
|
+
tuple[List[Trace], int]: Tuple of (list of matching traces, total count).
|
|
2921
|
+
"""
|
|
2922
|
+
try:
|
|
2923
|
+
from agno.tracing.schemas import Trace
|
|
2924
|
+
|
|
2925
|
+
table = self._get_table(table_type="traces")
|
|
2926
|
+
if table is None:
|
|
2927
|
+
log_debug("Traces table not found")
|
|
2928
|
+
return [], 0
|
|
2929
|
+
|
|
2930
|
+
# Get spans table for JOIN
|
|
2931
|
+
spans_table = self._get_table(table_type="spans")
|
|
2932
|
+
|
|
2933
|
+
with self.Session() as sess:
|
|
2934
|
+
# Build base query with aggregated span counts
|
|
2935
|
+
base_stmt = self._get_traces_base_query(table, spans_table)
|
|
2936
|
+
|
|
2937
|
+
# Apply filters
|
|
2938
|
+
if run_id:
|
|
2939
|
+
base_stmt = base_stmt.where(table.c.run_id == run_id)
|
|
2940
|
+
if session_id:
|
|
2941
|
+
base_stmt = base_stmt.where(table.c.session_id == session_id)
|
|
2942
|
+
if user_id:
|
|
2943
|
+
base_stmt = base_stmt.where(table.c.user_id == user_id)
|
|
2944
|
+
if agent_id:
|
|
2945
|
+
base_stmt = base_stmt.where(table.c.agent_id == agent_id)
|
|
2946
|
+
if team_id:
|
|
2947
|
+
base_stmt = base_stmt.where(table.c.team_id == team_id)
|
|
2948
|
+
if workflow_id:
|
|
2949
|
+
base_stmt = base_stmt.where(table.c.workflow_id == workflow_id)
|
|
2950
|
+
if status:
|
|
2951
|
+
base_stmt = base_stmt.where(table.c.status == status)
|
|
2952
|
+
if start_time:
|
|
2953
|
+
# Convert datetime to ISO string for comparison
|
|
2954
|
+
base_stmt = base_stmt.where(table.c.start_time >= start_time.isoformat())
|
|
2955
|
+
if end_time:
|
|
2956
|
+
# Convert datetime to ISO string for comparison
|
|
2957
|
+
base_stmt = base_stmt.where(table.c.end_time <= end_time.isoformat())
|
|
2958
|
+
|
|
2959
|
+
# Get total count
|
|
2960
|
+
count_stmt = select(func.count()).select_from(base_stmt.alias())
|
|
2961
|
+
total_count = sess.execute(count_stmt).scalar() or 0
|
|
2962
|
+
|
|
2963
|
+
# Apply pagination
|
|
2964
|
+
offset = (page - 1) * limit if page and limit else 0
|
|
2965
|
+
paginated_stmt = base_stmt.order_by(table.c.start_time.desc()).limit(limit).offset(offset)
|
|
2966
|
+
|
|
2967
|
+
results = sess.execute(paginated_stmt).fetchall()
|
|
2968
|
+
|
|
2969
|
+
traces = [Trace.from_dict(dict(row._mapping)) for row in results]
|
|
2970
|
+
return traces, total_count
|
|
2971
|
+
|
|
2972
|
+
except Exception as e:
|
|
2973
|
+
log_error(f"Error getting traces: {e}")
|
|
2974
|
+
return [], 0
|
|
2975
|
+
|
|
2976
|
+
def get_trace_stats(
|
|
2977
|
+
self,
|
|
2978
|
+
user_id: Optional[str] = None,
|
|
2979
|
+
agent_id: Optional[str] = None,
|
|
2980
|
+
team_id: Optional[str] = None,
|
|
2981
|
+
workflow_id: Optional[str] = None,
|
|
2982
|
+
start_time: Optional[datetime] = None,
|
|
2983
|
+
end_time: Optional[datetime] = None,
|
|
2984
|
+
limit: Optional[int] = 20,
|
|
2985
|
+
page: Optional[int] = 1,
|
|
2986
|
+
) -> tuple[List[Dict[str, Any]], int]:
|
|
2987
|
+
"""Get trace statistics grouped by session.
|
|
2988
|
+
|
|
2989
|
+
Args:
|
|
2990
|
+
user_id: Filter by user ID.
|
|
2991
|
+
agent_id: Filter by agent ID.
|
|
2992
|
+
team_id: Filter by team ID.
|
|
2993
|
+
workflow_id: Filter by workflow ID.
|
|
2994
|
+
start_time: Filter sessions with traces created after this datetime.
|
|
2995
|
+
end_time: Filter sessions with traces created before this datetime.
|
|
2996
|
+
limit: Maximum number of sessions to return per page.
|
|
2997
|
+
page: Page number (1-indexed).
|
|
2998
|
+
|
|
2999
|
+
Returns:
|
|
3000
|
+
tuple[List[Dict], int]: Tuple of (list of session stats dicts, total count).
|
|
3001
|
+
Each dict contains: session_id, user_id, agent_id, team_id, total_traces,
|
|
3002
|
+
first_trace_at, last_trace_at.
|
|
3003
|
+
"""
|
|
3004
|
+
try:
|
|
3005
|
+
table = self._get_table(table_type="traces")
|
|
3006
|
+
if table is None:
|
|
3007
|
+
log_debug("Traces table not found")
|
|
3008
|
+
return [], 0
|
|
3009
|
+
|
|
3010
|
+
with self.Session() as sess:
|
|
3011
|
+
# Build base query grouped by session_id
|
|
3012
|
+
base_stmt = (
|
|
3013
|
+
select(
|
|
3014
|
+
table.c.session_id,
|
|
3015
|
+
table.c.user_id,
|
|
3016
|
+
table.c.agent_id,
|
|
3017
|
+
table.c.team_id,
|
|
3018
|
+
table.c.workflow_id,
|
|
3019
|
+
func.count(table.c.trace_id).label("total_traces"),
|
|
3020
|
+
func.min(table.c.created_at).label("first_trace_at"),
|
|
3021
|
+
func.max(table.c.created_at).label("last_trace_at"),
|
|
3022
|
+
)
|
|
3023
|
+
.where(table.c.session_id.isnot(None)) # Only sessions with session_id
|
|
3024
|
+
.group_by(
|
|
3025
|
+
table.c.session_id, table.c.user_id, table.c.agent_id, table.c.team_id, table.c.workflow_id
|
|
3026
|
+
)
|
|
3027
|
+
)
|
|
3028
|
+
|
|
3029
|
+
# Apply filters
|
|
3030
|
+
if user_id:
|
|
3031
|
+
base_stmt = base_stmt.where(table.c.user_id == user_id)
|
|
3032
|
+
if workflow_id:
|
|
3033
|
+
base_stmt = base_stmt.where(table.c.workflow_id == workflow_id)
|
|
3034
|
+
if team_id:
|
|
3035
|
+
base_stmt = base_stmt.where(table.c.team_id == team_id)
|
|
3036
|
+
if agent_id:
|
|
3037
|
+
base_stmt = base_stmt.where(table.c.agent_id == agent_id)
|
|
3038
|
+
if start_time:
|
|
3039
|
+
# Convert datetime to ISO string for comparison
|
|
3040
|
+
base_stmt = base_stmt.where(table.c.created_at >= start_time.isoformat())
|
|
3041
|
+
if end_time:
|
|
3042
|
+
# Convert datetime to ISO string for comparison
|
|
3043
|
+
base_stmt = base_stmt.where(table.c.created_at <= end_time.isoformat())
|
|
3044
|
+
|
|
3045
|
+
# Get total count of sessions
|
|
3046
|
+
count_stmt = select(func.count()).select_from(base_stmt.alias())
|
|
3047
|
+
total_count = sess.execute(count_stmt).scalar() or 0
|
|
3048
|
+
|
|
3049
|
+
# Apply pagination and ordering
|
|
3050
|
+
offset = (page - 1) * limit if page and limit else 0
|
|
3051
|
+
paginated_stmt = base_stmt.order_by(func.max(table.c.created_at).desc()).limit(limit).offset(offset)
|
|
3052
|
+
|
|
3053
|
+
results = sess.execute(paginated_stmt).fetchall()
|
|
3054
|
+
|
|
3055
|
+
# Convert to list of dicts with datetime objects
|
|
3056
|
+
stats_list = []
|
|
3057
|
+
for row in results:
|
|
3058
|
+
# Convert ISO strings to datetime objects
|
|
3059
|
+
first_trace_at_str = row.first_trace_at
|
|
3060
|
+
last_trace_at_str = row.last_trace_at
|
|
3061
|
+
|
|
3062
|
+
# Parse ISO format strings to datetime objects
|
|
3063
|
+
first_trace_at = datetime.fromisoformat(first_trace_at_str.replace("Z", "+00:00"))
|
|
3064
|
+
last_trace_at = datetime.fromisoformat(last_trace_at_str.replace("Z", "+00:00"))
|
|
3065
|
+
|
|
3066
|
+
stats_list.append(
|
|
3067
|
+
{
|
|
3068
|
+
"session_id": row.session_id,
|
|
3069
|
+
"user_id": row.user_id,
|
|
3070
|
+
"agent_id": row.agent_id,
|
|
3071
|
+
"team_id": row.team_id,
|
|
3072
|
+
"workflow_id": row.workflow_id,
|
|
3073
|
+
"total_traces": row.total_traces,
|
|
3074
|
+
"first_trace_at": first_trace_at,
|
|
3075
|
+
"last_trace_at": last_trace_at,
|
|
3076
|
+
}
|
|
3077
|
+
)
|
|
3078
|
+
|
|
3079
|
+
return stats_list, total_count
|
|
3080
|
+
|
|
3081
|
+
except Exception as e:
|
|
3082
|
+
log_error(f"Error getting trace stats: {e}")
|
|
3083
|
+
return [], 0
|
|
3084
|
+
|
|
3085
|
+
# --- Spans ---
|
|
3086
|
+
def create_span(self, span: "Span") -> None:
|
|
3087
|
+
"""Create a single span in the database.
|
|
3088
|
+
|
|
3089
|
+
Args:
|
|
3090
|
+
span: The Span object to store.
|
|
3091
|
+
"""
|
|
3092
|
+
try:
|
|
3093
|
+
table = self._get_table(table_type="spans", create_table_if_not_found=True)
|
|
3094
|
+
if table is None:
|
|
3095
|
+
return
|
|
3096
|
+
|
|
3097
|
+
with self.Session() as sess, sess.begin():
|
|
3098
|
+
span_dict = span.to_dict()
|
|
3099
|
+
# Sanitize string fields and nested JSON structures
|
|
3100
|
+
if span_dict.get("name"):
|
|
3101
|
+
span_dict["name"] = sanitize_postgres_string(span_dict["name"])
|
|
3102
|
+
if span_dict.get("status_code"):
|
|
3103
|
+
span_dict["status_code"] = sanitize_postgres_string(span_dict["status_code"])
|
|
3104
|
+
# Sanitize any nested dict/JSON fields
|
|
3105
|
+
span_dict = cast(Dict[str, Any], sanitize_postgres_strings(span_dict))
|
|
3106
|
+
stmt = postgresql.insert(table).values(span_dict)
|
|
3107
|
+
sess.execute(stmt)
|
|
3108
|
+
|
|
3109
|
+
except Exception as e:
|
|
3110
|
+
log_error(f"Error creating span: {e}")
|
|
3111
|
+
|
|
3112
|
+
def create_spans(self, spans: List) -> None:
|
|
3113
|
+
"""Create multiple spans in the database as a batch.
|
|
3114
|
+
|
|
3115
|
+
Args:
|
|
3116
|
+
spans: List of Span objects to store.
|
|
3117
|
+
"""
|
|
3118
|
+
if not spans:
|
|
3119
|
+
return
|
|
3120
|
+
|
|
3121
|
+
try:
|
|
3122
|
+
table = self._get_table(table_type="spans", create_table_if_not_found=True)
|
|
3123
|
+
if table is None:
|
|
3124
|
+
return
|
|
3125
|
+
|
|
3126
|
+
with self.Session() as sess, sess.begin():
|
|
3127
|
+
for span in spans:
|
|
3128
|
+
span_dict = span.to_dict()
|
|
3129
|
+
# Sanitize string fields and nested JSON structures
|
|
3130
|
+
if span_dict.get("name"):
|
|
3131
|
+
span_dict["name"] = sanitize_postgres_string(span_dict["name"])
|
|
3132
|
+
if span_dict.get("status_code"):
|
|
3133
|
+
span_dict["status_code"] = sanitize_postgres_string(span_dict["status_code"])
|
|
3134
|
+
# Sanitize any nested dict/JSON fields
|
|
3135
|
+
span_dict = sanitize_postgres_strings(span_dict)
|
|
3136
|
+
stmt = postgresql.insert(table).values(span_dict)
|
|
3137
|
+
sess.execute(stmt)
|
|
3138
|
+
|
|
3139
|
+
except Exception as e:
|
|
3140
|
+
log_error(f"Error creating spans batch: {e}")
|
|
3141
|
+
|
|
3142
|
+
def get_span(self, span_id: str):
|
|
3143
|
+
"""Get a single span by its span_id.
|
|
3144
|
+
|
|
3145
|
+
Args:
|
|
3146
|
+
span_id: The unique span identifier.
|
|
3147
|
+
|
|
3148
|
+
Returns:
|
|
3149
|
+
Optional[Span]: The span if found, None otherwise.
|
|
3150
|
+
"""
|
|
3151
|
+
try:
|
|
3152
|
+
from agno.tracing.schemas import Span
|
|
3153
|
+
|
|
3154
|
+
table = self._get_table(table_type="spans")
|
|
3155
|
+
if table is None:
|
|
3156
|
+
return None
|
|
3157
|
+
|
|
3158
|
+
with self.Session() as sess:
|
|
3159
|
+
stmt = select(table).where(table.c.span_id == span_id)
|
|
3160
|
+
result = sess.execute(stmt).fetchone()
|
|
3161
|
+
if result:
|
|
3162
|
+
return Span.from_dict(dict(result._mapping))
|
|
3163
|
+
return None
|
|
3164
|
+
|
|
3165
|
+
except Exception as e:
|
|
3166
|
+
log_error(f"Error getting span: {e}")
|
|
3167
|
+
return None
|
|
3168
|
+
|
|
3169
|
+
def get_spans(
|
|
3170
|
+
self,
|
|
3171
|
+
trace_id: Optional[str] = None,
|
|
3172
|
+
parent_span_id: Optional[str] = None,
|
|
3173
|
+
limit: Optional[int] = 1000,
|
|
3174
|
+
) -> List:
|
|
3175
|
+
"""Get spans matching the provided filters.
|
|
3176
|
+
|
|
3177
|
+
Args:
|
|
3178
|
+
trace_id: Filter by trace ID.
|
|
3179
|
+
parent_span_id: Filter by parent span ID.
|
|
3180
|
+
limit: Maximum number of spans to return.
|
|
3181
|
+
|
|
3182
|
+
Returns:
|
|
3183
|
+
List[Span]: List of matching spans.
|
|
3184
|
+
"""
|
|
3185
|
+
try:
|
|
3186
|
+
from agno.tracing.schemas import Span
|
|
3187
|
+
|
|
3188
|
+
table = self._get_table(table_type="spans")
|
|
3189
|
+
if table is None:
|
|
3190
|
+
return []
|
|
3191
|
+
|
|
3192
|
+
with self.Session() as sess:
|
|
3193
|
+
stmt = select(table)
|
|
3194
|
+
|
|
3195
|
+
# Apply filters
|
|
3196
|
+
if trace_id:
|
|
3197
|
+
stmt = stmt.where(table.c.trace_id == trace_id)
|
|
3198
|
+
if parent_span_id:
|
|
3199
|
+
stmt = stmt.where(table.c.parent_span_id == parent_span_id)
|
|
3200
|
+
|
|
3201
|
+
if limit:
|
|
3202
|
+
stmt = stmt.limit(limit)
|
|
3203
|
+
|
|
3204
|
+
results = sess.execute(stmt).fetchall()
|
|
3205
|
+
return [Span.from_dict(dict(row._mapping)) for row in results]
|
|
3206
|
+
|
|
3207
|
+
except Exception as e:
|
|
3208
|
+
log_error(f"Error getting spans: {e}")
|
|
3209
|
+
return []
|
|
3210
|
+
|
|
3211
|
+
# --- Components ---
|
|
3212
|
+
def get_component(
|
|
3213
|
+
self,
|
|
3214
|
+
component_id: str,
|
|
3215
|
+
component_type: Optional[ComponentType] = None,
|
|
3216
|
+
) -> Optional[Dict[str, Any]]:
|
|
3217
|
+
try:
|
|
3218
|
+
table = self._get_table(table_type="components")
|
|
3219
|
+
if table is None:
|
|
3220
|
+
return None
|
|
3221
|
+
|
|
3222
|
+
with self.Session() as sess:
|
|
3223
|
+
stmt = select(table).where(
|
|
3224
|
+
table.c.component_id == component_id,
|
|
3225
|
+
table.c.deleted_at.is_(None),
|
|
3226
|
+
)
|
|
3227
|
+
|
|
3228
|
+
if component_type is not None:
|
|
3229
|
+
stmt = stmt.where(table.c.component_type == component_type.value)
|
|
3230
|
+
|
|
3231
|
+
row = sess.execute(stmt).mappings().one_or_none()
|
|
3232
|
+
return dict(row) if row else None
|
|
3233
|
+
|
|
3234
|
+
except Exception as e:
|
|
3235
|
+
log_error(f"Error getting component: {e}")
|
|
3236
|
+
raise
|
|
3237
|
+
|
|
3238
|
+
def upsert_component(
|
|
3239
|
+
self,
|
|
3240
|
+
component_id: str,
|
|
3241
|
+
component_type: Optional[ComponentType] = None,
|
|
3242
|
+
name: Optional[str] = None,
|
|
3243
|
+
description: Optional[str] = None,
|
|
3244
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
3245
|
+
) -> Dict[str, Any]:
|
|
3246
|
+
"""Create or update a component.
|
|
3247
|
+
|
|
3248
|
+
Args:
|
|
3249
|
+
component_id: Unique identifier.
|
|
3250
|
+
component_type: Type (agent|team|workflow). Required for create, optional for update.
|
|
3251
|
+
name: Display name.
|
|
3252
|
+
description: Optional description.
|
|
3253
|
+
metadata: Optional metadata dict.
|
|
3254
|
+
|
|
3255
|
+
Returns:
|
|
3256
|
+
Created/updated component dictionary.
|
|
3257
|
+
|
|
3258
|
+
Raises:
|
|
3259
|
+
ValueError: If creating and component_type is not provided.
|
|
3260
|
+
"""
|
|
3261
|
+
try:
|
|
3262
|
+
table = self._get_table(table_type="components", create_table_if_not_found=True)
|
|
3263
|
+
if table is None:
|
|
3264
|
+
raise ValueError("Components table not found")
|
|
3265
|
+
|
|
3266
|
+
with self.Session() as sess, sess.begin():
|
|
3267
|
+
existing = sess.execute(
|
|
3268
|
+
select(table).where(
|
|
3269
|
+
table.c.component_id == component_id,
|
|
3270
|
+
table.c.deleted_at.is_(None),
|
|
3271
|
+
)
|
|
3272
|
+
).fetchone()
|
|
3273
|
+
if existing is None:
|
|
3274
|
+
# Create new component
|
|
3275
|
+
if component_type is None:
|
|
3276
|
+
raise ValueError("component_type is required when creating a new component")
|
|
3277
|
+
|
|
3278
|
+
sess.execute(
|
|
3279
|
+
table.insert().values(
|
|
3280
|
+
component_id=component_id,
|
|
3281
|
+
component_type=component_type.value,
|
|
3282
|
+
name=name,
|
|
3283
|
+
description=description,
|
|
3284
|
+
current_version=None,
|
|
3285
|
+
metadata=metadata,
|
|
3286
|
+
created_at=int(time.time()),
|
|
3287
|
+
)
|
|
3288
|
+
)
|
|
3289
|
+
log_debug(f"Created component {component_id}")
|
|
3290
|
+
|
|
3291
|
+
elif existing.deleted_at is not None:
|
|
3292
|
+
# Reactivate soft-deleted
|
|
3293
|
+
if component_type is None:
|
|
3294
|
+
raise ValueError("component_type is required when reactivating a deleted component")
|
|
3295
|
+
|
|
3296
|
+
sess.execute(
|
|
3297
|
+
table.update()
|
|
3298
|
+
.where(table.c.component_id == component_id)
|
|
3299
|
+
.values(
|
|
3300
|
+
component_type=component_type.value,
|
|
3301
|
+
name=name or component_id,
|
|
3302
|
+
description=description,
|
|
3303
|
+
current_version=None,
|
|
3304
|
+
metadata=metadata,
|
|
3305
|
+
updated_at=int(time.time()),
|
|
3306
|
+
deleted_at=None,
|
|
3307
|
+
)
|
|
3308
|
+
)
|
|
3309
|
+
log_debug(f"Reactivated component {component_id}")
|
|
3310
|
+
|
|
3311
|
+
else:
|
|
3312
|
+
# Update existing
|
|
3313
|
+
updates: Dict[str, Any] = {"updated_at": int(time.time())}
|
|
3314
|
+
if component_type is not None:
|
|
3315
|
+
updates["component_type"] = component_type.value
|
|
3316
|
+
if name is not None:
|
|
3317
|
+
updates["name"] = name
|
|
3318
|
+
if description is not None:
|
|
3319
|
+
updates["description"] = description
|
|
3320
|
+
if metadata is not None:
|
|
3321
|
+
updates["metadata"] = metadata
|
|
3322
|
+
|
|
3323
|
+
sess.execute(table.update().where(table.c.component_id == component_id).values(**updates))
|
|
3324
|
+
log_debug(f"Updated component {component_id}")
|
|
3325
|
+
|
|
3326
|
+
result = self.get_component(component_id)
|
|
3327
|
+
if result is None:
|
|
3328
|
+
raise ValueError(f"Failed to get component {component_id} after upsert")
|
|
3329
|
+
return result
|
|
3330
|
+
|
|
3331
|
+
except Exception as e:
|
|
3332
|
+
log_error(f"Error upserting component: {e}")
|
|
3333
|
+
raise
|
|
3334
|
+
|
|
3335
|
+
def delete_component(
|
|
3336
|
+
self,
|
|
3337
|
+
component_id: str,
|
|
3338
|
+
hard_delete: bool = False,
|
|
3339
|
+
) -> bool:
|
|
3340
|
+
"""Delete a component and all its configs/links.
|
|
3341
|
+
|
|
3342
|
+
Args:
|
|
3343
|
+
component_id: The component ID.
|
|
3344
|
+
hard_delete: If True, permanently delete. Otherwise soft-delete.
|
|
3345
|
+
|
|
3346
|
+
Returns:
|
|
3347
|
+
True if deleted, False if not found or already deleted.
|
|
3348
|
+
"""
|
|
3349
|
+
try:
|
|
3350
|
+
components_table = self._get_table(table_type="components")
|
|
3351
|
+
configs_table = self._get_table(table_type="component_configs")
|
|
3352
|
+
links_table = self._get_table(table_type="component_links")
|
|
3353
|
+
|
|
3354
|
+
if components_table is None:
|
|
3355
|
+
return False
|
|
3356
|
+
|
|
3357
|
+
with self.Session() as sess, sess.begin():
|
|
3358
|
+
# Verify component exists (and not already soft-deleted for soft-delete)
|
|
3359
|
+
if hard_delete:
|
|
3360
|
+
exists = sess.execute(
|
|
3361
|
+
select(components_table.c.component_id).where(components_table.c.component_id == component_id)
|
|
3362
|
+
).scalar_one_or_none()
|
|
3363
|
+
else:
|
|
3364
|
+
exists = sess.execute(
|
|
3365
|
+
select(components_table.c.component_id).where(
|
|
3366
|
+
components_table.c.component_id == component_id,
|
|
3367
|
+
components_table.c.deleted_at.is_(None),
|
|
3368
|
+
)
|
|
3369
|
+
).scalar_one_or_none()
|
|
3370
|
+
|
|
3371
|
+
if exists is None:
|
|
3372
|
+
log_error(f"Component {component_id} not found")
|
|
3373
|
+
return False
|
|
3374
|
+
|
|
3375
|
+
if hard_delete:
|
|
3376
|
+
# Delete links where this component is parent or child
|
|
3377
|
+
if links_table is not None:
|
|
3378
|
+
sess.execute(links_table.delete().where(links_table.c.parent_component_id == component_id))
|
|
3379
|
+
sess.execute(links_table.delete().where(links_table.c.child_component_id == component_id))
|
|
3380
|
+
# Delete configs
|
|
3381
|
+
if configs_table is not None:
|
|
3382
|
+
sess.execute(configs_table.delete().where(configs_table.c.component_id == component_id))
|
|
3383
|
+
# Delete component
|
|
3384
|
+
sess.execute(components_table.delete().where(components_table.c.component_id == component_id))
|
|
3385
|
+
else:
|
|
3386
|
+
# Soft delete (preserve current_version for potential reactivation)
|
|
3387
|
+
sess.execute(
|
|
3388
|
+
components_table.update()
|
|
3389
|
+
.where(components_table.c.component_id == component_id)
|
|
3390
|
+
.values(deleted_at=int(time.time()))
|
|
3391
|
+
)
|
|
3392
|
+
|
|
3393
|
+
return True
|
|
3394
|
+
|
|
3395
|
+
except Exception as e:
|
|
3396
|
+
log_error(f"Error deleting component: {e}")
|
|
3397
|
+
raise
|
|
3398
|
+
|
|
3399
|
+
def list_components(
|
|
3400
|
+
self,
|
|
3401
|
+
component_type: Optional[ComponentType] = None,
|
|
3402
|
+
include_deleted: bool = False,
|
|
3403
|
+
limit: int = 20,
|
|
3404
|
+
offset: int = 0,
|
|
3405
|
+
) -> Tuple[List[Dict[str, Any]], int]:
|
|
3406
|
+
"""List components with pagination.
|
|
3407
|
+
|
|
3408
|
+
Args:
|
|
3409
|
+
component_type: Filter by type (agent|team|workflow).
|
|
3410
|
+
include_deleted: Include soft-deleted components.
|
|
3411
|
+
limit: Maximum number of items to return.
|
|
3412
|
+
offset: Number of items to skip.
|
|
3413
|
+
|
|
3414
|
+
Returns:
|
|
3415
|
+
Tuple of (list of component dicts, total count).
|
|
3416
|
+
"""
|
|
3417
|
+
try:
|
|
3418
|
+
table = self._get_table(table_type="components")
|
|
3419
|
+
if table is None:
|
|
3420
|
+
return [], 0
|
|
3421
|
+
|
|
3422
|
+
with self.Session() as sess:
|
|
3423
|
+
# Build base where clause
|
|
3424
|
+
where_clauses = []
|
|
3425
|
+
if component_type is not None:
|
|
3426
|
+
where_clauses.append(table.c.component_type == component_type.value)
|
|
3427
|
+
if not include_deleted:
|
|
3428
|
+
where_clauses.append(table.c.deleted_at.is_(None))
|
|
3429
|
+
|
|
3430
|
+
# Get total count
|
|
3431
|
+
count_stmt = select(func.count()).select_from(table)
|
|
3432
|
+
for clause in where_clauses:
|
|
3433
|
+
count_stmt = count_stmt.where(clause)
|
|
3434
|
+
total_count = sess.execute(count_stmt).scalar() or 0
|
|
3435
|
+
|
|
3436
|
+
# Get paginated results
|
|
3437
|
+
stmt = select(table).order_by(
|
|
3438
|
+
table.c.created_at.desc(),
|
|
3439
|
+
table.c.component_id,
|
|
3440
|
+
)
|
|
3441
|
+
for clause in where_clauses:
|
|
3442
|
+
stmt = stmt.where(clause)
|
|
3443
|
+
stmt = stmt.limit(limit).offset(offset)
|
|
3444
|
+
|
|
3445
|
+
rows = sess.execute(stmt).mappings().all()
|
|
3446
|
+
return [dict(r) for r in rows], total_count
|
|
3447
|
+
|
|
3448
|
+
except Exception as e:
|
|
3449
|
+
log_error(f"Error listing components: {e}")
|
|
3450
|
+
raise
|
|
3451
|
+
|
|
3452
|
+
def create_component_with_config(
|
|
3453
|
+
self,
|
|
3454
|
+
component_id: str,
|
|
3455
|
+
component_type: ComponentType,
|
|
3456
|
+
name: Optional[str],
|
|
3457
|
+
config: Dict[str, Any],
|
|
3458
|
+
description: Optional[str] = None,
|
|
3459
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
3460
|
+
label: Optional[str] = None,
|
|
3461
|
+
stage: str = "draft",
|
|
3462
|
+
notes: Optional[str] = None,
|
|
3463
|
+
links: Optional[List[Dict[str, Any]]] = None,
|
|
3464
|
+
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
|
3465
|
+
"""Create a component with its initial config atomically.
|
|
3466
|
+
|
|
3467
|
+
Args:
|
|
3468
|
+
component_id: Unique identifier.
|
|
3469
|
+
component_type: Type (agent|team|workflow).
|
|
3470
|
+
name: Display name.
|
|
3471
|
+
config: The config data.
|
|
3472
|
+
description: Optional description.
|
|
3473
|
+
metadata: Optional metadata dict.
|
|
3474
|
+
label: Optional config label.
|
|
3475
|
+
stage: "draft" or "published".
|
|
3476
|
+
notes: Optional notes.
|
|
3477
|
+
links: Optional list of links. Each must have child_version set.
|
|
3478
|
+
|
|
3479
|
+
Returns:
|
|
3480
|
+
Tuple of (component dict, config dict).
|
|
3481
|
+
|
|
3482
|
+
Raises:
|
|
3483
|
+
ValueError: If component already exists, invalid stage, or link missing child_version.
|
|
3484
|
+
"""
|
|
3485
|
+
if stage not in {"draft", "published"}:
|
|
3486
|
+
raise ValueError(f"Invalid stage: {stage}")
|
|
3487
|
+
|
|
3488
|
+
# Validate links have child_version
|
|
3489
|
+
if links:
|
|
3490
|
+
for link in links:
|
|
3491
|
+
if link.get("child_version") is None:
|
|
3492
|
+
raise ValueError(f"child_version is required for link to {link['child_component_id']}")
|
|
3493
|
+
|
|
3494
|
+
try:
|
|
3495
|
+
components_table = self._get_table(table_type="components", create_table_if_not_found=True)
|
|
3496
|
+
configs_table = self._get_table(table_type="component_configs", create_table_if_not_found=True)
|
|
3497
|
+
links_table = self._get_table(table_type="component_links", create_table_if_not_found=True)
|
|
3498
|
+
|
|
3499
|
+
if components_table is None:
|
|
3500
|
+
raise ValueError("Components table not found")
|
|
3501
|
+
if configs_table is None:
|
|
3502
|
+
raise ValueError("Component configs table not found")
|
|
3503
|
+
|
|
3504
|
+
with self.Session() as sess, sess.begin():
|
|
3505
|
+
# Check if component already exists
|
|
3506
|
+
existing = sess.execute(
|
|
3507
|
+
select(components_table.c.component_id).where(components_table.c.component_id == component_id)
|
|
3508
|
+
).scalar_one_or_none()
|
|
3509
|
+
|
|
3510
|
+
if existing is not None:
|
|
3511
|
+
raise ValueError(f"Component {component_id} already exists")
|
|
3512
|
+
|
|
3513
|
+
# Check label uniqueness
|
|
3514
|
+
if label is not None:
|
|
3515
|
+
existing_label = sess.execute(
|
|
3516
|
+
select(configs_table.c.version).where(
|
|
3517
|
+
configs_table.c.component_id == component_id,
|
|
3518
|
+
configs_table.c.label == label,
|
|
3519
|
+
)
|
|
3520
|
+
).first()
|
|
3521
|
+
if existing_label:
|
|
3522
|
+
raise ValueError(f"Label '{label}' already exists for {component_id}")
|
|
3523
|
+
|
|
3524
|
+
now = int(time.time())
|
|
3525
|
+
version = 1
|
|
3526
|
+
|
|
3527
|
+
# Create component
|
|
3528
|
+
sess.execute(
|
|
3529
|
+
components_table.insert().values(
|
|
3530
|
+
component_id=component_id,
|
|
3531
|
+
component_type=component_type.value,
|
|
3532
|
+
name=name,
|
|
3533
|
+
description=description,
|
|
3534
|
+
metadata=metadata,
|
|
3535
|
+
current_version=version if stage == "published" else None,
|
|
3536
|
+
created_at=now,
|
|
3537
|
+
)
|
|
3538
|
+
)
|
|
3539
|
+
|
|
3540
|
+
# Create initial config
|
|
3541
|
+
sess.execute(
|
|
3542
|
+
configs_table.insert().values(
|
|
3543
|
+
component_id=component_id,
|
|
3544
|
+
version=version,
|
|
3545
|
+
label=label,
|
|
3546
|
+
stage=stage,
|
|
3547
|
+
config=config,
|
|
3548
|
+
notes=notes,
|
|
3549
|
+
created_at=now,
|
|
3550
|
+
)
|
|
3551
|
+
)
|
|
3552
|
+
|
|
3553
|
+
# Create links if provided
|
|
3554
|
+
if links and links_table is not None:
|
|
3555
|
+
for link in links:
|
|
3556
|
+
sess.execute(
|
|
3557
|
+
links_table.insert().values(
|
|
3558
|
+
parent_component_id=component_id,
|
|
3559
|
+
parent_version=version,
|
|
3560
|
+
link_kind=link["link_kind"],
|
|
3561
|
+
link_key=link["link_key"],
|
|
3562
|
+
child_component_id=link["child_component_id"],
|
|
3563
|
+
child_version=link["child_version"],
|
|
3564
|
+
position=link["position"],
|
|
3565
|
+
meta=link.get("meta"),
|
|
3566
|
+
created_at=now,
|
|
3567
|
+
)
|
|
3568
|
+
)
|
|
3569
|
+
|
|
3570
|
+
# Fetch and return both
|
|
3571
|
+
component = self.get_component(component_id)
|
|
3572
|
+
config_result = self.get_config(component_id, version=version)
|
|
3573
|
+
|
|
3574
|
+
if component is None:
|
|
3575
|
+
raise ValueError(f"Failed to get component {component_id} after creation")
|
|
3576
|
+
if config_result is None:
|
|
3577
|
+
raise ValueError(f"Failed to get config for {component_id} after creation")
|
|
3578
|
+
|
|
3579
|
+
return component, config_result
|
|
3580
|
+
|
|
3581
|
+
except Exception as e:
|
|
3582
|
+
log_error(f"Error creating component with config: {e}")
|
|
3583
|
+
raise
|
|
3584
|
+
|
|
3585
|
+
# --- Component Configs ---
|
|
3586
|
+
def get_config(
|
|
3587
|
+
self,
|
|
3588
|
+
component_id: str,
|
|
3589
|
+
version: Optional[int] = None,
|
|
3590
|
+
label: Optional[str] = None,
|
|
3591
|
+
) -> Optional[Dict[str, Any]]:
|
|
3592
|
+
"""Get a config by component ID and version or label.
|
|
3593
|
+
|
|
3594
|
+
Args:
|
|
3595
|
+
component_id: The component ID.
|
|
3596
|
+
version: Specific version number. If None, uses current or latest draft.
|
|
3597
|
+
label: Config label to lookup. Ignored if version is provided.
|
|
3598
|
+
|
|
3599
|
+
Returns:
|
|
3600
|
+
Config dictionary or None if not found.
|
|
3601
|
+
"""
|
|
3602
|
+
try:
|
|
3603
|
+
configs_table = self._get_table(table_type="component_configs")
|
|
3604
|
+
components_table = self._get_table(table_type="components")
|
|
3605
|
+
|
|
3606
|
+
if configs_table is None or components_table is None:
|
|
3607
|
+
return None
|
|
3608
|
+
|
|
3609
|
+
with self.Session() as sess:
|
|
3610
|
+
# Verify component exists and get current_version
|
|
3611
|
+
component_row = (
|
|
3612
|
+
sess.execute(
|
|
3613
|
+
select(components_table.c.component_id, components_table.c.current_version).where(
|
|
3614
|
+
components_table.c.component_id == component_id,
|
|
3615
|
+
components_table.c.deleted_at.is_(None),
|
|
3616
|
+
)
|
|
3617
|
+
)
|
|
3618
|
+
.mappings()
|
|
3619
|
+
.one_or_none()
|
|
3620
|
+
)
|
|
3621
|
+
|
|
3622
|
+
if component_row is None:
|
|
3623
|
+
return None
|
|
3624
|
+
|
|
3625
|
+
current_version = component_row["current_version"]
|
|
3626
|
+
|
|
3627
|
+
if version is not None:
|
|
3628
|
+
stmt = select(configs_table).where(
|
|
3629
|
+
configs_table.c.component_id == component_id,
|
|
3630
|
+
configs_table.c.version == version,
|
|
3631
|
+
)
|
|
3632
|
+
elif label is not None:
|
|
3633
|
+
stmt = select(configs_table).where(
|
|
3634
|
+
configs_table.c.component_id == component_id,
|
|
3635
|
+
configs_table.c.label == label,
|
|
3636
|
+
)
|
|
3637
|
+
elif current_version is not None:
|
|
3638
|
+
# Use the current published version
|
|
3639
|
+
stmt = select(configs_table).where(
|
|
3640
|
+
configs_table.c.component_id == component_id,
|
|
3641
|
+
configs_table.c.version == current_version,
|
|
3642
|
+
)
|
|
3643
|
+
else:
|
|
3644
|
+
# No current_version set (draft only) - get the latest version
|
|
3645
|
+
stmt = (
|
|
3646
|
+
select(configs_table)
|
|
3647
|
+
.where(configs_table.c.component_id == component_id)
|
|
3648
|
+
.order_by(configs_table.c.version.desc())
|
|
3649
|
+
.limit(1)
|
|
3650
|
+
)
|
|
3651
|
+
|
|
3652
|
+
row = sess.execute(stmt).mappings().one_or_none()
|
|
3653
|
+
return dict(row) if row else None
|
|
3654
|
+
|
|
3655
|
+
except Exception as e:
|
|
3656
|
+
log_error(f"Error getting config: {e}")
|
|
3657
|
+
raise
|
|
3658
|
+
|
|
3659
|
+
def upsert_config(
|
|
3660
|
+
self,
|
|
3661
|
+
component_id: str,
|
|
3662
|
+
config: Optional[Dict[str, Any]] = None,
|
|
3663
|
+
version: Optional[int] = None,
|
|
3664
|
+
label: Optional[str] = None,
|
|
3665
|
+
stage: Optional[str] = None,
|
|
3666
|
+
notes: Optional[str] = None,
|
|
3667
|
+
links: Optional[List[Dict[str, Any]]] = None,
|
|
3668
|
+
) -> Dict[str, Any]:
|
|
3669
|
+
"""Create or update a config version for a component.
|
|
3670
|
+
|
|
3671
|
+
Rules:
|
|
3672
|
+
- Draft configs can be edited freely
|
|
3673
|
+
- Published configs are immutable
|
|
3674
|
+
- Publishing a config automatically sets it as current_version
|
|
3675
|
+
|
|
3676
|
+
Args:
|
|
3677
|
+
component_id: The component ID.
|
|
3678
|
+
config: The config data. Required for create, optional for update.
|
|
3679
|
+
version: If None, creates new version. If provided, updates that version.
|
|
3680
|
+
label: Optional human-readable label.
|
|
3681
|
+
stage: "draft" or "published". Defaults to "draft" for new configs.
|
|
3682
|
+
notes: Optional notes.
|
|
3683
|
+
links: Optional list of links. Each link must have child_version set.
|
|
3684
|
+
|
|
3685
|
+
Returns:
|
|
3686
|
+
Created/updated config dictionary.
|
|
3687
|
+
|
|
3688
|
+
Raises:
|
|
3689
|
+
ValueError: If component doesn't exist, version not found, label conflict,
|
|
3690
|
+
or attempting to update a published config.
|
|
3691
|
+
"""
|
|
3692
|
+
if stage is not None and stage not in {"draft", "published"}:
|
|
3693
|
+
raise ValueError(f"Invalid stage: {stage}")
|
|
3694
|
+
|
|
3695
|
+
try:
|
|
3696
|
+
configs_table = self._get_table(table_type="component_configs", create_table_if_not_found=True)
|
|
3697
|
+
components_table = self._get_table(table_type="components")
|
|
3698
|
+
links_table = self._get_table(table_type="component_links", create_table_if_not_found=True)
|
|
3699
|
+
|
|
3700
|
+
if components_table is None:
|
|
3701
|
+
raise ValueError("Components table not found")
|
|
3702
|
+
if configs_table is None:
|
|
3703
|
+
raise ValueError("Component configs table not found")
|
|
3704
|
+
|
|
3705
|
+
with self.Session() as sess, sess.begin():
|
|
3706
|
+
# Verify component exists and is not deleted
|
|
3707
|
+
component = sess.execute(
|
|
3708
|
+
select(components_table.c.component_id).where(
|
|
3709
|
+
components_table.c.component_id == component_id,
|
|
3710
|
+
components_table.c.deleted_at.is_(None),
|
|
3711
|
+
)
|
|
3712
|
+
).scalar_one_or_none()
|
|
3713
|
+
|
|
3714
|
+
if component is None:
|
|
3715
|
+
raise ValueError(f"Component {component_id} not found")
|
|
3716
|
+
|
|
3717
|
+
# Label uniqueness check
|
|
3718
|
+
if label is not None:
|
|
3719
|
+
label_query = select(configs_table.c.version).where(
|
|
3720
|
+
configs_table.c.component_id == component_id,
|
|
3721
|
+
configs_table.c.label == label,
|
|
3722
|
+
)
|
|
3723
|
+
if version is not None:
|
|
3724
|
+
label_query = label_query.where(configs_table.c.version != version)
|
|
3725
|
+
|
|
3726
|
+
if sess.execute(label_query).first():
|
|
3727
|
+
raise ValueError(f"Label '{label}' already exists for {component_id}")
|
|
3728
|
+
|
|
3729
|
+
# Validate links have child_version
|
|
3730
|
+
if links:
|
|
3731
|
+
for link in links:
|
|
3732
|
+
if link.get("child_version") is None:
|
|
3733
|
+
raise ValueError(f"child_version is required for link to {link['child_component_id']}")
|
|
3734
|
+
|
|
3735
|
+
if version is None:
|
|
3736
|
+
if config is None:
|
|
3737
|
+
raise ValueError("config is required when creating a new version")
|
|
3738
|
+
|
|
3739
|
+
# Default to draft for new configs
|
|
3740
|
+
if stage is None:
|
|
3741
|
+
stage = "draft"
|
|
3742
|
+
|
|
3743
|
+
max_version = sess.execute(
|
|
3744
|
+
select(configs_table.c.version)
|
|
3745
|
+
.where(configs_table.c.component_id == component_id)
|
|
3746
|
+
.order_by(configs_table.c.version.desc())
|
|
3747
|
+
.limit(1)
|
|
3748
|
+
).scalar()
|
|
3749
|
+
|
|
3750
|
+
final_version = (max_version or 0) + 1
|
|
3751
|
+
|
|
3752
|
+
sess.execute(
|
|
3753
|
+
configs_table.insert().values(
|
|
3754
|
+
component_id=component_id,
|
|
3755
|
+
version=final_version,
|
|
3756
|
+
label=label,
|
|
3757
|
+
stage=stage,
|
|
3758
|
+
config=config,
|
|
3759
|
+
notes=notes,
|
|
3760
|
+
created_at=int(time.time()),
|
|
3761
|
+
)
|
|
3762
|
+
)
|
|
3763
|
+
else:
|
|
3764
|
+
existing = (
|
|
3765
|
+
sess.execute(
|
|
3766
|
+
select(configs_table.c.version, configs_table.c.stage).where(
|
|
3767
|
+
configs_table.c.component_id == component_id,
|
|
3768
|
+
configs_table.c.version == version,
|
|
3769
|
+
)
|
|
3770
|
+
)
|
|
3771
|
+
.mappings()
|
|
3772
|
+
.one_or_none()
|
|
3773
|
+
)
|
|
3774
|
+
|
|
3775
|
+
if existing is None:
|
|
3776
|
+
raise ValueError(f"Config {component_id} v{version} not found")
|
|
3777
|
+
|
|
3778
|
+
# Published configs are immutable
|
|
3779
|
+
if existing["stage"] == "published":
|
|
3780
|
+
raise ValueError(f"Cannot update published config {component_id} v{version}")
|
|
3781
|
+
|
|
3782
|
+
# Build update dict with only provided fields
|
|
3783
|
+
updates: Dict[str, Any] = {"updated_at": int(time.time())}
|
|
3784
|
+
if label is not None:
|
|
3785
|
+
updates["label"] = label
|
|
3786
|
+
if stage is not None:
|
|
3787
|
+
updates["stage"] = stage
|
|
3788
|
+
if config is not None:
|
|
3789
|
+
updates["config"] = config
|
|
3790
|
+
if notes is not None:
|
|
3791
|
+
updates["notes"] = notes
|
|
3792
|
+
|
|
3793
|
+
sess.execute(
|
|
3794
|
+
configs_table.update()
|
|
3795
|
+
.where(
|
|
3796
|
+
configs_table.c.component_id == component_id,
|
|
3797
|
+
configs_table.c.version == version,
|
|
3798
|
+
)
|
|
3799
|
+
.values(**updates)
|
|
3800
|
+
)
|
|
3801
|
+
final_version = version
|
|
3802
|
+
|
|
3803
|
+
if links is not None and links_table is not None:
|
|
3804
|
+
sess.execute(
|
|
3805
|
+
links_table.delete().where(
|
|
3806
|
+
links_table.c.parent_component_id == component_id,
|
|
3807
|
+
links_table.c.parent_version == final_version,
|
|
3808
|
+
)
|
|
3809
|
+
)
|
|
3810
|
+
for link in links:
|
|
3811
|
+
sess.execute(
|
|
3812
|
+
links_table.insert().values(
|
|
3813
|
+
parent_component_id=component_id,
|
|
3814
|
+
parent_version=final_version,
|
|
3815
|
+
link_kind=link["link_kind"],
|
|
3816
|
+
link_key=link["link_key"],
|
|
3817
|
+
child_component_id=link["child_component_id"],
|
|
3818
|
+
child_version=link["child_version"],
|
|
3819
|
+
position=link["position"],
|
|
3820
|
+
meta=link.get("meta"),
|
|
3821
|
+
created_at=int(time.time()),
|
|
3822
|
+
)
|
|
3823
|
+
)
|
|
3824
|
+
|
|
3825
|
+
# Determine final stage (could be from update or create)
|
|
3826
|
+
final_stage = stage if stage is not None else (existing["stage"] if version is not None else "draft")
|
|
3827
|
+
|
|
3828
|
+
if final_stage == "published":
|
|
3829
|
+
sess.execute(
|
|
3830
|
+
components_table.update()
|
|
3831
|
+
.where(components_table.c.component_id == component_id)
|
|
3832
|
+
.values(current_version=final_version, updated_at=int(time.time()))
|
|
3833
|
+
)
|
|
3834
|
+
|
|
3835
|
+
result = self.get_config(component_id, version=final_version)
|
|
3836
|
+
if result is None:
|
|
3837
|
+
raise ValueError(f"Failed to get config {component_id} v{final_version} after upsert")
|
|
3838
|
+
return result
|
|
3839
|
+
|
|
3840
|
+
except Exception as e:
|
|
3841
|
+
log_error(f"Error upserting config: {e}")
|
|
3842
|
+
raise
|
|
3843
|
+
|
|
3844
|
+
def delete_config(
|
|
3845
|
+
self,
|
|
3846
|
+
component_id: str,
|
|
3847
|
+
version: int,
|
|
3848
|
+
) -> bool:
|
|
3849
|
+
"""Delete a specific config version.
|
|
3850
|
+
|
|
3851
|
+
Only draft configs can be deleted. Published configs are immutable.
|
|
3852
|
+
Cannot delete the current version.
|
|
3853
|
+
|
|
3854
|
+
Args:
|
|
3855
|
+
component_id: The component ID.
|
|
3856
|
+
version: The version to delete.
|
|
3857
|
+
|
|
3858
|
+
Returns:
|
|
3859
|
+
True if deleted, False if not found.
|
|
3860
|
+
|
|
3861
|
+
Raises:
|
|
3862
|
+
ValueError: If attempting to delete a published or current config.
|
|
3863
|
+
"""
|
|
3864
|
+
try:
|
|
3865
|
+
configs_table = self._get_table(table_type="component_configs")
|
|
3866
|
+
links_table = self._get_table(table_type="component_links")
|
|
3867
|
+
components_table = self._get_table(table_type="components")
|
|
3868
|
+
|
|
3869
|
+
if configs_table is None or components_table is None:
|
|
3870
|
+
return False
|
|
3871
|
+
|
|
3872
|
+
with self.Session() as sess, sess.begin():
|
|
3873
|
+
# Get config stage and check if it's current
|
|
3874
|
+
config_row = sess.execute(
|
|
3875
|
+
select(configs_table.c.stage).where(
|
|
3876
|
+
configs_table.c.component_id == component_id,
|
|
3877
|
+
configs_table.c.version == version,
|
|
3878
|
+
)
|
|
3879
|
+
).scalar_one_or_none()
|
|
3880
|
+
|
|
3881
|
+
if config_row is None:
|
|
3882
|
+
return False
|
|
3883
|
+
|
|
3884
|
+
# Cannot delete published configs
|
|
3885
|
+
if config_row == "published":
|
|
3886
|
+
raise ValueError(f"Cannot delete published config {component_id} v{version}")
|
|
3887
|
+
|
|
3888
|
+
# Check if it's current version
|
|
3889
|
+
current = sess.execute(
|
|
3890
|
+
select(components_table.c.current_version).where(components_table.c.component_id == component_id)
|
|
3891
|
+
).scalar_one_or_none()
|
|
3892
|
+
|
|
3893
|
+
if current == version:
|
|
3894
|
+
raise ValueError(f"Cannot delete current config {component_id} v{version}")
|
|
3895
|
+
|
|
3896
|
+
# Delete associated links
|
|
3897
|
+
if links_table is not None:
|
|
3898
|
+
sess.execute(
|
|
3899
|
+
links_table.delete().where(
|
|
3900
|
+
links_table.c.parent_component_id == component_id,
|
|
3901
|
+
links_table.c.parent_version == version,
|
|
3902
|
+
)
|
|
3903
|
+
)
|
|
3904
|
+
|
|
3905
|
+
# Delete the config
|
|
3906
|
+
sess.execute(
|
|
3907
|
+
configs_table.delete().where(
|
|
3908
|
+
configs_table.c.component_id == component_id,
|
|
3909
|
+
configs_table.c.version == version,
|
|
3910
|
+
)
|
|
3911
|
+
)
|
|
3912
|
+
|
|
3913
|
+
return True
|
|
3914
|
+
|
|
3915
|
+
except Exception as e:
|
|
3916
|
+
log_error(f"Error deleting config: {e}")
|
|
3917
|
+
raise
|
|
3918
|
+
|
|
3919
|
+
def list_configs(
|
|
3920
|
+
self,
|
|
3921
|
+
component_id: str,
|
|
3922
|
+
include_config: bool = False,
|
|
3923
|
+
) -> List[Dict[str, Any]]:
|
|
3924
|
+
"""List all config versions for a component.
|
|
3925
|
+
|
|
3926
|
+
Args:
|
|
3927
|
+
component_id: The component ID.
|
|
3928
|
+
include_config: If True, include full config blob. Otherwise just metadata.
|
|
3929
|
+
|
|
3930
|
+
Returns:
|
|
3931
|
+
List of config dictionaries, newest first.
|
|
3932
|
+
Returns empty list if component not found or deleted.
|
|
3933
|
+
"""
|
|
3934
|
+
try:
|
|
3935
|
+
configs_table = self._get_table(table_type="component_configs")
|
|
3936
|
+
components_table = self._get_table(table_type="components")
|
|
3937
|
+
|
|
3938
|
+
if configs_table is None or components_table is None:
|
|
3939
|
+
return []
|
|
3940
|
+
|
|
3941
|
+
with self.Session() as sess:
|
|
3942
|
+
# Verify component exists and is not deleted
|
|
3943
|
+
exists = sess.execute(
|
|
3944
|
+
select(components_table.c.component_id).where(
|
|
3945
|
+
components_table.c.component_id == component_id,
|
|
3946
|
+
components_table.c.deleted_at.is_(None),
|
|
3947
|
+
)
|
|
3948
|
+
).scalar_one_or_none()
|
|
3949
|
+
|
|
3950
|
+
if exists is None:
|
|
3951
|
+
return []
|
|
3952
|
+
|
|
3953
|
+
# Select columns based on include_config flag
|
|
3954
|
+
if include_config:
|
|
3955
|
+
stmt = select(configs_table)
|
|
3956
|
+
else:
|
|
3957
|
+
stmt = select(
|
|
3958
|
+
configs_table.c.component_id,
|
|
3959
|
+
configs_table.c.version,
|
|
3960
|
+
configs_table.c.label,
|
|
3961
|
+
configs_table.c.stage,
|
|
3962
|
+
configs_table.c.notes,
|
|
3963
|
+
configs_table.c.created_at,
|
|
3964
|
+
configs_table.c.updated_at,
|
|
3965
|
+
)
|
|
3966
|
+
|
|
3967
|
+
stmt = stmt.where(configs_table.c.component_id == component_id).order_by(configs_table.c.version.desc())
|
|
3968
|
+
|
|
3969
|
+
results = sess.execute(stmt).mappings().all()
|
|
3970
|
+
return [dict(row) for row in results]
|
|
3971
|
+
|
|
3972
|
+
except Exception as e:
|
|
3973
|
+
log_error(f"Error listing configs: {e}")
|
|
3974
|
+
raise
|
|
3975
|
+
|
|
3976
|
+
def set_current_version(
|
|
3977
|
+
self,
|
|
3978
|
+
component_id: str,
|
|
3979
|
+
version: int,
|
|
3980
|
+
) -> bool:
|
|
3981
|
+
"""Set a specific published version as current.
|
|
3982
|
+
|
|
3983
|
+
Only published configs can be set as current. This is used for
|
|
3984
|
+
rollback scenarios where you want to switch to a previous
|
|
3985
|
+
published version.
|
|
3986
|
+
|
|
3987
|
+
Args:
|
|
3988
|
+
component_id: The component ID.
|
|
3989
|
+
version: The version to set as current (must be published).
|
|
3990
|
+
|
|
3991
|
+
Returns:
|
|
3992
|
+
True if successful, False if component or version not found.
|
|
3993
|
+
|
|
3994
|
+
Raises:
|
|
3995
|
+
ValueError: If attempting to set a draft config as current.
|
|
3996
|
+
"""
|
|
3997
|
+
try:
|
|
3998
|
+
configs_table = self._get_table(table_type="component_configs")
|
|
3999
|
+
components_table = self._get_table(table_type="components")
|
|
4000
|
+
|
|
4001
|
+
if configs_table is None or components_table is None:
|
|
4002
|
+
return False
|
|
4003
|
+
|
|
4004
|
+
with self.Session() as sess, sess.begin():
|
|
4005
|
+
# Verify component exists and is not deleted
|
|
4006
|
+
component_exists = sess.execute(
|
|
4007
|
+
select(components_table.c.component_id).where(
|
|
4008
|
+
components_table.c.component_id == component_id,
|
|
4009
|
+
components_table.c.deleted_at.is_(None),
|
|
4010
|
+
)
|
|
4011
|
+
).scalar_one_or_none()
|
|
4012
|
+
|
|
4013
|
+
if component_exists is None:
|
|
4014
|
+
return False
|
|
4015
|
+
|
|
4016
|
+
# Verify version exists and get stage
|
|
4017
|
+
stage = sess.execute(
|
|
4018
|
+
select(configs_table.c.stage).where(
|
|
4019
|
+
configs_table.c.component_id == component_id,
|
|
4020
|
+
configs_table.c.version == version,
|
|
4021
|
+
)
|
|
4022
|
+
).scalar_one_or_none()
|
|
4023
|
+
|
|
4024
|
+
if stage is None:
|
|
4025
|
+
return False
|
|
4026
|
+
|
|
4027
|
+
# Only published configs can be set as current
|
|
4028
|
+
if stage != "published":
|
|
4029
|
+
raise ValueError(
|
|
4030
|
+
f"Cannot set draft config {component_id} v{version} as current. "
|
|
4031
|
+
"Only published configs can be current."
|
|
4032
|
+
)
|
|
4033
|
+
|
|
4034
|
+
# Update pointer
|
|
4035
|
+
result = sess.execute(
|
|
4036
|
+
components_table.update()
|
|
4037
|
+
.where(components_table.c.component_id == component_id)
|
|
4038
|
+
.values(current_version=version, updated_at=int(time.time()))
|
|
4039
|
+
)
|
|
4040
|
+
|
|
4041
|
+
if result.rowcount == 0:
|
|
4042
|
+
return False
|
|
4043
|
+
|
|
4044
|
+
log_debug(f"Set {component_id} current version to {version}")
|
|
4045
|
+
return True
|
|
4046
|
+
|
|
4047
|
+
except Exception as e:
|
|
4048
|
+
log_error(f"Error setting current version: {e}")
|
|
4049
|
+
raise
|
|
4050
|
+
|
|
4051
|
+
# --- Component Links ---
|
|
4052
|
+
def get_links(
|
|
4053
|
+
self,
|
|
4054
|
+
component_id: str,
|
|
4055
|
+
version: int,
|
|
4056
|
+
link_kind: Optional[str] = None,
|
|
4057
|
+
) -> List[Dict[str, Any]]:
|
|
4058
|
+
"""Get links for a config version.
|
|
4059
|
+
|
|
4060
|
+
Args:
|
|
4061
|
+
component_id: The component ID.
|
|
4062
|
+
version: The config version.
|
|
4063
|
+
link_kind: Optional filter by link kind (member|step).
|
|
4064
|
+
|
|
4065
|
+
Returns:
|
|
4066
|
+
List of link dictionaries, ordered by position.
|
|
4067
|
+
"""
|
|
4068
|
+
try:
|
|
4069
|
+
table = self._get_table(table_type="component_links")
|
|
4070
|
+
if table is None:
|
|
4071
|
+
return []
|
|
4072
|
+
|
|
4073
|
+
with self.Session() as sess:
|
|
4074
|
+
stmt = (
|
|
4075
|
+
select(table)
|
|
4076
|
+
.where(
|
|
4077
|
+
table.c.parent_component_id == component_id,
|
|
4078
|
+
table.c.parent_version == version,
|
|
4079
|
+
)
|
|
4080
|
+
.order_by(table.c.position)
|
|
4081
|
+
)
|
|
4082
|
+
if link_kind is not None:
|
|
4083
|
+
stmt = stmt.where(table.c.link_kind == link_kind)
|
|
4084
|
+
|
|
4085
|
+
rows = sess.execute(stmt).mappings().all()
|
|
4086
|
+
return [dict(r) for r in rows]
|
|
4087
|
+
|
|
4088
|
+
except Exception as e:
|
|
4089
|
+
log_error(f"Error getting links: {e}")
|
|
4090
|
+
raise
|
|
4091
|
+
|
|
4092
|
+
def get_dependents(
|
|
4093
|
+
self,
|
|
4094
|
+
component_id: str,
|
|
4095
|
+
version: Optional[int] = None,
|
|
4096
|
+
) -> List[Dict[str, Any]]:
|
|
4097
|
+
"""Find all components that reference this component.
|
|
4098
|
+
|
|
4099
|
+
Args:
|
|
4100
|
+
component_id: The component ID to find dependents of.
|
|
4101
|
+
version: Optional specific version. If None, finds links to any version.
|
|
4102
|
+
|
|
4103
|
+
Returns:
|
|
4104
|
+
List of link dictionaries showing what depends on this component.
|
|
4105
|
+
"""
|
|
4106
|
+
try:
|
|
4107
|
+
table = self._get_table(table_type="component_links")
|
|
4108
|
+
if table is None:
|
|
4109
|
+
return []
|
|
4110
|
+
|
|
4111
|
+
with self.Session() as sess:
|
|
4112
|
+
stmt = select(table).where(table.c.child_component_id == component_id)
|
|
4113
|
+
if version is not None:
|
|
4114
|
+
stmt = stmt.where(table.c.child_version == version)
|
|
4115
|
+
|
|
4116
|
+
rows = sess.execute(stmt).mappings().all()
|
|
4117
|
+
return [dict(r) for r in rows]
|
|
4118
|
+
|
|
4119
|
+
except Exception as e:
|
|
4120
|
+
log_error(f"Error getting dependents: {e}")
|
|
4121
|
+
raise
|
|
4122
|
+
|
|
4123
|
+
def _resolve_version(
|
|
4124
|
+
self,
|
|
4125
|
+
component_id: str,
|
|
4126
|
+
version: Optional[int],
|
|
4127
|
+
) -> Optional[int]:
|
|
4128
|
+
"""Resolve a version number, handling None as 'current'.
|
|
4129
|
+
|
|
4130
|
+
Args:
|
|
4131
|
+
component_id: The component ID.
|
|
4132
|
+
version: Version number or None for current.
|
|
4133
|
+
|
|
4134
|
+
Returns:
|
|
4135
|
+
Resolved version number, or None if component missing/deleted or no current.
|
|
4136
|
+
"""
|
|
4137
|
+
if version is not None:
|
|
4138
|
+
return version
|
|
4139
|
+
|
|
4140
|
+
try:
|
|
4141
|
+
components_table = self._get_table(table_type="components")
|
|
4142
|
+
if components_table is None:
|
|
4143
|
+
return None
|
|
4144
|
+
|
|
4145
|
+
with self.Session() as sess:
|
|
4146
|
+
return sess.execute(
|
|
4147
|
+
select(components_table.c.current_version).where(
|
|
4148
|
+
components_table.c.component_id == component_id,
|
|
4149
|
+
components_table.c.deleted_at.is_(None),
|
|
4150
|
+
)
|
|
4151
|
+
).scalar_one_or_none()
|
|
4152
|
+
|
|
4153
|
+
except Exception as e:
|
|
4154
|
+
log_error(f"Error resolving version: {e}")
|
|
4155
|
+
raise
|
|
4156
|
+
|
|
4157
|
+
def load_component_graph(
|
|
4158
|
+
self,
|
|
4159
|
+
component_id: str,
|
|
4160
|
+
version: Optional[int] = None,
|
|
4161
|
+
label: Optional[str] = None,
|
|
4162
|
+
*,
|
|
4163
|
+
_visited: Optional[Set[Tuple[str, int]]] = None,
|
|
4164
|
+
_max_depth: int = 50,
|
|
4165
|
+
) -> Optional[Dict[str, Any]]:
|
|
4166
|
+
"""Load a component with its full resolved graph.
|
|
4167
|
+
|
|
4168
|
+
Handles cycles by returning a stub with cycle_detected=True.
|
|
4169
|
+
Has a max depth guard to prevent stack overflow.
|
|
4170
|
+
|
|
4171
|
+
Args:
|
|
4172
|
+
component_id: The component ID.
|
|
4173
|
+
version: Specific version or None for current.
|
|
4174
|
+
label: Optional label of the component.
|
|
4175
|
+
_visited: Internal cycle tracking (do not pass).
|
|
4176
|
+
_max_depth: Internal depth limit (do not pass).
|
|
4177
|
+
|
|
4178
|
+
Returns:
|
|
4179
|
+
Dictionary with component, config, children, and resolved_versions.
|
|
4180
|
+
Returns None if component not found or depth exceeded.
|
|
4181
|
+
"""
|
|
4182
|
+
try:
|
|
4183
|
+
if _max_depth <= 0:
|
|
4184
|
+
return None
|
|
4185
|
+
|
|
4186
|
+
component = self.get_component(component_id)
|
|
4187
|
+
if component is None:
|
|
4188
|
+
return None
|
|
4189
|
+
|
|
4190
|
+
resolved_version = self._resolve_version(component_id, version)
|
|
4191
|
+
if resolved_version is None:
|
|
4192
|
+
return None
|
|
4193
|
+
|
|
4194
|
+
# Cycle detection
|
|
4195
|
+
if _visited is None:
|
|
4196
|
+
_visited = set()
|
|
4197
|
+
|
|
4198
|
+
node_key = (component_id, resolved_version)
|
|
4199
|
+
if node_key in _visited:
|
|
4200
|
+
return {
|
|
4201
|
+
"component": component,
|
|
4202
|
+
"config": self.get_config(component_id, version=resolved_version),
|
|
4203
|
+
"children": [],
|
|
4204
|
+
"resolved_versions": {component_id: resolved_version},
|
|
4205
|
+
"cycle_detected": True,
|
|
4206
|
+
}
|
|
4207
|
+
_visited.add(node_key)
|
|
4208
|
+
|
|
4209
|
+
config = self.get_config(component_id, version=resolved_version)
|
|
4210
|
+
if config is None:
|
|
4211
|
+
return None
|
|
4212
|
+
|
|
4213
|
+
links = self.get_links(component_id, resolved_version)
|
|
4214
|
+
|
|
4215
|
+
children: List[Dict[str, Any]] = []
|
|
4216
|
+
resolved_versions: Dict[str, Optional[int]] = {component_id: resolved_version}
|
|
4217
|
+
|
|
4218
|
+
for link in links:
|
|
4219
|
+
child_id = link["child_component_id"]
|
|
4220
|
+
child_ver = link.get("child_version")
|
|
4221
|
+
|
|
4222
|
+
resolved_child_ver = self._resolve_version(child_id, child_ver)
|
|
4223
|
+
resolved_versions[child_id] = resolved_child_ver
|
|
4224
|
+
|
|
4225
|
+
if resolved_child_ver is None:
|
|
4226
|
+
children.append(
|
|
4227
|
+
{
|
|
4228
|
+
"link": link,
|
|
4229
|
+
"graph": None,
|
|
4230
|
+
"error": "child_version_unresolvable",
|
|
4231
|
+
}
|
|
4232
|
+
)
|
|
4233
|
+
continue
|
|
4234
|
+
|
|
4235
|
+
child_graph = self.load_component_graph(
|
|
4236
|
+
child_id,
|
|
4237
|
+
version=resolved_child_ver,
|
|
4238
|
+
_visited=_visited,
|
|
4239
|
+
_max_depth=_max_depth - 1,
|
|
4240
|
+
)
|
|
4241
|
+
|
|
4242
|
+
if child_graph:
|
|
4243
|
+
resolved_versions.update(child_graph.get("resolved_versions", {}))
|
|
4244
|
+
|
|
4245
|
+
children.append({"link": link, "graph": child_graph})
|
|
4246
|
+
|
|
4247
|
+
return {
|
|
4248
|
+
"component": component,
|
|
4249
|
+
"config": config,
|
|
4250
|
+
"children": children,
|
|
4251
|
+
"resolved_versions": resolved_versions,
|
|
4252
|
+
}
|
|
4253
|
+
|
|
4254
|
+
except Exception as e:
|
|
4255
|
+
log_error(f"Error loading component graph: {e}")
|
|
4256
|
+
raise
|
|
4257
|
+
|
|
4258
|
+
# -- Learning methods --
|
|
4259
|
+
def get_learning(
|
|
4260
|
+
self,
|
|
4261
|
+
learning_type: str,
|
|
4262
|
+
user_id: Optional[str] = None,
|
|
4263
|
+
agent_id: Optional[str] = None,
|
|
4264
|
+
team_id: Optional[str] = None,
|
|
4265
|
+
workflow_id: Optional[str] = None,
|
|
4266
|
+
session_id: Optional[str] = None,
|
|
4267
|
+
namespace: Optional[str] = None,
|
|
4268
|
+
entity_id: Optional[str] = None,
|
|
4269
|
+
entity_type: Optional[str] = None,
|
|
4270
|
+
) -> Optional[Dict[str, Any]]:
|
|
4271
|
+
"""Retrieve a learning record.
|
|
4272
|
+
|
|
4273
|
+
Args:
|
|
4274
|
+
learning_type: Type of learning ('user_profile', 'session_context', etc.)
|
|
4275
|
+
user_id: Filter by user ID.
|
|
4276
|
+
agent_id: Filter by agent ID.
|
|
4277
|
+
team_id: Filter by team ID.
|
|
4278
|
+
workflow_id: Filter by workflow ID.
|
|
4279
|
+
session_id: Filter by session ID.
|
|
4280
|
+
namespace: Filter by namespace ('user', 'global', or custom).
|
|
4281
|
+
entity_id: Filter by entity ID (for entity-specific learnings).
|
|
4282
|
+
entity_type: Filter by entity type ('person', 'company', etc.).
|
|
4283
|
+
|
|
4284
|
+
Returns:
|
|
4285
|
+
Dict with 'content' key containing the learning data, or None.
|
|
4286
|
+
"""
|
|
4287
|
+
try:
|
|
4288
|
+
table = self._get_table(table_type="learnings")
|
|
4289
|
+
if table is None:
|
|
4290
|
+
return None
|
|
4291
|
+
|
|
4292
|
+
with self.Session() as sess:
|
|
4293
|
+
stmt = select(table).where(table.c.learning_type == learning_type)
|
|
4294
|
+
|
|
4295
|
+
if user_id is not None:
|
|
4296
|
+
stmt = stmt.where(table.c.user_id == user_id)
|
|
4297
|
+
if agent_id is not None:
|
|
4298
|
+
stmt = stmt.where(table.c.agent_id == agent_id)
|
|
4299
|
+
if team_id is not None:
|
|
4300
|
+
stmt = stmt.where(table.c.team_id == team_id)
|
|
4301
|
+
if workflow_id is not None:
|
|
4302
|
+
stmt = stmt.where(table.c.workflow_id == workflow_id)
|
|
4303
|
+
if session_id is not None:
|
|
4304
|
+
stmt = stmt.where(table.c.session_id == session_id)
|
|
4305
|
+
if namespace is not None:
|
|
4306
|
+
stmt = stmt.where(table.c.namespace == namespace)
|
|
4307
|
+
if entity_id is not None:
|
|
4308
|
+
stmt = stmt.where(table.c.entity_id == entity_id)
|
|
4309
|
+
if entity_type is not None:
|
|
4310
|
+
stmt = stmt.where(table.c.entity_type == entity_type)
|
|
4311
|
+
|
|
4312
|
+
result = sess.execute(stmt).fetchone()
|
|
4313
|
+
if result is None:
|
|
4314
|
+
return None
|
|
4315
|
+
|
|
4316
|
+
row = dict(result._mapping)
|
|
4317
|
+
return {"content": row.get("content")}
|
|
4318
|
+
|
|
4319
|
+
except Exception as e:
|
|
4320
|
+
log_debug(f"Error retrieving learning: {e}")
|
|
4321
|
+
return None
|
|
4322
|
+
|
|
4323
|
+
def upsert_learning(
|
|
4324
|
+
self,
|
|
4325
|
+
id: str,
|
|
4326
|
+
learning_type: str,
|
|
4327
|
+
content: Dict[str, Any],
|
|
4328
|
+
user_id: Optional[str] = None,
|
|
4329
|
+
agent_id: Optional[str] = None,
|
|
4330
|
+
team_id: Optional[str] = None,
|
|
4331
|
+
workflow_id: Optional[str] = None,
|
|
4332
|
+
session_id: Optional[str] = None,
|
|
4333
|
+
namespace: Optional[str] = None,
|
|
4334
|
+
entity_id: Optional[str] = None,
|
|
4335
|
+
entity_type: Optional[str] = None,
|
|
4336
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
4337
|
+
) -> None:
|
|
4338
|
+
"""Insert or update a learning record.
|
|
4339
|
+
|
|
4340
|
+
Args:
|
|
4341
|
+
id: Unique identifier for the learning.
|
|
4342
|
+
learning_type: Type of learning ('user_profile', 'session_context', etc.)
|
|
4343
|
+
content: The learning content as a dict.
|
|
4344
|
+
user_id: Associated user ID.
|
|
4345
|
+
agent_id: Associated agent ID.
|
|
4346
|
+
team_id: Associated team ID.
|
|
4347
|
+
workflow_id: Associated workflow ID.
|
|
4348
|
+
session_id: Associated session ID.
|
|
4349
|
+
namespace: Namespace for scoping ('user', 'global', or custom).
|
|
4350
|
+
entity_id: Associated entity ID (for entity-specific learnings).
|
|
4351
|
+
entity_type: Entity type ('person', 'company', etc.).
|
|
4352
|
+
metadata: Optional metadata.
|
|
4353
|
+
"""
|
|
4354
|
+
try:
|
|
4355
|
+
table = self._get_table(table_type="learnings", create_table_if_not_found=True)
|
|
4356
|
+
if table is None:
|
|
4357
|
+
return
|
|
4358
|
+
|
|
4359
|
+
current_time = int(time.time())
|
|
4360
|
+
|
|
4361
|
+
with self.Session() as sess, sess.begin():
|
|
4362
|
+
stmt = postgresql.insert(table).values(
|
|
4363
|
+
learning_id=id,
|
|
4364
|
+
learning_type=learning_type,
|
|
4365
|
+
namespace=namespace,
|
|
4366
|
+
user_id=user_id,
|
|
4367
|
+
agent_id=agent_id,
|
|
4368
|
+
team_id=team_id,
|
|
4369
|
+
workflow_id=workflow_id,
|
|
4370
|
+
session_id=session_id,
|
|
4371
|
+
entity_id=entity_id,
|
|
4372
|
+
entity_type=entity_type,
|
|
4373
|
+
content=content,
|
|
4374
|
+
metadata=metadata,
|
|
4375
|
+
created_at=current_time,
|
|
4376
|
+
updated_at=current_time,
|
|
4377
|
+
)
|
|
4378
|
+
stmt = stmt.on_conflict_do_update(
|
|
4379
|
+
index_elements=["learning_id"],
|
|
4380
|
+
set_=dict(
|
|
4381
|
+
content=content,
|
|
4382
|
+
metadata=metadata,
|
|
4383
|
+
updated_at=current_time,
|
|
4384
|
+
),
|
|
4385
|
+
)
|
|
4386
|
+
sess.execute(stmt)
|
|
4387
|
+
|
|
4388
|
+
log_debug(f"Upserted learning: {id}")
|
|
4389
|
+
|
|
4390
|
+
except Exception as e:
|
|
4391
|
+
log_debug(f"Error upserting learning: {e}")
|
|
4392
|
+
|
|
4393
|
+
def delete_learning(self, id: str) -> bool:
|
|
4394
|
+
"""Delete a learning record.
|
|
4395
|
+
|
|
4396
|
+
Args:
|
|
4397
|
+
id: The learning ID to delete.
|
|
4398
|
+
|
|
4399
|
+
Returns:
|
|
4400
|
+
True if deleted, False otherwise.
|
|
4401
|
+
"""
|
|
4402
|
+
try:
|
|
4403
|
+
table = self._get_table(table_type="learnings")
|
|
4404
|
+
if table is None:
|
|
4405
|
+
return False
|
|
4406
|
+
|
|
4407
|
+
with self.Session() as sess, sess.begin():
|
|
4408
|
+
stmt = table.delete().where(table.c.learning_id == id)
|
|
4409
|
+
result = sess.execute(stmt)
|
|
4410
|
+
return result.rowcount > 0
|
|
4411
|
+
|
|
4412
|
+
except Exception as e:
|
|
4413
|
+
log_debug(f"Error deleting learning: {e}")
|
|
4414
|
+
return False
|
|
4415
|
+
|
|
4416
|
+
def get_learnings(
|
|
4417
|
+
self,
|
|
4418
|
+
learning_type: Optional[str] = None,
|
|
4419
|
+
user_id: Optional[str] = None,
|
|
4420
|
+
agent_id: Optional[str] = None,
|
|
4421
|
+
team_id: Optional[str] = None,
|
|
4422
|
+
workflow_id: Optional[str] = None,
|
|
4423
|
+
session_id: Optional[str] = None,
|
|
4424
|
+
namespace: Optional[str] = None,
|
|
4425
|
+
entity_id: Optional[str] = None,
|
|
4426
|
+
entity_type: Optional[str] = None,
|
|
4427
|
+
limit: Optional[int] = None,
|
|
4428
|
+
) -> List[Dict[str, Any]]:
|
|
4429
|
+
"""Get multiple learning records.
|
|
4430
|
+
|
|
4431
|
+
Args:
|
|
4432
|
+
learning_type: Filter by learning type.
|
|
4433
|
+
user_id: Filter by user ID.
|
|
4434
|
+
agent_id: Filter by agent ID.
|
|
4435
|
+
team_id: Filter by team ID.
|
|
4436
|
+
workflow_id: Filter by workflow ID.
|
|
4437
|
+
session_id: Filter by session ID.
|
|
4438
|
+
namespace: Filter by namespace ('user', 'global', or custom).
|
|
4439
|
+
entity_id: Filter by entity ID (for entity-specific learnings).
|
|
4440
|
+
entity_type: Filter by entity type ('person', 'company', etc.).
|
|
4441
|
+
limit: Maximum number of records to return.
|
|
4442
|
+
|
|
4443
|
+
Returns:
|
|
4444
|
+
List of learning records.
|
|
4445
|
+
"""
|
|
4446
|
+
try:
|
|
4447
|
+
table = self._get_table(table_type="learnings")
|
|
4448
|
+
if table is None:
|
|
4449
|
+
return []
|
|
4450
|
+
|
|
4451
|
+
with self.Session() as sess:
|
|
4452
|
+
stmt = select(table)
|
|
4453
|
+
|
|
4454
|
+
if learning_type is not None:
|
|
4455
|
+
stmt = stmt.where(table.c.learning_type == learning_type)
|
|
4456
|
+
if user_id is not None:
|
|
4457
|
+
stmt = stmt.where(table.c.user_id == user_id)
|
|
4458
|
+
if agent_id is not None:
|
|
4459
|
+
stmt = stmt.where(table.c.agent_id == agent_id)
|
|
4460
|
+
if team_id is not None:
|
|
4461
|
+
stmt = stmt.where(table.c.team_id == team_id)
|
|
4462
|
+
if workflow_id is not None:
|
|
4463
|
+
stmt = stmt.where(table.c.workflow_id == workflow_id)
|
|
4464
|
+
if session_id is not None:
|
|
4465
|
+
stmt = stmt.where(table.c.session_id == session_id)
|
|
4466
|
+
if namespace is not None:
|
|
4467
|
+
stmt = stmt.where(table.c.namespace == namespace)
|
|
4468
|
+
if entity_id is not None:
|
|
4469
|
+
stmt = stmt.where(table.c.entity_id == entity_id)
|
|
4470
|
+
if entity_type is not None:
|
|
4471
|
+
stmt = stmt.where(table.c.entity_type == entity_type)
|
|
4472
|
+
|
|
4473
|
+
stmt = stmt.order_by(table.c.updated_at.desc())
|
|
4474
|
+
|
|
4475
|
+
if limit is not None:
|
|
4476
|
+
stmt = stmt.limit(limit)
|
|
4477
|
+
|
|
4478
|
+
result = sess.execute(stmt).fetchall()
|
|
4479
|
+
return [dict(row._mapping) for row in result]
|
|
4480
|
+
|
|
4481
|
+
except Exception as e:
|
|
4482
|
+
log_debug(f"Error getting learnings: {e}")
|
|
4483
|
+
return []
|