fuju-trace-sql 0.1.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,475 @@
|
|
|
1
|
+
"""Direct SQL stores for Fuju Trace.
|
|
2
|
+
|
|
3
|
+
Each store owns private tables under a validated prefix. Source events and the
|
|
4
|
+
folded span projection change in one transaction. Text search is a portable
|
|
5
|
+
substring query; native FTS and vector indexes are separate future capabilities.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
import sqlite3
|
|
13
|
+
import threading
|
|
14
|
+
from contextlib import contextmanager
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Callable, Iterator, Mapping, Sequence
|
|
17
|
+
|
|
18
|
+
from fuju_trace.event import EventType, event_id
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"SQLiteTraceStore", "DuckDBTraceStore", "PostgreSQLTraceStore",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
_PREFIX = re.compile(r"^[a-z][a-z0-9_]{0,35}$")
|
|
25
|
+
_FIELDS = (
|
|
26
|
+
"parent_span_id", "status", "duration_ns", "input_tokens", "output_tokens",
|
|
27
|
+
"cache_read_tokens", "cache_write_tokens", "session_id", "tenant_id",
|
|
28
|
+
"external_trace_id", "external_span_id", "external_parent_span_id",
|
|
29
|
+
"external_session_id", "span_name", "display_name", "agent_name",
|
|
30
|
+
"tool_name", "model", "input_text", "output_text", "eval_score", "eval_label",
|
|
31
|
+
)
|
|
32
|
+
_ALIASES = {"projectId": "project_id", "callSite": "call_site"}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _json(value: Any) -> str:
|
|
36
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True,
|
|
37
|
+
separators=(",", ":"), allow_nan=False)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _key(value: Any) -> str:
|
|
41
|
+
return hashlib.sha256(str(value).encode("utf-8")).hexdigest()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _fold(events: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
|
45
|
+
ordered = sorted(events, key=lambda e: (int(e["seq"]), int(e["event_type"])))
|
|
46
|
+
first = ordered[0]
|
|
47
|
+
span: dict[str, Any] = {
|
|
48
|
+
"trace_id": first["trace_id"], "span_id": first["span_id"],
|
|
49
|
+
"has_start": False, "has_end": False, "event_count": len(ordered),
|
|
50
|
+
"ts": min(int(e["ts"]) for e in ordered), "logs": [], "attrs": {},
|
|
51
|
+
}
|
|
52
|
+
seen_logs: set[str] = set()
|
|
53
|
+
for event in ordered:
|
|
54
|
+
kind = int(event["event_type"])
|
|
55
|
+
span["has_start"] |= kind == 1
|
|
56
|
+
span["has_end"] |= kind == 2
|
|
57
|
+
for field in _FIELDS:
|
|
58
|
+
value = event.get(field)
|
|
59
|
+
if value is not None and (field not in {"span_name", "display_name"} or kind == 1):
|
|
60
|
+
span[field] = value
|
|
61
|
+
for log in event.get("logs") or []:
|
|
62
|
+
if log not in seen_logs:
|
|
63
|
+
seen_logs.add(log)
|
|
64
|
+
span["logs"].append(log)
|
|
65
|
+
span["attrs"].update(event.get("attrs") or {})
|
|
66
|
+
return span
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _search_text(span: Mapping[str, Any]) -> str:
|
|
70
|
+
# Display fields must not silently become search terms.
|
|
71
|
+
return "\n".join(str(value) for value in (
|
|
72
|
+
span.get("input_text"), span.get("output_text"), *(span.get("logs") or []),
|
|
73
|
+
) if value) or " "
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _like(value: str) -> str:
|
|
77
|
+
return "%" + value.replace("!", "!!").replace("%", "!%").replace("_", "!_") + "%"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class SQLTraceStore:
|
|
81
|
+
"""One tenant, one DB-API connection, guarded by a thread lock."""
|
|
82
|
+
|
|
83
|
+
dialect = ""
|
|
84
|
+
|
|
85
|
+
def __init__(self, connection: Any, *, tenant_id: str | int,
|
|
86
|
+
table_prefix: str = "fuju_trace") -> None:
|
|
87
|
+
if not _PREFIX.fullmatch(table_prefix):
|
|
88
|
+
raise ValueError("table_prefix must be a short lowercase SQL identifier")
|
|
89
|
+
if tenant_id is None or str(tenant_id) == "":
|
|
90
|
+
raise ValueError("tenant_id is required")
|
|
91
|
+
self._conn = connection
|
|
92
|
+
self._lock = threading.RLock()
|
|
93
|
+
self._closed = False
|
|
94
|
+
self._tenant = str(tenant_id)
|
|
95
|
+
self._tenant_key = _key(self._tenant)
|
|
96
|
+
self.prefix = table_prefix
|
|
97
|
+
self._config = f"{table_prefix}_config"
|
|
98
|
+
self._events = f"{table_prefix}_events"
|
|
99
|
+
self._spans = f"{table_prefix}_spans"
|
|
100
|
+
self._attrs = f"{table_prefix}_attrs"
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def _bind(self) -> str:
|
|
104
|
+
return "?" if self.dialect in ("sqlite", "duckdb") else "%s"
|
|
105
|
+
|
|
106
|
+
def _execute(self, cur: Any, sql: str, params: Sequence[Any] = ()) -> Any:
|
|
107
|
+
return cur.execute(sql.replace("?", self._bind), tuple(params))
|
|
108
|
+
|
|
109
|
+
@contextmanager
|
|
110
|
+
def _tx(self, *, write: bool = False) -> Iterator[Any]:
|
|
111
|
+
with self._lock:
|
|
112
|
+
if self._closed:
|
|
113
|
+
raise RuntimeError("trace store is closed")
|
|
114
|
+
# DuckDB cursor() creates another connection. An unnamed in-memory
|
|
115
|
+
# database would then appear empty to that connection.
|
|
116
|
+
cur = self._conn if self.dialect == "duckdb" else self._conn.cursor()
|
|
117
|
+
try:
|
|
118
|
+
if write and self.dialect == "sqlite":
|
|
119
|
+
cur.execute("BEGIN IMMEDIATE")
|
|
120
|
+
elif write and self.dialect == "duckdb":
|
|
121
|
+
cur.execute("BEGIN TRANSACTION")
|
|
122
|
+
yield cur
|
|
123
|
+
if self.dialect != "duckdb" or write:
|
|
124
|
+
self._conn.commit()
|
|
125
|
+
except Exception:
|
|
126
|
+
if self.dialect != "duckdb" or write:
|
|
127
|
+
self._conn.rollback()
|
|
128
|
+
raise
|
|
129
|
+
finally:
|
|
130
|
+
if cur is not self._conn:
|
|
131
|
+
cur.close()
|
|
132
|
+
|
|
133
|
+
def _tenant_for(self, requested: str | int | None) -> str:
|
|
134
|
+
if requested is not None and str(requested) != self._tenant:
|
|
135
|
+
raise ValueError("tenant_id does not match this store")
|
|
136
|
+
return self._tenant_key
|
|
137
|
+
|
|
138
|
+
def initialize(self) -> None:
|
|
139
|
+
"""Create a private, versioned schema; never mutate another prefix."""
|
|
140
|
+
with self._tx(write=True) as cur:
|
|
141
|
+
self._execute(cur, f"CREATE TABLE IF NOT EXISTS {self._config} (name TEXT PRIMARY KEY, value VARCHAR(255) NOT NULL)")
|
|
142
|
+
self._execute(cur, f"SELECT value FROM {self._config} WHERE name='schema_version'")
|
|
143
|
+
row = cur.fetchone()
|
|
144
|
+
if row is None:
|
|
145
|
+
self._execute(cur, f"INSERT INTO {self._config} (name,value) VALUES ('schema_version','1')")
|
|
146
|
+
elif row[0] != "1":
|
|
147
|
+
raise ValueError(f"unsupported {self.prefix} schema version")
|
|
148
|
+
self._execute(cur, f"""CREATE TABLE IF NOT EXISTS {self._events} (
|
|
149
|
+
tenant_key CHAR(64) NOT NULL, event_key CHAR(64) NOT NULL,
|
|
150
|
+
trace_key CHAR(64) NOT NULL, span_key CHAR(64) NOT NULL,
|
|
151
|
+
seq BIGINT NOT NULL, ts BIGINT NOT NULL, event_json TEXT NOT NULL,
|
|
152
|
+
PRIMARY KEY (tenant_key,event_key))""")
|
|
153
|
+
self._execute(cur, f"""CREATE TABLE IF NOT EXISTS {self._spans} (
|
|
154
|
+
tenant_key CHAR(64) NOT NULL, trace_key CHAR(64) NOT NULL,
|
|
155
|
+
span_key CHAR(64) NOT NULL, trace_id TEXT NOT NULL,
|
|
156
|
+
span_id TEXT NOT NULL, ts BIGINT NOT NULL,
|
|
157
|
+
agent_name TEXT, status INTEGER, session_key CHAR(64),
|
|
158
|
+
search_text TEXT NOT NULL, data TEXT NOT NULL,
|
|
159
|
+
PRIMARY KEY (tenant_key,trace_key,span_key))""")
|
|
160
|
+
self._execute(cur, f"""CREATE TABLE IF NOT EXISTS {self._attrs} (
|
|
161
|
+
tenant_key CHAR(64) NOT NULL, trace_key CHAR(64) NOT NULL,
|
|
162
|
+
span_key CHAR(64) NOT NULL, attr_key_hash CHAR(64) NOT NULL,
|
|
163
|
+
value_hash CHAR(64) NOT NULL, attr_key TEXT NOT NULL,
|
|
164
|
+
value_json TEXT NOT NULL,
|
|
165
|
+
PRIMARY KEY (tenant_key,trace_key,span_key,attr_key_hash))""")
|
|
166
|
+
self._index(cur, f"{self.prefix}_event_span_idx", self._events,
|
|
167
|
+
"(tenant_key,trace_key,span_key,seq)")
|
|
168
|
+
self._index(cur, f"{self.prefix}_span_time_idx", self._spans,
|
|
169
|
+
"(tenant_key,ts)")
|
|
170
|
+
self._index(cur, f"{self.prefix}_span_session_idx", self._spans,
|
|
171
|
+
"(tenant_key,session_key,ts)")
|
|
172
|
+
self._index(cur, f"{self.prefix}_attrs_lookup_idx", self._attrs,
|
|
173
|
+
"(tenant_key,attr_key_hash,value_hash)")
|
|
174
|
+
|
|
175
|
+
def _index(self, cur: Any, name: str, table: str, columns: str) -> None:
|
|
176
|
+
self._execute(cur, f"CREATE INDEX IF NOT EXISTS {name} ON {table} {columns}")
|
|
177
|
+
|
|
178
|
+
def ingest(self, events: Sequence[Mapping[str, Any]], *,
|
|
179
|
+
tenant_id: str | int | None = None) -> dict[str, int]:
|
|
180
|
+
tenant = self._tenant_for(tenant_id)
|
|
181
|
+
prepared: list[tuple[str, str, str, str, str, int, int, str]] = []
|
|
182
|
+
for raw in events:
|
|
183
|
+
event = dict(raw)
|
|
184
|
+
if event.get("tenant_id") is not None and str(event["tenant_id"]) != self._tenant:
|
|
185
|
+
raise ValueError("event tenant_id does not match this store")
|
|
186
|
+
kind, seq = int(event["event_type"]), int(event["seq"])
|
|
187
|
+
if kind not in (1, 2, 3, 4, 5) or not 0 <= seq < 2**63:
|
|
188
|
+
raise ValueError("invalid event_type or seq")
|
|
189
|
+
ext_id = str(event["ext_span_id"]) if event.get("ext_span_id") is not None else ""
|
|
190
|
+
if not ext_id:
|
|
191
|
+
raise ValueError("ext_span_id is required")
|
|
192
|
+
expected_id = event_id(ext_id, seq, EventType(kind))
|
|
193
|
+
if event.get("event_id") is not None and int(event["event_id"]) != expected_id:
|
|
194
|
+
raise ValueError("event_id does not match ext_span_id, seq and event_type")
|
|
195
|
+
trace_id = str(event["trace_id"]) if event.get("trace_id") is not None else ""
|
|
196
|
+
span_id = str(event["span_id"]) if event.get("span_id") is not None else ""
|
|
197
|
+
if not trace_id or not span_id:
|
|
198
|
+
raise ValueError("trace_id and span_id are required")
|
|
199
|
+
event["tenant_id"] = self._tenant
|
|
200
|
+
identity = _key(_json([ext_id, seq, kind]))
|
|
201
|
+
prepared.append((identity, _key(trace_id), _key(span_id),
|
|
202
|
+
trace_id, span_id, seq, int(event["ts"]), _json(event)))
|
|
203
|
+
if not prepared:
|
|
204
|
+
return {"ingested": 0}
|
|
205
|
+
group_ids = {(trace_key, span_key): (trace_id, span_id)
|
|
206
|
+
for _, trace_key, span_key, trace_id, span_id, _, _, _ in prepared}
|
|
207
|
+
groups = sorted(group_ids)
|
|
208
|
+
inserted = 0
|
|
209
|
+
with self._tx(write=True) as cur:
|
|
210
|
+
for trace_key, span_key in groups:
|
|
211
|
+
trace_id, span_id = group_ids[(trace_key, span_key)]
|
|
212
|
+
self._execute(cur, f"""INSERT INTO {self._spans}
|
|
213
|
+
(tenant_key,trace_key,span_key,trace_id,span_id,ts,search_text,data)
|
|
214
|
+
VALUES (?,?,?,?,?,0,' ','{{}}') ON CONFLICT DO NOTHING""",
|
|
215
|
+
(tenant, trace_key, span_key, trace_id, span_id))
|
|
216
|
+
lock = " FOR UPDATE" if self.dialect == "postgresql" else ""
|
|
217
|
+
self._execute(cur, f"""SELECT 1 FROM {self._spans}
|
|
218
|
+
WHERE tenant_key=? AND trace_key=? AND span_key=?{lock}""",
|
|
219
|
+
(tenant, trace_key, span_key))
|
|
220
|
+
cur.fetchone()
|
|
221
|
+
for identity, trace_key, span_key, _trace_id, _span_id, seq, ts, body in prepared:
|
|
222
|
+
self._execute(cur, f"""SELECT trace_key,span_key FROM {self._events}
|
|
223
|
+
WHERE tenant_key=? AND event_key=?""", (tenant, identity))
|
|
224
|
+
existing = cur.fetchone()
|
|
225
|
+
if existing is not None:
|
|
226
|
+
if (str(existing[0]), str(existing[1])) != (trace_key, span_key):
|
|
227
|
+
raise ValueError("duplicate event identity belongs to another span")
|
|
228
|
+
continue
|
|
229
|
+
self._execute(cur, f"""INSERT INTO {self._events}
|
|
230
|
+
(tenant_key,event_key,trace_key,span_key,seq,ts,event_json)
|
|
231
|
+
VALUES (?,?,?,?,?,?,?)""",
|
|
232
|
+
(tenant, identity, trace_key, span_key, seq, ts, body))
|
|
233
|
+
inserted += 1
|
|
234
|
+
for trace_key, span_key in groups:
|
|
235
|
+
self._execute(cur, f"""SELECT event_json FROM {self._events}
|
|
236
|
+
WHERE tenant_key=? AND trace_key=? AND span_key=?
|
|
237
|
+
ORDER BY seq,event_key""", (tenant, trace_key, span_key))
|
|
238
|
+
sources = [json.loads(row[0]) for row in cur.fetchall()]
|
|
239
|
+
if not sources:
|
|
240
|
+
self._execute(cur, f"""DELETE FROM {self._spans}
|
|
241
|
+
WHERE tenant_key=? AND trace_key=? AND span_key=? AND data='{{}}'""",
|
|
242
|
+
(tenant, trace_key, span_key))
|
|
243
|
+
continue
|
|
244
|
+
span = _fold(sources)
|
|
245
|
+
session = span.get("external_session_id") or span.get("session_id")
|
|
246
|
+
self._execute(cur, f"""UPDATE {self._spans} SET
|
|
247
|
+
ts=?,agent_name=?,status=?,session_key=?,search_text=?,data=?
|
|
248
|
+
WHERE tenant_key=? AND trace_key=? AND span_key=?""",
|
|
249
|
+
(span["ts"], span.get("agent_name"), span.get("status"),
|
|
250
|
+
_key(session) if session is not None else None,
|
|
251
|
+
_search_text(span), _json(span), tenant, trace_key, span_key))
|
|
252
|
+
self._execute(cur, f"""DELETE FROM {self._attrs}
|
|
253
|
+
WHERE tenant_key=? AND trace_key=? AND span_key=?""",
|
|
254
|
+
(tenant, trace_key, span_key))
|
|
255
|
+
for attr_key, value in sorted((span.get("attrs") or {}).items()):
|
|
256
|
+
encoded = _json(value)
|
|
257
|
+
self._execute(cur, f"""INSERT INTO {self._attrs}
|
|
258
|
+
(tenant_key,trace_key,span_key,attr_key_hash,value_hash,attr_key,value_json)
|
|
259
|
+
VALUES (?,?,?,?,?,?,?)""",
|
|
260
|
+
(tenant, trace_key, span_key, _key(attr_key), _key(encoded),
|
|
261
|
+
str(attr_key), encoded))
|
|
262
|
+
return {"ingested": inserted}
|
|
263
|
+
|
|
264
|
+
def _where(self, filters: Mapping[str, Any]) -> tuple[str, list[Any]]:
|
|
265
|
+
clauses = ["s.tenant_key=?", "s.data<>'{}'"]
|
|
266
|
+
params: list[Any] = [self._tenant_key]
|
|
267
|
+
attrs: dict[str, Any] = {}
|
|
268
|
+
for key, value in filters.items():
|
|
269
|
+
if key in {"tenant_id", "tenantId"}:
|
|
270
|
+
raise ValueError("tenant cannot be set in a filter")
|
|
271
|
+
if value is None and key != "attrs":
|
|
272
|
+
continue
|
|
273
|
+
if key in {"trace_id", "externalTraceId", "external_trace_id"}:
|
|
274
|
+
clauses.append("s.trace_key=?")
|
|
275
|
+
params.append(_key(value))
|
|
276
|
+
elif key in {"externalSessionId", "external_session_id"}:
|
|
277
|
+
clauses.append("s.session_key=?")
|
|
278
|
+
params.append(_key(value))
|
|
279
|
+
elif key == "agent_name":
|
|
280
|
+
clauses.append("s.agent_name=?")
|
|
281
|
+
params.append(value)
|
|
282
|
+
elif key == "status":
|
|
283
|
+
clauses.append("s.status=?")
|
|
284
|
+
params.append(value)
|
|
285
|
+
elif key in {"time_from", "time_to"}:
|
|
286
|
+
clauses.append(f"s.ts {'>=' if key == 'time_from' else '<='}?")
|
|
287
|
+
params.append(int(value))
|
|
288
|
+
elif key == "attrs":
|
|
289
|
+
if not isinstance(value, Mapping):
|
|
290
|
+
raise ValueError("filter.attrs must be an object")
|
|
291
|
+
attrs.update(value)
|
|
292
|
+
elif key in {"project_id", "projectId", "skill", "mode", "call_site", "callSite"}:
|
|
293
|
+
attrs[_ALIASES.get(key, key)] = value
|
|
294
|
+
else:
|
|
295
|
+
raise ValueError(f"unsupported search filter: {key}")
|
|
296
|
+
for key, value in sorted(attrs.items()):
|
|
297
|
+
encoded = _json(value)
|
|
298
|
+
clauses.append(f"""EXISTS (SELECT 1 FROM {self._attrs} a
|
|
299
|
+
WHERE a.tenant_key=s.tenant_key AND a.trace_key=s.trace_key
|
|
300
|
+
AND a.span_key=s.span_key AND a.attr_key_hash=?
|
|
301
|
+
AND a.value_hash=? AND a.attr_key=? AND a.value_json=?)""")
|
|
302
|
+
params.extend((_key(key), _key(encoded), str(key), encoded))
|
|
303
|
+
return " AND ".join(clauses), params
|
|
304
|
+
|
|
305
|
+
def search(self, query: Mapping[str, Any] | None = None, *,
|
|
306
|
+
tenant_id: str | int | None = None, **kwargs: Any) -> list[dict[str, Any]]:
|
|
307
|
+
self._tenant_for(tenant_id)
|
|
308
|
+
body = {**dict(query or {}), **kwargs}
|
|
309
|
+
if set(body) - {"text", "vector", "k", "filter"}:
|
|
310
|
+
raise ValueError("unsupported search field")
|
|
311
|
+
if body.get("vector") is not None:
|
|
312
|
+
raise NotImplementedError(f"{self.dialect} vector search is not configured")
|
|
313
|
+
text = body.get("text")
|
|
314
|
+
if not isinstance(text, str) or not text:
|
|
315
|
+
raise ValueError("search requires non-empty text")
|
|
316
|
+
k = int(body.get("k", 10))
|
|
317
|
+
if not 1 <= k <= 1000:
|
|
318
|
+
raise ValueError("k must be between 1 and 1000")
|
|
319
|
+
filters = body.get("filter") or {}
|
|
320
|
+
if not isinstance(filters, Mapping):
|
|
321
|
+
raise ValueError("filter must be an object")
|
|
322
|
+
where, params = self._where(filters)
|
|
323
|
+
with self._tx() as cur:
|
|
324
|
+
self._execute(cur, f"""SELECT s.data FROM {self._spans} s
|
|
325
|
+
WHERE {where} AND s.search_text LIKE ? ESCAPE '!'
|
|
326
|
+
ORDER BY s.ts DESC,s.trace_key,s.span_key LIMIT ?""",
|
|
327
|
+
(*params, _like(text), k))
|
|
328
|
+
return [dict(json.loads(row[0]), score=1.0) for row in cur.fetchall()]
|
|
329
|
+
|
|
330
|
+
def trace(self, trace_id: str | int, *, tenant_id: str | int | None = None) -> list[dict[str, Any]]:
|
|
331
|
+
tenant = self._tenant_for(tenant_id)
|
|
332
|
+
with self._tx() as cur:
|
|
333
|
+
self._execute(cur, f"""SELECT data FROM {self._spans}
|
|
334
|
+
WHERE tenant_key=? AND trace_key=? AND data<>'{{}}'
|
|
335
|
+
ORDER BY ts,span_key""", (tenant, _key(trace_id)))
|
|
336
|
+
return [json.loads(row[0]) for row in cur.fetchall()]
|
|
337
|
+
|
|
338
|
+
def span(self, trace_id: str | int, span_id: str | int, *,
|
|
339
|
+
tenant_id: str | int | None = None) -> dict[str, Any] | None:
|
|
340
|
+
tenant = self._tenant_for(tenant_id)
|
|
341
|
+
with self._tx() as cur:
|
|
342
|
+
self._execute(cur, f"""SELECT data FROM {self._spans}
|
|
343
|
+
WHERE tenant_key=? AND trace_key=? AND span_key=? AND data<>'{{}}'""",
|
|
344
|
+
(tenant, _key(trace_id), _key(span_id)))
|
|
345
|
+
row = cur.fetchone()
|
|
346
|
+
return json.loads(row[0]) if row else None
|
|
347
|
+
|
|
348
|
+
def list_spans(self, *, filters: Mapping[str, Any] | None = None,
|
|
349
|
+
limit: int = 100, cursor: int = 0,
|
|
350
|
+
tenant_id: str | int | None = None) -> dict[str, Any]:
|
|
351
|
+
self._tenant_for(tenant_id)
|
|
352
|
+
if not 1 <= limit <= 1000 or cursor < 0:
|
|
353
|
+
raise ValueError("limit must be 1..1000 and cursor nonnegative")
|
|
354
|
+
where, params = self._where(filters or {})
|
|
355
|
+
with self._tx() as cur:
|
|
356
|
+
self._execute(cur, f"SELECT COUNT(*) FROM {self._spans} s WHERE {where}", params)
|
|
357
|
+
total = int(cur.fetchone()[0])
|
|
358
|
+
self._execute(cur, f"""SELECT s.data FROM {self._spans} s WHERE {where}
|
|
359
|
+
ORDER BY s.ts,s.trace_key,s.span_key LIMIT ? OFFSET ?""",
|
|
360
|
+
(*params, limit, cursor))
|
|
361
|
+
return {"items": [json.loads(row[0]) for row in cur.fetchall()], "total": total}
|
|
362
|
+
|
|
363
|
+
def list_trace_ids(self, *, filters: Mapping[str, Any] | None = None,
|
|
364
|
+
limit: int = 20, cursor: int = 0,
|
|
365
|
+
tenant_id: str | int | None = None) -> dict[str, Any]:
|
|
366
|
+
self._tenant_for(tenant_id)
|
|
367
|
+
if not 1 <= limit <= 1000 or cursor < 0:
|
|
368
|
+
raise ValueError("limit must be 1..1000 and cursor nonnegative")
|
|
369
|
+
where, params = self._where(filters or {})
|
|
370
|
+
with self._tx() as cur:
|
|
371
|
+
self._execute(cur, f"SELECT COUNT(DISTINCT s.trace_key) FROM {self._spans} s WHERE {where}", params)
|
|
372
|
+
total = int(cur.fetchone()[0])
|
|
373
|
+
self._execute(cur, f"""SELECT MIN(s.trace_id) FROM {self._spans} s WHERE {where}
|
|
374
|
+
GROUP BY s.trace_key ORDER BY MIN(s.ts),s.trace_key LIMIT ? OFFSET ?""",
|
|
375
|
+
(*params, limit, cursor))
|
|
376
|
+
return {"items": [str(row[0]) for row in cur.fetchall()], "total": total}
|
|
377
|
+
|
|
378
|
+
def capabilities(self) -> dict[str, Any]:
|
|
379
|
+
return {"backend": self.dialect, "text": "substring_scan",
|
|
380
|
+
"vector": None, "hybrid": None, "attrs_filter": "exact_sql",
|
|
381
|
+
"transactional_projection": True, "session_filter": True}
|
|
382
|
+
|
|
383
|
+
def ping(self) -> bool:
|
|
384
|
+
with self._tx() as cur:
|
|
385
|
+
self._execute(cur, f"SELECT value FROM {self._config} WHERE name='schema_version'")
|
|
386
|
+
row = cur.fetchone()
|
|
387
|
+
if row is None or row[0] != "1":
|
|
388
|
+
return False
|
|
389
|
+
for table in (self._events, self._spans, self._attrs):
|
|
390
|
+
self._execute(cur, f"SELECT 1 FROM {table} LIMIT 0")
|
|
391
|
+
return True
|
|
392
|
+
|
|
393
|
+
def close(self) -> None:
|
|
394
|
+
with self._lock:
|
|
395
|
+
if not self._closed:
|
|
396
|
+
self._closed = True
|
|
397
|
+
self._conn.close()
|
|
398
|
+
|
|
399
|
+
def __enter__(self) -> SQLTraceStore:
|
|
400
|
+
return self
|
|
401
|
+
|
|
402
|
+
def __exit__(self, *_: Any) -> None:
|
|
403
|
+
self.close()
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
class SQLiteTraceStore(SQLTraceStore):
|
|
407
|
+
dialect = "sqlite"
|
|
408
|
+
|
|
409
|
+
@classmethod
|
|
410
|
+
def open(cls, path: str | Path, *, tenant_id: str | int,
|
|
411
|
+
initialize: bool = False, table_prefix: str = "fuju_trace") -> SQLiteTraceStore:
|
|
412
|
+
conn = sqlite3.connect(str(path), timeout=30, check_same_thread=False)
|
|
413
|
+
conn.execute("PRAGMA busy_timeout=30000")
|
|
414
|
+
if str(path) != ":memory:":
|
|
415
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
416
|
+
store = cls(conn, tenant_id=tenant_id, table_prefix=table_prefix)
|
|
417
|
+
try:
|
|
418
|
+
if initialize:
|
|
419
|
+
store.initialize()
|
|
420
|
+
return store
|
|
421
|
+
except Exception:
|
|
422
|
+
store.close()
|
|
423
|
+
raise
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
class DuckDBTraceStore(SQLTraceStore):
|
|
427
|
+
dialect = "duckdb"
|
|
428
|
+
|
|
429
|
+
@classmethod
|
|
430
|
+
def open(cls, path: str | Path, *, tenant_id: str | int,
|
|
431
|
+
initialize: bool = False, table_prefix: str = "fuju_trace") -> DuckDBTraceStore:
|
|
432
|
+
try:
|
|
433
|
+
import duckdb
|
|
434
|
+
except ImportError as exc:
|
|
435
|
+
raise RuntimeError("install fuju-trace-sql[duckdb]") from exc
|
|
436
|
+
store = cls(duckdb.connect(str(path)), tenant_id=tenant_id, table_prefix=table_prefix)
|
|
437
|
+
try:
|
|
438
|
+
if initialize:
|
|
439
|
+
store.initialize()
|
|
440
|
+
return store
|
|
441
|
+
except Exception:
|
|
442
|
+
store.close()
|
|
443
|
+
raise
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
class PostgreSQLTraceStore(SQLTraceStore):
|
|
447
|
+
dialect = "postgresql"
|
|
448
|
+
|
|
449
|
+
@classmethod
|
|
450
|
+
def open(cls, dsn: str | None = None, *, tenant_id: str | int,
|
|
451
|
+
connection_params: Mapping[str, Any] | None = None,
|
|
452
|
+
initialize: bool = False, table_prefix: str = "fuju_trace",
|
|
453
|
+
connection_factory: Callable[..., Any] | None = None) -> PostgreSQLTraceStore:
|
|
454
|
+
if (dsn is None) == (connection_params is None):
|
|
455
|
+
raise ValueError("provide PostgreSQL dsn or connection_params")
|
|
456
|
+
if dsn == "" or connection_params == {}:
|
|
457
|
+
raise ValueError("PostgreSQL connection target must not be empty")
|
|
458
|
+
if connection_factory is None:
|
|
459
|
+
try:
|
|
460
|
+
import psycopg2
|
|
461
|
+
except ImportError as exc:
|
|
462
|
+
raise RuntimeError("install fuju-trace-sql[postgresql]") from exc
|
|
463
|
+
connection_factory = psycopg2.connect
|
|
464
|
+
conn = (connection_factory(dsn) if dsn is not None
|
|
465
|
+
else connection_factory(**dict(connection_params)))
|
|
466
|
+
if hasattr(conn, "autocommit"):
|
|
467
|
+
conn.autocommit = False
|
|
468
|
+
store = cls(conn, tenant_id=tenant_id, table_prefix=table_prefix)
|
|
469
|
+
try:
|
|
470
|
+
if initialize:
|
|
471
|
+
store.initialize()
|
|
472
|
+
return store
|
|
473
|
+
except Exception:
|
|
474
|
+
store.close()
|
|
475
|
+
raise
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fuju-trace-sql
|
|
3
|
+
Version: 0.1.10
|
|
4
|
+
Summary: SQLite, DuckDB and PostgreSQL storage adapters for Fuju Trace
|
|
5
|
+
Project-URL: Homepage, https://github.com/vibeinging/fuju-trace
|
|
6
|
+
Project-URL: Repository, https://github.com/vibeinging/fuju-trace
|
|
7
|
+
Project-URL: Issues, https://github.com/vibeinging/fuju-trace/issues
|
|
8
|
+
Author: Fuju Trace
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Requires-Dist: fuju-trace==0.1.10
|
|
12
|
+
Provides-Extra: duckdb
|
|
13
|
+
Requires-Dist: duckdb<3,>=1.0; extra == 'duckdb'
|
|
14
|
+
Provides-Extra: postgresql
|
|
15
|
+
Requires-Dist: psycopg2-binary<3,>=2.9.5; extra == 'postgresql'
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# Fuju Trace SQL adapters
|
|
19
|
+
|
|
20
|
+
This package adds three direct database adapters to the Fuju Trace Python
|
|
21
|
+
SDK: SQLite, DuckDB, and PostgreSQL. They have the same event ingestion
|
|
22
|
+
and basic read API as the VexDB adapter, but their current text search is a
|
|
23
|
+
portable **substring scan**. They do not provide BM25, ANN, vector writes, or
|
|
24
|
+
hybrid search. Use VexDB or the local TraceDB when those search features matter.
|
|
25
|
+
|
|
26
|
+
Install an adapter through a Fuju Trace extra:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
python -m pip install 'fuju-trace[sqlite]==0.1.10'
|
|
30
|
+
python -m pip install 'fuju-trace[duckdb]==0.1.10'
|
|
31
|
+
python -m pip install 'fuju-trace[postgresql]==0.1.10'
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Install from this checkout
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
python -m pip install -e ./fuju-trace-sdk/python -e ./fuju-trace-sql
|
|
38
|
+
# Add one or more optional drivers when needed:
|
|
39
|
+
python -m pip install 'duckdb>=1.0,<3' 'psycopg2-binary>=2.9.5,<3'
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
SQLite uses Python's standard library. The `[duckdb]` and `[postgresql]`
|
|
43
|
+
extras install their respective database drivers.
|
|
44
|
+
|
|
45
|
+
## Connect
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from fuju_trace import DbExporter, Tracer, connect
|
|
49
|
+
|
|
50
|
+
with connect(sqlite_path="./trace.sqlite", tenant_id=1, initialize=True) as db:
|
|
51
|
+
tracer = Tracer(exporter=DbExporter(db, tenant_id=1), node_id=1)
|
|
52
|
+
with tracer.trace("request", session_id=42, tenant_id=1) as trace:
|
|
53
|
+
with trace.span("tool call") as span:
|
|
54
|
+
span.log("query failed")
|
|
55
|
+
tracer.close()
|
|
56
|
+
print(db.search(text="failed", k=10))
|
|
57
|
+
print(db.list_spans(filters={"externalSessionId": 42}))
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Other connection forms:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
connect(duckdb_path="./trace.duckdb", tenant_id=1, initialize=True)
|
|
64
|
+
connect(postgresql_dsn="dbname=app user=trace", tenant_id=1, initialize=True)
|
|
65
|
+
connect(postgresql_params={"host": "localhost", "dbname": "app",
|
|
66
|
+
"user": "trace"}, tenant_id=1, initialize=True)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Supply credentials from your secret manager or environment, never from source
|
|
70
|
+
control. Set `initialize=True` only for first setup; it creates private
|
|
71
|
+
`fuju_trace_*` tables and indexes. Use `table_prefix` to isolate installations.
|
|
72
|
+
Opening a database without the schema requires initialization first.
|
|
73
|
+
|
|
74
|
+
## API and limits
|
|
75
|
+
|
|
76
|
+
- `ingest(events)` is transactional and deduplicates
|
|
77
|
+
`(ext_span_id, seq, event_type)` within the bound tenant. The source event
|
|
78
|
+
and folded span are committed together.
|
|
79
|
+
- `trace(id)`, `span(trace_id, span_id)`, `list_spans(filters=...)`, and
|
|
80
|
+
`list_trace_ids(filters=...)` read the folded model.
|
|
81
|
+
- `search(text=..., k=..., filter=...)` searches log and input/output text.
|
|
82
|
+
Supported filters include trace ID, session ID, time, agent, status, and
|
|
83
|
+
exact attributes. `%` and `_` in a query are treated literally.
|
|
84
|
+
- `capabilities()` reports `text="substring_scan"` and `vector=None`.
|
|
85
|
+
Passing a vector raises `NotImplementedError`.
|
|
86
|
+
- Each store binds one tenant and serializes one connection across threads.
|
|
87
|
+
Give each process its own connection and each writer process a distinct
|
|
88
|
+
`node_id` (0–1023).
|
|
89
|
+
- SQLite enables WAL for file databases; concurrent writes still serialize.
|
|
90
|
+
DuckDB is best used by one writer process for a database file. PostgreSQL
|
|
91
|
+
uses row locks when folding spans.
|
|
92
|
+
- The substring search has no text index and can scan many spans. It is a
|
|
93
|
+
correctness-first baseline, not a large-scale search benchmark result.
|
|
94
|
+
|
|
95
|
+
For DuckDB's file concurrency behavior and FTS index refresh requirement, see
|
|
96
|
+
its [concurrency](https://duckdb.org/docs/stable/connect/concurrency.html) and
|
|
97
|
+
[full-text search](https://duckdb.org/docs/current/core_extensions/full_text_search)
|
|
98
|
+
documentation.
|
|
99
|
+
|
|
100
|
+
## Verification
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
python -m unittest discover -s fuju-trace-sql/tests -p 'test_*.py'
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
SQLite and DuckDB tests use real database files. They cover writes, retries,
|
|
107
|
+
filters, tenant isolation, and reopen. The PostgreSQL test below uses a real
|
|
108
|
+
server and covers the same path plus two connections writing one span at the
|
|
109
|
+
same time. No MySQL server is needed or supported.
|
|
110
|
+
|
|
111
|
+
To start a temporary local PostgreSQL instance, run the following from the
|
|
112
|
+
repository root. It listens only on a private Unix socket and deletes its data
|
|
113
|
+
directory after the test. `initdb`, `pg_ctl`, `createdb`, and a Python with
|
|
114
|
+
`psycopg2` must be installed:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
FUJU_SQL_PYTHON=python3 ./fuju-trace-sql/tests/run_local_postgresql.sh
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
For an existing **disposable test database** with table create/drop permission,
|
|
121
|
+
set `POSTGRESQL_DSN` and run `python fuju-trace-sql/tests/live_smoke.py`.
|
|
122
|
+
The test creates and removes only four tables under its own random
|
|
123
|
+
`fuju_smoke_*` prefix. CI runs it against a PostgreSQL 16 service container.
|
|
124
|
+
These checks verify storage behavior; they do not benchmark search performance
|
|
125
|
+
or exercise VexDB-Lite vector extensions.
|
|
126
|
+
The [real database test report](../docs/reports/2026-09-25_sql-adapter-real-tests.md)
|
|
127
|
+
records the versions and results from the local run.
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
fuju_trace_sql/__init__.py,sha256=5Rk8awOzHvmKnGhjGAA2-QWyHjeeSskkB9qFHksKxZw,23141
|
|
2
|
+
fuju_trace_sql-0.1.10.dist-info/METADATA,sha256=Zdt3YLCPy0JKtgUtXuQnH-mxk1HEvTSiUgI0hLEq_oQ,5449
|
|
3
|
+
fuju_trace_sql-0.1.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
4
|
+
fuju_trace_sql-0.1.10.dist-info/RECORD,,
|