flock-core 0.5.0b65__py3-none-any.whl → 0.5.0b71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flock-core might be problematic. Click here for more details.
- flock/cli.py +74 -2
- flock/engines/dspy_engine.py +41 -5
- flock/examples.py +4 -1
- flock/frontend/README.md +15 -1
- flock/frontend/package-lock.json +2 -2
- flock/frontend/package.json +1 -1
- flock/frontend/src/App.tsx +74 -6
- flock/frontend/src/__tests__/e2e/critical-scenarios.test.tsx +4 -5
- flock/frontend/src/__tests__/integration/filtering-e2e.test.tsx +7 -3
- flock/frontend/src/components/filters/ArtifactTypeFilter.tsx +21 -0
- flock/frontend/src/components/filters/FilterFlyout.module.css +104 -0
- flock/frontend/src/components/filters/FilterFlyout.tsx +80 -0
- flock/frontend/src/components/filters/FilterPills.module.css +186 -45
- flock/frontend/src/components/filters/FilterPills.test.tsx +115 -99
- flock/frontend/src/components/filters/FilterPills.tsx +120 -44
- flock/frontend/src/components/filters/ProducerFilter.tsx +21 -0
- flock/frontend/src/components/filters/SavedFiltersControl.module.css +60 -0
- flock/frontend/src/components/filters/SavedFiltersControl.test.tsx +158 -0
- flock/frontend/src/components/filters/SavedFiltersControl.tsx +159 -0
- flock/frontend/src/components/filters/TagFilter.tsx +21 -0
- flock/frontend/src/components/filters/TimeRangeFilter.module.css +24 -0
- flock/frontend/src/components/filters/TimeRangeFilter.tsx +6 -1
- flock/frontend/src/components/filters/VisibilityFilter.tsx +21 -0
- flock/frontend/src/components/graph/GraphCanvas.tsx +24 -0
- flock/frontend/src/components/layout/DashboardLayout.css +13 -0
- flock/frontend/src/components/layout/DashboardLayout.tsx +8 -24
- flock/frontend/src/components/modules/HistoricalArtifactsModule.module.css +288 -0
- flock/frontend/src/components/modules/HistoricalArtifactsModule.tsx +460 -0
- flock/frontend/src/components/modules/HistoricalArtifactsModuleWrapper.tsx +13 -0
- flock/frontend/src/components/modules/ModuleRegistry.ts +7 -1
- flock/frontend/src/components/modules/registerModules.ts +9 -10
- flock/frontend/src/hooks/useModules.ts +11 -1
- flock/frontend/src/services/api.ts +140 -0
- flock/frontend/src/services/indexeddb.ts +56 -2
- flock/frontend/src/services/websocket.ts +129 -0
- flock/frontend/src/store/filterStore.test.ts +105 -185
- flock/frontend/src/store/filterStore.ts +173 -26
- flock/frontend/src/store/graphStore.test.ts +19 -0
- flock/frontend/src/store/graphStore.ts +166 -27
- flock/frontend/src/types/filters.ts +34 -1
- flock/frontend/src/types/graph.ts +7 -0
- flock/frontend/src/utils/artifacts.ts +24 -0
- flock/orchestrator.py +23 -1
- flock/service.py +146 -9
- flock/store.py +971 -24
- {flock_core-0.5.0b65.dist-info → flock_core-0.5.0b71.dist-info}/METADATA +26 -1
- {flock_core-0.5.0b65.dist-info → flock_core-0.5.0b71.dist-info}/RECORD +50 -43
- flock/frontend/src/components/filters/FilterBar.module.css +0 -29
- flock/frontend/src/components/filters/FilterBar.test.tsx +0 -133
- flock/frontend/src/components/filters/FilterBar.tsx +0 -33
- flock/frontend/src/components/modules/EventLogModule.test.tsx +0 -401
- flock/frontend/src/components/modules/EventLogModule.tsx +0 -396
- flock/frontend/src/components/modules/EventLogModuleWrapper.tsx +0 -17
- {flock_core-0.5.0b65.dist-info → flock_core-0.5.0b71.dist-info}/WHEEL +0 -0
- {flock_core-0.5.0b65.dist-info → flock_core-0.5.0b71.dist-info}/entry_points.txt +0 -0
- {flock_core-0.5.0b65.dist-info → flock_core-0.5.0b71.dist-info}/licenses/LICENSE +0 -0
flock/store.py
CHANGED
|
@@ -1,23 +1,112 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
"""Blackboard storage primitives.
|
|
4
|
+
"""Blackboard storage primitives and metadata envelopes.
|
|
5
5
|
|
|
6
|
+
Future backends should read the docstrings on :class:`FilterConfig`,
|
|
7
|
+
:class:`ConsumptionRecord`, and :class:`BlackboardStore` to understand the
|
|
8
|
+
contract expected by the REST layer and dashboard.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
6
14
|
from asyncio import Lock
|
|
7
15
|
from collections import defaultdict
|
|
8
|
-
from
|
|
16
|
+
from collections.abc import Iterable
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from datetime import datetime, timedelta, timezone
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, TypeVar
|
|
21
|
+
from uuid import UUID
|
|
22
|
+
|
|
23
|
+
import aiosqlite
|
|
24
|
+
from opentelemetry import trace
|
|
9
25
|
|
|
26
|
+
from flock.artifacts import Artifact
|
|
10
27
|
from flock.registry import type_registry
|
|
28
|
+
from flock.visibility import (
|
|
29
|
+
AfterVisibility,
|
|
30
|
+
LabelledVisibility,
|
|
31
|
+
PrivateVisibility,
|
|
32
|
+
PublicVisibility,
|
|
33
|
+
TenantVisibility,
|
|
34
|
+
Visibility,
|
|
35
|
+
)
|
|
11
36
|
|
|
12
37
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
from collections.abc import Iterable
|
|
16
|
-
from uuid import UUID
|
|
38
|
+
T = TypeVar("T")
|
|
39
|
+
tracer = trace.get_tracer(__name__)
|
|
17
40
|
|
|
18
|
-
|
|
41
|
+
ISO_DURATION_RE = re.compile(
|
|
42
|
+
r"^P(?:T?(?:(?P<hours>\d+)H)?(?:(?P<minutes>\d+)M)?(?:(?P<seconds>\d+)S)?)$"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _parse_iso_duration(value: str | None) -> timedelta:
|
|
47
|
+
if not value:
|
|
48
|
+
return timedelta(0)
|
|
49
|
+
match = ISO_DURATION_RE.match(value)
|
|
50
|
+
if not match:
|
|
51
|
+
return timedelta(0)
|
|
52
|
+
hours = int(match.group("hours") or 0)
|
|
53
|
+
minutes = int(match.group("minutes") or 0)
|
|
54
|
+
seconds = int(match.group("seconds") or 0)
|
|
55
|
+
return timedelta(hours=hours, minutes=minutes, seconds=seconds)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _deserialize_visibility(data: Any) -> Visibility:
|
|
59
|
+
if isinstance(data, Visibility):
|
|
60
|
+
return data
|
|
61
|
+
if not data:
|
|
62
|
+
return PublicVisibility()
|
|
63
|
+
kind = data.get("kind") if isinstance(data, dict) else None
|
|
64
|
+
if kind == "Public":
|
|
65
|
+
return PublicVisibility()
|
|
66
|
+
if kind == "Private":
|
|
67
|
+
return PrivateVisibility(agents=set(data.get("agents", [])))
|
|
68
|
+
if kind == "Labelled":
|
|
69
|
+
return LabelledVisibility(required_labels=set(data.get("required_labels", [])))
|
|
70
|
+
if kind == "Tenant":
|
|
71
|
+
return TenantVisibility(tenant_id=data.get("tenant_id"))
|
|
72
|
+
if kind == "After":
|
|
73
|
+
ttl = _parse_iso_duration(data.get("ttl"))
|
|
74
|
+
then_data = data.get("then") if isinstance(data, dict) else None
|
|
75
|
+
then_visibility = _deserialize_visibility(then_data) if then_data else None
|
|
76
|
+
return AfterVisibility(ttl=ttl, then=then_visibility)
|
|
77
|
+
return PublicVisibility()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(slots=True)
|
|
81
|
+
class ConsumptionRecord:
|
|
82
|
+
"""Historical record describing which agent consumed an artifact."""
|
|
83
|
+
|
|
84
|
+
artifact_id: UUID
|
|
85
|
+
consumer: str
|
|
86
|
+
run_id: str | None = None
|
|
87
|
+
correlation_id: str | None = None
|
|
88
|
+
consumed_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(slots=True)
|
|
92
|
+
class FilterConfig:
|
|
93
|
+
"""Shared filter configuration used by all stores."""
|
|
94
|
+
|
|
95
|
+
type_names: set[str] | None = None
|
|
96
|
+
produced_by: set[str] | None = None
|
|
97
|
+
correlation_id: str | None = None
|
|
98
|
+
tags: set[str] | None = None
|
|
99
|
+
visibility: set[str] | None = None
|
|
100
|
+
start: datetime | None = None
|
|
101
|
+
end: datetime | None = None
|
|
19
102
|
|
|
20
|
-
|
|
103
|
+
|
|
104
|
+
@dataclass(slots=True)
|
|
105
|
+
class ArtifactEnvelope:
|
|
106
|
+
"""Wrapper returned when ``embed_meta`` is requested."""
|
|
107
|
+
|
|
108
|
+
artifact: Artifact
|
|
109
|
+
consumptions: list[ConsumptionRecord] = field(default_factory=list)
|
|
21
110
|
|
|
22
111
|
|
|
23
112
|
class BlackboardStore:
|
|
@@ -27,13 +116,13 @@ class BlackboardStore:
|
|
|
27
116
|
async def get(self, artifact_id: UUID) -> Artifact | None:
|
|
28
117
|
raise NotImplementedError
|
|
29
118
|
|
|
30
|
-
async def list(self) ->
|
|
119
|
+
async def list(self) -> list[Artifact]:
|
|
31
120
|
raise NotImplementedError
|
|
32
121
|
|
|
33
|
-
async def list_by_type(self, type_name: str) ->
|
|
122
|
+
async def list_by_type(self, type_name: str) -> list[Artifact]:
|
|
34
123
|
raise NotImplementedError
|
|
35
124
|
|
|
36
|
-
async def get_by_type(self, artifact_type: type[T]) ->
|
|
125
|
+
async def get_by_type(self, artifact_type: type[T]) -> list[T]:
|
|
37
126
|
"""Get artifacts by Pydantic type, returning data already cast.
|
|
38
127
|
|
|
39
128
|
Args:
|
|
@@ -48,6 +137,39 @@ class BlackboardStore:
|
|
|
48
137
|
"""
|
|
49
138
|
raise NotImplementedError
|
|
50
139
|
|
|
140
|
+
async def record_consumptions(
|
|
141
|
+
self,
|
|
142
|
+
records: Iterable[ConsumptionRecord],
|
|
143
|
+
) -> None:
|
|
144
|
+
"""Persist one or more consumption events."""
|
|
145
|
+
raise NotImplementedError
|
|
146
|
+
|
|
147
|
+
async def query_artifacts(
|
|
148
|
+
self,
|
|
149
|
+
filters: FilterConfig | None = None,
|
|
150
|
+
*,
|
|
151
|
+
limit: int = 50,
|
|
152
|
+
offset: int = 0,
|
|
153
|
+
embed_meta: bool = False,
|
|
154
|
+
) -> tuple[list[Artifact | ArtifactEnvelope], int]:
|
|
155
|
+
"""Search artifacts with filtering and pagination."""
|
|
156
|
+
raise NotImplementedError
|
|
157
|
+
|
|
158
|
+
async def summarize_artifacts(
|
|
159
|
+
self,
|
|
160
|
+
filters: FilterConfig | None = None,
|
|
161
|
+
) -> dict[str, Any]:
|
|
162
|
+
"""Return aggregate artifact statistics for the given filters."""
|
|
163
|
+
raise NotImplementedError
|
|
164
|
+
|
|
165
|
+
async def agent_history_summary(
|
|
166
|
+
self,
|
|
167
|
+
agent_id: str,
|
|
168
|
+
filters: FilterConfig | None = None,
|
|
169
|
+
) -> dict[str, Any]:
|
|
170
|
+
"""Return produced/consumed counts for the specified agent."""
|
|
171
|
+
raise NotImplementedError
|
|
172
|
+
|
|
51
173
|
|
|
52
174
|
class InMemoryBlackboardStore(BlackboardStore):
|
|
53
175
|
"""Simple in-memory implementation suitable for local dev and tests."""
|
|
@@ -56,6 +178,7 @@ class InMemoryBlackboardStore(BlackboardStore):
|
|
|
56
178
|
self._lock = Lock()
|
|
57
179
|
self._by_id: dict[UUID, Artifact] = {}
|
|
58
180
|
self._by_type: dict[str, list[Artifact]] = defaultdict(list)
|
|
181
|
+
self._consumptions_by_artifact: dict[UUID, list[ConsumptionRecord]] = defaultdict(list)
|
|
59
182
|
|
|
60
183
|
async def publish(self, artifact: Artifact) -> None:
|
|
61
184
|
async with self._lock:
|
|
@@ -66,37 +189,861 @@ class InMemoryBlackboardStore(BlackboardStore):
|
|
|
66
189
|
async with self._lock:
|
|
67
190
|
return self._by_id.get(artifact_id)
|
|
68
191
|
|
|
69
|
-
async def list(self) ->
|
|
192
|
+
async def list(self) -> list[Artifact]:
|
|
70
193
|
async with self._lock:
|
|
71
194
|
return list(self._by_id.values())
|
|
72
195
|
|
|
73
|
-
async def list_by_type(self, type_name: str) ->
|
|
196
|
+
async def list_by_type(self, type_name: str) -> list[Artifact]:
|
|
74
197
|
async with self._lock:
|
|
75
198
|
canonical = type_registry.resolve_name(type_name)
|
|
76
199
|
return list(self._by_type.get(canonical, []))
|
|
77
200
|
|
|
78
|
-
async def get_by_type(self, artifact_type: type[T]) ->
|
|
79
|
-
"""Get artifacts by Pydantic type, returning data already cast.
|
|
80
|
-
|
|
81
|
-
Args:
|
|
82
|
-
artifact_type: The Pydantic model class (e.g., BugAnalysis)
|
|
83
|
-
|
|
84
|
-
Returns:
|
|
85
|
-
List of data objects of the specified type (not Artifact wrappers)
|
|
86
|
-
"""
|
|
201
|
+
async def get_by_type(self, artifact_type: type[T]) -> list[T]:
|
|
87
202
|
async with self._lock:
|
|
88
|
-
# Get canonical name from the type
|
|
89
203
|
canonical = type_registry.resolve_name(artifact_type.__name__)
|
|
90
204
|
artifacts = self._by_type.get(canonical, [])
|
|
91
|
-
# Reconstruct Pydantic models from payload dictionaries
|
|
92
205
|
return [artifact_type(**artifact.payload) for artifact in artifacts] # type: ignore
|
|
93
206
|
|
|
94
207
|
async def extend(self, artifacts: Iterable[Artifact]) -> None: # pragma: no cover - helper
|
|
95
208
|
for artifact in artifacts:
|
|
96
209
|
await self.publish(artifact)
|
|
97
210
|
|
|
211
|
+
async def record_consumptions(
|
|
212
|
+
self,
|
|
213
|
+
records: Iterable[ConsumptionRecord],
|
|
214
|
+
) -> None:
|
|
215
|
+
async with self._lock:
|
|
216
|
+
for record in records:
|
|
217
|
+
self._consumptions_by_artifact[record.artifact_id].append(record)
|
|
218
|
+
|
|
219
|
+
async def query_artifacts(
|
|
220
|
+
self,
|
|
221
|
+
filters: FilterConfig | None = None,
|
|
222
|
+
*,
|
|
223
|
+
limit: int = 50,
|
|
224
|
+
offset: int = 0,
|
|
225
|
+
embed_meta: bool = False,
|
|
226
|
+
) -> tuple[list[Artifact | ArtifactEnvelope], int]:
|
|
227
|
+
async with self._lock:
|
|
228
|
+
artifacts = list(self._by_id.values())
|
|
229
|
+
|
|
230
|
+
filters = filters or FilterConfig()
|
|
231
|
+
canonical: set[str] | None = None
|
|
232
|
+
if filters.type_names:
|
|
233
|
+
canonical = {type_registry.resolve_name(name) for name in filters.type_names}
|
|
234
|
+
|
|
235
|
+
visibility_filter = filters.visibility or set()
|
|
236
|
+
|
|
237
|
+
def _matches(artifact: Artifact) -> bool:
|
|
238
|
+
if canonical and artifact.type not in canonical:
|
|
239
|
+
return False
|
|
240
|
+
if filters.produced_by and artifact.produced_by not in filters.produced_by:
|
|
241
|
+
return False
|
|
242
|
+
if filters.correlation_id and (
|
|
243
|
+
artifact.correlation_id is None
|
|
244
|
+
or str(artifact.correlation_id) != filters.correlation_id
|
|
245
|
+
):
|
|
246
|
+
return False
|
|
247
|
+
if filters.tags and not filters.tags.issubset(artifact.tags):
|
|
248
|
+
return False
|
|
249
|
+
if visibility_filter and artifact.visibility.kind not in visibility_filter:
|
|
250
|
+
return False
|
|
251
|
+
if filters.start and artifact.created_at < filters.start:
|
|
252
|
+
return False
|
|
253
|
+
if filters.end and artifact.created_at > filters.end:
|
|
254
|
+
return False
|
|
255
|
+
return True
|
|
256
|
+
|
|
257
|
+
filtered = [artifact for artifact in artifacts if _matches(artifact)]
|
|
258
|
+
filtered.sort(key=lambda a: (a.created_at, a.id))
|
|
259
|
+
|
|
260
|
+
total = len(filtered)
|
|
261
|
+
offset = max(offset, 0)
|
|
262
|
+
if limit <= 0:
|
|
263
|
+
page = filtered[offset:]
|
|
264
|
+
else:
|
|
265
|
+
page = filtered[offset : offset + limit]
|
|
266
|
+
|
|
267
|
+
if not embed_meta:
|
|
268
|
+
return page, total
|
|
269
|
+
|
|
270
|
+
envelopes = [
|
|
271
|
+
ArtifactEnvelope(
|
|
272
|
+
artifact=artifact,
|
|
273
|
+
consumptions=list(self._consumptions_by_artifact.get(artifact.id, [])),
|
|
274
|
+
)
|
|
275
|
+
for artifact in page
|
|
276
|
+
]
|
|
277
|
+
return envelopes, total
|
|
278
|
+
|
|
279
|
+
async def summarize_artifacts(
|
|
280
|
+
self,
|
|
281
|
+
filters: FilterConfig | None = None,
|
|
282
|
+
) -> dict[str, Any]:
|
|
283
|
+
filters = filters or FilterConfig()
|
|
284
|
+
artifacts, total = await self.query_artifacts(
|
|
285
|
+
filters=filters,
|
|
286
|
+
limit=0,
|
|
287
|
+
offset=0,
|
|
288
|
+
embed_meta=False,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
by_type: dict[str, int] = {}
|
|
292
|
+
by_producer: dict[str, int] = {}
|
|
293
|
+
by_visibility: dict[str, int] = {}
|
|
294
|
+
tag_counts: dict[str, int] = {}
|
|
295
|
+
earliest: datetime | None = None
|
|
296
|
+
latest: datetime | None = None
|
|
297
|
+
|
|
298
|
+
for artifact in artifacts:
|
|
299
|
+
if not isinstance(artifact, Artifact):
|
|
300
|
+
raise TypeError("Expected Artifact instance")
|
|
301
|
+
by_type[artifact.type] = by_type.get(artifact.type, 0) + 1
|
|
302
|
+
by_producer[artifact.produced_by] = by_producer.get(artifact.produced_by, 0) + 1
|
|
303
|
+
kind = getattr(artifact.visibility, "kind", "Unknown")
|
|
304
|
+
by_visibility[kind] = by_visibility.get(kind, 0) + 1
|
|
305
|
+
for tag in artifact.tags:
|
|
306
|
+
tag_counts[tag] = tag_counts.get(tag, 0) + 1
|
|
307
|
+
if earliest is None or artifact.created_at < earliest:
|
|
308
|
+
earliest = artifact.created_at
|
|
309
|
+
if latest is None or artifact.created_at > latest:
|
|
310
|
+
latest = artifact.created_at
|
|
311
|
+
|
|
312
|
+
if earliest and latest:
|
|
313
|
+
span = latest - earliest
|
|
314
|
+
if span.days >= 2:
|
|
315
|
+
span_label = f"{span.days} days"
|
|
316
|
+
elif span.total_seconds() >= 3600:
|
|
317
|
+
hours = span.total_seconds() / 3600
|
|
318
|
+
span_label = f"{hours:.1f} hours"
|
|
319
|
+
elif span.total_seconds() > 0:
|
|
320
|
+
minutes = max(1, int(span.total_seconds() / 60))
|
|
321
|
+
span_label = f"{minutes} minutes"
|
|
322
|
+
else:
|
|
323
|
+
span_label = "moments"
|
|
324
|
+
else:
|
|
325
|
+
span_label = "empty"
|
|
326
|
+
|
|
327
|
+
return {
|
|
328
|
+
"total": total,
|
|
329
|
+
"by_type": by_type,
|
|
330
|
+
"by_producer": by_producer,
|
|
331
|
+
"by_visibility": by_visibility,
|
|
332
|
+
"tag_counts": tag_counts,
|
|
333
|
+
"earliest_created_at": earliest.isoformat() if earliest else None,
|
|
334
|
+
"latest_created_at": latest.isoformat() if latest else None,
|
|
335
|
+
"is_full_window": filters.start is None and filters.end is None,
|
|
336
|
+
"window_span_label": span_label,
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
async def agent_history_summary(
|
|
340
|
+
self,
|
|
341
|
+
agent_id: str,
|
|
342
|
+
filters: FilterConfig | None = None,
|
|
343
|
+
) -> dict[str, Any]:
|
|
344
|
+
filters = filters or FilterConfig()
|
|
345
|
+
envelopes, _ = await self.query_artifacts(
|
|
346
|
+
filters=filters,
|
|
347
|
+
limit=0,
|
|
348
|
+
offset=0,
|
|
349
|
+
embed_meta=True,
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
produced_total = 0
|
|
353
|
+
produced_by_type: dict[str, int] = defaultdict(int)
|
|
354
|
+
consumed_total = 0
|
|
355
|
+
consumed_by_type: dict[str, int] = defaultdict(int)
|
|
356
|
+
|
|
357
|
+
for envelope in envelopes:
|
|
358
|
+
if not isinstance(envelope, ArtifactEnvelope):
|
|
359
|
+
raise TypeError("Expected ArtifactEnvelope instance")
|
|
360
|
+
artifact = envelope.artifact
|
|
361
|
+
if artifact.produced_by == agent_id:
|
|
362
|
+
produced_total += 1
|
|
363
|
+
produced_by_type[artifact.type] += 1
|
|
364
|
+
for consumption in envelope.consumptions:
|
|
365
|
+
if consumption.consumer == agent_id:
|
|
366
|
+
consumed_total += 1
|
|
367
|
+
consumed_by_type[artifact.type] += 1
|
|
368
|
+
|
|
369
|
+
return {
|
|
370
|
+
"produced": {"total": produced_total, "by_type": dict(produced_by_type)},
|
|
371
|
+
"consumed": {"total": consumed_total, "by_type": dict(consumed_by_type)},
|
|
372
|
+
}
|
|
373
|
+
|
|
98
374
|
|
|
99
375
|
__all__ = [
|
|
100
376
|
"BlackboardStore",
|
|
101
377
|
"InMemoryBlackboardStore",
|
|
378
|
+
"SQLiteBlackboardStore",
|
|
102
379
|
]
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
class SQLiteBlackboardStore(BlackboardStore):
|
|
383
|
+
"""SQLite-backed implementation of :class:`BlackboardStore`."""
|
|
384
|
+
|
|
385
|
+
SCHEMA_VERSION = 2
|
|
386
|
+
|
|
387
|
+
def __init__(self, db_path: str, *, timeout: float = 5.0) -> None:
|
|
388
|
+
self._db_path = Path(db_path)
|
|
389
|
+
self._timeout = timeout
|
|
390
|
+
self._connection: aiosqlite.Connection | None = None
|
|
391
|
+
self._connection_lock = asyncio.Lock()
|
|
392
|
+
self._write_lock = asyncio.Lock()
|
|
393
|
+
self._schema_ready = False
|
|
394
|
+
|
|
395
|
+
async def publish(self, artifact: Artifact) -> None: # type: ignore[override]
|
|
396
|
+
with tracer.start_as_current_span("sqlite_store.publish"):
|
|
397
|
+
conn = await self._get_connection()
|
|
398
|
+
|
|
399
|
+
payload_json = json.dumps(artifact.payload)
|
|
400
|
+
visibility_json = json.dumps(artifact.visibility.model_dump(mode="json"))
|
|
401
|
+
tags_json = json.dumps(sorted(artifact.tags))
|
|
402
|
+
created_at = artifact.created_at.isoformat()
|
|
403
|
+
|
|
404
|
+
try:
|
|
405
|
+
canonical_type = type_registry.resolve_name(artifact.type)
|
|
406
|
+
except Exception:
|
|
407
|
+
canonical_type = artifact.type
|
|
408
|
+
|
|
409
|
+
record = {
|
|
410
|
+
"artifact_id": str(artifact.id),
|
|
411
|
+
"type": artifact.type,
|
|
412
|
+
"canonical_type": canonical_type,
|
|
413
|
+
"produced_by": artifact.produced_by,
|
|
414
|
+
"payload": payload_json,
|
|
415
|
+
"version": artifact.version,
|
|
416
|
+
"visibility": visibility_json,
|
|
417
|
+
"tags": tags_json,
|
|
418
|
+
"correlation_id": str(artifact.correlation_id) if artifact.correlation_id else None,
|
|
419
|
+
"partition_key": artifact.partition_key,
|
|
420
|
+
"created_at": created_at,
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async with self._write_lock:
|
|
424
|
+
await conn.execute(
|
|
425
|
+
"""
|
|
426
|
+
INSERT INTO artifacts (
|
|
427
|
+
artifact_id,
|
|
428
|
+
type,
|
|
429
|
+
canonical_type,
|
|
430
|
+
produced_by,
|
|
431
|
+
payload,
|
|
432
|
+
version,
|
|
433
|
+
visibility,
|
|
434
|
+
tags,
|
|
435
|
+
correlation_id,
|
|
436
|
+
partition_key,
|
|
437
|
+
created_at
|
|
438
|
+
) VALUES (
|
|
439
|
+
:artifact_id,
|
|
440
|
+
:type,
|
|
441
|
+
:canonical_type,
|
|
442
|
+
:produced_by,
|
|
443
|
+
:payload,
|
|
444
|
+
:version,
|
|
445
|
+
:visibility,
|
|
446
|
+
:tags,
|
|
447
|
+
:correlation_id,
|
|
448
|
+
:partition_key,
|
|
449
|
+
:created_at
|
|
450
|
+
)
|
|
451
|
+
ON CONFLICT(artifact_id) DO UPDATE SET
|
|
452
|
+
type=excluded.type,
|
|
453
|
+
canonical_type=excluded.canonical_type,
|
|
454
|
+
produced_by=excluded.produced_by,
|
|
455
|
+
payload=excluded.payload,
|
|
456
|
+
version=excluded.version,
|
|
457
|
+
visibility=excluded.visibility,
|
|
458
|
+
tags=excluded.tags,
|
|
459
|
+
correlation_id=excluded.correlation_id,
|
|
460
|
+
partition_key=excluded.partition_key,
|
|
461
|
+
created_at=excluded.created_at
|
|
462
|
+
""",
|
|
463
|
+
record,
|
|
464
|
+
)
|
|
465
|
+
await conn.commit()
|
|
466
|
+
|
|
467
|
+
async def record_consumptions( # type: ignore[override]
|
|
468
|
+
self,
|
|
469
|
+
records: Iterable[ConsumptionRecord],
|
|
470
|
+
) -> None:
|
|
471
|
+
with tracer.start_as_current_span("sqlite_store.record_consumptions"):
|
|
472
|
+
rows = [
|
|
473
|
+
(
|
|
474
|
+
str(record.artifact_id),
|
|
475
|
+
record.consumer,
|
|
476
|
+
record.run_id,
|
|
477
|
+
record.correlation_id,
|
|
478
|
+
record.consumed_at.isoformat(),
|
|
479
|
+
)
|
|
480
|
+
for record in records
|
|
481
|
+
]
|
|
482
|
+
if not rows:
|
|
483
|
+
return
|
|
484
|
+
|
|
485
|
+
conn = await self._get_connection()
|
|
486
|
+
async with self._write_lock:
|
|
487
|
+
await conn.executemany(
|
|
488
|
+
"""
|
|
489
|
+
INSERT OR REPLACE INTO artifact_consumptions (
|
|
490
|
+
artifact_id,
|
|
491
|
+
consumer,
|
|
492
|
+
run_id,
|
|
493
|
+
correlation_id,
|
|
494
|
+
consumed_at
|
|
495
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
496
|
+
""",
|
|
497
|
+
rows,
|
|
498
|
+
)
|
|
499
|
+
await conn.commit()
|
|
500
|
+
|
|
501
|
+
async def get(self, artifact_id: UUID) -> Artifact | None: # type: ignore[override]
|
|
502
|
+
with tracer.start_as_current_span("sqlite_store.get"):
|
|
503
|
+
conn = await self._get_connection()
|
|
504
|
+
cursor = await conn.execute(
|
|
505
|
+
"""
|
|
506
|
+
SELECT
|
|
507
|
+
artifact_id,
|
|
508
|
+
type,
|
|
509
|
+
canonical_type,
|
|
510
|
+
produced_by,
|
|
511
|
+
payload,
|
|
512
|
+
version,
|
|
513
|
+
visibility,
|
|
514
|
+
tags,
|
|
515
|
+
correlation_id,
|
|
516
|
+
partition_key,
|
|
517
|
+
created_at
|
|
518
|
+
FROM artifacts
|
|
519
|
+
WHERE artifact_id = ?
|
|
520
|
+
""",
|
|
521
|
+
(str(artifact_id),),
|
|
522
|
+
)
|
|
523
|
+
row = await cursor.fetchone()
|
|
524
|
+
await cursor.close()
|
|
525
|
+
if row is None:
|
|
526
|
+
return None
|
|
527
|
+
return self._row_to_artifact(row)
|
|
528
|
+
|
|
529
|
+
async def list(self) -> list[Artifact]: # type: ignore[override]
|
|
530
|
+
with tracer.start_as_current_span("sqlite_store.list"):
|
|
531
|
+
conn = await self._get_connection()
|
|
532
|
+
cursor = await conn.execute(
|
|
533
|
+
"""
|
|
534
|
+
SELECT
|
|
535
|
+
artifact_id,
|
|
536
|
+
type,
|
|
537
|
+
canonical_type,
|
|
538
|
+
produced_by,
|
|
539
|
+
payload,
|
|
540
|
+
version,
|
|
541
|
+
visibility,
|
|
542
|
+
tags,
|
|
543
|
+
correlation_id,
|
|
544
|
+
partition_key,
|
|
545
|
+
created_at
|
|
546
|
+
FROM artifacts
|
|
547
|
+
ORDER BY created_at ASC, rowid ASC
|
|
548
|
+
"""
|
|
549
|
+
)
|
|
550
|
+
rows = await cursor.fetchall()
|
|
551
|
+
await cursor.close()
|
|
552
|
+
return [self._row_to_artifact(row) for row in rows]
|
|
553
|
+
|
|
554
|
+
async def list_by_type(self, type_name: str) -> list[Artifact]: # type: ignore[override]
|
|
555
|
+
with tracer.start_as_current_span("sqlite_store.list_by_type"):
|
|
556
|
+
conn = await self._get_connection()
|
|
557
|
+
canonical = type_registry.resolve_name(type_name)
|
|
558
|
+
cursor = await conn.execute(
|
|
559
|
+
"""
|
|
560
|
+
SELECT
|
|
561
|
+
artifact_id,
|
|
562
|
+
type,
|
|
563
|
+
canonical_type,
|
|
564
|
+
produced_by,
|
|
565
|
+
payload,
|
|
566
|
+
version,
|
|
567
|
+
visibility,
|
|
568
|
+
tags,
|
|
569
|
+
correlation_id,
|
|
570
|
+
partition_key,
|
|
571
|
+
created_at
|
|
572
|
+
FROM artifacts
|
|
573
|
+
WHERE canonical_type = ?
|
|
574
|
+
ORDER BY created_at ASC, rowid ASC
|
|
575
|
+
""",
|
|
576
|
+
(canonical,),
|
|
577
|
+
)
|
|
578
|
+
rows = await cursor.fetchall()
|
|
579
|
+
await cursor.close()
|
|
580
|
+
return [self._row_to_artifact(row) for row in rows]
|
|
581
|
+
|
|
582
|
+
async def get_by_type(self, artifact_type: type[T]) -> list[T]: # type: ignore[override]
|
|
583
|
+
with tracer.start_as_current_span("sqlite_store.get_by_type"):
|
|
584
|
+
conn = await self._get_connection()
|
|
585
|
+
canonical = type_registry.resolve_name(artifact_type.__name__)
|
|
586
|
+
cursor = await conn.execute(
|
|
587
|
+
"""
|
|
588
|
+
SELECT payload
|
|
589
|
+
FROM artifacts
|
|
590
|
+
WHERE canonical_type = ?
|
|
591
|
+
ORDER BY created_at ASC, rowid ASC
|
|
592
|
+
""",
|
|
593
|
+
(canonical,),
|
|
594
|
+
)
|
|
595
|
+
rows = await cursor.fetchall()
|
|
596
|
+
await cursor.close()
|
|
597
|
+
results: list[T] = []
|
|
598
|
+
for row in rows:
|
|
599
|
+
payload = json.loads(row["payload"])
|
|
600
|
+
results.append(artifact_type(**payload)) # type: ignore[arg-type]
|
|
601
|
+
return results
|
|
602
|
+
|
|
603
|
+
async def query_artifacts(
|
|
604
|
+
self,
|
|
605
|
+
filters: FilterConfig | None = None,
|
|
606
|
+
*,
|
|
607
|
+
limit: int = 50,
|
|
608
|
+
offset: int = 0,
|
|
609
|
+
embed_meta: bool = False,
|
|
610
|
+
) -> tuple[list[Artifact | ArtifactEnvelope], int]:
|
|
611
|
+
filters = filters or FilterConfig()
|
|
612
|
+
conn = await self._get_connection()
|
|
613
|
+
|
|
614
|
+
where_clause, params = self._build_filters(filters)
|
|
615
|
+
count_query = f"SELECT COUNT(*) AS total FROM artifacts{where_clause}" # nosec B608 - where_clause contains only parameter placeholders from _build_filters
|
|
616
|
+
cursor = await conn.execute(count_query, tuple(params)) # nosec B608
|
|
617
|
+
total_row = await cursor.fetchone()
|
|
618
|
+
await cursor.close()
|
|
619
|
+
total = total_row["total"] if total_row else 0
|
|
620
|
+
|
|
621
|
+
query = f"""
|
|
622
|
+
SELECT
|
|
623
|
+
artifact_id,
|
|
624
|
+
type,
|
|
625
|
+
canonical_type,
|
|
626
|
+
produced_by,
|
|
627
|
+
payload,
|
|
628
|
+
version,
|
|
629
|
+
visibility,
|
|
630
|
+
tags,
|
|
631
|
+
correlation_id,
|
|
632
|
+
partition_key,
|
|
633
|
+
created_at
|
|
634
|
+
FROM artifacts
|
|
635
|
+
{where_clause}
|
|
636
|
+
ORDER BY created_at ASC, rowid ASC
|
|
637
|
+
""" # nosec B608 - where_clause contains only parameter placeholders from _build_filters
|
|
638
|
+
query_params: tuple[Any, ...]
|
|
639
|
+
if limit <= 0:
|
|
640
|
+
if offset > 0:
|
|
641
|
+
query += " LIMIT -1 OFFSET ?"
|
|
642
|
+
query_params = (*params, max(offset, 0))
|
|
643
|
+
else:
|
|
644
|
+
query_params = tuple(params)
|
|
645
|
+
else:
|
|
646
|
+
query += " LIMIT ? OFFSET ?"
|
|
647
|
+
query_params = (*params, limit, max(offset, 0))
|
|
648
|
+
|
|
649
|
+
cursor = await conn.execute(query, query_params)
|
|
650
|
+
rows = await cursor.fetchall()
|
|
651
|
+
await cursor.close()
|
|
652
|
+
artifacts = [self._row_to_artifact(row) for row in rows]
|
|
653
|
+
|
|
654
|
+
if not embed_meta or not artifacts:
|
|
655
|
+
return artifacts, total
|
|
656
|
+
|
|
657
|
+
artifact_ids = [str(artifact.id) for artifact in artifacts]
|
|
658
|
+
placeholders = ", ".join("?" for _ in artifact_ids)
|
|
659
|
+
consumption_query = f"""
|
|
660
|
+
SELECT
|
|
661
|
+
artifact_id,
|
|
662
|
+
consumer,
|
|
663
|
+
run_id,
|
|
664
|
+
correlation_id,
|
|
665
|
+
consumed_at
|
|
666
|
+
FROM artifact_consumptions
|
|
667
|
+
WHERE artifact_id IN ({placeholders})
|
|
668
|
+
ORDER BY consumed_at ASC
|
|
669
|
+
""" # nosec B608 - placeholders string contains only '?' characters
|
|
670
|
+
cursor = await conn.execute(consumption_query, artifact_ids)
|
|
671
|
+
consumption_rows = await cursor.fetchall()
|
|
672
|
+
await cursor.close()
|
|
673
|
+
|
|
674
|
+
consumptions_map: dict[UUID, list[ConsumptionRecord]] = defaultdict(list)
|
|
675
|
+
for row in consumption_rows:
|
|
676
|
+
artifact_uuid = UUID(row["artifact_id"])
|
|
677
|
+
consumptions_map[artifact_uuid].append(
|
|
678
|
+
ConsumptionRecord(
|
|
679
|
+
artifact_id=artifact_uuid,
|
|
680
|
+
consumer=row["consumer"],
|
|
681
|
+
run_id=row["run_id"],
|
|
682
|
+
correlation_id=row["correlation_id"],
|
|
683
|
+
consumed_at=datetime.fromisoformat(row["consumed_at"]),
|
|
684
|
+
)
|
|
685
|
+
)
|
|
686
|
+
|
|
687
|
+
envelopes: list[ArtifactEnvelope] = [
|
|
688
|
+
ArtifactEnvelope(
|
|
689
|
+
artifact=artifact,
|
|
690
|
+
consumptions=consumptions_map.get(artifact.id, []),
|
|
691
|
+
)
|
|
692
|
+
for artifact in artifacts
|
|
693
|
+
]
|
|
694
|
+
return envelopes, total
|
|
695
|
+
|
|
696
|
+
async def summarize_artifacts(
|
|
697
|
+
self,
|
|
698
|
+
filters: FilterConfig | None = None,
|
|
699
|
+
) -> dict[str, Any]:
|
|
700
|
+
filters = filters or FilterConfig()
|
|
701
|
+
conn = await self._get_connection()
|
|
702
|
+
|
|
703
|
+
where_clause, params = self._build_filters(filters)
|
|
704
|
+
params_tuple = tuple(params)
|
|
705
|
+
|
|
706
|
+
count_query = f"SELECT COUNT(*) AS total FROM artifacts{where_clause}" # nosec B608 - where_clause contains only parameter placeholders from _build_filters
|
|
707
|
+
cursor = await conn.execute(count_query, params_tuple) # nosec B608
|
|
708
|
+
total_row = await cursor.fetchone()
|
|
709
|
+
await cursor.close()
|
|
710
|
+
total = total_row["total"] if total_row else 0
|
|
711
|
+
|
|
712
|
+
by_type_query = f"""
|
|
713
|
+
SELECT canonical_type, COUNT(*) AS count
|
|
714
|
+
FROM artifacts
|
|
715
|
+
{where_clause}
|
|
716
|
+
GROUP BY canonical_type
|
|
717
|
+
""" # nosec B608 - where_clause contains only parameter placeholders from _build_filters
|
|
718
|
+
cursor = await conn.execute(by_type_query, params_tuple)
|
|
719
|
+
by_type_rows = await cursor.fetchall()
|
|
720
|
+
await cursor.close()
|
|
721
|
+
by_type = {row["canonical_type"]: row["count"] for row in by_type_rows}
|
|
722
|
+
|
|
723
|
+
by_producer_query = f"""
|
|
724
|
+
SELECT produced_by, COUNT(*) AS count
|
|
725
|
+
FROM artifacts
|
|
726
|
+
{where_clause}
|
|
727
|
+
GROUP BY produced_by
|
|
728
|
+
""" # nosec B608 - where_clause contains only parameter placeholders from _build_filters
|
|
729
|
+
cursor = await conn.execute(by_producer_query, params_tuple)
|
|
730
|
+
by_producer_rows = await cursor.fetchall()
|
|
731
|
+
await cursor.close()
|
|
732
|
+
by_producer = {row["produced_by"]: row["count"] for row in by_producer_rows}
|
|
733
|
+
|
|
734
|
+
by_visibility_query = f"""
|
|
735
|
+
SELECT json_extract(visibility, '$.kind') AS visibility_kind, COUNT(*) AS count
|
|
736
|
+
FROM artifacts
|
|
737
|
+
{where_clause}
|
|
738
|
+
GROUP BY json_extract(visibility, '$.kind')
|
|
739
|
+
""" # nosec B608 - where_clause contains only parameter placeholders from _build_filters
|
|
740
|
+
cursor = await conn.execute(by_visibility_query, params_tuple)
|
|
741
|
+
by_visibility_rows = await cursor.fetchall()
|
|
742
|
+
await cursor.close()
|
|
743
|
+
by_visibility = {
|
|
744
|
+
(row["visibility_kind"] or "Unknown"): row["count"] for row in by_visibility_rows
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
tag_query = f"""
|
|
748
|
+
SELECT json_each.value AS tag, COUNT(*) AS count
|
|
749
|
+
FROM artifacts
|
|
750
|
+
JOIN json_each(artifacts.tags)
|
|
751
|
+
{where_clause}
|
|
752
|
+
GROUP BY json_each.value
|
|
753
|
+
""" # nosec B608 - where_clause contains only parameter placeholders produced by _build_filters
|
|
754
|
+
cursor = await conn.execute(tag_query, params_tuple)
|
|
755
|
+
tag_rows = await cursor.fetchall()
|
|
756
|
+
await cursor.close()
|
|
757
|
+
tag_counts = {row["tag"]: row["count"] for row in tag_rows}
|
|
758
|
+
|
|
759
|
+
range_query = f"""
|
|
760
|
+
SELECT MIN(created_at) AS earliest, MAX(created_at) AS latest
|
|
761
|
+
FROM artifacts
|
|
762
|
+
{where_clause}
|
|
763
|
+
""" # nosec B608 - safe composition using parameterized where_clause
|
|
764
|
+
cursor = await conn.execute(range_query, params_tuple)
|
|
765
|
+
range_row = await cursor.fetchone()
|
|
766
|
+
await cursor.close()
|
|
767
|
+
earliest = range_row["earliest"] if range_row and range_row["earliest"] else None
|
|
768
|
+
latest = range_row["latest"] if range_row and range_row["latest"] else None
|
|
769
|
+
|
|
770
|
+
return {
|
|
771
|
+
"total": total,
|
|
772
|
+
"by_type": by_type,
|
|
773
|
+
"by_producer": by_producer,
|
|
774
|
+
"by_visibility": by_visibility,
|
|
775
|
+
"tag_counts": tag_counts,
|
|
776
|
+
"earliest_created_at": earliest,
|
|
777
|
+
"latest_created_at": latest,
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
async def agent_history_summary(
|
|
781
|
+
self,
|
|
782
|
+
agent_id: str,
|
|
783
|
+
filters: FilterConfig | None = None,
|
|
784
|
+
) -> dict[str, Any]:
|
|
785
|
+
filters = filters or FilterConfig()
|
|
786
|
+
conn = await self._get_connection()
|
|
787
|
+
|
|
788
|
+
produced_total = 0
|
|
789
|
+
produced_by_type: dict[str, int] = {}
|
|
790
|
+
|
|
791
|
+
if filters.produced_by and agent_id not in filters.produced_by:
|
|
792
|
+
produced_total = 0
|
|
793
|
+
else:
|
|
794
|
+
produced_filter = FilterConfig(
|
|
795
|
+
type_names=set(filters.type_names) if filters.type_names else None,
|
|
796
|
+
produced_by={agent_id},
|
|
797
|
+
correlation_id=filters.correlation_id,
|
|
798
|
+
tags=set(filters.tags) if filters.tags else None,
|
|
799
|
+
visibility=set(filters.visibility) if filters.visibility else None,
|
|
800
|
+
start=filters.start,
|
|
801
|
+
end=filters.end,
|
|
802
|
+
)
|
|
803
|
+
where_clause, params = self._build_filters(produced_filter)
|
|
804
|
+
produced_query = f"""
|
|
805
|
+
SELECT canonical_type, COUNT(*) AS count
|
|
806
|
+
FROM artifacts
|
|
807
|
+
{where_clause}
|
|
808
|
+
GROUP BY canonical_type
|
|
809
|
+
""" # nosec B608 - produced_filter yields parameter placeholders only
|
|
810
|
+
cursor = await conn.execute(produced_query, tuple(params))
|
|
811
|
+
rows = await cursor.fetchall()
|
|
812
|
+
await cursor.close()
|
|
813
|
+
produced_by_type = {row["canonical_type"]: row["count"] for row in rows}
|
|
814
|
+
produced_total = sum(produced_by_type.values())
|
|
815
|
+
|
|
816
|
+
where_clause, params = self._build_filters(filters, table_alias="a")
|
|
817
|
+
params_with_consumer = (*params, agent_id)
|
|
818
|
+
consumption_query = f"""
|
|
819
|
+
SELECT a.canonical_type AS canonical_type, COUNT(*) AS count
|
|
820
|
+
FROM artifact_consumptions c
|
|
821
|
+
JOIN artifacts a ON a.artifact_id = c.artifact_id
|
|
822
|
+
{where_clause}
|
|
823
|
+
{"AND" if where_clause else "WHERE"} c.consumer = ?
|
|
824
|
+
GROUP BY a.canonical_type
|
|
825
|
+
""" # nosec B608 - where_clause joins parameter placeholders only
|
|
826
|
+
cursor = await conn.execute(consumption_query, params_with_consumer)
|
|
827
|
+
consumption_rows = await cursor.fetchall()
|
|
828
|
+
await cursor.close()
|
|
829
|
+
|
|
830
|
+
consumed_by_type = {row["canonical_type"]: row["count"] for row in consumption_rows}
|
|
831
|
+
consumed_total = sum(consumed_by_type.values())
|
|
832
|
+
|
|
833
|
+
return {
|
|
834
|
+
"produced": {"total": produced_total, "by_type": produced_by_type},
|
|
835
|
+
"consumed": {"total": consumed_total, "by_type": consumed_by_type},
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
async def ensure_schema(self) -> None:
|
|
839
|
+
conn = await self._ensure_connection()
|
|
840
|
+
await self._apply_schema(conn)
|
|
841
|
+
|
|
842
|
+
async def close(self) -> None:
|
|
843
|
+
async with self._connection_lock:
|
|
844
|
+
if self._connection is not None:
|
|
845
|
+
await self._connection.close()
|
|
846
|
+
self._connection = None
|
|
847
|
+
self._schema_ready = False
|
|
848
|
+
|
|
849
|
+
async def vacuum(self) -> None:
|
|
850
|
+
"""Run SQLite VACUUM for maintenance."""
|
|
851
|
+
with tracer.start_as_current_span("sqlite_store.vacuum"):
|
|
852
|
+
conn = await self._get_connection()
|
|
853
|
+
async with self._write_lock:
|
|
854
|
+
await conn.execute("VACUUM")
|
|
855
|
+
await conn.commit()
|
|
856
|
+
|
|
857
|
+
async def delete_before(self, before: datetime) -> int:
|
|
858
|
+
"""Delete artifacts persisted before the given timestamp."""
|
|
859
|
+
with tracer.start_as_current_span("sqlite_store.delete_before"):
|
|
860
|
+
conn = await self._get_connection()
|
|
861
|
+
async with self._write_lock:
|
|
862
|
+
cursor = await conn.execute(
|
|
863
|
+
"DELETE FROM artifacts WHERE created_at < ?", (before.isoformat(),)
|
|
864
|
+
)
|
|
865
|
+
await conn.commit()
|
|
866
|
+
deleted = cursor.rowcount or 0
|
|
867
|
+
await cursor.close()
|
|
868
|
+
return deleted
|
|
869
|
+
|
|
870
|
+
async def _ensure_connection(self) -> aiosqlite.Connection:
|
|
871
|
+
async with self._connection_lock:
|
|
872
|
+
if self._connection is None:
|
|
873
|
+
self._db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
874
|
+
conn = await aiosqlite.connect(
|
|
875
|
+
str(self._db_path), timeout=self._timeout, isolation_level=None
|
|
876
|
+
)
|
|
877
|
+
conn.row_factory = aiosqlite.Row
|
|
878
|
+
await conn.execute("PRAGMA journal_mode=WAL;")
|
|
879
|
+
await conn.execute("PRAGMA synchronous=NORMAL;")
|
|
880
|
+
await conn.execute("PRAGMA foreign_keys=ON;")
|
|
881
|
+
self._connection = conn
|
|
882
|
+
self._schema_ready = False
|
|
883
|
+
return self._connection
|
|
884
|
+
|
|
885
|
+
async def _get_connection(self) -> aiosqlite.Connection:
|
|
886
|
+
conn = await self._ensure_connection()
|
|
887
|
+
if not self._schema_ready:
|
|
888
|
+
await self._apply_schema(conn)
|
|
889
|
+
return conn
|
|
890
|
+
|
|
891
|
+
async def _apply_schema(self, conn: aiosqlite.Connection) -> None:
|
|
892
|
+
async with self._connection_lock:
|
|
893
|
+
await conn.execute(
|
|
894
|
+
"""
|
|
895
|
+
CREATE TABLE IF NOT EXISTS schema_meta (
|
|
896
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
897
|
+
version INTEGER NOT NULL,
|
|
898
|
+
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
899
|
+
)
|
|
900
|
+
"""
|
|
901
|
+
)
|
|
902
|
+
await conn.execute(
|
|
903
|
+
"""
|
|
904
|
+
INSERT OR IGNORE INTO schema_meta (id, version)
|
|
905
|
+
VALUES (1, ?)
|
|
906
|
+
""",
|
|
907
|
+
(self.SCHEMA_VERSION,),
|
|
908
|
+
)
|
|
909
|
+
await conn.execute(
|
|
910
|
+
"""
|
|
911
|
+
CREATE TABLE IF NOT EXISTS artifacts (
|
|
912
|
+
artifact_id TEXT PRIMARY KEY,
|
|
913
|
+
type TEXT NOT NULL,
|
|
914
|
+
canonical_type TEXT NOT NULL,
|
|
915
|
+
produced_by TEXT NOT NULL,
|
|
916
|
+
payload TEXT NOT NULL,
|
|
917
|
+
version INTEGER NOT NULL,
|
|
918
|
+
visibility TEXT NOT NULL,
|
|
919
|
+
tags TEXT NOT NULL,
|
|
920
|
+
correlation_id TEXT,
|
|
921
|
+
partition_key TEXT,
|
|
922
|
+
created_at TEXT NOT NULL
|
|
923
|
+
)
|
|
924
|
+
"""
|
|
925
|
+
)
|
|
926
|
+
await conn.execute(
|
|
927
|
+
"""
|
|
928
|
+
CREATE INDEX IF NOT EXISTS idx_artifacts_canonical_type_created
|
|
929
|
+
ON artifacts(canonical_type, created_at)
|
|
930
|
+
"""
|
|
931
|
+
)
|
|
932
|
+
await conn.execute(
|
|
933
|
+
"""
|
|
934
|
+
CREATE INDEX IF NOT EXISTS idx_artifacts_produced_by_created
|
|
935
|
+
ON artifacts(produced_by, created_at)
|
|
936
|
+
"""
|
|
937
|
+
)
|
|
938
|
+
await conn.execute(
|
|
939
|
+
"""
|
|
940
|
+
CREATE INDEX IF NOT EXISTS idx_artifacts_correlation
|
|
941
|
+
ON artifacts(correlation_id)
|
|
942
|
+
"""
|
|
943
|
+
)
|
|
944
|
+
await conn.execute(
|
|
945
|
+
"""
|
|
946
|
+
CREATE INDEX IF NOT EXISTS idx_artifacts_partition
|
|
947
|
+
ON artifacts(partition_key)
|
|
948
|
+
"""
|
|
949
|
+
)
|
|
950
|
+
await conn.execute(
|
|
951
|
+
"""
|
|
952
|
+
CREATE TABLE IF NOT EXISTS artifact_consumptions (
|
|
953
|
+
artifact_id TEXT NOT NULL,
|
|
954
|
+
consumer TEXT NOT NULL,
|
|
955
|
+
run_id TEXT,
|
|
956
|
+
correlation_id TEXT,
|
|
957
|
+
consumed_at TEXT NOT NULL,
|
|
958
|
+
PRIMARY KEY (artifact_id, consumer, consumed_at)
|
|
959
|
+
)
|
|
960
|
+
"""
|
|
961
|
+
)
|
|
962
|
+
await conn.execute(
|
|
963
|
+
"""
|
|
964
|
+
CREATE INDEX IF NOT EXISTS idx_consumptions_artifact
|
|
965
|
+
ON artifact_consumptions(artifact_id)
|
|
966
|
+
"""
|
|
967
|
+
)
|
|
968
|
+
await conn.execute(
|
|
969
|
+
"""
|
|
970
|
+
CREATE INDEX IF NOT EXISTS idx_consumptions_consumer
|
|
971
|
+
ON artifact_consumptions(consumer)
|
|
972
|
+
"""
|
|
973
|
+
)
|
|
974
|
+
await conn.execute(
|
|
975
|
+
"""
|
|
976
|
+
CREATE INDEX IF NOT EXISTS idx_consumptions_correlation
|
|
977
|
+
ON artifact_consumptions(correlation_id)
|
|
978
|
+
"""
|
|
979
|
+
)
|
|
980
|
+
await conn.commit()
|
|
981
|
+
self._schema_ready = True
|
|
982
|
+
|
|
983
|
+
def _build_filters(
|
|
984
|
+
self,
|
|
985
|
+
filters: FilterConfig,
|
|
986
|
+
*,
|
|
987
|
+
table_alias: str | None = None,
|
|
988
|
+
) -> tuple[str, list[Any]]:
|
|
989
|
+
prefix = f"{table_alias}." if table_alias else ""
|
|
990
|
+
conditions: list[str] = []
|
|
991
|
+
params: list[Any] = []
|
|
992
|
+
|
|
993
|
+
if filters.type_names:
|
|
994
|
+
canonical = {type_registry.resolve_name(name) for name in filters.type_names}
|
|
995
|
+
placeholders = ", ".join("?" for _ in canonical)
|
|
996
|
+
conditions.append(f"{prefix}canonical_type IN ({placeholders})")
|
|
997
|
+
params.extend(sorted(canonical))
|
|
998
|
+
|
|
999
|
+
if filters.produced_by:
|
|
1000
|
+
placeholders = ", ".join("?" for _ in filters.produced_by)
|
|
1001
|
+
conditions.append(f"{prefix}produced_by IN ({placeholders})")
|
|
1002
|
+
params.extend(sorted(filters.produced_by))
|
|
1003
|
+
|
|
1004
|
+
if filters.correlation_id:
|
|
1005
|
+
conditions.append(f"{prefix}correlation_id = ?")
|
|
1006
|
+
params.append(filters.correlation_id)
|
|
1007
|
+
|
|
1008
|
+
if filters.visibility:
|
|
1009
|
+
placeholders = ", ".join("?" for _ in filters.visibility)
|
|
1010
|
+
conditions.append(f"json_extract({prefix}visibility, '$.kind') IN ({placeholders})")
|
|
1011
|
+
params.extend(sorted(filters.visibility))
|
|
1012
|
+
|
|
1013
|
+
if filters.start is not None:
|
|
1014
|
+
conditions.append(f"{prefix}created_at >= ?")
|
|
1015
|
+
params.append(filters.start.isoformat())
|
|
1016
|
+
|
|
1017
|
+
if filters.end is not None:
|
|
1018
|
+
conditions.append(f"{prefix}created_at <= ?")
|
|
1019
|
+
params.append(filters.end.isoformat())
|
|
1020
|
+
|
|
1021
|
+
if filters.tags:
|
|
1022
|
+
column = f"{prefix}tags" if table_alias else "artifacts.tags"
|
|
1023
|
+
for tag in sorted(filters.tags):
|
|
1024
|
+
conditions.append(
|
|
1025
|
+
f"EXISTS (SELECT 1 FROM json_each({column}) WHERE json_each.value = ?)" # nosec B608 - column is internal constant
|
|
1026
|
+
)
|
|
1027
|
+
params.append(tag)
|
|
1028
|
+
|
|
1029
|
+
where_clause = f" WHERE {' AND '.join(conditions)}" if conditions else ""
|
|
1030
|
+
return where_clause, params
|
|
1031
|
+
|
|
1032
|
+
def _row_to_artifact(self, row: Any) -> Artifact:
|
|
1033
|
+
payload = json.loads(row["payload"])
|
|
1034
|
+
visibility_data = json.loads(row["visibility"])
|
|
1035
|
+
tags = json.loads(row["tags"])
|
|
1036
|
+
correlation_raw = row["correlation_id"]
|
|
1037
|
+
correlation = UUID(correlation_raw) if correlation_raw else None
|
|
1038
|
+
return Artifact(
|
|
1039
|
+
id=UUID(row["artifact_id"]),
|
|
1040
|
+
type=row["type"],
|
|
1041
|
+
payload=payload,
|
|
1042
|
+
produced_by=row["produced_by"],
|
|
1043
|
+
visibility=_deserialize_visibility(visibility_data),
|
|
1044
|
+
tags=set(tags),
|
|
1045
|
+
correlation_id=correlation,
|
|
1046
|
+
partition_key=row["partition_key"],
|
|
1047
|
+
created_at=datetime.fromisoformat(row["created_at"]),
|
|
1048
|
+
version=row["version"],
|
|
1049
|
+
)
|