astrocyte-postgres 0.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- astrocyte_postgres/__init__.py +11 -0
- astrocyte_postgres/store.py +919 -0
- astrocyte_postgres/wiki_store.py +458 -0
- astrocyte_postgres-0.12.0.dist-info/METADATA +133 -0
- astrocyte_postgres-0.12.0.dist-info/RECORD +7 -0
- astrocyte_postgres-0.12.0.dist-info/WHEEL +4 -0
- astrocyte_postgres-0.12.0.dist-info/entry_points.txt +8 -0
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
"""PostgreSQL-backed WikiStore for durable compiled memory pages."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from typing import Any, ClassVar
|
|
10
|
+
|
|
11
|
+
import psycopg
|
|
12
|
+
from astrocyte.tenancy import fq_table, get_current_schema
|
|
13
|
+
from astrocyte.types import HealthStatus, WikiPage
|
|
14
|
+
from psycopg.rows import dict_row
|
|
15
|
+
from psycopg.types.json import Json
|
|
16
|
+
from psycopg_pool import AsyncConnectionPool
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PostgresWikiStore:
|
|
20
|
+
"""Durable WikiStore using the reference Astrocyte Postgres schema.
|
|
21
|
+
|
|
22
|
+
Per-tenant aware via :func:`astrocyte.tenancy.fq_table` — every SQL
|
|
23
|
+
string that references a table goes through :meth:`_fq` so the active
|
|
24
|
+
tenant's schema is honoured.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
SPI_VERSION: ClassVar[int] = 1
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
dsn: str | None = None,
|
|
32
|
+
*,
|
|
33
|
+
bootstrap_schema: bool = True,
|
|
34
|
+
**kwargs: Any,
|
|
35
|
+
) -> None:
|
|
36
|
+
self._dsn = dsn or os.environ.get("DATABASE_URL") or os.environ.get("ASTROCYTE_PG_DSN")
|
|
37
|
+
if not self._dsn:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
"PostgresWikiStore requires `dsn` in wiki_store_config or DATABASE_URL / ASTROCYTE_PG_DSN",
|
|
40
|
+
)
|
|
41
|
+
self._bootstrap_schema = bool(bootstrap_schema)
|
|
42
|
+
self._pool: AsyncConnectionPool | None = None
|
|
43
|
+
self._pool_lock = asyncio.Lock()
|
|
44
|
+
# Per-tenant-schema bootstrap tracking — see PostgresStore for rationale.
|
|
45
|
+
self._bootstrapped_schemas: set[str] = set()
|
|
46
|
+
self._schema_lock = asyncio.Lock()
|
|
47
|
+
|
|
48
|
+
def _fq(self, table: str) -> str:
|
|
49
|
+
"""Schema-qualify a table name using the current tenant context."""
|
|
50
|
+
return fq_table(table)
|
|
51
|
+
|
|
52
|
+
async def _ensure_pool(self) -> AsyncConnectionPool:
|
|
53
|
+
async with self._pool_lock:
|
|
54
|
+
if self._pool is None:
|
|
55
|
+
async def configure(conn: psycopg.AsyncConnection) -> None:
|
|
56
|
+
# Pin search_path to ``public`` first so wiki tables are
|
|
57
|
+
# routed to the canonical migrated schema. Without this,
|
|
58
|
+
# Postgres' default ``"$user", public`` order silently
|
|
59
|
+
# creates a duplicate set of wiki tables in the user-named
|
|
60
|
+
# schema (``astrocyte`` for the bench DB) and writes land
|
|
61
|
+
# there, breaking recall queries that read from ``public``.
|
|
62
|
+
await conn.execute('SET search_path = public, "$user"')
|
|
63
|
+
await conn.commit()
|
|
64
|
+
|
|
65
|
+
self._pool = AsyncConnectionPool(
|
|
66
|
+
conninfo=self._dsn,
|
|
67
|
+
configure=configure,
|
|
68
|
+
open=False,
|
|
69
|
+
min_size=2,
|
|
70
|
+
# Sized for parallel persona-compile tasks (each writes
|
|
71
|
+
# one wiki page + revision) running alongside retain.
|
|
72
|
+
max_size=40,
|
|
73
|
+
kwargs={"connect_timeout": 10},
|
|
74
|
+
)
|
|
75
|
+
await self._pool.open()
|
|
76
|
+
return self._pool
|
|
77
|
+
|
|
78
|
+
async def _ensure_schema(self, pool: AsyncConnectionPool) -> None:
|
|
79
|
+
"""Per-tenant-aware bootstrap of wiki tables in the active schema.
|
|
80
|
+
|
|
81
|
+
Same per-(schema, store) tracking pattern as
|
|
82
|
+
:meth:`PostgresStore._ensure_schema`. Mirrors ``007_wiki_tables.sql``
|
|
83
|
+
for the active tenant schema.
|
|
84
|
+
"""
|
|
85
|
+
if not self._bootstrap_schema:
|
|
86
|
+
return
|
|
87
|
+
active_schema = get_current_schema()
|
|
88
|
+
if active_schema in self._bootstrapped_schemas:
|
|
89
|
+
return
|
|
90
|
+
async with self._schema_lock:
|
|
91
|
+
if active_schema in self._bootstrapped_schemas:
|
|
92
|
+
return
|
|
93
|
+
banks = self._fq("astrocyte_banks")
|
|
94
|
+
pages = self._fq("astrocyte_wiki_pages")
|
|
95
|
+
revisions = self._fq("astrocyte_wiki_revisions")
|
|
96
|
+
sources = self._fq("astrocyte_wiki_revision_sources")
|
|
97
|
+
links = self._fq("astrocyte_wiki_links")
|
|
98
|
+
lint = self._fq("astrocyte_wiki_lint_issues")
|
|
99
|
+
async with pool.connection() as conn:
|
|
100
|
+
await conn.execute(f'CREATE SCHEMA IF NOT EXISTS "{active_schema}"')
|
|
101
|
+
await conn.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
|
|
102
|
+
await conn.execute(
|
|
103
|
+
f"""
|
|
104
|
+
CREATE TABLE IF NOT EXISTS {banks} (
|
|
105
|
+
id TEXT PRIMARY KEY,
|
|
106
|
+
tenant_id TEXT,
|
|
107
|
+
display_name TEXT,
|
|
108
|
+
description TEXT,
|
|
109
|
+
metadata JSONB NOT NULL DEFAULT '{{}}'::jsonb,
|
|
110
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
111
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
112
|
+
archived_at TIMESTAMPTZ
|
|
113
|
+
)
|
|
114
|
+
"""
|
|
115
|
+
)
|
|
116
|
+
await conn.execute(
|
|
117
|
+
f"""
|
|
118
|
+
CREATE TABLE IF NOT EXISTS {pages} (
|
|
119
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
120
|
+
page_id TEXT NOT NULL,
|
|
121
|
+
bank_id TEXT NOT NULL,
|
|
122
|
+
slug TEXT NOT NULL,
|
|
123
|
+
title TEXT NOT NULL,
|
|
124
|
+
kind TEXT NOT NULL CHECK (kind IN ('topic', 'entity', 'concept')),
|
|
125
|
+
scope TEXT NOT NULL,
|
|
126
|
+
current_revision_id UUID,
|
|
127
|
+
confidence DOUBLE PRECISION NOT NULL DEFAULT 0,
|
|
128
|
+
tags TEXT[],
|
|
129
|
+
metadata JSONB NOT NULL DEFAULT '{{}}'::jsonb,
|
|
130
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
131
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
132
|
+
deleted_at TIMESTAMPTZ,
|
|
133
|
+
UNIQUE (bank_id, page_id),
|
|
134
|
+
UNIQUE (bank_id, slug)
|
|
135
|
+
)
|
|
136
|
+
"""
|
|
137
|
+
)
|
|
138
|
+
await conn.execute(
|
|
139
|
+
f"""
|
|
140
|
+
CREATE TABLE IF NOT EXISTS {revisions} (
|
|
141
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
142
|
+
page_uuid UUID NOT NULL REFERENCES {pages}(id) ON DELETE CASCADE,
|
|
143
|
+
revision_number INTEGER NOT NULL,
|
|
144
|
+
markdown TEXT NOT NULL,
|
|
145
|
+
summary TEXT,
|
|
146
|
+
compiled_by TEXT,
|
|
147
|
+
source_count INTEGER NOT NULL DEFAULT 0,
|
|
148
|
+
tokens_used INTEGER NOT NULL DEFAULT 0,
|
|
149
|
+
metadata JSONB NOT NULL DEFAULT '{{}}'::jsonb,
|
|
150
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
151
|
+
UNIQUE (page_uuid, revision_number)
|
|
152
|
+
)
|
|
153
|
+
"""
|
|
154
|
+
)
|
|
155
|
+
await conn.execute(
|
|
156
|
+
f"""
|
|
157
|
+
CREATE TABLE IF NOT EXISTS {sources} (
|
|
158
|
+
revision_id UUID NOT NULL REFERENCES {revisions}(id) ON DELETE CASCADE,
|
|
159
|
+
memory_id TEXT NOT NULL,
|
|
160
|
+
bank_id TEXT NOT NULL,
|
|
161
|
+
quote TEXT,
|
|
162
|
+
relevance DOUBLE PRECISION,
|
|
163
|
+
metadata JSONB NOT NULL DEFAULT '{{}}'::jsonb,
|
|
164
|
+
PRIMARY KEY (revision_id, memory_id)
|
|
165
|
+
)
|
|
166
|
+
"""
|
|
167
|
+
)
|
|
168
|
+
await conn.execute(
|
|
169
|
+
f"""
|
|
170
|
+
CREATE TABLE IF NOT EXISTS {links} (
|
|
171
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
172
|
+
from_page_id UUID NOT NULL REFERENCES {pages}(id) ON DELETE CASCADE,
|
|
173
|
+
to_page_id UUID REFERENCES {pages}(id) ON DELETE SET NULL,
|
|
174
|
+
target_slug TEXT NOT NULL,
|
|
175
|
+
link_type TEXT NOT NULL DEFAULT 'related',
|
|
176
|
+
metadata JSONB NOT NULL DEFAULT '{{}}'::jsonb,
|
|
177
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
178
|
+
UNIQUE (from_page_id, target_slug, link_type)
|
|
179
|
+
)
|
|
180
|
+
"""
|
|
181
|
+
)
|
|
182
|
+
await conn.execute(
|
|
183
|
+
f"""
|
|
184
|
+
CREATE TABLE IF NOT EXISTS {lint} (
|
|
185
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
186
|
+
page_id UUID NOT NULL REFERENCES {pages}(id) ON DELETE CASCADE,
|
|
187
|
+
revision_id UUID REFERENCES {revisions}(id) ON DELETE SET NULL,
|
|
188
|
+
issue_type TEXT NOT NULL,
|
|
189
|
+
severity TEXT NOT NULL CHECK (severity IN ('low', 'medium', 'high')),
|
|
190
|
+
message TEXT NOT NULL,
|
|
191
|
+
evidence JSONB NOT NULL DEFAULT '{{}}'::jsonb,
|
|
192
|
+
status TEXT NOT NULL DEFAULT 'open',
|
|
193
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
194
|
+
resolved_at TIMESTAMPTZ
|
|
195
|
+
)
|
|
196
|
+
"""
|
|
197
|
+
)
|
|
198
|
+
await conn.execute(
|
|
199
|
+
f"""
|
|
200
|
+
CREATE INDEX IF NOT EXISTS astrocyte_wiki_pages_bank_kind_idx
|
|
201
|
+
ON {pages} (bank_id, kind)
|
|
202
|
+
WHERE deleted_at IS NULL
|
|
203
|
+
"""
|
|
204
|
+
)
|
|
205
|
+
await conn.commit()
|
|
206
|
+
self._bootstrapped_schemas.add(active_schema)
|
|
207
|
+
|
|
208
|
+
async def upsert_page(self, page: WikiPage, bank_id: str) -> str:
|
|
209
|
+
pool = await self._ensure_pool()
|
|
210
|
+
await self._ensure_schema(pool)
|
|
211
|
+
now = datetime.now(UTC)
|
|
212
|
+
async with pool.connection() as conn:
|
|
213
|
+
async with conn.cursor(row_factory=dict_row) as cur:
|
|
214
|
+
await cur.execute(
|
|
215
|
+
f"""
|
|
216
|
+
INSERT INTO {self._fq("astrocyte_banks")} (id, updated_at)
|
|
217
|
+
VALUES (%s, NOW())
|
|
218
|
+
ON CONFLICT (id) DO UPDATE SET updated_at = NOW()
|
|
219
|
+
""",
|
|
220
|
+
(bank_id,),
|
|
221
|
+
)
|
|
222
|
+
await cur.execute(
|
|
223
|
+
f"""
|
|
224
|
+
SELECT p.id, COALESCE(MAX(r.revision_number), 0) AS revision_number
|
|
225
|
+
FROM {self._fq("astrocyte_wiki_pages")} p
|
|
226
|
+
LEFT JOIN {self._fq("astrocyte_wiki_revisions")} r ON r.page_uuid = p.id
|
|
227
|
+
WHERE p.bank_id = %s AND p.page_id = %s
|
|
228
|
+
GROUP BY p.id
|
|
229
|
+
""",
|
|
230
|
+
(bank_id, page.page_id),
|
|
231
|
+
)
|
|
232
|
+
existing = await cur.fetchone()
|
|
233
|
+
if existing:
|
|
234
|
+
page_uuid = existing["id"]
|
|
235
|
+
revision = int(existing["revision_number"]) + 1
|
|
236
|
+
await cur.execute(
|
|
237
|
+
f"""
|
|
238
|
+
UPDATE {self._fq("astrocyte_wiki_pages")}
|
|
239
|
+
SET slug = %s,
|
|
240
|
+
title = %s,
|
|
241
|
+
kind = %s,
|
|
242
|
+
scope = %s,
|
|
243
|
+
tags = %s,
|
|
244
|
+
metadata = %s,
|
|
245
|
+
updated_at = %s,
|
|
246
|
+
deleted_at = NULL
|
|
247
|
+
WHERE id = %s
|
|
248
|
+
""",
|
|
249
|
+
(
|
|
250
|
+
_slug_for_page(page),
|
|
251
|
+
page.title,
|
|
252
|
+
page.kind,
|
|
253
|
+
page.scope,
|
|
254
|
+
page.tags,
|
|
255
|
+
Json(page.metadata or {}),
|
|
256
|
+
now,
|
|
257
|
+
page_uuid,
|
|
258
|
+
),
|
|
259
|
+
)
|
|
260
|
+
else:
|
|
261
|
+
revision = 1
|
|
262
|
+
await cur.execute(
|
|
263
|
+
f"""
|
|
264
|
+
INSERT INTO {self._fq("astrocyte_wiki_pages")}
|
|
265
|
+
(page_id, bank_id, slug, title, kind, scope, tags, metadata, created_at, updated_at)
|
|
266
|
+
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
267
|
+
RETURNING id
|
|
268
|
+
""",
|
|
269
|
+
(
|
|
270
|
+
page.page_id,
|
|
271
|
+
bank_id,
|
|
272
|
+
_slug_for_page(page),
|
|
273
|
+
page.title,
|
|
274
|
+
page.kind,
|
|
275
|
+
page.scope,
|
|
276
|
+
page.tags,
|
|
277
|
+
Json(page.metadata or {}),
|
|
278
|
+
now,
|
|
279
|
+
now,
|
|
280
|
+
),
|
|
281
|
+
)
|
|
282
|
+
page_uuid = (await cur.fetchone())["id"]
|
|
283
|
+
|
|
284
|
+
await cur.execute(
|
|
285
|
+
f"""
|
|
286
|
+
INSERT INTO {self._fq("astrocyte_wiki_revisions")}
|
|
287
|
+
(page_uuid, revision_number, markdown, source_count, metadata, created_at)
|
|
288
|
+
VALUES (%s, %s, %s, %s, %s, %s)
|
|
289
|
+
RETURNING id
|
|
290
|
+
""",
|
|
291
|
+
(
|
|
292
|
+
page_uuid,
|
|
293
|
+
revision,
|
|
294
|
+
page.content,
|
|
295
|
+
len(page.source_ids),
|
|
296
|
+
Json({"declared_revision": page.revision}),
|
|
297
|
+
page.revised_at or now,
|
|
298
|
+
),
|
|
299
|
+
)
|
|
300
|
+
revision_id = (await cur.fetchone())["id"]
|
|
301
|
+
|
|
302
|
+
for source_id in page.source_ids:
|
|
303
|
+
await cur.execute(
|
|
304
|
+
f"""
|
|
305
|
+
INSERT INTO {self._fq("astrocyte_wiki_revision_sources")} (revision_id, memory_id, bank_id)
|
|
306
|
+
VALUES (%s, %s, %s)
|
|
307
|
+
ON CONFLICT (revision_id, memory_id) DO NOTHING
|
|
308
|
+
""",
|
|
309
|
+
(revision_id, source_id, bank_id),
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
for link in page.cross_links:
|
|
313
|
+
await cur.execute(
|
|
314
|
+
f"""
|
|
315
|
+
INSERT INTO {self._fq("astrocyte_wiki_links")} (from_page_id, target_slug, link_type)
|
|
316
|
+
VALUES (%s, %s, 'related')
|
|
317
|
+
ON CONFLICT (from_page_id, target_slug, link_type) DO NOTHING
|
|
318
|
+
""",
|
|
319
|
+
(page_uuid, link),
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
await cur.execute(
|
|
323
|
+
f"""
|
|
324
|
+
UPDATE {self._fq("astrocyte_wiki_pages")}
|
|
325
|
+
SET current_revision_id = %s, updated_at = %s
|
|
326
|
+
WHERE id = %s
|
|
327
|
+
""",
|
|
328
|
+
(revision_id, now, page_uuid),
|
|
329
|
+
)
|
|
330
|
+
await conn.commit()
|
|
331
|
+
return page.page_id
|
|
332
|
+
|
|
333
|
+
async def get_page(self, page_id: str, bank_id: str) -> WikiPage | None:
|
|
334
|
+
pool = await self._ensure_pool()
|
|
335
|
+
await self._ensure_schema(pool)
|
|
336
|
+
async with pool.connection() as conn:
|
|
337
|
+
async with conn.cursor(row_factory=dict_row) as cur:
|
|
338
|
+
await cur.execute(
|
|
339
|
+
f"""
|
|
340
|
+
SELECT p.*, r.id AS revision_id, r.revision_number, r.markdown, r.created_at AS revised_at
|
|
341
|
+
FROM {self._fq("astrocyte_wiki_pages")} p
|
|
342
|
+
JOIN {self._fq("astrocyte_wiki_revisions")} r ON r.id = p.current_revision_id
|
|
343
|
+
WHERE p.bank_id = %s AND p.page_id = %s AND p.deleted_at IS NULL
|
|
344
|
+
""",
|
|
345
|
+
(bank_id, page_id),
|
|
346
|
+
)
|
|
347
|
+
row = await cur.fetchone()
|
|
348
|
+
if row is None:
|
|
349
|
+
return None
|
|
350
|
+
return await self._page_from_row(cur, row)
|
|
351
|
+
|
|
352
|
+
async def list_pages(
|
|
353
|
+
self,
|
|
354
|
+
bank_id: str,
|
|
355
|
+
scope: str | None = None,
|
|
356
|
+
kind: str | None = None,
|
|
357
|
+
) -> list[WikiPage]:
|
|
358
|
+
pool = await self._ensure_pool()
|
|
359
|
+
await self._ensure_schema(pool)
|
|
360
|
+
where = ["p.bank_id = %s", "p.deleted_at IS NULL"]
|
|
361
|
+
params: list[Any] = [bank_id]
|
|
362
|
+
if scope is not None:
|
|
363
|
+
where.append("p.scope = %s")
|
|
364
|
+
params.append(scope)
|
|
365
|
+
if kind is not None:
|
|
366
|
+
where.append("p.kind = %s")
|
|
367
|
+
params.append(kind)
|
|
368
|
+
async with pool.connection() as conn:
|
|
369
|
+
async with conn.cursor(row_factory=dict_row) as cur:
|
|
370
|
+
await cur.execute(
|
|
371
|
+
f"""
|
|
372
|
+
SELECT p.*, r.id AS revision_id, r.revision_number, r.markdown, r.created_at AS revised_at
|
|
373
|
+
FROM {self._fq("astrocyte_wiki_pages")} p
|
|
374
|
+
JOIN {self._fq("astrocyte_wiki_revisions")} r ON r.id = p.current_revision_id
|
|
375
|
+
WHERE {" AND ".join(where)}
|
|
376
|
+
ORDER BY p.page_id
|
|
377
|
+
""",
|
|
378
|
+
params,
|
|
379
|
+
)
|
|
380
|
+
rows = await cur.fetchall()
|
|
381
|
+
return [await self._page_from_row(cur, row) for row in rows]
|
|
382
|
+
|
|
383
|
+
async def delete_page(self, page_id: str, bank_id: str) -> bool:
|
|
384
|
+
pool = await self._ensure_pool()
|
|
385
|
+
await self._ensure_schema(pool)
|
|
386
|
+
async with pool.connection() as conn:
|
|
387
|
+
async with conn.cursor() as cur:
|
|
388
|
+
await cur.execute(
|
|
389
|
+
f"""
|
|
390
|
+
UPDATE {self._fq("astrocyte_wiki_pages")}
|
|
391
|
+
SET deleted_at = NOW()
|
|
392
|
+
WHERE bank_id = %s AND page_id = %s AND deleted_at IS NULL
|
|
393
|
+
""",
|
|
394
|
+
(bank_id, page_id),
|
|
395
|
+
)
|
|
396
|
+
deleted = bool(cur.rowcount)
|
|
397
|
+
await conn.commit()
|
|
398
|
+
return deleted
|
|
399
|
+
|
|
400
|
+
async def health(self) -> HealthStatus:
|
|
401
|
+
try:
|
|
402
|
+
pool = await self._ensure_pool()
|
|
403
|
+
async with pool.connection() as conn:
|
|
404
|
+
async with conn.cursor() as cur:
|
|
405
|
+
await cur.execute("SELECT 1")
|
|
406
|
+
return HealthStatus(healthy=True, message="pg wiki store connected")
|
|
407
|
+
except Exception as exc:
|
|
408
|
+
return HealthStatus(healthy=False, message=f"pg wiki store unhealthy: {exc!s}")
|
|
409
|
+
|
|
410
|
+
async def close(self) -> None:
|
|
411
|
+
async with self._pool_lock:
|
|
412
|
+
if self._pool is not None:
|
|
413
|
+
await self._pool.close()
|
|
414
|
+
self._pool = None
|
|
415
|
+
|
|
416
|
+
async def _page_from_row(self, cur: psycopg.AsyncCursor[dict[str, Any]], row: dict[str, Any]) -> WikiPage:
|
|
417
|
+
await cur.execute(
|
|
418
|
+
f"""
|
|
419
|
+
SELECT memory_id
|
|
420
|
+
FROM {self._fq("astrocyte_wiki_revision_sources")}
|
|
421
|
+
WHERE revision_id = %s
|
|
422
|
+
ORDER BY memory_id
|
|
423
|
+
""",
|
|
424
|
+
(row["revision_id"],),
|
|
425
|
+
)
|
|
426
|
+
sources = [source_row["memory_id"] for source_row in await cur.fetchall()]
|
|
427
|
+
|
|
428
|
+
await cur.execute(
|
|
429
|
+
f"""
|
|
430
|
+
SELECT target_slug
|
|
431
|
+
FROM {self._fq("astrocyte_wiki_links")}
|
|
432
|
+
WHERE from_page_id = %s
|
|
433
|
+
ORDER BY target_slug
|
|
434
|
+
""",
|
|
435
|
+
(row["id"],),
|
|
436
|
+
)
|
|
437
|
+
links = [link_row["target_slug"] for link_row in await cur.fetchall()]
|
|
438
|
+
metadata = row["metadata"]
|
|
439
|
+
if isinstance(metadata, str):
|
|
440
|
+
metadata = json.loads(metadata)
|
|
441
|
+
return WikiPage(
|
|
442
|
+
page_id=row["page_id"],
|
|
443
|
+
bank_id=row["bank_id"],
|
|
444
|
+
kind=row["kind"],
|
|
445
|
+
title=row["title"],
|
|
446
|
+
content=row["markdown"],
|
|
447
|
+
scope=row["scope"],
|
|
448
|
+
source_ids=sources,
|
|
449
|
+
cross_links=links,
|
|
450
|
+
revision=int(row["revision_number"]),
|
|
451
|
+
revised_at=row["revised_at"],
|
|
452
|
+
tags=list(row["tags"]) if row["tags"] else None,
|
|
453
|
+
metadata=metadata or None,
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _slug_for_page(page: WikiPage) -> str:
|
|
458
|
+
return page.page_id.split(":", 1)[-1] if ":" in page.page_id else page.page_id
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: astrocyte-postgres
|
|
3
|
+
Version: 0.12.0
|
|
4
|
+
Summary: PostgreSQL adapter for Astrocyte (vector + document + wiki stores backed by pgvector and tsvector)
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Requires-Dist: astrocyte<2,>=0.7.0
|
|
8
|
+
Requires-Dist: pgvector>=0.4
|
|
9
|
+
Requires-Dist: psycopg-pool>=3.2
|
|
10
|
+
Requires-Dist: psycopg[binary]>=3.1
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
13
|
+
Requires-Dist: pytest-timeout>=2.2; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# astrocyte-postgres
|
|
18
|
+
|
|
19
|
+
**PostgreSQL + [pgvector](https://github.com/pgvector/pgvector)** implementation of the Astrocyte **`VectorStore`** and **`WikiStore`** SPIs ([`provider-spi.md`](../../docs/_plugins/provider-spi.md)).
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
From the monorepo (with `astrocyte` available):
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
cd adapters-storage-py/astrocyte-postgres
|
|
27
|
+
uv sync
|
|
28
|
+
# or: pip install -e ../../astrocyte-py && pip install -e .
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Entry point names:
|
|
32
|
+
|
|
33
|
+
- **`postgres`** (group `astrocyte.vector_stores`) for raw/compiled memory vectors.
|
|
34
|
+
- **`postgres`** (group `astrocyte.document_stores`) for BM25 keyword retrieval over the same table.
|
|
35
|
+
- **`postgres`** (group `astrocyte.wiki_stores`) for durable wiki pages/revisions/provenance.
|
|
36
|
+
|
|
37
|
+
## PostgreSQL with Docker
|
|
38
|
+
|
|
39
|
+
Use the **combined** Compose stack in **[`../../astrocyte-services-py/docker-compose.yml`](../../astrocyte-services-py/docker-compose.yml)** to run **Postgres (pgvector) + the reference REST service** together:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
cd astrocyte-services-py
|
|
43
|
+
docker compose up -d
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
For **Postgres only** (no HTTP), start only `postgres`:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
cd astrocyte-services-py
|
|
50
|
+
docker compose up -d postgres
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Default DSN from your host (port **5433** maps to Postgres in the compose file):
|
|
54
|
+
|
|
55
|
+
```text
|
|
56
|
+
postgresql://astrocyte:astrocyte@127.0.0.1:5433/astrocyte
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Schema migrations (production)
|
|
60
|
+
|
|
61
|
+
DDL is shipped as **plain SQL** under [`migrations/`](migrations/) and applied with **`psql`** via [`scripts/migrate.sh`](scripts/migrate.sh) (no Python migration framework).
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
export DATABASE_URL='postgresql://astrocyte:astrocyte@127.0.0.1:5433/astrocyte'
|
|
65
|
+
cd adapters-storage-py/astrocyte-postgres
|
|
66
|
+
./scripts/migrate.sh
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Requirements: **PostgreSQL 15+** (for `CREATE INDEX CONCURRENTLY IF NOT EXISTS`), **psql** on `PATH`.
|
|
70
|
+
|
|
71
|
+
After migrations are applied, set **`bootstrap_schema: false`** in `vector_store_config` so the app does not run `CREATE TABLE` / indexes at runtime (see configuration table below). For a **single command** that starts Postgres, runs migrations, then starts the stack with runbook config, use **[`runbook-up.sh`](../../astrocyte-services-py/scripts/runbook-up.sh)** (see **[Runbook](../../astrocyte-services-py/README.md#runbook)**).
|
|
72
|
+
|
|
73
|
+
**Embedding width:** [`migrations/002_astrocyte_vectors.sql`](migrations/002_astrocyte_vectors.sql) creates `vector(${ASTROCYTE_EMBEDDING_DIMENSIONS:-128})`. That must match **`embedding_dimensions`** in config. For OpenAI `text-embedding-3-small`, run migrations with `ASTROCYTE_EMBEDDING_DIMENSIONS=1536`.
|
|
74
|
+
|
|
75
|
+
**Custom `table_name`:** The shipped SQL targets **`astrocyte_vectors`**. If you use another table name, copy and adjust the migration files accordingly.
|
|
76
|
+
|
|
77
|
+
The later migrations add the Hindsight-comparable Postgres substrate around vectors: bank metadata and access grants, lifecycle columns (`retained_at`, `forgotten_at`), durable wiki pages/revisions/provenance, canonical entity/link tables, and normalized temporal facts.
|
|
78
|
+
|
|
79
|
+
## Configuration
|
|
80
|
+
|
|
81
|
+
| Constructor / YAML `vector_store_config` | Meaning |
|
|
82
|
+
|--------------------------------------------|---------|
|
|
83
|
+
| `dsn` | PostgreSQL connection URI (or set `DATABASE_URL` / `ASTROCYTE_PG_DSN`) |
|
|
84
|
+
| `table_name` | Table name (default `astrocyte_vectors`; alphanumeric + underscore only) |
|
|
85
|
+
| `embedding_dimensions` | Fixed `vector(N)` width; must match your embedding model and the **`vector(N)`** in SQL migrations (default **128**) |
|
|
86
|
+
| `bootstrap_schema` | If **`true`** (default), create extension / table / btree index on first use (dev-friendly; no HNSW). If **`false`**, assume **`migrate.sh`** already applied [`migrations/`](migrations/) (production). |
|
|
87
|
+
|
|
88
|
+
## How this fits `astrocyte_gateway`
|
|
89
|
+
|
|
90
|
+
1. **`astrocyte-py`** defines the **`VectorStore`** protocol and discovers adapters by **entry point** (`astrocyte.vector_stores`).
|
|
91
|
+
2. **`astrocyte-postgres`** registers **`postgres` → `PostgresStore`**. Installing this package makes the name **`postgres`** available to **`resolve_provider()`**.
|
|
92
|
+
3. **`astrocyte_gateway/wiring.py`** calls **`resolve_vector_store(config)`**, which loads the class from the entry point and passes **`vector_store_config`** from YAML (or env-only defaults).
|
|
93
|
+
4. **`astrocyte_gateway/brain.py`** builds **`Astrocyte`** + **`PipelineOrchestrator`** with that store and your chosen **`llm_provider`** (still **`mock`** unless you configure a real LLM).
|
|
94
|
+
|
|
95
|
+
Example **`ASTROCYTE_CONFIG_PATH`** snippet:
|
|
96
|
+
|
|
97
|
+
```yaml
|
|
98
|
+
provider_tier: storage
|
|
99
|
+
vector_store: postgres
|
|
100
|
+
llm_provider: mock
|
|
101
|
+
vector_store_config:
|
|
102
|
+
dsn: postgresql://astrocyte:astrocyte@127.0.0.1:5433/astrocyte
|
|
103
|
+
embedding_dimensions: 128
|
|
104
|
+
bootstrap_schema: false
|
|
105
|
+
wiki_store: postgres
|
|
106
|
+
wiki_store_config:
|
|
107
|
+
dsn: postgresql://astrocyte:astrocyte@127.0.0.1:5433/astrocyte
|
|
108
|
+
bootstrap_schema: false
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Then run the REST service (from repo layout):
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
export ASTROCYTE_CONFIG_PATH=/path/to/that.yaml
|
|
115
|
+
cd astrocyte-services-py/astrocyte-gateway-py && uv run astrocyte-gateway-py
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Or set only env (no YAML file):
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
export ASTROCYTE_VECTOR_STORE=postgres
|
|
122
|
+
export DATABASE_URL=postgresql://astrocyte:astrocyte@127.0.0.1:5433/astrocyte
|
|
123
|
+
# embedding_dimensions default 128 — override via YAML if you add a file
|
|
124
|
+
cd astrocyte-services-py/astrocyte-gateway-py && uv sync --extra postgres
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
**Note:** `vector_store_config` for dimensions is only merged from YAML today; for env-only mode, add a small YAML or extend `brain.py` to pass `ASTROCYTE_EMBEDDING_DIMENSIONS` (future improvement).
|
|
128
|
+
|
|
129
|
+
## Production notes
|
|
130
|
+
|
|
131
|
+
- **HNSW** parameters (`m`, `ef_construction`) live in [`migrations/003_indexes.sql`](migrations/003_indexes.sql); tune with DBA guidance as load grows.
|
|
132
|
+
- **Embedding dimension** must match the **`LLMProvider.embed()`** output used by the pipeline.
|
|
133
|
+
- Use **secrets** for `dsn`, not committed YAML.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
astrocyte_postgres/__init__.py,sha256=FlBrbtKDI8L3_ruC1NLqqVupfoAyttoVXn_xysPCYKw,389
|
|
2
|
+
astrocyte_postgres/store.py,sha256=KPDLMV-3BxEyRzbAk5vshWPP2BhOrspoVSDe9JZI7b8,39059
|
|
3
|
+
astrocyte_postgres/wiki_store.py,sha256=kDX7q4OmxJCXsk000065NJSwwDQfMjwcE9A11C8Mqis,19615
|
|
4
|
+
astrocyte_postgres-0.12.0.dist-info/METADATA,sha256=uq7K2OT_73GVKAE_xeoHzSb9dQ4cQS9JuQfpaNWH-Lo,6302
|
|
5
|
+
astrocyte_postgres-0.12.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
6
|
+
astrocyte_postgres-0.12.0.dist-info/entry_points.txt,sha256=iHYotbk-W1WvoWfi-F3Xs2EkCdsPjUQU4mS6mU0YynU,239
|
|
7
|
+
astrocyte_postgres-0.12.0.dist-info/RECORD,,
|