community-agent-framework-postgres 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_framework_community_postgres/__init__.py +77 -0
- agent_framework_community_postgres/__main__.py +3 -0
- agent_framework_community_postgres/_checkpoint_storage.py +218 -0
- agent_framework_community_postgres/_cli.py +97 -0
- agent_framework_community_postgres/_client.py +390 -0
- agent_framework_community_postgres/_entra.py +178 -0
- agent_framework_community_postgres/_framework.py +148 -0
- agent_framework_community_postgres/_history_provider.py +261 -0
- agent_framework_community_postgres/_json.py +34 -0
- agent_framework_community_postgres/_leases.py +196 -0
- agent_framework_community_postgres/_migrations.py +235 -0
- agent_framework_community_postgres/_persistence.py +184 -0
- agent_framework_community_postgres/_retention.py +159 -0
- agent_framework_community_postgres/_session_store.py +135 -0
- agent_framework_community_postgres/_store.py +66 -0
- agent_framework_community_postgres/_thread_snapshot_store.py +162 -0
- agent_framework_community_postgres/py.typed +0 -0
- community_agent_framework_postgres-0.1.1.dist-info/METADATA +367 -0
- community_agent_framework_postgres-0.1.1.dist-info/RECORD +21 -0
- community_agent_framework_postgres-0.1.1.dist-info/WHEEL +4 -0
- community_agent_framework_postgres-0.1.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Every import of a private Agent Framework name, and the check against the tested MAF versions, in one place.
|
|
2
|
+
|
|
3
|
+
Microsoft's own Cosmos DB and Redis packages import these same names from the
|
|
4
|
+
same private modules. Keeping them here means an upstream rename breaks one
|
|
5
|
+
file, and ``tests/unit/test_framework.py`` fails before a user notices.
|
|
6
|
+
|
|
7
|
+
Verified against agent-framework-core 1.20.0 and 1.21.0 on 11 October 2026. 1.20 is the
|
|
8
|
+
floor because 1.19's ``filter_new_messages`` dropped any incoming message whose
|
|
9
|
+
content matched one already stored, so a repeated "yes" was lost.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
16
|
+
|
|
17
|
+
SUPPORTED_CORE = ">=1.20.0,<2"
|
|
18
|
+
"""The declared ``agent-framework-core`` range: MAF keeps breaking changes for major versions."""
|
|
19
|
+
|
|
20
|
+
TESTED_CORE = ("1.20", "1.21")
|
|
21
|
+
"""The ``agent-framework-core`` minors this release was tested on (major.minor)."""
|
|
22
|
+
|
|
23
|
+
TESTED_AG_UI = ("1.5",)
|
|
24
|
+
"""The ``agent-framework-ag-ui`` minors this release was tested on (major.minor)."""
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger("agent_framework_community_postgres")
|
|
27
|
+
_logged = False
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def installed_core_version() -> str:
|
|
31
|
+
"""The installed agent-framework-core version, or ``unknown``."""
|
|
32
|
+
try:
|
|
33
|
+
return version("agent-framework-core")
|
|
34
|
+
except PackageNotFoundError: # pragma: no cover
|
|
35
|
+
return "unknown"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _major_minor(text: str) -> tuple[int, int] | None:
|
|
39
|
+
"""The numeric major.minor at the start of a version string; pre-release and local parts are ignored."""
|
|
40
|
+
match = re.match(r"(\d+)\.(\d+)", text)
|
|
41
|
+
return (int(match[1]), int(match[2])) if match else None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class Compatibility:
|
|
46
|
+
"""Installed Agent Framework versions next to the minors this release was tested on."""
|
|
47
|
+
|
|
48
|
+
core_installed: str
|
|
49
|
+
core_tested: tuple[str, ...]
|
|
50
|
+
ag_ui_installed: str | None
|
|
51
|
+
ag_ui_tested: tuple[str, ...]
|
|
52
|
+
|
|
53
|
+
def _pairs(self) -> list[tuple[str, str, tuple[str, ...]]]:
|
|
54
|
+
pairs = [("agent-framework-core", self.core_installed, self.core_tested)]
|
|
55
|
+
if self.ag_ui_installed is not None:
|
|
56
|
+
pairs.append(("agent-framework-ag-ui", self.ag_ui_installed, self.ag_ui_tested))
|
|
57
|
+
return pairs
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def untested(self) -> tuple[str, ...]:
|
|
61
|
+
"""Distributions whose installed major.minor is newer than any tested one."""
|
|
62
|
+
return tuple(name for name, installed, tested in self._pairs() if _newer(installed, tested) is not None)
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def out_of_range(self) -> tuple[str, ...]:
|
|
66
|
+
"""Distributions whose installed major is newer than the tested major: outside the declared range."""
|
|
67
|
+
return tuple(name for name, installed, tested in self._pairs() if _newer(installed, tested) == "major")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _newer(installed: str, tested: tuple[str, ...]) -> str | None:
|
|
71
|
+
current = _major_minor(installed)
|
|
72
|
+
newest = max(filter(None, map(_major_minor, tested)))
|
|
73
|
+
if current is None or current <= newest:
|
|
74
|
+
return None
|
|
75
|
+
return "major" if current[0] > newest[0] else "minor"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def compatibility() -> Compatibility:
|
|
79
|
+
"""What is installed and what was tested; ``untested`` is empty when every version is covered."""
|
|
80
|
+
try:
|
|
81
|
+
ag_ui: str | None = version("agent-framework-ag-ui")
|
|
82
|
+
except PackageNotFoundError:
|
|
83
|
+
ag_ui = None
|
|
84
|
+
return Compatibility(installed_core_version(), TESTED_CORE, ag_ui, TESTED_AG_UI)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def log_untested_once() -> None:
|
|
88
|
+
"""Log one warning per process when an installed Agent Framework is newer than any tested minor.
|
|
89
|
+
|
|
90
|
+
Called by every store and hub constructor. Importing the package logs nothing.
|
|
91
|
+
"""
|
|
92
|
+
global _logged
|
|
93
|
+
if _logged:
|
|
94
|
+
return
|
|
95
|
+
_logged = True
|
|
96
|
+
report = compatibility()
|
|
97
|
+
for name, installed, tested in report._pairs(): # pyright: ignore[reportPrivateUsage]
|
|
98
|
+
kind = _newer(installed, tested)
|
|
99
|
+
if kind is None:
|
|
100
|
+
continue
|
|
101
|
+
current = _major_minor(installed)
|
|
102
|
+
if current is None:
|
|
103
|
+
continue
|
|
104
|
+
if kind == "major":
|
|
105
|
+
advice = (
|
|
106
|
+
f"{name} {installed} is a new major version, outside the declared range and likely to break;"
|
|
107
|
+
f" pin {name}<{current[0]} and open an issue."
|
|
108
|
+
)
|
|
109
|
+
else:
|
|
110
|
+
advice = (
|
|
111
|
+
"It is expected to work, since Agent Framework keeps breaking changes for major versions;"
|
|
112
|
+
f" if something fails, pin {name}<{current[0]}.{current[1]} and open an issue."
|
|
113
|
+
)
|
|
114
|
+
logger.warning(
|
|
115
|
+
"community-agent-framework-postgres has not been tested with %s %s (tested: %s). %s",
|
|
116
|
+
name,
|
|
117
|
+
installed,
|
|
118
|
+
", ".join(tested),
|
|
119
|
+
advice,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
try:
|
|
124
|
+
# The shared dedupe every history provider (Cosmos, Redis, File) uses.
|
|
125
|
+
from agent_framework._sessions import filter_new_messages
|
|
126
|
+
|
|
127
|
+
# The hybrid JSON + restricted-pickle encoding File and Cosmos storage use.
|
|
128
|
+
from agent_framework._workflows._checkpoint_encoding import decode_checkpoint_value, encode_checkpoint_value
|
|
129
|
+
except ImportError as exc: # pragma: no cover - exercised only by an incompatible upstream release
|
|
130
|
+
raise ImportError(
|
|
131
|
+
f"agent-framework-core {installed_core_version()} no longer provides a private name this package relies on "
|
|
132
|
+
f"({exc}). This release declares agent-framework-core {SUPPORTED_CORE} and was tested on "
|
|
133
|
+
f"{', '.join(TESTED_CORE)}; pin an earlier agent-framework-core or check for a newer "
|
|
134
|
+
"community-agent-framework-postgres, and open an issue."
|
|
135
|
+
) from exc
|
|
136
|
+
|
|
137
|
+
__all__ = [
|
|
138
|
+
"SUPPORTED_CORE",
|
|
139
|
+
"TESTED_AG_UI",
|
|
140
|
+
"TESTED_CORE",
|
|
141
|
+
"Compatibility",
|
|
142
|
+
"compatibility",
|
|
143
|
+
"decode_checkpoint_value",
|
|
144
|
+
"encode_checkpoint_value",
|
|
145
|
+
"filter_new_messages",
|
|
146
|
+
"installed_core_version",
|
|
147
|
+
"log_untested_once",
|
|
148
|
+
]
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""Conversation history in PostgreSQL, one row per message, scoped like the Redis provider."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from collections.abc import Mapping, Sequence
|
|
7
|
+
from typing import Any, ClassVar, cast
|
|
8
|
+
|
|
9
|
+
from agent_framework import HistoryProvider, Message, SecretString
|
|
10
|
+
from psycopg import AsyncConnection, sql
|
|
11
|
+
|
|
12
|
+
from ._client import ClientHandle, PostgresClient, PostgresStorageError, optional_text, require_text
|
|
13
|
+
from ._entra import EntraCredential
|
|
14
|
+
from ._framework import filter_new_messages
|
|
15
|
+
from ._json import encode_jsonb
|
|
16
|
+
from ._retention import EXPIRES_AT, PurgeReport, RetentionPolicy, purge_rows
|
|
17
|
+
from ._store import BaseStore
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PostgresHistoryProvider(HistoryProvider, BaseStore):
|
|
23
|
+
"""PostgreSQL-backed ``HistoryProvider``.
|
|
24
|
+
|
|
25
|
+
Rows are isolated by application, optional tenant and agent, provider ``source_id``
|
|
26
|
+
and session id, the same boundaries ``RedisHistoryProvider`` uses. These identifiers
|
|
27
|
+
select stored history; they are not authorization. Bind them to authenticated
|
|
28
|
+
context in the application.
|
|
29
|
+
|
|
30
|
+
Saves to one session are serialized by a transaction-scoped advisory lock, so
|
|
31
|
+
concurrent writers never duplicate history. When an agent runs without an explicit
|
|
32
|
+
session, MAF creates a new session id per call, so each stateless run writes a new
|
|
33
|
+
history; pass a session or set ``retention``.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
DEFAULT_SOURCE_ID: ClassVar[str] = "postgres_history"
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
source_id: str = DEFAULT_SOURCE_ID,
|
|
41
|
+
*,
|
|
42
|
+
application_id: str,
|
|
43
|
+
tenant_id: str | None = None,
|
|
44
|
+
agent_id: str | None = None,
|
|
45
|
+
max_messages: int | None = None,
|
|
46
|
+
load_messages: bool = True,
|
|
47
|
+
store_inputs: bool = True,
|
|
48
|
+
store_context_messages: bool = False,
|
|
49
|
+
store_context_from: set[str] | None = None,
|
|
50
|
+
store_outputs: bool = True,
|
|
51
|
+
connection_string: str | SecretString | None = None,
|
|
52
|
+
client: PostgresClient | ClientHandle | None = None,
|
|
53
|
+
credential: EntraCredential | None = None,
|
|
54
|
+
env_file_path: str | None = None,
|
|
55
|
+
env_file_encoding: str | None = None,
|
|
56
|
+
schema: str = "public",
|
|
57
|
+
table_prefix: str = "af_",
|
|
58
|
+
retention: RetentionPolicy | None = None,
|
|
59
|
+
pool_kwargs: Mapping[str, Any] | None = None,
|
|
60
|
+
) -> None:
|
|
61
|
+
HistoryProvider.__init__(
|
|
62
|
+
self,
|
|
63
|
+
require_text(source_id, "source_id"),
|
|
64
|
+
load_messages=load_messages,
|
|
65
|
+
store_inputs=store_inputs,
|
|
66
|
+
store_context_messages=store_context_messages,
|
|
67
|
+
store_context_from=store_context_from,
|
|
68
|
+
store_outputs=store_outputs,
|
|
69
|
+
)
|
|
70
|
+
BaseStore.__init__(
|
|
71
|
+
self,
|
|
72
|
+
application_id=application_id,
|
|
73
|
+
connection_string=connection_string,
|
|
74
|
+
client=client,
|
|
75
|
+
credential=credential,
|
|
76
|
+
env_file_path=env_file_path,
|
|
77
|
+
env_file_encoding=env_file_encoding,
|
|
78
|
+
schema=schema,
|
|
79
|
+
table_prefix=table_prefix,
|
|
80
|
+
retention=retention,
|
|
81
|
+
pool_kwargs=pool_kwargs,
|
|
82
|
+
)
|
|
83
|
+
if max_messages is not None and max_messages < 0:
|
|
84
|
+
raise ValueError("max_messages must be None (unlimited) or a non-negative integer.")
|
|
85
|
+
self.tenant_id = optional_text(tenant_id, "tenant_id")
|
|
86
|
+
self.agent_id = optional_text(agent_id, "agent_id")
|
|
87
|
+
self.max_messages = max_messages
|
|
88
|
+
|
|
89
|
+
_SCOPE = sql.SQL(
|
|
90
|
+
"application_id = %(app)s AND tenant_id = %(tenant)s AND agent_id = %(agent)s"
|
|
91
|
+
" AND source_id = %(source)s AND session_id = %(session)s"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
def _params(self, session_id: str | None) -> dict[str, Any]:
|
|
95
|
+
return {
|
|
96
|
+
"app": self.application_id,
|
|
97
|
+
"tenant": self.tenant_id,
|
|
98
|
+
"agent": self.agent_id,
|
|
99
|
+
"source": self.source_id,
|
|
100
|
+
"session": require_text(session_id, "session_id"),
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async def get_messages(
|
|
104
|
+
self, session_id: str | None, *, state: dict[str, Any] | None = None, **kwargs: Any
|
|
105
|
+
) -> list[Message]:
|
|
106
|
+
"""All stored messages for the session, oldest first; set ``max_messages`` to bound the session."""
|
|
107
|
+
params = self._params(session_id)
|
|
108
|
+
async with self._client.connection(transactional=False) as connection:
|
|
109
|
+
return await self._read(connection, params)
|
|
110
|
+
|
|
111
|
+
@staticmethod
|
|
112
|
+
def _decode(payload: Any) -> Message:
|
|
113
|
+
if not isinstance(payload, dict):
|
|
114
|
+
raise ValueError("not an object")
|
|
115
|
+
return Message.from_dict(cast("dict[str, Any]", payload))
|
|
116
|
+
|
|
117
|
+
async def _read(self, connection: AsyncConnection[Any], params: dict[str, Any]) -> list[Message]:
|
|
118
|
+
cursor = await connection.execute(
|
|
119
|
+
sql.SQL("SELECT message FROM {history} WHERE {scope} ORDER BY id").format(
|
|
120
|
+
history=self._names.table("history_messages"), scope=self._SCOPE
|
|
121
|
+
),
|
|
122
|
+
params,
|
|
123
|
+
)
|
|
124
|
+
messages: list[Message] = []
|
|
125
|
+
for (payload,) in await cursor.fetchall():
|
|
126
|
+
try:
|
|
127
|
+
messages.append(self._decode(payload))
|
|
128
|
+
except (ValueError, TypeError, KeyError, AttributeError):
|
|
129
|
+
logger.warning("Skipping a history row that failed to deserialize.")
|
|
130
|
+
return messages
|
|
131
|
+
|
|
132
|
+
async def _tail(self, connection: AsyncConnection[Any], params: dict[str, Any], limit: int) -> list[Message]:
|
|
133
|
+
"""The newest ``limit`` stored messages, oldest first; raises when one no longer deserializes."""
|
|
134
|
+
cursor = await connection.execute(
|
|
135
|
+
sql.SQL("SELECT message FROM {history} WHERE {scope} ORDER BY id DESC LIMIT %(limit)s").format(
|
|
136
|
+
history=self._names.table("history_messages"), scope=self._SCOPE
|
|
137
|
+
),
|
|
138
|
+
params | {"limit": limit},
|
|
139
|
+
)
|
|
140
|
+
rows = await cursor.fetchall()
|
|
141
|
+
try:
|
|
142
|
+
return [self._decode(payload) for (payload,) in reversed(rows)]
|
|
143
|
+
except (ValueError, TypeError, KeyError, AttributeError) as exc:
|
|
144
|
+
raise PostgresStorageError(
|
|
145
|
+
"A stored message in this session no longer deserializes; clear the session or repair the row."
|
|
146
|
+
) from exc
|
|
147
|
+
|
|
148
|
+
_LOCK = sql.SQL(
|
|
149
|
+
"SELECT pg_advisory_xact_lock(hashtextextended(concat_ws(chr(31), %(schema)s::text, %(prefix)s::text,"
|
|
150
|
+
" %(app)s::text, %(tenant)s::text, %(agent)s::text, %(source)s::text, %(session)s::text), 0))"
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
def _lock_params(self, params: dict[str, Any]) -> dict[str, Any]:
|
|
154
|
+
return params | {"schema": self._names.schema, "prefix": self._names.prefix}
|
|
155
|
+
|
|
156
|
+
async def save_messages(
|
|
157
|
+
self,
|
|
158
|
+
session_id: str | None,
|
|
159
|
+
messages: Sequence[Message],
|
|
160
|
+
*,
|
|
161
|
+
state: dict[str, Any] | None = None,
|
|
162
|
+
**kwargs: Any,
|
|
163
|
+
) -> None:
|
|
164
|
+
"""Append the messages not already stored; trim to ``max_messages`` when set.
|
|
165
|
+
|
|
166
|
+
Writers of one session take turns on a transaction-scoped advisory lock inside a
|
|
167
|
+
``READ COMMITTED`` transaction, so the dedupe read sees every save committed before the
|
|
168
|
+
lock was granted, whatever the server's default isolation level. Dedupe needs only the
|
|
169
|
+
newest ``len(messages)`` stored rows: MAF's ``filter_new_messages`` compares a suffix of the
|
|
170
|
+
stored history with a prefix of the batch, or the whole stored history inside the batch,
|
|
171
|
+
and both fit in that window.
|
|
172
|
+
"""
|
|
173
|
+
params = self._params(session_id)
|
|
174
|
+
if not messages or self.max_messages == 0:
|
|
175
|
+
return
|
|
176
|
+
history = self._names.table("history_messages")
|
|
177
|
+
# Encode before taking a connection, so a value JSONB refuses fails without holding one.
|
|
178
|
+
encoded = {id(message): encode_jsonb(message.to_dict(), what="History message") for message in messages}
|
|
179
|
+
async with self._client.connection(read_committed=True) as connection:
|
|
180
|
+
await connection.execute(self._LOCK, self._lock_params(params))
|
|
181
|
+
existing = await self._tail(connection, params, len(messages))
|
|
182
|
+
new_messages = filter_new_messages(existing, messages)
|
|
183
|
+
if not new_messages:
|
|
184
|
+
return
|
|
185
|
+
async with connection.cursor() as cursor:
|
|
186
|
+
await cursor.executemany(
|
|
187
|
+
sql.SQL(
|
|
188
|
+
"INSERT INTO {history}"
|
|
189
|
+
" (application_id, tenant_id, agent_id, source_id, session_id, message, expires_at)"
|
|
190
|
+
" VALUES (%(app)s, %(tenant)s, %(agent)s, %(source)s, %(session)s, %(message)s, {expires})"
|
|
191
|
+
).format(history=history, expires=EXPIRES_AT),
|
|
192
|
+
[params | {"message": encoded[id(message)], "ttl": self._ttl()} for message in new_messages],
|
|
193
|
+
)
|
|
194
|
+
if self.max_messages is not None:
|
|
195
|
+
await connection.execute(
|
|
196
|
+
sql.SQL(
|
|
197
|
+
"DELETE FROM {history} WHERE {scope} AND id NOT IN"
|
|
198
|
+
" (SELECT id FROM {history} WHERE {scope} ORDER BY id DESC LIMIT %(keep)s)"
|
|
199
|
+
).format(history=history, scope=self._SCOPE),
|
|
200
|
+
params | {"keep": self.max_messages},
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
async def clear(self, session_id: str | None) -> None:
|
|
204
|
+
"""Delete every stored message for the session; takes the session's save lock."""
|
|
205
|
+
params = self._params(session_id)
|
|
206
|
+
async with self._client.connection(read_committed=True) as connection:
|
|
207
|
+
await connection.execute(self._LOCK, self._lock_params(params))
|
|
208
|
+
await connection.execute(
|
|
209
|
+
sql.SQL("DELETE FROM {history} WHERE {scope}").format(
|
|
210
|
+
history=self._names.table("history_messages"), scope=self._SCOPE
|
|
211
|
+
),
|
|
212
|
+
params,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
def _scope_params(self) -> dict[str, Any]:
|
|
216
|
+
return {"app": self.application_id, "tenant": self.tenant_id, "agent": self.agent_id, "source": self.source_id}
|
|
217
|
+
|
|
218
|
+
async def list_sessions(self, *, limit: int | None = None, after: str | None = None) -> list[str]:
|
|
219
|
+
"""Session ids with stored history under this provider's scope, sorted; page with ``limit`` and ``after``."""
|
|
220
|
+
async with self._client.connection(transactional=False) as connection:
|
|
221
|
+
cursor = await connection.execute(
|
|
222
|
+
sql.SQL(
|
|
223
|
+
"SELECT DISTINCT session_id FROM {history}"
|
|
224
|
+
" WHERE application_id = %(app)s AND tenant_id = %(tenant)s AND agent_id = %(agent)s"
|
|
225
|
+
" AND source_id = %(source)s AND (%(after)s::text IS NULL OR session_id > %(after)s)"
|
|
226
|
+
" ORDER BY session_id LIMIT %(limit)s"
|
|
227
|
+
).format(history=self._names.table("history_messages")),
|
|
228
|
+
self._scope_params() | {"after": after, "limit": limit},
|
|
229
|
+
)
|
|
230
|
+
return [row[0] for row in await cursor.fetchall()]
|
|
231
|
+
|
|
232
|
+
async def purge(self) -> PurgeReport:
|
|
233
|
+
"""Delete the messages of sessions whose every message has expired, whatever policy stamped them."""
|
|
234
|
+
history = self._names.table("history_messages")
|
|
235
|
+
count = await purge_rows(
|
|
236
|
+
self._client,
|
|
237
|
+
table=history,
|
|
238
|
+
where=sql.SQL(
|
|
239
|
+
"c.tenant_id = %(tenant)s AND c.agent_id = %(agent)s AND c.source_id = %(source)s AND {whole}"
|
|
240
|
+
).format(whole=whole_session_expired(history)),
|
|
241
|
+
params=self._scope_params(),
|
|
242
|
+
mode="delete",
|
|
243
|
+
tombstone=None,
|
|
244
|
+
)
|
|
245
|
+
return PurgeReport({f"{self._names.prefix}history_messages": count})
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def whole_session_expired(history: sql.Identifier) -> sql.Composable:
|
|
249
|
+
"""Purge predicate on the candidate row ``c``: no message of its session is still live.
|
|
250
|
+
|
|
251
|
+
A session's rows are deleted only when no row of the session is still live; a row written while
|
|
252
|
+
``ttl`` was ``None`` has no expiry and keeps its whole session. A conversation that goes on
|
|
253
|
+
never loses its head, and a tool call never loses its result. Nothing is rewritten on save; the
|
|
254
|
+
rule is applied at purge time.
|
|
255
|
+
"""
|
|
256
|
+
return sql.SQL(
|
|
257
|
+
"c.application_id = %(app)s AND NOT EXISTS (SELECT 1 FROM {history} live"
|
|
258
|
+
" WHERE live.application_id = c.application_id AND live.tenant_id = c.tenant_id"
|
|
259
|
+
" AND live.agent_id = c.agent_id AND live.source_id = c.source_id AND live.session_id = c.session_id"
|
|
260
|
+
" AND (live.expires_at IS NULL OR live.expires_at >= now()))"
|
|
261
|
+
).format(history=history)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Encode a JSONB value once, refusing what PostgreSQL would reject, without quoting the value."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
from psycopg.types.json import Jsonb
|
|
9
|
+
|
|
10
|
+
# json.dumps writes U+0000 as the escape \u0000, which JSONB rejects. An even run of backslashes
|
|
11
|
+
# before it (none included) means the escape is real, not the literal text "\u0000" in a string.
|
|
12
|
+
_NUL_ESCAPE = re.compile(r"(?<!\\)(?:\\\\)*\\u0000")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def encode_jsonb(value: object, *, what: str) -> Jsonb:
|
|
16
|
+
"""Serialize ``value`` once and wrap the text for psycopg.
|
|
17
|
+
|
|
18
|
+
Raises:
|
|
19
|
+
ValueError: ``value`` holds a NUL character, a lone surrogate, a non-finite float, a circular
|
|
20
|
+
reference or something ``json`` cannot encode. The message names ``what`` and never the value.
|
|
21
|
+
"""
|
|
22
|
+
try:
|
|
23
|
+
text = json.dumps(value, allow_nan=False, ensure_ascii=False)
|
|
24
|
+
# A lone surrogate survives json.dumps but not psycopg's UTF-8 encoding; fail here, before any SQL.
|
|
25
|
+
text.encode("utf-8")
|
|
26
|
+
except UnicodeEncodeError:
|
|
27
|
+
raise ValueError(
|
|
28
|
+
f"{what} contains a lone surrogate, which is not valid UTF-8 and PostgreSQL cannot store."
|
|
29
|
+
) from None
|
|
30
|
+
except (ValueError, TypeError, RecursionError) as exc:
|
|
31
|
+
raise ValueError(f"{what} is not JSON-serializable ({type(exc).__name__}).") from None
|
|
32
|
+
if _NUL_ESCAPE.search(text):
|
|
33
|
+
raise ValueError(f"{what} contains a NUL character, which PostgreSQL JSONB cannot store.")
|
|
34
|
+
return Jsonb(value, dumps=lambda _obj: text)
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Short, renewable, owner-tagged leases: the one coordination primitive this package offers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
import time
|
|
8
|
+
import uuid
|
|
9
|
+
from collections.abc import AsyncGenerator, Mapping
|
|
10
|
+
from contextlib import asynccontextmanager
|
|
11
|
+
from datetime import timedelta
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from agent_framework import SecretString
|
|
15
|
+
from psycopg import sql
|
|
16
|
+
|
|
17
|
+
from ._client import ClientHandle, LeaseLost, LeaseUnavailable, PostgresClient, require_text
|
|
18
|
+
from ._entra import EntraCredential
|
|
19
|
+
from ._retention import PurgeReport, RetentionPolicy, purge_rows
|
|
20
|
+
from ._store import BaseStore
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
_FIRST_DELAY = 0.1
|
|
25
|
+
_MAX_DELAY = 2.0
|
|
26
|
+
_RELEASE_SECONDS = 5.0
|
|
27
|
+
# Releases still running after their bound; the event loop keeps only weak references to tasks.
|
|
28
|
+
_late_releases: set[asyncio.Task[None]] = set()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async def _release_after_failure(lease: Lease) -> None:
|
|
32
|
+
"""Release after a block raised, logging a failure here so the task never ends with an exception.
|
|
33
|
+
|
|
34
|
+
A shielded task that fails after its waiter gave up is otherwise logged by asyncio itself, with the
|
|
35
|
+
traceback, whatever its done callbacks retrieve.
|
|
36
|
+
"""
|
|
37
|
+
try:
|
|
38
|
+
await lease.release()
|
|
39
|
+
except Exception:
|
|
40
|
+
logger.warning("Could not release a lease after its block raised; it frees itself on expiry.")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Lease:
|
|
44
|
+
"""A lease the caller holds; ``renew`` extends it, ``release`` gives it up.
|
|
45
|
+
|
|
46
|
+
``token`` is new on every acquisition, so two holders with the same ``owner`` never
|
|
47
|
+
renew or release each other's lease.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def __init__(self, store: PostgresLeases, resource: str, owner: str, ttl: timedelta, token: str) -> None:
|
|
51
|
+
self._store = store
|
|
52
|
+
self.resource = resource
|
|
53
|
+
self.owner = owner
|
|
54
|
+
self.ttl = ttl
|
|
55
|
+
self.token = token
|
|
56
|
+
|
|
57
|
+
async def renew(self) -> None:
|
|
58
|
+
"""Extend the lease by ``ttl`` from now; raise ``LeaseLost`` if it expired or changed hands."""
|
|
59
|
+
await self._store._renew(self) # pyright: ignore[reportPrivateUsage]
|
|
60
|
+
|
|
61
|
+
async def release(self) -> None:
|
|
62
|
+
"""Give the lease up. Releasing a lease that is no longer ours does nothing."""
|
|
63
|
+
await self._store._release(self) # pyright: ignore[reportPrivateUsage]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class PostgresLeases(BaseStore):
|
|
67
|
+
"""Leases keyed by ``(application_id, resource)``, judged by the database clock.
|
|
68
|
+
|
|
69
|
+
Acquisition is one upsert that succeeds only when the row is absent or expired, so
|
|
70
|
+
there is one holder per acquisition, whatever ``owner`` says; a holder that crashed
|
|
71
|
+
is recovered when its lease expires. No advisory locks are used, so a connection is
|
|
72
|
+
never pinned while the caller does slow work.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(
|
|
76
|
+
self,
|
|
77
|
+
*,
|
|
78
|
+
application_id: str,
|
|
79
|
+
connection_string: str | SecretString | None = None,
|
|
80
|
+
client: PostgresClient | ClientHandle | None = None,
|
|
81
|
+
credential: EntraCredential | None = None,
|
|
82
|
+
env_file_path: str | None = None,
|
|
83
|
+
env_file_encoding: str | None = None,
|
|
84
|
+
schema: str = "public",
|
|
85
|
+
table_prefix: str = "af_",
|
|
86
|
+
pool_kwargs: Mapping[str, Any] | None = None,
|
|
87
|
+
) -> None:
|
|
88
|
+
super().__init__(
|
|
89
|
+
application_id=application_id,
|
|
90
|
+
connection_string=connection_string,
|
|
91
|
+
client=client,
|
|
92
|
+
credential=credential,
|
|
93
|
+
env_file_path=env_file_path,
|
|
94
|
+
env_file_encoding=env_file_encoding,
|
|
95
|
+
schema=schema,
|
|
96
|
+
table_prefix=table_prefix,
|
|
97
|
+
retention=RetentionPolicy(),
|
|
98
|
+
pool_kwargs=pool_kwargs,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
async def try_acquire(self, resource: str, *, owner: str, ttl: timedelta) -> Lease | None:
|
|
102
|
+
"""Take the lease if it is free or expired; ``None`` while anyone, this owner included, holds it."""
|
|
103
|
+
require_text(resource, "resource")
|
|
104
|
+
require_text(owner, "owner")
|
|
105
|
+
if ttl <= timedelta(0):
|
|
106
|
+
raise ValueError("ttl must be positive.")
|
|
107
|
+
token = uuid.uuid4().hex
|
|
108
|
+
async with self._client.connection(read_committed=True) as connection:
|
|
109
|
+
cursor = await connection.execute(
|
|
110
|
+
sql.SQL(
|
|
111
|
+
"INSERT INTO {leases} (application_id, resource, owner, token, expires_at)"
|
|
112
|
+
" VALUES (%(app)s, %(resource)s, %(owner)s, %(token)s, now() + %(ttl)s)"
|
|
113
|
+
" ON CONFLICT (application_id, resource) DO UPDATE"
|
|
114
|
+
" SET owner = EXCLUDED.owner, token = EXCLUDED.token, expires_at = EXCLUDED.expires_at"
|
|
115
|
+
" WHERE {leases}.expires_at < now()"
|
|
116
|
+
" RETURNING token"
|
|
117
|
+
).format(leases=self._names.table("leases")),
|
|
118
|
+
{"app": self.application_id, "resource": resource, "owner": owner, "token": token, "ttl": ttl},
|
|
119
|
+
)
|
|
120
|
+
row = await cursor.fetchone()
|
|
121
|
+
return Lease(self, resource, owner, ttl, token) if row is not None else None
|
|
122
|
+
|
|
123
|
+
@asynccontextmanager
|
|
124
|
+
async def acquire(
|
|
125
|
+
self, resource: str, *, owner: str, ttl: timedelta, wait: timedelta = timedelta(0)
|
|
126
|
+
) -> AsyncGenerator[Lease]:
|
|
127
|
+
"""Hold the lease for the block, waiting up to ``wait`` for it to free up; release on exit.
|
|
128
|
+
|
|
129
|
+
If the block raises and the release then fails, the release failure is logged and the
|
|
130
|
+
block's exception propagates. If the block succeeded, a release failure propagates.
|
|
131
|
+
"""
|
|
132
|
+
deadline = time.monotonic() + wait.total_seconds()
|
|
133
|
+
delay = _FIRST_DELAY
|
|
134
|
+
while True:
|
|
135
|
+
lease = await self.try_acquire(resource, owner=owner, ttl=ttl)
|
|
136
|
+
if lease is not None:
|
|
137
|
+
break
|
|
138
|
+
if time.monotonic() >= deadline:
|
|
139
|
+
raise LeaseUnavailable("The lease is held by another acquisition.", resource=resource)
|
|
140
|
+
await asyncio.sleep(min(delay, max(0.0, deadline - time.monotonic())))
|
|
141
|
+
delay = min(delay * 2, _MAX_DELAY)
|
|
142
|
+
try:
|
|
143
|
+
yield lease
|
|
144
|
+
except BaseException:
|
|
145
|
+
# The block may have been cancelled; the release still runs, shielded and bounded, so a slow
|
|
146
|
+
# database cannot hold a cancelling task. An unreleased lease frees itself on expiry.
|
|
147
|
+
release = asyncio.ensure_future(_release_after_failure(lease))
|
|
148
|
+
_late_releases.add(release)
|
|
149
|
+
release.add_done_callback(_late_releases.discard)
|
|
150
|
+
try:
|
|
151
|
+
await asyncio.wait_for(asyncio.shield(release), timeout=_RELEASE_SECONDS)
|
|
152
|
+
except TimeoutError:
|
|
153
|
+
logger.warning(
|
|
154
|
+
"A lease release did not finish in time after its block raised; it continues in the background."
|
|
155
|
+
)
|
|
156
|
+
raise
|
|
157
|
+
await lease.release()
|
|
158
|
+
|
|
159
|
+
async def purge(self) -> PurgeReport:
|
|
160
|
+
"""Delete leases whose expiry has passed; an expired lease is already free, so nothing changes hands."""
|
|
161
|
+
count = await purge_rows(
|
|
162
|
+
self._client,
|
|
163
|
+
table=self._names.table("leases"),
|
|
164
|
+
where=sql.SQL("application_id = %(app)s"),
|
|
165
|
+
params={"app": self.application_id},
|
|
166
|
+
mode="delete",
|
|
167
|
+
tombstone=None,
|
|
168
|
+
)
|
|
169
|
+
return PurgeReport({f"{self._names.prefix}leases": count})
|
|
170
|
+
|
|
171
|
+
def _lease_params(self, lease: Lease) -> dict[str, object]:
|
|
172
|
+
return {"app": self.application_id, "resource": lease.resource, "owner": lease.owner, "token": lease.token}
|
|
173
|
+
|
|
174
|
+
async def _renew(self, lease: Lease) -> None:
|
|
175
|
+
async with self._client.connection(read_committed=True) as connection:
|
|
176
|
+
cursor = await connection.execute(
|
|
177
|
+
sql.SQL(
|
|
178
|
+
"UPDATE {leases} SET expires_at = now() + %(ttl)s"
|
|
179
|
+
" WHERE application_id = %(app)s AND resource = %(resource)s AND owner = %(owner)s"
|
|
180
|
+
" AND token = %(token)s AND expires_at >= now() RETURNING owner"
|
|
181
|
+
).format(leases=self._names.table("leases")),
|
|
182
|
+
self._lease_params(lease) | {"ttl": lease.ttl},
|
|
183
|
+
)
|
|
184
|
+
if await cursor.fetchone() is None:
|
|
185
|
+
raise LeaseLost("The lease expired or was taken over.", resource=lease.resource)
|
|
186
|
+
|
|
187
|
+
async def _release(self, lease: Lease) -> None:
|
|
188
|
+
async with self._client.connection(read_committed=True) as connection:
|
|
189
|
+
await connection.execute(
|
|
190
|
+
sql.SQL(
|
|
191
|
+
"DELETE FROM {leases}"
|
|
192
|
+
" WHERE application_id = %(app)s AND resource = %(resource)s AND owner = %(owner)s"
|
|
193
|
+
" AND token = %(token)s"
|
|
194
|
+
).format(leases=self._names.table("leases")),
|
|
195
|
+
self._lease_params(lease),
|
|
196
|
+
)
|