agent-framework-sql-server 1.0.0a261002__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_framework_sql_server/__init__.py +22 -0
- agent_framework_sql_server/_sql.py +344 -0
- agent_framework_sql_server/_vector_store.py +705 -0
- agent_framework_sql_server/py.typed +0 -0
- agent_framework_sql_server-1.0.0a261002.dist-info/METADATA +195 -0
- agent_framework_sql_server-1.0.0a261002.dist-info/RECORD +8 -0
- agent_framework_sql_server-1.0.0a261002.dist-info/WHEEL +4 -0
- agent_framework_sql_server-1.0.0a261002.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Copyright (c) Microsoft. All rights reserved.
|
|
2
|
+
|
|
3
|
+
"""SQL Server native vector collections and stores for Agent Framework."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import importlib.metadata
|
|
8
|
+
|
|
9
|
+
from ._vector_store import SqlServerCollection, SqlServerCommittedCleanupException, SqlServerSettings, SqlServerStore
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
__version__ = importlib.metadata.version(__name__)
|
|
13
|
+
except importlib.metadata.PackageNotFoundError:
|
|
14
|
+
__version__ = "0.0.0"
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"SqlServerCollection",
|
|
18
|
+
"SqlServerCommittedCleanupException",
|
|
19
|
+
"SqlServerSettings",
|
|
20
|
+
"SqlServerStore",
|
|
21
|
+
"__version__",
|
|
22
|
+
]
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
# Copyright (c) Microsoft. All rights reserved.
|
|
2
|
+
|
|
3
|
+
"""SQL Server identifiers, values, and portable filter translation."""
|
|
4
|
+
|
|
5
|
+
# pyright: reportUnusedFunction=false, reportUnusedClass=false
|
|
6
|
+
# Package-private helpers are consumed by the sibling vector-store module.
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import math
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from datetime import date, datetime, timezone
|
|
14
|
+
from typing import Any, cast
|
|
15
|
+
from uuid import UUID
|
|
16
|
+
|
|
17
|
+
from agent_framework import FilterGroup, VectorStoreCollectionDefinition, VectorStoreField
|
|
18
|
+
from agent_framework._vector_filters import FilterExpression
|
|
19
|
+
from agent_framework.exceptions import IntegrationInvalidResponseException
|
|
20
|
+
|
|
21
|
+
MAX_PARAMETERS = 2000 # SQL Server's limit is 2100; leave room for paging and search arguments.
|
|
22
|
+
_FLOAT32_MAX = 3.4028234663852886e38
|
|
23
|
+
_COLLATION = "Latin1_General_100_BIN2"
|
|
24
|
+
_ORDER_OPERATORS = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}
|
|
25
|
+
_METRICS = {
|
|
26
|
+
"DEFAULT": ("cosine", "distance"),
|
|
27
|
+
"cosine_distance": ("cosine", "distance"),
|
|
28
|
+
"cosine_similarity": ("cosine", "similarity"),
|
|
29
|
+
"euclidean_distance": ("euclidean", "distance"),
|
|
30
|
+
"dot_prod": ("dot", "negative"),
|
|
31
|
+
"negative_dot_prod": ("dot", "distance"),
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _quote_identifier(name: str) -> str:
|
|
36
|
+
"""Quote one SQL identifier; data values must be passed as bound parameters."""
|
|
37
|
+
if not isinstance(name, str) or not name or "\0" in name or len(name.encode("utf-16-le")) // 2 > 128:
|
|
38
|
+
raise ValueError("SQL Server identifiers must contain 1-128 UTF-16 code units and no NUL.")
|
|
39
|
+
return f"[{name.replace(']', ']]')}]"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _metric_for(field: VectorStoreField) -> tuple[str, str]:
|
|
43
|
+
try:
|
|
44
|
+
return _METRICS[field.distance_function or "DEFAULT"]
|
|
45
|
+
except KeyError:
|
|
46
|
+
raise NotImplementedError(f"Unsupported SQL Server distance function '{field.distance_function}'.") from None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _column_type(field: VectorStoreField) -> str:
|
|
50
|
+
if field.field_type == "vector":
|
|
51
|
+
dimensions = field.dimensions
|
|
52
|
+
if type(dimensions) is not int or not 1 <= dimensions <= 1998:
|
|
53
|
+
raise ValueError("SQL Server vector dimensions must be an integer between 1 and 1998.")
|
|
54
|
+
if field.type_ not in (None, "float", "float32"):
|
|
55
|
+
raise NotImplementedError("SQL Server VECTOR columns support float32 vectors only.")
|
|
56
|
+
return f"VECTOR({dimensions})"
|
|
57
|
+
types = {
|
|
58
|
+
"int": "BIGINT",
|
|
59
|
+
"float": "FLOAT(53)",
|
|
60
|
+
"bool": "BIT",
|
|
61
|
+
"UUID": "UNIQUEIDENTIFIER",
|
|
62
|
+
"bytes": "VARBINARY(MAX)",
|
|
63
|
+
"date": "DATE",
|
|
64
|
+
"datetime": "DATETIME2(7)",
|
|
65
|
+
"list": "NVARCHAR(MAX)",
|
|
66
|
+
"dict": "NVARCHAR(MAX)",
|
|
67
|
+
}
|
|
68
|
+
if field.type_ == "str":
|
|
69
|
+
width = "450" if field.field_type == "key" or field.is_indexed else "MAX"
|
|
70
|
+
return f"NVARCHAR({width}) COLLATE {_COLLATION}"
|
|
71
|
+
if field.type_ not in types:
|
|
72
|
+
raise NotImplementedError(f"Field '{field.name}' needs a supported explicit type; got '{field.type_}'.")
|
|
73
|
+
return types[field.type_]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _validate_json(value: Any) -> None:
|
|
77
|
+
if value is None or type(value) in (str, bool, int):
|
|
78
|
+
return
|
|
79
|
+
if type(value) is float:
|
|
80
|
+
if not math.isfinite(value):
|
|
81
|
+
raise ValueError("JSON values must be finite.")
|
|
82
|
+
return
|
|
83
|
+
if isinstance(value, list):
|
|
84
|
+
for item in cast(list[Any], value):
|
|
85
|
+
_validate_json(item)
|
|
86
|
+
return
|
|
87
|
+
if isinstance(value, dict):
|
|
88
|
+
for key, item in cast(dict[Any, Any], value).items():
|
|
89
|
+
if not isinstance(key, str):
|
|
90
|
+
raise TypeError("JSON object keys must be strings.")
|
|
91
|
+
_validate_json(item)
|
|
92
|
+
return
|
|
93
|
+
raise TypeError("JSON fields support only JSON scalars, lists, and string-keyed dictionaries.")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _prepare_vector(field: VectorStoreField, value: Any) -> str:
|
|
97
|
+
if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)):
|
|
98
|
+
raise TypeError(f"Vector field '{field.name}' requires a dense numeric sequence.")
|
|
99
|
+
vector = cast(Sequence[float | int], value)
|
|
100
|
+
if len(vector) != field.dimensions:
|
|
101
|
+
raise ValueError(f"Vector field '{field.name}' requires {field.dimensions} dimensions.")
|
|
102
|
+
components: list[float] = []
|
|
103
|
+
for element in vector:
|
|
104
|
+
if type(element) not in (float, int):
|
|
105
|
+
raise TypeError(f"Vector field '{field.name}' requires numeric elements, not booleans or strings.")
|
|
106
|
+
try:
|
|
107
|
+
component = float(element)
|
|
108
|
+
except OverflowError as exc:
|
|
109
|
+
raise ValueError(f"Vector field '{field.name}' has an element outside the float32 range.") from exc
|
|
110
|
+
if not math.isfinite(component) or abs(component) > _FLOAT32_MAX:
|
|
111
|
+
raise ValueError(f"Vector field '{field.name}' requires finite float32 elements.")
|
|
112
|
+
components.append(component)
|
|
113
|
+
return json.dumps(components, separators=(",", ":"), allow_nan=False)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _prepare_value(field: VectorStoreField, value: Any) -> Any:
|
|
117
|
+
if value is None:
|
|
118
|
+
return None
|
|
119
|
+
if field.field_type == "vector":
|
|
120
|
+
return _prepare_vector(field, value)
|
|
121
|
+
kind = field.type_
|
|
122
|
+
if kind == "UUID" and isinstance(value, (str, UUID)):
|
|
123
|
+
return str(UUID(str(value)))
|
|
124
|
+
if kind == "int" and type(value) is int:
|
|
125
|
+
if not -(2**63) <= value < 2**63:
|
|
126
|
+
raise ValueError(f"Field '{field.name}' exceeds the SQL Server bigint range.")
|
|
127
|
+
return value
|
|
128
|
+
if kind == "float" and type(value) in (float, int):
|
|
129
|
+
try:
|
|
130
|
+
number = float(value)
|
|
131
|
+
except OverflowError as exc:
|
|
132
|
+
raise ValueError(f"Field '{field.name}' requires a finite number.") from exc
|
|
133
|
+
if not math.isfinite(number):
|
|
134
|
+
raise ValueError(f"Field '{field.name}' requires a finite number.")
|
|
135
|
+
return number
|
|
136
|
+
if kind == "bool" and type(value) is bool:
|
|
137
|
+
return value
|
|
138
|
+
if kind == "str" and isinstance(value, str):
|
|
139
|
+
if (field.field_type == "key" or field.is_indexed) and len(value.encode("utf-16-le")) // 2 > 450:
|
|
140
|
+
raise ValueError(f"Field '{field.name}' exceeds the indexed NVARCHAR(450) limit.")
|
|
141
|
+
if field.field_type == "key" and value.endswith(" "):
|
|
142
|
+
raise ValueError(
|
|
143
|
+
"SQL Server string keys cannot end in a space; SQL Server ignores trailing spaces in keys."
|
|
144
|
+
)
|
|
145
|
+
return value
|
|
146
|
+
if kind == "bytes" and isinstance(value, bytes):
|
|
147
|
+
return value
|
|
148
|
+
if kind == "date" and (type(value) is date or isinstance(value, str)):
|
|
149
|
+
return date.fromisoformat(value) if isinstance(value, str) else value
|
|
150
|
+
if kind == "datetime" and isinstance(value, (datetime, str)):
|
|
151
|
+
resolved = datetime.fromisoformat(value.replace("Z", "+00:00")) if isinstance(value, str) else value
|
|
152
|
+
if resolved.tzinfo is None or resolved.utcoffset() is None:
|
|
153
|
+
raise ValueError(f"Datetime field '{field.name}' requires a timezone.")
|
|
154
|
+
return resolved.astimezone(timezone.utc).replace(tzinfo=None)
|
|
155
|
+
if (kind == "list" and isinstance(value, list)) or (kind == "dict" and isinstance(value, dict)):
|
|
156
|
+
_validate_json(value)
|
|
157
|
+
return json.dumps(value, separators=(",", ":"), ensure_ascii=False, allow_nan=False)
|
|
158
|
+
raise TypeError(f"Field '{field.name}' requires a value of type '{kind}'.")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _parse_value(field: VectorStoreField, value: Any) -> Any:
|
|
162
|
+
if value is None:
|
|
163
|
+
return None
|
|
164
|
+
kind = field.type_
|
|
165
|
+
if field.field_type == "vector" or kind in ("list", "dict"):
|
|
166
|
+
if not isinstance(value, str):
|
|
167
|
+
raise IntegrationInvalidResponseException(f"SQL Server returned a non-JSON value for '{field.name}'.")
|
|
168
|
+
try:
|
|
169
|
+
parsed: Any = json.loads(value)
|
|
170
|
+
except json.JSONDecodeError as exc:
|
|
171
|
+
raise IntegrationInvalidResponseException(f"SQL Server returned invalid JSON for '{field.name}'.") from exc
|
|
172
|
+
if field.field_type == "vector":
|
|
173
|
+
if not isinstance(parsed, list):
|
|
174
|
+
raise IntegrationInvalidResponseException(f"SQL Server returned an invalid vector for '{field.name}'.")
|
|
175
|
+
vector_values = cast(list[Any], parsed)
|
|
176
|
+
if len(vector_values) != field.dimensions:
|
|
177
|
+
raise IntegrationInvalidResponseException(f"SQL Server returned an invalid vector for '{field.name}'.")
|
|
178
|
+
components: list[float] = []
|
|
179
|
+
for item in vector_values:
|
|
180
|
+
if type(item) not in (int, float):
|
|
181
|
+
raise IntegrationInvalidResponseException(
|
|
182
|
+
f"SQL Server returned invalid vector elements for '{field.name}'."
|
|
183
|
+
)
|
|
184
|
+
try:
|
|
185
|
+
number = float(item)
|
|
186
|
+
except OverflowError as exc:
|
|
187
|
+
raise IntegrationInvalidResponseException(
|
|
188
|
+
f"SQL Server returned invalid vector elements for '{field.name}'."
|
|
189
|
+
) from exc
|
|
190
|
+
if not math.isfinite(number) or abs(number) > _FLOAT32_MAX:
|
|
191
|
+
raise IntegrationInvalidResponseException(
|
|
192
|
+
f"SQL Server returned invalid vector elements for '{field.name}'."
|
|
193
|
+
)
|
|
194
|
+
components.append(number)
|
|
195
|
+
return components
|
|
196
|
+
if not isinstance(parsed, list if kind == "list" else dict):
|
|
197
|
+
raise IntegrationInvalidResponseException(f"SQL Server returned the wrong JSON type for '{field.name}'.")
|
|
198
|
+
return cast(list[Any] | dict[str, Any], parsed)
|
|
199
|
+
if kind == "UUID":
|
|
200
|
+
try:
|
|
201
|
+
return UUID(str(value))
|
|
202
|
+
except ValueError as exc:
|
|
203
|
+
raise IntegrationInvalidResponseException(
|
|
204
|
+
f"SQL Server returned an invalid UUID for '{field.name}'."
|
|
205
|
+
) from exc
|
|
206
|
+
if kind == "datetime":
|
|
207
|
+
if not isinstance(value, datetime):
|
|
208
|
+
raise IntegrationInvalidResponseException(f"SQL Server returned an invalid datetime for '{field.name}'.")
|
|
209
|
+
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
|
|
210
|
+
if kind == "bool" and type(value) is int and value in (0, 1):
|
|
211
|
+
return bool(value)
|
|
212
|
+
return value
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _filter_field(definition: VectorStoreCollectionDefinition, name: str) -> VectorStoreField:
|
|
216
|
+
if "." in name:
|
|
217
|
+
raise NotImplementedError("SQL Server filters and ordering do not support nested field paths.")
|
|
218
|
+
field = definition.try_get_field(name)
|
|
219
|
+
if field is None:
|
|
220
|
+
raise ValueError(f"Unknown SQL Server field '{name}'.")
|
|
221
|
+
if field.field_type == "vector":
|
|
222
|
+
raise NotImplementedError("Filtering and ordering vector columns is not supported.")
|
|
223
|
+
return field
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _numeric_filter_value(value: int | float) -> int | float:
|
|
227
|
+
try:
|
|
228
|
+
if not math.isfinite(value):
|
|
229
|
+
raise ValueError("Numeric filter values must be finite.")
|
|
230
|
+
except OverflowError as exc:
|
|
231
|
+
raise ValueError("Numeric filter values must fit in the SQL Server numeric range.") from exc
|
|
232
|
+
if type(value) is int and not -(2**63) <= value < 2**63:
|
|
233
|
+
raise ValueError("Integer filter values must fit in the SQL Server bigint range.")
|
|
234
|
+
return value
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _prepare_numeric_filter_value(field: VectorStoreField, value: int | float) -> int | float:
|
|
238
|
+
return _numeric_filter_value(value) if field.type_ == "int" else _prepare_value(field, value)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class _FilterCompiler:
|
|
242
|
+
"""Compile bounded data-only filter expressions into parameterized two-valued T-SQL."""
|
|
243
|
+
|
|
244
|
+
def __init__(self, definition: VectorStoreCollectionDefinition, *, alias: str = "") -> None:
|
|
245
|
+
self.definition = definition
|
|
246
|
+
self.alias = alias
|
|
247
|
+
self.parameters: list[Any] = []
|
|
248
|
+
|
|
249
|
+
def compile(self, expression: FilterExpression) -> tuple[str, list[Any]]:
|
|
250
|
+
"""Compile one filter with its parameters in placeholder order."""
|
|
251
|
+
self.parameters = []
|
|
252
|
+
return self._condition(expression), self.parameters
|
|
253
|
+
|
|
254
|
+
def _bind(self, value: Any) -> str:
|
|
255
|
+
if len(self.parameters) >= MAX_PARAMETERS:
|
|
256
|
+
raise ValueError(f"SQL Server queries support at most {MAX_PARAMETERS} bound parameters.")
|
|
257
|
+
self.parameters.append(value)
|
|
258
|
+
return "?"
|
|
259
|
+
|
|
260
|
+
def _equality(self, field: VectorStoreField, column: str, value: Any) -> str:
|
|
261
|
+
if value is None:
|
|
262
|
+
return f"{column} IS NULL"
|
|
263
|
+
kind = field.type_
|
|
264
|
+
if kind in ("list", "dict"):
|
|
265
|
+
raise NotImplementedError("Equality filtering JSON fields is not supported.")
|
|
266
|
+
if kind == "bool" and type(value) is not bool:
|
|
267
|
+
return "1 = 0"
|
|
268
|
+
if kind != "bool" and isinstance(value, bool):
|
|
269
|
+
return "1 = 0"
|
|
270
|
+
if kind in ("int", "float") and type(value) in (int, float):
|
|
271
|
+
adapted = _prepare_numeric_filter_value(field, value)
|
|
272
|
+
elif (
|
|
273
|
+
(kind == "str" and isinstance(value, str))
|
|
274
|
+
or (kind == "bytes" and isinstance(value, bytes))
|
|
275
|
+
or (kind == "UUID" and isinstance(value, (UUID, str)))
|
|
276
|
+
or (kind == "date" and (type(value) is date or isinstance(value, str)))
|
|
277
|
+
or (kind == "datetime" and isinstance(value, (datetime, str)))
|
|
278
|
+
or (kind == "bool" and type(value) is bool)
|
|
279
|
+
):
|
|
280
|
+
adapted = _prepare_value(field, value)
|
|
281
|
+
else:
|
|
282
|
+
return "1 = 0"
|
|
283
|
+
if kind == "str":
|
|
284
|
+
# SQL Server pads trailing spaces even with binary collations; compare bytes instead.
|
|
285
|
+
return (
|
|
286
|
+
f"({column} IS NOT NULL AND CONVERT(VARBINARY(MAX), {column}) = "
|
|
287
|
+
f"CONVERT(VARBINARY(MAX), CONVERT(NVARCHAR(MAX), {self._bind(adapted)})))"
|
|
288
|
+
)
|
|
289
|
+
return f"({column} IS NOT NULL AND {column} = {self._bind(adapted)})"
|
|
290
|
+
|
|
291
|
+
def _condition(self, expression: FilterExpression) -> str:
|
|
292
|
+
if isinstance(expression, FilterGroup):
|
|
293
|
+
conditions = [self._condition(child) for child in expression.filters]
|
|
294
|
+
if expression.operator == "not":
|
|
295
|
+
return f"(NOT ({conditions[0]}))"
|
|
296
|
+
joiner = " AND " if expression.operator == "and" else " OR "
|
|
297
|
+
return f"({joiner.join(conditions)})"
|
|
298
|
+
field = _filter_field(self.definition, expression.field_name)
|
|
299
|
+
column = f"{self.alias}{_quote_identifier(field.storage_name or field.name)}"
|
|
300
|
+
op, value = expression.operator, expression.value
|
|
301
|
+
if op == "exists":
|
|
302
|
+
return "1 = 1" # SQL columns exist even when their values are NULL.
|
|
303
|
+
if op == "is_null":
|
|
304
|
+
return f"{column} IS NULL"
|
|
305
|
+
if op == "is_not_null":
|
|
306
|
+
return f"{column} IS NOT NULL"
|
|
307
|
+
if op in ("eq", "ne"):
|
|
308
|
+
equality = self._equality(field, column, value)
|
|
309
|
+
return equality if op == "eq" else f"(NOT ({equality}))"
|
|
310
|
+
if op in ("in", "not_in"):
|
|
311
|
+
choices = " OR ".join(self._equality(field, column, item) for item in value) or "1 = 0"
|
|
312
|
+
membership = f"({choices})" if op == "in" else f"(NOT ({choices}))"
|
|
313
|
+
return f"({column} IS NOT NULL AND {membership})"
|
|
314
|
+
if op in _ORDER_OPERATORS or op == "between":
|
|
315
|
+
if field.type_ not in ("int", "float", "date", "datetime"):
|
|
316
|
+
raise NotImplementedError(f"Ordered SQL Server filtering is not supported for '{field.type_}'.")
|
|
317
|
+
values: Sequence[Any] = value if op == "between" else [value]
|
|
318
|
+
if any(item is None or isinstance(item, bool) for item in values):
|
|
319
|
+
raise TypeError("Ordered filter operands must be non-null scalars of the column's type.")
|
|
320
|
+
adapted = [
|
|
321
|
+
_prepare_numeric_filter_value(field, item)
|
|
322
|
+
if field.type_ in ("int", "float") and type(item) in (int, float)
|
|
323
|
+
else _prepare_value(field, item)
|
|
324
|
+
for item in values
|
|
325
|
+
]
|
|
326
|
+
comparison = (
|
|
327
|
+
f"{column} BETWEEN {self._bind(adapted[0])} AND {self._bind(adapted[1])}"
|
|
328
|
+
if op == "between"
|
|
329
|
+
else f"{column} {_ORDER_OPERATORS[op]} {self._bind(adapted[0])}"
|
|
330
|
+
)
|
|
331
|
+
return f"({column} IS NOT NULL AND {comparison})"
|
|
332
|
+
if op in ("contains_text", "starts_with", "ends_with"):
|
|
333
|
+
if field.type_ != "str":
|
|
334
|
+
raise TypeError("Text filtering requires a string column.")
|
|
335
|
+
if not isinstance(value, str):
|
|
336
|
+
raise TypeError("Text filtering requires a string operand.")
|
|
337
|
+
if "\0" in value:
|
|
338
|
+
raise ValueError("SQL Server LIKE does not support NUL characters.")
|
|
339
|
+
escaped = value.replace("!", "!!").replace("%", "!%").replace("_", "!_").replace("[", "[[]")
|
|
340
|
+
pattern = ("%" if op != "starts_with" else "") + escaped + ("%" if op != "ends_with" else "")
|
|
341
|
+
if len(pattern.encode("utf-16-le")) > 8000:
|
|
342
|
+
raise ValueError("SQL Server LIKE patterns cannot exceed 8000 bytes.")
|
|
343
|
+
return f"({column} IS NOT NULL AND {column} COLLATE {_COLLATION} LIKE {self._bind(pattern)} ESCAPE '!')"
|
|
344
|
+
raise NotImplementedError(f"Unsupported SQL Server filter operator '{op}'.")
|
|
@@ -0,0 +1,705 @@
|
|
|
1
|
+
# Copyright (c) Microsoft. All rights reserved.
|
|
2
|
+
|
|
3
|
+
"""Asynchronous Agent Framework collections backed by SQL Server VECTOR columns."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import hashlib
|
|
9
|
+
import logging
|
|
10
|
+
import math
|
|
11
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
12
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
13
|
+
from dataclasses import replace
|
|
14
|
+
from typing import Any, ClassVar, Generic, cast
|
|
15
|
+
|
|
16
|
+
import mssql_python
|
|
17
|
+
from agent_framework import (
|
|
18
|
+
BaseVectorCollection,
|
|
19
|
+
BaseVectorSearch,
|
|
20
|
+
BaseVectorStore,
|
|
21
|
+
SearchResults,
|
|
22
|
+
SecretString,
|
|
23
|
+
VectorStoreCollectionDefinition,
|
|
24
|
+
load_settings,
|
|
25
|
+
)
|
|
26
|
+
from agent_framework._vector_filters import FilterExpression
|
|
27
|
+
from agent_framework._vectors import EmbeddingClient, SearchType, Vector
|
|
28
|
+
from agent_framework.exceptions import IntegrationException, IntegrationInvalidResponseException
|
|
29
|
+
from typing_extensions import Self, TypedDict, TypeVar
|
|
30
|
+
|
|
31
|
+
from ._sql import (
|
|
32
|
+
MAX_PARAMETERS,
|
|
33
|
+
_column_type, # pyright: ignore[reportPrivateUsage]
|
|
34
|
+
_filter_field, # pyright: ignore[reportPrivateUsage]
|
|
35
|
+
_FilterCompiler, # pyright: ignore[reportPrivateUsage]
|
|
36
|
+
_metric_for, # pyright: ignore[reportPrivateUsage]
|
|
37
|
+
_parse_value, # pyright: ignore[reportPrivateUsage]
|
|
38
|
+
_prepare_value, # pyright: ignore[reportPrivateUsage]
|
|
39
|
+
_quote_identifier, # pyright: ignore[reportPrivateUsage]
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
KeyT = TypeVar("KeyT", default=Any)
|
|
43
|
+
ModelT = TypeVar("ModelT", default=Any)
|
|
44
|
+
ResultT = TypeVar("ResultT")
|
|
45
|
+
_KEY_BATCH_SIZE = 1000
|
|
46
|
+
logger = logging.getLogger(__name__)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class SqlServerCommittedCleanupException(IntegrationException):
|
|
50
|
+
"""The SQL Server transaction committed, but closing its connection failed."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class SqlServerSettings(TypedDict, total=False):
|
|
54
|
+
"""Connection settings resolved from explicit values, a selected .env file, or ``SQL_SERVER_`` variables."""
|
|
55
|
+
|
|
56
|
+
connection_string: SecretString | None
|
|
57
|
+
"""Driver connection string, resolved from ``SQL_SERVER_CONNECTION_STRING`` and kept masked."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _validate_options(options: Mapping[str, Any] | None) -> None:
|
|
61
|
+
if options:
|
|
62
|
+
raise NotImplementedError(f"Unsupported SQL Server operation option(s): {', '.join(sorted(options))}.")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _execute(cursor: mssql_python.Cursor, statement: str, parameters: Sequence[Any] = ()) -> None:
|
|
66
|
+
if parameters:
|
|
67
|
+
cursor.execute(statement, tuple(parameters)) # pyright: ignore[reportUnknownMemberType]
|
|
68
|
+
else:
|
|
69
|
+
cursor.execute(statement) # pyright: ignore[reportUnknownMemberType]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _rows(cursor: mssql_python.Cursor, count: int) -> list[Sequence[Any]]:
|
|
73
|
+
result: list[Sequence[Any]] = [tuple(row) for row in cursor.fetchall()]
|
|
74
|
+
for row in result:
|
|
75
|
+
if len(row) != count:
|
|
76
|
+
raise IntegrationInvalidResponseException("SQL Server returned a row with an unexpected column count.")
|
|
77
|
+
return result
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _check_parameter_count(parameters: Sequence[Any]) -> None:
|
|
81
|
+
if len(parameters) > MAX_PARAMETERS:
|
|
82
|
+
raise ValueError(f"SQL Server queries support at most {MAX_PARAMETERS} bound parameters.")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class _Client:
|
|
86
|
+
"""Own one worker; every operation creates and releases a connection on that worker."""
|
|
87
|
+
|
|
88
|
+
def __init__(self, connection_string: SecretString | None, *, query_timeout: int | None = None) -> None:
|
|
89
|
+
if connection_string is None:
|
|
90
|
+
raise ValueError("SQL_SERVER_CONNECTION_STRING or an explicit connection_string is required.")
|
|
91
|
+
if not connection_string.get_secret_value().strip():
|
|
92
|
+
raise ValueError("connection_string must not be empty.")
|
|
93
|
+
if query_timeout is not None and (type(query_timeout) is not int or query_timeout < 0):
|
|
94
|
+
raise ValueError("query_timeout must be a non-negative integer number of seconds.")
|
|
95
|
+
self.connection_string = connection_string
|
|
96
|
+
self.query_timeout = query_timeout
|
|
97
|
+
self.closed = False
|
|
98
|
+
self._executor: ThreadPoolExecutor | None = None
|
|
99
|
+
self._close_lock = asyncio.Lock()
|
|
100
|
+
|
|
101
|
+
def ensure_open(self) -> None:
|
|
102
|
+
if self.closed:
|
|
103
|
+
raise RuntimeError("The SQL Server client is closed.")
|
|
104
|
+
|
|
105
|
+
def _run_sync(self, operation: Callable[[mssql_python.Cursor], ResultT]) -> ResultT:
|
|
106
|
+
try:
|
|
107
|
+
connection = mssql_python.connect(self.connection_string.get_secret_value(), autocommit=False)
|
|
108
|
+
committed = False
|
|
109
|
+
failed = False
|
|
110
|
+
try:
|
|
111
|
+
try:
|
|
112
|
+
if self.query_timeout is not None:
|
|
113
|
+
connection.timeout = self.query_timeout
|
|
114
|
+
cursor = connection.cursor()
|
|
115
|
+
try:
|
|
116
|
+
result = operation(cursor)
|
|
117
|
+
finally:
|
|
118
|
+
cursor.close()
|
|
119
|
+
connection.commit()
|
|
120
|
+
committed = True
|
|
121
|
+
return result
|
|
122
|
+
except BaseException:
|
|
123
|
+
failed = True
|
|
124
|
+
connection.rollback()
|
|
125
|
+
raise
|
|
126
|
+
finally:
|
|
127
|
+
try:
|
|
128
|
+
connection.close()
|
|
129
|
+
except Exception as exc:
|
|
130
|
+
if committed:
|
|
131
|
+
raise SqlServerCommittedCleanupException(
|
|
132
|
+
"SQL Server transaction committed, but connection cleanup failed; "
|
|
133
|
+
"do not retry this operation automatically."
|
|
134
|
+
) from exc
|
|
135
|
+
if failed:
|
|
136
|
+
logger.warning(
|
|
137
|
+
"SQL Server connection cleanup also failed after an operation error.", exc_info=exc
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
raise
|
|
141
|
+
except mssql_python.Error as exc:
|
|
142
|
+
raise IntegrationException("SQL Server operation failed; inspect the chained driver exception.") from exc
|
|
143
|
+
|
|
144
|
+
async def run(self, operation: Callable[[mssql_python.Cursor], ResultT]) -> ResultT:
|
|
145
|
+
"""Offload a whole transaction and wait for worker cleanup on cancellation."""
|
|
146
|
+
self.ensure_open()
|
|
147
|
+
if self._executor is None:
|
|
148
|
+
self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="af-sql-server")
|
|
149
|
+
future = asyncio.get_running_loop().run_in_executor(self._executor, self._run_sync, operation)
|
|
150
|
+
try:
|
|
151
|
+
return await asyncio.shield(future)
|
|
152
|
+
except asyncio.CancelledError:
|
|
153
|
+
while not future.done():
|
|
154
|
+
try:
|
|
155
|
+
await asyncio.shield(future)
|
|
156
|
+
except asyncio.CancelledError:
|
|
157
|
+
continue
|
|
158
|
+
except Exception:
|
|
159
|
+
break
|
|
160
|
+
if not future.cancelled() and (error := future.exception()) is not None:
|
|
161
|
+
logger.warning("SQL Server worker failed after caller cancellation.", exc_info=error)
|
|
162
|
+
raise
|
|
163
|
+
|
|
164
|
+
async def close(self) -> None:
|
|
165
|
+
"""Wait for queued operations and shut down the owned worker."""
|
|
166
|
+
async with self._close_lock:
|
|
167
|
+
if self.closed:
|
|
168
|
+
return
|
|
169
|
+
self.closed = True
|
|
170
|
+
if self._executor is not None:
|
|
171
|
+
await asyncio.to_thread(self._executor.shutdown, wait=True)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _create_client(
|
|
175
|
+
connection_string: str | SecretString | None,
|
|
176
|
+
*,
|
|
177
|
+
query_timeout: int | None,
|
|
178
|
+
env_file_path: str | None,
|
|
179
|
+
env_file_encoding: str | None,
|
|
180
|
+
) -> _Client:
|
|
181
|
+
settings = load_settings(
|
|
182
|
+
SqlServerSettings,
|
|
183
|
+
env_prefix="SQL_SERVER_",
|
|
184
|
+
connection_string=connection_string,
|
|
185
|
+
env_file_path=env_file_path,
|
|
186
|
+
env_file_encoding=env_file_encoding,
|
|
187
|
+
)
|
|
188
|
+
resolved = settings.get("connection_string")
|
|
189
|
+
if resolved is not None and not isinstance(resolved, SecretString):
|
|
190
|
+
raise TypeError("connection_string must be a string or SecretString.")
|
|
191
|
+
return _Client(resolved, query_timeout=query_timeout)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class SqlServerCollection(
|
|
195
|
+
BaseVectorCollection[KeyT, ModelT],
|
|
196
|
+
BaseVectorSearch[KeyT, ModelT],
|
|
197
|
+
Generic[KeyT, ModelT],
|
|
198
|
+
):
|
|
199
|
+
"""Store typed rows and perform exact searches on SQL Server native VECTOR columns.
|
|
200
|
+
|
|
201
|
+
Tables are created in an existing schema and are never migrated. Connections
|
|
202
|
+
are owned by the collection, or by its store when created from a store.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
supported_key_types: ClassVar[set[str] | None] = {"str", "int", "UUID"}
|
|
206
|
+
supported_vector_types: ClassVar[set[str] | None] = {"float", "float32"}
|
|
207
|
+
supported_search_types: ClassVar[set[SearchType]] = {"vector"}
|
|
208
|
+
|
|
209
|
+
def __init__(
|
|
210
|
+
self,
|
|
211
|
+
record_type: type[ModelT],
|
|
212
|
+
*,
|
|
213
|
+
connection_string: str | SecretString | None = None,
|
|
214
|
+
query_timeout: int | None = None,
|
|
215
|
+
schema: str = "dbo",
|
|
216
|
+
definition: VectorStoreCollectionDefinition | None = None,
|
|
217
|
+
collection_name: str | None = None,
|
|
218
|
+
embedding_generator: EmbeddingClient | None = None,
|
|
219
|
+
env_file_path: str | None = None,
|
|
220
|
+
env_file_encoding: str | None = None,
|
|
221
|
+
) -> None:
|
|
222
|
+
"""Initialize a collection without connecting to the database.
|
|
223
|
+
|
|
224
|
+
Args:
|
|
225
|
+
record_type: Decorated/registered model type, or ``dict``.
|
|
226
|
+
connection_string: Driver connection string, or ``SQL_SERVER_CONNECTION_STRING``.
|
|
227
|
+
query_timeout: Optional per-statement timeout in seconds; ``0`` disables the timeout.
|
|
228
|
+
schema: Existing database schema containing the table.
|
|
229
|
+
definition: Explicit field definition for dictionary records.
|
|
230
|
+
collection_name: Table name overriding the model definition.
|
|
231
|
+
embedding_generator: Default local embedding generator.
|
|
232
|
+
env_file_path: Optional .env file.
|
|
233
|
+
env_file_encoding: Encoding of the selected .env file.
|
|
234
|
+
"""
|
|
235
|
+
super().__init__(
|
|
236
|
+
record_type,
|
|
237
|
+
definition=definition,
|
|
238
|
+
collection_name=collection_name,
|
|
239
|
+
embedding_generator=embedding_generator,
|
|
240
|
+
managed_client=True,
|
|
241
|
+
)
|
|
242
|
+
self.schema = schema
|
|
243
|
+
self._table = f"{_quote_identifier(schema)}.{_quote_identifier(self.collection_name)}"
|
|
244
|
+
self._fields = tuple(
|
|
245
|
+
replace(field, provider_annotations=field.provider_annotations) for field in self.definition.fields
|
|
246
|
+
)
|
|
247
|
+
for field in self._fields:
|
|
248
|
+
_quote_identifier(field.storage_name or field.name)
|
|
249
|
+
_column_type(field)
|
|
250
|
+
if field.is_full_text_indexed:
|
|
251
|
+
raise NotImplementedError("SQL Server full-text and keyword-hybrid search are not supported.")
|
|
252
|
+
if any(name.startswith("sql_server.") for name in field.provider_annotations):
|
|
253
|
+
raise NotImplementedError("SQL Server provider annotations are not supported.")
|
|
254
|
+
if field.field_type == "vector":
|
|
255
|
+
_metric_for(field)
|
|
256
|
+
if field.index_kind not in ("default", "flat"):
|
|
257
|
+
raise NotImplementedError("SQL Server approximate vector indexes are not supported.")
|
|
258
|
+
elif field.is_indexed and field.field_type == "data" and field.type_ in ("bytes", "list", "dict"):
|
|
259
|
+
raise NotImplementedError(f"SQL Server cannot index data fields of type '{field.type_}'.")
|
|
260
|
+
self._client = _create_client(
|
|
261
|
+
connection_string,
|
|
262
|
+
query_timeout=query_timeout,
|
|
263
|
+
env_file_path=env_file_path,
|
|
264
|
+
env_file_encoding=env_file_encoding,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
async def __aenter__(self) -> Self:
|
|
268
|
+
"""Enter the collection context."""
|
|
269
|
+
return self
|
|
270
|
+
|
|
271
|
+
async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
|
272
|
+
"""Release collection-owned resources."""
|
|
273
|
+
await self.close()
|
|
274
|
+
|
|
275
|
+
async def close(self) -> None:
|
|
276
|
+
"""Close a collection-owned worker, never its store's worker."""
|
|
277
|
+
if self.managed_client:
|
|
278
|
+
await self._client.close()
|
|
279
|
+
|
|
280
|
+
def _column_names(self, include_vectors: bool) -> list[str]:
|
|
281
|
+
return [
|
|
282
|
+
field.storage_name or field.name
|
|
283
|
+
for field in self._fields
|
|
284
|
+
if include_vectors or field.field_type != "vector"
|
|
285
|
+
]
|
|
286
|
+
|
|
287
|
+
def _column_definitions(self) -> str:
|
|
288
|
+
definitions: list[str] = []
|
|
289
|
+
for field in self._fields:
|
|
290
|
+
column = _quote_identifier(field.storage_name or field.name)
|
|
291
|
+
data_type = _column_type(field)
|
|
292
|
+
if field.field_type == "key":
|
|
293
|
+
generated = ""
|
|
294
|
+
if field.is_auto_generated:
|
|
295
|
+
generated = {
|
|
296
|
+
"int": " IDENTITY(1,1)",
|
|
297
|
+
"UUID": " DEFAULT NEWID()",
|
|
298
|
+
"str": " DEFAULT CONVERT(NVARCHAR(36), NEWID())",
|
|
299
|
+
}[field.type_ or ""]
|
|
300
|
+
definitions.append(f"{column} {data_type}{generated} NOT NULL PRIMARY KEY")
|
|
301
|
+
else:
|
|
302
|
+
definitions.append(f"{column} {data_type} NULL")
|
|
303
|
+
return ", ".join(definitions)
|
|
304
|
+
|
|
305
|
+
@staticmethod
|
|
306
|
+
def _index_name(schema: str, table: str, column: str) -> str:
|
|
307
|
+
digest = hashlib.sha256(f"{schema}\0{table}\0{column}".encode()).hexdigest()[:32]
|
|
308
|
+
return f"af_sql_{digest}"
|
|
309
|
+
|
|
310
|
+
async def ensure_collection_exists(self, *, operation_options: Mapping[str, Any] | None = None) -> None:
|
|
311
|
+
"""Create a table and requested scalar indexes, without modifying existing tables."""
|
|
312
|
+
_validate_options(operation_options)
|
|
313
|
+
self._client.ensure_open()
|
|
314
|
+
statement = (
|
|
315
|
+
"IF NOT EXISTS (SELECT 1 FROM sys.tables AS t " # nosec B608
|
|
316
|
+
"JOIN sys.schemas AS s ON s.schema_id = t.schema_id "
|
|
317
|
+
"WHERE s.name = ? AND t.name = ?) "
|
|
318
|
+
f"BEGIN CREATE TABLE {self._table} ({self._column_definitions()}) END"
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
def create(cursor: mssql_python.Cursor) -> None:
|
|
322
|
+
_execute(cursor, statement, [self.schema, self.collection_name])
|
|
323
|
+
for field in self._fields:
|
|
324
|
+
if field.field_type != "data" or not field.is_indexed:
|
|
325
|
+
continue
|
|
326
|
+
name = field.storage_name or field.name
|
|
327
|
+
index_name = self._index_name(self.schema, self.collection_name, name)
|
|
328
|
+
index_statement = (
|
|
329
|
+
"IF NOT EXISTS (SELECT 1 FROM sys.indexes AS i " # nosec B608
|
|
330
|
+
"JOIN sys.tables AS t ON t.object_id = i.object_id "
|
|
331
|
+
"JOIN sys.schemas AS s ON s.schema_id = t.schema_id "
|
|
332
|
+
"WHERE s.name = ? AND t.name = ? AND i.name = ?) "
|
|
333
|
+
f"CREATE INDEX {_quote_identifier(index_name)} ON {self._table} ({_quote_identifier(name)})"
|
|
334
|
+
)
|
|
335
|
+
_execute(cursor, index_statement, [self.schema, self.collection_name, index_name])
|
|
336
|
+
|
|
337
|
+
await self._client.run(create)
|
|
338
|
+
|
|
339
|
+
async def collection_exists(self, *, operation_options: Mapping[str, Any] | None = None) -> bool:
|
|
340
|
+
"""Return whether a base table exists in the configured schema."""
|
|
341
|
+
_validate_options(operation_options)
|
|
342
|
+
|
|
343
|
+
def exists(cursor: mssql_python.Cursor) -> bool:
|
|
344
|
+
_execute(
|
|
345
|
+
cursor,
|
|
346
|
+
"SELECT 1 FROM sys.tables AS t JOIN sys.schemas AS s ON s.schema_id = t.schema_id "
|
|
347
|
+
"WHERE s.name = ? AND t.name = ?",
|
|
348
|
+
[self.schema, self.collection_name],
|
|
349
|
+
)
|
|
350
|
+
return cursor.fetchone() is not None
|
|
351
|
+
|
|
352
|
+
return await self._client.run(exists)
|
|
353
|
+
|
|
354
|
+
async def ensure_collection_deleted(self, *, operation_options: Mapping[str, Any] | None = None) -> None:
|
|
355
|
+
"""Drop only the table in this schema; never drop the schema or other tables."""
|
|
356
|
+
_validate_options(operation_options)
|
|
357
|
+
await self._client.run(lambda cursor: _execute(cursor, f"DROP TABLE IF EXISTS {self._table}"))
|
|
358
|
+
|
|
359
|
+
def _deserialize_store_models_to_dicts(
|
|
360
|
+
self, records: Sequence[Any], *, context: Mapping[str, Any] | None = None
|
|
361
|
+
) -> Sequence[dict[str, Any]]:
|
|
362
|
+
result = super()._deserialize_store_models_to_dicts(records, context=context)
|
|
363
|
+
for record in result:
|
|
364
|
+
for field in self._fields:
|
|
365
|
+
name = field.storage_name or field.name
|
|
366
|
+
if name in record:
|
|
367
|
+
record[name] = _parse_value(field, record[name])
|
|
368
|
+
return result
|
|
369
|
+
|
|
370
|
+
def _prepare_key(self, key: Any) -> Any:
|
|
371
|
+
if key is None:
|
|
372
|
+
raise ValueError("SQL Server keys cannot be null.")
|
|
373
|
+
return _prepare_value(self.definition.key_field, key)
|
|
374
|
+
|
|
375
|
+
def _parsed_key(self, key: Any) -> Any:
|
|
376
|
+
return _parse_value(self.definition.key_field, key)
|
|
377
|
+
|
|
378
|
+
async def _inner_upsert(
|
|
379
|
+
self, records: Sequence[Any], *, operation_options: Mapping[str, Any] | None = None
|
|
380
|
+
) -> Sequence[KeyT]:
|
|
381
|
+
_validate_options(operation_options)
|
|
382
|
+
self._client.ensure_open()
|
|
383
|
+
prepared: list[dict[str, Any]] = []
|
|
384
|
+
for record in records:
|
|
385
|
+
row: dict[str, Any] = {}
|
|
386
|
+
for field in self._fields:
|
|
387
|
+
name = field.storage_name or field.name
|
|
388
|
+
if name not in record and field.field_type == "key" and field.is_auto_generated:
|
|
389
|
+
continue
|
|
390
|
+
row[name] = (
|
|
391
|
+
self._prepare_key(record[name])
|
|
392
|
+
if field.field_type == "key"
|
|
393
|
+
else _prepare_value(field, record[name])
|
|
394
|
+
)
|
|
395
|
+
prepared.append(row)
|
|
396
|
+
if not prepared:
|
|
397
|
+
return []
|
|
398
|
+
key_name = self.definition.key_field_storage_name
|
|
399
|
+
key_column = _quote_identifier(key_name)
|
|
400
|
+
|
|
401
|
+
def upsert(cursor: mssql_python.Cursor) -> list[KeyT]:
|
|
402
|
+
keys: list[KeyT] = []
|
|
403
|
+
for row in prepared:
|
|
404
|
+
names = tuple(row)
|
|
405
|
+
if key_name in row:
|
|
406
|
+
_execute(
|
|
407
|
+
cursor,
|
|
408
|
+
f"SELECT {key_column} FROM {self._table} WITH (UPDLOCK, HOLDLOCK) WHERE {key_column} = ?", # nosec B608
|
|
409
|
+
[row[key_name]],
|
|
410
|
+
)
|
|
411
|
+
if cursor.fetchone() is not None:
|
|
412
|
+
updates = [name for name in names if name != key_name]
|
|
413
|
+
assignments = ", ".join(f"{_quote_identifier(name)} = ?" for name in updates)
|
|
414
|
+
if not assignments:
|
|
415
|
+
assignments = f"{key_column} = {key_column}"
|
|
416
|
+
parameters = [*(row[name] for name in updates), row[key_name]]
|
|
417
|
+
_check_parameter_count(parameters)
|
|
418
|
+
_execute(
|
|
419
|
+
cursor,
|
|
420
|
+
f"UPDATE {self._table} SET {assignments} OUTPUT INSERTED.{key_column} " # nosec B608
|
|
421
|
+
f"WHERE {key_column} = ?",
|
|
422
|
+
parameters,
|
|
423
|
+
)
|
|
424
|
+
result = cursor.fetchone()
|
|
425
|
+
if result is None:
|
|
426
|
+
raise IntegrationInvalidResponseException("SQL Server did not return an updated key.")
|
|
427
|
+
keys.append(cast(KeyT, self._parsed_key(result[0])))
|
|
428
|
+
continue
|
|
429
|
+
if self.definition.key_field.is_auto_generated and self.definition.key_field.type_ == "int":
|
|
430
|
+
raise NotImplementedError(
|
|
431
|
+
"SQL Server IDENTITY columns cannot insert an explicit new key; omit it to generate a key."
|
|
432
|
+
)
|
|
433
|
+
if names:
|
|
434
|
+
columns = ", ".join(_quote_identifier(name) for name in names)
|
|
435
|
+
markers = ", ".join("?" for _ in names)
|
|
436
|
+
statement = f"INSERT INTO {self._table} ({columns}) OUTPUT INSERTED.{key_column} VALUES ({markers})" # nosec B608
|
|
437
|
+
parameters = [row[name] for name in names]
|
|
438
|
+
_check_parameter_count(parameters)
|
|
439
|
+
_execute(cursor, statement, parameters)
|
|
440
|
+
else:
|
|
441
|
+
_execute(cursor, f"INSERT INTO {self._table} OUTPUT INSERTED.{key_column} DEFAULT VALUES")
|
|
442
|
+
result = cursor.fetchone()
|
|
443
|
+
if result is None:
|
|
444
|
+
raise IntegrationInvalidResponseException("SQL Server did not return an inserted key.")
|
|
445
|
+
keys.append(cast(KeyT, self._parsed_key(result[0])))
|
|
446
|
+
return keys
|
|
447
|
+
|
|
448
|
+
return await self._client.run(upsert)
|
|
449
|
+
|
|
450
|
+
def _order_by(self, order_by: Mapping[str, bool] | None) -> str:
|
|
451
|
+
parts: list[str] = []
|
|
452
|
+
for name, ascending in (order_by or {}).items():
|
|
453
|
+
if type(ascending) is not bool:
|
|
454
|
+
raise TypeError("Order directions must be booleans.")
|
|
455
|
+
field = _filter_field(self.definition, name)
|
|
456
|
+
if field.type_ in ("list", "dict", "bytes"):
|
|
457
|
+
raise NotImplementedError(f"SQL Server ordering is not supported for '{field.type_}'.")
|
|
458
|
+
column = _quote_identifier(field.storage_name or field.name)
|
|
459
|
+
parts.append(f"CASE WHEN {column} IS NULL THEN 1 ELSE 0 END")
|
|
460
|
+
parts.append(f"{column} {'ASC' if ascending else 'DESC'}")
|
|
461
|
+
if self.definition.key_field.name not in (order_by or {}):
|
|
462
|
+
parts.append(f"{_quote_identifier(self.definition.key_field_storage_name)} ASC")
|
|
463
|
+
return ", ".join(parts)
|
|
464
|
+
|
|
465
|
+
async def _inner_get(
|
|
466
|
+
self,
|
|
467
|
+
*,
|
|
468
|
+
keys: Sequence[KeyT] | None = None,
|
|
469
|
+
filter: FilterExpression | None = None,
|
|
470
|
+
top: int = 10,
|
|
471
|
+
skip: int = 0,
|
|
472
|
+
order_by: Mapping[str, bool] | None = None,
|
|
473
|
+
include_vectors: bool = False,
|
|
474
|
+
operation_options: Mapping[str, Any] | None = None,
|
|
475
|
+
) -> Sequence[Any]:
|
|
476
|
+
_validate_options(operation_options)
|
|
477
|
+
self._client.ensure_open()
|
|
478
|
+
columns = self._column_names(include_vectors)
|
|
479
|
+
select = f"SELECT {', '.join(_quote_identifier(name) for name in columns)} FROM {self._table}" # nosec B608
|
|
480
|
+
if keys is not None:
|
|
481
|
+
if order_by:
|
|
482
|
+
raise ValueError("order_by applies only to filtered retrieval, not key lookup.")
|
|
483
|
+
prepared_keys = [self._prepare_key(key) for key in keys]
|
|
484
|
+
if not prepared_keys:
|
|
485
|
+
return []
|
|
486
|
+
|
|
487
|
+
def get_keys(cursor: mssql_python.Cursor) -> list[dict[str, Any]]:
|
|
488
|
+
found: dict[Any, dict[str, Any]] = {}
|
|
489
|
+
key_column = _quote_identifier(self.definition.key_field_storage_name)
|
|
490
|
+
for offset in range(0, len(prepared_keys), _KEY_BATCH_SIZE):
|
|
491
|
+
batch = prepared_keys[offset : offset + _KEY_BATCH_SIZE]
|
|
492
|
+
markers = ", ".join("?" for _ in batch)
|
|
493
|
+
_execute(cursor, f"{select} WHERE {key_column} IN ({markers})", batch)
|
|
494
|
+
for row in _rows(cursor, len(columns)):
|
|
495
|
+
record = dict(zip(columns, row, strict=True))
|
|
496
|
+
found[self._parsed_key(record[self.definition.key_field_storage_name])] = record
|
|
497
|
+
return [found[key] for raw in prepared_keys if (key := self._parsed_key(raw)) in found]
|
|
498
|
+
|
|
499
|
+
return await self._client.run(get_keys)
|
|
500
|
+
where, parameters = ("1 = 1", []) if filter is None else _FilterCompiler(self.definition).compile(filter)
|
|
501
|
+
parameters.extend((skip, top))
|
|
502
|
+
_check_parameter_count(parameters)
|
|
503
|
+
statement = f"{select} WHERE {where} ORDER BY {self._order_by(order_by)} OFFSET ? ROWS FETCH NEXT ? ROWS ONLY"
|
|
504
|
+
|
|
505
|
+
def get_page(cursor: mssql_python.Cursor) -> list[dict[str, Any]]:
|
|
506
|
+
_execute(cursor, statement, parameters)
|
|
507
|
+
return [dict(zip(columns, row, strict=True)) for row in _rows(cursor, len(columns))]
|
|
508
|
+
|
|
509
|
+
return await self._client.run(get_page)
|
|
510
|
+
|
|
511
|
+
async def _inner_delete(self, keys: Sequence[KeyT], *, operation_options: Mapping[str, Any] | None = None) -> None:
|
|
512
|
+
_validate_options(operation_options)
|
|
513
|
+
self._client.ensure_open()
|
|
514
|
+
prepared = [self._prepare_key(key) for key in keys]
|
|
515
|
+
if not prepared:
|
|
516
|
+
return
|
|
517
|
+
key_column = _quote_identifier(self.definition.key_field_storage_name)
|
|
518
|
+
|
|
519
|
+
def delete(cursor: mssql_python.Cursor) -> None:
|
|
520
|
+
for offset in range(0, len(prepared), _KEY_BATCH_SIZE):
|
|
521
|
+
batch = prepared[offset : offset + _KEY_BATCH_SIZE]
|
|
522
|
+
markers = ", ".join("?" for _ in batch)
|
|
523
|
+
_execute(cursor, f"DELETE FROM {self._table} WHERE {key_column} IN ({markers})", batch) # nosec B608
|
|
524
|
+
|
|
525
|
+
await self._client.run(delete)
|
|
526
|
+
|
|
527
|
+
async def _inner_search(
|
|
528
|
+
self,
|
|
529
|
+
*,
|
|
530
|
+
search_type: SearchType,
|
|
531
|
+
filter: FilterExpression | None = None,
|
|
532
|
+
values: Any | None = None,
|
|
533
|
+
vector: Vector | None = None,
|
|
534
|
+
top: int = 3,
|
|
535
|
+
skip: int = 0,
|
|
536
|
+
include_vectors: bool = False,
|
|
537
|
+
vector_property_name: str | None = None,
|
|
538
|
+
additional_property_name: str | None = None,
|
|
539
|
+
score_threshold: float | None = None,
|
|
540
|
+
operation_options: Mapping[str, Any] | None = None,
|
|
541
|
+
) -> SearchResults[Any]:
|
|
542
|
+
_validate_options(operation_options)
|
|
543
|
+
if search_type != "vector" or additional_property_name is not None:
|
|
544
|
+
raise NotImplementedError("SQL Server supports dense vector search only, not keyword-hybrid search.")
|
|
545
|
+
if vector is None:
|
|
546
|
+
raise NotImplementedError("SQL Server requires a vector or a local embedding generator.")
|
|
547
|
+
self._client.ensure_open()
|
|
548
|
+
resolved = self.definition.try_get_vector_field(vector_property_name)
|
|
549
|
+
if resolved is None:
|
|
550
|
+
raise ValueError("Select a vector_property_name from the collection definition.")
|
|
551
|
+
field = next(item for item in self._fields if item.name == resolved.name)
|
|
552
|
+
metric, result_kind = _metric_for(field)
|
|
553
|
+
where, filter_params = (
|
|
554
|
+
("1 = 1", []) if filter is None else _FilterCompiler(self.definition, alias="t.").compile(filter)
|
|
555
|
+
)
|
|
556
|
+
column_names = self._column_names(include_vectors)
|
|
557
|
+
projections = ", ".join(f"t.{_quote_identifier(name)}" for name in column_names)
|
|
558
|
+
vector_column = f"t.{_quote_identifier(field.storage_name or field.name)}"
|
|
559
|
+
key_column = f"t.{_quote_identifier(self.definition.key_field_storage_name)}"
|
|
560
|
+
dimensions = field.dimensions
|
|
561
|
+
statement = (
|
|
562
|
+
f"SELECT {projections}, d.[distance] FROM {self._table} AS t " # nosec B608
|
|
563
|
+
f"CROSS APPLY (VALUES (VECTOR_DISTANCE('{metric}', CAST(? AS VECTOR({dimensions})), "
|
|
564
|
+
f"{vector_column}))) AS d([distance]) "
|
|
565
|
+
f"WHERE {vector_column} IS NOT NULL AND ({where}) AND d.[distance] IS NOT NULL"
|
|
566
|
+
)
|
|
567
|
+
parameters: list[Any] = [_prepare_value(field, vector), *filter_params]
|
|
568
|
+
if score_threshold is not None:
|
|
569
|
+
if type(score_threshold) not in (int, float):
|
|
570
|
+
raise ValueError("score_threshold must be a finite number.")
|
|
571
|
+
try:
|
|
572
|
+
threshold = float(score_threshold)
|
|
573
|
+
except OverflowError as exc:
|
|
574
|
+
raise ValueError("score_threshold must be a finite number.") from exc
|
|
575
|
+
if not math.isfinite(threshold):
|
|
576
|
+
raise ValueError("score_threshold must be a finite number.")
|
|
577
|
+
cutoff = (
|
|
578
|
+
1 - threshold
|
|
579
|
+
if result_kind == "similarity"
|
|
580
|
+
else (-threshold if result_kind == "negative" else threshold)
|
|
581
|
+
)
|
|
582
|
+
statement += " AND d.[distance] <= ?"
|
|
583
|
+
parameters.append(cutoff)
|
|
584
|
+
statement += f" ORDER BY d.[distance] ASC, {key_column} ASC OFFSET ? ROWS FETCH NEXT ? ROWS ONLY"
|
|
585
|
+
parameters.extend((skip, top))
|
|
586
|
+
_check_parameter_count(parameters)
|
|
587
|
+
|
|
588
|
+
def search(cursor: mssql_python.Cursor) -> list[dict[str, Any]]:
|
|
589
|
+
_execute(cursor, statement, parameters)
|
|
590
|
+
results: list[dict[str, Any]] = []
|
|
591
|
+
for row in _rows(cursor, len(column_names) + 1):
|
|
592
|
+
distance = row[-1]
|
|
593
|
+
if type(distance) not in (float, int) or not math.isfinite(distance):
|
|
594
|
+
raise IntegrationInvalidResponseException("SQL Server returned a non-finite vector distance.")
|
|
595
|
+
score = (
|
|
596
|
+
1 - distance
|
|
597
|
+
if result_kind == "similarity"
|
|
598
|
+
else -distance
|
|
599
|
+
if result_kind == "negative"
|
|
600
|
+
else distance
|
|
601
|
+
)
|
|
602
|
+
results.append({"record": dict(zip(column_names, row[:-1], strict=True)), "score": float(score)})
|
|
603
|
+
return results
|
|
604
|
+
|
|
605
|
+
rows = await self._client.run(search)
|
|
606
|
+
return SearchResults(
|
|
607
|
+
rows,
|
|
608
|
+
metadata={"distance_function": field.distance_function or "DEFAULT", "approximate": False},
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
def _get_record_from_result(self, result: Any) -> Any:
|
|
612
|
+
return result["record"]
|
|
613
|
+
|
|
614
|
+
def _get_score_from_result(self, result: Any) -> float | None:
|
|
615
|
+
return cast(float, result["score"])
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
class SqlServerStore(BaseVectorStore):
|
|
619
|
+
"""Create SQL Server collections sharing one store-owned worker."""
|
|
620
|
+
|
|
621
|
+
def __init__(
|
|
622
|
+
self,
|
|
623
|
+
*,
|
|
624
|
+
connection_string: str | SecretString | None = None,
|
|
625
|
+
query_timeout: int | None = None,
|
|
626
|
+
schema: str = "dbo",
|
|
627
|
+
embedding_generator: EmbeddingClient | None = None,
|
|
628
|
+
env_file_path: str | None = None,
|
|
629
|
+
env_file_encoding: str | None = None,
|
|
630
|
+
) -> None:
|
|
631
|
+
"""Initialize a store without connecting to SQL Server.
|
|
632
|
+
|
|
633
|
+
Args:
|
|
634
|
+
connection_string: Driver connection string, or ``SQL_SERVER_CONNECTION_STRING``.
|
|
635
|
+
query_timeout: Optional per-statement timeout in seconds; ``0`` disables the timeout.
|
|
636
|
+
schema: Existing database schema used by all collections.
|
|
637
|
+
embedding_generator: Default local embedding generator.
|
|
638
|
+
env_file_path: Optional .env file.
|
|
639
|
+
env_file_encoding: Encoding of the selected .env file.
|
|
640
|
+
"""
|
|
641
|
+
super().__init__(embedding_generator=embedding_generator, managed_client=True)
|
|
642
|
+
_quote_identifier(schema)
|
|
643
|
+
self.schema = schema
|
|
644
|
+
self._client = _create_client(
|
|
645
|
+
connection_string,
|
|
646
|
+
query_timeout=query_timeout,
|
|
647
|
+
env_file_path=env_file_path,
|
|
648
|
+
env_file_encoding=env_file_encoding,
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
def get_collection(
|
|
652
|
+
self,
|
|
653
|
+
record_type: type[ModelT],
|
|
654
|
+
*,
|
|
655
|
+
definition: VectorStoreCollectionDefinition | None = None,
|
|
656
|
+
collection_name: str | None = None,
|
|
657
|
+
embedding_generator: EmbeddingClient | None = None,
|
|
658
|
+
) -> SqlServerCollection[Any, ModelT]:
|
|
659
|
+
"""Create a collection sharing this store's owned worker and lifecycle."""
|
|
660
|
+
collection = SqlServerCollection(
|
|
661
|
+
record_type,
|
|
662
|
+
connection_string=self._client.connection_string,
|
|
663
|
+
query_timeout=self._client.query_timeout,
|
|
664
|
+
schema=self.schema,
|
|
665
|
+
definition=definition,
|
|
666
|
+
collection_name=collection_name,
|
|
667
|
+
embedding_generator=embedding_generator if embedding_generator is not None else self.embedding_generator,
|
|
668
|
+
)
|
|
669
|
+
# Collection construction does no I/O; reuse the store's lazy worker.
|
|
670
|
+
collection._client = self._client # pyright: ignore[reportPrivateUsage]
|
|
671
|
+
collection.managed_client = False
|
|
672
|
+
return collection
|
|
673
|
+
|
|
674
|
+
async def list_collection_names(self, *, operation_options: Mapping[str, Any] | None = None) -> Sequence[str]:
|
|
675
|
+
"""List base tables in the configured schema."""
|
|
676
|
+
_validate_options(operation_options)
|
|
677
|
+
|
|
678
|
+
def list_names(cursor: mssql_python.Cursor) -> list[str]:
|
|
679
|
+
_execute(
|
|
680
|
+
cursor,
|
|
681
|
+
"SELECT t.name FROM sys.tables AS t JOIN sys.schemas AS s ON s.schema_id = t.schema_id "
|
|
682
|
+
"WHERE s.name = ? ORDER BY t.name",
|
|
683
|
+
[self.schema],
|
|
684
|
+
)
|
|
685
|
+
names = _rows(cursor, 1)
|
|
686
|
+
if any(not isinstance(row[0], str) for row in names):
|
|
687
|
+
raise IntegrationInvalidResponseException("SQL Server returned a non-string table name.")
|
|
688
|
+
return [cast(str, row[0]) for row in names]
|
|
689
|
+
|
|
690
|
+
return await self._client.run(list_names)
|
|
691
|
+
|
|
692
|
+
async def _inner_ensure_collection_deleted(
|
|
693
|
+
self, collection_name: str, *, operation_options: Mapping[str, Any] | None = None
|
|
694
|
+
) -> None:
|
|
695
|
+
_validate_options(operation_options)
|
|
696
|
+
table = f"{_quote_identifier(self.schema)}.{_quote_identifier(collection_name)}"
|
|
697
|
+
await self._client.run(lambda cursor: _execute(cursor, f"DROP TABLE IF EXISTS {table}"))
|
|
698
|
+
|
|
699
|
+
async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
|
700
|
+
"""Release store-owned resources."""
|
|
701
|
+
await self.close()
|
|
702
|
+
|
|
703
|
+
async def close(self) -> None:
|
|
704
|
+
"""Wait for in-flight operations and shut down the store-owned worker."""
|
|
705
|
+
await self._client.close()
|
|
File without changes
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agent-framework-sql-server
|
|
3
|
+
Version: 1.0.0a261002
|
|
4
|
+
Summary: SQL Server native vector integration for Microsoft Agent Framework.
|
|
5
|
+
Author-email: Microsoft <af-support@microsoft.com>
|
|
6
|
+
Requires-Python: >=3.10,<3.15
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: agent-framework-core>=1.19.0,<2
|
|
20
|
+
Requires-Dist: mssql-python>=1.15.0,<2
|
|
21
|
+
Project-URL: homepage, https://aka.ms/agent-framework
|
|
22
|
+
Project-URL: issues, https://github.com/microsoft/agent-framework/issues
|
|
23
|
+
Project-URL: source, https://github.com/microsoft/agent-framework/tree/main/python
|
|
24
|
+
|
|
25
|
+
# Agent Framework SQL Server vector store
|
|
26
|
+
|
|
27
|
+
Store typed Agent Framework records in SQL Server and Azure SQL native `VECTOR`
|
|
28
|
+
columns, with exact, database-side similarity search. This alpha package exports
|
|
29
|
+
`SqlServerCollection`, `SqlServerStore`, `SqlServerSettings`, and
|
|
30
|
+
`SqlServerCommittedCleanupException` directly from `agent_framework_sql_server`.
|
|
31
|
+
|
|
32
|
+
## Install and provision
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install agent-framework-sql-server --pre
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Requires Python 3.10–3.14 and a vector-enabled database: SQL Server 2025
|
|
39
|
+
(17.x), Azure SQL Database, Azure SQL Managed Instance on the SQL Server 2025
|
|
40
|
+
or Always-up-to-date update policy, or SQL database in Microsoft Fabric.
|
|
41
|
+
Older SQL Server releases do not support `VECTOR`/`VECTOR_DISTANCE`.
|
|
42
|
+
|
|
43
|
+
The package uses Microsoft's [`mssql-python` 1.15+ driver](https://pypi.org/project/mssql-python/).
|
|
44
|
+
It installs its `mssql-python-odbc` binary companion automatically; no external
|
|
45
|
+
ODBC Driver 18 or driver manager is required. Published wheels cover CPython
|
|
46
|
+
3.10–3.14 on Windows x64, Linux x64/ARM64, and **macOS 15+**
|
|
47
|
+
Intel/Apple Silicon; Windows ARM64 wheels start at Python 3.11. macOS 14 is
|
|
48
|
+
listed in the driver's support documentation, but the published macOS wheels
|
|
49
|
+
are tagged `macosx_15_0_universal2` and there is no source distribution.
|
|
50
|
+
Python 3.15 has no published wheel yet: this package caps `requires-python`
|
|
51
|
+
below 3.15, and the repository's non-blocking 3.15 CI lane excludes it.
|
|
52
|
+
The workspace lockfile likewise limits Python below 3.15 while this package
|
|
53
|
+
remains a member; the experimental 3.15 lane excludes it before resolving.
|
|
54
|
+
Follow the [driver's installation instructions](https://learn.microsoft.com/sql/connect/python/mssql-python/installation)
|
|
55
|
+
for system libraries on Linux and OpenSSL on macOS.
|
|
56
|
+
|
|
57
|
+
The database administrator must provide an existing schema (default `dbo`).
|
|
58
|
+
`ensure_collection_exists()` creates the requested table and scalar indexes
|
|
59
|
+
there but never creates a schema, alters an existing table, or changes database
|
|
60
|
+
settings. `ensure_collection_deleted()` drops only that table.
|
|
61
|
+
|
|
62
|
+
## Connection settings and ownership
|
|
63
|
+
|
|
64
|
+
For local Azure SQL development with passwordless Microsoft Entra authentication,
|
|
65
|
+
sign in with Azure CLI, then set the environment variable used by the
|
|
66
|
+
[sample](samples/sql_server_vectors.py):
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
az login
|
|
70
|
+
export SQL_SERVER_CONNECTION_STRING='Server=<host>;Database=<db>;Authentication=ActiveDirectoryDefault;Encrypt=yes;'
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Replace `<host>` with the Azure SQL server hostname (for example,
|
|
74
|
+
`my-server.database.windows.net`) and `<db>` with your existing database.
|
|
75
|
+
`ActiveDirectoryDefault` uses the driver's credential chain, which can use
|
|
76
|
+
your Azure CLI sign-in. The identity must be granted access to the database
|
|
77
|
+
and permission to create a table and index in the configured schema and
|
|
78
|
+
read/write its records. On an Azure-hosted app, use
|
|
79
|
+
`Authentication=ActiveDirectoryMSI` for managed identity; add
|
|
80
|
+
`UID=<client-id>` for a user-assigned identity. See
|
|
81
|
+
[Microsoft's Entra authentication guide](https://learn.microsoft.com/sql/connect/python/mssql-python/entra-authentication)
|
|
82
|
+
for setup, permissions, and other supported modes. This connector accepts
|
|
83
|
+
connection strings, not the driver's `token_provider=` credential argument.
|
|
84
|
+
|
|
85
|
+
Do not commit connection strings containing credentials. Alternatively, pass
|
|
86
|
+
`connection_string` as a string or Agent Framework `SecretString` to
|
|
87
|
+
`SqlServerStore` or `SqlServerCollection`. Settings precedence is **explicit
|
|
88
|
+
argument > selected `.env` file > process environment**. To read a `.env` file
|
|
89
|
+
in the run directory, pass `env_file_path=".env"` to `SqlServerStore` or
|
|
90
|
+
`SqlServerCollection`; `env_file_encoding` is optional. Missing or empty
|
|
91
|
+
connection strings are rejected.
|
|
92
|
+
|
|
93
|
+
The connector owns all connections. Each whole operation opens, uses, commits
|
|
94
|
+
or rolls back, and closes a `mssql-python` connection on a dedicated worker
|
|
95
|
+
thread, keeping Agent Framework's async calls nonblocking. The driver's
|
|
96
|
+
built-in pooling can reuse the underlying physical connection. A store and
|
|
97
|
+
its collections share one worker; a standalone collection owns its own.
|
|
98
|
+
Call `close()` or use an async context manager to release the worker. There is
|
|
99
|
+
no `client=` or connection-factory constructor argument: arbitrary caller-owned
|
|
100
|
+
`mssql-python` connections cannot safely cross threads (`threadsafety=1`).
|
|
101
|
+
This is intentionally narrower than connectors that support borrowed async
|
|
102
|
+
clients.
|
|
103
|
+
|
|
104
|
+
Batch writes commit or roll back together on one connection. Cancelling an
|
|
105
|
+
async operation waits for its worker to finish cleanup; it cannot interrupt an
|
|
106
|
+
already-running synchronous SQL statement, and the transaction may already
|
|
107
|
+
have committed. Set `query_timeout=30` (seconds, for example) on the store or
|
|
108
|
+
collection when bounding database calls; leaving it unset uses the driver's
|
|
109
|
+
default, and `0` disables the timeout. Prefer stable application-provided
|
|
110
|
+
keys when retrying writes.
|
|
111
|
+
If closing the connection fails after a successful commit,
|
|
112
|
+
`SqlServerCommittedCleanupException` explicitly signals that the transaction
|
|
113
|
+
**already committed**; do not automatically retry, especially with generated
|
|
114
|
+
keys. Cancellation remains `CancelledError` even if the worker fails while
|
|
115
|
+
finishing; that worker error is logged after cleanup.
|
|
116
|
+
|
|
117
|
+
## Example
|
|
118
|
+
|
|
119
|
+
With `SQL_SERVER_CONNECTION_STRING` configured, run the
|
|
120
|
+
[typed sample](samples/sql_server_vectors.py) from the `python/` directory:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
uv run --package agent-framework-sql-server \
|
|
124
|
+
python packages/sql-server/samples/sql_server_vectors.py
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The sample creates a uniquely named table, upserts precomputed embeddings,
|
|
128
|
+
filters/ranks in SQL Server, retrieves an optional vector, and drops its own
|
|
129
|
+
table. Pass `generate_vectors=False` to preserve precomputed vectors; to
|
|
130
|
+
generate them locally, configure an `embedding_generator`.
|
|
131
|
+
|
|
132
|
+
## Capabilities and limits
|
|
133
|
+
|
|
134
|
+
The connector supports typed decorated models and dictionary definitions;
|
|
135
|
+
string/integer/UUID keys (including generated keys); multiple nullable float32
|
|
136
|
+
vector columns with **1–1998 dimensions**; field storage aliases; batch
|
|
137
|
+
upsert/get/delete; paged and ordered retrieval; scalar data indexes; and
|
|
138
|
+
parameterized top-level `Filter`/`FilterGroup` expressions. String keys cannot
|
|
139
|
+
end with a space because SQL Server ignores trailing spaces in key comparisons.
|
|
140
|
+
Indexed strings use `NVARCHAR(450)`; other strings use `NVARCHAR(MAX)`. List and
|
|
141
|
+
dictionary fields are stored as JSON, and timezone-aware `datetime` values are
|
|
142
|
+
normalized to UTC in `DATETIME2(7)` columns.
|
|
143
|
+
For an auto-generated integer (`IDENTITY`) key, omit the key on insert;
|
|
144
|
+
explicit keys can update existing rows but cannot create new identity rows.
|
|
145
|
+
|
|
146
|
+
Supported filters: scalar `eq`, `ne`, `in`, `not_in`, `is_null`, `is_not_null`,
|
|
147
|
+
`exists`; numeric/date/datetime `gt`, `gte`, `lt`, `lte`, `between`; and string
|
|
148
|
+
`starts_with`, `ends_with`, `contains_text`. `AND`/`OR`/`NOT` groups preserve
|
|
149
|
+
two-valued null semantics; string equality is byte-exact and text patterns
|
|
150
|
+
escape SQL Server wildcards. JSON fields support `is_null`, `is_not_null`, and
|
|
151
|
+
`exists`, **not** equality or collection-membership filters. Nested paths,
|
|
152
|
+
full-text filtering, and unknown operation options fail explicitly. The SQL
|
|
153
|
+
Server 2100-parameter limit is respected by batching key reads/deletes and
|
|
154
|
+
limiting other statements to 2000 bound parameters.
|
|
155
|
+
|
|
156
|
+
Search uses SQL Server's exact `VECTOR_DISTANCE` on native `VECTOR` columns,
|
|
157
|
+
with filters and score thresholds applied **before** offset/limit. The default
|
|
158
|
+
metric is cosine **distance** (lower is better). Euclidean and negative dot
|
|
159
|
+
product also return distances (maximum thresholds); `cosine_similarity` and
|
|
160
|
+
`dot_prod` return similarity/positive-dot scores (minimum thresholds).
|
|
161
|
+
Scores are raw metric units, not probabilities. Retrieval excludes vectors by
|
|
162
|
+
default; use `include_vectors=True` to return them. Approximate DiskANN
|
|
163
|
+
indexes/search, preview-only float16 vectors, keyword-hybrid search, sparse or
|
|
164
|
+
binary vectors, server-side embedding generation, and schema migration are
|
|
165
|
+
not supported.
|
|
166
|
+
|
|
167
|
+
The server stores native `VECTOR` columns, but `mssql-python` 1.15 does not
|
|
168
|
+
expose a native Python vector type. The connector binds JSON-encoded vectors
|
|
169
|
+
as parameters and parses JSON on retrieval; SQL Server converts to/from the
|
|
170
|
+
native type. It does not enable native driver vector bindings.
|
|
171
|
+
|
|
172
|
+
## Service tests
|
|
173
|
+
|
|
174
|
+
Unit tests need no database. Integration tests are opt-in: set
|
|
175
|
+
`SQL_SERVER_TEST_CONNECTION_STRING` to a deliberately designated test database
|
|
176
|
+
with table creation permissions and run:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
uv run --package agent-framework-sql-server pytest \
|
|
180
|
+
packages/sql-server/tests/sql_server/test_integration.py -m integration
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
The tests create uniquely named tables and remove only those tables. They
|
|
184
|
+
skip when the variable is absent or empty (including an unconfigured CI
|
|
185
|
+
secret), and fail rather than silently skipping when an explicitly
|
|
186
|
+
designated server lacks vector support.
|
|
187
|
+
|
|
188
|
+
## References
|
|
189
|
+
|
|
190
|
+
- [SQL Server vector type and database availability](https://learn.microsoft.com/sql/t-sql/data-types/vector-data-type)
|
|
191
|
+
- [Exact vector distance metrics](https://learn.microsoft.com/sql/t-sql/functions/vector-distance-transact-sql)
|
|
192
|
+
- [Microsoft's Python vector JSON example](https://learn.microsoft.com/sql/t-sql/data-types/vector-data-type#python)
|
|
193
|
+
- [mssql-python asynchronous integration patterns](https://learn.microsoft.com/sql/connect/python/mssql-python/asynchronous-patterns)
|
|
194
|
+
- [Microsoft Agent Framework](https://learn.microsoft.com/agent-framework/)
|
|
195
|
+
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
agent_framework_sql_server/__init__.py,sha256=FU0pnNvCvhVvurpIk5Hw8_okV76LbuNhbPSdkeduJ4U,591
|
|
2
|
+
agent_framework_sql_server/_sql.py,sha256=19Lidl8195aeBgXuJa-88s293uh2lLoBDVR7bp3Hrmc,16911
|
|
3
|
+
agent_framework_sql_server/_vector_store.py,sha256=Wvz_akkjiy3Y93eprkGzeoOTQ87OUdbsZEGY6vNXYSg,32658
|
|
4
|
+
agent_framework_sql_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
agent_framework_sql_server-1.0.0a261002.dist-info/licenses/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
|
|
6
|
+
agent_framework_sql_server-1.0.0a261002.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
|
7
|
+
agent_framework_sql_server-1.0.0a261002.dist-info/METADATA,sha256=kunrA6pFHYxmG5yR3YsHsOiF0vKRGCOzd02B0EkcccE,10514
|
|
8
|
+
agent_framework_sql_server-1.0.0a261002.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) Microsoft Corporation.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE
|