sqlspec 0.67.0__py3-none-any.whl → 0.69.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlspec/adapters/adbc/adk/__init__.py +8 -2
- sqlspec/adapters/adbc/adk/store.py +366 -27
- sqlspec/adapters/adbc/config.py +31 -5
- sqlspec/adapters/adbc/core.py +23 -0
- sqlspec/adapters/adbc/driver.py +75 -8
- sqlspec/adapters/asyncpg/config.py +60 -3
- sqlspec/adapters/asyncpg/core.py +22 -0
- sqlspec/adapters/asyncpg/driver.py +12 -1
- sqlspec/adapters/asyncpg/events/_hub.py +45 -2
- sqlspec/adapters/duckdb/config.py +4 -0
- sqlspec/adapters/duckdb/core.py +61 -2
- sqlspec/adapters/duckdb/driver.py +75 -4
- sqlspec/adapters/duckdb/pool.py +35 -1
- sqlspec/adapters/oracledb/__init__.py +54 -0
- sqlspec/adapters/oracledb/_typing.py +18 -4
- sqlspec/adapters/oracledb/adk/store.py +241 -9
- sqlspec/adapters/oracledb/config.py +285 -27
- sqlspec/adapters/oracledb/core.py +1961 -56
- sqlspec/adapters/oracledb/data_dictionary.py +112 -0
- sqlspec/adapters/oracledb/driver.py +279 -16
- sqlspec/adapters/psqlpy/adk/store.py +76 -31
- sqlspec/adapters/psqlpy/config.py +59 -2
- sqlspec/adapters/psqlpy/core.py +56 -0
- sqlspec/adapters/psqlpy/driver.py +34 -5
- sqlspec/adapters/psqlpy/events/_hub.py +47 -2
- sqlspec/adapters/psycopg/adk/store.py +87 -32
- sqlspec/adapters/psycopg/config.py +117 -4
- sqlspec/adapters/psycopg/core.py +8 -0
- sqlspec/adapters/psycopg/driver.py +24 -0
- sqlspec/adapters/psycopg/events/_hub.py +89 -3
- sqlspec/builder/__init__.py +15 -1
- sqlspec/builder/_explain.py +70 -1
- sqlspec/builder/_select.py +18 -0
- sqlspec/builder/_vector_distance.py +200 -1
- sqlspec/config.py +87 -1
- sqlspec/core/explain.py +28 -1
- sqlspec/core/sqlcommenter.py +182 -10
- sqlspec/core/statement.py +4 -2
- sqlspec/data_dictionary/dialects/oracle/sql/objects.sql +8 -0
- sqlspec/data_dictionary/dialects/oracle/sql/system.sql +8 -0
- sqlspec/data_dictionary/dialects/oracle/sql/version.sql +7 -0
- sqlspec/data_dictionary/dialects/oracle/sql/views.sql +26 -0
- sqlspec/driver/_async.py +33 -6
- sqlspec/driver/_common.py +95 -11
- sqlspec/driver/_sync.py +34 -7
- sqlspec/extensions/adk/artifact/service.py +131 -40
- sqlspec/extensions/adk/converters.py +9 -4
- sqlspec/extensions/adk/memory/converters.py +13 -1
- sqlspec/extensions/adk/memory/service.py +22 -4
- sqlspec/extensions/adk/service.py +170 -77
- sqlspec/migrations/base.py +6 -0
- sqlspec/migrations/commands.py +2 -2
- sqlspec/observability/_observer.py +41 -1
- sqlspec/observability/_runtime.py +77 -22
- {sqlspec-0.67.0.dist-info → sqlspec-0.69.0.dist-info}/METADATA +2 -2
- {sqlspec-0.67.0.dist-info → sqlspec-0.69.0.dist-info}/RECORD +59 -59
- {sqlspec-0.67.0.dist-info → sqlspec-0.69.0.dist-info}/WHEEL +0 -0
- {sqlspec-0.67.0.dist-info → sqlspec-0.69.0.dist-info}/entry_points.txt +0 -0
- {sqlspec-0.67.0.dist-info → sqlspec-0.69.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
"""ADBC ADK integration for Google Agent Development Kit."""
|
|
2
2
|
|
|
3
|
-
from sqlspec.adapters.adbc.adk.store import
|
|
3
|
+
from sqlspec.adapters.adbc.adk.store import (
|
|
4
|
+
AdbcADKConfig,
|
|
5
|
+
AdbcADKMemoryStore,
|
|
6
|
+
AdbcADKStore,
|
|
7
|
+
decode_adbc_pgvector,
|
|
8
|
+
format_pgvector_literal,
|
|
9
|
+
)
|
|
4
10
|
|
|
5
|
-
__all__ = ("AdbcADKMemoryStore", "AdbcADKStore")
|
|
11
|
+
__all__ = ("AdbcADKConfig", "AdbcADKMemoryStore", "AdbcADKStore", "decode_adbc_pgvector", "format_pgvector_literal")
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
"""ADBC ADK store for Google Agent Development Kit session/event storage."""
|
|
2
2
|
|
|
3
3
|
import contextlib
|
|
4
|
+
import struct
|
|
5
|
+
from collections.abc import Mapping
|
|
4
6
|
from datetime import UTC, datetime, timedelta
|
|
5
7
|
from functools import lru_cache
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Final, Literal
|
|
8
|
+
from typing import TYPE_CHECKING, Any, Final, Literal, NotRequired, cast
|
|
7
9
|
|
|
8
10
|
from sqlspec.adapters.adbc.core import (
|
|
9
11
|
driver_kind_from_driver_name,
|
|
@@ -11,7 +13,9 @@ from sqlspec.adapters.adbc.core import (
|
|
|
11
13
|
is_shared_object_driver,
|
|
12
14
|
resolve_dialect_from_config,
|
|
13
15
|
)
|
|
16
|
+
from sqlspec.config import ADKConfig
|
|
14
17
|
from sqlspec.core.parameters._validator import PARAMETER_REGEX
|
|
18
|
+
from sqlspec.exceptions import ImproperConfigurationError
|
|
15
19
|
from sqlspec.extensions.adk import BaseSyncADKStore, StoredEvent, StoredSession, normalize_session_list_options
|
|
16
20
|
from sqlspec.extensions.adk.memory.store import BaseSyncADKMemoryStore
|
|
17
21
|
from sqlspec.utils.logging import get_logger
|
|
@@ -24,11 +28,33 @@ if TYPE_CHECKING:
|
|
|
24
28
|
from sqlspec.adapters.adbc.config import AdbcConfig
|
|
25
29
|
from sqlspec.extensions.adk import SessionOrderBy, StoredMemory
|
|
26
30
|
|
|
27
|
-
__all__ = ("AdbcADKMemoryStore", "AdbcADKStore")
|
|
31
|
+
__all__ = ("AdbcADKConfig", "AdbcADKMemoryStore", "AdbcADKStore", "decode_adbc_pgvector", "format_pgvector_literal")
|
|
28
32
|
|
|
29
33
|
logger = get_logger("sqlspec.adapters.adbc.adk.store")
|
|
30
34
|
|
|
31
35
|
|
|
36
|
+
class AdbcADKConfig(ADKConfig):
|
|
37
|
+
"""ADBC-specific ADK extension settings.
|
|
38
|
+
|
|
39
|
+
Use these keys inside ``extension_config["adk"]`` with the ADBC ADK stores.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
enable_bm25: NotRequired[bool]
|
|
43
|
+
"""Enable pg_textsearch BM25 indexing for hybrid vector search on PostgreSQL. Defaults to False."""
|
|
44
|
+
|
|
45
|
+
vector_dimensions: NotRequired[int]
|
|
46
|
+
"""Dimensions for the vector embedding column on PostgreSQL. Defaults to 768."""
|
|
47
|
+
|
|
48
|
+
vector_index_type: NotRequired[Literal["hnsw", "ivfflat", "scann", "none"]]
|
|
49
|
+
"""Vector index type for PostgreSQL. Defaults to 'hnsw'."""
|
|
50
|
+
|
|
51
|
+
scann_num_leaves: NotRequired[int]
|
|
52
|
+
"""Number of leaves for ScaNN index on PostgreSQL. Defaults to 50."""
|
|
53
|
+
|
|
54
|
+
scann_quantizer: NotRequired[Literal["SQ8", "FP32"]]
|
|
55
|
+
"""Quantizer for ScaNN index on PostgreSQL. Defaults to 'SQ8'."""
|
|
56
|
+
|
|
57
|
+
|
|
32
58
|
DIALECT_POSTGRESQL: Final = "postgresql"
|
|
33
59
|
DIALECT_SQLITE: Final = "sqlite"
|
|
34
60
|
DIALECT_DUCKDB: Final = "duckdb"
|
|
@@ -292,7 +318,7 @@ class AdbcADKStore(BaseSyncADKStore["AdbcConfig"]):
|
|
|
292
318
|
"""
|
|
293
319
|
if data is None:
|
|
294
320
|
return {}
|
|
295
|
-
return from_json(str(data))
|
|
321
|
+
return cast("dict[str, Any]", from_json(str(data)))
|
|
296
322
|
|
|
297
323
|
def _serialize_json_field(self, value: Any) -> "str | None":
|
|
298
324
|
"""Serialize optional JSON field for event storage.
|
|
@@ -348,7 +374,7 @@ class AdbcADKStore(BaseSyncADKStore["AdbcConfig"]):
|
|
|
348
374
|
"""
|
|
349
375
|
if data is None:
|
|
350
376
|
return None
|
|
351
|
-
return from_json(str(data))
|
|
377
|
+
return cast("dict[str, Any]", from_json(str(data)))
|
|
352
378
|
|
|
353
379
|
def _sessions_table_ddl(self) -> str:
|
|
354
380
|
"""Get CREATE TABLE SQL for sessions with dialect dispatch.
|
|
@@ -1662,6 +1688,9 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
1662
1688
|
def __init__(self, config: "AdbcConfig") -> None:
|
|
1663
1689
|
super().__init__(config)
|
|
1664
1690
|
self._dialect = self._detect_dialect()
|
|
1691
|
+
if self._vector_dimensions <= 0:
|
|
1692
|
+
msg = "vector_dimensions must be a positive integer"
|
|
1693
|
+
raise ImproperConfigurationError(msg)
|
|
1665
1694
|
|
|
1666
1695
|
@property
|
|
1667
1696
|
def dialect(self) -> str:
|
|
@@ -1691,8 +1720,8 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
1691
1720
|
scope_filter: Literal["all", "user", "app"] = "all",
|
|
1692
1721
|
embedding: "Sequence[float] | None" = None,
|
|
1693
1722
|
) -> "list[StoredMemory]":
|
|
1694
|
-
"""Search memory entries by text query."""
|
|
1695
|
-
return self._search_entries(query, app_name, user_id, limit, scope_filter)
|
|
1723
|
+
"""Search memory entries by text query or vector embedding."""
|
|
1724
|
+
return self._search_entries(query, app_name, user_id, limit, scope_filter, embedding=embedding)
|
|
1696
1725
|
|
|
1697
1726
|
def delete_entries_by_session(self, session_id: str) -> int:
|
|
1698
1727
|
"""Delete all memory entries for a specific session."""
|
|
@@ -1702,6 +1731,23 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
1702
1731
|
"""Delete memory entries older than specified days."""
|
|
1703
1732
|
return self._delete_entries_older_than(days, app_name, scope)
|
|
1704
1733
|
|
|
1734
|
+
def _pgvector_enabled(self) -> bool:
|
|
1735
|
+
if self._dialect != DIALECT_POSTGRESQL:
|
|
1736
|
+
return False
|
|
1737
|
+
pgvector_available = getattr(self._config, "_pgvector_available", None)
|
|
1738
|
+
return pgvector_available is not False
|
|
1739
|
+
|
|
1740
|
+
def _ensure_postgres_vector_supported(self) -> None:
|
|
1741
|
+
if self._dialect != DIALECT_POSTGRESQL:
|
|
1742
|
+
msg = f"Vector embeddings in AdbcADKMemoryStore are only supported on PostgreSQL (current dialect: {self._dialect})."
|
|
1743
|
+
raise ImproperConfigurationError(msg)
|
|
1744
|
+
if getattr(self._config, "_pgvector_available", None) is False:
|
|
1745
|
+
msg = "PostgreSQL extension 'vector' (pgvector) is required for vector embeddings in AdbcADKMemoryStore."
|
|
1746
|
+
raise ImproperConfigurationError(msg)
|
|
1747
|
+
if self._vector_dimensions <= 0:
|
|
1748
|
+
msg = "vector_dimensions must be a positive integer."
|
|
1749
|
+
raise ImproperConfigurationError(msg)
|
|
1750
|
+
|
|
1705
1751
|
def _detect_dialect(self) -> str:
|
|
1706
1752
|
driver_name = self._config.connection_config.get("driver_name", "")
|
|
1707
1753
|
if (
|
|
@@ -1799,6 +1845,9 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
1799
1845
|
|
|
1800
1846
|
def _memory_ddl_postgresql(self) -> str:
|
|
1801
1847
|
owner_id_ddl = f", {self._owner_id_column_ddl}" if self._owner_id_column_ddl else ""
|
|
1848
|
+
embedding_line = (
|
|
1849
|
+
f"\n embedding VECTOR({self._vector_dimensions})," if self._pgvector_enabled() else ""
|
|
1850
|
+
)
|
|
1802
1851
|
return f"""
|
|
1803
1852
|
CREATE TABLE IF NOT EXISTS {self._memory_table} (
|
|
1804
1853
|
id VARCHAR(128) PRIMARY KEY,
|
|
@@ -1808,7 +1857,7 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
1808
1857
|
scope VARCHAR(16) NOT NULL DEFAULT 'user',
|
|
1809
1858
|
event_id VARCHAR(128) NOT NULL UNIQUE,
|
|
1810
1859
|
author VARCHAR(256){owner_id_ddl},
|
|
1811
|
-
timestamp TIMESTAMPTZ NOT NULL,
|
|
1860
|
+
timestamp TIMESTAMPTZ NOT NULL,{embedding_line}
|
|
1812
1861
|
content_json JSONB NOT NULL,
|
|
1813
1862
|
content_text TEXT NOT NULL,
|
|
1814
1863
|
metadata_json JSONB,
|
|
@@ -1983,6 +2032,17 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
1983
2032
|
if not self._enabled:
|
|
1984
2033
|
return
|
|
1985
2034
|
|
|
2035
|
+
if self._dialect == DIALECT_POSTGRESQL:
|
|
2036
|
+
if getattr(self._config, "_pgvector_available", None) is None:
|
|
2037
|
+
detect_fn = getattr(self._config, "_detect_extensions_if_needed", None)
|
|
2038
|
+
if callable(detect_fn):
|
|
2039
|
+
with contextlib.suppress(Exception):
|
|
2040
|
+
detect_fn()
|
|
2041
|
+
if self._enable_bm25 and getattr(self._config, "_pg_textsearch_available", None) is False:
|
|
2042
|
+
ensure_bm25 = getattr(self._config, "_ensure_pg_textsearch_available", None)
|
|
2043
|
+
if callable(ensure_bm25):
|
|
2044
|
+
ensure_bm25()
|
|
2045
|
+
|
|
1986
2046
|
skip_indexes = self._dialect in {DIALECT_BIGQUERY, DIALECT_SNOWFLAKE}
|
|
1987
2047
|
with self._config.provide_connection() as conn:
|
|
1988
2048
|
cursor = conn.cursor()
|
|
@@ -2009,9 +2069,51 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
2009
2069
|
cursor, self._dialect, f"idx_{self._memory_table}_session", self._memory_table, "session_id"
|
|
2010
2070
|
)
|
|
2011
2071
|
conn.commit()
|
|
2072
|
+
|
|
2073
|
+
if self._dialect == DIALECT_POSTGRESQL:
|
|
2074
|
+
for index_sql in self._postgres_memory_extra_index_sql():
|
|
2075
|
+
cursor.execute(index_sql)
|
|
2076
|
+
conn.commit()
|
|
2012
2077
|
finally:
|
|
2013
2078
|
cursor.close()
|
|
2014
2079
|
|
|
2080
|
+
def _postgres_memory_extra_index_sql(self) -> "list[str]":
|
|
2081
|
+
indexes: list[str] = []
|
|
2082
|
+
if self._use_fts:
|
|
2083
|
+
indexes.append(
|
|
2084
|
+
f"CREATE INDEX IF NOT EXISTS idx_{self._memory_table}_fts ON {self._memory_table} USING GIN (to_tsvector('english', content_text))"
|
|
2085
|
+
)
|
|
2086
|
+
if self._enable_bm25:
|
|
2087
|
+
indexes.append(
|
|
2088
|
+
f"CREATE INDEX IF NOT EXISTS idx_{self._memory_table}_bm25 ON {self._memory_table} USING bm25 (content_text) WITH (text_config='english')"
|
|
2089
|
+
)
|
|
2090
|
+
if not self._pgvector_enabled():
|
|
2091
|
+
return indexes
|
|
2092
|
+
|
|
2093
|
+
if self._vector_index_type == "scann":
|
|
2094
|
+
if (
|
|
2095
|
+
not isinstance(self._scann_num_leaves, int)
|
|
2096
|
+
or isinstance(self._scann_num_leaves, bool)
|
|
2097
|
+
or self._scann_num_leaves <= 0
|
|
2098
|
+
):
|
|
2099
|
+
msg = "extension_config['adk']['scann_num_leaves'] must be a positive integer"
|
|
2100
|
+
raise ValueError(msg)
|
|
2101
|
+
if self._scann_quantizer not in {"SQ8", "FP32"}:
|
|
2102
|
+
msg = "extension_config['adk']['scann_quantizer'] must be 'SQ8' or 'FP32'"
|
|
2103
|
+
raise ValueError(msg)
|
|
2104
|
+
indexes.append(
|
|
2105
|
+
f"CREATE INDEX IF NOT EXISTS idx_{self._memory_table}_scann ON {self._memory_table} USING scann (embedding) WITH (num_leaves = {self._scann_num_leaves}, quantizer = '{self._scann_quantizer}')"
|
|
2106
|
+
)
|
|
2107
|
+
elif self._vector_index_type == "ivfflat":
|
|
2108
|
+
indexes.append(
|
|
2109
|
+
f"CREATE INDEX IF NOT EXISTS idx_{self._memory_table}_ivfflat ON {self._memory_table} USING ivfflat (embedding vector_cosine_ops)"
|
|
2110
|
+
)
|
|
2111
|
+
elif self._vector_index_type == "hnsw":
|
|
2112
|
+
indexes.append(
|
|
2113
|
+
f"CREATE INDEX IF NOT EXISTS idx_{self._memory_table}_hnsw ON {self._memory_table} USING hnsw (embedding vector_cosine_ops)"
|
|
2114
|
+
)
|
|
2115
|
+
return indexes
|
|
2116
|
+
|
|
2015
2117
|
def _insert_memory_entries(self, entries: "list[StoredMemory]", owner_id: "object | None" = None) -> int:
|
|
2016
2118
|
if not self._enabled:
|
|
2017
2119
|
msg = "Memory store is disabled"
|
|
@@ -2020,11 +2122,36 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
2020
2122
|
if not entries:
|
|
2021
2123
|
return 0
|
|
2022
2124
|
|
|
2125
|
+
has_non_null_embedding = any(entry.get("embedding") is not None for entry in entries)
|
|
2126
|
+
if has_non_null_embedding:
|
|
2127
|
+
self._ensure_postgres_vector_supported()
|
|
2128
|
+
|
|
2129
|
+
include_vector = self._pgvector_enabled()
|
|
2023
2130
|
inserted_count = 0
|
|
2024
2131
|
use_returning = self._dialect in {DIALECT_SQLITE, DIALECT_POSTGRESQL, DIALECT_DUCKDB}
|
|
2025
2132
|
json_placeholder = self._json_placeholder()
|
|
2026
2133
|
|
|
2027
|
-
if
|
|
2134
|
+
if include_vector:
|
|
2135
|
+
if self._owner_id_column_name:
|
|
2136
|
+
sql = f"""
|
|
2137
|
+
INSERT INTO {self._memory_table} (
|
|
2138
|
+
id, session_id, app_name, user_id, scope, event_id, author,
|
|
2139
|
+
{self._owner_id_column_name}, timestamp, embedding, content_json, content_text,
|
|
2140
|
+
metadata_json, inserted_at
|
|
2141
|
+
) VALUES (
|
|
2142
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?, ?::text::vector, {json_placeholder}, ?, {json_placeholder}, ?
|
|
2143
|
+
) ON CONFLICT(event_id) DO NOTHING RETURNING 1
|
|
2144
|
+
"""
|
|
2145
|
+
else:
|
|
2146
|
+
sql = f"""
|
|
2147
|
+
INSERT INTO {self._memory_table} (
|
|
2148
|
+
id, session_id, app_name, user_id, scope, event_id, author,
|
|
2149
|
+
timestamp, embedding, content_json, content_text, metadata_json, inserted_at
|
|
2150
|
+
) VALUES (
|
|
2151
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?::text::vector, {json_placeholder}, ?, {json_placeholder}, ?
|
|
2152
|
+
) ON CONFLICT(event_id) DO NOTHING RETURNING 1
|
|
2153
|
+
"""
|
|
2154
|
+
elif self._owner_id_column_name:
|
|
2028
2155
|
if use_returning:
|
|
2029
2156
|
sql = f"""
|
|
2030
2157
|
INSERT INTO {self._memory_table} (
|
|
@@ -2070,8 +2197,44 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
2070
2197
|
for entry in entries:
|
|
2071
2198
|
content_json = self._serialize_json_field(entry["content_json"])
|
|
2072
2199
|
metadata_json = self._serialize_json_field(entry.get("metadata_json"))
|
|
2200
|
+
raw_emb = entry.get("embedding")
|
|
2201
|
+
embedding_literal = format_pgvector_literal(raw_emb) if raw_emb is not None else None
|
|
2073
2202
|
params: tuple[Any, ...]
|
|
2074
|
-
if
|
|
2203
|
+
if include_vector:
|
|
2204
|
+
if self._owner_id_column_name:
|
|
2205
|
+
params = (
|
|
2206
|
+
entry["id"],
|
|
2207
|
+
entry["session_id"],
|
|
2208
|
+
entry["app_name"],
|
|
2209
|
+
entry["user_id"],
|
|
2210
|
+
entry.get("scope", "user"),
|
|
2211
|
+
entry["event_id"],
|
|
2212
|
+
entry.get("author"),
|
|
2213
|
+
owner_id,
|
|
2214
|
+
self._encode_timestamp(entry["timestamp"]),
|
|
2215
|
+
embedding_literal,
|
|
2216
|
+
content_json,
|
|
2217
|
+
entry["content_text"],
|
|
2218
|
+
metadata_json,
|
|
2219
|
+
self._encode_timestamp(entry["inserted_at"]),
|
|
2220
|
+
)
|
|
2221
|
+
else:
|
|
2222
|
+
params = (
|
|
2223
|
+
entry["id"],
|
|
2224
|
+
entry["session_id"],
|
|
2225
|
+
entry["app_name"],
|
|
2226
|
+
entry["user_id"],
|
|
2227
|
+
entry.get("scope", "user"),
|
|
2228
|
+
entry["event_id"],
|
|
2229
|
+
entry.get("author"),
|
|
2230
|
+
self._encode_timestamp(entry["timestamp"]),
|
|
2231
|
+
embedding_literal,
|
|
2232
|
+
content_json,
|
|
2233
|
+
entry["content_text"],
|
|
2234
|
+
metadata_json,
|
|
2235
|
+
self._encode_timestamp(entry["inserted_at"]),
|
|
2236
|
+
)
|
|
2237
|
+
elif self._owner_id_column_name:
|
|
2075
2238
|
params = (
|
|
2076
2239
|
entry["id"],
|
|
2077
2240
|
entry["session_id"],
|
|
@@ -2139,20 +2302,44 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
2139
2302
|
user_id: str,
|
|
2140
2303
|
limit: "int | None" = None,
|
|
2141
2304
|
scope_filter: Literal["all", "user", "app"] = "all",
|
|
2305
|
+
*,
|
|
2306
|
+
embedding: "Sequence[float] | None" = None,
|
|
2142
2307
|
) -> "list[StoredMemory]":
|
|
2143
2308
|
if not self._enabled:
|
|
2144
2309
|
msg = "Memory store is disabled"
|
|
2145
2310
|
raise RuntimeError(msg)
|
|
2146
2311
|
|
|
2147
|
-
|
|
2312
|
+
has_query = bool(query and query.strip())
|
|
2313
|
+
if not has_query and embedding is None:
|
|
2148
2314
|
return []
|
|
2149
2315
|
|
|
2150
|
-
if self.
|
|
2316
|
+
effective_limit = limit if limit is not None else self._max_results
|
|
2317
|
+
|
|
2318
|
+
if embedding is not None:
|
|
2319
|
+
self._ensure_postgres_vector_supported()
|
|
2320
|
+
try:
|
|
2321
|
+
if self._enable_bm25 and has_query:
|
|
2322
|
+
ensure_bm25 = getattr(self._config, "_ensure_pg_textsearch_available", None)
|
|
2323
|
+
if callable(ensure_bm25) and getattr(self._config, "_pg_textsearch_available", None) is False:
|
|
2324
|
+
ensure_bm25()
|
|
2325
|
+
return self._search_entries_hybrid(
|
|
2326
|
+
query, app_name, user_id, effective_limit, scope_filter, embedding
|
|
2327
|
+
)
|
|
2328
|
+
return self._search_entries_vector(app_name, user_id, effective_limit, scope_filter, embedding)
|
|
2329
|
+
except Exception as exc:
|
|
2330
|
+
error_msg = str(exc).lower()
|
|
2331
|
+
if any(pattern in error_msg for pattern in ADBC_TABLE_NOT_FOUND_PATTERNS):
|
|
2332
|
+
return []
|
|
2333
|
+
raise
|
|
2334
|
+
|
|
2335
|
+
if self._use_fts and self._dialect != DIALECT_POSTGRESQL:
|
|
2151
2336
|
logger.warning("ADBC memory store does not support FTS, falling back to simple search")
|
|
2152
2337
|
|
|
2153
|
-
effective_limit = limit if limit is not None else self._max_results
|
|
2154
2338
|
where_scope, scope_params = _build_adbc_scope_where(app_name, user_id, scope_filter)
|
|
2155
|
-
if self._dialect ==
|
|
2339
|
+
if self._dialect == DIALECT_POSTGRESQL and self._use_fts:
|
|
2340
|
+
match_clause = "to_tsvector('english', content_text) @@ plainto_tsquery('english', ?)"
|
|
2341
|
+
search_param = query
|
|
2342
|
+
elif self._dialect == DIALECT_DB2:
|
|
2156
2343
|
match_clause = "POSSTR(LOWER(content_text), LOWER(?)) > 0"
|
|
2157
2344
|
search_param = query
|
|
2158
2345
|
else:
|
|
@@ -2164,9 +2351,10 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
2164
2351
|
else:
|
|
2165
2352
|
limit_clause = "LIMIT ?"
|
|
2166
2353
|
|
|
2354
|
+
embedding_projection = ", embedding::text AS embedding" if self._pgvector_enabled() else ""
|
|
2167
2355
|
sql = f"""
|
|
2168
2356
|
SELECT id, session_id, app_name, user_id, scope, event_id, author,
|
|
2169
|
-
timestamp, content_json, content_text, metadata_json, inserted_at
|
|
2357
|
+
timestamp, content_json, content_text, metadata_json, inserted_at{embedding_projection}
|
|
2170
2358
|
FROM {self._memory_table}
|
|
2171
2359
|
WHERE {where_scope}
|
|
2172
2360
|
AND {match_clause}
|
|
@@ -2190,6 +2378,81 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
2190
2378
|
|
|
2191
2379
|
return self._rows_to_records(rows)
|
|
2192
2380
|
|
|
2381
|
+
def _search_entries_vector(
|
|
2382
|
+
self,
|
|
2383
|
+
app_name: str,
|
|
2384
|
+
user_id: str,
|
|
2385
|
+
limit: int,
|
|
2386
|
+
scope_filter: Literal["all", "user", "app"],
|
|
2387
|
+
embedding: "Sequence[float]",
|
|
2388
|
+
) -> "list[StoredMemory]":
|
|
2389
|
+
where_scope, scope_params = _build_adbc_scope_where(app_name, user_id, scope_filter)
|
|
2390
|
+
vector_literal = format_pgvector_literal(embedding)
|
|
2391
|
+
sql = f"""
|
|
2392
|
+
SELECT id, session_id, app_name, user_id, scope, event_id, author,
|
|
2393
|
+
timestamp, content_json, content_text, metadata_json, inserted_at,
|
|
2394
|
+
embedding::text AS embedding
|
|
2395
|
+
FROM {self._memory_table}
|
|
2396
|
+
WHERE {where_scope} AND embedding IS NOT NULL
|
|
2397
|
+
ORDER BY embedding <=> ?::text::vector ASC, timestamp DESC
|
|
2398
|
+
LIMIT ?
|
|
2399
|
+
"""
|
|
2400
|
+
params = (*scope_params, vector_literal, limit)
|
|
2401
|
+
with self._config.provide_connection() as conn:
|
|
2402
|
+
cursor = conn.cursor()
|
|
2403
|
+
try:
|
|
2404
|
+
self._execute(cursor, sql, params)
|
|
2405
|
+
rows = cursor.fetchall()
|
|
2406
|
+
finally:
|
|
2407
|
+
cursor.close()
|
|
2408
|
+
return self._rows_to_records(rows)
|
|
2409
|
+
|
|
2410
|
+
def _search_entries_hybrid(
|
|
2411
|
+
self,
|
|
2412
|
+
query: str,
|
|
2413
|
+
app_name: str,
|
|
2414
|
+
user_id: str,
|
|
2415
|
+
limit: int,
|
|
2416
|
+
scope_filter: Literal["all", "user", "app"],
|
|
2417
|
+
embedding: "Sequence[float]",
|
|
2418
|
+
) -> "list[StoredMemory]":
|
|
2419
|
+
where_scope, scope_params = _build_adbc_scope_where(app_name, user_id, scope_filter)
|
|
2420
|
+
vector_literal = format_pgvector_literal(embedding)
|
|
2421
|
+
candidate_limit = max(limit * 2, 50)
|
|
2422
|
+
sql = f"""
|
|
2423
|
+
WITH vector_matches AS (
|
|
2424
|
+
SELECT id, RANK() OVER (ORDER BY embedding <=> ?::text::vector) AS rank_vec
|
|
2425
|
+
FROM {self._memory_table}
|
|
2426
|
+
WHERE {where_scope} AND embedding IS NOT NULL
|
|
2427
|
+
LIMIT ?
|
|
2428
|
+
),
|
|
2429
|
+
text_matches AS (
|
|
2430
|
+
SELECT id, RANK() OVER (ORDER BY content_text <@> ?) AS rank_txt
|
|
2431
|
+
FROM {self._memory_table}
|
|
2432
|
+
WHERE {where_scope}
|
|
2433
|
+
LIMIT ?
|
|
2434
|
+
)
|
|
2435
|
+
SELECT m.id, m.session_id, m.app_name, m.user_id, m.scope, m.event_id, m.author,
|
|
2436
|
+
m.timestamp, m.content_json, m.content_text, m.metadata_json, m.inserted_at,
|
|
2437
|
+
m.embedding::text AS embedding,
|
|
2438
|
+
(COALESCE(1.0 / (60 + v.rank_vec), 0.0) + COALESCE(1.0 / (60 + t.rank_txt), 0.0)) AS rrf_score
|
|
2439
|
+
FROM {self._memory_table} m
|
|
2440
|
+
LEFT JOIN vector_matches v ON m.id = v.id
|
|
2441
|
+
LEFT JOIN text_matches t ON m.id = t.id
|
|
2442
|
+
WHERE v.id IS NOT NULL OR t.id IS NOT NULL
|
|
2443
|
+
ORDER BY rrf_score DESC, m.timestamp DESC
|
|
2444
|
+
LIMIT ?
|
|
2445
|
+
"""
|
|
2446
|
+
params = (vector_literal, *scope_params, candidate_limit, query, *scope_params, candidate_limit, limit)
|
|
2447
|
+
with self._config.provide_connection() as conn:
|
|
2448
|
+
cursor = conn.cursor()
|
|
2449
|
+
try:
|
|
2450
|
+
self._execute(cursor, sql, params)
|
|
2451
|
+
rows = cursor.fetchall()
|
|
2452
|
+
finally:
|
|
2453
|
+
cursor.close()
|
|
2454
|
+
return self._rows_to_records(rows)
|
|
2455
|
+
|
|
2193
2456
|
def _delete_entries_by_session(self, session_id: str) -> int:
|
|
2194
2457
|
if not self._enabled:
|
|
2195
2458
|
msg = "Memory store is disabled"
|
|
@@ -2249,13 +2512,40 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
2249
2512
|
def _rows_to_records(self, rows: "list[Any]") -> "list[StoredMemory]":
|
|
2250
2513
|
records: list[StoredMemory] = []
|
|
2251
2514
|
for row in rows:
|
|
2252
|
-
|
|
2515
|
+
if isinstance(row, Mapping):
|
|
2516
|
+
content_json = row["content_json"]
|
|
2517
|
+
metadata_json = row.get("metadata_json")
|
|
2518
|
+
raw_embedding = row.get("embedding")
|
|
2519
|
+
row_id = row["id"]
|
|
2520
|
+
session_id = row["session_id"]
|
|
2521
|
+
app_name = row["app_name"]
|
|
2522
|
+
user_id = row["user_id"]
|
|
2523
|
+
scope = row.get("scope", "user")
|
|
2524
|
+
event_id = row["event_id"]
|
|
2525
|
+
author = row.get("author")
|
|
2526
|
+
timestamp_raw = row["timestamp"]
|
|
2527
|
+
content_text = row["content_text"]
|
|
2528
|
+
inserted_at_raw = row["inserted_at"]
|
|
2529
|
+
else:
|
|
2530
|
+
content_json = row[8]
|
|
2531
|
+
metadata_json = row[10]
|
|
2532
|
+
raw_embedding = row[_ADBC_EMBEDDING_COLUMN_INDEX] if len(row) > _ADBC_EMBEDDING_COLUMN_INDEX else None
|
|
2533
|
+
row_id = row[0]
|
|
2534
|
+
session_id = row[1]
|
|
2535
|
+
app_name = row[2]
|
|
2536
|
+
user_id = row[3]
|
|
2537
|
+
scope = row[4]
|
|
2538
|
+
event_id = row[5]
|
|
2539
|
+
author = row[6]
|
|
2540
|
+
timestamp_raw = row[7]
|
|
2541
|
+
content_text = row[9]
|
|
2542
|
+
inserted_at_raw = row[11]
|
|
2543
|
+
|
|
2253
2544
|
if isinstance(content_json, dict):
|
|
2254
2545
|
content_value = content_json
|
|
2255
2546
|
else:
|
|
2256
2547
|
content_value = from_json(content_json if isinstance(content_json, (str, bytes)) else str(content_json))
|
|
2257
2548
|
|
|
2258
|
-
metadata_json = row[10]
|
|
2259
2549
|
if metadata_json is None:
|
|
2260
2550
|
metadata_value = None
|
|
2261
2551
|
elif isinstance(metadata_json, dict):
|
|
@@ -2266,23 +2556,72 @@ class AdbcADKMemoryStore(BaseSyncADKMemoryStore["AdbcConfig"]):
|
|
|
2266
2556
|
)
|
|
2267
2557
|
|
|
2268
2558
|
records.append({
|
|
2269
|
-
"id":
|
|
2270
|
-
"session_id":
|
|
2271
|
-
"app_name":
|
|
2272
|
-
"user_id":
|
|
2273
|
-
"scope":
|
|
2274
|
-
"event_id":
|
|
2275
|
-
"author":
|
|
2276
|
-
"timestamp": self._decode_timestamp(
|
|
2559
|
+
"id": row_id,
|
|
2560
|
+
"session_id": session_id,
|
|
2561
|
+
"app_name": app_name,
|
|
2562
|
+
"user_id": user_id,
|
|
2563
|
+
"scope": scope,
|
|
2564
|
+
"event_id": event_id,
|
|
2565
|
+
"author": author,
|
|
2566
|
+
"timestamp": self._decode_timestamp(timestamp_raw),
|
|
2277
2567
|
"content_json": content_value,
|
|
2278
|
-
"content_text":
|
|
2568
|
+
"content_text": content_text,
|
|
2279
2569
|
"metadata_json": metadata_value,
|
|
2280
|
-
"inserted_at": self._decode_timestamp(
|
|
2281
|
-
"embedding":
|
|
2570
|
+
"inserted_at": self._decode_timestamp(inserted_at_raw),
|
|
2571
|
+
"embedding": decode_adbc_pgvector(raw_embedding),
|
|
2282
2572
|
})
|
|
2283
2573
|
return records
|
|
2284
2574
|
|
|
2285
2575
|
|
|
2576
|
+
def format_pgvector_literal(embedding: "Sequence[float]") -> str:
|
|
2577
|
+
"""Serialize a sequence of floats into a canonical pgvector literal ``'[v1,v2,...]'``.
|
|
2578
|
+
|
|
2579
|
+
Args:
|
|
2580
|
+
embedding: Sequence of numeric values representing a vector embedding.
|
|
2581
|
+
|
|
2582
|
+
Returns:
|
|
2583
|
+
Canonical pgvector text literal string.
|
|
2584
|
+
"""
|
|
2585
|
+
return "[" + ",".join(str(float(v)) for v in embedding) + "]"
|
|
2586
|
+
|
|
2587
|
+
|
|
2588
|
+
_ADBC_EMBEDDING_COLUMN_INDEX: Final = 12
|
|
2589
|
+
_PGVECTOR_BINARY_HEADER_BYTES: Final = 4
|
|
2590
|
+
_PGVECTOR_FLOAT32_BYTES: Final = 4
|
|
2591
|
+
|
|
2592
|
+
|
|
2593
|
+
def decode_adbc_pgvector(raw_embedding: Any) -> "list[float] | None":
|
|
2594
|
+
"""Decode a PostgreSQL ``vector`` value returned over ADBC into ``list[float]``.
|
|
2595
|
+
|
|
2596
|
+
Supports text literals (``'[0.1,0.2,...]'``), UTF-8 byte strings, pgvector
|
|
2597
|
+
binary wire frames (``>hh{dim}f``), ``PgVector`` objects, and Python sequences.
|
|
2598
|
+
|
|
2599
|
+
Args:
|
|
2600
|
+
raw_embedding: Raw column value returned by the ADBC PostgreSQL cursor.
|
|
2601
|
+
|
|
2602
|
+
Returns:
|
|
2603
|
+
Decoded list of floats, or ``None`` if input is ``None``.
|
|
2604
|
+
"""
|
|
2605
|
+
if raw_embedding is None:
|
|
2606
|
+
return None
|
|
2607
|
+
if isinstance(raw_embedding, (bytes, bytearray, memoryview)):
|
|
2608
|
+
buf = bytes(raw_embedding)
|
|
2609
|
+
if len(buf) >= _PGVECTOR_BINARY_HEADER_BYTES:
|
|
2610
|
+
dim, unused = struct.unpack_from(">hh", buf, 0)
|
|
2611
|
+
if unused == 0 and dim >= 0 and len(buf) == _PGVECTOR_BINARY_HEADER_BYTES + _PGVECTOR_FLOAT32_BYTES * dim:
|
|
2612
|
+
return list(struct.unpack_from(f">{dim}f", buf, _PGVECTOR_BINARY_HEADER_BYTES))
|
|
2613
|
+
raw_embedding = buf.decode("utf-8")
|
|
2614
|
+
if isinstance(raw_embedding, str):
|
|
2615
|
+
stripped = raw_embedding.strip().strip("[]").strip()
|
|
2616
|
+
if not stripped:
|
|
2617
|
+
return []
|
|
2618
|
+
return [float(part) for part in stripped.split(",") if part.strip()]
|
|
2619
|
+
to_list_fn = getattr(raw_embedding, "to_list", None)
|
|
2620
|
+
if callable(to_list_fn):
|
|
2621
|
+
return [float(v) for v in cast("Sequence[Any]", to_list_fn())]
|
|
2622
|
+
return [float(v) for v in raw_embedding]
|
|
2623
|
+
|
|
2624
|
+
|
|
2286
2625
|
_DB2_TABLE_EXISTS_SQL: Final = (
|
|
2287
2626
|
"SELECT 1 FROM SYSCAT.TABLES WHERE TABSCHEMA = COALESCE(CAST(? AS VARCHAR(128)), CURRENT SCHEMA) AND TABNAME = ?"
|
|
2288
2627
|
)
|
sqlspec/adapters/adbc/config.py
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
"""ADBC database configuration."""
|
|
2
2
|
|
|
3
|
+
from pathlib import Path
|
|
3
4
|
from typing import TYPE_CHECKING, Any, ClassVar, Literal, NotRequired, TypedDict, cast
|
|
4
5
|
|
|
5
6
|
from sqlspec.adapters.adbc._typing import AdbcConnection, AdbcCursor, AdbcSessionContext
|
|
6
7
|
from sqlspec.adapters.adbc.core import (
|
|
7
8
|
apply_driver_features,
|
|
8
9
|
build_connection_config,
|
|
10
|
+
configure_adbc_duckdb_profiling,
|
|
9
11
|
detect_postgres_extensions,
|
|
10
12
|
get_statement_config,
|
|
11
13
|
is_postgres_dialect,
|
|
@@ -15,6 +17,7 @@ from sqlspec.adapters.adbc.core import (
|
|
|
15
17
|
resolve_driver_connect_func,
|
|
16
18
|
)
|
|
17
19
|
from sqlspec.adapters.adbc.driver import AdbcDriver, AdbcExceptionHandler
|
|
20
|
+
from sqlspec.adapters.duckdb.core import cleanup_duckdb_profile_path, create_duckdb_profile_path
|
|
18
21
|
from sqlspec.config import ExtensionConfigs, NoPoolSyncConfig
|
|
19
22
|
from sqlspec.core import StatementConfig
|
|
20
23
|
from sqlspec.core.capabilities import TypeCoercionCapabilities
|
|
@@ -135,6 +138,8 @@ class AdbcDriverFeatures(TypedDict):
|
|
|
135
138
|
Only option: "poll_queue" (durable table-backed queue with lease-based retries and acknowledgements).
|
|
136
139
|
ADBC does not have native pub/sub, so poll_queue is the only backend.
|
|
137
140
|
Defaults to "poll_queue".
|
|
141
|
+
enable_statement_profiling: Enable DuckDB JSON statement profiling when using a DuckDB ADBC backend.
|
|
142
|
+
Defaults to False.
|
|
138
143
|
"""
|
|
139
144
|
|
|
140
145
|
json_serializer: "NotRequired[Callable[[Any], str]]"
|
|
@@ -150,6 +155,7 @@ class AdbcDriverFeatures(TypedDict):
|
|
|
150
155
|
enable_events: NotRequired[bool]
|
|
151
156
|
on_connection_create: "NotRequired[Callable[[AdbcConnection], None]]"
|
|
152
157
|
events_backend: NotRequired[Literal["poll_queue"]]
|
|
158
|
+
enable_statement_profiling: NotRequired[bool]
|
|
153
159
|
|
|
154
160
|
|
|
155
161
|
class AdbcConnectionContext(SyncPoolConnectionContext):
|
|
@@ -176,22 +182,42 @@ class AdbcConnectionContext(SyncPoolConnectionContext):
|
|
|
176
182
|
|
|
177
183
|
|
|
178
184
|
class _AdbcSessionConnectionHandler(SyncPoolSessionFactory):
|
|
179
|
-
__slots__ = ("_connection",)
|
|
185
|
+
__slots__ = ("_connection", "_driver_features", "_profile_path")
|
|
180
186
|
|
|
181
187
|
def __init__(self, config: "AdbcConfig") -> None:
|
|
182
188
|
super().__init__(config)
|
|
183
189
|
self._connection: AdbcConnection | None = None
|
|
190
|
+
self._driver_features: dict[str, Any] = dict(config.driver_features)
|
|
191
|
+
self._profile_path: Path | None = None
|
|
192
|
+
|
|
193
|
+
@property
|
|
194
|
+
def driver_features(self) -> "dict[str, Any]":
|
|
195
|
+
return self._driver_features
|
|
184
196
|
|
|
185
197
|
def acquire_connection(self) -> "AdbcConnection":
|
|
186
198
|
self._connection = self._config.create_connection()
|
|
199
|
+
if self._config._resolved_dialect == "duckdb" and bool(
|
|
200
|
+
self._driver_features.get("enable_statement_profiling", False)
|
|
201
|
+
):
|
|
202
|
+
self._profile_path = create_duckdb_profile_path()
|
|
203
|
+
configure_adbc_duckdb_profiling(self._connection, self._profile_path)
|
|
204
|
+
self._driver_features["_duckdb_profile_path"] = str(self._profile_path)
|
|
187
205
|
return cast("AdbcConnection", self._connection)
|
|
188
206
|
|
|
189
207
|
def release_connection(self, _conn: "AdbcConnection", **kwargs: Any) -> None:
|
|
190
208
|
if self._connection is None:
|
|
191
209
|
return
|
|
192
|
-
|
|
193
|
-
self._connection.
|
|
194
|
-
|
|
210
|
+
try:
|
|
211
|
+
if self._connection is not self._config.connection_instance:
|
|
212
|
+
self._connection.close()
|
|
213
|
+
finally:
|
|
214
|
+
self._connection = None
|
|
215
|
+
if self._profile_path is not None:
|
|
216
|
+
cleanup_duckdb_profile_path(self._profile_path)
|
|
217
|
+
self._profile_path = None
|
|
218
|
+
raw_path = self._driver_features.pop("_duckdb_profile_path", None)
|
|
219
|
+
if isinstance(raw_path, str):
|
|
220
|
+
cleanup_duckdb_profile_path(Path(raw_path))
|
|
195
221
|
|
|
196
222
|
|
|
197
223
|
class AdbcConfig(NoPoolSyncConfig[AdbcConnection, AdbcDriver]):
|
|
@@ -369,7 +395,7 @@ class AdbcConfig(NoPoolSyncConfig[AdbcConnection, AdbcDriver]):
|
|
|
369
395
|
acquire_connection=handler.acquire_connection,
|
|
370
396
|
release_connection=handler.release_connection,
|
|
371
397
|
statement_config=statement_config,
|
|
372
|
-
driver_features=
|
|
398
|
+
driver_features=handler.driver_features,
|
|
373
399
|
prepare_driver=self._prepare_driver,
|
|
374
400
|
)
|
|
375
401
|
|