pgvector-haystack 6.3.1__tar.gz → 6.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/CHANGELOG.md +39 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/PKG-INFO +1 -1
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/examples/embedding_retrieval.py +4 -1
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/examples/hybrid_retrieval.py +4 -1
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +15 -2
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +12 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/document_stores/pgvector/document_store.py +34 -12
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/document_stores/pgvector/filters.py +5 -5
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/test_document_store.py +105 -9
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/test_document_store_async.py +43 -1
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/test_filters.py +40 -1
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/test_retrievers.py +41 -1
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/.gitignore +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/LICENSE.txt +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/README.md +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/pydoc/config_docusaurus.yml +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/pyproject.toml +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/components/retrievers/pgvector/__init__.py +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/document_stores/pgvector/__init__.py +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/document_stores/pgvector/converters.py +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/src/haystack_integrations/document_stores/py.typed +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/__init__.py +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/conftest.py +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/test_converters.py +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/test_retrieval.py +0 -0
- {pgvector_haystack-6.3.1 → pgvector_haystack-6.4.1}/tests/test_retrieval_async.py +0 -0
|
@@ -1,5 +1,44 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/pgvector-v6.4.0] - 2026-07-23
|
|
4
|
+
|
|
5
|
+
### 🚀 Features
|
|
6
|
+
|
|
7
|
+
- Pgvector - add closing methods (#3650)
|
|
8
|
+
|
|
9
|
+
### 🐛 Bug Fixes
|
|
10
|
+
|
|
11
|
+
- *(pgvector)* NO_VALUE sentinel collides with the literal string 'no_value' (#3593)
|
|
12
|
+
|
|
13
|
+
### 📚 Documentation
|
|
14
|
+
|
|
15
|
+
- Replace old haystack core imports with haystack_integrations paths (#3545)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
## [integrations/pgvector-v6.3.1] - 2026-06-01
|
|
19
|
+
|
|
20
|
+
### 🐛 Bug Fixes
|
|
21
|
+
|
|
22
|
+
- Replace in place dataclass mutations for pgvector #11087 (#3142)
|
|
23
|
+
- *(pgvector)* Order retrieval by distance operator (#3370)
|
|
24
|
+
|
|
25
|
+
### 🚜 Refactor
|
|
26
|
+
|
|
27
|
+
- *(pgvector)* Use async DocumentStore mixin tests (#3094)
|
|
28
|
+
|
|
29
|
+
### 🧪 Testing
|
|
30
|
+
|
|
31
|
+
- `PGVectorDocumentStore`e use Mixin tests (#3003)
|
|
32
|
+
- Test compatible integrations with python 3.14; update pyproject (#3001)
|
|
33
|
+
- Track test coverage for all integrations (#3065)
|
|
34
|
+
- Add tests for pgvector document store (#3156)
|
|
35
|
+
|
|
36
|
+
### 🧹 Chores
|
|
37
|
+
|
|
38
|
+
- Add ANN ruff ruleset to optimum, paddleocr, pgvector, pinecone, pyversity, qdrant, ragas, snowflake (#2992)
|
|
39
|
+
- Enforce ruff docstring rules in integrations 31-40 (openrouter, opensearch, optimum, paddleocr, pgvector, pinecone, pyversity, qdrant, ragas, snowflake) (#3011)
|
|
40
|
+
|
|
41
|
+
|
|
3
42
|
## [integrations/pgvector-v6.3.0] - 2026-03-17
|
|
4
43
|
|
|
5
44
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.4.1
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
@@ -14,10 +14,13 @@ import glob
|
|
|
14
14
|
|
|
15
15
|
from haystack import Pipeline
|
|
16
16
|
from haystack.components.converters import MarkdownToDocument
|
|
17
|
-
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
|
18
17
|
from haystack.components.preprocessors import DocumentSplitter
|
|
19
18
|
from haystack.components.writers import DocumentWriter
|
|
20
19
|
|
|
20
|
+
from haystack_integrations.components.embedders.sentence_transformers import (
|
|
21
|
+
SentenceTransformersDocumentEmbedder,
|
|
22
|
+
SentenceTransformersTextEmbedder,
|
|
23
|
+
)
|
|
21
24
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever
|
|
22
25
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
23
26
|
|
|
@@ -14,11 +14,14 @@ import glob
|
|
|
14
14
|
|
|
15
15
|
from haystack import Pipeline
|
|
16
16
|
from haystack.components.converters import MarkdownToDocument
|
|
17
|
-
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
|
18
17
|
from haystack.components.joiners import DocumentJoiner
|
|
19
18
|
from haystack.components.preprocessors import DocumentSplitter
|
|
20
19
|
from haystack.components.writers import DocumentWriter
|
|
21
20
|
|
|
21
|
+
from haystack_integrations.components.embedders.sentence_transformers import (
|
|
22
|
+
SentenceTransformersDocumentEmbedder,
|
|
23
|
+
SentenceTransformersTextEmbedder,
|
|
24
|
+
)
|
|
22
25
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever
|
|
23
26
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
24
27
|
|
|
@@ -21,7 +21,9 @@ class PgvectorEmbeddingRetriever:
|
|
|
21
21
|
```python
|
|
22
22
|
from haystack.document_stores import DuplicatePolicy
|
|
23
23
|
from haystack import Document, Pipeline
|
|
24
|
-
|
|
24
|
+
# Requires: pip install sentence-transformers-haystack
|
|
25
|
+
from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder
|
|
26
|
+
from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder
|
|
25
27
|
|
|
26
28
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
27
29
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever
|
|
@@ -40,7 +42,6 @@ class PgvectorEmbeddingRetriever:
|
|
|
40
42
|
Document(content="In certain places, you can witness the phenomenon of bioluminescent waves.")]
|
|
41
43
|
|
|
42
44
|
document_embedder = SentenceTransformersDocumentEmbedder()
|
|
43
|
-
document_embedder.warm_up()
|
|
44
45
|
documents_with_embeddings = document_embedder.run(documents)
|
|
45
46
|
|
|
46
47
|
document_store.write_documents(documents_with_embeddings.get("documents"), policy=DuplicatePolicy.OVERWRITE)
|
|
@@ -135,6 +136,18 @@ class PgvectorEmbeddingRetriever:
|
|
|
135
136
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
|
|
136
137
|
return default_from_dict(cls, data)
|
|
137
138
|
|
|
139
|
+
def close(self) -> None:
|
|
140
|
+
"""
|
|
141
|
+
Release the synchronous resources of the underlying Document Store.
|
|
142
|
+
"""
|
|
143
|
+
self.document_store.close()
|
|
144
|
+
|
|
145
|
+
async def close_async(self) -> None:
|
|
146
|
+
"""
|
|
147
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
148
|
+
"""
|
|
149
|
+
await self.document_store.close_async()
|
|
150
|
+
|
|
138
151
|
@component.output_types(documents=list[Document])
|
|
139
152
|
def run(
|
|
140
153
|
self,
|
|
@@ -109,6 +109,18 @@ class PgvectorKeywordRetriever:
|
|
|
109
109
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
|
|
110
110
|
return default_from_dict(cls, data)
|
|
111
111
|
|
|
112
|
+
def close(self) -> None:
|
|
113
|
+
"""
|
|
114
|
+
Release the synchronous resources of the underlying Document Store.
|
|
115
|
+
"""
|
|
116
|
+
self.document_store.close()
|
|
117
|
+
|
|
118
|
+
async def close_async(self) -> None:
|
|
119
|
+
"""
|
|
120
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
121
|
+
"""
|
|
122
|
+
await self.document_store.close_async()
|
|
123
|
+
|
|
112
124
|
@component.output_types(documents=list[Document])
|
|
113
125
|
def run(
|
|
114
126
|
self,
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
+
from contextlib import suppress
|
|
5
6
|
from typing import Any, Literal, overload
|
|
6
7
|
|
|
7
8
|
from haystack import default_from_dict, default_to_dict, logging
|
|
@@ -223,6 +224,28 @@ class PgvectorDocumentStore:
|
|
|
223
224
|
deserialize_secrets_inplace(data["init_parameters"], ["connection_string"])
|
|
224
225
|
return default_from_dict(cls, data)
|
|
225
226
|
|
|
227
|
+
def close(self) -> None:
|
|
228
|
+
"""
|
|
229
|
+
Release the associated synchronous resources.
|
|
230
|
+
"""
|
|
231
|
+
if self._connection is not None:
|
|
232
|
+
with suppress(Exception):
|
|
233
|
+
self._connection.close()
|
|
234
|
+
self._connection = None
|
|
235
|
+
self._cursor = None
|
|
236
|
+
self._dict_cursor = None
|
|
237
|
+
|
|
238
|
+
async def close_async(self) -> None:
|
|
239
|
+
"""
|
|
240
|
+
Release the associated asynchronous resources.
|
|
241
|
+
"""
|
|
242
|
+
if self._async_connection is not None:
|
|
243
|
+
with suppress(Exception):
|
|
244
|
+
await self._async_connection.close()
|
|
245
|
+
self._async_connection = None
|
|
246
|
+
self._async_cursor = None
|
|
247
|
+
self._async_dict_cursor = None
|
|
248
|
+
|
|
226
249
|
@staticmethod
|
|
227
250
|
def _connection_is_valid(connection: Connection) -> bool:
|
|
228
251
|
"""
|
|
@@ -1909,7 +1932,8 @@ class PgvectorDocumentStore:
|
|
|
1909
1932
|
Builds SQL queries for getting unique metadata field values.
|
|
1910
1933
|
|
|
1911
1934
|
:param normalized_field: The normalized metadata field name.
|
|
1912
|
-
:param search_term: Optional search term to filter
|
|
1935
|
+
:param search_term: Optional search term to filter unique values by a case-insensitive substring match
|
|
1936
|
+
against the metadata field's own value.
|
|
1913
1937
|
:param from_: The offset for pagination (0-based).
|
|
1914
1938
|
:param size: The number of unique values to return.
|
|
1915
1939
|
:returns: A tuple containing (count_query, select_query, params).
|
|
@@ -1925,11 +1949,9 @@ class PgvectorDocumentStore:
|
|
|
1925
1949
|
|
|
1926
1950
|
params: tuple = ()
|
|
1927
1951
|
if search_term:
|
|
1928
|
-
#
|
|
1929
|
-
sql_where += SQL(" AND
|
|
1930
|
-
|
|
1931
|
-
)
|
|
1932
|
-
params = (search_term,)
|
|
1952
|
+
# Case-insensitive substring match against the metadata field's own value.
|
|
1953
|
+
sql_where += SQL(" AND meta->>{} ILIKE %s").format(field_literal)
|
|
1954
|
+
params = (f"%{search_term}%",)
|
|
1933
1955
|
|
|
1934
1956
|
# count query
|
|
1935
1957
|
sql_count = SQL("SELECT COUNT(DISTINCT meta->>{} ) AS total").format(field_literal)
|
|
@@ -1959,14 +1981,14 @@ class PgvectorDocumentStore:
|
|
|
1959
1981
|
return unique_values, total_count
|
|
1960
1982
|
|
|
1961
1983
|
def get_metadata_field_unique_values(
|
|
1962
|
-
self, metadata_field: str, search_term: str | None, from_: int, size: int
|
|
1984
|
+
self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
|
|
1963
1985
|
) -> tuple[list[str], int]:
|
|
1964
1986
|
"""
|
|
1965
1987
|
Returns unique values for a given metadata field, optionally filtered by a search term.
|
|
1966
1988
|
|
|
1967
1989
|
:param metadata_field: The name of the metadata field. Can include or omit the "meta." prefix.
|
|
1968
|
-
:param search_term: Optional search term to filter
|
|
1969
|
-
If None, all
|
|
1990
|
+
:param search_term: Optional search term to filter unique values by a case-insensitive substring
|
|
1991
|
+
match against the metadata field's own value. If None, all values are considered.
|
|
1970
1992
|
:param from_: The offset for pagination (0-based).
|
|
1971
1993
|
:param size: The number of unique values to return.
|
|
1972
1994
|
:returns: A tuple containing:
|
|
@@ -1997,14 +2019,14 @@ class PgvectorDocumentStore:
|
|
|
1997
2019
|
return PgvectorDocumentStore._process_unique_values_result(count_result, records)
|
|
1998
2020
|
|
|
1999
2021
|
async def get_metadata_field_unique_values_async(
|
|
2000
|
-
self, metadata_field: str, search_term: str | None, from_: int, size: int
|
|
2022
|
+
self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
|
|
2001
2023
|
) -> tuple[list[str], int]:
|
|
2002
2024
|
"""
|
|
2003
2025
|
Asynchronously returns unique values for a given metadata field, optionally filtered by a search term.
|
|
2004
2026
|
|
|
2005
2027
|
:param metadata_field: The name of the metadata field. Can include or omit the "meta." prefix.
|
|
2006
|
-
:param search_term: Optional search term to filter
|
|
2007
|
-
If None, all
|
|
2028
|
+
:param search_term: Optional search term to filter unique values by a case-insensitive substring
|
|
2029
|
+
match against the metadata field's own value. If None, all values are considered.
|
|
2008
2030
|
:param from_: The offset for pagination (0-based).
|
|
2009
2031
|
:param size: The number of unique values to return.
|
|
2010
2032
|
:returns: A tuple containing:
|
|
@@ -20,7 +20,7 @@ PYTHON_TYPES_TO_PG_TYPES = {
|
|
|
20
20
|
bool: "boolean",
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
NO_VALUE =
|
|
23
|
+
NO_VALUE = object() # unique sentinel; can never equal a user-supplied filter value
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def _validate_filters(filters: dict[str, Any] | None = None) -> None:
|
|
@@ -48,7 +48,7 @@ def _convert_filters_to_where_clause_and_params(
|
|
|
48
48
|
query, values = _parse_logical_condition(filters)
|
|
49
49
|
|
|
50
50
|
where_clause = SQL(f" {operator} ") + query
|
|
51
|
-
params = tuple(value for value in values if value
|
|
51
|
+
params = tuple(value for value in values if value is not NO_VALUE)
|
|
52
52
|
|
|
53
53
|
return where_clause, params
|
|
54
54
|
|
|
@@ -227,14 +227,14 @@ def _less_than_equal(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
|
227
227
|
|
|
228
228
|
def _not_in(field: Composable, value: Any) -> tuple[Composed, list]:
|
|
229
229
|
if not isinstance(value, list):
|
|
230
|
-
msg = f"{field}'s value must be a list when using 'not in' comparator in
|
|
230
|
+
msg = f"{field}'s value must be a list when using 'not in' comparator in PgVector"
|
|
231
231
|
raise FilterError(msg)
|
|
232
|
-
return SQL("{} IS NULL OR {} != ALL(%s)").format(field, field), [value]
|
|
232
|
+
return SQL("({} IS NULL OR {} != ALL(%s))").format(field, field), [value]
|
|
233
233
|
|
|
234
234
|
|
|
235
235
|
def _in(field: Composable, value: Any) -> tuple[Composed, list]:
|
|
236
236
|
if not isinstance(value, list):
|
|
237
|
-
msg = f"{field}'s value must be a list when using 'in' comparator in
|
|
237
|
+
msg = f"{field}'s value must be a list when using 'in' comparator in PgVector"
|
|
238
238
|
raise FilterError(msg)
|
|
239
239
|
|
|
240
240
|
# see https://www.psycopg.org/psycopg3/docs/basic/adapt.html#lists-adaptation
|
|
@@ -103,6 +103,18 @@ class TestDocumentStore(
|
|
|
103
103
|
document_store._ensure_db_setup()
|
|
104
104
|
assert "Failed to connect to PostgreSQL database" in str(e)
|
|
105
105
|
|
|
106
|
+
def test_close_and_reopen(self, document_store: PgvectorDocumentStore):
|
|
107
|
+
assert document_store.count_documents() == 0
|
|
108
|
+
assert document_store._connection is not None
|
|
109
|
+
|
|
110
|
+
document_store.close()
|
|
111
|
+
assert document_store._connection is None
|
|
112
|
+
assert document_store._cursor is None
|
|
113
|
+
assert document_store._dict_cursor is None
|
|
114
|
+
|
|
115
|
+
assert document_store.count_documents() == 0
|
|
116
|
+
assert document_store._connection is not None
|
|
117
|
+
|
|
106
118
|
|
|
107
119
|
@pytest.mark.usefixtures("patches_for_unit_tests")
|
|
108
120
|
def test_init(monkeypatch):
|
|
@@ -353,6 +365,33 @@ def test_process_count_unique_metadata_result_uses_zero_for_missing_keys():
|
|
|
353
365
|
assert counts == {"category": 5, "language": 0}
|
|
354
366
|
|
|
355
367
|
|
|
368
|
+
def test_close(mock_store):
|
|
369
|
+
mock_connection = Mock(spec=Connection)
|
|
370
|
+
mock_store._connection = mock_connection
|
|
371
|
+
mock_store._cursor = Mock(spec=Cursor)
|
|
372
|
+
mock_store._dict_cursor = Mock(spec=Cursor)
|
|
373
|
+
|
|
374
|
+
mock_store.close()
|
|
375
|
+
|
|
376
|
+
mock_connection.close.assert_called_once()
|
|
377
|
+
assert mock_store._connection is None
|
|
378
|
+
assert mock_store._cursor is None
|
|
379
|
+
assert mock_store._dict_cursor is None
|
|
380
|
+
|
|
381
|
+
mock_store.close()
|
|
382
|
+
mock_connection.close.assert_called_once()
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def test_close_is_exception_safe(mock_store):
|
|
386
|
+
mock_connection = Mock(spec=Connection)
|
|
387
|
+
mock_connection.close.side_effect = RuntimeError("boom")
|
|
388
|
+
mock_store._connection = mock_connection
|
|
389
|
+
|
|
390
|
+
mock_store.close()
|
|
391
|
+
|
|
392
|
+
assert mock_store._connection is None
|
|
393
|
+
|
|
394
|
+
|
|
356
395
|
@pytest.mark.integration
|
|
357
396
|
def test_halfvec_hnsw_write_documents(document_store_w_halfvec_hnsw_index: PgvectorDocumentStore):
|
|
358
397
|
documents = [
|
|
@@ -623,17 +662,30 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
|
|
|
623
662
|
# All three pages should be different
|
|
624
663
|
assert len(set(unique_values_single1 + unique_values_single2 + unique_values_single3)) == 3
|
|
625
664
|
|
|
626
|
-
# Test with search term -
|
|
665
|
+
# Test with search term - case-insensitive substring match against the metadata field's OWN value
|
|
666
|
+
# (not against document content).
|
|
627
667
|
unique_values_filtered, total_filtered = document_store.get_metadata_field_unique_values(
|
|
628
|
-
"meta.
|
|
668
|
+
"meta.language", "Python", 0, 10
|
|
629
669
|
)
|
|
630
|
-
assert set(unique_values_filtered) == {"
|
|
670
|
+
assert set(unique_values_filtered) == {"Python"}
|
|
631
671
|
assert total_filtered == 1
|
|
632
672
|
|
|
633
|
-
#
|
|
634
|
-
|
|
635
|
-
assert set(
|
|
636
|
-
assert
|
|
673
|
+
# Case-insensitivity: lowercase search term should still match "Python"
|
|
674
|
+
unique_values_lower, total_lower = document_store.get_metadata_field_unique_values("meta.language", "python", 0, 10)
|
|
675
|
+
assert set(unique_values_lower) == {"Python"}
|
|
676
|
+
assert total_lower == 1
|
|
677
|
+
|
|
678
|
+
# Substring matching: "Java" is a substring of both "Java" and "JavaScript"
|
|
679
|
+
unique_values_java, total_java = document_store.get_metadata_field_unique_values("meta.language", "Java", 0, 10)
|
|
680
|
+
assert set(unique_values_java) == {"Java", "JavaScript"}
|
|
681
|
+
assert total_java == 2
|
|
682
|
+
|
|
683
|
+
# Substring matching in the middle/end of a value
|
|
684
|
+
unique_values_script, total_script = document_store.get_metadata_field_unique_values(
|
|
685
|
+
"meta.language", "Script", 0, 10
|
|
686
|
+
)
|
|
687
|
+
assert set(unique_values_script) == {"JavaScript"}
|
|
688
|
+
assert total_script == 1
|
|
637
689
|
|
|
638
690
|
# Test pagination with search term
|
|
639
691
|
unique_values_search_page1, total_search = document_store.get_metadata_field_unique_values(
|
|
@@ -650,6 +702,17 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
|
|
|
650
702
|
assert len(unique_values_search_empty) == 0
|
|
651
703
|
assert total_search_empty == 1
|
|
652
704
|
|
|
705
|
+
# Test that search_term matches metadata VALUE, not content
|
|
706
|
+
content_vs_metadata_docs = [
|
|
707
|
+
Document(content="This document mentions Python explicitly", meta={"topic": "cooking"}),
|
|
708
|
+
Document(content="Unrelated content about recipes", meta={"topic": "python-tutorial"}),
|
|
709
|
+
]
|
|
710
|
+
document_store.write_documents(content_vs_metadata_docs)
|
|
711
|
+
|
|
712
|
+
unique_topics, total_topics = document_store.get_metadata_field_unique_values("meta.topic", "python", 0, 10)
|
|
713
|
+
assert set(unique_topics) == {"python-tutorial"}
|
|
714
|
+
assert total_topics == 1
|
|
715
|
+
|
|
653
716
|
# Test with integer values
|
|
654
717
|
int_docs = [
|
|
655
718
|
Document(content="Doc 1", meta={"priority": 1}),
|
|
@@ -662,9 +725,42 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
|
|
|
662
725
|
assert set(unique_priorities) == {"1", "2", "3"}
|
|
663
726
|
assert total_priorities == 3
|
|
664
727
|
|
|
665
|
-
# Test with search term on integer field
|
|
728
|
+
# Test with search term on integer field - substring match against the field's own (stringified)
|
|
729
|
+
# value, e.g. "Doc 1" (content) no longer matches; "1" (the value itself) does.
|
|
666
730
|
unique_priorities_filtered, total_priorities_filtered = document_store.get_metadata_field_unique_values(
|
|
667
|
-
"meta.priority", "
|
|
731
|
+
"meta.priority", "1", 0, 10
|
|
668
732
|
)
|
|
669
733
|
assert set(unique_priorities_filtered) == {"1"}
|
|
670
734
|
assert total_priorities_filtered == 1
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
@pytest.mark.integration
|
|
738
|
+
def test_get_metadata_field_unique_values_no_search_term(document_store: PgvectorDocumentStore):
|
|
739
|
+
docs = [
|
|
740
|
+
Document(content="mentions tsvector and content in the body", meta={"category": "A"}),
|
|
741
|
+
Document(content="another document", meta={"category": "B"}),
|
|
742
|
+
]
|
|
743
|
+
document_store.write_documents(docs)
|
|
744
|
+
|
|
745
|
+
values, total = document_store.get_metadata_field_unique_values("meta.category")
|
|
746
|
+
|
|
747
|
+
assert set(values) == {"A", "B"}
|
|
748
|
+
assert total == 2
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
@pytest.mark.integration
|
|
752
|
+
def test_get_metadata_field_unique_values_search_term_matches_field_value_not_content(
|
|
753
|
+
document_store: PgvectorDocumentStore,
|
|
754
|
+
):
|
|
755
|
+
docs = [
|
|
756
|
+
# "python" appears in the content but NOT in the category value -> must be EXCLUDED
|
|
757
|
+
Document(content="Python programming guide", meta={"category": "Backend"}),
|
|
758
|
+
# "python" appears in the category value but NOT in the content -> must be INCLUDED
|
|
759
|
+
Document(content="General purpose scripting language", meta={"category": "Python-based"}),
|
|
760
|
+
]
|
|
761
|
+
document_store.write_documents(docs)
|
|
762
|
+
|
|
763
|
+
values, total = document_store.get_metadata_field_unique_values("meta.category", search_term="python")
|
|
764
|
+
|
|
765
|
+
assert values == ["Python-based"]
|
|
766
|
+
assert total == 1
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from unittest.mock import Mock, patch
|
|
5
|
+
from unittest.mock import AsyncMock, Mock, patch
|
|
6
6
|
|
|
7
7
|
import psycopg
|
|
8
8
|
import pytest
|
|
@@ -131,6 +131,18 @@ class TestDocumentStoreAsync(
|
|
|
131
131
|
"""
|
|
132
132
|
await document_store.delete_table_async() # if throw error, test fails
|
|
133
133
|
|
|
134
|
+
async def test_close_async_and_reopen(self, document_store: PgvectorDocumentStore):
|
|
135
|
+
assert await document_store.count_documents_async() == 0
|
|
136
|
+
assert document_store._async_connection is not None
|
|
137
|
+
|
|
138
|
+
await document_store.close_async()
|
|
139
|
+
assert document_store._async_connection is None
|
|
140
|
+
assert document_store._async_cursor is None
|
|
141
|
+
assert document_store._async_dict_cursor is None
|
|
142
|
+
|
|
143
|
+
assert await document_store.count_documents_async() == 0
|
|
144
|
+
assert document_store._async_connection is not None
|
|
145
|
+
|
|
134
146
|
|
|
135
147
|
@pytest.mark.integration
|
|
136
148
|
@pytest.mark.asyncio
|
|
@@ -302,3 +314,33 @@ async def test_write_documents_async_rejects_non_document_items(mock_store):
|
|
|
302
314
|
async def test_count_unique_metadata_by_filter_async_rejects_empty_fields(mock_store):
|
|
303
315
|
with pytest.raises(ValueError, match="metadata_fields must be a non-empty list"):
|
|
304
316
|
await mock_store.count_unique_metadata_by_filter_async(filters={}, metadata_fields=[])
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
@pytest.mark.asyncio
|
|
320
|
+
async def test_close_async(mock_store):
|
|
321
|
+
mock_connection = Mock(spec=AsyncConnection)
|
|
322
|
+
mock_connection.close = AsyncMock()
|
|
323
|
+
mock_store._async_connection = mock_connection
|
|
324
|
+
mock_store._async_cursor = Mock(spec=AsyncCursor)
|
|
325
|
+
mock_store._async_dict_cursor = Mock(spec=AsyncCursor)
|
|
326
|
+
|
|
327
|
+
await mock_store.close_async()
|
|
328
|
+
|
|
329
|
+
mock_connection.close.assert_awaited_once()
|
|
330
|
+
assert mock_store._async_connection is None
|
|
331
|
+
assert mock_store._async_cursor is None
|
|
332
|
+
assert mock_store._async_dict_cursor is None
|
|
333
|
+
|
|
334
|
+
await mock_store.close_async()
|
|
335
|
+
mock_connection.close.assert_awaited_once()
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
@pytest.mark.asyncio
|
|
339
|
+
async def test_close_async_is_exception_safe(mock_store):
|
|
340
|
+
mock_connection = Mock(spec=AsyncConnection)
|
|
341
|
+
mock_connection.close = AsyncMock(side_effect=RuntimeError("boom"))
|
|
342
|
+
mock_store._async_connection = mock_connection
|
|
343
|
+
|
|
344
|
+
await mock_store.close_async()
|
|
345
|
+
|
|
346
|
+
assert mock_store._async_connection is None
|
|
@@ -183,6 +183,14 @@ def test_comparison_condition_unknown_operator():
|
|
|
183
183
|
_parse_comparison_condition(condition)
|
|
184
184
|
|
|
185
185
|
|
|
186
|
+
@pytest.mark.parametrize("operator", ["like", "not like"])
|
|
187
|
+
def test_comparison_condition_like_operator_requires_str_value(operator):
|
|
188
|
+
# pgvector-specific: LIKE / NOT LIKE require a string operand.
|
|
189
|
+
condition = {"field": "meta.name", "operator": operator, "value": 123}
|
|
190
|
+
with pytest.raises(FilterError, match="must be a str when using 'LIKE'"):
|
|
191
|
+
_parse_comparison_condition(condition)
|
|
192
|
+
|
|
193
|
+
|
|
186
194
|
def test_logical_condition_missing_operator():
|
|
187
195
|
condition = {"conditions": []}
|
|
188
196
|
with pytest.raises(FilterError):
|
|
@@ -228,13 +236,32 @@ def test_logical_condition_nested():
|
|
|
228
236
|
"("
|
|
229
237
|
"(meta->>'domain' IS DISTINCT FROM %s OR meta->>'chapter' = ANY(%s))"
|
|
230
238
|
" AND "
|
|
231
|
-
"((meta->>'number')::integer >= %s OR meta->>'author' IS NULL OR meta->>'author' != ALL(%s))"
|
|
239
|
+
"((meta->>'number')::integer >= %s OR (meta->>'author' IS NULL OR meta->>'author' != ALL(%s)))"
|
|
232
240
|
")"
|
|
233
241
|
)
|
|
234
242
|
assert _render(query) == expected_sql
|
|
235
243
|
assert values == ["science", [["intro", "conclusion"]], 90, [["John", "Jane"]]]
|
|
236
244
|
|
|
237
245
|
|
|
246
|
+
def test_logical_condition_not_in_is_parenthesized_when_and_combined():
|
|
247
|
+
# `not in` renders an internal `OR` (`IS NULL OR != ALL`). Without surrounding
|
|
248
|
+
# parentheses that `OR` escapes an enclosing `AND` (AND binds tighter than OR in
|
|
249
|
+
# SQL), so an AND-combined `not in` would wrongly match documents.
|
|
250
|
+
condition = {
|
|
251
|
+
"operator": "AND",
|
|
252
|
+
"conditions": [
|
|
253
|
+
{"field": "meta.number", "operator": "==", "value": 5},
|
|
254
|
+
{"field": "meta.author", "operator": "not in", "value": ["John", "Jane"]},
|
|
255
|
+
],
|
|
256
|
+
}
|
|
257
|
+
query, values = _parse_logical_condition(condition)
|
|
258
|
+
assert isinstance(query, Composed)
|
|
259
|
+
|
|
260
|
+
expected_sql = "((meta->>'number')::integer = %s AND (meta->>'author' IS NULL OR meta->>'author' != ALL(%s)))"
|
|
261
|
+
assert _render(query) == expected_sql
|
|
262
|
+
assert values == [5, [["John", "Jane"]]]
|
|
263
|
+
|
|
264
|
+
|
|
238
265
|
def test_convert_filters_to_where_clause_and_params():
|
|
239
266
|
filters = {
|
|
240
267
|
"operator": "AND",
|
|
@@ -265,6 +292,18 @@ def test_convert_filters_to_where_clause_and_params_handle_null():
|
|
|
265
292
|
assert params == ("intro",)
|
|
266
293
|
|
|
267
294
|
|
|
295
|
+
def test_convert_filters_to_where_clause_and_params_value_equal_to_no_value_sentinel():
|
|
296
|
+
"""A legitimate equality filter whose value is the literal string the internal
|
|
297
|
+
NO_VALUE sentinel used to be ("no_value") must not be dropped from params --
|
|
298
|
+
the query still has a placeholder for it."""
|
|
299
|
+
filters = {"field": "meta.tag", "operator": "==", "value": "no_value"}
|
|
300
|
+
where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
301
|
+
|
|
302
|
+
expected_sql = " WHERE meta->>'tag' = %s"
|
|
303
|
+
assert _render(where_clause) == expected_sql
|
|
304
|
+
assert params == ("no_value",)
|
|
305
|
+
|
|
306
|
+
|
|
268
307
|
def test_validate_filters():
|
|
269
308
|
filters = {"field": "meta.number", "operator": "==", "value": 100}
|
|
270
309
|
_validate_filters(filters)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from unittest.mock import Mock
|
|
4
|
+
from unittest.mock import AsyncMock, Mock
|
|
5
5
|
|
|
6
6
|
import pytest
|
|
7
7
|
from haystack.dataclasses import Document
|
|
@@ -126,6 +126,26 @@ class TestEmbeddingRetriever:
|
|
|
126
126
|
assert retriever.filter_policy == FilterPolicy.REPLACE
|
|
127
127
|
assert retriever.vector_function == "l2_distance"
|
|
128
128
|
|
|
129
|
+
def test_close(self):
|
|
130
|
+
mock_store = Mock(spec=PgvectorDocumentStore)
|
|
131
|
+
retriever = PgvectorEmbeddingRetriever(document_store=mock_store, vector_function="l2_distance")
|
|
132
|
+
|
|
133
|
+
retriever.close()
|
|
134
|
+
|
|
135
|
+
mock_store.close.assert_called_once()
|
|
136
|
+
assert retriever.document_store is mock_store
|
|
137
|
+
|
|
138
|
+
@pytest.mark.asyncio
|
|
139
|
+
async def test_close_async(self):
|
|
140
|
+
mock_store = Mock(spec=PgvectorDocumentStore)
|
|
141
|
+
mock_store.close_async = AsyncMock()
|
|
142
|
+
retriever = PgvectorEmbeddingRetriever(document_store=mock_store, vector_function="l2_distance")
|
|
143
|
+
|
|
144
|
+
await retriever.close_async()
|
|
145
|
+
|
|
146
|
+
mock_store.close_async.assert_awaited_once()
|
|
147
|
+
assert retriever.document_store is mock_store
|
|
148
|
+
|
|
129
149
|
def test_run(self):
|
|
130
150
|
mock_store = Mock(spec=PgvectorDocumentStore)
|
|
131
151
|
doc = Document(content="Test doc", embedding=[0.1, 0.2])
|
|
@@ -315,6 +335,26 @@ class TestKeywordRetriever:
|
|
|
315
335
|
assert retriever.filter_policy == FilterPolicy.REPLACE # defaults to REPLACE
|
|
316
336
|
assert retriever.top_k == 5
|
|
317
337
|
|
|
338
|
+
def test_close(self):
|
|
339
|
+
mock_store = Mock(spec=PgvectorDocumentStore)
|
|
340
|
+
retriever = PgvectorKeywordRetriever(document_store=mock_store)
|
|
341
|
+
|
|
342
|
+
retriever.close()
|
|
343
|
+
|
|
344
|
+
mock_store.close.assert_called_once()
|
|
345
|
+
assert retriever.document_store is mock_store
|
|
346
|
+
|
|
347
|
+
@pytest.mark.asyncio
|
|
348
|
+
async def test_close_async(self):
|
|
349
|
+
mock_store = Mock(spec=PgvectorDocumentStore)
|
|
350
|
+
mock_store.close_async = AsyncMock()
|
|
351
|
+
retriever = PgvectorKeywordRetriever(document_store=mock_store)
|
|
352
|
+
|
|
353
|
+
await retriever.close_async()
|
|
354
|
+
|
|
355
|
+
mock_store.close_async.assert_awaited_once()
|
|
356
|
+
assert retriever.document_store is mock_store
|
|
357
|
+
|
|
318
358
|
def test_run(self):
|
|
319
359
|
mock_store = Mock(spec=PgvectorDocumentStore)
|
|
320
360
|
doc = Document(content="Test doc", embedding=[0.1, 0.2])
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|