pgvector-haystack 6.4.0__tar.gz → 6.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/CHANGELOG.md +24 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/PKG-INFO +1 -1
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/pyproject.toml +1 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/document_store.py +22 -20
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/filters.py +74 -4
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_document_store.py +69 -11
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_document_store_async.py +16 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_filters.py +196 -1
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/.gitignore +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/LICENSE.txt +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/README.md +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/examples/embedding_retrieval.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/examples/hybrid_retrieval.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/pydoc/config_docusaurus.yml +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/components/retrievers/pgvector/__init__.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/__init__.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/converters.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/__init__.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/conftest.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_converters.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_retrieval.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_retrieval_async.py +0 -0
- {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_retrievers.py +0 -0
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/pgvector-v6.4.1] - 2026-07-23
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- *(pgvector)* Parenthesize `not in` clause so it does not escape an AND filter (#3586)
|
|
8
|
+
- `PGVectorDocumentStore` match `search_term` against metadata field value, not content (#3648)
|
|
9
|
+
- Fix (pgvector): fix error messages to say pgvector instead of pinecone (#3654)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
## [integrations/pgvector-v6.4.0] - 2026-07-23
|
|
13
|
+
|
|
14
|
+
### 🚀 Features
|
|
15
|
+
|
|
16
|
+
- Pgvector - add closing methods (#3650)
|
|
17
|
+
|
|
18
|
+
### 🐛 Bug Fixes
|
|
19
|
+
|
|
20
|
+
- *(pgvector)* NO_VALUE sentinel collides with the literal string 'no_value' (#3593)
|
|
21
|
+
|
|
22
|
+
### 📚 Documentation
|
|
23
|
+
|
|
24
|
+
- Replace old haystack core imports with haystack_integrations paths (#3545)
|
|
25
|
+
|
|
26
|
+
|
|
3
27
|
## [integrations/pgvector-v6.3.1] - 2026-06-01
|
|
4
28
|
|
|
5
29
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.5.0
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
@@ -1932,7 +1932,8 @@ class PgvectorDocumentStore:
|
|
|
1932
1932
|
Builds SQL queries for getting unique metadata field values.
|
|
1933
1933
|
|
|
1934
1934
|
:param normalized_field: The normalized metadata field name.
|
|
1935
|
-
:param search_term: Optional search term to filter
|
|
1935
|
+
:param search_term: Optional search term to filter unique values by a case-insensitive substring match
|
|
1936
|
+
against the metadata field's own value.
|
|
1936
1937
|
:param from_: The offset for pagination (0-based).
|
|
1937
1938
|
:param size: The number of unique values to return.
|
|
1938
1939
|
:returns: A tuple containing (count_query, select_query, params).
|
|
@@ -1940,7 +1941,10 @@ class PgvectorDocumentStore:
|
|
|
1940
1941
|
field_literal = SQLLiteral(normalized_field)
|
|
1941
1942
|
|
|
1942
1943
|
# base query components
|
|
1943
|
-
|
|
1944
|
+
# Use the `->` (jsonb) operator rather than `->>` (text) so DISTINCT/ORDER BY operate on the
|
|
1945
|
+
# value's original JSON type (numbers sort numerically, not lexicographically) and psycopg
|
|
1946
|
+
# decodes the result to its native Python type instead of a string.
|
|
1947
|
+
sql_select = SQL("SELECT DISTINCT meta->{} AS value").format(field_literal)
|
|
1944
1948
|
sql_from = SQL(" FROM {schema_name}.{table_name}").format(
|
|
1945
1949
|
schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
|
|
1946
1950
|
)
|
|
@@ -1948,14 +1952,12 @@ class PgvectorDocumentStore:
|
|
|
1948
1952
|
|
|
1949
1953
|
params: tuple = ()
|
|
1950
1954
|
if search_term:
|
|
1951
|
-
#
|
|
1952
|
-
sql_where += SQL(" AND
|
|
1953
|
-
|
|
1954
|
-
)
|
|
1955
|
-
params = (search_term,)
|
|
1955
|
+
# Case-insensitive substring match against the metadata field's own (text) value.
|
|
1956
|
+
sql_where += SQL(" AND meta->>{} ILIKE %s").format(field_literal)
|
|
1957
|
+
params = (f"%{search_term}%",)
|
|
1956
1958
|
|
|
1957
1959
|
# count query
|
|
1958
|
-
sql_count = SQL("SELECT COUNT(DISTINCT meta
|
|
1960
|
+
sql_count = SQL("SELECT COUNT(DISTINCT meta->{} ) AS total").format(field_literal)
|
|
1959
1961
|
sql_count += sql_from + sql_where
|
|
1960
1962
|
|
|
1961
1963
|
# paginated select query
|
|
@@ -1969,7 +1971,7 @@ class PgvectorDocumentStore:
|
|
|
1969
1971
|
@staticmethod
|
|
1970
1972
|
def _process_unique_values_result(
|
|
1971
1973
|
count_result: dict[str, Any] | None, records: list[dict[str, Any]]
|
|
1972
|
-
) -> tuple[list[
|
|
1974
|
+
) -> tuple[list[Any], int]:
|
|
1973
1975
|
"""
|
|
1974
1976
|
Processes the results from unique values queries.
|
|
1975
1977
|
|
|
@@ -1978,22 +1980,22 @@ class PgvectorDocumentStore:
|
|
|
1978
1980
|
:returns: A tuple containing (unique_values, total_count).
|
|
1979
1981
|
"""
|
|
1980
1982
|
total_count = count_result.get("total", 0) if count_result else 0
|
|
1981
|
-
unique_values = [
|
|
1983
|
+
unique_values = [record.get("value") for record in records if record.get("value") is not None]
|
|
1982
1984
|
return unique_values, total_count
|
|
1983
1985
|
|
|
1984
1986
|
def get_metadata_field_unique_values(
|
|
1985
|
-
self, metadata_field: str, search_term: str | None, from_: int, size: int
|
|
1986
|
-
) -> tuple[list[
|
|
1987
|
+
self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
|
|
1988
|
+
) -> tuple[list[Any], int]:
|
|
1987
1989
|
"""
|
|
1988
1990
|
Returns unique values for a given metadata field, optionally filtered by a search term.
|
|
1989
1991
|
|
|
1990
1992
|
:param metadata_field: The name of the metadata field. Can include or omit the "meta." prefix.
|
|
1991
|
-
:param search_term: Optional search term to filter
|
|
1992
|
-
If None, all
|
|
1993
|
+
:param search_term: Optional search term to filter unique values by a case-insensitive substring
|
|
1994
|
+
match against the metadata field's own value. If None, all values are considered.
|
|
1993
1995
|
:param from_: The offset for pagination (0-based).
|
|
1994
1996
|
:param size: The number of unique values to return.
|
|
1995
1997
|
:returns: A tuple containing:
|
|
1996
|
-
- A list of unique values
|
|
1998
|
+
- A list of unique values in their original type
|
|
1997
1999
|
- The total count of unique values
|
|
1998
2000
|
"""
|
|
1999
2001
|
normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
|
|
@@ -2020,18 +2022,18 @@ class PgvectorDocumentStore:
|
|
|
2020
2022
|
return PgvectorDocumentStore._process_unique_values_result(count_result, records)
|
|
2021
2023
|
|
|
2022
2024
|
async def get_metadata_field_unique_values_async(
|
|
2023
|
-
self, metadata_field: str, search_term: str | None, from_: int, size: int
|
|
2024
|
-
) -> tuple[list[
|
|
2025
|
+
self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
|
|
2026
|
+
) -> tuple[list[Any], int]:
|
|
2025
2027
|
"""
|
|
2026
2028
|
Asynchronously returns unique values for a given metadata field, optionally filtered by a search term.
|
|
2027
2029
|
|
|
2028
2030
|
:param metadata_field: The name of the metadata field. Can include or omit the "meta." prefix.
|
|
2029
|
-
:param search_term: Optional search term to filter
|
|
2030
|
-
If None, all
|
|
2031
|
+
:param search_term: Optional search term to filter unique values by a case-insensitive substring
|
|
2032
|
+
match against the metadata field's own value. If None, all values are considered.
|
|
2031
2033
|
:param from_: The offset for pagination (0-based).
|
|
2032
2034
|
:param size: The number of unique values to return.
|
|
2033
2035
|
:returns: A tuple containing:
|
|
2034
|
-
- A list of unique values
|
|
2036
|
+
- A list of unique values in their original type
|
|
2035
2037
|
- The total count of unique values
|
|
2036
2038
|
"""
|
|
2037
2039
|
normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
|
|
@@ -22,6 +22,11 @@ PYTHON_TYPES_TO_PG_TYPES = {
|
|
|
22
22
|
|
|
23
23
|
NO_VALUE = object() # unique sentinel; can never equal a user-supplied filter value
|
|
24
24
|
|
|
25
|
+
# Operators that match against a JSONB array stored in the meta field.
|
|
26
|
+
# They need the JSONB value (via the `->` accessor) as their left operand,
|
|
27
|
+
# unlike the scalar operators which access meta as text (via `->>`).
|
|
28
|
+
ARRAY_OPERATORS = ("array_contains", "array_overlaps")
|
|
29
|
+
|
|
25
30
|
|
|
26
31
|
def _validate_filters(filters: dict[str, Any] | None = None) -> None:
|
|
27
32
|
"""
|
|
@@ -106,7 +111,18 @@ def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[Composed, li
|
|
|
106
111
|
value: Any = condition["value"]
|
|
107
112
|
|
|
108
113
|
if field.startswith("meta."):
|
|
109
|
-
|
|
114
|
+
# array operators need the JSONB value (->), the other operators access meta as text (->>)
|
|
115
|
+
sql_field: Composable = (
|
|
116
|
+
_treat_meta_field_as_jsonb(field) if operator in ARRAY_OPERATORS else _treat_meta_field(field, value)
|
|
117
|
+
)
|
|
118
|
+
elif operator in ARRAY_OPERATORS:
|
|
119
|
+
# top-level columns are never JSONB arrays, so these operators would render SQL that the
|
|
120
|
+
# database rejects with an opaque UndefinedFunction error. Fail with a FilterError instead.
|
|
121
|
+
msg = (
|
|
122
|
+
f"Operator '{operator}' is only supported on meta fields, but got field '{field}'. "
|
|
123
|
+
f"Prefix the field name with 'meta.' to filter on an array stored in a document's metadata."
|
|
124
|
+
)
|
|
125
|
+
raise FilterError(msg)
|
|
110
126
|
else:
|
|
111
127
|
sql_field = Identifier(field)
|
|
112
128
|
|
|
@@ -149,6 +165,24 @@ def _treat_meta_field(field: str, value: Any) -> Composed:
|
|
|
149
165
|
return composed
|
|
150
166
|
|
|
151
167
|
|
|
168
|
+
def _treat_meta_field_as_jsonb(field: str) -> Composed:
|
|
169
|
+
"""
|
|
170
|
+
Internal method that returns a psycopg Composed object accessing a meta key as a JSONB value.
|
|
171
|
+
|
|
172
|
+
Unlike `_treat_meta_field`, this uses the `->` operator (which returns a JSONB value) instead of
|
|
173
|
+
`->>` (which returns text) and applies no type cast. The array operators need a JSONB left operand
|
|
174
|
+
for their JSONB operators (`@>`, `?|`) to work.
|
|
175
|
+
|
|
176
|
+
Uses psycopg.sql.Literal to embed the field name, preventing SQL injection via metadata field names.
|
|
177
|
+
|
|
178
|
+
Example:
|
|
179
|
+
>>> _treat_meta_field_as_jsonb(field="meta.tags")
|
|
180
|
+
Composed([SQL('meta->'), Literal('tags')])
|
|
181
|
+
"""
|
|
182
|
+
field_name = field.split(".", 1)[-1]
|
|
183
|
+
return SQL("meta->") + SQLLiteral(field_name)
|
|
184
|
+
|
|
185
|
+
|
|
152
186
|
def _equal(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
153
187
|
if value is None:
|
|
154
188
|
return SQL("{} IS NULL").format(field), NO_VALUE
|
|
@@ -227,14 +261,14 @@ def _less_than_equal(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
|
227
261
|
|
|
228
262
|
def _not_in(field: Composable, value: Any) -> tuple[Composed, list]:
|
|
229
263
|
if not isinstance(value, list):
|
|
230
|
-
msg = f"{field}'s value must be a list when using 'not in' comparator in
|
|
264
|
+
msg = f"{field}'s value must be a list when using 'not in' comparator in PgVector"
|
|
231
265
|
raise FilterError(msg)
|
|
232
|
-
return SQL("{} IS NULL OR {} != ALL(%s)").format(field, field), [value]
|
|
266
|
+
return SQL("({} IS NULL OR {} != ALL(%s))").format(field, field), [value]
|
|
233
267
|
|
|
234
268
|
|
|
235
269
|
def _in(field: Composable, value: Any) -> tuple[Composed, list]:
|
|
236
270
|
if not isinstance(value, list):
|
|
237
|
-
msg = f"{field}'s value must be a list when using 'in' comparator in
|
|
271
|
+
msg = f"{field}'s value must be a list when using 'in' comparator in PgVector"
|
|
238
272
|
raise FilterError(msg)
|
|
239
273
|
|
|
240
274
|
# see https://www.psycopg.org/psycopg3/docs/basic/adapt.html#lists-adaptation
|
|
@@ -255,6 +289,40 @@ def _not_like(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
|
255
289
|
return SQL("{} NOT LIKE %s").format(field), value
|
|
256
290
|
|
|
257
291
|
|
|
292
|
+
def _validate_array_operator_value(field: Composable, value: Any, operator: str) -> None:
|
|
293
|
+
if not isinstance(value, list):
|
|
294
|
+
msg = f"{field}'s value must be a list when using '{operator}' comparator in PgVector"
|
|
295
|
+
raise FilterError(msg)
|
|
296
|
+
# an empty list has no useful match semantics: `@> '[]'` matches every array and `?| '{}'` matches
|
|
297
|
+
# nothing, so the same empty filter would mean "everything" for one operator and "nothing" for the
|
|
298
|
+
# other. Callers building the list dynamically should omit the condition instead.
|
|
299
|
+
if not value:
|
|
300
|
+
msg = f"{field}'s value must be a non-empty list when using '{operator}' comparator in PgVector"
|
|
301
|
+
raise FilterError(msg)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _array_contains(field: Composable, value: Any) -> tuple[Composed, Jsonb]:
|
|
305
|
+
_validate_array_operator_value(field, value, "array_contains")
|
|
306
|
+
# `jsonb @> jsonb` is true when the left array contains every element of the right array,
|
|
307
|
+
# so the meta array must contain all of the given values. Containment compares elements by value,
|
|
308
|
+
# so a non-array meta value (a bare string, an object) never matches a list of values.
|
|
309
|
+
return SQL("{} @> %s").format(field), Jsonb(value)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _array_overlaps(field: Composable, value: Any) -> tuple[Composed, list]:
|
|
313
|
+
_validate_array_operator_value(field, value, "array_overlaps")
|
|
314
|
+
# `jsonb ?| text[]` only matches string elements, so we reject non-string values to fail fast
|
|
315
|
+
# instead of silently returning no results.
|
|
316
|
+
if not all(isinstance(item, str) for item in value):
|
|
317
|
+
msg = f"{field}'s value must be a list of strings when using 'array_overlaps' comparator in PgVector"
|
|
318
|
+
raise FilterError(msg)
|
|
319
|
+
# `?|` is true when a given string exists as a top-level key OR array element, so on its own it
|
|
320
|
+
# also matches objects (by key) and bare strings. Restricting it to arrays with jsonb_typeof keeps
|
|
321
|
+
# it in agreement with array_contains. The parentheses stop this AND from being split by an
|
|
322
|
+
# enclosing OR (see _not_in for the same reason).
|
|
323
|
+
return SQL("(jsonb_typeof({}) = 'array' AND {} ?| %s)").format(field, field), [value]
|
|
324
|
+
|
|
325
|
+
|
|
258
326
|
COMPARISON_OPERATORS = {
|
|
259
327
|
"==": _equal,
|
|
260
328
|
"!=": _not_equal,
|
|
@@ -266,4 +334,6 @@ COMPARISON_OPERATORS = {
|
|
|
266
334
|
"not in": _not_in,
|
|
267
335
|
"like": _like,
|
|
268
336
|
"not like": _not_like,
|
|
337
|
+
"array_contains": _array_contains,
|
|
338
|
+
"array_overlaps": _array_overlaps,
|
|
269
339
|
}
|
|
@@ -662,17 +662,30 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
|
|
|
662
662
|
# All three pages should be different
|
|
663
663
|
assert len(set(unique_values_single1 + unique_values_single2 + unique_values_single3)) == 3
|
|
664
664
|
|
|
665
|
-
# Test with search term -
|
|
665
|
+
# Test with search term - case-insensitive substring match against the metadata field's OWN value
|
|
666
|
+
# (not against document content).
|
|
666
667
|
unique_values_filtered, total_filtered = document_store.get_metadata_field_unique_values(
|
|
667
|
-
"meta.
|
|
668
|
+
"meta.language", "Python", 0, 10
|
|
668
669
|
)
|
|
669
|
-
assert set(unique_values_filtered) == {"
|
|
670
|
+
assert set(unique_values_filtered) == {"Python"}
|
|
670
671
|
assert total_filtered == 1
|
|
671
672
|
|
|
672
|
-
#
|
|
673
|
-
|
|
674
|
-
assert set(
|
|
675
|
-
assert
|
|
673
|
+
# Case-insensitivity: lowercase search term should still match "Python"
|
|
674
|
+
unique_values_lower, total_lower = document_store.get_metadata_field_unique_values("meta.language", "python", 0, 10)
|
|
675
|
+
assert set(unique_values_lower) == {"Python"}
|
|
676
|
+
assert total_lower == 1
|
|
677
|
+
|
|
678
|
+
# Substring matching: "Java" is a substring of both "Java" and "JavaScript"
|
|
679
|
+
unique_values_java, total_java = document_store.get_metadata_field_unique_values("meta.language", "Java", 0, 10)
|
|
680
|
+
assert set(unique_values_java) == {"Java", "JavaScript"}
|
|
681
|
+
assert total_java == 2
|
|
682
|
+
|
|
683
|
+
# Substring matching in the middle/end of a value
|
|
684
|
+
unique_values_script, total_script = document_store.get_metadata_field_unique_values(
|
|
685
|
+
"meta.language", "Script", 0, 10
|
|
686
|
+
)
|
|
687
|
+
assert set(unique_values_script) == {"JavaScript"}
|
|
688
|
+
assert total_script == 1
|
|
676
689
|
|
|
677
690
|
# Test pagination with search term
|
|
678
691
|
unique_values_search_page1, total_search = document_store.get_metadata_field_unique_values(
|
|
@@ -689,6 +702,17 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
|
|
|
689
702
|
assert len(unique_values_search_empty) == 0
|
|
690
703
|
assert total_search_empty == 1
|
|
691
704
|
|
|
705
|
+
# Test that search_term matches metadata VALUE, not content
|
|
706
|
+
content_vs_metadata_docs = [
|
|
707
|
+
Document(content="This document mentions Python explicitly", meta={"topic": "cooking"}),
|
|
708
|
+
Document(content="Unrelated content about recipes", meta={"topic": "python-tutorial"}),
|
|
709
|
+
]
|
|
710
|
+
document_store.write_documents(content_vs_metadata_docs)
|
|
711
|
+
|
|
712
|
+
unique_topics, total_topics = document_store.get_metadata_field_unique_values("meta.topic", "python", 0, 10)
|
|
713
|
+
assert set(unique_topics) == {"python-tutorial"}
|
|
714
|
+
assert total_topics == 1
|
|
715
|
+
|
|
692
716
|
# Test with integer values
|
|
693
717
|
int_docs = [
|
|
694
718
|
Document(content="Doc 1", meta={"priority": 1}),
|
|
@@ -698,12 +722,46 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
|
|
|
698
722
|
]
|
|
699
723
|
document_store.write_documents(int_docs)
|
|
700
724
|
unique_priorities, total_priorities = document_store.get_metadata_field_unique_values("meta.priority", None, 0, 10)
|
|
701
|
-
assert set(unique_priorities) == {
|
|
725
|
+
assert set(unique_priorities) == {1, 2, 3}
|
|
702
726
|
assert total_priorities == 3
|
|
703
727
|
|
|
704
|
-
# Test with search term on integer field
|
|
728
|
+
# Test with search term on integer field - substring match against the field's own (stringified)
|
|
729
|
+
# value, e.g. "Doc 1" (content) no longer matches; "1" (the value itself) does. The returned
|
|
730
|
+
# values themselves keep their original type (int here), only the match is textual.
|
|
705
731
|
unique_priorities_filtered, total_priorities_filtered = document_store.get_metadata_field_unique_values(
|
|
706
|
-
"meta.priority", "
|
|
732
|
+
"meta.priority", "1", 0, 10
|
|
707
733
|
)
|
|
708
|
-
assert set(unique_priorities_filtered) == {
|
|
734
|
+
assert set(unique_priorities_filtered) == {1}
|
|
709
735
|
assert total_priorities_filtered == 1
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
@pytest.mark.integration
|
|
739
|
+
def test_get_metadata_field_unique_values_no_search_term(document_store: PgvectorDocumentStore):
|
|
740
|
+
docs = [
|
|
741
|
+
Document(content="mentions tsvector and content in the body", meta={"category": "A"}),
|
|
742
|
+
Document(content="another document", meta={"category": "B"}),
|
|
743
|
+
]
|
|
744
|
+
document_store.write_documents(docs)
|
|
745
|
+
|
|
746
|
+
values, total = document_store.get_metadata_field_unique_values("meta.category")
|
|
747
|
+
|
|
748
|
+
assert set(values) == {"A", "B"}
|
|
749
|
+
assert total == 2
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
@pytest.mark.integration
|
|
753
|
+
def test_get_metadata_field_unique_values_search_term_matches_field_value_not_content(
|
|
754
|
+
document_store: PgvectorDocumentStore,
|
|
755
|
+
):
|
|
756
|
+
docs = [
|
|
757
|
+
# "python" appears in the content but NOT in the category value -> must be EXCLUDED
|
|
758
|
+
Document(content="Python programming guide", meta={"category": "Backend"}),
|
|
759
|
+
# "python" appears in the category value but NOT in the content -> must be INCLUDED
|
|
760
|
+
Document(content="General purpose scripting language", meta={"category": "Python-based"}),
|
|
761
|
+
]
|
|
762
|
+
document_store.write_documents(docs)
|
|
763
|
+
|
|
764
|
+
values, total = document_store.get_metadata_field_unique_values("meta.category", search_term="python")
|
|
765
|
+
|
|
766
|
+
assert values == ["Python-based"]
|
|
767
|
+
assert total == 1
|
|
@@ -124,6 +124,22 @@ class TestDocumentStoreAsync(
|
|
|
124
124
|
filters={"field": "meta.category", "operator": "==", "value": "A"}, meta={}
|
|
125
125
|
)
|
|
126
126
|
|
|
127
|
+
async def test_get_metadata_field_unique_values_async_preserves_non_string_types(
|
|
128
|
+
self, document_store: PgvectorDocumentStore
|
|
129
|
+
):
|
|
130
|
+
"""Non-string metadata values (e.g. ints) are returned in their original type, not stringified."""
|
|
131
|
+
docs = [
|
|
132
|
+
Document(content="Doc 1", meta={"priority": 1}),
|
|
133
|
+
Document(content="Doc 2", meta={"priority": 2}),
|
|
134
|
+
Document(content="Doc 3", meta={"priority": 1}),
|
|
135
|
+
]
|
|
136
|
+
await document_store.write_documents_async(docs)
|
|
137
|
+
|
|
138
|
+
values, total = await document_store.get_metadata_field_unique_values_async("meta.priority")
|
|
139
|
+
|
|
140
|
+
assert set(values) == {1, 2}
|
|
141
|
+
assert total == 2
|
|
142
|
+
|
|
127
143
|
async def test_delete_table_async_first_call(self, document_store: PgvectorDocumentStore):
|
|
128
144
|
"""
|
|
129
145
|
Test that delete_table_async can be executed as the initial operation on the Document Store
|
|
@@ -10,6 +10,7 @@ from haystack.testing.document_store import FilterDocumentsTest
|
|
|
10
10
|
from psycopg.adapt import Transformer
|
|
11
11
|
from psycopg.sql import SQL, Composed
|
|
12
12
|
from psycopg.sql import Literal as SQLLiteral
|
|
13
|
+
from psycopg.types.json import Jsonb
|
|
13
14
|
|
|
14
15
|
from haystack_integrations.document_stores.pgvector.filters import (
|
|
15
16
|
FilterError,
|
|
@@ -17,6 +18,7 @@ from haystack_integrations.document_stores.pgvector.filters import (
|
|
|
17
18
|
_parse_comparison_condition,
|
|
18
19
|
_parse_logical_condition,
|
|
19
20
|
_treat_meta_field,
|
|
21
|
+
_treat_meta_field_as_jsonb,
|
|
20
22
|
_validate_filters,
|
|
21
23
|
)
|
|
22
24
|
|
|
@@ -141,6 +143,86 @@ class TestFilters(FilterDocumentsTest):
|
|
|
141
143
|
],
|
|
142
144
|
)
|
|
143
145
|
|
|
146
|
+
@staticmethod
|
|
147
|
+
def _docs_with_array_meta():
|
|
148
|
+
return [
|
|
149
|
+
Document(content="doc with tag1 and tag2", meta={"tags": ["tag1", "tag2"]}),
|
|
150
|
+
Document(content="doc with tag1 and tag3", meta={"tags": ["tag1", "tag3"]}),
|
|
151
|
+
Document(content="doc with tag4", meta={"tags": ["tag4"]}),
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
def test_array_contains_operator(self, document_store):
|
|
155
|
+
docs = self._docs_with_array_meta()
|
|
156
|
+
document_store.write_documents(docs)
|
|
157
|
+
|
|
158
|
+
# array_contains requires ALL listed values to be present in the meta array
|
|
159
|
+
result = document_store.filter_documents(
|
|
160
|
+
filters={"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
|
|
161
|
+
)
|
|
162
|
+
assert {doc.content for doc in result} == {"doc with tag1 and tag2"}
|
|
163
|
+
|
|
164
|
+
def test_array_contains_operator_single_value(self, document_store):
|
|
165
|
+
docs = self._docs_with_array_meta()
|
|
166
|
+
document_store.write_documents(docs)
|
|
167
|
+
|
|
168
|
+
result = document_store.filter_documents(
|
|
169
|
+
filters={"field": "meta.tags", "operator": "array_contains", "value": ["tag1"]}
|
|
170
|
+
)
|
|
171
|
+
assert {doc.content for doc in result} == {"doc with tag1 and tag2", "doc with tag1 and tag3"}
|
|
172
|
+
|
|
173
|
+
def test_array_overlaps_operator(self, document_store):
|
|
174
|
+
docs = self._docs_with_array_meta()
|
|
175
|
+
document_store.write_documents(docs)
|
|
176
|
+
|
|
177
|
+
# array_overlaps requires AT LEAST ONE listed value to be present in the meta array
|
|
178
|
+
result = document_store.filter_documents(
|
|
179
|
+
filters={"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag3"]}
|
|
180
|
+
)
|
|
181
|
+
assert {doc.content for doc in result} == {"doc with tag1 and tag2", "doc with tag1 and tag3"}
|
|
182
|
+
|
|
183
|
+
def test_array_overlaps_operator_no_match(self, document_store):
|
|
184
|
+
docs = self._docs_with_array_meta()
|
|
185
|
+
document_store.write_documents(docs)
|
|
186
|
+
|
|
187
|
+
result = document_store.filter_documents(
|
|
188
|
+
filters={"field": "meta.tags", "operator": "array_overlaps", "value": ["nonexistent"]}
|
|
189
|
+
)
|
|
190
|
+
assert result == []
|
|
191
|
+
|
|
192
|
+
def test_array_operators_combined(self, document_store):
|
|
193
|
+
docs = self._docs_with_array_meta()
|
|
194
|
+
document_store.write_documents(docs)
|
|
195
|
+
|
|
196
|
+
result = document_store.filter_documents(
|
|
197
|
+
filters={
|
|
198
|
+
"operator": "AND",
|
|
199
|
+
"conditions": [
|
|
200
|
+
{"field": "meta.tags", "operator": "array_contains", "value": ["tag1"]},
|
|
201
|
+
{"field": "meta.tags", "operator": "array_overlaps", "value": ["tag3"]},
|
|
202
|
+
],
|
|
203
|
+
}
|
|
204
|
+
)
|
|
205
|
+
assert {doc.content for doc in result} == {"doc with tag1 and tag3"}
|
|
206
|
+
|
|
207
|
+
@pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
|
|
208
|
+
def test_array_operators_ignore_non_array_meta(self, document_store, operator):
|
|
209
|
+
# meta shapes vary across sources, so the same key can hold an array in one document and a
|
|
210
|
+
# bare string or an object in another. `?|` matches a top-level key or array element, so
|
|
211
|
+
# without the jsonb_typeof guard the object and the string below would both match.
|
|
212
|
+
document_store.write_documents(
|
|
213
|
+
[
|
|
214
|
+
Document(content="array meta", meta={"tags": ["tag1", "tag2"]}),
|
|
215
|
+
Document(content="string meta", meta={"tags": "tag1"}),
|
|
216
|
+
Document(content="object meta", meta={"tags": {"tag1": "value"}}),
|
|
217
|
+
Document(content="missing key", meta={"other": "tag1"}),
|
|
218
|
+
]
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
result = document_store.filter_documents(
|
|
222
|
+
filters={"field": "meta.tags", "operator": operator, "value": ["tag1"]}
|
|
223
|
+
)
|
|
224
|
+
assert {doc.content for doc in result} == {"array meta"}
|
|
225
|
+
|
|
144
226
|
|
|
145
227
|
def test_treat_meta_field():
|
|
146
228
|
cast_integer = SQL("(") + SQL("meta->>") + SQLLiteral("number") + SQL(")::integer")
|
|
@@ -165,6 +247,92 @@ def test_treat_meta_field_sql_injection_is_safely_escaped():
|
|
|
165
247
|
assert result == SQL("meta->>") + SQLLiteral("name' OR 1=1 --")
|
|
166
248
|
|
|
167
249
|
|
|
250
|
+
def test_treat_meta_field_as_jsonb():
|
|
251
|
+
# array operators access the meta value as JSONB (->) without a cast, unlike _treat_meta_field (->>)
|
|
252
|
+
assert _treat_meta_field_as_jsonb(field="meta.tags") == SQL("meta->") + SQLLiteral("tags")
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def test_treat_meta_field_as_jsonb_sql_injection_is_safely_escaped():
|
|
256
|
+
result = _treat_meta_field_as_jsonb(field="meta.tags' OR 1=1 --")
|
|
257
|
+
assert isinstance(result, Composed)
|
|
258
|
+
assert result == SQL("meta->") + SQLLiteral("tags' OR 1=1 --")
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def test_array_contains_condition():
|
|
262
|
+
condition = {"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
|
|
263
|
+
query, values = _parse_comparison_condition(condition)
|
|
264
|
+
|
|
265
|
+
assert _render(query) == "meta->'tags' @> %s"
|
|
266
|
+
# the value is wrapped as a single Jsonb param holding the original list
|
|
267
|
+
assert len(values) == 1
|
|
268
|
+
assert isinstance(values[0], Jsonb)
|
|
269
|
+
assert values[0].obj == ["tag1", "tag2"]
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def test_array_overlaps_condition():
|
|
273
|
+
condition = {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag2"]}
|
|
274
|
+
query, values = _parse_comparison_condition(condition)
|
|
275
|
+
|
|
276
|
+
# the jsonb_typeof guard keeps `?|` from matching objects (by key) or bare strings
|
|
277
|
+
assert _render(query) == "(jsonb_typeof(meta->'tags') = 'array' AND meta->'tags' ?| %s)"
|
|
278
|
+
# the list is wrapped twice (once here, once by _parse_comparison_condition), exactly like 'in',
|
|
279
|
+
# so psycopg adapts it to a nested Postgres text[]; `?|` deconstructs it to the same elements
|
|
280
|
+
assert values == [[["tag1", "tag2"]]]
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def test_array_contains_non_list_value_raises():
|
|
284
|
+
condition = {"field": "meta.tags", "operator": "array_contains", "value": "tag1"}
|
|
285
|
+
with pytest.raises(FilterError):
|
|
286
|
+
_parse_comparison_condition(condition)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def test_array_overlaps_non_list_value_raises():
|
|
290
|
+
condition = {"field": "meta.tags", "operator": "array_overlaps", "value": "tag1"}
|
|
291
|
+
with pytest.raises(FilterError):
|
|
292
|
+
_parse_comparison_condition(condition)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def test_array_overlaps_non_string_elements_raises():
|
|
296
|
+
condition = {"field": "meta.tags", "operator": "array_overlaps", "value": [1, 2]}
|
|
297
|
+
with pytest.raises(FilterError):
|
|
298
|
+
_parse_comparison_condition(condition)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
@pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
|
|
302
|
+
def test_array_operators_empty_list_raises(operator):
|
|
303
|
+
# `@> '[]'` matches every array and `?| '{}'` matches nothing, so an empty list would mean
|
|
304
|
+
# "everything" for one operator and "nothing" for the other. Reject it instead.
|
|
305
|
+
condition = {"field": "meta.tags", "operator": operator, "value": []}
|
|
306
|
+
with pytest.raises(FilterError, match="non-empty list"):
|
|
307
|
+
_parse_comparison_condition(condition)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
@pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
|
|
311
|
+
def test_array_operators_on_non_meta_field_raises(operator):
|
|
312
|
+
# top-level columns are never JSONB arrays, so this must fail before reaching the database
|
|
313
|
+
condition = {"field": "content", "operator": operator, "value": ["tag1"]}
|
|
314
|
+
with pytest.raises(FilterError, match="only supported on meta fields"):
|
|
315
|
+
_parse_comparison_condition(condition)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def test_convert_filters_array_contains():
|
|
319
|
+
filters = {"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
|
|
320
|
+
where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
321
|
+
|
|
322
|
+
assert _render(where_clause) == " WHERE meta->'tags' @> %s"
|
|
323
|
+
assert len(params) == 1
|
|
324
|
+
assert isinstance(params[0], Jsonb)
|
|
325
|
+
assert params[0].obj == ["tag1", "tag2"]
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def test_convert_filters_array_overlaps():
|
|
329
|
+
filters = {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag2"]}
|
|
330
|
+
where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
331
|
+
|
|
332
|
+
assert _render(where_clause) == " WHERE (jsonb_typeof(meta->'tags') = 'array' AND meta->'tags' ?| %s)"
|
|
333
|
+
assert params == ([["tag1", "tag2"]],)
|
|
334
|
+
|
|
335
|
+
|
|
168
336
|
def test_comparison_condition_missing_operator():
|
|
169
337
|
condition = {"field": "meta.type", "value": "article"}
|
|
170
338
|
with pytest.raises(FilterError):
|
|
@@ -183,6 +351,14 @@ def test_comparison_condition_unknown_operator():
|
|
|
183
351
|
_parse_comparison_condition(condition)
|
|
184
352
|
|
|
185
353
|
|
|
354
|
+
@pytest.mark.parametrize("operator", ["like", "not like"])
|
|
355
|
+
def test_comparison_condition_like_operator_requires_str_value(operator):
|
|
356
|
+
# pgvector-specific: LIKE / NOT LIKE require a string operand.
|
|
357
|
+
condition = {"field": "meta.name", "operator": operator, "value": 123}
|
|
358
|
+
with pytest.raises(FilterError, match="must be a str when using 'LIKE'"):
|
|
359
|
+
_parse_comparison_condition(condition)
|
|
360
|
+
|
|
361
|
+
|
|
186
362
|
def test_logical_condition_missing_operator():
|
|
187
363
|
condition = {"conditions": []}
|
|
188
364
|
with pytest.raises(FilterError):
|
|
@@ -228,13 +404,32 @@ def test_logical_condition_nested():
|
|
|
228
404
|
"("
|
|
229
405
|
"(meta->>'domain' IS DISTINCT FROM %s OR meta->>'chapter' = ANY(%s))"
|
|
230
406
|
" AND "
|
|
231
|
-
"((meta->>'number')::integer >= %s OR meta->>'author' IS NULL OR meta->>'author' != ALL(%s))"
|
|
407
|
+
"((meta->>'number')::integer >= %s OR (meta->>'author' IS NULL OR meta->>'author' != ALL(%s)))"
|
|
232
408
|
")"
|
|
233
409
|
)
|
|
234
410
|
assert _render(query) == expected_sql
|
|
235
411
|
assert values == ["science", [["intro", "conclusion"]], 90, [["John", "Jane"]]]
|
|
236
412
|
|
|
237
413
|
|
|
414
|
+
def test_logical_condition_not_in_is_parenthesized_when_and_combined():
|
|
415
|
+
# `not in` renders an internal `OR` (`IS NULL OR != ALL`). Without surrounding
|
|
416
|
+
# parentheses that `OR` escapes an enclosing `AND` (AND binds tighter than OR in
|
|
417
|
+
# SQL), so an AND-combined `not in` would wrongly match documents.
|
|
418
|
+
condition = {
|
|
419
|
+
"operator": "AND",
|
|
420
|
+
"conditions": [
|
|
421
|
+
{"field": "meta.number", "operator": "==", "value": 5},
|
|
422
|
+
{"field": "meta.author", "operator": "not in", "value": ["John", "Jane"]},
|
|
423
|
+
],
|
|
424
|
+
}
|
|
425
|
+
query, values = _parse_logical_condition(condition)
|
|
426
|
+
assert isinstance(query, Composed)
|
|
427
|
+
|
|
428
|
+
expected_sql = "((meta->>'number')::integer = %s AND (meta->>'author' IS NULL OR meta->>'author' != ALL(%s)))"
|
|
429
|
+
assert _render(query) == expected_sql
|
|
430
|
+
assert values == [5, [["John", "Jane"]]]
|
|
431
|
+
|
|
432
|
+
|
|
238
433
|
def test_convert_filters_to_where_clause_and_params():
|
|
239
434
|
filters = {
|
|
240
435
|
"operator": "AND",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|