ibm-db-haystack 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ibm_db_haystack-0.4.0/CHANGELOG.md +47 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/PKG-INFO +5 -2
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/README.md +3 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/src/haystack_integrations/document_stores/ibm_db/document_store.py +112 -41
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/tests/test_document_store.py +317 -0
- ibm_db_haystack-0.2.0/CHANGELOG.md +0 -16
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/.gitignore +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/LICENSE.txt +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/docker-compose.yml +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/pydoc/config_docusaurus.yml +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/pyproject.toml +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/src/haystack_integrations/components/retrievers/ibm_db/__init__.py +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/src/haystack_integrations/components/retrievers/ibm_db/embedding_retriever.py +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/src/haystack_integrations/document_stores/ibm_db/__init__.py +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/src/haystack_integrations/document_stores/ibm_db/filters.py +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/tests/__init__.py +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/tests/conftest.py +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/tests/test_embedding_retriever.py +0 -0
- {ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/tests/test_filters.py +0 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [integrations/ibm_db-v0.3.0] - 2026-09-01
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- `IBMDB2DocumentStore` search_term + pagination (#3689)
|
|
8
|
+
- IBMDB adding optional parameter `filters` to `get_metadata_field_unique_values()` (#3739)
|
|
9
|
+
|
|
10
|
+
### 🧪 Testing
|
|
11
|
+
|
|
12
|
+
- `IBMDBDocumentStore` removing duplicated tests already covered by haystack Mixin tests (#3779)
|
|
13
|
+
|
|
14
|
+
### 🧹 Chores
|
|
15
|
+
|
|
16
|
+
- Re-standardize READMEs (#3768)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
## [integrations/ibm_db-v0.2.0] - 2026-07-27
|
|
20
|
+
|
|
21
|
+
### 🚀 Features
|
|
22
|
+
|
|
23
|
+
- IBM Db2 - lazy connection + closing methods (#3664)
|
|
24
|
+
|
|
25
|
+
### 🐛 Bug Fixes
|
|
26
|
+
|
|
27
|
+
- Fix new issues raised by ruff 0.16.0 (#3670)
|
|
28
|
+
|
|
29
|
+
### 📚 Documentation
|
|
30
|
+
|
|
31
|
+
- Update DB2 to Db2 in README (#3575)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
## [integrations/ibm_db-v0.1.1] - 2026-07-10
|
|
35
|
+
|
|
36
|
+
### 📚 Documentation
|
|
37
|
+
|
|
38
|
+
- Update docstrings for IBM Db2 integration (#3574)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
## [integrations/ibm_db-v0.1.0] - 2026-07-09
|
|
42
|
+
|
|
43
|
+
### 🚀 Features
|
|
44
|
+
|
|
45
|
+
- Added IBM Db2 vector store integration (#3518)
|
|
46
|
+
|
|
47
|
+
<!-- generated by git-cliff -->
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: ibm-db-haystack
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Haystack integration for IBM Db2
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/ibm_db#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -28,10 +28,13 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
[](https://pypi.org/project/ibm-db-haystack)
|
|
29
29
|
[](https://pypi.org/project/ibm-db-haystack)
|
|
30
30
|
|
|
31
|
+
- [Integration page](https://haystack.deepset.ai/integrations/ibm-db-document-store)
|
|
31
32
|
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/ibm_db/CHANGELOG.md)
|
|
32
33
|
|
|
33
34
|
---
|
|
34
35
|
|
|
36
|
+
## Contributing
|
|
37
|
+
|
|
35
38
|
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
36
39
|
|
|
37
40
|
To run integration tests locally, you need to have an IBM Db2 instance running.
|
|
@@ -3,10 +3,13 @@
|
|
|
3
3
|
[](https://pypi.org/project/ibm-db-haystack)
|
|
4
4
|
[](https://pypi.org/project/ibm-db-haystack)
|
|
5
5
|
|
|
6
|
+
- [Integration page](https://haystack.deepset.ai/integrations/ibm-db-document-store)
|
|
6
7
|
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/ibm_db/CHANGELOG.md)
|
|
7
8
|
|
|
8
9
|
---
|
|
9
10
|
|
|
11
|
+
## Contributing
|
|
12
|
+
|
|
10
13
|
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
11
14
|
|
|
12
15
|
To run integration tests locally, you need to have an IBM Db2 instance running.
|
|
@@ -5,14 +5,13 @@
|
|
|
5
5
|
"""IBM Db2 Document Store for Haystack."""
|
|
6
6
|
|
|
7
7
|
import json
|
|
8
|
-
import logging
|
|
9
8
|
import threading
|
|
10
9
|
from collections.abc import Iterator
|
|
11
10
|
from contextlib import contextmanager, suppress
|
|
12
11
|
from typing import Any, Literal
|
|
13
12
|
|
|
14
13
|
import ibm_db_dbi # type: ignore[import-untyped]
|
|
15
|
-
from haystack import default_from_dict, default_to_dict
|
|
14
|
+
from haystack import default_from_dict, default_to_dict, logging
|
|
16
15
|
from haystack.dataclasses import Document
|
|
17
16
|
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
18
17
|
from haystack.document_stores.types import DuplicatePolicy
|
|
@@ -22,6 +21,8 @@ from .filters import FilterTranslator
|
|
|
22
21
|
|
|
23
22
|
logger = logging.getLogger(__name__)
|
|
24
23
|
|
|
24
|
+
VALID_DISTANCE_METRICS = frozenset({"COSINE", "DOT", "EUCLIDEAN", "EUCLIDEAN_SQUARED", "HAMMING", "MANHATTAN"})
|
|
25
|
+
|
|
25
26
|
|
|
26
27
|
def _parse_embedding(embedding: Any) -> list[float] | None:
|
|
27
28
|
"""
|
|
@@ -103,7 +104,14 @@ class IBMDb2DocumentStore:
|
|
|
103
104
|
connection_options: dict[str, Any] | None = None,
|
|
104
105
|
table_name: str = "haystack_documents",
|
|
105
106
|
embedding_dim: int = 768,
|
|
106
|
-
distance_metric: Literal[
|
|
107
|
+
distance_metric: Literal[
|
|
108
|
+
"COSINE",
|
|
109
|
+
"DOT",
|
|
110
|
+
"EUCLIDEAN",
|
|
111
|
+
"EUCLIDEAN_SQUARED",
|
|
112
|
+
"HAMMING",
|
|
113
|
+
"MANHATTAN",
|
|
114
|
+
] = "COSINE",
|
|
107
115
|
recreate_table: bool = False,
|
|
108
116
|
):
|
|
109
117
|
"""
|
|
@@ -121,7 +129,8 @@ class IBMDb2DocumentStore:
|
|
|
121
129
|
:param connection_options: Additional connection options as dict (optional)
|
|
122
130
|
:param table_name: Name of the table to store documents (default: "haystack_documents")
|
|
123
131
|
:param embedding_dim: Dimension of embedding vectors (default: 768)
|
|
124
|
-
:param distance_metric: Distance metric for similarity search (default: "COSINE")
|
|
132
|
+
:param distance_metric: Distance metric for similarity search (default: "COSINE"). Supported metrics in Db2:
|
|
133
|
+
"COSINE", "DOT", "EUCLIDEAN", "EUCLIDEAN_SQUARED", "HAMMING", "MANHATTAN".
|
|
125
134
|
:param recreate_table: If True, drop and recreate the table (default: False)
|
|
126
135
|
"""
|
|
127
136
|
self.database = database
|
|
@@ -136,6 +145,9 @@ class IBMDb2DocumentStore:
|
|
|
136
145
|
self.connection_options = connection_options
|
|
137
146
|
self.table_name = table_name
|
|
138
147
|
self.embedding_dim = embedding_dim
|
|
148
|
+
if distance_metric not in VALID_DISTANCE_METRICS:
|
|
149
|
+
msg = f"Invalid distance_metric '{distance_metric}'. Must be one of {sorted(VALID_DISTANCE_METRICS)}."
|
|
150
|
+
raise ValueError(msg)
|
|
139
151
|
self.distance_metric = distance_metric
|
|
140
152
|
self.recreate_table = recreate_table
|
|
141
153
|
|
|
@@ -269,7 +281,7 @@ class IBMDb2DocumentStore:
|
|
|
269
281
|
try:
|
|
270
282
|
cur.execute(create_sql)
|
|
271
283
|
conn.commit()
|
|
272
|
-
logger.info(
|
|
284
|
+
logger.info("Created table {table_name}", table_name=self.table_name)
|
|
273
285
|
except Exception:
|
|
274
286
|
conn.rollback()
|
|
275
287
|
# If it still fails, raise the error
|
|
@@ -579,50 +591,109 @@ class IBMDb2DocumentStore:
|
|
|
579
591
|
|
|
580
592
|
return updated_count
|
|
581
593
|
|
|
582
|
-
|
|
594
|
+
@staticmethod
|
|
595
|
+
def _normalize_metadata_field_name(field_name: str) -> str:
|
|
583
596
|
"""
|
|
584
|
-
|
|
597
|
+
Normalize a metadata field name and validate its characters.
|
|
585
598
|
|
|
586
|
-
|
|
587
|
-
|
|
599
|
+
Removes the 'meta.' prefix, if present, and validates the remaining characters to prevent
|
|
600
|
+
SQL/JSONPath injection when the name is embedded in a SQL string literal.
|
|
601
|
+
|
|
602
|
+
:param field_name: The field name to normalize.
|
|
603
|
+
:return: The normalized field name.
|
|
604
|
+
:raises ValueError: If the field name contains characters other than alphanumerics, underscores,
|
|
605
|
+
or hyphens.
|
|
588
606
|
"""
|
|
589
|
-
|
|
590
|
-
field_name = field.removeprefix("meta.")
|
|
607
|
+
field_name = field_name.removeprefix("meta.")
|
|
591
608
|
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
609
|
+
if not all(c.isalnum() or c in ("_", "-") for c in field_name):
|
|
610
|
+
msg = (
|
|
611
|
+
f"Invalid metadata field name: '{field_name}'. Field names can only contain alphanumeric "
|
|
612
|
+
f"characters, underscores, and hyphens."
|
|
613
|
+
)
|
|
614
|
+
raise ValueError(msg)
|
|
615
|
+
|
|
616
|
+
return field_name
|
|
617
|
+
|
|
618
|
+
def get_metadata_field_unique_values(
|
|
619
|
+
self,
|
|
620
|
+
metadata_field: str,
|
|
621
|
+
search_term: str | None = None,
|
|
622
|
+
from_: int = 0,
|
|
623
|
+
size: int = 10,
|
|
624
|
+
filters: dict[str, Any] | None = None,
|
|
625
|
+
) -> tuple[list[Any], int]:
|
|
626
|
+
"""
|
|
627
|
+
Get unique values for a given metadata field, optionally filtered by a search term.
|
|
628
|
+
|
|
629
|
+
**Note**: values of different types are kept distinct even when they compare equal in Python
|
|
630
|
+
or share a textual form (e.g. the int `1`, the bool `True` and the str `"1"` are returned as
|
|
631
|
+
three separate values, and a whole-number float like `1.0` stays distinct from the int `1`).
|
|
632
|
+
|
|
633
|
+
:param metadata_field: The metadata field name (can include or omit the 'meta.' prefix).
|
|
634
|
+
:param search_term: Optional term to filter the returned values by, matching as a case-insensitive
|
|
635
|
+
substring of the metadata field's own value (not the document content). If None, all values
|
|
636
|
+
are considered.
|
|
637
|
+
:param from_: The offset for pagination (0-based).
|
|
638
|
+
:param size: The number of unique values to return.
|
|
639
|
+
:param filters: Optional filters to restrict the documents considered.
|
|
640
|
+
:return: A tuple containing (list of unique values in their original JSON type, total count of
|
|
641
|
+
unique values matching `search_term`).
|
|
642
|
+
"""
|
|
643
|
+
field_name = self._normalize_metadata_field_name(metadata_field)
|
|
644
|
+
|
|
645
|
+
# Db2 raises SQL0134N ("a LOB column can't be used with DISTINCT/GROUP BY") when DISTINCT is
|
|
646
|
+
# applied directly to a JSON_QUERY(...) expression derived from the BLOB `meta` column. Wrapping
|
|
647
|
+
# the extraction in a derived table materializes it as a plain VARCHAR column first, so DISTINCT
|
|
648
|
+
# and ORDER BY in the outer query operate on that column instead of the LOB-derived expression.
|
|
649
|
+
# `filters` is applied inside this subquery (not the outer one) since it references the raw
|
|
650
|
+
# `meta` column, which only exists at this level — the outer query only sees the extracted `value`.
|
|
651
|
+
#
|
|
652
|
+
# JSON_VALUE dequotes strings and stringifies numbers/booleans, so a JSON string "1" and the
|
|
653
|
+
# number 1 both become the identical SQL text '1' - merging distinct values and, once re-parsed,
|
|
654
|
+
# corrupting a numeric-looking string into a number. JSON_QUERY preserves the scalar's literal
|
|
655
|
+
# JSON form instead (quotes for strings, `true`/`false` for booleans), so json.loads() below can
|
|
656
|
+
# recover the original type, and DISTINCT compares literal text instead of collapsing values.
|
|
657
|
+
params: list[Any] = []
|
|
658
|
+
filter_where = ""
|
|
659
|
+
if filters:
|
|
660
|
+
filter_expression = FilterTranslator().translate(filters, params)
|
|
661
|
+
filter_where = f" WHERE {filter_expression}"
|
|
662
|
+
|
|
663
|
+
value_subquery = (
|
|
664
|
+
f"SELECT JSON_QUERY(SYSTOOLS.BSON2JSON(meta), '$.{field_name}' RETURNING VARCHAR(1000)) AS value "
|
|
665
|
+
f"FROM {self.table_name}{filter_where}"
|
|
600
666
|
)
|
|
601
|
-
|
|
602
|
-
|
|
667
|
+
|
|
668
|
+
search_clause = ""
|
|
669
|
+
if search_term is not None:
|
|
670
|
+
search_clause = " AND LOCATE(UPPER(?), UPPER(value)) > 0"
|
|
671
|
+
params.append(search_term)
|
|
672
|
+
|
|
673
|
+
count_sql = f"SELECT COUNT(DISTINCT value) FROM ({value_subquery}) AS t WHERE value IS NOT NULL{search_clause}"
|
|
674
|
+
select_sql = (
|
|
675
|
+
f"SELECT DISTINCT value FROM ({value_subquery}) AS t WHERE value IS NOT NULL{search_clause} "
|
|
676
|
+
f"ORDER BY value OFFSET ? ROWS FETCH FIRST ? ROWS ONLY"
|
|
677
|
+
)
|
|
678
|
+
|
|
679
|
+
with self._transaction(f"Failed to get unique values for field '{metadata_field}'") as cur:
|
|
680
|
+
cur.execute(count_sql, params)
|
|
681
|
+
count_row = cur.fetchone()
|
|
682
|
+
total_count = count_row[0] if count_row and count_row[0] is not None else 0
|
|
683
|
+
|
|
684
|
+
cur.execute(select_sql, [*params, from_, size])
|
|
603
685
|
rows = cur.fetchall()
|
|
604
686
|
|
|
605
|
-
|
|
606
|
-
seen = set()
|
|
607
|
-
values = []
|
|
687
|
+
unique_values: list[Any] = []
|
|
608
688
|
for row in rows:
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
seen.add(value_key)
|
|
618
|
-
values.append(parsed_value)
|
|
619
|
-
except (json.JSONDecodeError, TypeError):
|
|
620
|
-
# If it's not valid JSON, use the string value
|
|
621
|
-
if value not in seen:
|
|
622
|
-
seen.add(value)
|
|
623
|
-
values.append(value)
|
|
624
|
-
|
|
625
|
-
return values
|
|
689
|
+
if row[0] is None:
|
|
690
|
+
continue
|
|
691
|
+
try:
|
|
692
|
+
unique_values.append(json.loads(row[0]))
|
|
693
|
+
except (json.JSONDecodeError, TypeError):
|
|
694
|
+
unique_values.append(row[0])
|
|
695
|
+
|
|
696
|
+
return unique_values, total_count
|
|
626
697
|
|
|
627
698
|
def get_metadata_field_min_max(self, field: str) -> dict[str, Any]:
|
|
628
699
|
"""
|
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
"""Integration tests for IBM DB2 Document Store using Haystack mixin tests."""
|
|
6
6
|
|
|
7
7
|
import math
|
|
8
|
+
import sys
|
|
8
9
|
from datetime import datetime, timedelta, timezone
|
|
9
10
|
from unittest.mock import MagicMock, Mock
|
|
10
11
|
|
|
11
12
|
import pytest
|
|
12
13
|
from haystack.dataclasses import Document
|
|
13
14
|
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
15
|
+
from haystack.document_stores.types import DuplicatePolicy
|
|
14
16
|
from haystack.testing.document_store import (
|
|
15
17
|
CountDocumentsByFilterTest,
|
|
16
18
|
CountDocumentsTest,
|
|
@@ -239,6 +241,176 @@ class TestDocumentStore(
|
|
|
239
241
|
assert [doc.id for doc in results] == ["a", "c"]
|
|
240
242
|
assert all(doc.score is not None for doc in results)
|
|
241
243
|
|
|
244
|
+
@pytest.mark.parametrize(
|
|
245
|
+
"metric",
|
|
246
|
+
["COSINE", "DOT", "EUCLIDEAN", "EUCLIDEAN_SQUARED", "HAMMING", "MANHATTAN"],
|
|
247
|
+
)
|
|
248
|
+
def test_all_distance_metrics_return_ranked_results(self, metric: str, connection_config):
|
|
249
|
+
"""
|
|
250
|
+
Verify that _embedding_retrieval works correctly for each of the 6 Db2
|
|
251
|
+
distance metrics: correct ranking, finite scores, top_k respected.
|
|
252
|
+
|
|
253
|
+
A fresh store with embedding_dim=4 is created for each metric so the
|
|
254
|
+
test is independent of the default `document_store` fixture.
|
|
255
|
+
All Db2 distance metrics return scores ordered ASC (smaller = more similar).
|
|
256
|
+
"""
|
|
257
|
+
table_name = f"haystack_metric_{metric.lower()}_{sys.version_info.major}_{sys.version_info.minor}"
|
|
258
|
+
store = IBMDb2DocumentStore(
|
|
259
|
+
**connection_config,
|
|
260
|
+
table_name=table_name,
|
|
261
|
+
embedding_dim=4,
|
|
262
|
+
distance_metric=metric,
|
|
263
|
+
recreate_table=True,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
try:
|
|
267
|
+
# Embeddings chosen so ranking is unambiguous for every metric:
|
|
268
|
+
# most_similar == query → distance 0 (or near 0) for all metrics
|
|
269
|
+
# second_best is close → small distance
|
|
270
|
+
# least_similar is far → large distance
|
|
271
|
+
query = [1.0, 1.0, 1.0, 1.0]
|
|
272
|
+
most_similar = [1.0, 1.0, 1.0, 1.0]
|
|
273
|
+
second_best = [1.0, 1.0, 0.5, 0.5]
|
|
274
|
+
least_similar = [0.0, 0.0, 0.0, 1.0]
|
|
275
|
+
|
|
276
|
+
docs = [
|
|
277
|
+
Document(content="most_similar", embedding=most_similar),
|
|
278
|
+
Document(content="second_best", embedding=second_best),
|
|
279
|
+
Document(content="least_similar", embedding=least_similar),
|
|
280
|
+
]
|
|
281
|
+
store.write_documents(docs, policy=DuplicatePolicy.OVERWRITE)
|
|
282
|
+
|
|
283
|
+
results = store._embedding_retrieval(query_embedding=query, top_k=2)
|
|
284
|
+
|
|
285
|
+
# Exactly top_k documents returned
|
|
286
|
+
assert len(results) == 2, f"[{metric}] Expected 2 results, got {len(results)}"
|
|
287
|
+
|
|
288
|
+
# most_similar must be in the top-2; least_similar must be excluded
|
|
289
|
+
contents = [r.content for r in results]
|
|
290
|
+
assert "most_similar" in contents, f"[{metric}] 'most_similar' missing from top-2: {contents}"
|
|
291
|
+
assert "least_similar" not in contents, f"[{metric}] 'least_similar' appeared in top-2: {contents}"
|
|
292
|
+
|
|
293
|
+
# Scores must be finite floats and ordered ASC (ORDER BY score ASC)
|
|
294
|
+
for doc in results:
|
|
295
|
+
assert doc.score is not None, f"[{metric}] score is None"
|
|
296
|
+
assert math.isfinite(doc.score), f"[{metric}] non-finite score: {doc.score}"
|
|
297
|
+
assert results[0].content == "most_similar", (
|
|
298
|
+
f"[{metric}] Expected 'most_similar' at rank-1, got '{results[0].content}' "
|
|
299
|
+
f"(scores: {[r.score for r in results]})"
|
|
300
|
+
)
|
|
301
|
+
assert results[0].score <= results[1].score, (
|
|
302
|
+
f"[{metric}] Score ordering wrong: {results[0].score} > {results[1].score}"
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
finally:
|
|
306
|
+
try:
|
|
307
|
+
conn = store._get_connection()
|
|
308
|
+
with conn.cursor() as cur:
|
|
309
|
+
cur.execute(f"DROP TABLE {store.table_name}")
|
|
310
|
+
conn.commit()
|
|
311
|
+
except Exception:
|
|
312
|
+
pass
|
|
313
|
+
store.close()
|
|
314
|
+
|
|
315
|
+
@pytest.mark.parametrize(
|
|
316
|
+
"metric",
|
|
317
|
+
["COSINE", "DOT", "EUCLIDEAN", "EUCLIDEAN_SQUARED", "HAMMING", "MANHATTAN"],
|
|
318
|
+
)
|
|
319
|
+
def test_all_distance_metrics_respect_filters(self, metric: str, connection_config):
|
|
320
|
+
"""
|
|
321
|
+
Verify that metadata filters work correctly alongside each distance metric.
|
|
322
|
+
"""
|
|
323
|
+
table_name = f"haystack_metric_flt_{metric.lower()}_{sys.version_info.major}_{sys.version_info.minor}"
|
|
324
|
+
store = IBMDb2DocumentStore(
|
|
325
|
+
**connection_config,
|
|
326
|
+
table_name=table_name,
|
|
327
|
+
embedding_dim=4,
|
|
328
|
+
distance_metric=metric,
|
|
329
|
+
recreate_table=True,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
try:
|
|
333
|
+
query = [1.0, 1.0, 1.0, 1.0]
|
|
334
|
+
docs = [
|
|
335
|
+
Document(content="in_1", embedding=[1.0, 1.0, 1.0, 1.0], meta={"cat": "A"}),
|
|
336
|
+
Document(content="in_2", embedding=[0.9, 0.9, 0.9, 0.9], meta={"cat": "A"}),
|
|
337
|
+
Document(content="out", embedding=[1.0, 1.0, 1.0, 1.0], meta={"cat": "B"}),
|
|
338
|
+
]
|
|
339
|
+
store.write_documents(docs, policy=DuplicatePolicy.OVERWRITE)
|
|
340
|
+
|
|
341
|
+
filters = {"field": "meta.cat", "operator": "==", "value": "A"}
|
|
342
|
+
results = store._embedding_retrieval(query_embedding=query, top_k=5, filters=filters)
|
|
343
|
+
|
|
344
|
+
assert len(results) == 2, f"[{metric}] Expected 2 filtered results, got {len(results)}"
|
|
345
|
+
for doc in results:
|
|
346
|
+
assert doc.meta.get("cat") == "A", f"[{metric}] Doc with wrong category returned: {doc.meta}"
|
|
347
|
+
|
|
348
|
+
finally:
|
|
349
|
+
try:
|
|
350
|
+
conn = store._get_connection()
|
|
351
|
+
with conn.cursor() as cur:
|
|
352
|
+
cur.execute(f"DROP TABLE {store.table_name}")
|
|
353
|
+
conn.commit()
|
|
354
|
+
except Exception:
|
|
355
|
+
pass
|
|
356
|
+
store.close()
|
|
357
|
+
|
|
358
|
+
def test_get_metadata_field_unique_values_pagination(self, document_store: IBMDb2DocumentStore):
|
|
359
|
+
docs = [
|
|
360
|
+
Document(content=f"doc {i}", meta={"category": c}) for i, c in enumerate(["A", "B", "A", "C", "B", "A"])
|
|
361
|
+
]
|
|
362
|
+
document_store.write_documents(docs)
|
|
363
|
+
|
|
364
|
+
page1, total1 = document_store.get_metadata_field_unique_values("category", from_=0, size=2)
|
|
365
|
+
page2, total2 = document_store.get_metadata_field_unique_values("category", from_=2, size=2)
|
|
366
|
+
|
|
367
|
+
assert len(page1) == 2
|
|
368
|
+
assert len(page2) == 1
|
|
369
|
+
assert total1 == total2 == 3
|
|
370
|
+
assert not set(page1).intersection(page2)
|
|
371
|
+
assert set(page1) | set(page2) == {"A", "B", "C"}
|
|
372
|
+
|
|
373
|
+
all_values, total = document_store.get_metadata_field_unique_values("category", from_=0, size=100)
|
|
374
|
+
assert set(all_values) == {"A", "B", "C"}
|
|
375
|
+
assert total == 3
|
|
376
|
+
|
|
377
|
+
beyond, total_beyond = document_store.get_metadata_field_unique_values("category", from_=10, size=10)
|
|
378
|
+
assert beyond == []
|
|
379
|
+
assert total_beyond == 3
|
|
380
|
+
|
|
381
|
+
def test_get_metadata_field_unique_values_invalid_field_name(self, document_store: IBMDb2DocumentStore):
|
|
382
|
+
with pytest.raises(ValueError, match="Invalid metadata field name"):
|
|
383
|
+
document_store.get_metadata_field_unique_values("field' OR '1'='1")
|
|
384
|
+
|
|
385
|
+
def test_get_metadata_field_unique_values_hyphenated_field_name(self, document_store: IBMDb2DocumentStore):
|
|
386
|
+
docs = [
|
|
387
|
+
Document(content="one", meta={"some-field": "A"}),
|
|
388
|
+
Document(content="two", meta={"some-field": "B"}),
|
|
389
|
+
Document(content="three", meta={"some-field": "A"}),
|
|
390
|
+
]
|
|
391
|
+
document_store.write_documents(docs)
|
|
392
|
+
|
|
393
|
+
values, total_count = document_store.get_metadata_field_unique_values("some-field")
|
|
394
|
+
assert set(values) == {"A", "B"}
|
|
395
|
+
assert total_count == 2
|
|
396
|
+
|
|
397
|
+
def test_case_insensitive_and_special_characters_are_allowed(self, document_store: IBMDb2DocumentStore):
|
|
398
|
+
docs = [
|
|
399
|
+
Document(meta={"category": "100% complete"}),
|
|
400
|
+
Document(meta={"category": "plain"}),
|
|
401
|
+
Document(meta={"category": "under_score"}),
|
|
402
|
+
Document(meta={"category": "underXscore"}),
|
|
403
|
+
]
|
|
404
|
+
document_store.write_documents(docs)
|
|
405
|
+
|
|
406
|
+
values, total = document_store.get_metadata_field_unique_values("category", search_term="%")
|
|
407
|
+
assert values == ["100% complete"]
|
|
408
|
+
assert total == 1
|
|
409
|
+
|
|
410
|
+
values, total = document_store.get_metadata_field_unique_values("category", search_term="_")
|
|
411
|
+
assert values == ["under_score"]
|
|
412
|
+
assert total == 1
|
|
413
|
+
|
|
242
414
|
|
|
243
415
|
class TestIBMDb2DocumentStoreUtilMethods:
|
|
244
416
|
"""Unit tests for pure utility methods: _parse_embedding, _validate_embedding, _infer_field_type."""
|
|
@@ -300,6 +472,32 @@ class TestIBMDb2DocumentStoreUtilMethods:
|
|
|
300
472
|
def test_infer_field_type(self, value, expected):
|
|
301
473
|
assert IBMDb2DocumentStore._infer_field_type(value) == expected
|
|
302
474
|
|
|
475
|
+
# _normalize_metadata_field_name
|
|
476
|
+
def test_normalize_metadata_field_name_strips_meta_prefix(self):
|
|
477
|
+
assert IBMDb2DocumentStore._normalize_metadata_field_name("meta.category") == "category"
|
|
478
|
+
|
|
479
|
+
def test_normalize_metadata_field_name_without_prefix_unchanged(self):
|
|
480
|
+
assert IBMDb2DocumentStore._normalize_metadata_field_name("category") == "category"
|
|
481
|
+
|
|
482
|
+
def test_normalize_metadata_field_name_allows_underscore_and_hyphen(self):
|
|
483
|
+
assert IBMDb2DocumentStore._normalize_metadata_field_name("meta.some_field-name") == "some_field-name"
|
|
484
|
+
|
|
485
|
+
@pytest.mark.parametrize(
|
|
486
|
+
"bad_field",
|
|
487
|
+
[
|
|
488
|
+
"field@invalid",
|
|
489
|
+
"field with spaces",
|
|
490
|
+
"field;drop",
|
|
491
|
+
"meta.field!",
|
|
492
|
+
"field' OR '1'='1",
|
|
493
|
+
"../etc/passwd",
|
|
494
|
+
"field/* comment */",
|
|
495
|
+
],
|
|
496
|
+
)
|
|
497
|
+
def test_normalize_metadata_field_name_rejects_invalid_chars(self, bad_field):
|
|
498
|
+
with pytest.raises(ValueError, match="Invalid metadata field name"):
|
|
499
|
+
IBMDb2DocumentStore._normalize_metadata_field_name(bad_field)
|
|
500
|
+
|
|
303
501
|
|
|
304
502
|
@pytest.fixture
|
|
305
503
|
def unit_store(monkeypatch) -> IBMDb2DocumentStore:
|
|
@@ -459,6 +657,47 @@ class TestIBMDb2DocumentStoreUnit:
|
|
|
459
657
|
assert isinstance(new_store.username, Secret)
|
|
460
658
|
assert isinstance(new_store.password, Secret)
|
|
461
659
|
|
|
660
|
+
@pytest.mark.parametrize(
|
|
661
|
+
"metric",
|
|
662
|
+
[
|
|
663
|
+
"COSINE",
|
|
664
|
+
"DOT",
|
|
665
|
+
"EUCLIDEAN",
|
|
666
|
+
"EUCLIDEAN_SQUARED",
|
|
667
|
+
"HAMMING",
|
|
668
|
+
"MANHATTAN",
|
|
669
|
+
],
|
|
670
|
+
)
|
|
671
|
+
def test_supported_distance_metrics(self, metric, monkeypatch):
|
|
672
|
+
"""Test that all 6 supported Db2 distance metrics can be configured and serialized."""
|
|
673
|
+
monkeypatch.setenv("DB2_USERNAME", "db2inst1")
|
|
674
|
+
monkeypatch.setenv("DB2_PASSWORD", "Passw0rd123!")
|
|
675
|
+
store = IBMDb2DocumentStore(
|
|
676
|
+
database="testdb",
|
|
677
|
+
hostname="localhost",
|
|
678
|
+
username=Secret.from_env_var("DB2_USERNAME"),
|
|
679
|
+
password=Secret.from_env_var("DB2_PASSWORD"),
|
|
680
|
+
distance_metric=metric,
|
|
681
|
+
)
|
|
682
|
+
assert store.distance_metric == metric
|
|
683
|
+
data = store.to_dict()
|
|
684
|
+
assert data["init_parameters"]["distance_metric"] == metric
|
|
685
|
+
restored = IBMDb2DocumentStore.from_dict(data)
|
|
686
|
+
assert restored.distance_metric == metric
|
|
687
|
+
|
|
688
|
+
def test_invalid_distance_metric_raises(self, monkeypatch):
|
|
689
|
+
"""Test that an invalid distance_metric raises ValueError at construction time."""
|
|
690
|
+
monkeypatch.setenv("DB2_USERNAME", "db2inst1")
|
|
691
|
+
monkeypatch.setenv("DB2_PASSWORD", "Passw0rd123!")
|
|
692
|
+
with pytest.raises(ValueError, match="Invalid distance_metric"):
|
|
693
|
+
IBMDb2DocumentStore(
|
|
694
|
+
database="testdb",
|
|
695
|
+
hostname="localhost",
|
|
696
|
+
username=Secret.from_env_var("DB2_USERNAME"),
|
|
697
|
+
password=Secret.from_env_var("DB2_PASSWORD"),
|
|
698
|
+
distance_metric="L2", # type: ignore[arg-type]
|
|
699
|
+
)
|
|
700
|
+
|
|
462
701
|
def test_to_row_with_none_metadata(self, unit_store):
|
|
463
702
|
"""Test _to_row with None metadata."""
|
|
464
703
|
doc = Document(id="1", content="test", meta=None, embedding=[0.1] * 768)
|
|
@@ -556,6 +795,30 @@ class TestIBMDb2DocumentStoreUnit:
|
|
|
556
795
|
executed_sql = cur.execute.call_args.args[0]
|
|
557
796
|
assert "AND embedding IS NOT NULL" in executed_sql
|
|
558
797
|
|
|
798
|
+
@pytest.mark.parametrize(
|
|
799
|
+
"metric",
|
|
800
|
+
[
|
|
801
|
+
"COSINE",
|
|
802
|
+
"DOT",
|
|
803
|
+
"EUCLIDEAN",
|
|
804
|
+
"EUCLIDEAN_SQUARED",
|
|
805
|
+
"HAMMING",
|
|
806
|
+
"MANHATTAN",
|
|
807
|
+
],
|
|
808
|
+
)
|
|
809
|
+
def test_embedding_retrieval_uses_configured_metric(self, mocked_store, metric):
|
|
810
|
+
store, _, cur = mocked_store
|
|
811
|
+
store.distance_metric = metric
|
|
812
|
+
cur.fetchall.return_value = []
|
|
813
|
+
|
|
814
|
+
store._embedding_retrieval([0.1, 0.2], top_k=5)
|
|
815
|
+
|
|
816
|
+
executed_sql = cur.execute.call_args.args[0]
|
|
817
|
+
expected_fragment = (
|
|
818
|
+
f"VECTOR_DISTANCE(embedding, VECTOR(CAST(? AS CLOB(100000)), {store.embedding_dim}, FLOAT32), {metric})"
|
|
819
|
+
)
|
|
820
|
+
assert expected_fragment in executed_sql
|
|
821
|
+
|
|
559
822
|
@pytest.mark.parametrize(
|
|
560
823
|
"fetch_error",
|
|
561
824
|
[
|
|
@@ -575,3 +838,57 @@ class TestIBMDb2DocumentStoreUnit:
|
|
|
575
838
|
|
|
576
839
|
with pytest.raises(RuntimeError, match="connection reset"):
|
|
577
840
|
store._embedding_retrieval([0.1, 0.2], top_k=1)
|
|
841
|
+
|
|
842
|
+
def test_get_metadata_field_unique_values_builds_paginated_sql(self, mocked_store):
|
|
843
|
+
store, _, cur = mocked_store
|
|
844
|
+
cur.fetchone.return_value = (2,)
|
|
845
|
+
# JSON_QUERY returns the literal JSON form of a string - quoted - unlike JSON_VALUE which
|
|
846
|
+
# dequotes it; json.loads() below relies on that to tell a string apart from a number.
|
|
847
|
+
cur.fetchall.return_value = [('"A"',), ('"B"',)]
|
|
848
|
+
|
|
849
|
+
values, total_count = store.get_metadata_field_unique_values("meta.category", from_=5, size=10)
|
|
850
|
+
|
|
851
|
+
assert values == ["A", "B"]
|
|
852
|
+
assert total_count == 2
|
|
853
|
+
|
|
854
|
+
count_sql, count_params = cur.execute.call_args_list[0].args
|
|
855
|
+
select_sql, select_params = cur.execute.call_args_list[1].args
|
|
856
|
+
|
|
857
|
+
assert "COUNT(DISTINCT value)" in count_sql
|
|
858
|
+
assert "$.category" in count_sql
|
|
859
|
+
assert count_params == []
|
|
860
|
+
|
|
861
|
+
assert "SELECT DISTINCT value" in select_sql
|
|
862
|
+
assert "ORDER BY value OFFSET ? ROWS FETCH FIRST ? ROWS ONLY" in select_sql
|
|
863
|
+
assert select_params == [5, 10]
|
|
864
|
+
|
|
865
|
+
def test_get_metadata_field_unique_values_applies_search_term(self, mocked_store):
|
|
866
|
+
store, _, cur = mocked_store
|
|
867
|
+
cur.fetchone.return_value = (1,)
|
|
868
|
+
cur.fetchall.return_value = [('"Python-based"',)]
|
|
869
|
+
|
|
870
|
+
store.get_metadata_field_unique_values("meta.category", search_term="python")
|
|
871
|
+
|
|
872
|
+
count_sql, count_params = cur.execute.call_args_list[0].args
|
|
873
|
+
select_sql, select_params = cur.execute.call_args_list[1].args
|
|
874
|
+
|
|
875
|
+
assert "LOCATE(UPPER(?), UPPER(value)) > 0" in count_sql
|
|
876
|
+
assert count_params == ["python"]
|
|
877
|
+
assert "LOCATE(UPPER(?), UPPER(value)) > 0" in select_sql
|
|
878
|
+
assert select_params == ["python", 0, 10]
|
|
879
|
+
|
|
880
|
+
def test_get_metadata_field_unique_values_rejects_invalid_field_name(self, mocked_store):
|
|
881
|
+
store, _, _ = mocked_store
|
|
882
|
+
|
|
883
|
+
with pytest.raises(ValueError, match="Invalid metadata field name"):
|
|
884
|
+
store.get_metadata_field_unique_values("field' OR '1'='1")
|
|
885
|
+
|
|
886
|
+
def test_get_metadata_field_unique_values_handles_no_rows(self, mocked_store):
|
|
887
|
+
store, _, cur = mocked_store
|
|
888
|
+
cur.fetchone.return_value = None
|
|
889
|
+
cur.fetchall.return_value = []
|
|
890
|
+
|
|
891
|
+
values, total_count = store.get_metadata_field_unique_values("meta.category")
|
|
892
|
+
|
|
893
|
+
assert values == []
|
|
894
|
+
assert total_count == 0
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
|
-
## [integrations/ibm_db-v0.1.1] - 2026-07-10
|
|
4
|
-
|
|
5
|
-
### 📚 Documentation
|
|
6
|
-
|
|
7
|
-
- Update docstrings for IBM Db2 integration (#3574)
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
## [integrations/ibm_db-v0.1.0] - 2026-07-09
|
|
11
|
-
|
|
12
|
-
### 🚀 Features
|
|
13
|
-
|
|
14
|
-
- Added IBM Db2 vector store integration (#3518)
|
|
15
|
-
|
|
16
|
-
<!-- generated by git-cliff -->
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ibm_db_haystack-0.2.0 → ibm_db_haystack-0.4.0}/src/haystack_integrations/document_stores/py.typed
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|