ibm-db-haystack 0.2.0__tar.gz → 0.3.0__tar.gz

{ibm_db_haystack-0.2.0 → ibm_db_haystack-0.3.0}/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [integrations/ibm_db-v0.2.0] - 2026-07-27
+
+### 🚀 Features
+
+- IBM Db2 - lazy connection + closing methods (#3664)
+
+### 🐛 Bug Fixes
+
+- Fix new issues raised by ruff 0.16.0 (#3670)
+
+### 📚 Documentation
+
+- Update DB2 to Db2 in README (#3575)
+
+
 ## [integrations/ibm_db-v0.1.1] - 2026-07-10
 
 ### 📚 Documentation

{ibm_db_haystack-0.2.0 → ibm_db_haystack-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: ibm-db-haystack
-Version: 0.2.0
+Version: 0.3.0
 Summary: Haystack integration for IBM Db2
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/ibm_db#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -28,10 +28,13 @@ Description-Content-Type: text/markdown
 [![PyPI - Version](https://img.shields.io/pypi/v/ibm-db-haystack.svg)](https://pypi.org/project/ibm-db-haystack)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/ibm-db-haystack.svg)](https://pypi.org/project/ibm-db-haystack)
 
+- [Integration page](https://haystack.deepset.ai/integrations/ibm-db-document-store)
 - [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/ibm_db/CHANGELOG.md)
 
 ---
 
+## Contributing
+
 Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
 
 To run integration tests locally, you need to have an IBM Db2 instance running.

{ibm_db_haystack-0.2.0 → ibm_db_haystack-0.3.0}/README.md

@@ -3,10 +3,13 @@
 [![PyPI - Version](https://img.shields.io/pypi/v/ibm-db-haystack.svg)](https://pypi.org/project/ibm-db-haystack)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/ibm-db-haystack.svg)](https://pypi.org/project/ibm-db-haystack)
 
+- [Integration page](https://haystack.deepset.ai/integrations/ibm-db-document-store)
 - [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/ibm_db/CHANGELOG.md)
 
 ---
 
+## Contributing
+
 Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
 
 To run integration tests locally, you need to have an IBM Db2 instance running.

{ibm_db_haystack-0.2.0 → ibm_db_haystack-0.3.0}/src/haystack_integrations/document_stores/ibm_db/document_store.py

@@ -579,50 +579,109 @@ class IBMDb2DocumentStore:
 
             return updated_count
 
-    def get_metadata_field_unique_values(self, field: str) -> list[Any]:
+    @staticmethod
+    def _normalize_metadata_field_name(field_name: str) -> str:
         """
-        Get all unique values for a given metadata field.
+        Normalize a metadata field name and validate its characters.
 
-        :param field: The metadata field name (can include 'meta.' prefix)
-        :return: List of unique values for the field
+        Removes the 'meta.' prefix, if present, and validates the remaining characters to prevent
+        SQL/JSONPath injection when the name is embedded in a SQL string literal.
+
+        :param field_name: The field name to normalize.
+        :return: The normalized field name.
+        :raises ValueError: If the field name contains characters other than alphanumerics, underscores,
+            or hyphens.
         """
-        # Strip 'meta.' prefix if present
-        field_name = field.removeprefix("meta.")
+        field_name = field_name.removeprefix("meta.")
 
-        # Extract values from the JSON metadata field
-        # Db2 has issues with DISTINCT/GROUP BY on JSON_VALUE results (SQL0134N error)
-        # So we fetch all values and deduplicate in Python
-        # Use RETURNING VARCHAR to explicitly specify the return type
-        sql = (
-            f"SELECT JSON_VALUE(SYSTOOLS.BSON2JSON(meta), '$.{field_name}' RETURNING VARCHAR(1000)) "
-            f"FROM {self.table_name} "
-            f"WHERE JSON_VALUE(SYSTOOLS.BSON2JSON(meta), '$.{field_name}' RETURNING VARCHAR(1000)) IS NOT NULL"
+        if not all(c.isalnum() or c in ("_", "-") for c in field_name):
+            msg = (
+                f"Invalid metadata field name: '{field_name}'. Field names can only contain alphanumeric "
+                f"characters, underscores, and hyphens."
+            )
+            raise ValueError(msg)
+
+        return field_name
+
+    def get_metadata_field_unique_values(
+        self,
+        metadata_field: str,
+        search_term: str | None = None,
+        from_: int = 0,
+        size: int = 10,
+        filters: dict[str, Any] | None = None,
+    ) -> tuple[list[Any], int]:
+        """
+        Get unique values for a given metadata field, optionally filtered by a search term.
+
+        **Note**: values of different types are kept distinct even when they compare equal in Python
+        or share a textual form (e.g. the int `1`, the bool `True` and the str `"1"` are returned as
+        three separate values, and a whole-number float like `1.0` stays distinct from the int `1`).
+
+        :param metadata_field: The metadata field name (can include or omit the 'meta.' prefix).
+        :param search_term: Optional term to filter the returned values by, matching as a case-insensitive
+            substring of the metadata field's own value (not the document content). If None, all values
+            are considered.
+        :param from_: The offset for pagination (0-based).
+        :param size: The number of unique values to return.
+        :param filters: Optional filters to restrict the documents considered.
+        :return: A tuple containing (list of unique values in their original JSON type, total count of
+            unique values matching `search_term`).
+        """
+        field_name = self._normalize_metadata_field_name(metadata_field)
+
+        # Db2 raises SQL0134N ("a LOB column can't be used with DISTINCT/GROUP BY") when DISTINCT is
+        # applied directly to a JSON_QUERY(...) expression derived from the BLOB `meta` column. Wrapping
+        # the extraction in a derived table materializes it as a plain VARCHAR column first, so DISTINCT
+        # and ORDER BY in the outer query operate on that column instead of the LOB-derived expression.
+        # `filters` is applied inside this subquery (not the outer one) since it references the raw
+        # `meta` column, which only exists at this level — the outer query only sees the extracted `value`.
+        #
+        # JSON_VALUE dequotes strings and stringifies numbers/booleans, so a JSON string "1" and the
+        # number 1 both become the identical SQL text '1' - merging distinct values and, once re-parsed,
+        # corrupting a numeric-looking string into a number. JSON_QUERY preserves the scalar's literal
+        # JSON form instead (quotes for strings, `true`/`false` for booleans), so json.loads() below can
+        # recover the original type, and DISTINCT compares literal text instead of collapsing values.
+        params: list[Any] = []
+        filter_where = ""
+        if filters:
+            filter_expression = FilterTranslator().translate(filters, params)
+            filter_where = f" WHERE {filter_expression}"
+
+        value_subquery = (
+            f"SELECT JSON_QUERY(SYSTOOLS.BSON2JSON(meta), '$.{field_name}' RETURNING VARCHAR(1000)) AS value "
+            f"FROM {self.table_name}{filter_where}"
         )
-        with self._transaction(f"Failed to get unique values for field '{field}'") as cur:
-            cur.execute(sql)
+
+        search_clause = ""
+        if search_term is not None:
+            search_clause = " AND LOCATE(UPPER(?), UPPER(value)) > 0"
+            params.append(search_term)
+
+        count_sql = f"SELECT COUNT(DISTINCT value) FROM ({value_subquery}) AS t WHERE value IS NOT NULL{search_clause}"
+        select_sql = (
+            f"SELECT DISTINCT value FROM ({value_subquery}) AS t WHERE value IS NOT NULL{search_clause} "
+            f"ORDER BY value OFFSET ? ROWS FETCH FIRST ? ROWS ONLY"
+        )
+
+        with self._transaction(f"Failed to get unique values for field '{metadata_field}'") as cur:
+            cur.execute(count_sql, params)
+            count_row = cur.fetchone()
+            total_count = count_row[0] if count_row and count_row[0] is not None else 0
+
+            cur.execute(select_sql, [*params, from_, size])
             rows = cur.fetchall()
 
-        # Parse and deduplicate the values
-        seen = set()
-        values = []
+        unique_values: list[Any] = []
         for row in rows:
-            value = row[0]
-            if value is not None:
-                # Try to parse as JSON to get the actual type
-                try:
-                    parsed_value = json.loads(value)
-                    # Use JSON string for deduplication to handle unhashable types
-                    value_key = json.dumps(parsed_value, sort_keys=True)
-                    if value_key not in seen:
-                        seen.add(value_key)
-                        values.append(parsed_value)
-                except (json.JSONDecodeError, TypeError):
-                    # If it's not valid JSON, use the string value
-                    if value not in seen:
-                        seen.add(value)
-                        values.append(value)
-
-        return values
+            if row[0] is None:
+                continue
+            try:
+                unique_values.append(json.loads(row[0]))
+            except (json.JSONDecodeError, TypeError):
+                unique_values.append(row[0])
+
+        return unique_values, total_count
 
     def get_metadata_field_min_max(self, field: str) -> dict[str, Any]:
         """

{ibm_db_haystack-0.2.0 → ibm_db_haystack-0.3.0}/tests/test_document_store.py

@@ -239,6 +239,62 @@ class TestDocumentStore(
         assert [doc.id for doc in results] == ["a", "c"]
         assert all(doc.score is not None for doc in results)
 
+    def test_get_metadata_field_unique_values_pagination(self, document_store: IBMDb2DocumentStore):
+        docs = [
+            Document(content=f"doc {i}", meta={"category": c}) for i, c in enumerate(["A", "B", "A", "C", "B", "A"])
+        ]
+        document_store.write_documents(docs)
+
+        page1, total1 = document_store.get_metadata_field_unique_values("category", from_=0, size=2)
+        page2, total2 = document_store.get_metadata_field_unique_values("category", from_=2, size=2)
+
+        assert len(page1) == 2
+        assert len(page2) == 1
+        assert total1 == total2 == 3
+        assert not set(page1).intersection(page2)
+        assert set(page1) | set(page2) == {"A", "B", "C"}
+
+        all_values, total = document_store.get_metadata_field_unique_values("category", from_=0, size=100)
+        assert set(all_values) == {"A", "B", "C"}
+        assert total == 3
+
+        beyond, total_beyond = document_store.get_metadata_field_unique_values("category", from_=10, size=10)
+        assert beyond == []
+        assert total_beyond == 3
+
+    def test_get_metadata_field_unique_values_invalid_field_name(self, document_store: IBMDb2DocumentStore):
+        with pytest.raises(ValueError, match="Invalid metadata field name"):
+            document_store.get_metadata_field_unique_values("field' OR '1'='1")
+
+    def test_get_metadata_field_unique_values_hyphenated_field_name(self, document_store: IBMDb2DocumentStore):
+        docs = [
+            Document(content="one", meta={"some-field": "A"}),
+            Document(content="two", meta={"some-field": "B"}),
+            Document(content="three", meta={"some-field": "A"}),
+        ]
+        document_store.write_documents(docs)
+
+        values, total_count = document_store.get_metadata_field_unique_values("some-field")
+        assert set(values) == {"A", "B"}
+        assert total_count == 2
+
+    def test_case_insensitive_and_special_characters_are_allowed(self, document_store: IBMDb2DocumentStore):
+        docs = [
+            Document(meta={"category": "100% complete"}),
+            Document(meta={"category": "plain"}),
+            Document(meta={"category": "under_score"}),
+            Document(meta={"category": "underXscore"}),
+        ]
+        document_store.write_documents(docs)
+
+        values, total = document_store.get_metadata_field_unique_values("category", search_term="%")
+        assert values == ["100% complete"]
+        assert total == 1
+
+        values, total = document_store.get_metadata_field_unique_values("category", search_term="_")
+        assert values == ["under_score"]
+        assert total == 1
+
 
 class TestIBMDb2DocumentStoreUtilMethods:
     """Unit tests for pure utility methods: _parse_embedding, _validate_embedding, _infer_field_type."""
@@ -300,6 +356,32 @@ class TestIBMDb2DocumentStoreUtilMethods:
     def test_infer_field_type(self, value, expected):
         assert IBMDb2DocumentStore._infer_field_type(value) == expected
 
+    # _normalize_metadata_field_name
+    def test_normalize_metadata_field_name_strips_meta_prefix(self):
+        assert IBMDb2DocumentStore._normalize_metadata_field_name("meta.category") == "category"
+
+    def test_normalize_metadata_field_name_without_prefix_unchanged(self):
+        assert IBMDb2DocumentStore._normalize_metadata_field_name("category") == "category"
+
+    def test_normalize_metadata_field_name_allows_underscore_and_hyphen(self):
+        assert IBMDb2DocumentStore._normalize_metadata_field_name("meta.some_field-name") == "some_field-name"
+
+    @pytest.mark.parametrize(
+        "bad_field",
+        [
+            "field@invalid",
+            "field with spaces",
+            "field;drop",
+            "meta.field!",
+            "field' OR '1'='1",
+            "../etc/passwd",
+            "field/* comment */",
+        ],
+    )
+    def test_normalize_metadata_field_name_rejects_invalid_chars(self, bad_field):
+        with pytest.raises(ValueError, match="Invalid metadata field name"):
+            IBMDb2DocumentStore._normalize_metadata_field_name(bad_field)
+
 
 @pytest.fixture
 def unit_store(monkeypatch) -> IBMDb2DocumentStore:
@@ -575,3 +657,57 @@ class TestIBMDb2DocumentStoreUnit:
 
         with pytest.raises(RuntimeError, match="connection reset"):
             store._embedding_retrieval([0.1, 0.2], top_k=1)
+
+    def test_get_metadata_field_unique_values_builds_paginated_sql(self, mocked_store):
+        store, _, cur = mocked_store
+        cur.fetchone.return_value = (2,)
+        # JSON_QUERY returns the literal JSON form of a string - quoted - unlike JSON_VALUE which
+        # dequotes it; json.loads() below relies on that to tell a string apart from a number.
+        cur.fetchall.return_value = [('"A"',), ('"B"',)]
+
+        values, total_count = store.get_metadata_field_unique_values("meta.category", from_=5, size=10)
+
+        assert values == ["A", "B"]
+        assert total_count == 2
+
+        count_sql, count_params = cur.execute.call_args_list[0].args
+        select_sql, select_params = cur.execute.call_args_list[1].args
+
+        assert "COUNT(DISTINCT value)" in count_sql
+        assert "$.category" in count_sql
+        assert count_params == []
+
+        assert "SELECT DISTINCT value" in select_sql
+        assert "ORDER BY value OFFSET ? ROWS FETCH FIRST ? ROWS ONLY" in select_sql
+        assert select_params == [5, 10]
+
+    def test_get_metadata_field_unique_values_applies_search_term(self, mocked_store):
+        store, _, cur = mocked_store
+        cur.fetchone.return_value = (1,)
+        cur.fetchall.return_value = [('"Python-based"',)]
+
+        store.get_metadata_field_unique_values("meta.category", search_term="python")
+
+        count_sql, count_params = cur.execute.call_args_list[0].args
+        select_sql, select_params = cur.execute.call_args_list[1].args
+
+        assert "LOCATE(UPPER(?), UPPER(value)) > 0" in count_sql
+        assert count_params == ["python"]
+        assert "LOCATE(UPPER(?), UPPER(value)) > 0" in select_sql
+        assert select_params == ["python", 0, 10]
+
+    def test_get_metadata_field_unique_values_rejects_invalid_field_name(self, mocked_store):
+        store, _, _ = mocked_store
+
+        with pytest.raises(ValueError, match="Invalid metadata field name"):
+            store.get_metadata_field_unique_values("field' OR '1'='1")
+
+    def test_get_metadata_field_unique_values_handles_no_rows(self, mocked_store):
+        store, _, cur = mocked_store
+        cur.fetchone.return_value = None
+        cur.fetchall.return_value = []
+
+        values, total_count = store.get_metadata_field_unique_values("meta.category")
+
+        assert values == []
+        assert total_count == 0