oracle-haystack 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oracle_haystack-0.4.0/CHANGELOG.md +26 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/PKG-INFO +1 -1
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/pyproject.toml +1 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/src/haystack_integrations/components/retrievers/oracle/embedding_retriever.py +6 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/src/haystack_integrations/components/retrievers/oracle/keyword_retriever.py +6 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/src/haystack_integrations/document_stores/oracle/document_store.py +35 -26
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/src/haystack_integrations/document_stores/oracle/filters.py +11 -8
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/tests/conftest.py +21 -16
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/tests/test_document_store.py +216 -77
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/tests/test_embedding_retriever.py +7 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/tests/test_filter_translator.py +24 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/tests/test_keyword_retriever.py +7 -0
- oracle_haystack-0.2.0/CHANGELOG.md +0 -9
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/.gitignore +0 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/README.md +0 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/docker-compose.yml +0 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/init/01_vector_memory.sql +0 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/pydoc/config_docusaurus.yml +0 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/src/haystack_integrations/components/retrievers/oracle/__init__.py +0 -0
- {oracle_haystack-0.2.0/src/haystack_integrations/components/retrievers/oracle → oracle_haystack-0.4.0/src/haystack_integrations/components/retrievers}/py.typed +0 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/src/haystack_integrations/document_stores/oracle/__about__.py +0 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/src/haystack_integrations/document_stores/oracle/__init__.py +0 -0
- {oracle_haystack-0.2.0/src/haystack_integrations/document_stores/oracle → oracle_haystack-0.4.0/src/haystack_integrations/document_stores}/py.typed +0 -0
- {oracle_haystack-0.2.0 → oracle_haystack-0.4.0}/tests/__init__.py +0 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [unreleased]
|
|
4
|
+
|
|
5
|
+
### 🧹 Chores
|
|
6
|
+
|
|
7
|
+
- Improve consistency of integrations folder structure (#3430)
|
|
8
|
+
|
|
9
|
+
### 🌀 Miscellaneous
|
|
10
|
+
|
|
11
|
+
- Validating meta fields to prevent SQL injection (#3511)
|
|
12
|
+
|
|
13
|
+
## [integrations/oracle-v0.2.0] - 2026-04-23
|
|
14
|
+
|
|
15
|
+
### 🚀 Features
|
|
16
|
+
|
|
17
|
+
- Complete `OracleDocumentStore` operations (#3179)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
## [integrations/oracle-v0.1.0] - 2026-04-16
|
|
21
|
+
|
|
22
|
+
### 🚀 Features
|
|
23
|
+
|
|
24
|
+
- Add Oracle AI Vector Search DocumentStore (oracle-haystack) (#3096)
|
|
25
|
+
|
|
26
|
+
<!-- generated by git-cliff -->
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: oracle-haystack
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Oracle AI Vector Search DocumentStore integration for Haystack
|
|
5
5
|
Project-URL: Source Code, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/oracle
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -84,6 +84,12 @@ class OracleEmbeddingRetriever:
|
|
|
84
84
|
)
|
|
85
85
|
return {"documents": docs}
|
|
86
86
|
|
|
87
|
+
def close(self) -> None:
|
|
88
|
+
"""
|
|
89
|
+
Release the synchronous resources of the underlying Document Store.
|
|
90
|
+
"""
|
|
91
|
+
self.document_store.close()
|
|
92
|
+
|
|
87
93
|
def to_dict(self) -> dict[str, Any]:
|
|
88
94
|
"""
|
|
89
95
|
Serializes the component to a dictionary.
|
|
@@ -82,6 +82,12 @@ class OracleKeywordRetriever:
|
|
|
82
82
|
)
|
|
83
83
|
return {"documents": docs}
|
|
84
84
|
|
|
85
|
+
def close(self) -> None:
|
|
86
|
+
"""
|
|
87
|
+
Release the synchronous resources of the underlying Document Store.
|
|
88
|
+
"""
|
|
89
|
+
self.document_store.close()
|
|
90
|
+
|
|
85
91
|
def to_dict(self) -> dict[str, Any]:
|
|
86
92
|
"""
|
|
87
93
|
Serializes the component to a dictionary.
|
|
@@ -8,6 +8,7 @@ import json
|
|
|
8
8
|
import logging
|
|
9
9
|
import re
|
|
10
10
|
import threading
|
|
11
|
+
from contextlib import suppress
|
|
11
12
|
from dataclasses import dataclass
|
|
12
13
|
from typing import Any, Literal
|
|
13
14
|
|
|
@@ -18,21 +19,14 @@ from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumen
|
|
|
18
19
|
from haystack.document_stores.types import DuplicatePolicy
|
|
19
20
|
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
20
21
|
|
|
21
|
-
from .filters import FilterTranslator
|
|
22
|
+
from .filters import FilterTranslator, _validate_field_path
|
|
22
23
|
|
|
23
24
|
logger = logging.getLogger(__name__)
|
|
24
25
|
|
|
25
26
|
_SAFE_TABLE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_$#]{0,127}$")
|
|
26
|
-
_SAFE_FIELD_PATH = re.compile(r"^[A-Za-z0-9_.]+$")
|
|
27
27
|
MAX_INDEX_NAME_LEN = 128
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
def _validate_field_path(field_path: str) -> None:
|
|
31
|
-
if not _SAFE_FIELD_PATH.match(field_path):
|
|
32
|
-
msg = f"Invalid metadata field name: {field_path!r}"
|
|
33
|
-
raise ValueError(msg)
|
|
34
|
-
|
|
35
|
-
|
|
36
30
|
def _try_parse_number(value: Any) -> Any:
|
|
37
31
|
"""
|
|
38
32
|
Attempt to parse a string as a number.
|
|
@@ -184,11 +178,7 @@ class OracleDocumentStore:
|
|
|
184
178
|
|
|
185
179
|
self._pool: oracledb.ConnectionPool | None = None
|
|
186
180
|
self._pool_lock = threading.Lock()
|
|
187
|
-
|
|
188
|
-
if create_table_if_not_exists:
|
|
189
|
-
self._ensure_table()
|
|
190
|
-
if create_index:
|
|
191
|
-
self.create_hnsw_index()
|
|
181
|
+
self._setup_done = False
|
|
192
182
|
|
|
193
183
|
def _get_pool(self) -> oracledb.ConnectionPool:
|
|
194
184
|
if self._pool is not None:
|
|
@@ -219,14 +209,33 @@ class OracleDocumentStore:
|
|
|
219
209
|
return self._pool
|
|
220
210
|
|
|
221
211
|
def _get_connection(self) -> oracledb.Connection:
|
|
212
|
+
self._ensure_setup()
|
|
222
213
|
return self._get_pool().acquire()
|
|
223
214
|
|
|
224
|
-
def
|
|
215
|
+
def _ensure_setup(self) -> None:
|
|
216
|
+
"""
|
|
217
|
+
Create the backing table and optional HNSW index once, on first use if needed.
|
|
218
|
+
"""
|
|
219
|
+
if self._setup_done:
|
|
220
|
+
return
|
|
221
|
+
self._setup_done = True
|
|
222
|
+
try:
|
|
223
|
+
if self.create_table_if_not_exists:
|
|
224
|
+
self._ensure_table()
|
|
225
|
+
if self.create_index:
|
|
226
|
+
self.create_hnsw_index()
|
|
227
|
+
except Exception:
|
|
228
|
+
self._setup_done = False
|
|
229
|
+
raise
|
|
230
|
+
|
|
231
|
+
def close(self) -> None:
|
|
232
|
+
"""
|
|
233
|
+
Release the associated synchronous resources.
|
|
234
|
+
"""
|
|
225
235
|
if self._pool is not None:
|
|
226
|
-
|
|
236
|
+
with suppress(Exception):
|
|
227
237
|
self._pool.close()
|
|
228
|
-
|
|
229
|
-
logger.warning("Failed to close Oracle connection pool during cleanup.", exc_info=True)
|
|
238
|
+
self._pool = None
|
|
230
239
|
|
|
231
240
|
def _ensure_table(self) -> None:
|
|
232
241
|
sql = f"""
|
|
@@ -773,17 +782,17 @@ class OracleDocumentStore:
|
|
|
773
782
|
return {"min": _try_parse_number(row[0]), "max": _try_parse_number(row[1])}
|
|
774
783
|
|
|
775
784
|
def get_metadata_field_unique_values(
|
|
776
|
-
self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int | None =
|
|
785
|
+
self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int | None = 10
|
|
777
786
|
) -> tuple[list[str], int]:
|
|
778
787
|
"""
|
|
779
788
|
Return a paginated list of distinct values for a metadata field, plus the total distinct count.
|
|
780
789
|
|
|
781
790
|
:param metadata_field: Metadata field name. May be prefixed with ``"meta."``
|
|
782
791
|
(e.g. ``"meta.lang"`` or ``"lang"``).
|
|
783
|
-
:param search_term: Optional substring filter applied to
|
|
792
|
+
:param search_term: Optional case-insensitive substring filter applied to the metadata field's own value.
|
|
784
793
|
:param from_: Zero-based offset for pagination. Defaults to ``0``.
|
|
785
|
-
:param size: Maximum number of values to return. When ``None`` all values
|
|
786
|
-
are returned.
|
|
794
|
+
:param size: Maximum number of values to return. Defaults to ``10``. When ``None`` all values
|
|
795
|
+
from ``from_`` onward are returned.
|
|
787
796
|
:returns: A tuple ``(values, total)`` where ``values`` is the paginated list of distinct field
|
|
788
797
|
values as strings and ``total`` is the overall distinct count (before pagination).
|
|
789
798
|
:raises ValueError: If ``metadata_field`` contains characters outside ``[A-Za-z0-9_.]``.
|
|
@@ -793,7 +802,7 @@ class OracleDocumentStore:
|
|
|
793
802
|
base_sql = f"FROM {self.table_name} WHERE JSON_VALUE(metadata, '$.{field_path}') IS NOT NULL"
|
|
794
803
|
params: dict[str, Any] = {}
|
|
795
804
|
if search_term:
|
|
796
|
-
base_sql += f" AND (
|
|
805
|
+
base_sql += f" AND UPPER(JSON_VALUE(metadata, '$.{field_path}')) LIKE UPPER(:search)"
|
|
797
806
|
params["search"] = f"%{search_term}%"
|
|
798
807
|
|
|
799
808
|
sql_count = f"SELECT COUNT(DISTINCT JSON_VALUE(metadata, '$.{field_path}')) {base_sql}"
|
|
@@ -841,17 +850,17 @@ class OracleDocumentStore:
|
|
|
841
850
|
return await asyncio.to_thread(self.get_metadata_field_min_max, metadata_field)
|
|
842
851
|
|
|
843
852
|
async def get_metadata_field_unique_values_async(
|
|
844
|
-
self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int | None =
|
|
853
|
+
self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int | None = 10
|
|
845
854
|
) -> tuple[list[str], int]:
|
|
846
855
|
"""
|
|
847
856
|
Asynchronously returns a paginated list of distinct values for a metadata field, plus the total count.
|
|
848
857
|
|
|
849
858
|
:param metadata_field: Metadata field name. May be prefixed with ``"meta."``
|
|
850
859
|
(e.g. ``"meta.lang"`` or ``"lang"``).
|
|
851
|
-
:param search_term: Optional substring filter applied to
|
|
860
|
+
:param search_term: Optional case-insensitive substring filter applied to the metadata field's own value.
|
|
852
861
|
:param from_: Zero-based offset for pagination. Defaults to ``0``.
|
|
853
|
-
:param size: Maximum number of values to return. When ``None`` all values
|
|
854
|
-
are returned.
|
|
862
|
+
:param size: Maximum number of values to return. Defaults to ``10``. When ``None`` all values
|
|
863
|
+
from ``from_`` onward are returned.
|
|
855
864
|
:returns: A tuple ``(values, total)`` where ``values`` is the paginated list of distinct field
|
|
856
865
|
values as strings and ``total`` is the overall distinct count (before pagination).
|
|
857
866
|
:raises ValueError: If ``metadata_field`` contains characters outside ``[A-Za-z0-9_.]``.
|
|
@@ -2,12 +2,20 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
+
import re
|
|
5
6
|
from datetime import datetime
|
|
6
7
|
from typing import Any, ClassVar
|
|
7
8
|
|
|
8
9
|
from haystack.errors import FilterError
|
|
9
10
|
|
|
10
11
|
_RANGE_OPS = {">", ">=", "<", "<="}
|
|
12
|
+
_SAFE_FIELD_PATH = re.compile(r"^[A-Za-z0-9_.]+$")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _validate_field_path(field_path: str) -> None:
|
|
16
|
+
if not _SAFE_FIELD_PATH.match(field_path):
|
|
17
|
+
msg = f"Invalid metadata field name: {field_path!r}"
|
|
18
|
+
raise FilterError(msg)
|
|
11
19
|
|
|
12
20
|
|
|
13
21
|
class FilterTranslator:
|
|
@@ -123,14 +131,9 @@ class FilterTranslator:
|
|
|
123
131
|
return "id"
|
|
124
132
|
if field == "content":
|
|
125
133
|
return "text"
|
|
126
|
-
if field.startswith("meta.")
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
130
|
-
return f"TO_NUMBER({json_path})"
|
|
131
|
-
return json_path
|
|
132
|
-
# Fallback: treat as top-level JSON key
|
|
133
|
-
json_path = f"JSON_VALUE(metadata, '$.{field}')"
|
|
134
|
+
key = field[len("meta.") :] if field.startswith("meta.") else field
|
|
135
|
+
_validate_field_path(key)
|
|
136
|
+
json_path = f"JSON_VALUE(metadata, '$.{key}')"
|
|
134
137
|
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
135
138
|
return f"TO_NUMBER({json_path})"
|
|
136
139
|
return json_path
|
|
@@ -16,25 +16,30 @@ _PASSWORD = "haystack"
|
|
|
16
16
|
_DSN = "localhost:1521/freepdb1"
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
19
|
+
@pytest.fixture
|
|
20
|
+
def make_store():
|
|
21
|
+
"""Factory for a real OracleDocumentStore. Uses token secrets, so instances are not serialization-safe."""
|
|
22
|
+
|
|
23
|
+
def _make(table: str, embedding_dim: int = 4, *, create_table_if_not_exists: bool = True) -> OracleDocumentStore:
|
|
24
|
+
return OracleDocumentStore(
|
|
25
|
+
connection_config=OracleConnectionConfig(
|
|
26
|
+
user=Secret.from_token(_USER),
|
|
27
|
+
password=Secret.from_token(_PASSWORD),
|
|
28
|
+
dsn=Secret.from_token(_DSN),
|
|
29
|
+
),
|
|
30
|
+
table_name=table,
|
|
31
|
+
embedding_dim=embedding_dim,
|
|
32
|
+
create_table_if_not_exists=create_table_if_not_exists,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
return _make
|
|
31
36
|
|
|
32
37
|
|
|
33
38
|
@pytest.fixture
|
|
34
|
-
def document_store():
|
|
39
|
+
def document_store(make_store):
|
|
35
40
|
"""768-dim store required by the mixin's filterable_docs fixture."""
|
|
36
41
|
table = f"hs_sync_{uuid.uuid4().hex[:8]}"
|
|
37
|
-
s =
|
|
42
|
+
s = make_store(table, 768)
|
|
38
43
|
yield s
|
|
39
44
|
with s._get_connection() as conn, conn.cursor() as cur:
|
|
40
45
|
cur.execute(f"DROP TABLE {table} PURGE")
|
|
@@ -42,10 +47,10 @@ def document_store():
|
|
|
42
47
|
|
|
43
48
|
|
|
44
49
|
@pytest.fixture
|
|
45
|
-
def embedding_store():
|
|
50
|
+
def embedding_store(make_store):
|
|
46
51
|
"""4-dim store for embedding-retrieval, HNSW, and async tests."""
|
|
47
52
|
table = f"hs_emb_{uuid.uuid4().hex[:8]}"
|
|
48
|
-
s =
|
|
53
|
+
s = make_store(table)
|
|
49
54
|
yield s
|
|
50
55
|
with s._get_connection() as conn, conn.cursor() as cur:
|
|
51
56
|
cur.execute(f"DROP TABLE {table} PURGE")
|
|
@@ -3,11 +3,12 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
import uuid
|
|
6
|
+
from unittest.mock import MagicMock
|
|
6
7
|
|
|
7
8
|
import oracledb as _oracledb
|
|
8
9
|
import pytest
|
|
9
10
|
from haystack.dataclasses import Document
|
|
10
|
-
from haystack.document_stores.errors import DuplicateDocumentError
|
|
11
|
+
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
11
12
|
from haystack.document_stores.types import DuplicatePolicy
|
|
12
13
|
from haystack.testing.document_store import (
|
|
13
14
|
CountDocumentsByFilterTest,
|
|
@@ -27,13 +28,8 @@ from haystack.testing.document_store_async import (
|
|
|
27
28
|
FilterableDocsFixtureMixin,
|
|
28
29
|
UpdateByFilterAsyncTest,
|
|
29
30
|
)
|
|
30
|
-
from haystack.utils import Secret
|
|
31
31
|
|
|
32
|
-
from haystack_integrations.document_stores.oracle import
|
|
33
|
-
|
|
34
|
-
_USER = "haystack"
|
|
35
|
-
_PASSWORD = "haystack"
|
|
36
|
-
_DSN = "localhost:1521/freepdb1"
|
|
32
|
+
from haystack_integrations.document_stores.oracle import OracleDocumentStore
|
|
37
33
|
|
|
38
34
|
|
|
39
35
|
def _doc(doc_id: str, content: str = "hello", meta: dict | None = None, embedding: list[float] | None = None):
|
|
@@ -47,72 +43,54 @@ def _uid(suffix: str = "") -> str:
|
|
|
47
43
|
return f"{base}{suffix.upper():>4}"[:32]
|
|
48
44
|
|
|
49
45
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
GetMetadataFieldMinMaxTest,
|
|
59
|
-
GetMetadataFieldsInfoTest,
|
|
60
|
-
GetMetadataFieldUniqueValuesTest,
|
|
61
|
-
UpdateByFilterTest,
|
|
62
|
-
):
|
|
46
|
+
def _lob(value: str) -> MagicMock:
|
|
47
|
+
"""Mock for an oracledb Large Object (LOB)"""
|
|
48
|
+
lob = MagicMock()
|
|
49
|
+
lob.read.return_value = value
|
|
50
|
+
return lob
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TestOracleDocumentStoreUnit:
|
|
63
54
|
@staticmethod
|
|
64
55
|
def _mock_doc(content="hello", embedding=None, doc_id="AABB" * 8):
|
|
65
56
|
"""Lightweight document builder for mock-based tests."""
|
|
66
57
|
return Document(id=doc_id, content=content, meta={"k": "v"}, embedding=embedding)
|
|
67
58
|
|
|
68
|
-
@pytest.
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
dsn=Secret.from_token(_DSN),
|
|
77
|
-
),
|
|
78
|
-
table_name=table,
|
|
79
|
-
embedding_dim=768,
|
|
80
|
-
distance_metric="COSINE",
|
|
81
|
-
create_table_if_not_exists=True,
|
|
82
|
-
)
|
|
83
|
-
yield s
|
|
84
|
-
with s._get_connection() as conn, conn.cursor() as cur:
|
|
85
|
-
cur.execute(f"DROP TABLE {table} PURGE")
|
|
86
|
-
conn.commit()
|
|
87
|
-
|
|
88
|
-
# Mixin override
|
|
89
|
-
def assert_documents_are_equal(self, received: list[Document], expected: list[Document]) -> None:
|
|
90
|
-
# filter_documents does not SELECT the embedding column — ignore it when comparing
|
|
91
|
-
assert len(received) == len(expected)
|
|
92
|
-
received_sorted = sorted(received, key=lambda d: d.id)
|
|
93
|
-
expected_sorted = sorted(expected, key=lambda d: d.id)
|
|
94
|
-
for r, e in zip(received_sorted, expected_sorted, strict=True):
|
|
95
|
-
assert r.id == e.id
|
|
96
|
-
assert r.content == e.content
|
|
97
|
-
assert r.meta == e.meta
|
|
98
|
-
|
|
99
|
-
# Mixin override
|
|
100
|
-
def test_write_documents(self, document_store: OracleDocumentStore) -> None:
|
|
101
|
-
# Default policy is NONE — a second write of the same doc raises DuplicateDocumentError
|
|
102
|
-
doc = Document(content="test doc")
|
|
103
|
-
assert document_store.write_documents([doc]) == 1
|
|
104
|
-
with pytest.raises(DuplicateDocumentError):
|
|
105
|
-
document_store.write_documents([doc])
|
|
106
|
-
self.assert_documents_are_equal(document_store.filter_documents(), [doc])
|
|
107
|
-
|
|
108
|
-
# test_comparison_equal_with_none → IS NULL
|
|
109
|
-
# test_comparison_not_equal_with_none → IS NOT NULL
|
|
110
|
-
# test_comparison_not_equal → col != x OR IS NULL
|
|
111
|
-
# test_comparison_not_in → IS NULL OR NOT IN
|
|
112
|
-
@pytest.mark.skip(
|
|
113
|
-
reason="Oracle NULL propagation in NOT(...) cannot match Python 'not (None == x) is True' semantics"
|
|
59
|
+
@pytest.mark.parametrize(
|
|
60
|
+
("table_name", "embedding_dim", "match"),
|
|
61
|
+
[
|
|
62
|
+
pytest.param("1invalid", 4, "Invalid table_name", id="table_name_starts_with_digit"),
|
|
63
|
+
pytest.param("bad-name", 4, "Invalid table_name", id="table_name_illegal_char"),
|
|
64
|
+
pytest.param("valid", 0, "embedding_dim must be a positive integer", id="embedding_dim_zero"),
|
|
65
|
+
pytest.param("valid", -1, "embedding_dim must be a positive integer", id="embedding_dim_negative"),
|
|
66
|
+
],
|
|
114
67
|
)
|
|
115
|
-
def
|
|
68
|
+
def test_init_rejects_invalid_arguments(self, make_store, table_name, embedding_dim, match):
|
|
69
|
+
with pytest.raises(ValueError, match=match):
|
|
70
|
+
make_store(table_name, embedding_dim)
|
|
71
|
+
|
|
72
|
+
def test_init_opens_no_connection(self, make_store, mock_pool): # noqa: ARG002
|
|
73
|
+
store = make_store("lazy_docs")
|
|
74
|
+
assert store._pool is None
|
|
75
|
+
assert store._setup_done is False
|
|
76
|
+
|
|
77
|
+
def test_setup_is_deferred_and_runs_once(self, make_store, mock_pool):
|
|
78
|
+
store = make_store("lazy_docs")
|
|
79
|
+
store.count_documents()
|
|
80
|
+
store.count_documents()
|
|
81
|
+
assert store._pool is not None
|
|
82
|
+
assert store._setup_done is True
|
|
83
|
+
_, _, cursor = mock_pool
|
|
84
|
+
create_calls = [c for c in cursor.execute.call_args_list if "CREATE TABLE IF NOT EXISTS" in c[0][0]]
|
|
85
|
+
assert len(create_calls) == 1
|
|
86
|
+
|
|
87
|
+
def test_setup_failure_resets_flag(self, make_store, mock_pool):
|
|
88
|
+
_, _, cursor = mock_pool
|
|
89
|
+
cursor.execute.side_effect = RuntimeError("boom")
|
|
90
|
+
store = make_store("lazy_docs")
|
|
91
|
+
with pytest.raises(RuntimeError):
|
|
92
|
+
store.count_documents()
|
|
93
|
+
assert store._setup_done is False
|
|
116
94
|
|
|
117
95
|
def test_write_documents_none_policy_calls_insert(self, patched_store, mock_pool):
|
|
118
96
|
_, _, cursor = mock_pool
|
|
@@ -207,6 +185,78 @@ class TestOracleDocumentStore(
|
|
|
207
185
|
cursor.fetchone.return_value = (42,)
|
|
208
186
|
assert patched_store.count_documents() == 42
|
|
209
187
|
|
|
188
|
+
def test_get_metadata_field_unique_values_search_term_filters_value_only(self, patched_store, mock_pool):
|
|
189
|
+
"""search_term must filter on the metadata field's own JSON value, not on document text."""
|
|
190
|
+
_, _, cursor = mock_pool
|
|
191
|
+
cursor.fetchone.return_value = (1,)
|
|
192
|
+
cursor.fetchall.return_value = [("bar",)]
|
|
193
|
+
|
|
194
|
+
patched_store.get_metadata_field_unique_values("category", search_term="bar")
|
|
195
|
+
|
|
196
|
+
count_sql, count_params = cursor.execute.call_args_list[0][0]
|
|
197
|
+
vals_sql, vals_params = cursor.execute.call_args_list[1][0]
|
|
198
|
+
|
|
199
|
+
for sql in (count_sql, vals_sql):
|
|
200
|
+
assert "UPPER(JSON_VALUE(metadata, '$.category')) LIKE UPPER(:search)" in sql
|
|
201
|
+
|
|
202
|
+
assert count_params["search"] == "%bar%"
|
|
203
|
+
assert vals_params["search"] == "%bar%"
|
|
204
|
+
|
|
205
|
+
def test_delete_table_executes_drop_and_index_sql(self, patched_store, mock_pool):
|
|
206
|
+
_, _, cursor = mock_pool
|
|
207
|
+
patched_store.delete_table()
|
|
208
|
+
executed = [c[0][0] for c in cursor.execute.call_args_list]
|
|
209
|
+
assert any("DROP TABLE" in sql and "PURGE" in sql for sql in executed)
|
|
210
|
+
assert any("DBMS_SEARCH.DROP_INDEX" in sql for sql in executed)
|
|
211
|
+
|
|
212
|
+
def test_keyword_retrieval_builds_contains_query(self, patched_store, mock_pool):
|
|
213
|
+
_, _, cursor = mock_pool
|
|
214
|
+
cursor.fetchall.return_value = [("AABB" * 8, "hello", '{"k": "v"}', 1.5)]
|
|
215
|
+
docs = patched_store._keyword_retrieval("hello world", top_k=5)
|
|
216
|
+
sql, params = cursor.execute.call_args[0]
|
|
217
|
+
assert "CONTAINS(DATA, :query, 1)" in sql
|
|
218
|
+
assert "SCORE(1)" in sql
|
|
219
|
+
assert params["query"] == "hello world"
|
|
220
|
+
assert params["top_k"] == 5
|
|
221
|
+
assert docs[0].score == 1.5
|
|
222
|
+
|
|
223
|
+
@pytest.mark.parametrize(
|
|
224
|
+
"operation",
|
|
225
|
+
[
|
|
226
|
+
pytest.param(lambda s: s._embedding_retrieval([0.1, 0.2, 0.3, 0.4]), id="embedding_retrieval"),
|
|
227
|
+
pytest.param(lambda s: s._keyword_retrieval("query"), id="keyword_retrieval"),
|
|
228
|
+
pytest.param(lambda s: s.delete_table(), id="delete_table"),
|
|
229
|
+
],
|
|
230
|
+
)
|
|
231
|
+
def test_query_methods_wrap_database_errors(self, patched_store, mock_pool, operation):
|
|
232
|
+
_, _, cursor = mock_pool
|
|
233
|
+
cursor.execute.side_effect = _oracledb.DatabaseError("ORA-00942")
|
|
234
|
+
with pytest.raises(DocumentStoreError):
|
|
235
|
+
operation(patched_store)
|
|
236
|
+
|
|
237
|
+
@pytest.mark.parametrize(
|
|
238
|
+
("text_raw", "metadata_raw", "expected_content", "expected_meta"),
|
|
239
|
+
[
|
|
240
|
+
pytest.param("plain", '{"k": "v"}', "plain", {"k": "v"}, id="str_text_and_json_metadata"),
|
|
241
|
+
pytest.param(_lob("from_lob"), _lob('{"k": "v"}'), "from_lob", {"k": "v"}, id="lob_text_and_metadata"),
|
|
242
|
+
pytest.param("plain", {"k": "v"}, "plain", {"k": "v"}, id="dict_metadata"),
|
|
243
|
+
pytest.param("plain", None, "plain", {}, id="unsupported_metadata_type"),
|
|
244
|
+
],
|
|
245
|
+
)
|
|
246
|
+
def test_row_to_document_reads_lob_and_maps_metadata(self, text_raw, metadata_raw, expected_content, expected_meta):
|
|
247
|
+
doc = OracleDocumentStore._row_to_document(("id1", text_raw, metadata_raw))
|
|
248
|
+
assert doc.content == expected_content
|
|
249
|
+
assert doc.meta == expected_meta
|
|
250
|
+
|
|
251
|
+
def test_create_hnsw_index_sql(self, patched_store, mock_pool):
|
|
252
|
+
_, _, cursor = mock_pool
|
|
253
|
+
patched_store.create_hnsw_index()
|
|
254
|
+
sql = cursor.execute.call_args[0][0]
|
|
255
|
+
assert "CREATE VECTOR INDEX" in sql
|
|
256
|
+
assert "HNSW" in sql
|
|
257
|
+
assert str(patched_store.hnsw_neighbors) in sql
|
|
258
|
+
assert str(patched_store.hnsw_ef_construction) in sql
|
|
259
|
+
|
|
210
260
|
def test_to_dict_does_not_expose_plain_password(self, patched_store):
|
|
211
261
|
d = patched_store.to_dict()
|
|
212
262
|
pw = d["init_parameters"]["connection_config"]["password"]
|
|
@@ -220,17 +270,74 @@ class TestOracleDocumentStore(
|
|
|
220
270
|
assert restored.embedding_dim == patched_store.embedding_dim
|
|
221
271
|
assert restored.distance_metric == patched_store.distance_metric
|
|
222
272
|
|
|
223
|
-
def
|
|
224
|
-
|
|
225
|
-
patched_store.
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
assert
|
|
229
|
-
|
|
230
|
-
|
|
273
|
+
def test_close(self, patched_store):
|
|
274
|
+
pool = MagicMock()
|
|
275
|
+
patched_store._pool = pool
|
|
276
|
+
patched_store.close()
|
|
277
|
+
pool.close.assert_called_once()
|
|
278
|
+
assert patched_store._pool is None
|
|
279
|
+
patched_store.close()
|
|
280
|
+
pool.close.assert_called_once()
|
|
281
|
+
|
|
282
|
+
def test_close_is_exception_safe(self, patched_store):
|
|
283
|
+
pool = MagicMock()
|
|
284
|
+
pool.close.side_effect = RuntimeError("boom")
|
|
285
|
+
patched_store._pool = pool
|
|
286
|
+
patched_store.close()
|
|
287
|
+
assert patched_store._pool is None
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
@pytest.mark.integration
|
|
291
|
+
class TestOracleDocumentStore(
|
|
292
|
+
DocumentStoreBaseTests,
|
|
293
|
+
CountDocumentsByFilterTest,
|
|
294
|
+
CountUniqueMetadataByFilterTest,
|
|
295
|
+
DeleteAllTest,
|
|
296
|
+
DeleteByFilterTest,
|
|
297
|
+
DeleteDocumentsTest,
|
|
298
|
+
GetMetadataFieldMinMaxTest,
|
|
299
|
+
GetMetadataFieldsInfoTest,
|
|
300
|
+
GetMetadataFieldUniqueValuesTest,
|
|
301
|
+
UpdateByFilterTest,
|
|
302
|
+
):
|
|
303
|
+
@pytest.fixture
|
|
304
|
+
def document_store(self, make_store):
|
|
305
|
+
"""768-dim store — overrides the mixin's NotImplementedError stub."""
|
|
306
|
+
table = f"hs_sync_{uuid.uuid4().hex[:8]}"
|
|
307
|
+
s = make_store(table, 768)
|
|
308
|
+
yield s
|
|
309
|
+
with s._get_connection() as conn, conn.cursor() as cur:
|
|
310
|
+
cur.execute(f"DROP TABLE {table} PURGE")
|
|
311
|
+
conn.commit()
|
|
312
|
+
|
|
313
|
+
# Mixin override
|
|
314
|
+
def assert_documents_are_equal(self, received: list[Document], expected: list[Document]) -> None:
|
|
315
|
+
# filter_documents does not SELECT the embedding column — ignore it when comparing
|
|
316
|
+
assert len(received) == len(expected)
|
|
317
|
+
received_sorted = sorted(received, key=lambda d: d.id)
|
|
318
|
+
expected_sorted = sorted(expected, key=lambda d: d.id)
|
|
319
|
+
for r, e in zip(received_sorted, expected_sorted, strict=True):
|
|
320
|
+
assert r.id == e.id
|
|
321
|
+
assert r.content == e.content
|
|
322
|
+
assert r.meta == e.meta
|
|
231
323
|
|
|
232
|
-
|
|
233
|
-
|
|
324
|
+
# Mixin override
|
|
325
|
+
def test_write_documents(self, document_store: OracleDocumentStore) -> None:
|
|
326
|
+
# Default policy is NONE — a second write of the same doc raises DuplicateDocumentError
|
|
327
|
+
doc = Document(content="test doc")
|
|
328
|
+
assert document_store.write_documents([doc]) == 1
|
|
329
|
+
with pytest.raises(DuplicateDocumentError):
|
|
330
|
+
document_store.write_documents([doc])
|
|
331
|
+
self.assert_documents_are_equal(document_store.filter_documents(), [doc])
|
|
332
|
+
|
|
333
|
+
# test_comparison_equal_with_none → IS NULL
|
|
334
|
+
# test_comparison_not_equal_with_none → IS NOT NULL
|
|
335
|
+
# test_comparison_not_equal → col != x OR IS NULL
|
|
336
|
+
# test_comparison_not_in → IS NULL OR NOT IN
|
|
337
|
+
@pytest.mark.skip(
|
|
338
|
+
reason="Oracle NULL propagation in NOT(...) cannot match Python 'not (None == x) is True' semantics"
|
|
339
|
+
)
|
|
340
|
+
def test_not_operator(self, document_store, filterable_docs): ...
|
|
234
341
|
|
|
235
342
|
def test_filter_documents_not_operator(self, document_store):
|
|
236
343
|
# Scoped to two fresh docs — no NULL-valued rows, so NOT works correctly.
|
|
@@ -311,6 +418,38 @@ class TestOracleDocumentStore(
|
|
|
311
418
|
"""Calling _ensure_table() a second time must not raise."""
|
|
312
419
|
document_store._ensure_table()
|
|
313
420
|
|
|
421
|
+
def test_close_and_reopen(self, document_store):
|
|
422
|
+
document_store.count_documents()
|
|
423
|
+
assert document_store._pool is not None
|
|
424
|
+
document_store.close()
|
|
425
|
+
assert document_store._pool is None
|
|
426
|
+
assert document_store.count_documents() == 0
|
|
427
|
+
assert document_store._pool is not None
|
|
428
|
+
|
|
429
|
+
def test_get_metadata_field_unique_values_search_term_matches_value_not_content(self, document_store):
|
|
430
|
+
"""search_term filters on the metadata field's own value; document content is not considered."""
|
|
431
|
+
document_store.write_documents(
|
|
432
|
+
[
|
|
433
|
+
_doc(_uid("Q001"), content="this text mentions apple", meta={"category": "dessert"}),
|
|
434
|
+
_doc(_uid("Q002"), content="unrelated content", meta={"category": "apple-tart"}),
|
|
435
|
+
]
|
|
436
|
+
)
|
|
437
|
+
values, total = document_store.get_metadata_field_unique_values("category", search_term="apple")
|
|
438
|
+
# Q001: content contains "apple" but its category value ("dessert") does not -> excluded.
|
|
439
|
+
# Q002: category value ("apple-tart") contains "apple", content does not -> still included.
|
|
440
|
+
assert values == ["apple-tart"]
|
|
441
|
+
assert total == 1
|
|
442
|
+
|
|
443
|
+
def test_get_metadata_field_unique_values_search_term_case_insensitive(self, document_store):
|
|
444
|
+
document_store.write_documents(
|
|
445
|
+
[
|
|
446
|
+
_doc(_uid("Q003"), content="n/a", meta={"category": "Apple-Tart"}),
|
|
447
|
+
]
|
|
448
|
+
)
|
|
449
|
+
values, total = document_store.get_metadata_field_unique_values("category", search_term="APPLE")
|
|
450
|
+
assert values == ["Apple-Tart"]
|
|
451
|
+
assert total == 1
|
|
452
|
+
|
|
314
453
|
|
|
315
454
|
@pytest.mark.integration
|
|
316
455
|
class TestOracleDocumentStoreAsync(
|
|
@@ -67,6 +67,13 @@ def test_to_dict_from_dict_roundtrip(mock_store):
|
|
|
67
67
|
assert restored.document_store.embedding_dim == 4
|
|
68
68
|
|
|
69
69
|
|
|
70
|
+
def test_close(mock_store):
|
|
71
|
+
retriever = OracleEmbeddingRetriever(document_store=mock_store)
|
|
72
|
+
retriever.close()
|
|
73
|
+
mock_store.close.assert_called_once()
|
|
74
|
+
assert retriever.document_store is mock_store
|
|
75
|
+
|
|
76
|
+
|
|
70
77
|
def test_invalid_document_store_raises_type_error():
|
|
71
78
|
with pytest.raises(TypeError, match="must be an instance of OracleDocumentStore"):
|
|
72
79
|
OracleEmbeddingRetriever(document_store="not_a_store")
|
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
+
import pytest
|
|
6
|
+
from haystack.errors import FilterError
|
|
7
|
+
|
|
5
8
|
from haystack_integrations.document_stores.oracle.filters import FilterTranslator
|
|
6
9
|
|
|
7
10
|
|
|
@@ -134,3 +137,24 @@ def test_param_counter_increments_correctly():
|
|
|
134
137
|
}
|
|
135
138
|
)
|
|
136
139
|
assert set(params.keys()) == {"p0", "p1", "p2"}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_sql_injection_via_field_name_raises():
|
|
143
|
+
with pytest.raises(FilterError, match="Invalid metadata field name"):
|
|
144
|
+
_translate({"field": "meta.x') = 'x' OR 1=1 OR ('a", "operator": "==", "value": "y"})
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_sql_injection_via_fallback_field_raises():
|
|
148
|
+
# Injection via the non-meta fallback path
|
|
149
|
+
with pytest.raises(FilterError, match="Invalid metadata field name"):
|
|
150
|
+
_translate({"field": "x') OR 1=1--", "operator": "==", "value": "y"})
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def test_sql_injection_in_not_in_raises():
|
|
154
|
+
with pytest.raises(FilterError, match="Invalid metadata field name"):
|
|
155
|
+
_translate({"field": "meta.x') OR 1=1--", "operator": "in", "value": ["a", "b"]})
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_valid_nested_field_path_accepted():
|
|
159
|
+
sql, _ = _translate({"field": "meta.author.city", "operator": "==", "value": "Berlin"})
|
|
160
|
+
assert "'$.author.city'" in sql
|
|
@@ -67,6 +67,13 @@ def test_to_dict_from_dict_roundtrip(mock_store):
|
|
|
67
67
|
assert restored.document_store.embedding_dim == 4
|
|
68
68
|
|
|
69
69
|
|
|
70
|
+
def test_close(mock_store):
|
|
71
|
+
retriever = OracleKeywordRetriever(document_store=mock_store)
|
|
72
|
+
retriever.close()
|
|
73
|
+
mock_store.close.assert_called_once()
|
|
74
|
+
assert retriever.document_store is mock_store
|
|
75
|
+
|
|
76
|
+
|
|
70
77
|
def test_invalid_document_store_raises_type_error():
|
|
71
78
|
with pytest.raises(TypeError, match="must be an instance of OracleDocumentStore"):
|
|
72
79
|
OracleKeywordRetriever(document_store="not_a_store")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|