opensolr-haystack 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opensolr_haystack-0.1.0/LICENSE +21 -0
- opensolr_haystack-0.1.0/PKG-INFO +95 -0
- opensolr_haystack-0.1.0/README.md +77 -0
- opensolr_haystack-0.1.0/haystack_integrations/components/retrievers/opensolr/__init__.py +7 -0
- opensolr_haystack-0.1.0/haystack_integrations/components/retrievers/opensolr/retriever.py +81 -0
- opensolr_haystack-0.1.0/haystack_integrations/document_stores/opensolr/__init__.py +6 -0
- opensolr_haystack-0.1.0/haystack_integrations/document_stores/opensolr/client.py +227 -0
- opensolr_haystack-0.1.0/haystack_integrations/document_stores/opensolr/store.py +287 -0
- opensolr_haystack-0.1.0/opensolr_haystack.egg-info/PKG-INFO +95 -0
- opensolr_haystack-0.1.0/opensolr_haystack.egg-info/SOURCES.txt +13 -0
- opensolr_haystack-0.1.0/opensolr_haystack.egg-info/dependency_links.txt +1 -0
- opensolr_haystack-0.1.0/opensolr_haystack.egg-info/requires.txt +2 -0
- opensolr_haystack-0.1.0/opensolr_haystack.egg-info/top_level.txt +1 -0
- opensolr_haystack-0.1.0/pyproject.toml +28 -0
- opensolr_haystack-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Opensolr SRL
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: opensolr-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Haystack integration for Opensolr — managed Apache Solr DocumentStore with server-side embeddings and hybrid BM25+kNN retrieval
|
|
5
|
+
Author-email: Opensolr <support@opensolr.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://opensolr.com/langchain
|
|
8
|
+
Project-URL: Repository, https://github.com/phpcip/opensolr-haystack
|
|
9
|
+
Keywords: haystack,opensolr,solr,vector,hybrid-search,rag,document-store
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Requires-Python: >=3.9
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: haystack-ai>=2.0.0
|
|
16
|
+
Requires-Dist: httpx>=0.25.0
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# opensolr-haystack
|
|
20
|
+
|
|
21
|
+
[Haystack](https://haystack.deepset.ai) integration for
|
|
22
|
+
[Opensolr](https://opensolr.com) — managed Apache Solr as a DocumentStore,
|
|
23
|
+
with **server-side embeddings** and native **hybrid (BM25 + kNN) retrieval**.
|
|
24
|
+
|
|
25
|
+
No embedder components needed in your pipeline — texts and queries are
|
|
26
|
+
embedded on Opensolr's GPU infrastructure (multilingual E5-large-instruct,
|
|
27
|
+
1024 dimensions, cosine).
|
|
28
|
+
|
|
29
|
+
**Product page:** [opensolr.com/langchain](https://opensolr.com/langchain) ·
|
|
30
|
+
free 15-day trial, no card, at [opensolr.com](https://opensolr.com)
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install opensolr-haystack
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Quickstart
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from haystack import Document, Pipeline
|
|
40
|
+
from haystack_integrations.document_stores.opensolr import OpensolrDocumentStore
|
|
41
|
+
from haystack_integrations.components.retrievers.opensolr import OpensolrHybridRetriever
|
|
42
|
+
|
|
43
|
+
# credentials default to OPENSOLR_EMAIL / OPENSOLR_API_KEY env vars
|
|
44
|
+
store = OpensolrDocumentStore(index="mysite__dense", create_if_missing=True)
|
|
45
|
+
|
|
46
|
+
store.write_documents([
|
|
47
|
+
Document(content="Hybrid search fuses BM25 with vector similarity"),
|
|
48
|
+
Document(content="Cats sleep sixteen hours a day"),
|
|
49
|
+
])
|
|
50
|
+
|
|
51
|
+
pipe = Pipeline()
|
|
52
|
+
pipe.add_component("retriever", OpensolrHybridRetriever(document_store=store))
|
|
53
|
+
result = pipe.run({"retriever": {"query": "how do keyword and semantic search combine?"}})
|
|
54
|
+
print(result["retriever"]["documents"])
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Note there is **no embedder** in the pipeline — not for documents, not for
|
|
58
|
+
the query. The store embeds server-side at both index and query time.
|
|
59
|
+
|
|
60
|
+
## Hybrid retrieval
|
|
61
|
+
|
|
62
|
+
`OpensolrHybridRetriever` fuses BM25 and kNN scores per document via
|
|
63
|
+
Opensolr's native `{!hybrid}` Solr query parser:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
OpensolrHybridRetriever(
|
|
67
|
+
document_store=store,
|
|
68
|
+
top_k=10,
|
|
69
|
+
hybrid=True, # False = pure semantic kNN
|
|
70
|
+
alpha=0.5, # 0 = all semantic … 1 = all lexical
|
|
71
|
+
)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Standard Haystack filters are supported and map to Solr `fq`:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
pipe.run({"retriever": {
|
|
78
|
+
"query": "search engines",
|
|
79
|
+
"filters": {"field": "meta.category", "operator": "==", "value": "docs"},
|
|
80
|
+
}})
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Notes
|
|
84
|
+
|
|
85
|
+
- Vector-enabled indexes run on Opensolr's Solr 9.x environments — currently
|
|
86
|
+
`us` (Chicago), `de` (Germany), `fi` (Finland). **Additional dedicated
|
|
87
|
+
regions can be deployed on request** (paid add-on):
|
|
88
|
+
[support@opensolr.com](mailto:support@opensolr.com).
|
|
89
|
+
- Every index is also plain Apache Solr with the native `/select` API —
|
|
90
|
+
facets, highlighting, spellcheck included.
|
|
91
|
+
- Siblings: [`langchain-opensolr`](https://pypi.org/project/langchain-opensolr/) ·
|
|
92
|
+
[`llama-index-opensolr`](https://pypi.org/project/llama-index-opensolr/) ·
|
|
93
|
+
[`opensolr-mcp`](https://pypi.org/project/opensolr-mcp/)
|
|
94
|
+
|
|
95
|
+
MIT license.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# opensolr-haystack
|
|
2
|
+
|
|
3
|
+
[Haystack](https://haystack.deepset.ai) integration for
|
|
4
|
+
[Opensolr](https://opensolr.com) — managed Apache Solr as a DocumentStore,
|
|
5
|
+
with **server-side embeddings** and native **hybrid (BM25 + kNN) retrieval**.
|
|
6
|
+
|
|
7
|
+
No embedder components needed in your pipeline — texts and queries are
|
|
8
|
+
embedded on Opensolr's GPU infrastructure (multilingual E5-large-instruct,
|
|
9
|
+
1024 dimensions, cosine).
|
|
10
|
+
|
|
11
|
+
**Product page:** [opensolr.com/langchain](https://opensolr.com/langchain) ·
|
|
12
|
+
free 15-day trial, no card, at [opensolr.com](https://opensolr.com)
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install opensolr-haystack
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Quickstart
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from haystack import Document, Pipeline
|
|
22
|
+
from haystack_integrations.document_stores.opensolr import OpensolrDocumentStore
|
|
23
|
+
from haystack_integrations.components.retrievers.opensolr import OpensolrHybridRetriever
|
|
24
|
+
|
|
25
|
+
# credentials default to OPENSOLR_EMAIL / OPENSOLR_API_KEY env vars
|
|
26
|
+
store = OpensolrDocumentStore(index="mysite__dense", create_if_missing=True)
|
|
27
|
+
|
|
28
|
+
store.write_documents([
|
|
29
|
+
Document(content="Hybrid search fuses BM25 with vector similarity"),
|
|
30
|
+
Document(content="Cats sleep sixteen hours a day"),
|
|
31
|
+
])
|
|
32
|
+
|
|
33
|
+
pipe = Pipeline()
|
|
34
|
+
pipe.add_component("retriever", OpensolrHybridRetriever(document_store=store))
|
|
35
|
+
result = pipe.run({"retriever": {"query": "how do keyword and semantic search combine?"}})
|
|
36
|
+
print(result["retriever"]["documents"])
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Note there is **no embedder** in the pipeline — not for documents, not for
|
|
40
|
+
the query. The store embeds server-side at both index and query time.
|
|
41
|
+
|
|
42
|
+
## Hybrid retrieval
|
|
43
|
+
|
|
44
|
+
`OpensolrHybridRetriever` fuses BM25 and kNN scores per document via
|
|
45
|
+
Opensolr's native `{!hybrid}` Solr query parser:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
OpensolrHybridRetriever(
|
|
49
|
+
document_store=store,
|
|
50
|
+
top_k=10,
|
|
51
|
+
hybrid=True, # False = pure semantic kNN
|
|
52
|
+
alpha=0.5, # 0 = all semantic … 1 = all lexical
|
|
53
|
+
)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Standard Haystack filters are supported and map to Solr `fq`:
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
pipe.run({"retriever": {
|
|
60
|
+
"query": "search engines",
|
|
61
|
+
"filters": {"field": "meta.category", "operator": "==", "value": "docs"},
|
|
62
|
+
}})
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Notes
|
|
66
|
+
|
|
67
|
+
- Vector-enabled indexes run on Opensolr's Solr 9.x environments — currently
|
|
68
|
+
`us` (Chicago), `de` (Germany), `fi` (Finland). **Additional dedicated
|
|
69
|
+
regions can be deployed on request** (paid add-on):
|
|
70
|
+
[support@opensolr.com](mailto:support@opensolr.com).
|
|
71
|
+
- Every index is also plain Apache Solr with the native `/select` API —
|
|
72
|
+
facets, highlighting, spellcheck included.
|
|
73
|
+
- Siblings: [`langchain-opensolr`](https://pypi.org/project/langchain-opensolr/) ·
|
|
74
|
+
[`llama-index-opensolr`](https://pypi.org/project/llama-index-opensolr/) ·
|
|
75
|
+
[`opensolr-mcp`](https://pypi.org/project/opensolr-mcp/)
|
|
76
|
+
|
|
77
|
+
MIT license.
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Hybrid retriever component over :class:`OpensolrDocumentStore`.
|
|
2
|
+
|
|
3
|
+
Takes a plain-text query — embedding happens server-side on Opensolr's GPU
|
|
4
|
+
infrastructure, so no query-embedder component is needed in the pipeline.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any, Dict, List, Optional
|
|
10
|
+
|
|
11
|
+
from haystack import Document, component, default_from_dict, default_to_dict
|
|
12
|
+
|
|
13
|
+
from haystack_integrations.document_stores.opensolr import OpensolrDocumentStore
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@component
|
|
17
|
+
class OpensolrHybridRetriever:
|
|
18
|
+
"""Retrieve documents from Opensolr with hybrid BM25 + kNN scoring.
|
|
19
|
+
|
|
20
|
+
Example:
|
|
21
|
+
```python
|
|
22
|
+
from haystack import Pipeline
|
|
23
|
+
from haystack_integrations.document_stores.opensolr import OpensolrDocumentStore
|
|
24
|
+
from haystack_integrations.components.retrievers.opensolr import OpensolrHybridRetriever
|
|
25
|
+
|
|
26
|
+
store = OpensolrDocumentStore(index="mysite__dense")
|
|
27
|
+
pipe = Pipeline()
|
|
28
|
+
pipe.add_component("retriever", OpensolrHybridRetriever(document_store=store))
|
|
29
|
+
result = pipe.run({"retriever": {"query": "affordable restaurants"}})
|
|
30
|
+
```
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
document_store: OpensolrDocumentStore,
|
|
36
|
+
top_k: int = 10,
|
|
37
|
+
hybrid: bool = True,
|
|
38
|
+
alpha: float = 0.5,
|
|
39
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
40
|
+
) -> None:
|
|
41
|
+
self.document_store = document_store
|
|
42
|
+
self.top_k = top_k
|
|
43
|
+
self.hybrid = hybrid
|
|
44
|
+
self.alpha = alpha
|
|
45
|
+
self.filters = filters
|
|
46
|
+
|
|
47
|
+
@component.output_types(documents=List[Document])
|
|
48
|
+
def run(
|
|
49
|
+
self,
|
|
50
|
+
query: str,
|
|
51
|
+
top_k: Optional[int] = None,
|
|
52
|
+
hybrid: Optional[bool] = None,
|
|
53
|
+
alpha: Optional[float] = None,
|
|
54
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
55
|
+
) -> Dict[str, List[Document]]:
|
|
56
|
+
"""Run the retriever. ``alpha``: 0 = all semantic, 1 = all lexical."""
|
|
57
|
+
docs = self.document_store.search(
|
|
58
|
+
query=query,
|
|
59
|
+
top_k=top_k if top_k is not None else self.top_k,
|
|
60
|
+
hybrid=hybrid if hybrid is not None else self.hybrid,
|
|
61
|
+
alpha=alpha if alpha is not None else self.alpha,
|
|
62
|
+
filters=filters if filters is not None else self.filters,
|
|
63
|
+
)
|
|
64
|
+
return {"documents": docs}
|
|
65
|
+
|
|
66
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
67
|
+
return default_to_dict(
|
|
68
|
+
self,
|
|
69
|
+
document_store=self.document_store.to_dict(),
|
|
70
|
+
top_k=self.top_k,
|
|
71
|
+
hybrid=self.hybrid,
|
|
72
|
+
alpha=self.alpha,
|
|
73
|
+
filters=self.filters,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def from_dict(cls, data: Dict[str, Any]) -> "OpensolrHybridRetriever":
|
|
78
|
+
data["init_parameters"]["document_store"] = OpensolrDocumentStore.from_dict(
|
|
79
|
+
data["init_parameters"]["document_store"]
|
|
80
|
+
)
|
|
81
|
+
return default_from_dict(cls, data)
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""Thin REST client for the Opensolr platform APIs.
|
|
2
|
+
|
|
3
|
+
Two base URLs, by platform design:
|
|
4
|
+
- Management API (index list/info/create): https://opensolr.com/solr_manager/api
|
|
5
|
+
- AI API (embed, batch_embed, embed_and_search, ai_summary): https://api.opensolr.com/solr_manager/api
|
|
6
|
+
|
|
7
|
+
Direct Solr access (select/update) goes to the index's own host, resolved via
|
|
8
|
+
``get_core_info`` (``connection_url`` + HTTP basic auth).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
15
|
+
|
|
16
|
+
import httpx
|
|
17
|
+
|
|
18
|
+
MGMT_BASE = "https://opensolr.com/solr_manager/api"
|
|
19
|
+
AI_BASE = "https://api.opensolr.com/solr_manager/api"
|
|
20
|
+
|
|
21
|
+
#: Convenience aliases for Opensolr's vector-enabled environments. The
|
|
22
|
+
#: authoritative list is served live by the platform (``vector_regions``
|
|
23
|
+
#: endpoint) — new regions become valid automatically, and additional
|
|
24
|
+
#: dedicated regions can be deployed on request (paid): support@opensolr.com.
|
|
25
|
+
VECTOR_LOCATIONS: Dict[str, str] = {
|
|
26
|
+
"us": "CHICAGO-96",
|
|
27
|
+
"de": "DE-SOLR-9",
|
|
28
|
+
"fi": "FINLAND9",
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def resolve_location(location: str) -> str:
|
|
33
|
+
"""Map a friendly alias ("us"/"de"/"fi") to its environment identifier.
|
|
34
|
+
|
|
35
|
+
Unknown values pass through unchanged — validity is decided against the
|
|
36
|
+
live ``vector_regions`` list (or, ultimately, by the server), so newly
|
|
37
|
+
deployed vector regions work without a package upgrade.
|
|
38
|
+
"""
|
|
39
|
+
return VECTOR_LOCATIONS.get(location.strip().lower(), location.strip())
|
|
40
|
+
|
|
41
|
+
#: Server-side limit for one batch_embed call.
|
|
42
|
+
BATCH_EMBED_MAX = 50
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class OpensolrError(RuntimeError):
|
|
46
|
+
"""Raised when an Opensolr API call fails."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class OpensolrClient:
|
|
50
|
+
"""Authenticated client for Opensolr management + AI endpoints.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
email: Opensolr account email.
|
|
54
|
+
api_key: Opensolr API key (Account > API in the control panel).
|
|
55
|
+
timeout: Per-request timeout in seconds. Embedding calls run on GPU
|
|
56
|
+
infrastructure and are usually fast, but cold starts happen.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, email: str, api_key: str, timeout: float = 120.0) -> None:
|
|
60
|
+
self.email = email
|
|
61
|
+
self.api_key = api_key
|
|
62
|
+
self._http = httpx.Client(timeout=timeout, follow_redirects=True)
|
|
63
|
+
self._core_info_cache: Dict[str, Dict[str, Any]] = {}
|
|
64
|
+
|
|
65
|
+
# ------------------------------------------------------------------ #
|
|
66
|
+
# low level #
|
|
67
|
+
# ------------------------------------------------------------------ #
|
|
68
|
+
|
|
69
|
+
def _auth_params(self) -> Dict[str, str]:
|
|
70
|
+
return {"email": self.email, "api_key": self.api_key}
|
|
71
|
+
|
|
72
|
+
def _request(self, base: str, method: str, params: Dict[str, Any]) -> Any:
|
|
73
|
+
url = f"{base}/{method}"
|
|
74
|
+
data = {**self._auth_params(), **params}
|
|
75
|
+
resp = self._http.post(url, data=data)
|
|
76
|
+
if resp.status_code >= 500:
|
|
77
|
+
raise OpensolrError(f"{method}: HTTP {resp.status_code}: {resp.text[:200]}")
|
|
78
|
+
try:
|
|
79
|
+
body = resp.json()
|
|
80
|
+
except json.JSONDecodeError as exc:
|
|
81
|
+
raise OpensolrError(f"{method}: non-JSON response: {resp.text[:200]}") from exc
|
|
82
|
+
if isinstance(body, dict) and body.get("status") is False:
|
|
83
|
+
raise OpensolrError(f"{method}: {body.get('msg', body)}")
|
|
84
|
+
return body
|
|
85
|
+
|
|
86
|
+
def mgmt(self, method: str, **params: Any) -> Any:
|
|
87
|
+
return self._request(MGMT_BASE, method, params)
|
|
88
|
+
|
|
89
|
+
def ai(self, method: str, **params: Any) -> Any:
|
|
90
|
+
return self._request(AI_BASE, method, params)
|
|
91
|
+
|
|
92
|
+
# ------------------------------------------------------------------ #
|
|
93
|
+
# management #
|
|
94
|
+
# ------------------------------------------------------------------ #
|
|
95
|
+
|
|
96
|
+
def get_index_list(self) -> List[Dict[str, str]]:
|
|
97
|
+
return self.mgmt("get_index_list")
|
|
98
|
+
|
|
99
|
+
def get_core_info(self, index: str, refresh: bool = False) -> Dict[str, Any]:
|
|
100
|
+
"""Resolve an index's Solr endpoint + HTTP auth. Cached per client."""
|
|
101
|
+
if not refresh and index in self._core_info_cache:
|
|
102
|
+
return self._core_info_cache[index]
|
|
103
|
+
body = self.mgmt("get_core_info", core_name=index)
|
|
104
|
+
msg = body.get("msg") if isinstance(body, dict) else None
|
|
105
|
+
if not isinstance(msg, dict) or "info" not in msg:
|
|
106
|
+
raise OpensolrError(f"get_core_info({index}): unexpected response: {str(body)[:200]}")
|
|
107
|
+
info = msg["info"]
|
|
108
|
+
self._core_info_cache[index] = info
|
|
109
|
+
return info
|
|
110
|
+
|
|
111
|
+
def vector_regions(self) -> List[Dict[str, str]]:
|
|
112
|
+
"""Live list of vector-enabled environments (Solr 9.x + knn_vector +
|
|
113
|
+
hybrid parser): ``[{environment, country, solr_version}, ...]``.
|
|
114
|
+
|
|
115
|
+
Cached per client. Additional dedicated regions can be deployed on
|
|
116
|
+
request (paid) — contact support@opensolr.com.
|
|
117
|
+
"""
|
|
118
|
+
if not hasattr(self, "_vector_regions_cache"):
|
|
119
|
+
body = self.mgmt("vector_regions")
|
|
120
|
+
self._vector_regions_cache = body if isinstance(body, list) else []
|
|
121
|
+
return self._vector_regions_cache
|
|
122
|
+
|
|
123
|
+
def create_index(self, index: str, location: str = "us") -> Dict[str, Any]:
|
|
124
|
+
"""Create a vector-enabled index in a vector location.
|
|
125
|
+
|
|
126
|
+
``location`` is an alias ("us", "de", "fi") or a raw Opensolr
|
|
127
|
+
environment identifier. Validated against the live ``vector_regions``
|
|
128
|
+
list when reachable; otherwise the server has the final word.
|
|
129
|
+
"""
|
|
130
|
+
env = resolve_location(location)
|
|
131
|
+
try:
|
|
132
|
+
live = {r["environment"] for r in self.vector_regions()}
|
|
133
|
+
except OpensolrError:
|
|
134
|
+
live = set(VECTOR_LOCATIONS.values()) # offline fallback
|
|
135
|
+
if live and env not in live:
|
|
136
|
+
raise ValueError(
|
|
137
|
+
f"{location!r} is not a vector-enabled Opensolr location. "
|
|
138
|
+
f"Currently available: {sorted(live)}. Additional regions can "
|
|
139
|
+
f"be deployed on request — contact support@opensolr.com."
|
|
140
|
+
)
|
|
141
|
+
return self.mgmt(
|
|
142
|
+
"create_index", index_name=index, core_type="generic", server_country=env
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
# ------------------------------------------------------------------ #
|
|
146
|
+
# AI #
|
|
147
|
+
# ------------------------------------------------------------------ #
|
|
148
|
+
|
|
149
|
+
def embed(self, index: str, text: str, is_query: bool = False) -> List[float]:
|
|
150
|
+
body = self.ai(
|
|
151
|
+
"embed", index_name=index, payload=text, is_query="1" if is_query else "0"
|
|
152
|
+
)
|
|
153
|
+
if not isinstance(body, list) or not body:
|
|
154
|
+
raise OpensolrError(f"embed: unexpected response: {str(body)[:200]}")
|
|
155
|
+
return body
|
|
156
|
+
|
|
157
|
+
def batch_embed(self, index: str, texts: List[str]) -> List[List[float]]:
|
|
158
|
+
"""Embed many texts. Chunks transparently at the server's batch limit."""
|
|
159
|
+
out: List[List[float]] = []
|
|
160
|
+
for i in range(0, len(texts), BATCH_EMBED_MAX):
|
|
161
|
+
chunk = texts[i : i + BATCH_EMBED_MAX]
|
|
162
|
+
resp = self._http.post(
|
|
163
|
+
f"{AI_BASE}/batch_embed",
|
|
164
|
+
json={
|
|
165
|
+
**self._auth_params(),
|
|
166
|
+
"index_name": index,
|
|
167
|
+
"payloads": chunk,
|
|
168
|
+
},
|
|
169
|
+
)
|
|
170
|
+
try:
|
|
171
|
+
body = resp.json()
|
|
172
|
+
except json.JSONDecodeError as exc:
|
|
173
|
+
raise OpensolrError(f"batch_embed: non-JSON response: {resp.text[:200]}") from exc
|
|
174
|
+
if isinstance(body, dict) and body.get("status") is False:
|
|
175
|
+
raise OpensolrError(f"batch_embed: {body.get('msg', body)}")
|
|
176
|
+
embeddings = body.get("embeddings") if isinstance(body, dict) else None
|
|
177
|
+
if not isinstance(embeddings, list) or len(embeddings) != len(chunk):
|
|
178
|
+
raise OpensolrError(f"batch_embed: unexpected response: {str(body)[:200]}")
|
|
179
|
+
out.extend(embeddings)
|
|
180
|
+
return out
|
|
181
|
+
|
|
182
|
+
def embed_and_search(self, index: str, query: str, rows: int = 10, **params: Any) -> Dict[str, Any]:
|
|
183
|
+
"""Server-side one-shot: embed the query, run hybrid search, return docs."""
|
|
184
|
+
body = self.ai(
|
|
185
|
+
"embed_and_search",
|
|
186
|
+
index_name=index,
|
|
187
|
+
q=query,
|
|
188
|
+
rows=rows,
|
|
189
|
+
**{"in": "all", "fresh": "no", **params},
|
|
190
|
+
)
|
|
191
|
+
return body
|
|
192
|
+
|
|
193
|
+
# ------------------------------------------------------------------ #
|
|
194
|
+
# direct Solr #
|
|
195
|
+
# ------------------------------------------------------------------ #
|
|
196
|
+
|
|
197
|
+
def solr_endpoint(self, index: str) -> Tuple[str, Optional[Tuple[str, str]]]:
|
|
198
|
+
"""Return (base_url, basic_auth) for the index's native Solr API."""
|
|
199
|
+
info = self.get_core_info(index)
|
|
200
|
+
url = info.get("connection_url")
|
|
201
|
+
if not url:
|
|
202
|
+
raise OpensolrError(f"No connection_url for index {index!r}")
|
|
203
|
+
auth = None
|
|
204
|
+
if info.get("auth_username"):
|
|
205
|
+
auth = (info["auth_username"], info.get("auth_password") or "")
|
|
206
|
+
return url, auth
|
|
207
|
+
|
|
208
|
+
def solr_select(self, index: str, params: Dict[str, Any]) -> Dict[str, Any]:
|
|
209
|
+
base, auth = self.solr_endpoint(index)
|
|
210
|
+
resp = self._http.post(f"{base}/select", data={"wt": "json", **params}, auth=auth)
|
|
211
|
+
resp.raise_for_status()
|
|
212
|
+
return resp.json()
|
|
213
|
+
|
|
214
|
+
def solr_update(self, index: str, payload: Any, commit: bool = True) -> Dict[str, Any]:
|
|
215
|
+
base, auth = self.solr_endpoint(index)
|
|
216
|
+
params = {"commit": "true"} if commit else {"commitWithin": "10000"}
|
|
217
|
+
resp = self._http.post(
|
|
218
|
+
f"{base}/update",
|
|
219
|
+
params=params,
|
|
220
|
+
json=payload,
|
|
221
|
+
auth=auth,
|
|
222
|
+
)
|
|
223
|
+
resp.raise_for_status()
|
|
224
|
+
return resp.json()
|
|
225
|
+
|
|
226
|
+
def close(self) -> None:
|
|
227
|
+
self._http.close()
|
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
"""Opensolr DocumentStore for Haystack.
|
|
2
|
+
|
|
3
|
+
Managed, vector-enabled Apache Solr 9.x (knn_vector 1024-dim, cosine) with
|
|
4
|
+
embeddings computed **server-side** on Opensolr's GPU infrastructure — no
|
|
5
|
+
local embedder component needed at indexing time.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
from typing import Any, Dict, List, Optional
|
|
13
|
+
|
|
14
|
+
from haystack import Document, default_from_dict, default_to_dict
|
|
15
|
+
from haystack.document_stores.types import DuplicatePolicy
|
|
16
|
+
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
17
|
+
|
|
18
|
+
from haystack_integrations.document_stores.opensolr.client import (
|
|
19
|
+
OpensolrClient,
|
|
20
|
+
OpensolrError,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
_META_KEY_RE = re.compile(r"[^a-z0-9_]+")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _meta_field(key: str) -> str:
|
|
27
|
+
return f"meta_{_META_KEY_RE.sub('_', key.lower()).strip('_')}"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _escape(value: Any) -> str:
|
|
31
|
+
return str(value).replace("\\", "\\\\").replace('"', '\\"')
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _filters_to_fq(filters: Optional[Dict[str, Any]]) -> List[str]:
|
|
35
|
+
"""Translate Haystack's standard filter dict into Solr fq clauses.
|
|
36
|
+
|
|
37
|
+
Supports field conditions with operators ==, !=, in, not in, and logical
|
|
38
|
+
AND groups (the common cases). OR groups become a single fq with OR.
|
|
39
|
+
"""
|
|
40
|
+
if not filters:
|
|
41
|
+
return []
|
|
42
|
+
|
|
43
|
+
def _cond(f: Dict[str, Any]) -> str:
|
|
44
|
+
field = f["field"].split("meta.", 1)[-1]
|
|
45
|
+
solr_field = _meta_field(field)
|
|
46
|
+
op = f["operator"]
|
|
47
|
+
value = f.get("value")
|
|
48
|
+
if op == "==":
|
|
49
|
+
return f'{solr_field}:"{_escape(value)}"'
|
|
50
|
+
if op == "!=":
|
|
51
|
+
return f'-{solr_field}:"{_escape(value)}"'
|
|
52
|
+
if op == "in":
|
|
53
|
+
joined = " OR ".join(f'"{_escape(v)}"' for v in value)
|
|
54
|
+
return f"{solr_field}:({joined})"
|
|
55
|
+
if op == "not in":
|
|
56
|
+
joined = " OR ".join(f'"{_escape(v)}"' for v in value)
|
|
57
|
+
return f"-{solr_field}:({joined})"
|
|
58
|
+
if op in (">", ">="):
|
|
59
|
+
return f'{solr_field}:{"{" if op == ">" else "["}"{_escape(value)}" TO *]'
|
|
60
|
+
if op in ("<", "<="):
|
|
61
|
+
return f'{solr_field}:[* TO "{_escape(value)}"{"}" if op == "<" else "]"}'
|
|
62
|
+
raise ValueError(f"Unsupported filter operator: {op}")
|
|
63
|
+
|
|
64
|
+
if "operator" in filters and "conditions" in filters:
|
|
65
|
+
parts = []
|
|
66
|
+
for c in filters["conditions"]:
|
|
67
|
+
if "conditions" in c:
|
|
68
|
+
sub = _filters_to_fq(c)
|
|
69
|
+
parts.append("(" + " AND ".join(sub) + ")")
|
|
70
|
+
else:
|
|
71
|
+
parts.append(_cond(c))
|
|
72
|
+
if filters["operator"] == "AND":
|
|
73
|
+
return parts
|
|
74
|
+
if filters["operator"] == "OR":
|
|
75
|
+
return ["(" + " OR ".join(parts) + ")"]
|
|
76
|
+
raise ValueError(f"Unsupported logical operator: {filters['operator']}")
|
|
77
|
+
return [_cond(filters)]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class OpensolrDocumentStore:
|
|
81
|
+
"""Haystack DocumentStore backed by a managed Opensolr vector index.
|
|
82
|
+
|
|
83
|
+
Example:
|
|
84
|
+
```python
|
|
85
|
+
from haystack_integrations.document_stores.opensolr import OpensolrDocumentStore
|
|
86
|
+
|
|
87
|
+
store = OpensolrDocumentStore(index="mysite__dense")
|
|
88
|
+
# credentials default to OPENSOLR_EMAIL / OPENSOLR_API_KEY env vars
|
|
89
|
+
```
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
def __init__(
|
|
93
|
+
self,
|
|
94
|
+
index: str,
|
|
95
|
+
email: Secret = Secret.from_env_var("OPENSOLR_EMAIL"),
|
|
96
|
+
api_key: Secret = Secret.from_env_var("OPENSOLR_API_KEY"),
|
|
97
|
+
create_if_missing: bool = False,
|
|
98
|
+
location: str = "us",
|
|
99
|
+
) -> None:
|
|
100
|
+
self.index = index
|
|
101
|
+
self.email = email
|
|
102
|
+
self.api_key = api_key
|
|
103
|
+
self.create_if_missing = create_if_missing
|
|
104
|
+
self.location = location
|
|
105
|
+
self._client: Optional[OpensolrClient] = None
|
|
106
|
+
self._checked = False
|
|
107
|
+
|
|
108
|
+
# ------------------------------------------------------------------ #
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def client(self) -> OpensolrClient:
|
|
112
|
+
if self._client is None:
|
|
113
|
+
self._client = OpensolrClient(
|
|
114
|
+
self.email.resolve_value(), self.api_key.resolve_value()
|
|
115
|
+
)
|
|
116
|
+
return self._client
|
|
117
|
+
|
|
118
|
+
def _ensure_index(self) -> None:
|
|
119
|
+
if self._checked:
|
|
120
|
+
return
|
|
121
|
+
try:
|
|
122
|
+
self.client.get_core_info(self.index)
|
|
123
|
+
except OpensolrError:
|
|
124
|
+
if not self.create_if_missing:
|
|
125
|
+
raise
|
|
126
|
+
self.client.create_index(self.index, self.location)
|
|
127
|
+
import time
|
|
128
|
+
|
|
129
|
+
for _ in range(5):
|
|
130
|
+
time.sleep(2)
|
|
131
|
+
try:
|
|
132
|
+
self.client.get_core_info(self.index, refresh=True)
|
|
133
|
+
break
|
|
134
|
+
except OpensolrError:
|
|
135
|
+
continue
|
|
136
|
+
self._checked = True
|
|
137
|
+
|
|
138
|
+
def _doc_from_solr(self, solr_doc: Dict[str, Any]) -> Document:
|
|
139
|
+
def _flat(v: Any) -> Any:
|
|
140
|
+
return v[0] if isinstance(v, list) and len(v) == 1 else v
|
|
141
|
+
|
|
142
|
+
meta: Dict[str, Any] = {}
|
|
143
|
+
raw = _flat(solr_doc.get("meta_lc_json"))
|
|
144
|
+
if raw:
|
|
145
|
+
try:
|
|
146
|
+
meta = json.loads(raw)
|
|
147
|
+
except (TypeError, json.JSONDecodeError):
|
|
148
|
+
meta = {}
|
|
149
|
+
content = _flat(solr_doc.get("text", "")) or ""
|
|
150
|
+
if isinstance(content, list):
|
|
151
|
+
content = " ".join(str(c) for c in content)
|
|
152
|
+
score = solr_doc.get("score")
|
|
153
|
+
return Document(
|
|
154
|
+
id=str(_flat(solr_doc.get("id", ""))),
|
|
155
|
+
content=str(content),
|
|
156
|
+
meta=meta,
|
|
157
|
+
score=float(_flat(score)) if score is not None else None,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
# ------------------------------------------------------------------ #
|
|
161
|
+
# DocumentStore protocol #
|
|
162
|
+
# ------------------------------------------------------------------ #
|
|
163
|
+
|
|
164
|
+
def count_documents(self) -> int:
|
|
165
|
+
self._ensure_index()
|
|
166
|
+
body = self.client.solr_select(self.index, {"q": "*:*", "rows": 0})
|
|
167
|
+
return int(body["response"]["numFound"])
|
|
168
|
+
|
|
169
|
+
def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
|
|
170
|
+
self._ensure_index()
|
|
171
|
+
params: Dict[str, Any] = {"q": "*:*", "rows": 1000, "fl": "*"}
|
|
172
|
+
fq = _filters_to_fq(filters)
|
|
173
|
+
if fq:
|
|
174
|
+
params["fq"] = fq
|
|
175
|
+
body = self.client.solr_select(self.index, params)
|
|
176
|
+
return [self._doc_from_solr(d) for d in body["response"]["docs"]]
|
|
177
|
+
|
|
178
|
+
def write_documents(
|
|
179
|
+
self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
|
|
180
|
+
) -> int:
|
|
181
|
+
if not documents:
|
|
182
|
+
return 0
|
|
183
|
+
self._ensure_index()
|
|
184
|
+
|
|
185
|
+
if policy in (DuplicatePolicy.SKIP, DuplicatePolicy.FAIL):
|
|
186
|
+
ids = [d.id for d in documents]
|
|
187
|
+
joined = " OR ".join(f'"{_escape(i)}"' for i in ids)
|
|
188
|
+
body = self.client.solr_select(
|
|
189
|
+
self.index, {"q": f"id:({joined})", "rows": len(ids), "fl": "id"}
|
|
190
|
+
)
|
|
191
|
+
existing = {
|
|
192
|
+
str(d["id"][0] if isinstance(d["id"], list) else d["id"])
|
|
193
|
+
for d in body["response"]["docs"]
|
|
194
|
+
}
|
|
195
|
+
if existing and policy == DuplicatePolicy.FAIL:
|
|
196
|
+
from haystack.document_stores.errors import DuplicateDocumentError
|
|
197
|
+
|
|
198
|
+
raise DuplicateDocumentError(f"IDs already in the store: {sorted(existing)}")
|
|
199
|
+
documents = [d for d in documents if d.id not in existing]
|
|
200
|
+
if not documents:
|
|
201
|
+
return 0
|
|
202
|
+
|
|
203
|
+
texts = [d.content or " " for d in documents]
|
|
204
|
+
embeddings: List[Optional[List[float]]] = [d.embedding for d in documents]
|
|
205
|
+
missing = [i for i, e in enumerate(embeddings) if e is None]
|
|
206
|
+
if missing:
|
|
207
|
+
computed = self.client.batch_embed(self.index, [texts[i] for i in missing])
|
|
208
|
+
for i, vec in zip(missing, computed):
|
|
209
|
+
embeddings[i] = vec
|
|
210
|
+
|
|
211
|
+
docs = []
|
|
212
|
+
for doc, text, vector in zip(documents, texts, embeddings):
|
|
213
|
+
meta = dict(doc.meta or {})
|
|
214
|
+
solr_doc: Dict[str, Any] = {
|
|
215
|
+
"id": doc.id,
|
|
216
|
+
"text": text,
|
|
217
|
+
"embeddings": vector,
|
|
218
|
+
"meta_lc_json": json.dumps(meta, ensure_ascii=False),
|
|
219
|
+
"title": str(meta.get("title") or text[:100]),
|
|
220
|
+
}
|
|
221
|
+
for key, value in meta.items():
|
|
222
|
+
if isinstance(value, (str, int, float, bool)):
|
|
223
|
+
solr_doc[_meta_field(str(key))] = str(value)
|
|
224
|
+
docs.append(solr_doc)
|
|
225
|
+
|
|
226
|
+
self.client.solr_update(self.index, docs)
|
|
227
|
+
return len(docs)
|
|
228
|
+
|
|
229
|
+
def delete_documents(self, document_ids: List[str]) -> None:
|
|
230
|
+
if not document_ids:
|
|
231
|
+
return
|
|
232
|
+
self._ensure_index()
|
|
233
|
+
self.client.solr_update(self.index, {"delete": list(document_ids)})
|
|
234
|
+
|
|
235
|
+
# ------------------------------------------------------------------ #
|
|
236
|
+
# search (used by the retriever component) #
|
|
237
|
+
# ------------------------------------------------------------------ #
|
|
238
|
+
|
|
239
|
+
def search(
|
|
240
|
+
self,
|
|
241
|
+
query: str,
|
|
242
|
+
top_k: int = 10,
|
|
243
|
+
hybrid: bool = True,
|
|
244
|
+
alpha: float = 0.5,
|
|
245
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
246
|
+
) -> List[Document]:
|
|
247
|
+
self._ensure_index()
|
|
248
|
+
vector = self.client.embed(self.index, query, is_query=True)
|
|
249
|
+
compact = json.dumps(vector, separators=(",", ":"))
|
|
250
|
+
knn = f"{{!knn f=embeddings topK={max(top_k, 10)}}}{compact}"
|
|
251
|
+
|
|
252
|
+
params: Dict[str, Any] = {"rows": top_k, "fl": "*,score"}
|
|
253
|
+
if hybrid:
|
|
254
|
+
clean = query.replace("{", " ").replace("}", " ").replace('"', " ")
|
|
255
|
+
params["q"] = (
|
|
256
|
+
f"{{!hybrid lexical=$lexicalRaw vector=$vectorQuery "
|
|
257
|
+
f"mode=union alpha={alpha} topN={max(top_k, 10)}}}"
|
|
258
|
+
)
|
|
259
|
+
params["lexicalRaw"] = f'{{!edismax qf="title^100 text^1"}}{clean}'
|
|
260
|
+
params["vectorQuery"] = knn
|
|
261
|
+
else:
|
|
262
|
+
params["q"] = knn
|
|
263
|
+
fq = _filters_to_fq(filters)
|
|
264
|
+
if fq:
|
|
265
|
+
params["fq"] = fq
|
|
266
|
+
|
|
267
|
+
body = self.client.solr_select(self.index, params)
|
|
268
|
+
return [self._doc_from_solr(d) for d in body["response"]["docs"]]
|
|
269
|
+
|
|
270
|
+
# ------------------------------------------------------------------ #
|
|
271
|
+
# serialization #
|
|
272
|
+
# ------------------------------------------------------------------ #
|
|
273
|
+
|
|
274
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
275
|
+
return default_to_dict(
|
|
276
|
+
self,
|
|
277
|
+
index=self.index,
|
|
278
|
+
email=self.email.to_dict(),
|
|
279
|
+
api_key=self.api_key.to_dict(),
|
|
280
|
+
create_if_missing=self.create_if_missing,
|
|
281
|
+
location=self.location,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
@classmethod
|
|
285
|
+
def from_dict(cls, data: Dict[str, Any]) -> "OpensolrDocumentStore":
|
|
286
|
+
deserialize_secrets_inplace(data["init_parameters"], keys=["email", "api_key"])
|
|
287
|
+
return default_from_dict(cls, data)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: opensolr-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Haystack integration for Opensolr — managed Apache Solr DocumentStore with server-side embeddings and hybrid BM25+kNN retrieval
|
|
5
|
+
Author-email: Opensolr <support@opensolr.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://opensolr.com/langchain
|
|
8
|
+
Project-URL: Repository, https://github.com/phpcip/opensolr-haystack
|
|
9
|
+
Keywords: haystack,opensolr,solr,vector,hybrid-search,rag,document-store
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Requires-Python: >=3.9
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: haystack-ai>=2.0.0
|
|
16
|
+
Requires-Dist: httpx>=0.25.0
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# opensolr-haystack
|
|
20
|
+
|
|
21
|
+
[Haystack](https://haystack.deepset.ai) integration for
|
|
22
|
+
[Opensolr](https://opensolr.com) — managed Apache Solr as a DocumentStore,
|
|
23
|
+
with **server-side embeddings** and native **hybrid (BM25 + kNN) retrieval**.
|
|
24
|
+
|
|
25
|
+
No embedder components needed in your pipeline — texts and queries are
|
|
26
|
+
embedded on Opensolr's GPU infrastructure (multilingual E5-large-instruct,
|
|
27
|
+
1024 dimensions, cosine).
|
|
28
|
+
|
|
29
|
+
**Product page:** [opensolr.com/langchain](https://opensolr.com/langchain) ·
|
|
30
|
+
free 15-day trial, no card, at [opensolr.com](https://opensolr.com)
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install opensolr-haystack
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Quickstart
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from haystack import Document, Pipeline
|
|
40
|
+
from haystack_integrations.document_stores.opensolr import OpensolrDocumentStore
|
|
41
|
+
from haystack_integrations.components.retrievers.opensolr import OpensolrHybridRetriever
|
|
42
|
+
|
|
43
|
+
# credentials default to OPENSOLR_EMAIL / OPENSOLR_API_KEY env vars
|
|
44
|
+
store = OpensolrDocumentStore(index="mysite__dense", create_if_missing=True)
|
|
45
|
+
|
|
46
|
+
store.write_documents([
|
|
47
|
+
Document(content="Hybrid search fuses BM25 with vector similarity"),
|
|
48
|
+
Document(content="Cats sleep sixteen hours a day"),
|
|
49
|
+
])
|
|
50
|
+
|
|
51
|
+
pipe = Pipeline()
|
|
52
|
+
pipe.add_component("retriever", OpensolrHybridRetriever(document_store=store))
|
|
53
|
+
result = pipe.run({"retriever": {"query": "how do keyword and semantic search combine?"}})
|
|
54
|
+
print(result["retriever"]["documents"])
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Note there is **no embedder** in the pipeline — not for documents, not for
|
|
58
|
+
the query. The store embeds server-side at both index and query time.
|
|
59
|
+
|
|
60
|
+
## Hybrid retrieval
|
|
61
|
+
|
|
62
|
+
`OpensolrHybridRetriever` fuses BM25 and kNN scores per document via
|
|
63
|
+
Opensolr's native `{!hybrid}` Solr query parser:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
OpensolrHybridRetriever(
|
|
67
|
+
document_store=store,
|
|
68
|
+
top_k=10,
|
|
69
|
+
hybrid=True, # False = pure semantic kNN
|
|
70
|
+
alpha=0.5, # 0 = all semantic … 1 = all lexical
|
|
71
|
+
)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Standard Haystack filters are supported and map to Solr `fq`:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
pipe.run({"retriever": {
|
|
78
|
+
"query": "search engines",
|
|
79
|
+
"filters": {"field": "meta.category", "operator": "==", "value": "docs"},
|
|
80
|
+
}})
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Notes
|
|
84
|
+
|
|
85
|
+
- Vector-enabled indexes run on Opensolr's Solr 9.x environments — currently
|
|
86
|
+
`us` (Chicago), `de` (Germany), `fi` (Finland). **Additional dedicated
|
|
87
|
+
regions can be deployed on request** (paid add-on):
|
|
88
|
+
[support@opensolr.com](mailto:support@opensolr.com).
|
|
89
|
+
- Every index is also plain Apache Solr with the native `/select` API —
|
|
90
|
+
facets, highlighting, spellcheck included.
|
|
91
|
+
- Siblings: [`langchain-opensolr`](https://pypi.org/project/langchain-opensolr/) ·
|
|
92
|
+
[`llama-index-opensolr`](https://pypi.org/project/llama-index-opensolr/) ·
|
|
93
|
+
[`opensolr-mcp`](https://pypi.org/project/opensolr-mcp/)
|
|
94
|
+
|
|
95
|
+
MIT license.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
haystack_integrations/components/retrievers/opensolr/__init__.py
|
|
5
|
+
haystack_integrations/components/retrievers/opensolr/retriever.py
|
|
6
|
+
haystack_integrations/document_stores/opensolr/__init__.py
|
|
7
|
+
haystack_integrations/document_stores/opensolr/client.py
|
|
8
|
+
haystack_integrations/document_stores/opensolr/store.py
|
|
9
|
+
opensolr_haystack.egg-info/PKG-INFO
|
|
10
|
+
opensolr_haystack.egg-info/SOURCES.txt
|
|
11
|
+
opensolr_haystack.egg-info/dependency_links.txt
|
|
12
|
+
opensolr_haystack.egg-info/requires.txt
|
|
13
|
+
opensolr_haystack.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
haystack_integrations
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "opensolr-haystack"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Haystack integration for Opensolr — managed Apache Solr DocumentStore with server-side embeddings and hybrid BM25+kNN retrieval"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [{ name = "Opensolr", email = "support@opensolr.com" }]
|
|
13
|
+
dependencies = [
|
|
14
|
+
"haystack-ai>=2.0.0",
|
|
15
|
+
"httpx>=0.25.0",
|
|
16
|
+
]
|
|
17
|
+
keywords = ["haystack", "opensolr", "solr", "vector", "hybrid-search", "rag", "document-store"]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://opensolr.com/langchain"
|
|
25
|
+
Repository = "https://github.com/phpcip/opensolr-haystack"
|
|
26
|
+
|
|
27
|
+
[tool.setuptools.packages.find]
|
|
28
|
+
include = ["haystack_integrations.*"]
|