opensolr-haystack 0.2.0__tar.gz → 0.2.2__tar.gz

{opensolr_haystack-0.2.0/opensolr_haystack.egg-info → opensolr_haystack-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensolr-haystack
-Version: 0.2.0
+Version: 0.2.2
 Summary: Haystack integration for Opensolr — managed Apache Solr DocumentStore with server-side embeddings and hybrid BM25+kNN retrieval
 Author-email: Opensolr <support@opensolr.com>
 License: MIT
@@ -121,4 +121,42 @@ entry? Configure the **Web Crawler** in the Control Panel (Index Tools →
 WebCrawler): add your site URL, validate it, and Opensolr indexes the whole
 site for you.
 
+## Grounded RAG answers
+
+One call: hybrid retrieval picks the top hits, whose content becomes the LLM
+context, and Opensolr's server-side LLM answers — no generator component,
+no LLM key:
+
+```python
+answer = store.ai_answer(
+    "what does the refund policy say?",
+    rag_docs=3,        # how many hybrid hits feed the LLM (default 3)
+    rag_words=1500,    # words of text taken from each hit (default 1500)
+    # instruction="Answer in German, cite the exact titles you used",  # optional
+)
+```
+
+## How it's tested
+
+Every release is validated against **live Opensolr infrastructure** — no mocks:
+
+- **Unit tests** (offline): location aliases, filter→fq mapping, query building, escaping.
+- **End-to-end suite**: the full write path through the async Data Ingestion
+  queue (queued → server-side enrichment → searchable), semantic / hybrid /
+  lexical retrieval, metadata round-trip, filters, id round-trip (your ids
+  and the Solr `md5(uri)` ids), deletes by id and by query.
+- **Real-corpus validation**: searches run against a 340-document replica of
+  opensolr.com's own production search index. Verified: pure-semantic hits
+  with zero keyword overlap ("how do I get my data back after a disaster" →
+  backup &amp; restore docs), cross-lingual queries (Romanian query → English
+  content), exact-term surfacing in hybrid mode, all four hybrid modes, and
+  the full alpha range 0 → 1.
+- **PDF ingestion**: a real PDF ingested via `rtf:true` — server-side text
+  extraction (13k+ chars), automatic content-type detection, then retrieved
+  with a purely semantic query against its contents.
+- **Grounded RAG answers**: `ai_answer` verified end-to-end — a question answerable only from the ingested PDF returns the correct answer, sourced from the PDF's extracted text via hybrid retrieval.
+
+The store is exercised live (write via ingestion, DuplicatePolicy SKIP/FAIL,
+hybrid + lexical retrieval, filters, serde round-trip) before every release.
+
 MIT license.

{opensolr_haystack-0.2.0 → opensolr_haystack-0.2.2}/README.md

@@ -103,4 +103,42 @@ entry? Configure the **Web Crawler** in the Control Panel (Index Tools →
 WebCrawler): add your site URL, validate it, and Opensolr indexes the whole
 site for you.
 
+## Grounded RAG answers
+
+One call: hybrid retrieval picks the top hits, whose content becomes the LLM
+context, and Opensolr's server-side LLM answers — no generator component,
+no LLM key:
+
+```python
+answer = store.ai_answer(
+    "what does the refund policy say?",
+    rag_docs=3,        # how many hybrid hits feed the LLM (default 3)
+    rag_words=1500,    # words of text taken from each hit (default 1500)
+    # instruction="Answer in German, cite the exact titles you used",  # optional
+)
+```
+
+## How it's tested
+
+Every release is validated against **live Opensolr infrastructure** — no mocks:
+
+- **Unit tests** (offline): location aliases, filter→fq mapping, query building, escaping.
+- **End-to-end suite**: the full write path through the async Data Ingestion
+  queue (queued → server-side enrichment → searchable), semantic / hybrid /
+  lexical retrieval, metadata round-trip, filters, id round-trip (your ids
+  and the Solr `md5(uri)` ids), deletes by id and by query.
+- **Real-corpus validation**: searches run against a 340-document replica of
+  opensolr.com's own production search index. Verified: pure-semantic hits
+  with zero keyword overlap ("how do I get my data back after a disaster" →
+  backup & restore docs), cross-lingual queries (Romanian query → English
+  content), exact-term surfacing in hybrid mode, all four hybrid modes, and
+  the full alpha range 0 → 1.
+- **PDF ingestion**: a real PDF ingested via `rtf:true` — server-side text
+  extraction (13k+ chars), automatic content-type detection, then retrieved
+  with a purely semantic query against its contents.
+- **Grounded RAG answers**: `ai_answer` verified end-to-end — a question answerable only from the ingested PDF returns the correct answer, sourced from the PDF's extracted text via hybrid retrieval.
+
+The store is exercised live (write via ingestion, DuplicatePolicy SKIP/FAIL,
+hybrid + lexical retrieval, filters, serde round-trip) before every release.
+
 MIT license.

{opensolr_haystack-0.2.0 → opensolr_haystack-0.2.2}/haystack_integrations/document_stores/opensolr/client.py

@@ -138,9 +138,16 @@ class OpensolrClient:
                 f"Currently available: {sorted(live)}. Additional regions can "
                 f"be deployed on request — contact support@opensolr.com."
             )
-        return self.mgmt(
-            "create_index", index_name=index, core_type="generic", server_country=env
+        # create_index reads its params from the query string (GET) server-side
+        resp = self._http.post(
+            f"{MGMT_BASE}/create_index",
+            params={"index_name": index, "core_type": "generic", "server_country": env},
+            data=self._auth_params(),
         )
+        body = resp.json()
+        if isinstance(body, dict) and body.get("status") is False:
+            raise OpensolrError(f"create_index: {body.get('msg', body)}")
+        return body
 
     # ------------------------------------------------------------------ #
     # AI                                                                 #
@@ -257,6 +264,125 @@ class OpensolrClient:
         resp.raise_for_status()
         return resp.json()
 
+    def hybrid_search(
+        self,
+        index: str,
+        query: str,
+        rows: int = 5,
+        mode: str = "union",
+        alpha: float = 0.5,
+        fl: str = "*,score",
+        fq: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Hybrid (BM25 + kNN) search via the native ``{!hybrid}`` parser.
+
+        The query is embedded server-side; lexical and vector scores are
+        fused per document on the Solr side.
+        """
+        clean = query.replace("{", " ").replace("}", " ").replace('"', " ")
+        vector = self.embed(index, query, is_query=True)
+        compact = json.dumps(vector, separators=(",", ":"))
+        params: Dict[str, Any] = {
+            "q": (
+                f"{{!hybrid lexical=$lexicalRaw vector=$vectorQuery "
+                f"mode={mode} alpha={alpha} topN={max(rows, 10)}}}"
+            ),
+            "lexicalRaw": f'{{!edismax qf="title^100 text^1"}}{clean}',
+            "vectorQuery": f"{{!knn f=embeddings topK={max(rows, 10)}}}{compact}",
+            "rows": rows,
+            "fl": fl,
+        }
+        if fq:
+            params["fq"] = fq
+        return self.solr_select(index, params)
+
+    #: RAG context defaults — how many hybrid hits feed the LLM, and how many
+    #: words of each hit's text are included. Both overridable per call.
+    RAG_DOCS = 3
+    RAG_WORDS = 1500
+
+    def _rag_context(
+        self,
+        index: str,
+        query: str,
+        fq: Optional[str] = None,
+        docs: Optional[int] = None,
+        words: Optional[int] = None,
+    ) -> str:
+        """Build the LLM context from the top hybrid search hits."""
+
+        def _flat(v: Any) -> str:
+            if isinstance(v, list):
+                v = " ".join(str(x) for x in v)
+            return str(v or "")
+
+        docs = docs or self.RAG_DOCS
+        words = words or self.RAG_WORDS
+        body = self.hybrid_search(
+            index, query, rows=docs, fl="title,description,text", fq=fq
+        )
+        parts: List[str] = []
+        for doc in body.get("response", {}).get("docs", []):
+            text_words = _flat(doc.get("text")).split()[:words]
+            parts.append(
+                _flat(doc.get("title")) + " - "
+                + _flat(doc.get("description")) + " - "
+                + " ".join(text_words) + " - "
+            )
+        return "".join(parts)
+
+    def ai_summary(
+        self,
+        index: str,
+        query: str,
+        filter_query: Optional[str] = None,
+        rag_docs: Optional[int] = None,
+        rag_words: Optional[int] = None,
+        instruction: Optional[str] = None,
+        **params: Any,
+    ) -> str:
+        """Grounded RAG answer: hybrid retrieval over the index feeds the LLM.
+
+        Retrieval runs client-side via ``hybrid_search`` (same pipeline as the
+        hosted search UI): the top ``rag_docs`` hits' title/description/text
+        (first ``rag_words`` words each) become the LLM context. Pass
+        ``instruction`` to fully control the prompt (e.g. "Answer in German",
+        "Extract a list of people"). If retrieval fails or returns nothing,
+        the server falls back to its own retrieval. Returns plain text.
+        """
+        data = {
+            **self._auth_params(),
+            "index_name": index,
+            "query": query,
+            "stream": "false",
+            **params,
+        }
+        if instruction:
+            data["instruction"] = instruction
+        if "context" not in data:
+            try:
+                context = self._rag_context(
+                    index, query, fq=filter_query, docs=rag_docs, words=rag_words
+                )
+            except (OpensolrError, httpx.HTTPError):
+                context = ""
+            if context:
+                data["context"] = context
+                data.setdefault(
+                    "instruction",
+                    "Read and understand the full context below, and formulate "
+                    f"a clear, concise and factual answer to: '{query}'.\n"
+                    "Answer ONLY from the context. Format the answer in "
+                    "Markdown, use bold section headers where they help, and "
+                    "cite exact titles or names from the context when "
+                    "referring to them.\n",
+                )
+        resp = self._http.post(f"{AI_BASE}/ai_summary", data=data)
+        if resp.status_code >= 400:
+            raise OpensolrError(f"ai_summary: HTTP {resp.status_code}: {resp.text[:200]}")
+        # The stream is prefixed with flush-padding whitespace — strip it.
+        return resp.text.strip()
+
     def solr_update(self, index: str, payload: Any, commit: bool = True) -> Dict[str, Any]:
         base, auth = self.solr_endpoint(index)
         params = {"commit": "true"} if commit else {"commitWithin": "10000"}

{opensolr_haystack-0.2.0 → opensolr_haystack-0.2.2}/haystack_integrations/document_stores/opensolr/store.py

@@ -248,6 +248,32 @@ class OpensolrDocumentStore:
             self.client.ingest(self.index, docs[i : i + 50], wait=self.ingest_wait)
         return len(docs)
 
+    def ai_answer(
+        self,
+        query: str,
+        filters: Optional[Dict[str, Any]] = None,
+        rag_docs: int = 3,
+        rag_words: int = 1500,
+        instruction: Optional[str] = None,
+        **kwargs: Any,
+    ) -> str:
+        """Grounded RAG answer generated only from this index's content.
+
+        Two-step pattern: hybrid (BM25 + kNN) retrieval picks the top
+        ``rag_docs`` hits (first ``rag_words`` words of text each), whose
+        title/description/text become the LLM context — the same pipeline as
+        Opensolr's hosted search UI. Pass ``instruction`` to fully control
+        the prompt (e.g. "Answer in German, cite the sources you used").
+        Returns plain text.
+        """
+        fqs = _filters_to_fq(filters)
+        fq = " AND ".join(f"({f})" for f in fqs) if fqs else None
+        return self.client.ai_summary(
+            self.index, query, filter_query=fq,
+            rag_docs=rag_docs, rag_words=rag_words, instruction=instruction,
+            **kwargs,
+        )
+
     def delete_documents(self, document_ids: List[str]) -> None:
         if not document_ids:
             return

{opensolr_haystack-0.2.0 → opensolr_haystack-0.2.2/opensolr_haystack.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensolr-haystack
-Version: 0.2.0
+Version: 0.2.2
 Summary: Haystack integration for Opensolr — managed Apache Solr DocumentStore with server-side embeddings and hybrid BM25+kNN retrieval
 Author-email: Opensolr <support@opensolr.com>
 License: MIT
@@ -121,4 +121,42 @@ entry? Configure the **Web Crawler** in the Control Panel (Index Tools →
 WebCrawler): add your site URL, validate it, and Opensolr indexes the whole
 site for you.
 
+## Grounded RAG answers
+
+One call: hybrid retrieval picks the top hits, whose content becomes the LLM
+context, and Opensolr's server-side LLM answers — no generator component,
+no LLM key:
+
+```python
+answer = store.ai_answer(
+    "what does the refund policy say?",
+    rag_docs=3,        # how many hybrid hits feed the LLM (default 3)
+    rag_words=1500,    # words of text taken from each hit (default 1500)
+    # instruction="Answer in German, cite the exact titles you used",  # optional
+)
+```
+
+## How it's tested
+
+Every release is validated against **live Opensolr infrastructure** — no mocks:
+
+- **Unit tests** (offline): location aliases, filter→fq mapping, query building, escaping.
+- **End-to-end suite**: the full write path through the async Data Ingestion
+  queue (queued → server-side enrichment → searchable), semantic / hybrid /
+  lexical retrieval, metadata round-trip, filters, id round-trip (your ids
+  and the Solr `md5(uri)` ids), deletes by id and by query.
+- **Real-corpus validation**: searches run against a 340-document replica of
+  opensolr.com's own production search index. Verified: pure-semantic hits
+  with zero keyword overlap ("how do I get my data back after a disaster" →
+  backup &amp; restore docs), cross-lingual queries (Romanian query → English
+  content), exact-term surfacing in hybrid mode, all four hybrid modes, and
+  the full alpha range 0 → 1.
+- **PDF ingestion**: a real PDF ingested via `rtf:true` — server-side text
+  extraction (13k+ chars), automatic content-type detection, then retrieved
+  with a purely semantic query against its contents.
+- **Grounded RAG answers**: `ai_answer` verified end-to-end — a question answerable only from the ingested PDF returns the correct answer, sourced from the PDF's extracted text via hybrid retrieval.
+
+The store is exercised live (write via ingestion, DuplicatePolicy SKIP/FAIL,
+hybrid + lexical retrieval, filters, serde round-trip) before every release.
+
 MIT license.

{opensolr_haystack-0.2.0 → opensolr_haystack-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opensolr-haystack"
-version = "0.2.0"
+version = "0.2.2"
 description = "Haystack integration for Opensolr — managed Apache Solr DocumentStore with server-side embeddings and hybrid BM25+kNN retrieval"
 readme = "README.md"
 license = { text = "MIT" }