polias-haystack 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polias_haystack-0.1.0/.github/workflows/publish.yml +20 -0
- polias_haystack-0.1.0/.gitignore +7 -0
- polias_haystack-0.1.0/LICENSE +21 -0
- polias_haystack-0.1.0/PKG-INFO +79 -0
- polias_haystack-0.1.0/README.md +65 -0
- polias_haystack-0.1.0/pyproject.toml +25 -0
- polias_haystack-0.1.0/src/haystack_integrations/components/websearch/polias/__init__.py +3 -0
- polias_haystack-0.1.0/src/haystack_integrations/components/websearch/polias/polias_websearch.py +306 -0
- polias_haystack-0.1.0/tests/test_polias_websearch.py +78 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Publishes to PyPI via Trusted Publishing when a version tag (v*) is pushed. No token needed.
|
|
2
|
+
name: Publish
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ['v*']
|
|
6
|
+
jobs:
|
|
7
|
+
publish:
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
environment: pypi
|
|
10
|
+
permissions:
|
|
11
|
+
id-token: write
|
|
12
|
+
contents: read
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: '3.12'
|
|
18
|
+
- run: python -m pip install build
|
|
19
|
+
- run: python -m build
|
|
20
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Polias
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: polias-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Haystack integration for Polias search: web, news, official sources and law
|
|
5
|
+
Project-URL: Homepage, https://api.polias.eu/
|
|
6
|
+
Project-URL: Source, https://github.com/Polias-Team/polias-haystack
|
|
7
|
+
Author-email: Polias <hallo@polias.eu>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: Haystack,Legal,Polias,Web Search
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Requires-Dist: haystack-ai>=2.12.0
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
# polias-haystack
|
|
16
|
+
|
|
17
|
+
This package connects Haystack to [Polias](https://api.polias.eu/), a search engine with its own index, hosted in the EU.
|
|
18
|
+
Use it in RAG pipelines or with Haystack Agents to search the web, news, official sources and law.
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install polias-haystack
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
You need a Polias API key. Create an account at [api.polias.eu/account](https://api.polias.eu/account), as a company or as a
|
|
27
|
+
private person (private accounts: residents of Germany, Austria or Switzerland, 18 or older, non-commercial use only).
|
|
28
|
+
Each month, 1,000 searches are free for companies and 100 for private persons. More is available as packages, see
|
|
29
|
+
[pricing](https://api.polias.eu/pricing).
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
export POLIAS_API_KEY="pol_..."
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Usage
|
|
36
|
+
|
|
37
|
+
### PoliasWebSearch
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from haystack_integrations.components.websearch.polias import PoliasWebSearch
|
|
41
|
+
|
|
42
|
+
websearch = PoliasWebSearch(top_k=5, areas=["web", "legal"])
|
|
43
|
+
result = websearch.run(query="Kündigung Eigenbedarf")
|
|
44
|
+
documents = result["documents"] # snippet as content; title, link, area in meta
|
|
45
|
+
links = result["links"]
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The areas are:
|
|
49
|
+
- `web`;
|
|
50
|
+
- `news`: articles from the last 30 days;
|
|
51
|
+
- `legal`: court decisions and legislation;
|
|
52
|
+
- `official`: websites of public authorities in Germany, Austria, Switzerland and the EU. Use it without `web`.
|
|
53
|
+
|
|
54
|
+
### PoliasAISearch
|
|
55
|
+
|
|
56
|
+
This component returns documents for AI answers. Each document has licence details. Where the licence allows, it holds a
|
|
57
|
+
longer excerpt, and for legal results the full text.
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from haystack_integrations.components.websearch.polias import PoliasAISearch
|
|
61
|
+
|
|
62
|
+
search = PoliasAISearch(top_k=3, areas=["web", "legal"])
|
|
63
|
+
documents = search.run(query="Mietminderung Schimmel")["documents"]
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Please show the source notice (`meta["notice"]`) with legal results.
|
|
67
|
+
|
|
68
|
+
### Options
|
|
69
|
+
|
|
70
|
+
- `lang`: `"de"` (the default) or `"en"`. English focuses on technical topics such as developer documentation, software
|
|
71
|
+
and IT.
|
|
72
|
+
- `country`: `"de"`, `"at"` or `"ch"`, to prefer results from that country.
|
|
73
|
+
- `api_key`: a `Secret`. It defaults to the environment variable `POLIAS_API_KEY`.
|
|
74
|
+
|
|
75
|
+
Both components support `run_async`.
|
|
76
|
+
|
|
77
|
+
## License
|
|
78
|
+
|
|
79
|
+
MIT. Documentation: [api.polias.eu](https://api.polias.eu/) · Questions: hallo@polias.eu
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# polias-haystack
|
|
2
|
+
|
|
3
|
+
This package connects Haystack to [Polias](https://api.polias.eu/), a search engine with its own index, hosted in the EU.
|
|
4
|
+
Use it in RAG pipelines or with Haystack Agents to search the web, news, official sources and law.
|
|
5
|
+
|
|
6
|
+
## Installation
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install polias-haystack
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
You need a Polias API key. Create an account at [api.polias.eu/account](https://api.polias.eu/account), as a company or as a
|
|
13
|
+
private person (private accounts: residents of Germany, Austria or Switzerland, 18 or older, non-commercial use only).
|
|
14
|
+
Each month, 1,000 searches are free for companies and 100 for private persons. More is available as packages, see
|
|
15
|
+
[pricing](https://api.polias.eu/pricing).
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
export POLIAS_API_KEY="pol_..."
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
### PoliasWebSearch
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from haystack_integrations.components.websearch.polias import PoliasWebSearch
|
|
27
|
+
|
|
28
|
+
websearch = PoliasWebSearch(top_k=5, areas=["web", "legal"])
|
|
29
|
+
result = websearch.run(query="Kündigung Eigenbedarf")
|
|
30
|
+
documents = result["documents"] # snippet as content; title, link, area in meta
|
|
31
|
+
links = result["links"]
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The areas are:
|
|
35
|
+
- `web`;
|
|
36
|
+
- `news`: articles from the last 30 days;
|
|
37
|
+
- `legal`: court decisions and legislation;
|
|
38
|
+
- `official`: websites of public authorities in Germany, Austria, Switzerland and the EU. Use it without `web`.
|
|
39
|
+
|
|
40
|
+
### PoliasAISearch
|
|
41
|
+
|
|
42
|
+
This component returns documents for AI answers. Each document has licence details. Where the licence allows, it holds a
|
|
43
|
+
longer excerpt, and for legal results the full text.
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from haystack_integrations.components.websearch.polias import PoliasAISearch
|
|
47
|
+
|
|
48
|
+
search = PoliasAISearch(top_k=3, areas=["web", "legal"])
|
|
49
|
+
documents = search.run(query="Mietminderung Schimmel")["documents"]
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Please show the source notice (`meta["notice"]`) with legal results.
|
|
53
|
+
|
|
54
|
+
### Options
|
|
55
|
+
|
|
56
|
+
- `lang`: `"de"` (the default) or `"en"`. English focuses on technical topics such as developer documentation, software
|
|
57
|
+
and IT.
|
|
58
|
+
- `country`: `"de"`, `"at"` or `"ch"`, to prefer results from that country.
|
|
59
|
+
- `api_key`: a `Secret`. It defaults to the environment variable `POLIAS_API_KEY`.
|
|
60
|
+
|
|
61
|
+
Both components support `run_async`.
|
|
62
|
+
|
|
63
|
+
## License
|
|
64
|
+
|
|
65
|
+
MIT. Documentation: [api.polias.eu](https://api.polias.eu/) · Questions: hallo@polias.eu
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "polias-haystack"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Haystack integration for Polias search: web, news, official sources and law"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
keywords = ["Haystack", "Polias", "Web Search", "Legal"]
|
|
13
|
+
authors = [{ name = "Polias", email = "hallo@polias.eu" }]
|
|
14
|
+
dependencies = ["haystack-ai>=2.12.0"]
|
|
15
|
+
|
|
16
|
+
[project.urls]
|
|
17
|
+
Homepage = "https://api.polias.eu/"
|
|
18
|
+
Source = "https://github.com/Polias-Team/polias-haystack"
|
|
19
|
+
|
|
20
|
+
[tool.hatch.build.targets.wheel]
|
|
21
|
+
packages = ["src/haystack_integrations"]
|
|
22
|
+
|
|
23
|
+
[tool.pytest.ini_options]
|
|
24
|
+
asyncio_mode = "auto"
|
|
25
|
+
markers = ["integration: needs POLIAS_API_KEY"]
|
polias_haystack-0.1.0/src/haystack_integrations/components/websearch/polias/polias_websearch.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026-present Polias <hallo@polias.eu>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
|
8
|
+
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
9
|
+
from haystack.utils.requests_utils import async_request_with_retry, request_with_retry
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
POLIAS_API_URL = "https://api.polias.eu/v1"
|
|
14
|
+
|
|
15
|
+
_AREA_PARAM = {"legal": "recht", "official": "amtlich"}
|
|
16
|
+
_AREA_NAME = {"recht": "legal", "amtlich": "official"}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _clean(meta: dict[str, Any]) -> dict[str, Any]:
|
|
20
|
+
return {key: val for key, val in meta.items() if val not in (None, "")}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class _PoliasBase:
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
api_key: Secret,
|
|
27
|
+
top_k: int | None,
|
|
28
|
+
areas: list[str] | None,
|
|
29
|
+
lang: str | None,
|
|
30
|
+
country: str | None,
|
|
31
|
+
timeout: int,
|
|
32
|
+
max_retries: int,
|
|
33
|
+
) -> None:
|
|
34
|
+
self.api_key = api_key
|
|
35
|
+
self.top_k = top_k
|
|
36
|
+
self.areas = areas
|
|
37
|
+
self.lang = lang
|
|
38
|
+
self.country = country
|
|
39
|
+
self.timeout = timeout
|
|
40
|
+
self.max_retries = max_retries
|
|
41
|
+
|
|
42
|
+
def to_dict(self) -> dict[str, Any]:
|
|
43
|
+
"""
|
|
44
|
+
Serializes the component to a dictionary.
|
|
45
|
+
|
|
46
|
+
:returns: Dictionary with serialized data.
|
|
47
|
+
"""
|
|
48
|
+
return default_to_dict(
|
|
49
|
+
self,
|
|
50
|
+
api_key=self.api_key.to_dict(),
|
|
51
|
+
top_k=self.top_k,
|
|
52
|
+
areas=self.areas,
|
|
53
|
+
lang=self.lang,
|
|
54
|
+
country=self.country,
|
|
55
|
+
timeout=self.timeout,
|
|
56
|
+
max_retries=self.max_retries,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
@classmethod
|
|
60
|
+
def from_dict(cls, data: dict[str, Any]) -> Any:
|
|
61
|
+
"""
|
|
62
|
+
Deserializes the component from a dictionary.
|
|
63
|
+
|
|
64
|
+
:param data: Dictionary to deserialize from.
|
|
65
|
+
:returns: Deserialized component.
|
|
66
|
+
"""
|
|
67
|
+
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
|
|
68
|
+
return default_from_dict(cls, data)
|
|
69
|
+
|
|
70
|
+
def _headers(self) -> dict[str, str]:
|
|
71
|
+
return {"Accept": "application/json", "Authorization": f"Bearer {self.api_key.resolve_value() or ''}"}
|
|
72
|
+
|
|
73
|
+
def _get(self, path: str, params: dict[str, Any]) -> dict[str, Any]:
|
|
74
|
+
response = request_with_retry(
|
|
75
|
+
attempts=self.max_retries,
|
|
76
|
+
method="GET",
|
|
77
|
+
url=POLIAS_API_URL + path,
|
|
78
|
+
params={k: v for k, v in params.items() if v not in (None, "")},
|
|
79
|
+
headers=self._headers(),
|
|
80
|
+
timeout=self.timeout,
|
|
81
|
+
)
|
|
82
|
+
response.raise_for_status()
|
|
83
|
+
return response.json()
|
|
84
|
+
|
|
85
|
+
async def _get_async(self, path: str, params: dict[str, Any]) -> dict[str, Any]:
|
|
86
|
+
response = await async_request_with_retry(
|
|
87
|
+
attempts=self.max_retries,
|
|
88
|
+
method="GET",
|
|
89
|
+
url=POLIAS_API_URL + path,
|
|
90
|
+
params={k: v for k, v in params.items() if v not in (None, "")},
|
|
91
|
+
headers=self._headers(),
|
|
92
|
+
timeout=self.timeout,
|
|
93
|
+
)
|
|
94
|
+
response.raise_for_status()
|
|
95
|
+
return response.json()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@component
|
|
99
|
+
class PoliasWebSearch(_PoliasBase):
|
|
100
|
+
"""
|
|
101
|
+
Searches the web, news, official sources and law with Polias and returns the results as Haystack Documents.
|
|
102
|
+
|
|
103
|
+
Polias is a search engine with its own index, hosted in the EU. You need an API key from
|
|
104
|
+
[api.polias.eu/account](https://api.polias.eu/account).
|
|
105
|
+
|
|
106
|
+
### Usage example
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from haystack.utils import Secret
|
|
110
|
+
from haystack_integrations.components.websearch.polias import PoliasWebSearch
|
|
111
|
+
|
|
112
|
+
websearch = PoliasWebSearch(api_key=Secret.from_env_var("POLIAS_API_KEY"), top_k=5, areas=["web", "legal"])
|
|
113
|
+
result = websearch.run(query="Kündigung Eigenbedarf")
|
|
114
|
+
documents = result["documents"]
|
|
115
|
+
links = result["links"]
|
|
116
|
+
```
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
def __init__(
|
|
120
|
+
self,
|
|
121
|
+
api_key: Secret = Secret.from_env_var("POLIAS_API_KEY"),
|
|
122
|
+
top_k: int | None = 10,
|
|
123
|
+
areas: list[str] | None = None,
|
|
124
|
+
lang: str | None = None,
|
|
125
|
+
country: str | None = None,
|
|
126
|
+
timeout: int = 20,
|
|
127
|
+
max_retries: int = 3,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""
|
|
130
|
+
Initialize the PoliasWebSearch component.
|
|
131
|
+
|
|
132
|
+
:param api_key: Polias API key. Defaults to the `POLIAS_API_KEY` environment variable.
|
|
133
|
+
:param top_k: Maximum number of results (1–10).
|
|
134
|
+
:param areas: Areas to search: `"web"`, `"news"` (last 30 days), `"legal"` (court decisions and legislation)
|
|
135
|
+
and `"official"` (websites of public authorities; not together with `"web"`). Defaults to web search.
|
|
136
|
+
:param lang: Index language: `"de"` (default) or `"en"` (focus on technical topics).
|
|
137
|
+
:param country: Prefer results from a country: `"de"`, `"at"` or `"ch"`.
|
|
138
|
+
:param timeout: Timeout in seconds for the HTTP request.
|
|
139
|
+
:param max_retries: Maximum number of retry attempts on transient failures.
|
|
140
|
+
"""
|
|
141
|
+
_PoliasBase.__init__(self, api_key, top_k, areas, lang, country, timeout, max_retries)
|
|
142
|
+
|
|
143
|
+
def _params(self, query: str, top_k: int | None) -> dict[str, Any]:
|
|
144
|
+
return {
|
|
145
|
+
"q": query,
|
|
146
|
+
"count": top_k if top_k is not None else self.top_k,
|
|
147
|
+
"result_filter": ",".join(_AREA_PARAM.get(a, a) for a in (self.areas or [])),
|
|
148
|
+
"lang": self.lang,
|
|
149
|
+
"land": self.country,
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
@component.output_types(documents=list[Document], links=list[str])
|
|
153
|
+
def run(self, query: str, top_k: int | None = None) -> dict[str, Any]:
|
|
154
|
+
"""
|
|
155
|
+
Search with Polias and return the results as Documents.
|
|
156
|
+
|
|
157
|
+
:param query: Search query, up to 200 characters.
|
|
158
|
+
:param top_k: Optional per-run override of the maximum number of results.
|
|
159
|
+
:returns: A dictionary with:
|
|
160
|
+
- `documents`: List of Documents with the snippet as content.
|
|
161
|
+
- `links`: List of URLs from the search results.
|
|
162
|
+
"""
|
|
163
|
+
return self._parse(self._get("/search", self._params(query, top_k)))
|
|
164
|
+
|
|
165
|
+
@component.output_types(documents=list[Document], links=list[str])
|
|
166
|
+
async def run_async(self, query: str, top_k: int | None = None) -> dict[str, Any]:
|
|
167
|
+
"""
|
|
168
|
+
Asynchronously search with Polias and return the results as Documents.
|
|
169
|
+
|
|
170
|
+
:param query: Search query, up to 200 characters.
|
|
171
|
+
:param top_k: Optional per-run override of the maximum number of results.
|
|
172
|
+
:returns: A dictionary with `documents` and `links`.
|
|
173
|
+
"""
|
|
174
|
+
return self._parse(await self._get_async("/search", self._params(query, top_k)))
|
|
175
|
+
|
|
176
|
+
@staticmethod
|
|
177
|
+
def _parse(data: dict[str, Any]) -> dict[str, Any]:
|
|
178
|
+
if "results" in data:
|
|
179
|
+
blocks = {"web": data["results"]}
|
|
180
|
+
else:
|
|
181
|
+
blocks = {a: data.get(a, {}).get("results", []) for a in data.get("result_filter", [])}
|
|
182
|
+
documents: list[Document] = []
|
|
183
|
+
links: list[str] = []
|
|
184
|
+
for area, results in blocks.items():
|
|
185
|
+
for r in results:
|
|
186
|
+
url = r.get("url", "")
|
|
187
|
+
documents.append(
|
|
188
|
+
Document(
|
|
189
|
+
content=r.get("snippet", ""),
|
|
190
|
+
meta=_clean(
|
|
191
|
+
{
|
|
192
|
+
"title": r.get("title"),
|
|
193
|
+
"link": url,
|
|
194
|
+
"area": _AREA_NAME.get(area, area),
|
|
195
|
+
"published": r.get("page_age"),
|
|
196
|
+
"notice": r.get("hinweis"),
|
|
197
|
+
}
|
|
198
|
+
),
|
|
199
|
+
)
|
|
200
|
+
)
|
|
201
|
+
if url:
|
|
202
|
+
links.append(url)
|
|
203
|
+
return {"documents": documents, "links": links}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@component
|
|
207
|
+
class PoliasAISearch(_PoliasBase):
|
|
208
|
+
"""
|
|
209
|
+
Searches with Polias for use in AI answers.
|
|
210
|
+
|
|
211
|
+
Each result carries licence details and, where the licence allows, a longer excerpt. Legal results include the
|
|
212
|
+
full text where the licence allows it.
|
|
213
|
+
|
|
214
|
+
### Usage example
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
from haystack_integrations.components.websearch.polias import PoliasAISearch
|
|
218
|
+
|
|
219
|
+
search = PoliasAISearch(top_k=3, areas=["web", "legal"])
|
|
220
|
+
documents = search.run(query="Mietminderung Schimmel")["documents"]
|
|
221
|
+
```
|
|
222
|
+
"""
|
|
223
|
+
|
|
224
|
+
def __init__(
|
|
225
|
+
self,
|
|
226
|
+
api_key: Secret = Secret.from_env_var("POLIAS_API_KEY"),
|
|
227
|
+
top_k: int | None = 5,
|
|
228
|
+
areas: list[str] | None = None,
|
|
229
|
+
lang: str | None = None,
|
|
230
|
+
country: str | None = None,
|
|
231
|
+
timeout: int = 30,
|
|
232
|
+
max_retries: int = 3,
|
|
233
|
+
) -> None:
|
|
234
|
+
"""
|
|
235
|
+
Initialize the PoliasAISearch component.
|
|
236
|
+
|
|
237
|
+
:param api_key: Polias API key. Defaults to the `POLIAS_API_KEY` environment variable.
|
|
238
|
+
:param top_k: Maximum number of results per area (1–10).
|
|
239
|
+
:param areas: Areas: `"web"`, `"news"` and `"legal"`. Defaults to web and news.
|
|
240
|
+
:param lang: Index language: `"de"` (default) or `"en"`.
|
|
241
|
+
:param country: Prefer results from a country: `"de"`, `"at"` or `"ch"`.
|
|
242
|
+
:param timeout: Timeout in seconds for the HTTP request.
|
|
243
|
+
:param max_retries: Maximum number of retry attempts on transient failures.
|
|
244
|
+
"""
|
|
245
|
+
_PoliasBase.__init__(self, api_key, top_k, areas, lang, country, timeout, max_retries)
|
|
246
|
+
|
|
247
|
+
def _params(self, query: str, top_k: int | None) -> dict[str, Any]:
|
|
248
|
+
return {
|
|
249
|
+
"q": query,
|
|
250
|
+
"count": top_k if top_k is not None else self.top_k,
|
|
251
|
+
"scope": ",".join(self.areas or []),
|
|
252
|
+
"lang": self.lang,
|
|
253
|
+
"land": self.country,
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
@component.output_types(documents=list[Document], links=list[str])
|
|
257
|
+
def run(self, query: str, top_k: int | None = None) -> dict[str, Any]:
|
|
258
|
+
"""
|
|
259
|
+
Search with Polias and return Documents for AI answers.
|
|
260
|
+
|
|
261
|
+
:param query: Search query, up to 200 characters.
|
|
262
|
+
:param top_k: Optional per-run override of the maximum number of results per area.
|
|
263
|
+
:returns: A dictionary with `documents` (excerpt or full text as content) and `links`.
|
|
264
|
+
"""
|
|
265
|
+
return self._parse(self._get("/ki", self._params(query, top_k)))
|
|
266
|
+
|
|
267
|
+
@component.output_types(documents=list[Document], links=list[str])
|
|
268
|
+
async def run_async(self, query: str, top_k: int | None = None) -> dict[str, Any]:
|
|
269
|
+
"""
|
|
270
|
+
Asynchronously search with Polias and return Documents for AI answers.
|
|
271
|
+
|
|
272
|
+
:param query: Search query, up to 200 characters.
|
|
273
|
+
:param top_k: Optional per-run override of the maximum number of results per area.
|
|
274
|
+
:returns: A dictionary with `documents` and `links`.
|
|
275
|
+
"""
|
|
276
|
+
return self._parse(await self._get_async("/ki", self._params(query, top_k)))
|
|
277
|
+
|
|
278
|
+
@staticmethod
|
|
279
|
+
def _parse(data: dict[str, Any]) -> dict[str, Any]:
|
|
280
|
+
documents: list[Document] = []
|
|
281
|
+
links: list[str] = []
|
|
282
|
+
for area in ("web", "news", "official", "legal"):
|
|
283
|
+
for r in data.get(area) or []:
|
|
284
|
+
url = r.get("url", "")
|
|
285
|
+
documents.append(
|
|
286
|
+
Document(
|
|
287
|
+
content=r.get("volltext") or r.get("excerpt") or r.get("snippet") or "",
|
|
288
|
+
meta=_clean(
|
|
289
|
+
{
|
|
290
|
+
"title": r.get("title") or r.get("titel"),
|
|
291
|
+
"link": url,
|
|
292
|
+
"area": area,
|
|
293
|
+
"date": r.get("as_of") or r.get("date"),
|
|
294
|
+
"license": r.get("license") or r.get("lizenz"),
|
|
295
|
+
"notice": r.get("notice") or r.get("hinweis"),
|
|
296
|
+
"court": r.get("court"),
|
|
297
|
+
"case_number": r.get("case_number"),
|
|
298
|
+
"ecli": r.get("ecli"),
|
|
299
|
+
"full_text": bool(r.get("volltext")),
|
|
300
|
+
}
|
|
301
|
+
),
|
|
302
|
+
)
|
|
303
|
+
)
|
|
304
|
+
if url:
|
|
305
|
+
links.append(url)
|
|
306
|
+
return {"documents": documents, "links": links}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from unittest.mock import Mock, patch
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
from haystack import Pipeline
|
|
6
|
+
from haystack.utils import Secret
|
|
7
|
+
|
|
8
|
+
from haystack_integrations.components.websearch.polias import PoliasAISearch, PoliasWebSearch
|
|
9
|
+
|
|
10
|
+
MOD = "haystack_integrations.components.websearch.polias.polias_websearch"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _resp(payload):
|
|
14
|
+
r = Mock()
|
|
15
|
+
r.json.return_value = payload
|
|
16
|
+
r.raise_for_status.return_value = None
|
|
17
|
+
return r
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_to_from_dict(monkeypatch):
|
|
21
|
+
monkeypatch.setenv("POLIAS_API_KEY", "pol_env")
|
|
22
|
+
c = PoliasWebSearch(top_k=3, areas=["web", "legal"], country="at")
|
|
23
|
+
d = c.to_dict()
|
|
24
|
+
assert d["init_parameters"]["api_key"] == {"type": "env_var", "env_vars": ["POLIAS_API_KEY"], "strict": True}
|
|
25
|
+
assert "pol_env" not in str(d)
|
|
26
|
+
c2 = PoliasWebSearch.from_dict(d)
|
|
27
|
+
assert c2.areas == ["web", "legal"] and c2.country == "at"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_pipeline_roundtrip(monkeypatch):
|
|
31
|
+
monkeypatch.setenv("POLIAS_API_KEY", "pol_env")
|
|
32
|
+
p = Pipeline()
|
|
33
|
+
p.add_component("search", PoliasAISearch(areas=["legal"]))
|
|
34
|
+
assert Pipeline.loads(p.dumps()).get_component("search").areas == ["legal"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@patch(f"{MOD}.request_with_retry")
|
|
38
|
+
def test_web_search(mock_req):
|
|
39
|
+
mock_req.return_value = _resp(
|
|
40
|
+
{
|
|
41
|
+
"result_filter": ["web", "legal"],
|
|
42
|
+
"web": {"results": [{"title": "W", "url": "https://w", "snippet": "s"}]},
|
|
43
|
+
"legal": {"results": [{"title": "L", "url": "https://l", "snippet": "l", "hinweis": "Quelle"}]},
|
|
44
|
+
}
|
|
45
|
+
)
|
|
46
|
+
out = PoliasWebSearch(api_key=Secret.from_token("pol_t"), areas=["web", "legal"]).run("q", top_k=4)
|
|
47
|
+
kwargs = mock_req.call_args.kwargs
|
|
48
|
+
assert kwargs["url"] == "https://api.polias.eu/v1/search"
|
|
49
|
+
assert kwargs["params"] == {"q": "q", "count": 4, "result_filter": "web,recht"}
|
|
50
|
+
assert kwargs["headers"]["Authorization"] == "Bearer pol_t"
|
|
51
|
+
assert out["links"] == ["https://w", "https://l"]
|
|
52
|
+
assert [d.meta["area"] for d in out["documents"]] == ["web", "legal"]
|
|
53
|
+
assert out["documents"][1].meta["notice"] == "Quelle"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@patch(f"{MOD}.request_with_retry")
|
|
57
|
+
def test_ai_search_full_text(mock_req):
|
|
58
|
+
mock_req.return_value = _resp({"legal": [{"titel": "U", "url": "https://u", "volltext": "VT", "lizenz": "CC0"}]})
|
|
59
|
+
out = PoliasAISearch(api_key=Secret.from_token("pol_t"), areas=["legal"]).run("q")
|
|
60
|
+
assert mock_req.call_args.kwargs["params"]["scope"] == "legal"
|
|
61
|
+
doc = out["documents"][0]
|
|
62
|
+
assert doc.content == "VT" and doc.meta["full_text"] is True and doc.meta["license"] == "CC0"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@pytest.mark.skipif(not os.environ.get("POLIAS_API_KEY"), reason="needs POLIAS_API_KEY")
|
|
66
|
+
@pytest.mark.integration
|
|
67
|
+
def test_live():
|
|
68
|
+
out = PoliasWebSearch(top_k=3, areas=["web", "legal"]).run("VIII ZR 15/23")
|
|
69
|
+
assert out["documents"] and out["links"]
|
|
70
|
+
ai = PoliasAISearch(top_k=2, areas=["legal"]).run("Mietminderung Schimmel")
|
|
71
|
+
assert any(d.meta.get("full_text") for d in ai["documents"])
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@pytest.mark.skipif(not os.environ.get("POLIAS_API_KEY"), reason="needs POLIAS_API_KEY")
|
|
75
|
+
@pytest.mark.integration
|
|
76
|
+
async def test_live_async():
|
|
77
|
+
out = await PoliasWebSearch(top_k=2, areas=["news"]).run_async("bundestag")
|
|
78
|
+
assert all(d.meta["area"] == "news" for d in out["documents"])
|