cloro-haystack 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloro_haystack-0.1.0.dist-info/METADATA +134 -0
- cloro_haystack-0.1.0.dist-info/RECORD +9 -0
- cloro_haystack-0.1.0.dist-info/WHEEL +4 -0
- cloro_haystack-0.1.0.dist-info/licenses/LICENSE +21 -0
- haystack_integrations/components/websearch/cloro/__init__.py +9 -0
- haystack_integrations/components/websearch/cloro/_api.py +76 -0
- haystack_integrations/components/websearch/cloro/ai_answer.py +157 -0
- haystack_integrations/components/websearch/cloro/websearch.py +175 -0
- haystack_integrations/components/websearch/py.typed +0 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: cloro-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Haystack components for cloro: Google search results with the AI Overview, and ChatGPT, Google AI Mode and Gemini answers with sources.
|
|
5
|
+
Project-URL: Homepage, https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
|
|
6
|
+
Project-URL: Documentation, https://github.com/cloro-dev/cloro-haystack#readme
|
|
7
|
+
Project-URL: Issues, https://github.com/cloro-dev/cloro-haystack/issues
|
|
8
|
+
Project-URL: Source, https://github.com/cloro-dev/cloro-haystack
|
|
9
|
+
Author-email: cloro <support@cloro.dev>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: AI Mode,AI Overview,ChatGPT,Gemini,Google,Haystack,Web Search,cloro
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Requires-Dist: haystack-ai>=2.24.1
|
|
21
|
+
Requires-Dist: httpx>=0.27.0
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# cloro-haystack
|
|
25
|
+
|
|
26
|
+
[Haystack](https://haystack.deepset.ai) components for
|
|
27
|
+
[cloro](https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok).
|
|
28
|
+
|
|
29
|
+
| Component | cloro endpoint | Output | Credits per run |
|
|
30
|
+
|---|---|---|--:|
|
|
31
|
+
| `CloroWebSearch` | `POST /v1/monitor/google` | `documents` and `links` from Google organic results. With `include_ai_overview=True`, the Google AI Overview is the first Document. | 5, or 7 with the AI Overview |
|
|
32
|
+
| `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode` or `/gemini` | `replies` (the answer as Markdown), `documents` and `links` (the cited sources) | ChatGPT 7, AI Mode 6, Gemini 6 |
|
|
33
|
+
|
|
34
|
+
## Installation
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install cloro-haystack
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Get an API key at [dashboard.cloro.dev](https://dashboard.cloro.dev/api-keys?utm_source=cloro-haystack&utm_medium=byok).
|
|
41
|
+
The free plan includes 500 credits each month. Set the key as `CLORO_API_KEY`, or pass
|
|
42
|
+
`api_key=Secret.from_token(...)`.
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
### CloroWebSearch
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from haystack_integrations.components.websearch.cloro import CloroWebSearch
|
|
50
|
+
|
|
51
|
+
websearch = CloroWebSearch(top_k=5, country="US", include_ai_overview=True)
|
|
52
|
+
result = websearch.run(query="What is retrieval-augmented generation?")
|
|
53
|
+
|
|
54
|
+
for doc in result["documents"]:
|
|
55
|
+
print(doc.meta.get("title"), doc.meta.get("link"))
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Each organic result is a `Document`. Its `content` is the Google snippet and its `meta` has `title`,
|
|
59
|
+
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"` and the
|
|
60
|
+
URLs it cites in `meta["links"]`. Results with an empty link or a Google redirect link are skipped.
|
|
61
|
+
|
|
62
|
+
Parameters:
|
|
63
|
+
|
|
64
|
+
- `api_key`: defaults to the `CLORO_API_KEY` environment variable.
|
|
65
|
+
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of Google
|
|
66
|
+
results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
|
|
67
|
+
- `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
|
|
68
|
+
- `allowed_domains`: domains to limit the search to, sent as `site:` filters.
|
|
69
|
+
- `include_ai_overview`: request the Google AI Overview. Defaults to `False`.
|
|
70
|
+
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages`.
|
|
71
|
+
- `timeout`: request timeout in seconds. Defaults to 60.
|
|
72
|
+
|
|
73
|
+
### CloroAIAnswer
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from haystack_integrations.components.websearch.cloro import CloroAIAnswer
|
|
77
|
+
|
|
78
|
+
ai_answer = CloroAIAnswer(engine="aimode") # "chatgpt", "aimode" or "gemini"
|
|
79
|
+
result = ai_answer.run(query="What is retrieval-augmented generation?")
|
|
80
|
+
|
|
81
|
+
print(result["replies"][0])
|
|
82
|
+
print(result["links"])
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`replies` has one string: the answer as Markdown. `documents` has one `Document` for each source that
|
|
86
|
+
the answer cites, with `title`, `link` and `position` in `meta`. An answer can take 15 to 60 seconds,
|
|
87
|
+
so the default `timeout` is 180 seconds.
|
|
88
|
+
|
|
89
|
+
### In a pipeline
|
|
90
|
+
|
|
91
|
+
The output names of `CloroAIAnswer` match `AnswerBuilder`:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from haystack import Pipeline
|
|
95
|
+
from haystack.components.builders import AnswerBuilder
|
|
96
|
+
from haystack_integrations.components.websearch.cloro import CloroAIAnswer
|
|
97
|
+
|
|
98
|
+
pipe = Pipeline()
|
|
99
|
+
pipe.add_component("ai", CloroAIAnswer(engine="chatgpt"))
|
|
100
|
+
pipe.add_component("answer", AnswerBuilder())
|
|
101
|
+
pipe.connect("ai.replies", "answer.replies")
|
|
102
|
+
pipe.connect("ai.documents", "answer.documents")
|
|
103
|
+
|
|
104
|
+
query = "Which vector databases support hybrid search?"
|
|
105
|
+
result = pipe.run({"ai": {"query": query}, "answer": {"query": query}})
|
|
106
|
+
print(result["answer"]["answers"][0].data)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`CloroWebSearch` connects like the other Haystack web search components, for example to a
|
|
110
|
+
`ChatPromptBuilder` through `documents`, or to a fetcher through `links`.
|
|
111
|
+
|
|
112
|
+
Both components have `run_async` for `AsyncPipeline`.
|
|
113
|
+
|
|
114
|
+
### Errors
|
|
115
|
+
|
|
116
|
+
A cloro error response raises `CloroError` with the cloro error code, for example
|
|
117
|
+
`INVALID_API_KEY_FORMAT` (401), `INSUFFICIENT_CREDITS` (403) or `CONCURRENCY_LIMIT_EXCEEDED` (429).
|
|
118
|
+
A timeout raises `TimeoutError`.
|
|
119
|
+
|
|
120
|
+
## Development
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
uv sync
|
|
124
|
+
uv run pytest
|
|
125
|
+
uv run ruff check . && uv run ruff format --check .
|
|
126
|
+
uv run mypy src
|
|
127
|
+
uv build
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
The tests use `httpx.MockTransport` and do not call the cloro API.
|
|
131
|
+
|
|
132
|
+
## License
|
|
133
|
+
|
|
134
|
+
MIT
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
haystack_integrations/components/websearch/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
haystack_integrations/components/websearch/cloro/__init__.py,sha256=wb0fGi_5jhHrmXX5pfIhWeU6KIYJynrGPyzco_B_gOY,408
|
|
3
|
+
haystack_integrations/components/websearch/cloro/_api.py,sha256=mnSSgJuIa5OvUYQYpCYKtotaumFj6TyOeJ1OEKwzkW8,2994
|
|
4
|
+
haystack_integrations/components/websearch/cloro/ai_answer.py,sha256=P8oABrxNFCFwlgSgM8YEpU83evi2KC9oDBmqdRJuEM8,6237
|
|
5
|
+
haystack_integrations/components/websearch/cloro/websearch.py,sha256=at1kkjgU5kh-VF3GX6ljKqwrSUslnZw9gF5zb4fmya0,6862
|
|
6
|
+
cloro_haystack-0.1.0.dist-info/METADATA,sha256=b0tWbPfl2CArpq7ukOFDGHjmybr636Irl2G5TdL0qxE,5182
|
|
7
|
+
cloro_haystack-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
8
|
+
cloro_haystack-0.1.0.dist-info/licenses/LICENSE,sha256=4sdWt3wrB2gINd2nu57FAMdVqjr_bnpwB3aK4NJ5LxE,1062
|
|
9
|
+
cloro_haystack-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 cloro
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026-present cloro <support@cloro.dev>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
5
|
+
from haystack_integrations.components.websearch.cloro._api import CloroError
|
|
6
|
+
from haystack_integrations.components.websearch.cloro.ai_answer import CloroAIAnswer
|
|
7
|
+
from haystack_integrations.components.websearch.cloro.websearch import CloroWebSearch
|
|
8
|
+
|
|
9
|
+
__all__ = ["CloroAIAnswer", "CloroError", "CloroWebSearch"]
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026-present cloro <support@cloro.dev>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
5
|
+
from typing import Any, TypeGuard
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
from haystack import ComponentError
|
|
9
|
+
from haystack.utils import Secret
|
|
10
|
+
|
|
11
|
+
CLORO_API_BASE = "https://api.cloro.dev"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CloroError(ComponentError):
|
|
15
|
+
"""Raised when the cloro API returns an error or a response that cannot be read."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _headers(api_key: Secret) -> dict[str, str]:
|
|
19
|
+
return {"Authorization": f"Bearer {api_key.resolve_value()}", "Content-Type": "application/json"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _result(response: httpx.Response, path: str) -> dict[str, Any]:
|
|
23
|
+
try:
|
|
24
|
+
data = response.json()
|
|
25
|
+
except ValueError:
|
|
26
|
+
data = None
|
|
27
|
+
if response.is_error or not isinstance(data, dict) or not data.get("success"):
|
|
28
|
+
error = data.get("error") if isinstance(data, dict) else None
|
|
29
|
+
code = error.get("code") if isinstance(error, dict) else None
|
|
30
|
+
message = error.get("message") if isinstance(error, dict) else None
|
|
31
|
+
msg = f"cloro {path} returned HTTP {response.status_code}: {code or 'no error code'} {message or ''}".strip()
|
|
32
|
+
raise CloroError(msg)
|
|
33
|
+
result = data.get("result")
|
|
34
|
+
if not isinstance(result, dict):
|
|
35
|
+
msg = f"cloro {path} returned no result."
|
|
36
|
+
raise CloroError(msg)
|
|
37
|
+
return result
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def post(path: str, payload: dict[str, Any], api_key: Secret, timeout: float) -> dict[str, Any]:
|
|
41
|
+
"""Send a POST request to the cloro API and return the `result` object."""
|
|
42
|
+
try:
|
|
43
|
+
response = httpx.post(f"{CLORO_API_BASE}{path}", json=payload, headers=_headers(api_key), timeout=timeout)
|
|
44
|
+
except httpx.TimeoutException as error:
|
|
45
|
+
msg = f"Request to cloro {path} timed out."
|
|
46
|
+
raise TimeoutError(msg) from error
|
|
47
|
+
except httpx.HTTPError as error:
|
|
48
|
+
msg = f"Request to cloro {path} failed: {error}"
|
|
49
|
+
raise CloroError(msg) from error
|
|
50
|
+
return _result(response, path)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def post_async(path: str, payload: dict[str, Any], api_key: Secret, timeout: float) -> dict[str, Any]:
|
|
54
|
+
"""Send a POST request to the cloro API without blocking and return the `result` object."""
|
|
55
|
+
try:
|
|
56
|
+
async with httpx.AsyncClient() as client:
|
|
57
|
+
response = await client.post(
|
|
58
|
+
f"{CLORO_API_BASE}{path}", json=payload, headers=_headers(api_key), timeout=timeout
|
|
59
|
+
)
|
|
60
|
+
except httpx.TimeoutException as error:
|
|
61
|
+
msg = f"Request to cloro {path} timed out."
|
|
62
|
+
raise TimeoutError(msg) from error
|
|
63
|
+
except httpx.HTTPError as error:
|
|
64
|
+
msg = f"Request to cloro {path} failed: {error}"
|
|
65
|
+
raise CloroError(msg) from error
|
|
66
|
+
return _result(response, path)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def is_direct_link(url: Any) -> TypeGuard[str]:
|
|
70
|
+
"""Return True for an http(s) link that is not a Google redirect (/goto) or a Google search page (/search)."""
|
|
71
|
+
return (
|
|
72
|
+
isinstance(url, str)
|
|
73
|
+
and url.startswith(("http://", "https://"))
|
|
74
|
+
and "google.com/goto" not in url
|
|
75
|
+
and "google.com/search" not in url
|
|
76
|
+
)
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026-present cloro <support@cloro.dev>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
5
|
+
from typing import Any, Literal
|
|
6
|
+
|
|
7
|
+
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
|
8
|
+
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
9
|
+
|
|
10
|
+
from haystack_integrations.components.websearch.cloro._api import is_direct_link, post, post_async
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
Engine = Literal["chatgpt", "aimode", "gemini"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@component
|
|
18
|
+
class CloroAIAnswer:
|
|
19
|
+
"""
|
|
20
|
+
Gets the answer that an AI search engine gives to a query, with the sources it cites.
|
|
21
|
+
|
|
22
|
+
The component sends the query to ChatGPT, Google AI Mode or Gemini through the
|
|
23
|
+
[cloro](https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok) API.
|
|
24
|
+
`replies` has the answer as Markdown. `documents` has one
|
|
25
|
+
Document for each cited source, with `title`, `link` and `position` in its meta.
|
|
26
|
+
The output names match `AnswerBuilder`, so you can connect the component to it directly.
|
|
27
|
+
|
|
28
|
+
### Usage example
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from haystack_integrations.components.websearch.cloro import CloroAIAnswer
|
|
32
|
+
|
|
33
|
+
ai_answer = CloroAIAnswer(engine="aimode")
|
|
34
|
+
result = ai_answer.run(query="What is retrieval-augmented generation?")
|
|
35
|
+
print(result["replies"][0])
|
|
36
|
+
print(result["links"])
|
|
37
|
+
```
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
api_key: Secret = Secret.from_env_var("CLORO_API_KEY"),
|
|
43
|
+
engine: Engine = "aimode",
|
|
44
|
+
country: str = "US",
|
|
45
|
+
search_params: dict[str, Any] | None = None,
|
|
46
|
+
timeout: float = 180.0,
|
|
47
|
+
) -> None:
|
|
48
|
+
"""
|
|
49
|
+
Initialize the CloroAIAnswer component.
|
|
50
|
+
|
|
51
|
+
:param api_key:
|
|
52
|
+
cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
|
|
53
|
+
:param engine:
|
|
54
|
+
The AI search engine: `"chatgpt"`, `"aimode"` (Google AI Mode) or `"gemini"`.
|
|
55
|
+
:param country:
|
|
56
|
+
ISO 3166-1 alpha-2 country code for the engine location.
|
|
57
|
+
:param search_params:
|
|
58
|
+
Other fields for the `POST /v1/monitor/<engine>` request body. See the cloro API reference.
|
|
59
|
+
:param timeout:
|
|
60
|
+
Request timeout in seconds. An answer can take 15 to 60 seconds.
|
|
61
|
+
"""
|
|
62
|
+
if engine not in ("chatgpt", "aimode", "gemini"):
|
|
63
|
+
msg = f"engine must be 'chatgpt', 'aimode' or 'gemini', not {engine!r}"
|
|
64
|
+
raise ValueError(msg)
|
|
65
|
+
self.api_key = api_key
|
|
66
|
+
self.engine = engine
|
|
67
|
+
self.country = country
|
|
68
|
+
self.search_params = search_params
|
|
69
|
+
self.timeout = timeout
|
|
70
|
+
|
|
71
|
+
def to_dict(self) -> dict[str, Any]:
|
|
72
|
+
"""
|
|
73
|
+
Serialize the component to a dictionary.
|
|
74
|
+
|
|
75
|
+
:returns: Dictionary with serialized data.
|
|
76
|
+
"""
|
|
77
|
+
return default_to_dict(
|
|
78
|
+
self,
|
|
79
|
+
api_key=self.api_key.to_dict(),
|
|
80
|
+
engine=self.engine,
|
|
81
|
+
country=self.country,
|
|
82
|
+
search_params=self.search_params,
|
|
83
|
+
timeout=self.timeout,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
@classmethod
|
|
87
|
+
def from_dict(cls, data: dict[str, Any]) -> "CloroAIAnswer":
|
|
88
|
+
"""
|
|
89
|
+
Deserialize the component from a dictionary.
|
|
90
|
+
|
|
91
|
+
:param data: Dictionary to deserialize from.
|
|
92
|
+
:returns: Deserialized component.
|
|
93
|
+
"""
|
|
94
|
+
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
|
|
95
|
+
return default_from_dict(cls, data)
|
|
96
|
+
|
|
97
|
+
@component.output_types(replies=list[str], documents=list[Document], links=list[str])
|
|
98
|
+
def run(self, query: str) -> dict[str, Any]:
|
|
99
|
+
"""
|
|
100
|
+
Get the AI search engine answer for a query.
|
|
101
|
+
|
|
102
|
+
:param query: The prompt to send to the engine.
|
|
103
|
+
:returns: A dictionary with:
|
|
104
|
+
- `replies`: A list with the answer as Markdown.
|
|
105
|
+
- `documents`: One Document for each cited source.
|
|
106
|
+
- `links`: The URLs of the cited sources.
|
|
107
|
+
:raises TimeoutError: If the request times out.
|
|
108
|
+
:raises CloroError: If the cloro API returns an error.
|
|
109
|
+
"""
|
|
110
|
+
result = post(self._path(), self._payload(query), self.api_key, self.timeout)
|
|
111
|
+
return self._parse(result)
|
|
112
|
+
|
|
113
|
+
@component.output_types(replies=list[str], documents=list[Document], links=list[str])
|
|
114
|
+
async def run_async(self, query: str) -> dict[str, Any]:
|
|
115
|
+
"""
|
|
116
|
+
Get the AI search engine answer for a query without blocking.
|
|
117
|
+
|
|
118
|
+
This is the asynchronous version of `run`, with the same parameters and return values.
|
|
119
|
+
|
|
120
|
+
:param query: The prompt to send to the engine.
|
|
121
|
+
:returns: A dictionary with:
|
|
122
|
+
- `replies`: A list with the answer as Markdown.
|
|
123
|
+
- `documents`: One Document for each cited source.
|
|
124
|
+
- `links`: The URLs of the cited sources.
|
|
125
|
+
:raises TimeoutError: If the request times out.
|
|
126
|
+
:raises CloroError: If the cloro API returns an error.
|
|
127
|
+
"""
|
|
128
|
+
result = await post_async(self._path(), self._payload(query), self.api_key, self.timeout)
|
|
129
|
+
return self._parse(result)
|
|
130
|
+
|
|
131
|
+
def _path(self) -> str:
|
|
132
|
+
return f"/v1/monitor/{self.engine}"
|
|
133
|
+
|
|
134
|
+
def _payload(self, query: str) -> dict[str, Any]:
|
|
135
|
+
payload: dict[str, Any] = {"prompt": query, "country": self.country, **(self.search_params or {})}
|
|
136
|
+
include = dict(payload.get("include") or {})
|
|
137
|
+
include["markdown"] = True
|
|
138
|
+
payload["include"] = include
|
|
139
|
+
return payload
|
|
140
|
+
|
|
141
|
+
def _parse(self, result: dict[str, Any]) -> dict[str, Any]:
|
|
142
|
+
answer = result.get("markdown") or result.get("text") or ""
|
|
143
|
+
documents: list[Document] = []
|
|
144
|
+
links: list[str] = []
|
|
145
|
+
for source in result.get("sources") or []:
|
|
146
|
+
url = source.get("url") if isinstance(source, dict) else None
|
|
147
|
+
if not is_direct_link(url) or url in links:
|
|
148
|
+
continue
|
|
149
|
+
links.append(url)
|
|
150
|
+
documents.append(
|
|
151
|
+
Document(
|
|
152
|
+
content=source.get("description") or source.get("label") or "",
|
|
153
|
+
meta={"title": source.get("label") or "", "link": url, "position": source.get("position")},
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
logger.debug("cloro {engine} returned {count} sources", engine=self.engine, count=len(documents))
|
|
157
|
+
return {"replies": [answer] if answer else [], "documents": documents, "links": links}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026-present cloro <support@cloro.dev>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
|
8
|
+
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
9
|
+
|
|
10
|
+
from haystack_integrations.components.websearch.cloro._api import is_direct_link, post, post_async
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
GOOGLE_PATH = "/v1/monitor/google"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@component
|
|
18
|
+
class CloroWebSearch:
|
|
19
|
+
"""
|
|
20
|
+
Searches Google with the cloro API and returns the results as Documents.
|
|
21
|
+
|
|
22
|
+
cloro: https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
|
|
23
|
+
|
|
24
|
+
Each organic result becomes a Document. Its content is the Google snippet. Its meta has
|
|
25
|
+
`title`, `link` and `position`. With `include_ai_overview=True`, the Google AI Overview
|
|
26
|
+
(when Google shows one) is the first Document.
|
|
27
|
+
|
|
28
|
+
### Usage example
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from haystack_integrations.components.websearch.cloro import CloroWebSearch
|
|
32
|
+
|
|
33
|
+
websearch = CloroWebSearch(top_k=5, include_ai_overview=True)
|
|
34
|
+
result = websearch.run(query="What is retrieval-augmented generation?")
|
|
35
|
+
documents = result["documents"]
|
|
36
|
+
links = result["links"]
|
|
37
|
+
```
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
api_key: Secret = Secret.from_env_var("CLORO_API_KEY"),
|
|
43
|
+
top_k: int | None = 10,
|
|
44
|
+
country: str = "US",
|
|
45
|
+
allowed_domains: list[str] | None = None,
|
|
46
|
+
include_ai_overview: bool = False,
|
|
47
|
+
search_params: dict[str, Any] | None = None,
|
|
48
|
+
timeout: float = 60.0,
|
|
49
|
+
) -> None:
|
|
50
|
+
"""
|
|
51
|
+
Initialize the CloroWebSearch component.
|
|
52
|
+
|
|
53
|
+
:param api_key:
|
|
54
|
+
cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
|
|
55
|
+
:param top_k:
|
|
56
|
+
Maximum number of organic results to return. One request returns one page of Google
|
|
57
|
+
results (about 10). Set `search_params={"pages": 2}` or more to get more results.
|
|
58
|
+
:param country:
|
|
59
|
+
ISO 3166-1 alpha-2 country code for the Google location.
|
|
60
|
+
:param allowed_domains:
|
|
61
|
+
Domains to limit the search to. They are added to the query as `site:` filters.
|
|
62
|
+
:param include_ai_overview:
|
|
63
|
+
If True, request the Google AI Overview and return it as the first Document.
|
|
64
|
+
:param search_params:
|
|
65
|
+
Other fields for the `POST /v1/monitor/google` request body, for example `device`,
|
|
66
|
+
`location`, `hl` or `pages`. See the cloro API reference.
|
|
67
|
+
:param timeout:
|
|
68
|
+
Request timeout in seconds.
|
|
69
|
+
"""
|
|
70
|
+
self.api_key = api_key
|
|
71
|
+
self.top_k = top_k
|
|
72
|
+
self.country = country
|
|
73
|
+
self.allowed_domains = allowed_domains
|
|
74
|
+
self.include_ai_overview = include_ai_overview
|
|
75
|
+
self.search_params = search_params
|
|
76
|
+
self.timeout = timeout
|
|
77
|
+
|
|
78
|
+
def to_dict(self) -> dict[str, Any]:
|
|
79
|
+
"""
|
|
80
|
+
Serialize the component to a dictionary.
|
|
81
|
+
|
|
82
|
+
:returns: Dictionary with serialized data.
|
|
83
|
+
"""
|
|
84
|
+
return default_to_dict(
|
|
85
|
+
self,
|
|
86
|
+
api_key=self.api_key.to_dict(),
|
|
87
|
+
top_k=self.top_k,
|
|
88
|
+
country=self.country,
|
|
89
|
+
allowed_domains=self.allowed_domains,
|
|
90
|
+
include_ai_overview=self.include_ai_overview,
|
|
91
|
+
search_params=self.search_params,
|
|
92
|
+
timeout=self.timeout,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
@classmethod
|
|
96
|
+
def from_dict(cls, data: dict[str, Any]) -> "CloroWebSearch":
|
|
97
|
+
"""
|
|
98
|
+
Deserialize the component from a dictionary.
|
|
99
|
+
|
|
100
|
+
:param data: Dictionary to deserialize from.
|
|
101
|
+
:returns: Deserialized component.
|
|
102
|
+
"""
|
|
103
|
+
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
|
|
104
|
+
return default_from_dict(cls, data)
|
|
105
|
+
|
|
106
|
+
@component.output_types(documents=list[Document], links=list[str])
|
|
107
|
+
def run(self, query: str) -> dict[str, Any]:
|
|
108
|
+
"""
|
|
109
|
+
Search Google with cloro.
|
|
110
|
+
|
|
111
|
+
:param query: Search query.
|
|
112
|
+
:returns: A dictionary with:
|
|
113
|
+
- `documents`: The AI Overview (if requested and present), then the organic results.
|
|
114
|
+
- `links`: The links of the organic results.
|
|
115
|
+
:raises TimeoutError: If the request times out.
|
|
116
|
+
:raises CloroError: If the cloro API returns an error.
|
|
117
|
+
"""
|
|
118
|
+
result = post(GOOGLE_PATH, self._payload(query), self.api_key, self.timeout)
|
|
119
|
+
return self._parse(result)
|
|
120
|
+
|
|
121
|
+
@component.output_types(documents=list[Document], links=list[str])
|
|
122
|
+
async def run_async(self, query: str) -> dict[str, Any]:
|
|
123
|
+
"""
|
|
124
|
+
Search Google with cloro without blocking.
|
|
125
|
+
|
|
126
|
+
This is the asynchronous version of `run`, with the same parameters and return values.
|
|
127
|
+
|
|
128
|
+
:param query: Search query.
|
|
129
|
+
:returns: A dictionary with:
|
|
130
|
+
- `documents`: The AI Overview (if requested and present), then the organic results.
|
|
131
|
+
- `links`: The links of the organic results.
|
|
132
|
+
:raises TimeoutError: If the request times out.
|
|
133
|
+
:raises CloroError: If the cloro API returns an error.
|
|
134
|
+
"""
|
|
135
|
+
result = await post_async(GOOGLE_PATH, self._payload(query), self.api_key, self.timeout)
|
|
136
|
+
return self._parse(result)
|
|
137
|
+
|
|
138
|
+
def _payload(self, query: str) -> dict[str, Any]:
|
|
139
|
+
if self.allowed_domains:
|
|
140
|
+
query = f"{query} " + " OR ".join(f"site:{domain}" for domain in self.allowed_domains)
|
|
141
|
+
payload: dict[str, Any] = {"query": query, "country": self.country, **(self.search_params or {})}
|
|
142
|
+
if self.include_ai_overview:
|
|
143
|
+
include = dict(payload.get("include") or {})
|
|
144
|
+
include["aioverview"] = True
|
|
145
|
+
payload["include"] = include
|
|
146
|
+
return payload
|
|
147
|
+
|
|
148
|
+
def _parse(self, result: dict[str, Any]) -> dict[str, Any]:
|
|
149
|
+
organic = [
|
|
150
|
+
item
|
|
151
|
+
for item in result.get("organicResults") or []
|
|
152
|
+
if isinstance(item, dict) and is_direct_link(item.get("link"))
|
|
153
|
+
][: self.top_k]
|
|
154
|
+
documents = [
|
|
155
|
+
Document(
|
|
156
|
+
content=item.get("snippet") or "",
|
|
157
|
+
meta={"title": item.get("title") or "", "link": item["link"], "position": item.get("position")},
|
|
158
|
+
)
|
|
159
|
+
for item in organic
|
|
160
|
+
]
|
|
161
|
+
links = [item["link"] for item in organic]
|
|
162
|
+
|
|
163
|
+
overview = result.get("aioverview")
|
|
164
|
+
if self.include_ai_overview and isinstance(overview, dict) and overview.get("text"):
|
|
165
|
+
sources = [s["url"] for s in overview.get("sources") or [] if is_direct_link(s.get("url"))]
|
|
166
|
+
documents.insert(
|
|
167
|
+
0,
|
|
168
|
+
Document(
|
|
169
|
+
content=overview["text"],
|
|
170
|
+
meta={"title": "Google AI Overview", "source": "google_ai_overview", "links": sources},
|
|
171
|
+
),
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
logger.debug("cloro returned {count} documents", count=len(documents))
|
|
175
|
+
return {"documents": documents, "links": links}
|
|
File without changes
|