cloro-haystack 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ __pycache__/
2
+ *.egg-info/
3
+ .venv/
4
+ dist/
5
+ .pytest_cache/
6
+ .ruff_cache/
7
+ .mypy_cache/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 cloro
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,134 @@
1
+ Metadata-Version: 2.5
2
+ Name: cloro-haystack
3
+ Version: 0.1.0
4
+ Summary: Haystack components for cloro: Google search results with the AI Overview, and ChatGPT, Google AI Mode and Gemini answers with sources.
5
+ Project-URL: Homepage, https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
6
+ Project-URL: Documentation, https://github.com/cloro-dev/cloro-haystack#readme
7
+ Project-URL: Issues, https://github.com/cloro-dev/cloro-haystack/issues
8
+ Project-URL: Source, https://github.com/cloro-dev/cloro-haystack
9
+ Author-email: cloro <support@cloro.dev>
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: AI Mode,AI Overview,ChatGPT,Gemini,Google,Haystack,Web Search,cloro
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Requires-Python: >=3.11
20
+ Requires-Dist: haystack-ai>=2.24.1
21
+ Requires-Dist: httpx>=0.27.0
22
+ Description-Content-Type: text/markdown
23
+
24
+ # cloro-haystack
25
+
26
+ [Haystack](https://haystack.deepset.ai) components for
27
+ [cloro](https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok).
28
+
29
+ | Component | cloro endpoint | Output | Credits per run |
30
+ |---|---|---|--:|
31
+ | `CloroWebSearch` | `POST /v1/monitor/google` | `documents` and `links` from Google organic results. With `include_ai_overview=True`, the Google AI Overview is the first Document. | 5, or 7 with the AI Overview |
32
+ | `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode` or `/gemini` | `replies` (the answer as Markdown), `documents` and `links` (the cited sources) | ChatGPT 7, AI Mode 6, Gemini 6 |
33
+
34
+ ## Installation
35
+
36
+ ```bash
37
+ pip install cloro-haystack
38
+ ```
39
+
40
+ Get an API key at [dashboard.cloro.dev](https://dashboard.cloro.dev/api-keys?utm_source=cloro-haystack&utm_medium=byok).
41
+ The free plan includes 500 credits each month. Set the key as `CLORO_API_KEY`, or pass
42
+ `api_key=Secret.from_token(...)`.
43
+
44
+ ## Usage
45
+
46
+ ### CloroWebSearch
47
+
48
+ ```python
49
+ from haystack_integrations.components.websearch.cloro import CloroWebSearch
50
+
51
+ websearch = CloroWebSearch(top_k=5, country="US", include_ai_overview=True)
52
+ result = websearch.run(query="What is retrieval-augmented generation?")
53
+
54
+ for doc in result["documents"]:
55
+ print(doc.meta.get("title"), doc.meta.get("link"))
56
+ ```
57
+
58
+ Each organic result is a `Document`. Its `content` is the Google snippet and its `meta` has `title`,
59
+ `link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"` and the
60
+ URLs it cites in `meta["links"]`. Results with an empty link or a Google redirect link are skipped.
61
+
62
+ Parameters:
63
+
64
+ - `api_key`: defaults to the `CLORO_API_KEY` environment variable.
65
+ - `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of Google
66
+ results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
67
+ - `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
68
+ - `allowed_domains`: domains to limit the search to, sent as `site:` filters.
69
+ - `include_ai_overview`: request the Google AI Overview. Defaults to `False`.
70
+ - `search_params`: other request fields, for example `device`, `location`, `hl` or `pages`.
71
+ - `timeout`: request timeout in seconds. Defaults to 60.
72
+
73
+ ### CloroAIAnswer
74
+
75
+ ```python
76
+ from haystack_integrations.components.websearch.cloro import CloroAIAnswer
77
+
78
+ ai_answer = CloroAIAnswer(engine="aimode") # "chatgpt", "aimode" or "gemini"
79
+ result = ai_answer.run(query="What is retrieval-augmented generation?")
80
+
81
+ print(result["replies"][0])
82
+ print(result["links"])
83
+ ```
84
+
85
+ `replies` has one string: the answer as Markdown. `documents` has one `Document` for each source that
86
+ the answer cites, with `title`, `link` and `position` in `meta`. An answer can take 15 to 60 seconds,
87
+ so the default `timeout` is 180 seconds.
88
+
89
+ ### In a pipeline
90
+
91
+ The output names of `CloroAIAnswer` match `AnswerBuilder`:
92
+
93
+ ```python
94
+ from haystack import Pipeline
95
+ from haystack.components.builders import AnswerBuilder
96
+ from haystack_integrations.components.websearch.cloro import CloroAIAnswer
97
+
98
+ pipe = Pipeline()
99
+ pipe.add_component("ai", CloroAIAnswer(engine="chatgpt"))
100
+ pipe.add_component("answer", AnswerBuilder())
101
+ pipe.connect("ai.replies", "answer.replies")
102
+ pipe.connect("ai.documents", "answer.documents")
103
+
104
+ query = "Which vector databases support hybrid search?"
105
+ result = pipe.run({"ai": {"query": query}, "answer": {"query": query}})
106
+ print(result["answer"]["answers"][0].data)
107
+ ```
108
+
109
+ `CloroWebSearch` connects like the other Haystack web search components, for example to a
110
+ `ChatPromptBuilder` through `documents`, or to a fetcher through `links`.
111
+
112
+ Both components have `run_async` for `AsyncPipeline`.
113
+
114
+ ### Errors
115
+
116
+ A cloro error response raises `CloroError` with the cloro error code, for example
117
+ `INVALID_API_KEY_FORMAT` (401), `INSUFFICIENT_CREDITS` (403) or `CONCURRENCY_LIMIT_EXCEEDED` (429).
118
+ A timeout raises `TimeoutError`.
119
+
120
+ ## Development
121
+
122
+ ```bash
123
+ uv sync
124
+ uv run pytest
125
+ uv run ruff check . && uv run ruff format --check .
126
+ uv run mypy src
127
+ uv build
128
+ ```
129
+
130
+ The tests use `httpx.MockTransport` and do not call the cloro API.
131
+
132
+ ## License
133
+
134
+ MIT
@@ -0,0 +1,111 @@
1
+ # cloro-haystack
2
+
3
+ [Haystack](https://haystack.deepset.ai) components for
4
+ [cloro](https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok).
5
+
6
+ | Component | cloro endpoint | Output | Credits per run |
7
+ |---|---|---|--:|
8
+ | `CloroWebSearch` | `POST /v1/monitor/google` | `documents` and `links` from Google organic results. With `include_ai_overview=True`, the Google AI Overview is the first Document. | 5, or 7 with the AI Overview |
9
+ | `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode` or `/gemini` | `replies` (the answer as Markdown), `documents` and `links` (the cited sources) | ChatGPT 7, AI Mode 6, Gemini 6 |
10
+
11
+ ## Installation
12
+
13
+ ```bash
14
+ pip install cloro-haystack
15
+ ```
16
+
17
+ Get an API key at [dashboard.cloro.dev](https://dashboard.cloro.dev/api-keys?utm_source=cloro-haystack&utm_medium=byok).
18
+ The free plan includes 500 credits each month. Set the key as `CLORO_API_KEY`, or pass
19
+ `api_key=Secret.from_token(...)`.
20
+
21
+ ## Usage
22
+
23
+ ### CloroWebSearch
24
+
25
+ ```python
26
+ from haystack_integrations.components.websearch.cloro import CloroWebSearch
27
+
28
+ websearch = CloroWebSearch(top_k=5, country="US", include_ai_overview=True)
29
+ result = websearch.run(query="What is retrieval-augmented generation?")
30
+
31
+ for doc in result["documents"]:
32
+ print(doc.meta.get("title"), doc.meta.get("link"))
33
+ ```
34
+
35
+ Each organic result is a `Document`. Its `content` is the Google snippet and its `meta` has `title`,
36
+ `link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"` and the
37
+ URLs it cites in `meta["links"]`. Results with an empty link or a Google redirect link are skipped.
38
+
39
+ Parameters:
40
+
41
+ - `api_key`: defaults to the `CLORO_API_KEY` environment variable.
42
+ - `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of Google
43
+ results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
44
+ - `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
45
+ - `allowed_domains`: domains to limit the search to, sent as `site:` filters.
46
+ - `include_ai_overview`: request the Google AI Overview. Defaults to `False`.
47
+ - `search_params`: other request fields, for example `device`, `location`, `hl` or `pages`.
48
+ - `timeout`: request timeout in seconds. Defaults to 60.
49
+
50
+ ### CloroAIAnswer
51
+
52
+ ```python
53
+ from haystack_integrations.components.websearch.cloro import CloroAIAnswer
54
+
55
+ ai_answer = CloroAIAnswer(engine="aimode") # "chatgpt", "aimode" or "gemini"
56
+ result = ai_answer.run(query="What is retrieval-augmented generation?")
57
+
58
+ print(result["replies"][0])
59
+ print(result["links"])
60
+ ```
61
+
62
+ `replies` has one string: the answer as Markdown. `documents` has one `Document` for each source that
63
+ the answer cites, with `title`, `link` and `position` in `meta`. An answer can take 15 to 60 seconds,
64
+ so the default `timeout` is 180 seconds.
65
+
66
+ ### In a pipeline
67
+
68
+ The output names of `CloroAIAnswer` match `AnswerBuilder`:
69
+
70
+ ```python
71
+ from haystack import Pipeline
72
+ from haystack.components.builders import AnswerBuilder
73
+ from haystack_integrations.components.websearch.cloro import CloroAIAnswer
74
+
75
+ pipe = Pipeline()
76
+ pipe.add_component("ai", CloroAIAnswer(engine="chatgpt"))
77
+ pipe.add_component("answer", AnswerBuilder())
78
+ pipe.connect("ai.replies", "answer.replies")
79
+ pipe.connect("ai.documents", "answer.documents")
80
+
81
+ query = "Which vector databases support hybrid search?"
82
+ result = pipe.run({"ai": {"query": query}, "answer": {"query": query}})
83
+ print(result["answer"]["answers"][0].data)
84
+ ```
85
+
86
+ `CloroWebSearch` connects like the other Haystack web search components, for example to a
87
+ `ChatPromptBuilder` through `documents`, or to a fetcher through `links`.
88
+
89
+ Both components have `run_async` for `AsyncPipeline`.
90
+
91
+ ### Errors
92
+
93
+ A cloro error response raises `CloroError` with the cloro error code, for example
94
+ `INVALID_API_KEY_FORMAT` (401), `INSUFFICIENT_CREDITS` (403) or `CONCURRENCY_LIMIT_EXCEEDED` (429).
95
+ A timeout raises `TimeoutError`.
96
+
97
+ ## Development
98
+
99
+ ```bash
100
+ uv sync
101
+ uv run pytest
102
+ uv run ruff check . && uv run ruff format --check .
103
+ uv run mypy src
104
+ uv build
105
+ ```
106
+
107
+ The tests use `httpx.MockTransport` and do not call the cloro API.
108
+
109
+ ## License
110
+
111
+ MIT
@@ -0,0 +1,58 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "cloro-haystack"
7
+ version = "0.1.0"
8
+ description = "Haystack components for cloro: Google search results with the AI Overview, and ChatGPT, Google AI Mode and Gemini answers with sources."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ keywords = ["Haystack", "cloro", "Web Search", "Google", "AI Overview", "AI Mode", "ChatGPT", "Gemini"]
14
+ authors = [{ name = "cloro", email = "support@cloro.dev" }]
15
+ classifiers = [
16
+ "Development Status :: 4 - Beta",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python",
19
+ "Programming Language :: Python :: 3.11",
20
+ "Programming Language :: Python :: 3.12",
21
+ "Programming Language :: Python :: 3.13",
22
+ ]
23
+ dependencies = ["haystack-ai>=2.24.1", "httpx>=0.27.0"]
24
+
25
+ [project.urls]
26
+ Homepage = "https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok"
27
+ Documentation = "https://github.com/cloro-dev/cloro-haystack#readme"
28
+ Issues = "https://github.com/cloro-dev/cloro-haystack/issues"
29
+ Source = "https://github.com/cloro-dev/cloro-haystack"
30
+
31
+ [dependency-groups]
32
+ dev = ["pytest>=8.0", "ruff>=0.11", "mypy>=1.10"]
33
+
34
+ [tool.hatch.build.targets.sdist]
35
+ include = ["src/", "tests/", "LICENSE", "README.md"]
36
+
37
+ [tool.hatch.build.targets.wheel]
38
+ packages = ["src/haystack_integrations"]
39
+
40
+ [tool.mypy]
41
+ check_untyped_defs = true
42
+ disallow_incomplete_defs = true
43
+
44
+ [tool.ruff]
45
+ line-length = 120
46
+
47
+ [tool.ruff.lint]
48
+ select = ["A", "ANN", "ARG", "B", "C", "D102", "D103", "D205", "D209", "D213", "D417", "D419", "DTZ", "E", "EM", "F", "I", "ICN", "ISC", "N", "PLC", "PLE", "PLR", "PLW", "Q", "RUF", "S", "T", "TID", "UP", "W", "YTT"]
49
+ ignore = ["B008", "S105", "S106", "S107", "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", "PLR0917", "ANN401"]
50
+
51
+ [tool.ruff.lint.isort]
52
+ known-first-party = ["haystack_integrations"]
53
+
54
+ [tool.ruff.lint.flake8-tidy-imports]
55
+ ban-relative-imports = "parents"
56
+
57
+ [tool.ruff.lint.per-file-ignores]
58
+ "tests/**/*" = ["PLR2004", "S101", "TID252", "D", "ANN"]
@@ -0,0 +1,9 @@
1
+ # SPDX-FileCopyrightText: 2026-present cloro <support@cloro.dev>
2
+ #
3
+ # SPDX-License-Identifier: MIT
4
+
5
+ from haystack_integrations.components.websearch.cloro._api import CloroError
6
+ from haystack_integrations.components.websearch.cloro.ai_answer import CloroAIAnswer
7
+ from haystack_integrations.components.websearch.cloro.websearch import CloroWebSearch
8
+
9
+ __all__ = ["CloroAIAnswer", "CloroError", "CloroWebSearch"]
@@ -0,0 +1,76 @@
1
+ # SPDX-FileCopyrightText: 2026-present cloro <support@cloro.dev>
2
+ #
3
+ # SPDX-License-Identifier: MIT
4
+
5
+ from typing import Any, TypeGuard
6
+
7
+ import httpx
8
+ from haystack import ComponentError
9
+ from haystack.utils import Secret
10
+
11
+ CLORO_API_BASE = "https://api.cloro.dev"
12
+
13
+
14
+ class CloroError(ComponentError):
15
+ """Raised when the cloro API returns an error or a response that cannot be read."""
16
+
17
+
18
+ def _headers(api_key: Secret) -> dict[str, str]:
19
+ return {"Authorization": f"Bearer {api_key.resolve_value()}", "Content-Type": "application/json"}
20
+
21
+
22
+ def _result(response: httpx.Response, path: str) -> dict[str, Any]:
23
+ try:
24
+ data = response.json()
25
+ except ValueError:
26
+ data = None
27
+ if response.is_error or not isinstance(data, dict) or not data.get("success"):
28
+ error = data.get("error") if isinstance(data, dict) else None
29
+ code = error.get("code") if isinstance(error, dict) else None
30
+ message = error.get("message") if isinstance(error, dict) else None
31
+ msg = f"cloro {path} returned HTTP {response.status_code}: {code or 'no error code'} {message or ''}".strip()
32
+ raise CloroError(msg)
33
+ result = data.get("result")
34
+ if not isinstance(result, dict):
35
+ msg = f"cloro {path} returned no result."
36
+ raise CloroError(msg)
37
+ return result
38
+
39
+
40
+ def post(path: str, payload: dict[str, Any], api_key: Secret, timeout: float) -> dict[str, Any]:
41
+ """Send a POST request to the cloro API and return the `result` object."""
42
+ try:
43
+ response = httpx.post(f"{CLORO_API_BASE}{path}", json=payload, headers=_headers(api_key), timeout=timeout)
44
+ except httpx.TimeoutException as error:
45
+ msg = f"Request to cloro {path} timed out."
46
+ raise TimeoutError(msg) from error
47
+ except httpx.HTTPError as error:
48
+ msg = f"Request to cloro {path} failed: {error}"
49
+ raise CloroError(msg) from error
50
+ return _result(response, path)
51
+
52
+
53
+ async def post_async(path: str, payload: dict[str, Any], api_key: Secret, timeout: float) -> dict[str, Any]:
54
+ """Send a POST request to the cloro API without blocking and return the `result` object."""
55
+ try:
56
+ async with httpx.AsyncClient() as client:
57
+ response = await client.post(
58
+ f"{CLORO_API_BASE}{path}", json=payload, headers=_headers(api_key), timeout=timeout
59
+ )
60
+ except httpx.TimeoutException as error:
61
+ msg = f"Request to cloro {path} timed out."
62
+ raise TimeoutError(msg) from error
63
+ except httpx.HTTPError as error:
64
+ msg = f"Request to cloro {path} failed: {error}"
65
+ raise CloroError(msg) from error
66
+ return _result(response, path)
67
+
68
+
69
+ def is_direct_link(url: Any) -> TypeGuard[str]:
70
+ """Return True for an http(s) link that is not a Google redirect (/goto) or a Google search page (/search)."""
71
+ return (
72
+ isinstance(url, str)
73
+ and url.startswith(("http://", "https://"))
74
+ and "google.com/goto" not in url
75
+ and "google.com/search" not in url
76
+ )
@@ -0,0 +1,157 @@
1
+ # SPDX-FileCopyrightText: 2026-present cloro <support@cloro.dev>
2
+ #
3
+ # SPDX-License-Identifier: MIT
4
+
5
+ from typing import Any, Literal
6
+
7
+ from haystack import Document, component, default_from_dict, default_to_dict, logging
8
+ from haystack.utils import Secret, deserialize_secrets_inplace
9
+
10
+ from haystack_integrations.components.websearch.cloro._api import is_direct_link, post, post_async
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+ Engine = Literal["chatgpt", "aimode", "gemini"]
15
+
16
+
17
+ @component
18
+ class CloroAIAnswer:
19
+ """
20
+ Gets the answer that an AI search engine gives to a query, with the sources it cites.
21
+
22
+ The component sends the query to ChatGPT, Google AI Mode or Gemini through the
23
+ [cloro](https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok) API.
24
+ `replies` has the answer as Markdown. `documents` has one
25
+ Document for each cited source, with `title`, `link` and `position` in its meta.
26
+ The output names match `AnswerBuilder`, so you can connect the component to it directly.
27
+
28
+ ### Usage example
29
+
30
+ ```python
31
+ from haystack_integrations.components.websearch.cloro import CloroAIAnswer
32
+
33
+ ai_answer = CloroAIAnswer(engine="aimode")
34
+ result = ai_answer.run(query="What is retrieval-augmented generation?")
35
+ print(result["replies"][0])
36
+ print(result["links"])
37
+ ```
38
+ """
39
+
40
+ def __init__(
41
+ self,
42
+ api_key: Secret = Secret.from_env_var("CLORO_API_KEY"),
43
+ engine: Engine = "aimode",
44
+ country: str = "US",
45
+ search_params: dict[str, Any] | None = None,
46
+ timeout: float = 180.0,
47
+ ) -> None:
48
+ """
49
+ Initialize the CloroAIAnswer component.
50
+
51
+ :param api_key:
52
+ cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
53
+ :param engine:
54
+ The AI search engine: `"chatgpt"`, `"aimode"` (Google AI Mode) or `"gemini"`.
55
+ :param country:
56
+ ISO 3166-1 alpha-2 country code for the engine location.
57
+ :param search_params:
58
+ Other fields for the `POST /v1/monitor/<engine>` request body. See the cloro API reference.
59
+ :param timeout:
60
+ Request timeout in seconds. An answer can take 15 to 60 seconds.
61
+ """
62
+ if engine not in ("chatgpt", "aimode", "gemini"):
63
+ msg = f"engine must be 'chatgpt', 'aimode' or 'gemini', not {engine!r}"
64
+ raise ValueError(msg)
65
+ self.api_key = api_key
66
+ self.engine = engine
67
+ self.country = country
68
+ self.search_params = search_params
69
+ self.timeout = timeout
70
+
71
+ def to_dict(self) -> dict[str, Any]:
72
+ """
73
+ Serialize the component to a dictionary.
74
+
75
+ :returns: Dictionary with serialized data.
76
+ """
77
+ return default_to_dict(
78
+ self,
79
+ api_key=self.api_key.to_dict(),
80
+ engine=self.engine,
81
+ country=self.country,
82
+ search_params=self.search_params,
83
+ timeout=self.timeout,
84
+ )
85
+
86
+ @classmethod
87
+ def from_dict(cls, data: dict[str, Any]) -> "CloroAIAnswer":
88
+ """
89
+ Deserialize the component from a dictionary.
90
+
91
+ :param data: Dictionary to deserialize from.
92
+ :returns: Deserialized component.
93
+ """
94
+ deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
95
+ return default_from_dict(cls, data)
96
+
97
+ @component.output_types(replies=list[str], documents=list[Document], links=list[str])
98
+ def run(self, query: str) -> dict[str, Any]:
99
+ """
100
+ Get the AI search engine answer for a query.
101
+
102
+ :param query: The prompt to send to the engine.
103
+ :returns: A dictionary with:
104
+ - `replies`: A list with the answer as Markdown.
105
+ - `documents`: One Document for each cited source.
106
+ - `links`: The URLs of the cited sources.
107
+ :raises TimeoutError: If the request times out.
108
+ :raises CloroError: If the cloro API returns an error.
109
+ """
110
+ result = post(self._path(), self._payload(query), self.api_key, self.timeout)
111
+ return self._parse(result)
112
+
113
+ @component.output_types(replies=list[str], documents=list[Document], links=list[str])
114
+ async def run_async(self, query: str) -> dict[str, Any]:
115
+ """
116
+ Get the AI search engine answer for a query without blocking.
117
+
118
+ This is the asynchronous version of `run`, with the same parameters and return values.
119
+
120
+ :param query: The prompt to send to the engine.
121
+ :returns: A dictionary with:
122
+ - `replies`: A list with the answer as Markdown.
123
+ - `documents`: One Document for each cited source.
124
+ - `links`: The URLs of the cited sources.
125
+ :raises TimeoutError: If the request times out.
126
+ :raises CloroError: If the cloro API returns an error.
127
+ """
128
+ result = await post_async(self._path(), self._payload(query), self.api_key, self.timeout)
129
+ return self._parse(result)
130
+
131
+ def _path(self) -> str:
132
+ return f"/v1/monitor/{self.engine}"
133
+
134
+ def _payload(self, query: str) -> dict[str, Any]:
135
+ payload: dict[str, Any] = {"prompt": query, "country": self.country, **(self.search_params or {})}
136
+ include = dict(payload.get("include") or {})
137
+ include["markdown"] = True
138
+ payload["include"] = include
139
+ return payload
140
+
141
+ def _parse(self, result: dict[str, Any]) -> dict[str, Any]:
142
+ answer = result.get("markdown") or result.get("text") or ""
143
+ documents: list[Document] = []
144
+ links: list[str] = []
145
+ for source in result.get("sources") or []:
146
+ url = source.get("url") if isinstance(source, dict) else None
147
+ if not is_direct_link(url) or url in links:
148
+ continue
149
+ links.append(url)
150
+ documents.append(
151
+ Document(
152
+ content=source.get("description") or source.get("label") or "",
153
+ meta={"title": source.get("label") or "", "link": url, "position": source.get("position")},
154
+ )
155
+ )
156
+ logger.debug("cloro {engine} returned {count} sources", engine=self.engine, count=len(documents))
157
+ return {"replies": [answer] if answer else [], "documents": documents, "links": links}
@@ -0,0 +1,175 @@
1
+ # SPDX-FileCopyrightText: 2026-present cloro <support@cloro.dev>
2
+ #
3
+ # SPDX-License-Identifier: MIT
4
+
5
+ from typing import Any
6
+
7
+ from haystack import Document, component, default_from_dict, default_to_dict, logging
8
+ from haystack.utils import Secret, deserialize_secrets_inplace
9
+
10
+ from haystack_integrations.components.websearch.cloro._api import is_direct_link, post, post_async
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+ GOOGLE_PATH = "/v1/monitor/google"
15
+
16
+
17
+ @component
18
+ class CloroWebSearch:
19
+ """
20
+ Searches Google with the cloro API and returns the results as Documents.
21
+
22
+ cloro: https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
23
+
24
+ Each organic result becomes a Document. Its content is the Google snippet. Its meta has
25
+ `title`, `link` and `position`. With `include_ai_overview=True`, the Google AI Overview
26
+ (when Google shows one) is the first Document.
27
+
28
+ ### Usage example
29
+
30
+ ```python
31
+ from haystack_integrations.components.websearch.cloro import CloroWebSearch
32
+
33
+ websearch = CloroWebSearch(top_k=5, include_ai_overview=True)
34
+ result = websearch.run(query="What is retrieval-augmented generation?")
35
+ documents = result["documents"]
36
+ links = result["links"]
37
+ ```
38
+ """
39
+
40
+ def __init__(
41
+ self,
42
+ api_key: Secret = Secret.from_env_var("CLORO_API_KEY"),
43
+ top_k: int | None = 10,
44
+ country: str = "US",
45
+ allowed_domains: list[str] | None = None,
46
+ include_ai_overview: bool = False,
47
+ search_params: dict[str, Any] | None = None,
48
+ timeout: float = 60.0,
49
+ ) -> None:
50
+ """
51
+ Initialize the CloroWebSearch component.
52
+
53
+ :param api_key:
54
+ cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
55
+ :param top_k:
56
+ Maximum number of organic results to return. One request returns one page of Google
57
+ results (about 10). Set `search_params={"pages": 2}` or more to get more results.
58
+ :param country:
59
+ ISO 3166-1 alpha-2 country code for the Google location.
60
+ :param allowed_domains:
61
+ Domains to limit the search to. They are added to the query as `site:` filters.
62
+ :param include_ai_overview:
63
+ If True, request the Google AI Overview and return it as the first Document.
64
+ :param search_params:
65
+ Other fields for the `POST /v1/monitor/google` request body, for example `device`,
66
+ `location`, `hl` or `pages`. See the cloro API reference.
67
+ :param timeout:
68
+ Request timeout in seconds.
69
+ """
70
+ self.api_key = api_key
71
+ self.top_k = top_k
72
+ self.country = country
73
+ self.allowed_domains = allowed_domains
74
+ self.include_ai_overview = include_ai_overview
75
+ self.search_params = search_params
76
+ self.timeout = timeout
77
+
78
+ def to_dict(self) -> dict[str, Any]:
79
+ """
80
+ Serialize the component to a dictionary.
81
+
82
+ :returns: Dictionary with serialized data.
83
+ """
84
+ return default_to_dict(
85
+ self,
86
+ api_key=self.api_key.to_dict(),
87
+ top_k=self.top_k,
88
+ country=self.country,
89
+ allowed_domains=self.allowed_domains,
90
+ include_ai_overview=self.include_ai_overview,
91
+ search_params=self.search_params,
92
+ timeout=self.timeout,
93
+ )
94
+
95
+ @classmethod
96
+ def from_dict(cls, data: dict[str, Any]) -> "CloroWebSearch":
97
+ """
98
+ Deserialize the component from a dictionary.
99
+
100
+ :param data: Dictionary to deserialize from.
101
+ :returns: Deserialized component.
102
+ """
103
+ deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
104
+ return default_from_dict(cls, data)
105
+
106
+ @component.output_types(documents=list[Document], links=list[str])
107
+ def run(self, query: str) -> dict[str, Any]:
108
+ """
109
+ Search Google with cloro.
110
+
111
+ :param query: Search query.
112
+ :returns: A dictionary with:
113
+ - `documents`: The AI Overview (if requested and present), then the organic results.
114
+ - `links`: The links of the organic results.
115
+ :raises TimeoutError: If the request times out.
116
+ :raises CloroError: If the cloro API returns an error.
117
+ """
118
+ result = post(GOOGLE_PATH, self._payload(query), self.api_key, self.timeout)
119
+ return self._parse(result)
120
+
121
+ @component.output_types(documents=list[Document], links=list[str])
122
+ async def run_async(self, query: str) -> dict[str, Any]:
123
+ """
124
+ Search Google with cloro without blocking.
125
+
126
+ This is the asynchronous version of `run`, with the same parameters and return values.
127
+
128
+ :param query: Search query.
129
+ :returns: A dictionary with:
130
+ - `documents`: The AI Overview (if requested and present), then the organic results.
131
+ - `links`: The links of the organic results.
132
+ :raises TimeoutError: If the request times out.
133
+ :raises CloroError: If the cloro API returns an error.
134
+ """
135
+ result = await post_async(GOOGLE_PATH, self._payload(query), self.api_key, self.timeout)
136
+ return self._parse(result)
137
+
138
+ def _payload(self, query: str) -> dict[str, Any]:
139
+ if self.allowed_domains:
140
+ query = f"{query} " + " OR ".join(f"site:{domain}" for domain in self.allowed_domains)
141
+ payload: dict[str, Any] = {"query": query, "country": self.country, **(self.search_params or {})}
142
+ if self.include_ai_overview:
143
+ include = dict(payload.get("include") or {})
144
+ include["aioverview"] = True
145
+ payload["include"] = include
146
+ return payload
147
+
148
+ def _parse(self, result: dict[str, Any]) -> dict[str, Any]:
149
+ organic = [
150
+ item
151
+ for item in result.get("organicResults") or []
152
+ if isinstance(item, dict) and is_direct_link(item.get("link"))
153
+ ][: self.top_k]
154
+ documents = [
155
+ Document(
156
+ content=item.get("snippet") or "",
157
+ meta={"title": item.get("title") or "", "link": item["link"], "position": item.get("position")},
158
+ )
159
+ for item in organic
160
+ ]
161
+ links = [item["link"] for item in organic]
162
+
163
+ overview = result.get("aioverview")
164
+ if self.include_ai_overview and isinstance(overview, dict) and overview.get("text"):
165
+ sources = [s["url"] for s in overview.get("sources") or [] if is_direct_link(s.get("url"))]
166
+ documents.insert(
167
+ 0,
168
+ Document(
169
+ content=overview["text"],
170
+ meta={"title": "Google AI Overview", "source": "google_ai_overview", "links": sources},
171
+ ),
172
+ )
173
+
174
+ logger.debug("cloro returned {count} documents", count=len(documents))
175
+ return {"documents": documents, "links": links}
@@ -0,0 +1,31 @@
1
+ {
2
+ "success": true,
3
+ "result": {
4
+ "text": "Choosing the best rank tracking tool depends on factors like team size, agency reporting needs, budget, and tracking for emerging AI-driven search results.\n\nTop Categories and Tools\n\nEnterprise & Premium Suites\n\nSemrush: Ideal for large teams needing comprehensive multi-channel tracking, daily updates across devices, local and multilingual data, and built-in competitor analytics.",
5
+ "markdown": "Choosing the best rank tracking tool depends on factors like team size, agency reporting needs, budget, and tracking for emerging AI-driven search results.\n\nTop Categories and Tools\n\n * **Enterprise & Premium Suites**\n * **Semrush**: Ideal for large teams needing comprehensive multi-channel tracking, daily updates across devices, local and multilingual data, and built-in competitor analytics.",
6
+ "sources": [
7
+ {
8
+ "position": 1,
9
+ "label": "The 12 best rank tracker tools | Zapier",
10
+ "url": "https://zapier.com/blog/best-rank-tracker/",
11
+ "description": "Low-volume keyword tracking | Simple, intuitive tool suite for the price | From $49/month; has free SEO tools. All of its tools ar...",
12
+ "redirectLink": "https://www.google.com/goto?url=CAESZwHEIkXXHMk2gUQTN1YAHtshd18wZydVNBNOhAIqv2vk1ecw4EWNKfFZXSC7XzlsutUFtGEsSIzuXKCF8GbbR3tB5wBLb_wLxvBsz-dOJhtIgLmV4ngHwvkhFvLvZ2vrEFMpOMGAa28"
13
+ },
14
+ {
15
+ "position": 2,
16
+ "label": "15 Best Rank Tracking Software (Tested on Real Campaigns)",
17
+ "url": "https://onelittleweb.com/top-tools/best-rank-tracking-software/",
18
+ "description": "Google Search Console \u2014 Best free diagnostic tracker 13. Clean dashboards with historical graphs and landing page insights. Basic:",
19
+ "redirectLink": "https://www.google.com/goto?url=CAESfAHEIkXX3vTWWBJYDPXr_HA6IUEhTQLFybPBNdh9vVusAPlxk8Tr07Ze-rWF0Dw96M4hYlrSApMx2w0Q2_hr_NUiGGc12Q6fH8CAhnG5e8cvl3tn84TNcztIS9Gw9mQXb17O0J9hHu77-sXV4ytyt3a4oyrSZdkAOsB3xzA"
20
+ },
21
+ {
22
+ "position": 3,
23
+ "label": "What are the best rank tracker tools? : r/SEO - Reddit",
24
+ "url": "https://www.reddit.com/r/SEO/comments/1b3q9a4/what_are_the_best_rank_tracker_tools/",
25
+ "description": "Budget-friendly alternatives include Mangools KW Tracker, Wincher, SERPstat, and Ubersuggest. Serpnest.com is presented as a cost-",
26
+ "redirectLink": "https://www.google.com/goto?url=CAESkAEBxCJF14qM7Z1s1Xl32FFOcpnwLIIOX2Laa-YADKrADFxl47WQhbu2jOvmXgG7D-R-JmMKL_7m7cKxjxIBvZcwWBP3XHvsb0_n8cWEpDI3dyGzhvJPGYZY4BEgnnTxYQfDyy9dnCHGtztcB_n1sLZ4RPqCfenRjZAT8mjPdVchXsET452VShsFPjARZGcMiCs"
27
+ }
28
+ ],
29
+ "citationPills": []
30
+ }
31
+ }
@@ -0,0 +1,28 @@
1
+ {
2
+ "success": true,
3
+ "result": {
4
+ "model": null,
5
+ "text": "If you're looking for the best SEO keyword rank tracking tools in 2026, there are several excellent options. The best choice depends on whether you prioritize daily tracking, local SEO, agency reporting, competitor monitoring, or affordability.\n\nHere are my top picks.\n\nTop 7 rank tracking tools\n\n1. Semrush\n\nBest overall\n\nDaily keyword position tracking\n\nCompetitor rankings and share of voice\n\nMobile, desktop, and local tracking\n\nSERP feature and AI Overview visibility\n\nPricing: from approximately $117/month billed annually.\n\nWebsite and pricing\n\n2. Ahrefs\n\nBest for competitor analysis\n\nKeyword position history\n\nCompetitor ranking comparisons\n\nSERP feature tracking\n\nHistorical ranking data\n\nPricing: Lite from $129/month.\n\nWebsite and pricing\n\n3. SE Ranking\n\nBest value for agencies\n\nDaily ke",
6
+ "sources": [
7
+ {
8
+ "position": 1,
9
+ "label": "Website and pricing",
10
+ "url": "https://www.semrush.com/pricing/seo-ai-search/",
11
+ "footnote": true
12
+ },
13
+ {
14
+ "position": 2,
15
+ "label": "Website and pricing",
16
+ "url": "https://ahrefs.com/pricing?so=1",
17
+ "footnote": true
18
+ },
19
+ {
20
+ "position": 3,
21
+ "label": "Website and pricing",
22
+ "url": "https://seranking.com/subscription.html",
23
+ "footnote": true
24
+ }
25
+ ],
26
+ "citationPills": []
27
+ }
28
+ }
@@ -0,0 +1,8 @@
1
+ {
2
+ "success": false,
3
+ "error": {
4
+ "code": "INVALID_API_KEY_FORMAT",
5
+ "message": "Invalid API key format",
6
+ "timestamp": "2026-11-05T21:11:31.240Z"
7
+ }
8
+ }
@@ -0,0 +1,102 @@
1
+ {
2
+ "success": true,
3
+ "result": {
4
+ "organicResults": [
5
+ {
6
+ "position": 1,
7
+ "title": "The 12 best rank tracker tools",
8
+ "link": "https://zapier.com/blog/best-rank-tracker/",
9
+ "displayedLink": "https://zapier.com > App picks > Best apps",
10
+ "snippet": "Nov 27, 2024 \u2014 Best overall rank tracker Semrush (Web) KWFinder: Keyword research and ideation SERPChecker: SERP competitor analysis SERPWatcher: Rank ...",
11
+ "page": 1,
12
+ "redirectLink": "https://www.google.com/goto?url=CAESZwHEIkXXEPzfBAXr_G7ZilnrSgFFJIPBGEj4P490RYY-bdfrCgY02YcCB0a3DiMm4pCKSyLM9BCK3IzOmg9vbBOMaNJvTnzvvmzkrbHvvmdlIpW1OlQuzRGZHV93lUatBU6aChtfKFc"
13
+ },
14
+ {
15
+ "position": 2,
16
+ "title": "15 Best Rank Tracking Software We're Using for 2026",
17
+ "link": "https://onelittleweb.com/top-tools/best-rank-tracking-software/",
18
+ "displayedLink": "https://onelittleweb.com > Top Tools",
19
+ "snippet": "Aug 26, 2026 \u2014 The best tools like Semrush, Ahrefs, SE Ranking, and AccuRanker now track how your brand shows up in AI Overviews, ChatGPT, and Gemini, not just ...",
20
+ "page": 1,
21
+ "redirectLink": "https://www.google.com/goto?url=CAESfAHEIkXX0xLNZ5xH0S-ou0gHfPaOH7J1V9EUugn4wX3US0D8i_7OeHSII6ifddxm3QKqcbRKDUMTvQDwzcWhToNw2Ci_SFYL_LZfQ4EHY941BAw_Hv_AixAfHWypMwWR2xuy5wr6AOk4l0X_pIDxEt4dSCOz0sk3ZNiHNos"
22
+ },
23
+ {
24
+ "position": 3,
25
+ "title": "14 best rank tracker software I'm using for SEO in 2026",
26
+ "link": "https://www.marketermilk.com/blog/best-rank-tracker-software",
27
+ "displayedLink": "https://www.marketermilk.com > blog > best-rank-track...",
28
+ "snippet": "Jan 28, 2026 \u2014 Here are the best rank trackers I recommend: Nightwatch (best for local keyword tracking) SERPWatcher by Mangools (best for SERP monitor) ...",
29
+ "page": 1,
30
+ "redirectLink": "https://www.google.com/goto?url=CAESeQHEIkXXtn9-F5G1rDOxV0S25iM_-3zWGjtCRFexuj62rif1tst3meVeTUoVk1oGdZPQeiumIN_T87VFXIoGnXxv_u72yj9YynfAmB0q8v9BX0rJcvafTQNKHJR5nrIYdY31reUl29RFG-z-lvKcdJ67j04pba8rDf8"
31
+ },
32
+ {
33
+ "position": 4,
34
+ "title": "15 Best Keyword Tracking Tools for 2026: AI & Pixel Rank",
35
+ "link": "https://www.yotpo.com/blog/best-keyword-tracking-tools/",
36
+ "displayedLink": "https://www.yotpo.com > blog > best-keyword-tracking...",
37
+ "snippet": "Oct 20, 2026 \u2014 SEO PowerSuite (Rank Tracker) For small businesses or lean teams, Wincher and SE Ranking are the top recommendations.",
38
+ "page": 1,
39
+ "redirectLink": "https://www.google.com/goto?url=CAESdAHEIkXXpXurgKiiFTwID9Dg3Dq_UbWLGZqMNs-WIhYwDq_KWlQ7DO2AVP6dKdKMiGU8C061woImkLX7co4XnHUVj4femt7VkjML7XrpMBP8j7INg0hYoTAe_ONmPIu_WDyeW-ZODgcXYtjgvZmnpr6EuAWv"
40
+ },
41
+ {
42
+ "position": 5,
43
+ "title": "Empty link",
44
+ "link": "",
45
+ "displayedLink": "",
46
+ "snippet": "x",
47
+ "page": 1
48
+ },
49
+ {
50
+ "position": 6,
51
+ "title": "Redirect link",
52
+ "link": "https://www.google.com/goto?url=abc",
53
+ "displayedLink": "",
54
+ "snippet": "x",
55
+ "page": 1
56
+ }
57
+ ],
58
+ "peopleAlsoAsk": [
59
+ {
60
+ "question": "What is the best rank tracker tool?",
61
+ "type": "AIOVERVIEW"
62
+ }
63
+ ],
64
+ "relatedSearches": [
65
+ {
66
+ "query": "Best rank tracking tools free",
67
+ "link": "https://www.google.com/search?sca_esv=514e63bafcc8d03f&gl=us&hl=en&sxsrf=AKL5yDhCo1f9wA2fbmrEeKauc3btkhEzsw:1793913014958&q=Best+rank+tracking+tools+free&sa=X&ved=2ahUKEwjDtY7g3vGXAxXpg_0HHaCWJHgQ1QJ6BAhDEAE"
68
+ },
69
+ {
70
+ "query": "Best rank tracking tools html",
71
+ "link": "https://www.google.com/search?sca_esv=514e63bafcc8d03f&gl=us&hl=en&sxsrf=AKL5yDhCo1f9wA2fbmrEeKauc3btkhEzsw:1793913014958&q=Best+rank+tracking+tools+html&sa=X&ved=2ahUKEwjDtY7g3vGXAxXpg_0HHaCWJHgQ1QJ6BAhKEAE"
72
+ }
73
+ ],
74
+ "aioverview": {
75
+ "text": "The best rank tracker tools cater to different needs, from all-in-one SEO suites to budget-friendly or real-time specialized options:\n\nSemrush: Best overall functionality and large-scale marketing suites for teams.\n\nAhrefs: Best for deep keyword analysis, backlink tracking, and SEO visibility.\n\nSE Ranking: Best balance of affordability, daily tracking across 140+ countries, and AI-overviews visibility for agencies and small-to-medium businesses (starting around $65/month).\n\nAccuRanker: Best for real-time, on-demand updates and high-precision tracking.\n\nMangools (SERPWatcher): Best for clean, simple reporting and low-volume or beginner-friendly keyword tracking (from $49/month)\n\n.\n\nAdvanced Web Ranking: Best for white-label reporting and large agencies with high keyword volumes (from $99/mo",
76
+ "sources": [
77
+ {
78
+ "label": "The 12 best rank tracker tools | Zapier",
79
+ "url": "https://zapier.com/blog/best-rank-tracker/",
80
+ "description": "The 12 best rank trackers * Semrush for overall functionality * SEO Tester Online for an all-in-one SEO suite * Ahrefs for keyword...",
81
+ "position": 1,
82
+ "redirectLink": "https://www.google.com/goto?url=CAESZwHEIkXXoe3bD3eaDC68bpV2ju6caEHnGLqWQ0bQYsSJO2ASn217d4kn87kQ1Rmaau-NHVI5fdKpI5hOb1_82TMlgF8hJEx4I5rJhDMeRD4GPOqnlKqq8IxqkqI5uw20b7bSXEdnSYc"
83
+ },
84
+ {
85
+ "label": "What are the best rank tracker tools? : r/SEO - Reddit",
86
+ "url": "https://www.reddit.com/r/SEO/comments/1b3q9a4/what_are_the_best_rank_tracker_tools/",
87
+ "description": "Paid Rank Tracking Tools: Premium options like SEMrush and Ahrefs provide detailed data, though weekly updates may require expensi...",
88
+ "position": 2,
89
+ "redirectLink": "https://www.google.com/goto?url=CAESkAEBxCJF13jxB1ZV6Sx_0WvsMMRPcBm9tz365bB86dHzGF5oa47LQdpNiPtmVPcfGeHXqDZ_lGSmgpB1FkLbMk7BAV76eq_7gUxRu6hSRAHa4xEUAdTj1cYEB26LT9MB7gzMs5MCUxVArYg2EEyMDH3Sl8nEvmxtDUy6RGZAtJ7UugdIQVcIbreUWOSmJ7Z4y5E"
90
+ },
91
+ {
92
+ "label": "15 Best Rank Tracking Software (Tested on Real Campaigns)",
93
+ "url": "https://onelittleweb.com/top-tools/best-rank-tracking-software/",
94
+ "description": "Each of these tools cover different roles in real SEO workflows. 1. Semrush \u2014 Best all-in-one tracking for large teams 2. Ahrefs \u2014...",
95
+ "position": 3,
96
+ "redirectLink": "https://www.google.com/goto?url=CAESfAHEIkXXfaUHy5q3Kuq7S3ZMrkSQkEpzFHRNDh4LVQrG3iwbPxidFPPVQaeCqpZWkPjI0ddT2vsy8ak6ECQo4MHXlOq5fbMPCoDGZggFP1REQxbeFm8ejFpUTVUAzggVas4boQ9L1ltz91VzuVe1SUlp-9EbjjKRoqgLw88"
97
+ }
98
+ ],
99
+ "citationPills": []
100
+ }
101
+ }
102
+ }
@@ -0,0 +1,251 @@
1
+ """Offline tests for cloro-haystack.
2
+
3
+ All HTTP goes through an httpx.MockTransport, so no test calls the cloro API.
4
+ The fixtures are shortened copies of real cloro API responses.
5
+ """
6
+
7
+ import asyncio
8
+ import copy
9
+ import json
10
+ from pathlib import Path
11
+
12
+ import httpx
13
+ import pytest
14
+ from haystack import Document, Pipeline
15
+ from haystack.components.builders import AnswerBuilder
16
+ from haystack.utils import Secret
17
+
18
+ from haystack_integrations.components.websearch.cloro import CloroAIAnswer, CloroError, CloroWebSearch, _api
19
+ from haystack_integrations.components.websearch.cloro._api import is_direct_link
20
+
21
+ FIXTURES = Path(__file__).parent / "fixtures"
22
+
23
+
24
+ def _load(name):
25
+ return json.loads((FIXTURES / name).read_text())
26
+
27
+
28
+ @pytest.fixture(autouse=True)
29
+ def _env(monkeypatch):
30
+ monkeypatch.setenv("CLORO_API_KEY", "test-key")
31
+
32
+
33
+ @pytest.fixture
34
+ def api(monkeypatch):
35
+ """Route every cloro request to a handler that returns `api.response`."""
36
+ state = {"calls": [], "response": httpx.Response(200, json={"success": True, "result": {}})}
37
+
38
+ def handler(request):
39
+ state["calls"].append(request)
40
+ response = state["response"]
41
+ if isinstance(response, Exception):
42
+ raise response
43
+ return response
44
+
45
+ transport = httpx.MockTransport(handler)
46
+ real_async_client = httpx.AsyncClient
47
+ monkeypatch.setattr(_api.httpx, "post", lambda url, **kw: httpx.Client(transport=transport).post(url, **kw))
48
+ monkeypatch.setattr(_api.httpx, "AsyncClient", lambda **kw: real_async_client(transport=transport, **kw))
49
+
50
+ class Api:
51
+ @property
52
+ def calls(self):
53
+ return state["calls"]
54
+
55
+ def respond(self, response):
56
+ state["response"] = response
57
+
58
+ return Api()
59
+
60
+
61
+ def _body(request):
62
+ return json.loads(request.content)
63
+
64
+
65
+ # CloroWebSearch
66
+
67
+
68
+ def test_websearch_request(api):
69
+ api.respond(httpx.Response(200, json=_load("google_aioverview.json")))
70
+ CloroWebSearch().run(query="rank tracker")
71
+ request = api.calls[0]
72
+ assert str(request.url) == "https://api.cloro.dev/v1/monitor/google"
73
+ assert request.headers["Authorization"] == "Bearer test-key"
74
+ assert _body(request) == {"query": "rank tracker", "country": "US"}
75
+
76
+
77
+ def test_websearch_documents_and_links(api):
78
+ api.respond(httpx.Response(200, json=_load("google_aioverview.json")))
79
+ result = CloroWebSearch().run(query="rank tracker")
80
+ assert len(result["documents"]) == 4
81
+ assert result["links"] == [d.meta["link"] for d in result["documents"]]
82
+ first = result["documents"][0]
83
+ assert first.meta == {
84
+ "title": "The 12 best rank tracker tools",
85
+ "link": "https://zapier.com/blog/best-rank-tracker/",
86
+ "position": 1,
87
+ }
88
+ assert first.content.startswith("Nov 27, 2024")
89
+ assert all("google.com/goto" not in link for link in result["links"])
90
+
91
+
92
+ def test_websearch_ai_overview_first(api):
93
+ payload = _load("google_aioverview.json")
94
+ api.respond(httpx.Response(200, json=payload))
95
+ result = CloroWebSearch(include_ai_overview=True).run(query="rank tracker")
96
+ assert _body(api.calls[0])["include"] == {"aioverview": True}
97
+ overview = result["documents"][0]
98
+ assert overview.content == payload["result"]["aioverview"]["text"]
99
+ assert overview.meta["source"] == "google_ai_overview"
100
+ assert overview.meta["links"] == [s["url"] for s in payload["result"]["aioverview"]["sources"]]
101
+ assert len(result["documents"]) == 5
102
+ assert len(result["links"]) == 4
103
+
104
+
105
+ def test_websearch_ai_overview_ignored_when_not_requested(api):
106
+ api.respond(httpx.Response(200, json=_load("google_aioverview.json")))
107
+ result = CloroWebSearch().run(query="q")
108
+ assert all(d.meta.get("source") != "google_ai_overview" for d in result["documents"])
109
+
110
+
111
+ def test_websearch_top_k_domains_and_params(api):
112
+ api.respond(httpx.Response(200, json=_load("google_aioverview.json")))
113
+ search = CloroWebSearch(
114
+ top_k=2, country="DE", allowed_domains=["a.com", "b.org"], search_params={"device": "mobile", "pages": 2}
115
+ )
116
+ result = search.run(query="q")
117
+ assert _body(api.calls[0]) == {
118
+ "query": "q site:a.com OR site:b.org",
119
+ "country": "DE",
120
+ "device": "mobile",
121
+ "pages": 2,
122
+ }
123
+ assert len(result["documents"]) == 2
124
+ assert len(result["links"]) == 2
125
+
126
+
127
+ def test_websearch_run_async(api):
128
+ api.respond(httpx.Response(200, json=_load("google_aioverview.json")))
129
+ result = asyncio.run(CloroWebSearch().run_async(query="q"))
130
+ assert len(result["documents"]) == 4
131
+
132
+
133
+ # CloroAIAnswer
134
+
135
+
136
+ @pytest.mark.parametrize("engine", ["chatgpt", "aimode", "gemini"])
137
+ def test_ai_answer_request(api, engine):
138
+ api.respond(httpx.Response(200, json=_load("aimode.json")))
139
+ CloroAIAnswer(engine=engine).run(query="rank tracker")
140
+ request = api.calls[0]
141
+ assert str(request.url) == f"https://api.cloro.dev/v1/monitor/{engine}"
142
+ assert _body(request) == {"prompt": "rank tracker", "country": "US", "include": {"markdown": True}}
143
+
144
+
145
+ def test_ai_answer_rejects_other_engines():
146
+ with pytest.raises(ValueError, match="engine"):
147
+ CloroAIAnswer(engine="other") # type: ignore[arg-type]
148
+
149
+
150
+ def test_ai_answer_output(api):
151
+ payload = _load("aimode.json")
152
+ api.respond(httpx.Response(200, json=payload))
153
+ result = CloroAIAnswer().run(query="rank tracker")
154
+ assert result["replies"] == [payload["result"]["markdown"]]
155
+ sources = payload["result"]["sources"]
156
+ assert result["links"] == [s["url"] for s in sources]
157
+ assert result["documents"][0].meta == {"title": sources[0]["label"], "link": sources[0]["url"], "position": 1}
158
+ assert result["documents"][0].content == sources[0]["description"]
159
+
160
+
161
+ def test_ai_answer_chatgpt_without_markdown_or_description(api):
162
+ payload = _load("chatgpt.json")
163
+ api.respond(httpx.Response(200, json=payload))
164
+ result = CloroAIAnswer(engine="chatgpt").run(query="q")
165
+ assert result["replies"] == [payload["result"]["text"]]
166
+ assert result["documents"][0].content == payload["result"]["sources"][0]["label"]
167
+
168
+
169
+ def test_ai_answer_run_async(api):
170
+ api.respond(httpx.Response(200, json=_load("aimode.json")))
171
+ result = asyncio.run(CloroAIAnswer().run_async(query="q"))
172
+ assert len(result["replies"]) == 1
173
+
174
+
175
+ def test_ai_answer_connects_to_answer_builder(api):
176
+ api.respond(httpx.Response(200, json=_load("aimode.json")))
177
+ pipe = Pipeline()
178
+ pipe.add_component("ai", CloroAIAnswer())
179
+ pipe.add_component("answer", AnswerBuilder())
180
+ pipe.connect("ai.replies", "answer.replies")
181
+ pipe.connect("ai.documents", "answer.documents")
182
+ result = pipe.run({"ai": {"query": "rank tracker"}, "answer": {"query": "rank tracker"}})
183
+ answer = result["answer"]["answers"][0]
184
+ assert answer.data
185
+ assert all(isinstance(d, Document) for d in answer.documents)
186
+
187
+
188
+ # Errors
189
+
190
+
191
+ @pytest.mark.parametrize("component", [CloroWebSearch(), CloroAIAnswer()])
192
+ @pytest.mark.parametrize(
193
+ ("response", "match"),
194
+ [
195
+ (httpx.Response(401, json=_load("error_401.json")), "INVALID_API_KEY_FORMAT"),
196
+ (
197
+ httpx.Response(403, json={"success": False, "error": {"code": "INSUFFICIENT_CREDITS"}}),
198
+ "INSUFFICIENT_CREDITS",
199
+ ),
200
+ (httpx.Response(502, text="Bad gateway"), "HTTP 502"),
201
+ (httpx.Response(200, json={"success": True, "result": None}), "no result"),
202
+ (httpx.ConnectError("down"), "failed"),
203
+ ],
204
+ )
205
+ def test_errors_raise_cloro_error(api, component, response, match):
206
+ api.respond(response)
207
+ with pytest.raises(CloroError, match=match):
208
+ component.run(query="q")
209
+
210
+
211
+ def test_timeout_raises_timeout_error(api):
212
+ api.respond(httpx.ReadTimeout("slow"))
213
+ with pytest.raises(TimeoutError):
214
+ CloroWebSearch().run(query="q")
215
+
216
+
217
+ def test_error_message_has_no_key(api):
218
+ api.respond(httpx.Response(401, json=_load("error_401.json")))
219
+ with pytest.raises(CloroError) as info:
220
+ CloroWebSearch().run(query="q")
221
+ assert "test-key" not in str(info.value)
222
+
223
+
224
+ # Serialization
225
+
226
+
227
+ @pytest.mark.parametrize(
228
+ "component",
229
+ [
230
+ CloroWebSearch(top_k=3, country="GB", include_ai_overview=True, search_params={"device": "mobile"}),
231
+ CloroAIAnswer(engine="gemini", country="FR"),
232
+ ],
233
+ )
234
+ def test_to_dict_from_dict_round_trip(component):
235
+ data = component.to_dict()
236
+ assert data["init_parameters"]["api_key"] == {"type": "env_var", "env_vars": ["CLORO_API_KEY"], "strict": True}
237
+ assert "test-key" not in json.dumps(data)
238
+ restored = type(component).from_dict(copy.deepcopy(data))
239
+ assert restored.to_dict() == data
240
+
241
+
242
+ def test_token_secret_is_not_serializable():
243
+ with pytest.raises(ValueError):
244
+ CloroWebSearch(api_key=Secret.from_token("test-key")).to_dict()
245
+
246
+
247
+ def test_google_search_and_shopping_links_are_not_sources():
248
+ assert is_direct_link("https://aws.amazon.com/what-is/retrieval-augmented-generation/")
249
+ assert not is_direct_link("https://www.google.com/search?q=product&ibp=oshop")
250
+ assert not is_direct_link("https://www.google.com/goto?url=x")
251
+ assert not is_direct_link("")