cloro-haystack 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/PKG-INFO +26 -11
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/README.md +23 -8
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/pyproject.toml +3 -3
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/src/haystack_integrations/components/websearch/cloro/ai_answer.py +17 -7
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/src/haystack_integrations/components/websearch/cloro/websearch.py +33 -17
- cloro_haystack-0.3.0/tests/fixtures/bing.json +59 -0
- cloro_haystack-0.3.0/tests/fixtures/bing_aimode.json +36 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/tests/test_cloro.py +38 -3
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/.gitignore +0 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/LICENSE +0 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/src/haystack_integrations/components/websearch/cloro/__init__.py +0 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/src/haystack_integrations/components/websearch/cloro/_api.py +0 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/src/haystack_integrations/components/websearch/py.typed +0 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/tests/fixtures/aimode.json +0 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/tests/fixtures/chatgpt.json +0 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/tests/fixtures/error_401.json +0 -0
- {cloro_haystack-0.1.0 → cloro_haystack-0.3.0}/tests/fixtures/google_aioverview.json +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloro-haystack
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Haystack components for cloro: Google search results with the AI Overview, and ChatGPT, Google AI Mode and
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Haystack components for cloro: Google and Bing search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources.
|
|
5
5
|
Project-URL: Homepage, https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
|
|
6
6
|
Project-URL: Documentation, https://github.com/cloro-dev/cloro-haystack#readme
|
|
7
7
|
Project-URL: Issues, https://github.com/cloro-dev/cloro-haystack/issues
|
|
@@ -9,7 +9,7 @@ Project-URL: Source, https://github.com/cloro-dev/cloro-haystack
|
|
|
9
9
|
Author-email: cloro <support@cloro.dev>
|
|
10
10
|
License-Expression: MIT
|
|
11
11
|
License-File: LICENSE
|
|
12
|
-
Keywords: AI Mode,AI Overview,ChatGPT,Gemini,Google,Haystack,Web Search,cloro
|
|
12
|
+
Keywords: AI Mode,AI Overview,Bing,ChatGPT,Gemini,Google,Haystack,Web Search,cloro
|
|
13
13
|
Classifier: Development Status :: 4 - Beta
|
|
14
14
|
Classifier: License :: OSI Approved :: MIT License
|
|
15
15
|
Classifier: Programming Language :: Python
|
|
@@ -28,8 +28,8 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
|
|
29
29
|
| Component | cloro endpoint | Output | Credits per run |
|
|
30
30
|
|---|---|---|--:|
|
|
31
|
-
| `CloroWebSearch` | `POST /v1/monitor/google` | `documents` and `links` from Google organic results. With `include_ai_overview=True`, the
|
|
32
|
-
| `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode` or `/
|
|
31
|
+
| `CloroWebSearch` | `POST /v1/monitor/google` or `/bing` | `documents` and `links` from Google or Bing organic results. With `include_ai_overview=True`, the AI Overview is the first Document. | 5, or 7 with the AI Overview |
|
|
32
|
+
| `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode`, `/gemini` or `/bing/aimode` | `replies` (the answer as Markdown), `documents` and `links` (the cited sources) | ChatGPT 7, Google AI Mode 6, Gemini 6, Bing AI Mode 6 |
|
|
33
33
|
|
|
34
34
|
## Installation
|
|
35
35
|
|
|
@@ -55,9 +55,10 @@ for doc in result["documents"]:
|
|
|
55
55
|
print(doc.meta.get("title"), doc.meta.get("link"))
|
|
56
56
|
```
|
|
57
57
|
|
|
58
|
-
Each organic result is a `Document`. Its `content` is the
|
|
59
|
-
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"`
|
|
60
|
-
URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
58
|
+
Each organic result is a `Document`. Its `content` is the result snippet and its `meta` has `title`,
|
|
59
|
+
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"` (or
|
|
60
|
+
`"bing_ai_overview"`) and the URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
61
|
+
redirect link are skipped.
|
|
61
62
|
|
|
62
63
|
Parameters:
|
|
63
64
|
|
|
@@ -66,16 +67,26 @@ Parameters:
|
|
|
66
67
|
results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
|
|
67
68
|
- `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
|
|
68
69
|
- `allowed_domains`: domains to limit the search to, sent as `site:` filters.
|
|
69
|
-
- `include_ai_overview`: request the
|
|
70
|
-
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages
|
|
70
|
+
- `include_ai_overview`: request the AI Overview. Defaults to `False`.
|
|
71
|
+
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google.
|
|
71
72
|
- `timeout`: request timeout in seconds. Defaults to 60.
|
|
73
|
+
- `engine`: `"google"` (Google Search) or `"bing"` (Bing Search). Defaults to `"google"`.
|
|
74
|
+
|
|
75
|
+
With `engine="bing"`, the component searches Bing. The country sets the Bing market. Bing Search accepts
|
|
76
|
+
all countries except CN, KR and RU. It has no `device`, `location`, `hl` or `pages` fields and returns
|
|
77
|
+
one page of results. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
|
|
78
|
+
non-Latin scripts.
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
websearch = CloroWebSearch(engine="bing", country="GB", include_ai_overview=True)
|
|
82
|
+
```
|
|
72
83
|
|
|
73
84
|
### CloroAIAnswer
|
|
74
85
|
|
|
75
86
|
```python
|
|
76
87
|
from haystack_integrations.components.websearch.cloro import CloroAIAnswer
|
|
77
88
|
|
|
78
|
-
ai_answer = CloroAIAnswer(engine="aimode") # "chatgpt", "aimode" or "
|
|
89
|
+
ai_answer = CloroAIAnswer(engine="aimode") # "chatgpt", "aimode", "gemini" or "bing_aimode"
|
|
79
90
|
result = ai_answer.run(query="What is retrieval-augmented generation?")
|
|
80
91
|
|
|
81
92
|
print(result["replies"][0])
|
|
@@ -86,6 +97,10 @@ print(result["links"])
|
|
|
86
97
|
the answer cites, with `title`, `link` and `position` in `meta`. An answer can take 15 to 60 seconds,
|
|
87
98
|
so the default `timeout` is 180 seconds.
|
|
88
99
|
|
|
100
|
+
With `engine="bing_aimode"`, the query can have a maximum of 3,500 characters after URL encoding,
|
|
101
|
+
and fewer for non-Latin scripts. To send the query from a US state, set
|
|
102
|
+
`search_params={"state": "CA"}` with `country="US"` (2 more credits).
|
|
103
|
+
|
|
89
104
|
### In a pipeline
|
|
90
105
|
|
|
91
106
|
The output names of `CloroAIAnswer` match `AnswerBuilder`:
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
|
|
6
6
|
| Component | cloro endpoint | Output | Credits per run |
|
|
7
7
|
|---|---|---|--:|
|
|
8
|
-
| `CloroWebSearch` | `POST /v1/monitor/google` | `documents` and `links` from Google organic results. With `include_ai_overview=True`, the
|
|
9
|
-
| `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode` or `/
|
|
8
|
+
| `CloroWebSearch` | `POST /v1/monitor/google` or `/bing` | `documents` and `links` from Google or Bing organic results. With `include_ai_overview=True`, the AI Overview is the first Document. | 5, or 7 with the AI Overview |
|
|
9
|
+
| `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode`, `/gemini` or `/bing/aimode` | `replies` (the answer as Markdown), `documents` and `links` (the cited sources) | ChatGPT 7, Google AI Mode 6, Gemini 6, Bing AI Mode 6 |
|
|
10
10
|
|
|
11
11
|
## Installation
|
|
12
12
|
|
|
@@ -32,9 +32,10 @@ for doc in result["documents"]:
|
|
|
32
32
|
print(doc.meta.get("title"), doc.meta.get("link"))
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
-
Each organic result is a `Document`. Its `content` is the
|
|
36
|
-
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"`
|
|
37
|
-
URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
35
|
+
Each organic result is a `Document`. Its `content` is the result snippet and its `meta` has `title`,
|
|
36
|
+
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"` (or
|
|
37
|
+
`"bing_ai_overview"`) and the URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
38
|
+
redirect link are skipped.
|
|
38
39
|
|
|
39
40
|
Parameters:
|
|
40
41
|
|
|
@@ -43,16 +44,26 @@ Parameters:
|
|
|
43
44
|
results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
|
|
44
45
|
- `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
|
|
45
46
|
- `allowed_domains`: domains to limit the search to, sent as `site:` filters.
|
|
46
|
-
- `include_ai_overview`: request the
|
|
47
|
-
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages
|
|
47
|
+
- `include_ai_overview`: request the AI Overview. Defaults to `False`.
|
|
48
|
+
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google.
|
|
48
49
|
- `timeout`: request timeout in seconds. Defaults to 60.
|
|
50
|
+
- `engine`: `"google"` (Google Search) or `"bing"` (Bing Search). Defaults to `"google"`.
|
|
51
|
+
|
|
52
|
+
With `engine="bing"`, the component searches Bing. The country sets the Bing market. Bing Search accepts
|
|
53
|
+
all countries except CN, KR and RU. It has no `device`, `location`, `hl` or `pages` fields and returns
|
|
54
|
+
one page of results. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
|
|
55
|
+
non-Latin scripts.
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
websearch = CloroWebSearch(engine="bing", country="GB", include_ai_overview=True)
|
|
59
|
+
```
|
|
49
60
|
|
|
50
61
|
### CloroAIAnswer
|
|
51
62
|
|
|
52
63
|
```python
|
|
53
64
|
from haystack_integrations.components.websearch.cloro import CloroAIAnswer
|
|
54
65
|
|
|
55
|
-
ai_answer = CloroAIAnswer(engine="aimode") # "chatgpt", "aimode" or "
|
|
66
|
+
ai_answer = CloroAIAnswer(engine="aimode") # "chatgpt", "aimode", "gemini" or "bing_aimode"
|
|
56
67
|
result = ai_answer.run(query="What is retrieval-augmented generation?")
|
|
57
68
|
|
|
58
69
|
print(result["replies"][0])
|
|
@@ -63,6 +74,10 @@ print(result["links"])
|
|
|
63
74
|
the answer cites, with `title`, `link` and `position` in `meta`. An answer can take 15 to 60 seconds,
|
|
64
75
|
so the default `timeout` is 180 seconds.
|
|
65
76
|
|
|
77
|
+
With `engine="bing_aimode"`, the query can have a maximum of 3,500 characters after URL encoding,
|
|
78
|
+
and fewer for non-Latin scripts. To send the query from a US state, set
|
|
79
|
+
`search_params={"state": "CA"}` with `country="US"` (2 more credits).
|
|
80
|
+
|
|
66
81
|
### In a pipeline
|
|
67
82
|
|
|
68
83
|
The output names of `CloroAIAnswer` match `AnswerBuilder`:
|
|
@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cloro-haystack"
|
|
7
|
-
version = "0.
|
|
8
|
-
description = "Haystack components for cloro: Google search results with the AI Overview, and ChatGPT, Google AI Mode and
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "Haystack components for cloro: Google and Bing search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
11
11
|
license = "MIT"
|
|
12
12
|
license-files = ["LICENSE"]
|
|
13
|
-
keywords = ["Haystack", "cloro", "Web Search", "Google", "AI Overview", "AI Mode", "ChatGPT", "Gemini"]
|
|
13
|
+
keywords = ["Haystack", "cloro", "Web Search", "Google", "AI Overview", "AI Mode", "ChatGPT", "Gemini", "Bing"]
|
|
14
14
|
authors = [{ name = "cloro", email = "support@cloro.dev" }]
|
|
15
15
|
classifiers = [
|
|
16
16
|
"Development Status :: 4 - Beta",
|
|
@@ -11,7 +11,15 @@ from haystack_integrations.components.websearch.cloro._api import is_direct_link
|
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
14
|
-
Engine = Literal["chatgpt", "aimode", "gemini"]
|
|
14
|
+
Engine = Literal["chatgpt", "aimode", "gemini", "bing_aimode"]
|
|
15
|
+
|
|
16
|
+
# The API path for each engine.
|
|
17
|
+
_PATHS: dict[str, str] = {
|
|
18
|
+
"chatgpt": "/v1/monitor/chatgpt",
|
|
19
|
+
"aimode": "/v1/monitor/aimode",
|
|
20
|
+
"gemini": "/v1/monitor/gemini",
|
|
21
|
+
"bing_aimode": "/v1/monitor/bing/aimode",
|
|
22
|
+
}
|
|
15
23
|
|
|
16
24
|
|
|
17
25
|
@component
|
|
@@ -19,7 +27,7 @@ class CloroAIAnswer:
|
|
|
19
27
|
"""
|
|
20
28
|
Gets the answer that an AI search engine gives to a query, with the sources it cites.
|
|
21
29
|
|
|
22
|
-
The component sends the query to ChatGPT, Google AI Mode or
|
|
30
|
+
The component sends the query to ChatGPT, Google AI Mode, Gemini or Bing AI Mode through the
|
|
23
31
|
[cloro](https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok) API.
|
|
24
32
|
`replies` has the answer as Markdown. `documents` has one
|
|
25
33
|
Document for each cited source, with `title`, `link` and `position` in its meta.
|
|
@@ -51,16 +59,18 @@ class CloroAIAnswer:
|
|
|
51
59
|
:param api_key:
|
|
52
60
|
cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
|
|
53
61
|
:param engine:
|
|
54
|
-
The AI search engine: `"chatgpt"`, `"aimode"` (Google AI Mode)
|
|
62
|
+
The AI search engine: `"chatgpt"`, `"aimode"` (Google AI Mode), `"gemini"` or
|
|
63
|
+
`"bing_aimode"` (Bing AI Mode).
|
|
55
64
|
:param country:
|
|
56
65
|
ISO 3166-1 alpha-2 country code for the engine location.
|
|
57
66
|
:param search_params:
|
|
58
|
-
Other fields for the
|
|
67
|
+
Other fields for the engine request body, for example `state` for ChatGPT, Gemini or
|
|
68
|
+
Bing AI Mode. See the cloro API reference.
|
|
59
69
|
:param timeout:
|
|
60
70
|
Request timeout in seconds. An answer can take 15 to 60 seconds.
|
|
61
71
|
"""
|
|
62
|
-
if engine not in
|
|
63
|
-
msg = f"engine must be 'chatgpt', 'aimode' or '
|
|
72
|
+
if engine not in _PATHS:
|
|
73
|
+
msg = f"engine must be 'chatgpt', 'aimode', 'gemini' or 'bing_aimode', not {engine!r}"
|
|
64
74
|
raise ValueError(msg)
|
|
65
75
|
self.api_key = api_key
|
|
66
76
|
self.engine = engine
|
|
@@ -129,7 +139,7 @@ class CloroAIAnswer:
|
|
|
129
139
|
return self._parse(result)
|
|
130
140
|
|
|
131
141
|
def _path(self) -> str:
|
|
132
|
-
return
|
|
142
|
+
return _PATHS[self.engine]
|
|
133
143
|
|
|
134
144
|
def _payload(self, query: str) -> dict[str, Any]:
|
|
135
145
|
payload: dict[str, Any] = {"prompt": query, "country": self.country, **(self.search_params or {})}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: MIT
|
|
4
4
|
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any, Literal
|
|
6
6
|
|
|
7
7
|
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
|
8
8
|
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
@@ -11,19 +11,23 @@ from haystack_integrations.components.websearch.cloro._api import is_direct_link
|
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
Engine = Literal["google", "bing"]
|
|
15
|
+
|
|
16
|
+
# The API path and the display name for each engine.
|
|
17
|
+
_PATHS: dict[str, str] = {"google": "/v1/monitor/google", "bing": "/v1/monitor/bing"}
|
|
18
|
+
_NAMES: dict[str, str] = {"google": "Google", "bing": "Bing"}
|
|
15
19
|
|
|
16
20
|
|
|
17
21
|
@component
|
|
18
22
|
class CloroWebSearch:
|
|
19
23
|
"""
|
|
20
|
-
Searches Google with the cloro API and returns the results as Documents.
|
|
24
|
+
Searches Google or Bing with the cloro API and returns the results as Documents.
|
|
21
25
|
|
|
22
26
|
cloro: https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
|
|
23
27
|
|
|
24
|
-
Each organic result becomes a Document. Its content is the
|
|
25
|
-
`title`, `link` and `position`. With `include_ai_overview=True`, the
|
|
26
|
-
(when
|
|
28
|
+
Each organic result becomes a Document. Its content is the result snippet. Its meta has
|
|
29
|
+
`title`, `link` and `position`. With `include_ai_overview=True`, the AI Overview of the
|
|
30
|
+
engine (when the engine shows one) is the first Document.
|
|
27
31
|
|
|
28
32
|
### Usage example
|
|
29
33
|
|
|
@@ -46,6 +50,7 @@ class CloroWebSearch:
|
|
|
46
50
|
include_ai_overview: bool = False,
|
|
47
51
|
search_params: dict[str, Any] | None = None,
|
|
48
52
|
timeout: float = 60.0,
|
|
53
|
+
engine: Engine = "google",
|
|
49
54
|
) -> None:
|
|
50
55
|
"""
|
|
51
56
|
Initialize the CloroWebSearch component.
|
|
@@ -53,22 +58,28 @@ class CloroWebSearch:
|
|
|
53
58
|
:param api_key:
|
|
54
59
|
cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
|
|
55
60
|
:param top_k:
|
|
56
|
-
Maximum number of organic results to return. One request returns one page of
|
|
57
|
-
|
|
61
|
+
Maximum number of organic results to return. One request returns one page of results
|
|
62
|
+
(about 10). For Google, set `search_params={"pages": 2}` or more to get more results.
|
|
58
63
|
:param country:
|
|
59
|
-
ISO 3166-1 alpha-2 country code for the
|
|
64
|
+
ISO 3166-1 alpha-2 country code for the engine location.
|
|
60
65
|
:param allowed_domains:
|
|
61
66
|
Domains to limit the search to. They are added to the query as `site:` filters.
|
|
62
67
|
:param include_ai_overview:
|
|
63
|
-
If True, request the
|
|
68
|
+
If True, request the AI Overview and return it as the first Document.
|
|
64
69
|
:param search_params:
|
|
65
|
-
Other fields for the
|
|
66
|
-
`
|
|
70
|
+
Other fields for the engine request body, for example `device`, `location`, `hl` or
|
|
71
|
+
`pages` for Google. See the cloro API reference.
|
|
67
72
|
:param timeout:
|
|
68
73
|
Request timeout in seconds.
|
|
74
|
+
:param engine:
|
|
75
|
+
The search engine: `"google"` (Google Search) or `"bing"` (Bing Search).
|
|
69
76
|
"""
|
|
77
|
+
if engine not in _PATHS:
|
|
78
|
+
msg = f"engine must be 'google' or 'bing', not {engine!r}"
|
|
79
|
+
raise ValueError(msg)
|
|
70
80
|
self.api_key = api_key
|
|
71
81
|
self.top_k = top_k
|
|
82
|
+
self.engine = engine
|
|
72
83
|
self.country = country
|
|
73
84
|
self.allowed_domains = allowed_domains
|
|
74
85
|
self.include_ai_overview = include_ai_overview
|
|
@@ -90,6 +101,7 @@ class CloroWebSearch:
|
|
|
90
101
|
include_ai_overview=self.include_ai_overview,
|
|
91
102
|
search_params=self.search_params,
|
|
92
103
|
timeout=self.timeout,
|
|
104
|
+
engine=self.engine,
|
|
93
105
|
)
|
|
94
106
|
|
|
95
107
|
@classmethod
|
|
@@ -106,7 +118,7 @@ class CloroWebSearch:
|
|
|
106
118
|
@component.output_types(documents=list[Document], links=list[str])
|
|
107
119
|
def run(self, query: str) -> dict[str, Any]:
|
|
108
120
|
"""
|
|
109
|
-
Search Google with cloro.
|
|
121
|
+
Search Google or Bing with cloro.
|
|
110
122
|
|
|
111
123
|
:param query: Search query.
|
|
112
124
|
:returns: A dictionary with:
|
|
@@ -115,13 +127,13 @@ class CloroWebSearch:
|
|
|
115
127
|
:raises TimeoutError: If the request times out.
|
|
116
128
|
:raises CloroError: If the cloro API returns an error.
|
|
117
129
|
"""
|
|
118
|
-
result = post(
|
|
130
|
+
result = post(_PATHS[self.engine], self._payload(query), self.api_key, self.timeout)
|
|
119
131
|
return self._parse(result)
|
|
120
132
|
|
|
121
133
|
@component.output_types(documents=list[Document], links=list[str])
|
|
122
134
|
async def run_async(self, query: str) -> dict[str, Any]:
|
|
123
135
|
"""
|
|
124
|
-
Search Google with cloro without blocking.
|
|
136
|
+
Search Google or Bing with cloro without blocking.
|
|
125
137
|
|
|
126
138
|
This is the asynchronous version of `run`, with the same parameters and return values.
|
|
127
139
|
|
|
@@ -132,7 +144,7 @@ class CloroWebSearch:
|
|
|
132
144
|
:raises TimeoutError: If the request times out.
|
|
133
145
|
:raises CloroError: If the cloro API returns an error.
|
|
134
146
|
"""
|
|
135
|
-
result = await post_async(
|
|
147
|
+
result = await post_async(_PATHS[self.engine], self._payload(query), self.api_key, self.timeout)
|
|
136
148
|
return self._parse(result)
|
|
137
149
|
|
|
138
150
|
def _payload(self, query: str) -> dict[str, Any]:
|
|
@@ -167,7 +179,11 @@ class CloroWebSearch:
|
|
|
167
179
|
0,
|
|
168
180
|
Document(
|
|
169
181
|
content=overview["text"],
|
|
170
|
-
meta={
|
|
182
|
+
meta={
|
|
183
|
+
"title": f"{_NAMES[self.engine]} AI Overview",
|
|
184
|
+
"source": f"{self.engine}_ai_overview",
|
|
185
|
+
"links": sources,
|
|
186
|
+
},
|
|
171
187
|
),
|
|
172
188
|
)
|
|
173
189
|
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"success": true,
|
|
3
|
+
"result": {
|
|
4
|
+
"organicResults": [
|
|
5
|
+
{
|
|
6
|
+
"position": 1,
|
|
7
|
+
"title": "How to Season a Wok",
|
|
8
|
+
"link": "https://www.wokandskillet.com/how-to-season-a-wok/",
|
|
9
|
+
"displayedLink": "https://www.wokandskillet.com › how-to-season-a-wok",
|
|
10
|
+
"snippet": "Wash the wok, heat it until the water evaporates, then coat it with a thin layer of oil.",
|
|
11
|
+
"page": 1
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"position": 2,
|
|
15
|
+
"title": "How to Season a Wok - The Woks of Life",
|
|
16
|
+
"link": "https://thewoksoflife.com/how-to-season-a-wok/",
|
|
17
|
+
"displayedLink": "https://thewoksoflife.com › how-to-season-a-wok",
|
|
18
|
+
"snippet": "Seasoning a carbon steel wok gives it a natural nonstick surface.",
|
|
19
|
+
"page": 1
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"position": 3,
|
|
23
|
+
"title": "3 Ways to Season a Wok - wikiHow",
|
|
24
|
+
"link": "https://www.wikihow.com/Season-a-Wok",
|
|
25
|
+
"displayedLink": "https://www.wikihow.com › Season-a-Wok",
|
|
26
|
+
"snippet": "Heat the wok over high heat, add oil and swirl it to coat the inside evenly.",
|
|
27
|
+
"page": 1,
|
|
28
|
+
"date": "Apr 5, 2024"
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
"aioverview": {
|
|
32
|
+
"text": "Seasoning a carbon steel or cast iron wok means heating a thin layer of oil until it polymerizes.",
|
|
33
|
+
"sources": [
|
|
34
|
+
{
|
|
35
|
+
"position": 1,
|
|
36
|
+
"label": "Wok & Skillet",
|
|
37
|
+
"url": "https://www.wokandskillet.com/how-to-season-a-wok/",
|
|
38
|
+
"description": null
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"position": 2,
|
|
42
|
+
"label": "Umamicart",
|
|
43
|
+
"url": "https://www.umamicart.com/how-to-season-a-wok/",
|
|
44
|
+
"description": null
|
|
45
|
+
}
|
|
46
|
+
],
|
|
47
|
+
"citationPills": [
|
|
48
|
+
{
|
|
49
|
+
"position": 1,
|
|
50
|
+
"label": "Wok & Skillet",
|
|
51
|
+
"url": "https://www.wokandskillet.com/how-to-season-a-wok/",
|
|
52
|
+
"domain": "wokandskillet.com",
|
|
53
|
+
"citationPillId": 1
|
|
54
|
+
}
|
|
55
|
+
],
|
|
56
|
+
"markdown": "Seasoning a carbon steel or cast iron wok means **heating a thin layer of oil until it polymerizes**.[Wok & Skillet](https://www.wokandskillet.com/how-to-season-a-wok/)"
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"success": true,
|
|
3
|
+
"result": {
|
|
4
|
+
"text": "For most beginners, a neutral road shoe with soft cushioning is the safest start.",
|
|
5
|
+
"sources": [
|
|
6
|
+
{
|
|
7
|
+
"position": 1,
|
|
8
|
+
"label": "The Best Running Shoes for Beginners",
|
|
9
|
+
"url": "https://www.runnersworld.com/gear/a20865766/best-running-shoes-for-beginners/",
|
|
10
|
+
"description": "Our test team picked the best shoes for new runners."
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"position": 2,
|
|
14
|
+
"label": "How to Choose Running Shoes",
|
|
15
|
+
"url": "https://www.rei.com/learn/expert-advice/running-shoes.html"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"citationPills": [
|
|
19
|
+
{
|
|
20
|
+
"position": 1,
|
|
21
|
+
"label": "The Best Running Shoes for Beginners",
|
|
22
|
+
"url": "https://www.runnersworld.com/gear/a20865766/best-running-shoes-for-beginners/",
|
|
23
|
+
"domain": "runnersworld.com",
|
|
24
|
+
"citationPillId": 1
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"position": 2,
|
|
28
|
+
"label": "How to Choose Running Shoes",
|
|
29
|
+
"url": "https://www.rei.com/learn/expert-advice/running-shoes.html",
|
|
30
|
+
"domain": "rei.com",
|
|
31
|
+
"citationPillId": 1
|
|
32
|
+
}
|
|
33
|
+
],
|
|
34
|
+
"markdown": "For most beginners, a **neutral road shoe with soft cushioning** is the safest start.[The Best Running Shoes for Beginners](https://www.runnersworld.com/gear/a20865766/best-running-shoes-for-beginners/)[How to Choose Running Shoes](https://www.rei.com/learn/expert-advice/running-shoes.html)"
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -124,6 +124,27 @@ def test_websearch_top_k_domains_and_params(api):
|
|
|
124
124
|
assert len(result["links"]) == 2
|
|
125
125
|
|
|
126
126
|
|
|
127
|
+
def test_websearch_bing_request_and_ai_overview(api):
|
|
128
|
+
payload = _load("bing.json")
|
|
129
|
+
api.respond(httpx.Response(200, json=payload))
|
|
130
|
+
result = CloroWebSearch(engine="bing", country="GB", include_ai_overview=True).run(query="season a wok")
|
|
131
|
+
request = api.calls[0]
|
|
132
|
+
assert str(request.url) == "https://api.cloro.dev/v1/monitor/bing"
|
|
133
|
+
assert _body(request) == {"query": "season a wok", "country": "GB", "include": {"aioverview": True}}
|
|
134
|
+
overview = result["documents"][0]
|
|
135
|
+
assert overview.content == payload["result"]["aioverview"]["text"]
|
|
136
|
+
assert overview.meta["title"] == "Bing AI Overview"
|
|
137
|
+
assert overview.meta["source"] == "bing_ai_overview"
|
|
138
|
+
assert overview.meta["links"] == [s["url"] for s in payload["result"]["aioverview"]["sources"]]
|
|
139
|
+
assert result["links"] == [r["link"] for r in payload["result"]["organicResults"]]
|
|
140
|
+
assert len(result["documents"]) == 4
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def test_websearch_rejects_other_engines():
|
|
144
|
+
with pytest.raises(ValueError, match="engine"):
|
|
145
|
+
CloroWebSearch(engine="other") # type: ignore[arg-type]
|
|
146
|
+
|
|
147
|
+
|
|
127
148
|
def test_websearch_run_async(api):
|
|
128
149
|
api.respond(httpx.Response(200, json=_load("google_aioverview.json")))
|
|
129
150
|
result = asyncio.run(CloroWebSearch().run_async(query="q"))
|
|
@@ -133,12 +154,15 @@ def test_websearch_run_async(api):
|
|
|
133
154
|
# CloroAIAnswer
|
|
134
155
|
|
|
135
156
|
|
|
136
|
-
@pytest.mark.parametrize(
|
|
137
|
-
|
|
157
|
+
@pytest.mark.parametrize(
|
|
158
|
+
("engine", "path"),
|
|
159
|
+
[("chatgpt", "chatgpt"), ("aimode", "aimode"), ("gemini", "gemini"), ("bing_aimode", "bing/aimode")],
|
|
160
|
+
)
|
|
161
|
+
def test_ai_answer_request(api, engine, path):
|
|
138
162
|
api.respond(httpx.Response(200, json=_load("aimode.json")))
|
|
139
163
|
CloroAIAnswer(engine=engine).run(query="rank tracker")
|
|
140
164
|
request = api.calls[0]
|
|
141
|
-
assert str(request.url) == f"https://api.cloro.dev/v1/monitor/{
|
|
165
|
+
assert str(request.url) == f"https://api.cloro.dev/v1/monitor/{path}"
|
|
142
166
|
assert _body(request) == {"prompt": "rank tracker", "country": "US", "include": {"markdown": True}}
|
|
143
167
|
|
|
144
168
|
|
|
@@ -158,6 +182,16 @@ def test_ai_answer_output(api):
|
|
|
158
182
|
assert result["documents"][0].content == sources[0]["description"]
|
|
159
183
|
|
|
160
184
|
|
|
185
|
+
def test_ai_answer_bing_aimode_output(api):
|
|
186
|
+
payload = _load("bing_aimode.json")
|
|
187
|
+
api.respond(httpx.Response(200, json=payload))
|
|
188
|
+
result = CloroAIAnswer(engine="bing_aimode", search_params={"state": "CA"}).run(query="running shoes")
|
|
189
|
+
assert _body(api.calls[0])["state"] == "CA"
|
|
190
|
+
assert result["replies"] == [payload["result"]["markdown"]]
|
|
191
|
+
assert result["links"] == [s["url"] for s in payload["result"]["sources"]]
|
|
192
|
+
assert result["documents"][1].content == payload["result"]["sources"][1]["label"]
|
|
193
|
+
|
|
194
|
+
|
|
161
195
|
def test_ai_answer_chatgpt_without_markdown_or_description(api):
|
|
162
196
|
payload = _load("chatgpt.json")
|
|
163
197
|
api.respond(httpx.Response(200, json=payload))
|
|
@@ -228,6 +262,7 @@ def test_error_message_has_no_key(api):
|
|
|
228
262
|
"component",
|
|
229
263
|
[
|
|
230
264
|
CloroWebSearch(top_k=3, country="GB", include_ai_overview=True, search_params={"device": "mobile"}),
|
|
265
|
+
CloroWebSearch(engine="bing", country="DE"),
|
|
231
266
|
CloroAIAnswer(engine="gemini", country="FR"),
|
|
232
267
|
],
|
|
233
268
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|