cloro-haystack 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/PKG-INFO +28 -9
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/README.md +26 -7
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/pyproject.toml +2 -2
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/websearch.py +63 -19
- cloro_haystack-0.4.0/tests/fixtures/bing.json +59 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/tests/test_cloro.py +84 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/.gitignore +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/LICENSE +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/__init__.py +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/_api.py +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/ai_answer.py +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/py.typed +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/tests/fixtures/aimode.json +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/tests/fixtures/bing_aimode.json +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/tests/fixtures/chatgpt.json +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/tests/fixtures/error_401.json +0 -0
- {cloro_haystack-0.2.0 → cloro_haystack-0.4.0}/tests/fixtures/google_aioverview.json +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloro-haystack
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Haystack components for cloro: Google search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources.
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Haystack components for cloro: Google and Bing search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources.
|
|
5
5
|
Project-URL: Homepage, https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
|
|
6
6
|
Project-URL: Documentation, https://github.com/cloro-dev/cloro-haystack#readme
|
|
7
7
|
Project-URL: Issues, https://github.com/cloro-dev/cloro-haystack/issues
|
|
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
|
|
29
29
|
| Component | cloro endpoint | Output | Credits per run |
|
|
30
30
|
|---|---|---|--:|
|
|
31
|
-
| `CloroWebSearch` | `POST /v1/monitor/google` | `documents` and `links` from Google organic results. With `include_ai_overview=True`, the
|
|
31
|
+
| `CloroWebSearch` | `POST /v1/monitor/google` or `/bing` | `documents` and `links` from Google or Bing organic results. With `include_ai_overview=True`, the AI Overview is the first Document. | 5, or 7 with the AI Overview |
|
|
32
32
|
| `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode`, `/gemini` or `/bing/aimode` | `replies` (the answer as Markdown), `documents` and `links` (the cited sources) | ChatGPT 7, Google AI Mode 6, Gemini 6, Bing AI Mode 6 |
|
|
33
33
|
|
|
34
34
|
## Installation
|
|
@@ -55,20 +55,39 @@ for doc in result["documents"]:
|
|
|
55
55
|
print(doc.meta.get("title"), doc.meta.get("link"))
|
|
56
56
|
```
|
|
57
57
|
|
|
58
|
-
Each organic result is a `Document`. Its `content` is the
|
|
59
|
-
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"`
|
|
60
|
-
URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
58
|
+
Each organic result is a `Document`. Its `content` is the result snippet and its `meta` has `title`,
|
|
59
|
+
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"` (or
|
|
60
|
+
`"bing_ai_overview"`) and the URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
61
|
+
redirect link are skipped.
|
|
62
|
+
|
|
63
|
+
With `include_ads=True`, the ads on the Google or Bing page are the last Documents. The content of an ad
|
|
64
|
+
starts with `Ad:`. Its `meta["source"]` is `"google_ad"` or `"bing_ad"`, and its `meta` has `title`, `link`,
|
|
65
|
+
`position`, `block_position`, `ad_type`, `price` and `store`, when the ad has them. `links` has no ads.
|
|
61
66
|
|
|
62
67
|
Parameters:
|
|
63
68
|
|
|
64
69
|
- `api_key`: defaults to the `CLORO_API_KEY` environment variable.
|
|
65
|
-
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
|
|
70
|
+
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
|
|
66
71
|
results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
|
|
67
72
|
- `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
|
|
68
73
|
- `allowed_domains`: domains to limit the search to, sent as `site:` filters.
|
|
69
|
-
- `include_ai_overview`: request the
|
|
70
|
-
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages
|
|
74
|
+
- `include_ai_overview`: request the AI Overview. Defaults to `False`.
|
|
75
|
+
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google, and
|
|
76
|
+
`pages` or `first` for Bing.
|
|
71
77
|
- `timeout`: request timeout in seconds. Defaults to 60.
|
|
78
|
+
- `engine`: `"google"` (Google Search) or `"bing"` (Bing Search). Defaults to `"google"`.
|
|
79
|
+
- `include_ads`: return the ads on the page. Defaults to `False`. The ads cost no extra credits and do not
|
|
80
|
+
count towards `top_k`.
|
|
81
|
+
|
|
82
|
+
With `engine="bing"`, the component searches Bing. The country sets the Bing market. Bing Search accepts
|
|
83
|
+
all countries except CN, KR and RU. It has no `device`, `location` or `hl` fields. Set
|
|
84
|
+
`search_params={"pages": 2}` for more results (2 credits for each extra page) or `search_params={"first": 11}`
|
|
85
|
+
to start at a later result. Bing shows its AI summary only on the first page. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
|
|
86
|
+
non-Latin scripts.
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
websearch = CloroWebSearch(engine="bing", country="GB", include_ai_overview=True)
|
|
90
|
+
```
|
|
72
91
|
|
|
73
92
|
### CloroAIAnswer
|
|
74
93
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
| Component | cloro endpoint | Output | Credits per run |
|
|
7
7
|
|---|---|---|--:|
|
|
8
|
-
| `CloroWebSearch` | `POST /v1/monitor/google` | `documents` and `links` from Google organic results. With `include_ai_overview=True`, the
|
|
8
|
+
| `CloroWebSearch` | `POST /v1/monitor/google` or `/bing` | `documents` and `links` from Google or Bing organic results. With `include_ai_overview=True`, the AI Overview is the first Document. | 5, or 7 with the AI Overview |
|
|
9
9
|
| `CloroAIAnswer` | `POST /v1/monitor/chatgpt`, `/aimode`, `/gemini` or `/bing/aimode` | `replies` (the answer as Markdown), `documents` and `links` (the cited sources) | ChatGPT 7, Google AI Mode 6, Gemini 6, Bing AI Mode 6 |
|
|
10
10
|
|
|
11
11
|
## Installation
|
|
@@ -32,20 +32,39 @@ for doc in result["documents"]:
|
|
|
32
32
|
print(doc.meta.get("title"), doc.meta.get("link"))
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
-
Each organic result is a `Document`. Its `content` is the
|
|
36
|
-
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"`
|
|
37
|
-
URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
35
|
+
Each organic result is a `Document`. Its `content` is the result snippet and its `meta` has `title`,
|
|
36
|
+
`link` and `position`. The AI Overview Document has `meta["source"] == "google_ai_overview"` (or
|
|
37
|
+
`"bing_ai_overview"`) and the URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
38
|
+
redirect link are skipped.
|
|
39
|
+
|
|
40
|
+
With `include_ads=True`, the ads on the Google or Bing page are the last Documents. The content of an ad
|
|
41
|
+
starts with `Ad:`. Its `meta["source"]` is `"google_ad"` or `"bing_ad"`, and its `meta` has `title`, `link`,
|
|
42
|
+
`position`, `block_position`, `ad_type`, `price` and `store`, when the ad has them. `links` has no ads.
|
|
38
43
|
|
|
39
44
|
Parameters:
|
|
40
45
|
|
|
41
46
|
- `api_key`: defaults to the `CLORO_API_KEY` environment variable.
|
|
42
|
-
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
|
|
47
|
+
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
|
|
43
48
|
results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
|
|
44
49
|
- `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
|
|
45
50
|
- `allowed_domains`: domains to limit the search to, sent as `site:` filters.
|
|
46
|
-
- `include_ai_overview`: request the
|
|
47
|
-
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages
|
|
51
|
+
- `include_ai_overview`: request the AI Overview. Defaults to `False`.
|
|
52
|
+
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google, and
|
|
53
|
+
`pages` or `first` for Bing.
|
|
48
54
|
- `timeout`: request timeout in seconds. Defaults to 60.
|
|
55
|
+
- `engine`: `"google"` (Google Search) or `"bing"` (Bing Search). Defaults to `"google"`.
|
|
56
|
+
- `include_ads`: return the ads on the page. Defaults to `False`. The ads cost no extra credits and do not
|
|
57
|
+
count towards `top_k`.
|
|
58
|
+
|
|
59
|
+
With `engine="bing"`, the component searches Bing. The country sets the Bing market. Bing Search accepts
|
|
60
|
+
all countries except CN, KR and RU. It has no `device`, `location` or `hl` fields. Set
|
|
61
|
+
`search_params={"pages": 2}` for more results (2 credits for each extra page) or `search_params={"first": 11}`
|
|
62
|
+
to start at a later result. Bing shows its AI summary only on the first page. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
|
|
63
|
+
non-Latin scripts.
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
websearch = CloroWebSearch(engine="bing", country="GB", include_ai_overview=True)
|
|
67
|
+
```
|
|
49
68
|
|
|
50
69
|
### CloroAIAnswer
|
|
51
70
|
|
|
@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cloro-haystack"
|
|
7
|
-
version = "0.
|
|
8
|
-
description = "Haystack components for cloro: Google search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources."
|
|
7
|
+
version = "0.4.0"
|
|
8
|
+
description = "Haystack components for cloro: Google and Bing search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
11
11
|
license = "MIT"
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: MIT
|
|
4
4
|
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any, Literal
|
|
6
6
|
|
|
7
7
|
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
|
8
8
|
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
@@ -11,19 +11,24 @@ from haystack_integrations.components.websearch.cloro._api import is_direct_link
|
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
Engine = Literal["google", "bing"]
|
|
15
|
+
|
|
16
|
+
# The API path and the display name for each engine.
|
|
17
|
+
_PATHS: dict[str, str] = {"google": "/v1/monitor/google", "bing": "/v1/monitor/bing"}
|
|
18
|
+
_NAMES: dict[str, str] = {"google": "Google", "bing": "Bing"}
|
|
15
19
|
|
|
16
20
|
|
|
17
21
|
@component
|
|
18
22
|
class CloroWebSearch:
|
|
19
23
|
"""
|
|
20
|
-
Searches Google with the cloro API and returns the results as Documents.
|
|
24
|
+
Searches Google or Bing with the cloro API and returns the results as Documents.
|
|
21
25
|
|
|
22
26
|
cloro: https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
|
|
23
27
|
|
|
24
|
-
Each organic result becomes a Document. Its content is the
|
|
25
|
-
`title`, `link` and `position`. With `include_ai_overview=True`, the
|
|
26
|
-
(when
|
|
28
|
+
Each organic result becomes a Document. Its content is the result snippet. Its meta has
|
|
29
|
+
`title`, `link` and `position`. With `include_ai_overview=True`, the AI Overview of the
|
|
30
|
+
engine (when the engine shows one) is the first Document. With `include_ads=True`, the ads
|
|
31
|
+
on the page are the last Documents.
|
|
27
32
|
|
|
28
33
|
### Usage example
|
|
29
34
|
|
|
@@ -46,6 +51,8 @@ class CloroWebSearch:
|
|
|
46
51
|
include_ai_overview: bool = False,
|
|
47
52
|
search_params: dict[str, Any] | None = None,
|
|
48
53
|
timeout: float = 60.0,
|
|
54
|
+
engine: Engine = "google",
|
|
55
|
+
include_ads: bool = False,
|
|
49
56
|
) -> None:
|
|
50
57
|
"""
|
|
51
58
|
Initialize the CloroWebSearch component.
|
|
@@ -53,27 +60,37 @@ class CloroWebSearch:
|
|
|
53
60
|
:param api_key:
|
|
54
61
|
cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
|
|
55
62
|
:param top_k:
|
|
56
|
-
Maximum number of organic results to return. One request returns one page of
|
|
57
|
-
|
|
63
|
+
Maximum number of organic results to return. One request returns one page of results
|
|
64
|
+
(about 10). Set `search_params={"pages": 2}` or more to get more results.
|
|
58
65
|
:param country:
|
|
59
|
-
ISO 3166-1 alpha-2 country code for the
|
|
66
|
+
ISO 3166-1 alpha-2 country code for the engine location.
|
|
60
67
|
:param allowed_domains:
|
|
61
68
|
Domains to limit the search to. They are added to the query as `site:` filters.
|
|
62
69
|
:param include_ai_overview:
|
|
63
|
-
If True, request the
|
|
70
|
+
If True, request the AI Overview and return it as the first Document.
|
|
64
71
|
:param search_params:
|
|
65
|
-
Other fields for the
|
|
66
|
-
`
|
|
72
|
+
Other fields for the engine request body, for example `device`, `location`, `hl` or
|
|
73
|
+
`pages` for Google, and `pages` or `first` for Bing. See the cloro API reference.
|
|
67
74
|
:param timeout:
|
|
68
75
|
Request timeout in seconds.
|
|
76
|
+
:param engine:
|
|
77
|
+
The search engine: `"google"` (Google Search) or `"bing"` (Bing Search).
|
|
78
|
+
:param include_ads:
|
|
79
|
+
If True, return the ads on the page after the organic results. Each ad is a Document with
|
|
80
|
+
`meta["source"]` set to `"google_ad"` or `"bing_ad"`. The ads do not count towards `top_k`.
|
|
69
81
|
"""
|
|
82
|
+
if engine not in _PATHS:
|
|
83
|
+
msg = f"engine must be 'google' or 'bing', not {engine!r}"
|
|
84
|
+
raise ValueError(msg)
|
|
70
85
|
self.api_key = api_key
|
|
71
86
|
self.top_k = top_k
|
|
87
|
+
self.engine = engine
|
|
72
88
|
self.country = country
|
|
73
89
|
self.allowed_domains = allowed_domains
|
|
74
90
|
self.include_ai_overview = include_ai_overview
|
|
75
91
|
self.search_params = search_params
|
|
76
92
|
self.timeout = timeout
|
|
93
|
+
self.include_ads = include_ads
|
|
77
94
|
|
|
78
95
|
def to_dict(self) -> dict[str, Any]:
|
|
79
96
|
"""
|
|
@@ -90,6 +107,8 @@ class CloroWebSearch:
|
|
|
90
107
|
include_ai_overview=self.include_ai_overview,
|
|
91
108
|
search_params=self.search_params,
|
|
92
109
|
timeout=self.timeout,
|
|
110
|
+
engine=self.engine,
|
|
111
|
+
include_ads=self.include_ads,
|
|
93
112
|
)
|
|
94
113
|
|
|
95
114
|
@classmethod
|
|
@@ -106,33 +125,35 @@ class CloroWebSearch:
|
|
|
106
125
|
@component.output_types(documents=list[Document], links=list[str])
|
|
107
126
|
def run(self, query: str) -> dict[str, Any]:
|
|
108
127
|
"""
|
|
109
|
-
Search Google with cloro.
|
|
128
|
+
Search Google or Bing with cloro.
|
|
110
129
|
|
|
111
130
|
:param query: Search query.
|
|
112
131
|
:returns: A dictionary with:
|
|
113
|
-
- `documents`: The AI Overview (if requested and present), then the organic results
|
|
132
|
+
- `documents`: The AI Overview (if requested and present), then the organic results, then
|
|
133
|
+
the ads (if requested and present).
|
|
114
134
|
- `links`: The links of the organic results.
|
|
115
135
|
:raises TimeoutError: If the request times out.
|
|
116
136
|
:raises CloroError: If the cloro API returns an error.
|
|
117
137
|
"""
|
|
118
|
-
result = post(
|
|
138
|
+
result = post(_PATHS[self.engine], self._payload(query), self.api_key, self.timeout)
|
|
119
139
|
return self._parse(result)
|
|
120
140
|
|
|
121
141
|
@component.output_types(documents=list[Document], links=list[str])
|
|
122
142
|
async def run_async(self, query: str) -> dict[str, Any]:
|
|
123
143
|
"""
|
|
124
|
-
Search Google with cloro without blocking.
|
|
144
|
+
Search Google or Bing with cloro without blocking.
|
|
125
145
|
|
|
126
146
|
This is the asynchronous version of `run`, with the same parameters and return values.
|
|
127
147
|
|
|
128
148
|
:param query: Search query.
|
|
129
149
|
:returns: A dictionary with:
|
|
130
|
-
- `documents`: The AI Overview (if requested and present), then the organic results
|
|
150
|
+
- `documents`: The AI Overview (if requested and present), then the organic results, then
|
|
151
|
+
the ads (if requested and present).
|
|
131
152
|
- `links`: The links of the organic results.
|
|
132
153
|
:raises TimeoutError: If the request times out.
|
|
133
154
|
:raises CloroError: If the cloro API returns an error.
|
|
134
155
|
"""
|
|
135
|
-
result = await post_async(
|
|
156
|
+
result = await post_async(_PATHS[self.engine], self._payload(query), self.api_key, self.timeout)
|
|
136
157
|
return self._parse(result)
|
|
137
158
|
|
|
138
159
|
def _payload(self, query: str) -> dict[str, Any]:
|
|
@@ -159,6 +180,8 @@ class CloroWebSearch:
|
|
|
159
180
|
for item in organic
|
|
160
181
|
]
|
|
161
182
|
links = [item["link"] for item in organic]
|
|
183
|
+
if self.include_ads:
|
|
184
|
+
documents += [self._ad_document(ad) for ad in result.get("ads") or [] if isinstance(ad, dict)]
|
|
162
185
|
|
|
163
186
|
overview = result.get("aioverview")
|
|
164
187
|
if self.include_ai_overview and isinstance(overview, dict) and overview.get("text"):
|
|
@@ -167,9 +190,30 @@ class CloroWebSearch:
|
|
|
167
190
|
0,
|
|
168
191
|
Document(
|
|
169
192
|
content=overview["text"],
|
|
170
|
-
meta={
|
|
193
|
+
meta={
|
|
194
|
+
"title": f"{_NAMES[self.engine]} AI Overview",
|
|
195
|
+
"source": f"{self.engine}_ai_overview",
|
|
196
|
+
"links": sources,
|
|
197
|
+
},
|
|
171
198
|
),
|
|
172
199
|
)
|
|
173
200
|
|
|
174
201
|
logger.debug("cloro returned {count} documents", count=len(documents))
|
|
175
202
|
return {"documents": documents, "links": links}
|
|
203
|
+
|
|
204
|
+
def _ad_document(self, ad: dict[str, Any]) -> Document:
|
|
205
|
+
# Every field of an ad can be absent.
|
|
206
|
+
price = ad.get("price")
|
|
207
|
+
meta = {
|
|
208
|
+
"title": ad.get("title"),
|
|
209
|
+
"link": ad.get("url") if is_direct_link(ad.get("url")) else None,
|
|
210
|
+
"position": ad.get("position"),
|
|
211
|
+
"block_position": ad.get("blockPosition"),
|
|
212
|
+
"ad_type": ad.get("type"),
|
|
213
|
+
"price": price.get("raw") if isinstance(price, dict) else None,
|
|
214
|
+
"store": ad.get("store"),
|
|
215
|
+
}
|
|
216
|
+
return Document(
|
|
217
|
+
content=f"Ad: {ad.get('description') or ad.get('title') or ''}".strip(),
|
|
218
|
+
meta={"source": f"{self.engine}_ad", **{key: value for key, value in meta.items() if value}},
|
|
219
|
+
)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"success": true,
|
|
3
|
+
"result": {
|
|
4
|
+
"organicResults": [
|
|
5
|
+
{
|
|
6
|
+
"position": 1,
|
|
7
|
+
"title": "How to Season a Wok",
|
|
8
|
+
"link": "https://www.wokandskillet.com/how-to-season-a-wok/",
|
|
9
|
+
"displayedLink": "https://www.wokandskillet.com › how-to-season-a-wok",
|
|
10
|
+
"snippet": "Wash the wok, heat it until the water evaporates, then coat it with a thin layer of oil.",
|
|
11
|
+
"page": 1
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"position": 2,
|
|
15
|
+
"title": "How to Season a Wok - The Woks of Life",
|
|
16
|
+
"link": "https://thewoksoflife.com/how-to-season-a-wok/",
|
|
17
|
+
"displayedLink": "https://thewoksoflife.com › how-to-season-a-wok",
|
|
18
|
+
"snippet": "Seasoning a carbon steel wok gives it a natural nonstick surface.",
|
|
19
|
+
"page": 1
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"position": 3,
|
|
23
|
+
"title": "3 Ways to Season a Wok - wikiHow",
|
|
24
|
+
"link": "https://www.wikihow.com/Season-a-Wok",
|
|
25
|
+
"displayedLink": "https://www.wikihow.com › Season-a-Wok",
|
|
26
|
+
"snippet": "Heat the wok over high heat, add oil and swirl it to coat the inside evenly.",
|
|
27
|
+
"page": 1,
|
|
28
|
+
"date": "Apr 5, 2024"
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
"aioverview": {
|
|
32
|
+
"text": "Seasoning a carbon steel or cast iron wok means heating a thin layer of oil until it polymerizes.",
|
|
33
|
+
"sources": [
|
|
34
|
+
{
|
|
35
|
+
"position": 1,
|
|
36
|
+
"label": "Wok & Skillet",
|
|
37
|
+
"url": "https://www.wokandskillet.com/how-to-season-a-wok/",
|
|
38
|
+
"description": null
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"position": 2,
|
|
42
|
+
"label": "Umamicart",
|
|
43
|
+
"url": "https://www.umamicart.com/how-to-season-a-wok/",
|
|
44
|
+
"description": null
|
|
45
|
+
}
|
|
46
|
+
],
|
|
47
|
+
"citationPills": [
|
|
48
|
+
{
|
|
49
|
+
"position": 1,
|
|
50
|
+
"label": "Wok & Skillet",
|
|
51
|
+
"url": "https://www.wokandskillet.com/how-to-season-a-wok/",
|
|
52
|
+
"domain": "wokandskillet.com",
|
|
53
|
+
"citationPillId": 1
|
|
54
|
+
}
|
|
55
|
+
],
|
|
56
|
+
"markdown": "Seasoning a carbon steel or cast iron wok means **heating a thin layer of oil until it polymerizes**.[Wok & Skillet](https://www.wokandskillet.com/how-to-season-a-wok/)"
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -124,6 +124,89 @@ def test_websearch_top_k_domains_and_params(api):
|
|
|
124
124
|
assert len(result["links"]) == 2
|
|
125
125
|
|
|
126
126
|
|
|
127
|
+
def test_websearch_bing_request_and_ai_overview(api):
|
|
128
|
+
payload = _load("bing.json")
|
|
129
|
+
api.respond(httpx.Response(200, json=payload))
|
|
130
|
+
result = CloroWebSearch(engine="bing", country="GB", include_ai_overview=True).run(query="season a wok")
|
|
131
|
+
request = api.calls[0]
|
|
132
|
+
assert str(request.url) == "https://api.cloro.dev/v1/monitor/bing"
|
|
133
|
+
assert _body(request) == {"query": "season a wok", "country": "GB", "include": {"aioverview": True}}
|
|
134
|
+
overview = result["documents"][0]
|
|
135
|
+
assert overview.content == payload["result"]["aioverview"]["text"]
|
|
136
|
+
assert overview.meta["title"] == "Bing AI Overview"
|
|
137
|
+
assert overview.meta["source"] == "bing_ai_overview"
|
|
138
|
+
assert overview.meta["links"] == [s["url"] for s in payload["result"]["aioverview"]["sources"]]
|
|
139
|
+
assert result["links"] == [r["link"] for r in payload["result"]["organicResults"]]
|
|
140
|
+
assert len(result["documents"]) == 4
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
ADS = [
|
|
144
|
+
{
|
|
145
|
+
"position": 1,
|
|
146
|
+
"blockPosition": "top",
|
|
147
|
+
"type": "RESULT",
|
|
148
|
+
"title": "Wok Sale",
|
|
149
|
+
"url": "https://shop.example/woks",
|
|
150
|
+
"page": 1,
|
|
151
|
+
"domain": "shop.example",
|
|
152
|
+
"description": "Carbon steel woks.",
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"position": 1,
|
|
156
|
+
"blockPosition": "rhs",
|
|
157
|
+
"type": "SHOPPING_CARD",
|
|
158
|
+
"title": "14 in Wok",
|
|
159
|
+
"url": "https://shop.example/wok-14",
|
|
160
|
+
"page": 1,
|
|
161
|
+
"price": {"value": 29, "currency": "$", "raw": "$29.00"},
|
|
162
|
+
"store": "Shop",
|
|
163
|
+
},
|
|
164
|
+
{},
|
|
165
|
+
]
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@pytest.mark.parametrize(("engine", "fixture"), [("google", "google_aioverview.json"), ("bing", "bing.json")])
|
|
169
|
+
def test_websearch_ads(api, engine, fixture):
|
|
170
|
+
payload = _load(fixture)
|
|
171
|
+
payload["result"]["ads"] = ADS
|
|
172
|
+
api.respond(httpx.Response(200, json=payload))
|
|
173
|
+
result = CloroWebSearch(engine=engine, top_k=1, include_ads=True).run(query="wok")
|
|
174
|
+
assert "include" not in _body(api.calls[0])
|
|
175
|
+
assert len(result["links"]) == 1
|
|
176
|
+
text_ad, card, empty = result["documents"][1:]
|
|
177
|
+
assert text_ad.content == "Ad: Carbon steel woks."
|
|
178
|
+
assert text_ad.meta == {
|
|
179
|
+
"source": f"{engine}_ad",
|
|
180
|
+
"title": "Wok Sale",
|
|
181
|
+
"link": "https://shop.example/woks",
|
|
182
|
+
"position": 1,
|
|
183
|
+
"block_position": "top",
|
|
184
|
+
"ad_type": "RESULT",
|
|
185
|
+
}
|
|
186
|
+
assert card.content == "Ad: 14 in Wok"
|
|
187
|
+
assert card.meta["price"] == "$29.00"
|
|
188
|
+
assert card.meta["store"] == "Shop"
|
|
189
|
+
assert empty.meta == {"source": f"{engine}_ad"}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def test_websearch_ads_off_by_default(api):
|
|
193
|
+
payload = _load("bing.json")
|
|
194
|
+
payload["result"]["ads"] = ADS
|
|
195
|
+
api.respond(httpx.Response(200, json=payload))
|
|
196
|
+
result = CloroWebSearch(engine="bing").run(query="wok")
|
|
197
|
+
assert [d.meta["link"] for d in result["documents"]] == result["links"]
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_websearch_include_ads_without_ads(api):
|
|
201
|
+
api.respond(httpx.Response(200, json=_load("bing.json")))
|
|
202
|
+
assert len(CloroWebSearch(engine="bing", include_ads=True).run(query="wok")["documents"]) == 3
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def test_websearch_rejects_other_engines():
|
|
206
|
+
with pytest.raises(ValueError, match="engine"):
|
|
207
|
+
CloroWebSearch(engine="other") # type: ignore[arg-type]
|
|
208
|
+
|
|
209
|
+
|
|
127
210
|
def test_websearch_run_async(api):
|
|
128
211
|
api.respond(httpx.Response(200, json=_load("google_aioverview.json")))
|
|
129
212
|
result = asyncio.run(CloroWebSearch().run_async(query="q"))
|
|
@@ -241,6 +324,7 @@ def test_error_message_has_no_key(api):
|
|
|
241
324
|
"component",
|
|
242
325
|
[
|
|
243
326
|
CloroWebSearch(top_k=3, country="GB", include_ai_overview=True, search_params={"device": "mobile"}),
|
|
327
|
+
CloroWebSearch(engine="bing", country="DE"),
|
|
244
328
|
CloroAIAnswer(engine="gemini", country="FR"),
|
|
245
329
|
],
|
|
246
330
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|