cloro-haystack 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/PKG-INFO +13 -5
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/README.md +12 -4
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/pyproject.toml +1 -1
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/websearch.py +33 -5
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/test_cloro.py +62 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/.gitignore +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/LICENSE +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/__init__.py +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/_api.py +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/ai_answer.py +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/py.typed +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/aimode.json +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/bing.json +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/bing_aimode.json +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/chatgpt.json +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/error_401.json +0 -0
- {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/google_aioverview.json +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloro-haystack
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Haystack components for cloro: Google and Bing search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources.
|
|
5
5
|
Project-URL: Homepage, https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
|
|
6
6
|
Project-URL: Documentation, https://github.com/cloro-dev/cloro-haystack#readme
|
|
@@ -60,21 +60,29 @@ Each organic result is a `Document`. Its `content` is the result snippet and its
|
|
|
60
60
|
`"bing_ai_overview"`) and the URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
61
61
|
redirect link are skipped.
|
|
62
62
|
|
|
63
|
+
With `include_ads=True`, the ads on the Google or Bing page are the last Documents. The content of an ad
|
|
64
|
+
starts with `Ad:`. Its `meta["source"]` is `"google_ad"` or `"bing_ad"`, and its `meta` has `title`, `link`,
|
|
65
|
+
`position`, `block_position`, `ad_type`, `price` and `store`, when the ad has them. `links` has no ads.
|
|
66
|
+
|
|
63
67
|
Parameters:
|
|
64
68
|
|
|
65
69
|
- `api_key`: defaults to the `CLORO_API_KEY` environment variable.
|
|
66
|
-
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
|
|
70
|
+
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
|
|
67
71
|
results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
|
|
68
72
|
- `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
|
|
69
73
|
- `allowed_domains`: domains to limit the search to, sent as `site:` filters.
|
|
70
74
|
- `include_ai_overview`: request the AI Overview. Defaults to `False`.
|
|
71
|
-
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google
|
|
75
|
+
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google, and
|
|
76
|
+
`pages` or `first` for Bing.
|
|
72
77
|
- `timeout`: request timeout in seconds. Defaults to 60.
|
|
73
78
|
- `engine`: `"google"` (Google Search) or `"bing"` (Bing Search). Defaults to `"google"`.
|
|
79
|
+
- `include_ads`: return the ads on the page. Defaults to `False`. The ads cost no extra credits and do not
|
|
80
|
+
count towards `top_k`.
|
|
74
81
|
|
|
75
82
|
With `engine="bing"`, the component searches Bing. The country sets the Bing market. Bing Search accepts
|
|
76
|
-
all countries except CN, KR and RU. It has no `device`, `location
|
|
77
|
-
|
|
83
|
+
all countries except CN, KR and RU. It has no `device`, `location` or `hl` fields. Set
|
|
84
|
+
`search_params={"pages": 2}` for more results (2 credits for each extra page) or `search_params={"first": 11}`
|
|
85
|
+
to start at a later result. Bing shows its AI summary only on the first page. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
|
|
78
86
|
non-Latin scripts.
|
|
79
87
|
|
|
80
88
|
```python
|
|
@@ -37,21 +37,29 @@ Each organic result is a `Document`. Its `content` is the result snippet and its
|
|
|
37
37
|
`"bing_ai_overview"`) and the URLs it cites in `meta["links"]`. Results with an empty link or a Google
|
|
38
38
|
redirect link are skipped.
|
|
39
39
|
|
|
40
|
+
With `include_ads=True`, the ads on the Google or Bing page are the last Documents. The content of an ad
|
|
41
|
+
starts with `Ad:`. Its `meta["source"]` is `"google_ad"` or `"bing_ad"`, and its `meta` has `title`, `link`,
|
|
42
|
+
`position`, `block_position`, `ad_type`, `price` and `store`, when the ad has them. `links` has no ads.
|
|
43
|
+
|
|
40
44
|
Parameters:
|
|
41
45
|
|
|
42
46
|
- `api_key`: defaults to the `CLORO_API_KEY` environment variable.
|
|
43
|
-
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
|
|
47
|
+
- `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
|
|
44
48
|
results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
|
|
45
49
|
- `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
|
|
46
50
|
- `allowed_domains`: domains to limit the search to, sent as `site:` filters.
|
|
47
51
|
- `include_ai_overview`: request the AI Overview. Defaults to `False`.
|
|
48
|
-
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google
|
|
52
|
+
- `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google, and
|
|
53
|
+
`pages` or `first` for Bing.
|
|
49
54
|
- `timeout`: request timeout in seconds. Defaults to 60.
|
|
50
55
|
- `engine`: `"google"` (Google Search) or `"bing"` (Bing Search). Defaults to `"google"`.
|
|
56
|
+
- `include_ads`: return the ads on the page. Defaults to `False`. The ads cost no extra credits and do not
|
|
57
|
+
count towards `top_k`.
|
|
51
58
|
|
|
52
59
|
With `engine="bing"`, the component searches Bing. The country sets the Bing market. Bing Search accepts
|
|
53
|
-
all countries except CN, KR and RU. It has no `device`, `location
|
|
54
|
-
|
|
60
|
+
all countries except CN, KR and RU. It has no `device`, `location` or `hl` fields. Set
|
|
61
|
+
`search_params={"pages": 2}` for more results (2 credits for each extra page) or `search_params={"first": 11}`
|
|
62
|
+
to start at a later result. Bing shows its AI summary only on the first page. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
|
|
55
63
|
non-Latin scripts.
|
|
56
64
|
|
|
57
65
|
```python
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cloro-haystack"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Haystack components for cloro: Google and Bing search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -27,7 +27,8 @@ class CloroWebSearch:
|
|
|
27
27
|
|
|
28
28
|
Each organic result becomes a Document. Its content is the result snippet. Its meta has
|
|
29
29
|
`title`, `link` and `position`. With `include_ai_overview=True`, the AI Overview of the
|
|
30
|
-
engine (when the engine shows one) is the first Document.
|
|
30
|
+
engine (when the engine shows one) is the first Document. With `include_ads=True`, the ads
|
|
31
|
+
on the page are the last Documents.
|
|
31
32
|
|
|
32
33
|
### Usage example
|
|
33
34
|
|
|
@@ -51,6 +52,7 @@ class CloroWebSearch:
|
|
|
51
52
|
search_params: dict[str, Any] | None = None,
|
|
52
53
|
timeout: float = 60.0,
|
|
53
54
|
engine: Engine = "google",
|
|
55
|
+
include_ads: bool = False,
|
|
54
56
|
) -> None:
|
|
55
57
|
"""
|
|
56
58
|
Initialize the CloroWebSearch component.
|
|
@@ -59,7 +61,7 @@ class CloroWebSearch:
|
|
|
59
61
|
cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
|
|
60
62
|
:param top_k:
|
|
61
63
|
Maximum number of organic results to return. One request returns one page of results
|
|
62
|
-
(about 10).
|
|
64
|
+
(about 10). Set `search_params={"pages": 2}` or more to get more results.
|
|
63
65
|
:param country:
|
|
64
66
|
ISO 3166-1 alpha-2 country code for the engine location.
|
|
65
67
|
:param allowed_domains:
|
|
@@ -68,11 +70,14 @@ class CloroWebSearch:
|
|
|
68
70
|
If True, request the AI Overview and return it as the first Document.
|
|
69
71
|
:param search_params:
|
|
70
72
|
Other fields for the engine request body, for example `device`, `location`, `hl` or
|
|
71
|
-
`pages` for Google. See the cloro API reference.
|
|
73
|
+
`pages` for Google, and `pages` or `first` for Bing. See the cloro API reference.
|
|
72
74
|
:param timeout:
|
|
73
75
|
Request timeout in seconds.
|
|
74
76
|
:param engine:
|
|
75
77
|
The search engine: `"google"` (Google Search) or `"bing"` (Bing Search).
|
|
78
|
+
:param include_ads:
|
|
79
|
+
If True, return the ads on the page after the organic results. Each ad is a Document with
|
|
80
|
+
`meta["source"]` set to `"google_ad"` or `"bing_ad"`. The ads do not count towards `top_k`.
|
|
76
81
|
"""
|
|
77
82
|
if engine not in _PATHS:
|
|
78
83
|
msg = f"engine must be 'google' or 'bing', not {engine!r}"
|
|
@@ -85,6 +90,7 @@ class CloroWebSearch:
|
|
|
85
90
|
self.include_ai_overview = include_ai_overview
|
|
86
91
|
self.search_params = search_params
|
|
87
92
|
self.timeout = timeout
|
|
93
|
+
self.include_ads = include_ads
|
|
88
94
|
|
|
89
95
|
def to_dict(self) -> dict[str, Any]:
|
|
90
96
|
"""
|
|
@@ -102,6 +108,7 @@ class CloroWebSearch:
|
|
|
102
108
|
search_params=self.search_params,
|
|
103
109
|
timeout=self.timeout,
|
|
104
110
|
engine=self.engine,
|
|
111
|
+
include_ads=self.include_ads,
|
|
105
112
|
)
|
|
106
113
|
|
|
107
114
|
@classmethod
|
|
@@ -122,7 +129,8 @@ class CloroWebSearch:
|
|
|
122
129
|
|
|
123
130
|
:param query: Search query.
|
|
124
131
|
:returns: A dictionary with:
|
|
125
|
-
- `documents`: The AI Overview (if requested and present), then the organic results
|
|
132
|
+
- `documents`: The AI Overview (if requested and present), then the organic results, then
|
|
133
|
+
the ads (if requested and present).
|
|
126
134
|
- `links`: The links of the organic results.
|
|
127
135
|
:raises TimeoutError: If the request times out.
|
|
128
136
|
:raises CloroError: If the cloro API returns an error.
|
|
@@ -139,7 +147,8 @@ class CloroWebSearch:
|
|
|
139
147
|
|
|
140
148
|
:param query: Search query.
|
|
141
149
|
:returns: A dictionary with:
|
|
142
|
-
- `documents`: The AI Overview (if requested and present), then the organic results
|
|
150
|
+
- `documents`: The AI Overview (if requested and present), then the organic results, then
|
|
151
|
+
the ads (if requested and present).
|
|
143
152
|
- `links`: The links of the organic results.
|
|
144
153
|
:raises TimeoutError: If the request times out.
|
|
145
154
|
:raises CloroError: If the cloro API returns an error.
|
|
@@ -171,6 +180,8 @@ class CloroWebSearch:
|
|
|
171
180
|
for item in organic
|
|
172
181
|
]
|
|
173
182
|
links = [item["link"] for item in organic]
|
|
183
|
+
if self.include_ads:
|
|
184
|
+
documents += [self._ad_document(ad) for ad in result.get("ads") or [] if isinstance(ad, dict)]
|
|
174
185
|
|
|
175
186
|
overview = result.get("aioverview")
|
|
176
187
|
if self.include_ai_overview and isinstance(overview, dict) and overview.get("text"):
|
|
@@ -189,3 +200,20 @@ class CloroWebSearch:
|
|
|
189
200
|
|
|
190
201
|
logger.debug("cloro returned {count} documents", count=len(documents))
|
|
191
202
|
return {"documents": documents, "links": links}
|
|
203
|
+
|
|
204
|
+
def _ad_document(self, ad: dict[str, Any]) -> Document:
|
|
205
|
+
# Every field of an ad can be absent.
|
|
206
|
+
price = ad.get("price")
|
|
207
|
+
meta = {
|
|
208
|
+
"title": ad.get("title"),
|
|
209
|
+
"link": ad.get("url") if is_direct_link(ad.get("url")) else None,
|
|
210
|
+
"position": ad.get("position"),
|
|
211
|
+
"block_position": ad.get("blockPosition"),
|
|
212
|
+
"ad_type": ad.get("type"),
|
|
213
|
+
"price": price.get("raw") if isinstance(price, dict) else None,
|
|
214
|
+
"store": ad.get("store"),
|
|
215
|
+
}
|
|
216
|
+
return Document(
|
|
217
|
+
content=f"Ad: {ad.get('description') or ad.get('title') or ''}".strip(),
|
|
218
|
+
meta={"source": f"{self.engine}_ad", **{key: value for key, value in meta.items() if value}},
|
|
219
|
+
)
|
|
@@ -140,6 +140,68 @@ def test_websearch_bing_request_and_ai_overview(api):
|
|
|
140
140
|
assert len(result["documents"]) == 4
|
|
141
141
|
|
|
142
142
|
|
|
143
|
+
ADS = [
|
|
144
|
+
{
|
|
145
|
+
"position": 1,
|
|
146
|
+
"blockPosition": "top",
|
|
147
|
+
"type": "RESULT",
|
|
148
|
+
"title": "Wok Sale",
|
|
149
|
+
"url": "https://shop.example/woks",
|
|
150
|
+
"page": 1,
|
|
151
|
+
"domain": "shop.example",
|
|
152
|
+
"description": "Carbon steel woks.",
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"position": 1,
|
|
156
|
+
"blockPosition": "rhs",
|
|
157
|
+
"type": "SHOPPING_CARD",
|
|
158
|
+
"title": "14 in Wok",
|
|
159
|
+
"url": "https://shop.example/wok-14",
|
|
160
|
+
"page": 1,
|
|
161
|
+
"price": {"value": 29, "currency": "$", "raw": "$29.00"},
|
|
162
|
+
"store": "Shop",
|
|
163
|
+
},
|
|
164
|
+
{},
|
|
165
|
+
]
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@pytest.mark.parametrize(("engine", "fixture"), [("google", "google_aioverview.json"), ("bing", "bing.json")])
|
|
169
|
+
def test_websearch_ads(api, engine, fixture):
|
|
170
|
+
payload = _load(fixture)
|
|
171
|
+
payload["result"]["ads"] = ADS
|
|
172
|
+
api.respond(httpx.Response(200, json=payload))
|
|
173
|
+
result = CloroWebSearch(engine=engine, top_k=1, include_ads=True).run(query="wok")
|
|
174
|
+
assert "include" not in _body(api.calls[0])
|
|
175
|
+
assert len(result["links"]) == 1
|
|
176
|
+
text_ad, card, empty = result["documents"][1:]
|
|
177
|
+
assert text_ad.content == "Ad: Carbon steel woks."
|
|
178
|
+
assert text_ad.meta == {
|
|
179
|
+
"source": f"{engine}_ad",
|
|
180
|
+
"title": "Wok Sale",
|
|
181
|
+
"link": "https://shop.example/woks",
|
|
182
|
+
"position": 1,
|
|
183
|
+
"block_position": "top",
|
|
184
|
+
"ad_type": "RESULT",
|
|
185
|
+
}
|
|
186
|
+
assert card.content == "Ad: 14 in Wok"
|
|
187
|
+
assert card.meta["price"] == "$29.00"
|
|
188
|
+
assert card.meta["store"] == "Shop"
|
|
189
|
+
assert empty.meta == {"source": f"{engine}_ad"}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def test_websearch_ads_off_by_default(api):
|
|
193
|
+
payload = _load("bing.json")
|
|
194
|
+
payload["result"]["ads"] = ADS
|
|
195
|
+
api.respond(httpx.Response(200, json=payload))
|
|
196
|
+
result = CloroWebSearch(engine="bing").run(query="wok")
|
|
197
|
+
assert [d.meta["link"] for d in result["documents"]] == result["links"]
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_websearch_include_ads_without_ads(api):
|
|
201
|
+
api.respond(httpx.Response(200, json=_load("bing.json")))
|
|
202
|
+
assert len(CloroWebSearch(engine="bing", include_ads=True).run(query="wok")["documents"]) == 3
|
|
203
|
+
|
|
204
|
+
|
|
143
205
|
def test_websearch_rejects_other_engines():
|
|
144
206
|
with pytest.raises(ValueError, match="engine"):
|
|
145
207
|
CloroWebSearch(engine="other") # type: ignore[arg-type]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|