cloro-haystack 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/PKG-INFO +13 -5
  2. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/README.md +12 -4
  3. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/pyproject.toml +1 -1
  4. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/websearch.py +33 -5
  5. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/test_cloro.py +62 -0
  6. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/.gitignore +0 -0
  7. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/LICENSE +0 -0
  8. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/__init__.py +0 -0
  9. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/_api.py +0 -0
  10. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/cloro/ai_answer.py +0 -0
  11. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/src/haystack_integrations/components/websearch/py.typed +0 -0
  12. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/aimode.json +0 -0
  13. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/bing.json +0 -0
  14. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/bing_aimode.json +0 -0
  15. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/chatgpt.json +0 -0
  16. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/error_401.json +0 -0
  17. {cloro_haystack-0.3.0 → cloro_haystack-0.4.0}/tests/fixtures/google_aioverview.json +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cloro-haystack
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Haystack components for cloro: Google and Bing search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources.
5
5
  Project-URL: Homepage, https://cloro.dev/?utm_source=cloro-haystack&utm_medium=byok
6
6
  Project-URL: Documentation, https://github.com/cloro-dev/cloro-haystack#readme
@@ -60,21 +60,29 @@ Each organic result is a `Document`. Its `content` is the result snippet and its
60
60
  `"bing_ai_overview"`) and the URLs it cites in `meta["links"]`. Results with an empty link or a Google
61
61
  redirect link are skipped.
62
62
 
63
+ With `include_ads=True`, the ads on the Google or Bing page are the last Documents. The content of an ad
64
+ starts with `Ad:`. Its `meta["source"]` is `"google_ad"` or `"bing_ad"`, and its `meta` has `title`, `link`,
65
+ `position`, `block_position`, `ad_type`, `price` and `store`, when the ad has them. `links` has no ads.
66
+
63
67
  Parameters:
64
68
 
65
69
  - `api_key`: defaults to the `CLORO_API_KEY` environment variable.
66
- - `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of Google
70
+ - `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
67
71
  results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
68
72
  - `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
69
73
  - `allowed_domains`: domains to limit the search to, sent as `site:` filters.
70
74
  - `include_ai_overview`: request the AI Overview. Defaults to `False`.
71
- - `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google.
75
+ - `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google, and
76
+ `pages` or `first` for Bing.
72
77
  - `timeout`: request timeout in seconds. Defaults to 60.
73
78
  - `engine`: `"google"` (Google Search) or `"bing"` (Bing Search). Defaults to `"google"`.
79
+ - `include_ads`: return the ads on the page. Defaults to `False`. The ads cost no extra credits and do not
80
+ count towards `top_k`.
74
81
 
75
82
  With `engine="bing"`, the component searches Bing. The country sets the Bing market. Bing Search accepts
76
- all countries except CN, KR and RU. It has no `device`, `location`, `hl` or `pages` fields and returns
77
- one page of results. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
83
+ all countries except CN, KR and RU. It has no `device`, `location` or `hl` fields. Set
84
+ `search_params={"pages": 2}` for more results (2 credits for each extra page) or `search_params={"first": 11}`
85
+ to start at a later result. Bing shows its AI summary only on the first page. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
78
86
  non-Latin scripts.
79
87
 
80
88
  ```python
@@ -37,21 +37,29 @@ Each organic result is a `Document`. Its `content` is the result snippet and its
37
37
  `"bing_ai_overview"`) and the URLs it cites in `meta["links"]`. Results with an empty link or a Google
38
38
  redirect link are skipped.
39
39
 
40
+ With `include_ads=True`, the ads on the Google or Bing page are the last Documents. The content of an ad
41
+ starts with `Ad:`. Its `meta["source"]` is `"google_ad"` or `"bing_ad"`, and its `meta` has `title`, `link`,
42
+ `position`, `block_position`, `ad_type`, `price` and `store`, when the ad has them. `links` has no ads.
43
+
40
44
  Parameters:
41
45
 
42
46
  - `api_key`: defaults to the `CLORO_API_KEY` environment variable.
43
- - `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of Google
47
+ - `top_k`: maximum number of organic results. Defaults to 10. One request returns one page of
44
48
  results (about 10). Set `search_params={"pages": 2}` for more (2 credits for each extra page).
45
49
  - `country`: ISO 3166-1 alpha-2 country code. Defaults to `"US"`.
46
50
  - `allowed_domains`: domains to limit the search to, sent as `site:` filters.
47
51
  - `include_ai_overview`: request the AI Overview. Defaults to `False`.
48
- - `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google.
52
+ - `search_params`: other request fields, for example `device`, `location`, `hl` or `pages` for Google, and
53
+ `pages` or `first` for Bing.
49
54
  - `timeout`: request timeout in seconds. Defaults to 60.
50
55
  - `engine`: `"google"` (Google Search) or `"bing"` (Bing Search). Defaults to `"google"`.
56
+ - `include_ads`: return the ads on the page. Defaults to `False`. The ads cost no extra credits and do not
57
+ count towards `top_k`.
51
58
 
52
59
  With `engine="bing"`, the component searches Bing. The country sets the Bing market. Bing Search accepts
53
- all countries except CN, KR and RU. It has no `device`, `location`, `hl` or `pages` fields and returns
54
- one page of results. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
60
+ all countries except CN, KR and RU. It has no `device`, `location` or `hl` fields. Set
61
+ `search_params={"pages": 2}` for more results (2 credits for each extra page) or `search_params={"first": 11}`
62
+ to start at a later result. Bing shows its AI summary only on the first page. The query can have a maximum of 3,500 characters after URL encoding, and fewer for
55
63
  non-Latin scripts.
56
64
 
57
65
  ```python
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cloro-haystack"
7
- version = "0.3.0"
7
+ version = "0.4.0"
8
8
  description = "Haystack components for cloro: Google and Bing search results with the AI Overview, and ChatGPT, Google AI Mode, Gemini and Bing AI Mode answers with sources."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -27,7 +27,8 @@ class CloroWebSearch:
27
27
 
28
28
  Each organic result becomes a Document. Its content is the result snippet. Its meta has
29
29
  `title`, `link` and `position`. With `include_ai_overview=True`, the AI Overview of the
30
- engine (when the engine shows one) is the first Document.
30
+ engine (when the engine shows one) is the first Document. With `include_ads=True`, the ads
31
+ on the page are the last Documents.
31
32
 
32
33
  ### Usage example
33
34
 
@@ -51,6 +52,7 @@ class CloroWebSearch:
51
52
  search_params: dict[str, Any] | None = None,
52
53
  timeout: float = 60.0,
53
54
  engine: Engine = "google",
55
+ include_ads: bool = False,
54
56
  ) -> None:
55
57
  """
56
58
  Initialize the CloroWebSearch component.
@@ -59,7 +61,7 @@ class CloroWebSearch:
59
61
  cloro API key. Defaults to the `CLORO_API_KEY` environment variable.
60
62
  :param top_k:
61
63
  Maximum number of organic results to return. One request returns one page of results
62
- (about 10). For Google, set `search_params={"pages": 2}` or more to get more results.
64
+ (about 10). Set `search_params={"pages": 2}` or more to get more results.
63
65
  :param country:
64
66
  ISO 3166-1 alpha-2 country code for the engine location.
65
67
  :param allowed_domains:
@@ -68,11 +70,14 @@ class CloroWebSearch:
68
70
  If True, request the AI Overview and return it as the first Document.
69
71
  :param search_params:
70
72
  Other fields for the engine request body, for example `device`, `location`, `hl` or
71
- `pages` for Google. See the cloro API reference.
73
+ `pages` for Google, and `pages` or `first` for Bing. See the cloro API reference.
72
74
  :param timeout:
73
75
  Request timeout in seconds.
74
76
  :param engine:
75
77
  The search engine: `"google"` (Google Search) or `"bing"` (Bing Search).
78
+ :param include_ads:
79
+ If True, return the ads on the page after the organic results. Each ad is a Document with
80
+ `meta["source"]` set to `"google_ad"` or `"bing_ad"`. The ads do not count towards `top_k`.
76
81
  """
77
82
  if engine not in _PATHS:
78
83
  msg = f"engine must be 'google' or 'bing', not {engine!r}"
@@ -85,6 +90,7 @@ class CloroWebSearch:
85
90
  self.include_ai_overview = include_ai_overview
86
91
  self.search_params = search_params
87
92
  self.timeout = timeout
93
+ self.include_ads = include_ads
88
94
 
89
95
  def to_dict(self) -> dict[str, Any]:
90
96
  """
@@ -102,6 +108,7 @@ class CloroWebSearch:
102
108
  search_params=self.search_params,
103
109
  timeout=self.timeout,
104
110
  engine=self.engine,
111
+ include_ads=self.include_ads,
105
112
  )
106
113
 
107
114
  @classmethod
@@ -122,7 +129,8 @@ class CloroWebSearch:
122
129
 
123
130
  :param query: Search query.
124
131
  :returns: A dictionary with:
125
- - `documents`: The AI Overview (if requested and present), then the organic results.
132
+ - `documents`: The AI Overview (if requested and present), then the organic results, then
133
+ the ads (if requested and present).
126
134
  - `links`: The links of the organic results.
127
135
  :raises TimeoutError: If the request times out.
128
136
  :raises CloroError: If the cloro API returns an error.
@@ -139,7 +147,8 @@ class CloroWebSearch:
139
147
 
140
148
  :param query: Search query.
141
149
  :returns: A dictionary with:
142
- - `documents`: The AI Overview (if requested and present), then the organic results.
150
+ - `documents`: The AI Overview (if requested and present), then the organic results, then
151
+ the ads (if requested and present).
143
152
  - `links`: The links of the organic results.
144
153
  :raises TimeoutError: If the request times out.
145
154
  :raises CloroError: If the cloro API returns an error.
@@ -171,6 +180,8 @@ class CloroWebSearch:
171
180
  for item in organic
172
181
  ]
173
182
  links = [item["link"] for item in organic]
183
+ if self.include_ads:
184
+ documents += [self._ad_document(ad) for ad in result.get("ads") or [] if isinstance(ad, dict)]
174
185
 
175
186
  overview = result.get("aioverview")
176
187
  if self.include_ai_overview and isinstance(overview, dict) and overview.get("text"):
@@ -189,3 +200,20 @@ class CloroWebSearch:
189
200
 
190
201
  logger.debug("cloro returned {count} documents", count=len(documents))
191
202
  return {"documents": documents, "links": links}
203
+
204
+ def _ad_document(self, ad: dict[str, Any]) -> Document:
205
+ # Every field of an ad can be absent.
206
+ price = ad.get("price")
207
+ meta = {
208
+ "title": ad.get("title"),
209
+ "link": ad.get("url") if is_direct_link(ad.get("url")) else None,
210
+ "position": ad.get("position"),
211
+ "block_position": ad.get("blockPosition"),
212
+ "ad_type": ad.get("type"),
213
+ "price": price.get("raw") if isinstance(price, dict) else None,
214
+ "store": ad.get("store"),
215
+ }
216
+ return Document(
217
+ content=f"Ad: {ad.get('description') or ad.get('title') or ''}".strip(),
218
+ meta={"source": f"{self.engine}_ad", **{key: value for key, value in meta.items() if value}},
219
+ )
@@ -140,6 +140,68 @@ def test_websearch_bing_request_and_ai_overview(api):
140
140
  assert len(result["documents"]) == 4
141
141
 
142
142
 
143
+ ADS = [
144
+ {
145
+ "position": 1,
146
+ "blockPosition": "top",
147
+ "type": "RESULT",
148
+ "title": "Wok Sale",
149
+ "url": "https://shop.example/woks",
150
+ "page": 1,
151
+ "domain": "shop.example",
152
+ "description": "Carbon steel woks.",
153
+ },
154
+ {
155
+ "position": 1,
156
+ "blockPosition": "rhs",
157
+ "type": "SHOPPING_CARD",
158
+ "title": "14 in Wok",
159
+ "url": "https://shop.example/wok-14",
160
+ "page": 1,
161
+ "price": {"value": 29, "currency": "$", "raw": "$29.00"},
162
+ "store": "Shop",
163
+ },
164
+ {},
165
+ ]
166
+
167
+
168
+ @pytest.mark.parametrize(("engine", "fixture"), [("google", "google_aioverview.json"), ("bing", "bing.json")])
169
+ def test_websearch_ads(api, engine, fixture):
170
+ payload = _load(fixture)
171
+ payload["result"]["ads"] = ADS
172
+ api.respond(httpx.Response(200, json=payload))
173
+ result = CloroWebSearch(engine=engine, top_k=1, include_ads=True).run(query="wok")
174
+ assert "include" not in _body(api.calls[0])
175
+ assert len(result["links"]) == 1
176
+ text_ad, card, empty = result["documents"][1:]
177
+ assert text_ad.content == "Ad: Carbon steel woks."
178
+ assert text_ad.meta == {
179
+ "source": f"{engine}_ad",
180
+ "title": "Wok Sale",
181
+ "link": "https://shop.example/woks",
182
+ "position": 1,
183
+ "block_position": "top",
184
+ "ad_type": "RESULT",
185
+ }
186
+ assert card.content == "Ad: 14 in Wok"
187
+ assert card.meta["price"] == "$29.00"
188
+ assert card.meta["store"] == "Shop"
189
+ assert empty.meta == {"source": f"{engine}_ad"}
190
+
191
+
192
+ def test_websearch_ads_off_by_default(api):
193
+ payload = _load("bing.json")
194
+ payload["result"]["ads"] = ADS
195
+ api.respond(httpx.Response(200, json=payload))
196
+ result = CloroWebSearch(engine="bing").run(query="wok")
197
+ assert [d.meta["link"] for d in result["documents"]] == result["links"]
198
+
199
+
200
+ def test_websearch_include_ads_without_ads(api):
201
+ api.respond(httpx.Response(200, json=_load("bing.json")))
202
+ assert len(CloroWebSearch(engine="bing", include_ads=True).run(query="wok")["documents"]) == 3
203
+
204
+
143
205
  def test_websearch_rejects_other_engines():
144
206
  with pytest.raises(ValueError, match="engine"):
145
207
  CloroWebSearch(engine="other") # type: ignore[arg-type]
File without changes