search-parser 0.3.0__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/codeql.yml +1 -1
  2. {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/publish.yml +1 -0
  3. {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/test.yml +1 -1
  4. {search_parser-0.3.0 → search_parser-0.4.1}/.gitignore +2 -1
  5. {search_parser-0.3.0 → search_parser-0.4.1}/CHANGELOG.md +29 -1
  6. {search_parser-0.3.0 → search_parser-0.4.1}/PKG-INFO +88 -3
  7. {search_parser-0.3.0 → search_parser-0.4.1}/README.md +86 -1
  8. {search_parser-0.3.0 → search_parser-0.4.1}/docs/examples/advanced_usage.md +19 -0
  9. {search_parser-0.3.0 → search_parser-0.4.1}/docs/examples/basic_usage.md +23 -0
  10. {search_parser-0.3.0 → search_parser-0.4.1}/docs/getting_started.md +33 -0
  11. {search_parser-0.3.0 → search_parser-0.4.1}/docs/index.md +9 -2
  12. {search_parser-0.3.0 → search_parser-0.4.1}/pyproject.toml +2 -2
  13. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/__version__.py +1 -1
  14. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/core/models.py +6 -0
  15. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/markdown_formatter.py +50 -0
  16. search_parser-0.4.1/src/search_parser/parsers/google.py +671 -0
  17. {search_parser-0.3.0 → search_parser-0.4.1}/tests/conftest.py +10 -0
  18. search_parser-0.4.1/tests/fixtures/google/supply-chain-director-jobs.html +92 -0
  19. search_parser-0.4.1/tests/fixtures/google/weekly_contacts_with_mobile.html +35 -0
  20. {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/test_google_parser.py +235 -0
  21. {search_parser-0.3.0 → search_parser-0.4.1}/uv.lock +1 -1
  22. search_parser-0.3.0/src/search_parser/parsers/google.py +0 -379
  23. {search_parser-0.3.0 → search_parser-0.4.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  24. {search_parser-0.3.0 → search_parser-0.4.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  25. {search_parser-0.3.0 → search_parser-0.4.1}/.github/dependabot.yml +0 -0
  26. {search_parser-0.3.0 → search_parser-0.4.1}/.github/pull_request_template.md +0 -0
  27. {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/coverage-badge.yml +0 -0
  28. {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/lint.yml +0 -0
  29. {search_parser-0.3.0 → search_parser-0.4.1}/.pre-commit-config.yaml +0 -0
  30. {search_parser-0.3.0 → search_parser-0.4.1}/CLAUDE.md +0 -0
  31. {search_parser-0.3.0 → search_parser-0.4.1}/CODE_OF_CONDUCT.md +0 -0
  32. {search_parser-0.3.0 → search_parser-0.4.1}/CONTRIBUTING.md +0 -0
  33. {search_parser-0.3.0 → search_parser-0.4.1}/LICENSE +0 -0
  34. {search_parser-0.3.0 → search_parser-0.4.1}/docs/CNAME +0 -0
  35. {search_parser-0.3.0 → search_parser-0.4.1}/docs/adding_search_engine.md +0 -0
  36. {search_parser-0.3.0 → search_parser-0.4.1}/docs/api_reference.md +0 -0
  37. {search_parser-0.3.0 → search_parser-0.4.1}/docs/contributing.md +0 -0
  38. {search_parser-0.3.0 → search_parser-0.4.1}/examples/basic_parsing.py +0 -0
  39. {search_parser-0.3.0 → search_parser-0.4.1}/examples/batch_processing.py +0 -0
  40. {search_parser-0.3.0 → search_parser-0.4.1}/examples/custom_formatter.py +0 -0
  41. {search_parser-0.3.0 → search_parser-0.4.1}/mkdocs.yml +0 -0
  42. {search_parser-0.3.0 → search_parser-0.4.1}/scripts/update_fixtures.py +0 -0
  43. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/__init__.py +0 -0
  44. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/cli.py +0 -0
  45. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/core/__init__.py +0 -0
  46. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/core/detector.py +0 -0
  47. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/core/parser.py +0 -0
  48. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/exceptions.py +0 -0
  49. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/__init__.py +0 -0
  50. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/base.py +0 -0
  51. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/json_formatter.py +0 -0
  52. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/parsers/__init__.py +0 -0
  53. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/parsers/base.py +0 -0
  54. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/parsers/bing.py +0 -0
  55. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/parsers/duckduckgo.py +0 -0
  56. {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/utils.py +0 -0
  57. {search_parser-0.3.0 → search_parser-0.4.1}/tests/__init__.py +0 -0
  58. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/bing/organic_results.html +0 -0
  59. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/bing/search_github_repos.html +0 -0
  60. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/duckduckgo/organic_results.html +0 -0
  61. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/duckduckgo/search_github_repos.html +0 -0
  62. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/featured_snippet.html +0 -0
  63. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/knowledge_panel.html +0 -0
  64. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/need_turn_on_javascript.html +0 -0
  65. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/organic_results.html +0 -0
  66. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/search_best_employee_scheduling_app.html +0 -0
  67. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/search_github_repos.html +0 -0
  68. {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/search_python_web_scraping.html +0 -0
  69. {search_parser-0.3.0 → search_parser-0.4.1}/tests/test_cli.py +0 -0
  70. {search_parser-0.3.0 → search_parser-0.4.1}/tests/test_coverage_boost.py +0 -0
  71. {search_parser-0.3.0 → search_parser-0.4.1}/tests/test_formatters.py +0 -0
  72. {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/__init__.py +0 -0
  73. {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/test_bing_parser.py +0 -0
  74. {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/test_detector.py +0 -0
  75. {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/test_duckduckgo_parser.py +0 -0
@@ -57,7 +57,7 @@ jobs:
57
57
  # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
58
58
  steps:
59
59
  - name: Checkout repository
60
- uses: actions/checkout@v4
60
+ uses: actions/checkout@v6
61
61
 
62
62
  # Add any setup steps before running the `github/codeql-action/init` action.
63
63
  # This includes steps like installing compilers or runtimes (`actions/setup-node`
@@ -3,6 +3,7 @@ name: Publish to PyPI
3
3
  on:
4
4
  release:
5
5
  types: [published]
6
+ workflow_dispatch:
6
7
 
7
8
  jobs:
8
9
  deploy:
@@ -41,7 +41,7 @@ jobs:
41
41
  token: ${{ secrets.CODECOV_TOKEN }}
42
42
 
43
43
  - name: Upload coverage HTML artifact
44
- uses: actions/upload-artifact@v6
44
+ uses: actions/upload-artifact@v7
45
45
  if: matrix.python-version == '3.11'
46
46
  with:
47
47
  name: coverage-report
@@ -48,4 +48,5 @@ Thumbs.db
48
48
 
49
49
  # uv
50
50
 
51
- search_parser_prompt.md
51
+ search_parser_prompt.md
52
+
@@ -9,6 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
9
9
 
10
10
  ---
11
11
 
12
+ ## [0.4.1] - 2026-03-05
13
+
14
+ ### Added
15
+
16
+ - **Google parser: Mobile HTML support** — organic results from mobile Google pages (using `div.xpd` / `egMi0` layout) are now parsed correctly. URLs are decoded from Google's `/url?q=...` redirect format to the actual destination. People Also Ask, People Also Search For, and AI Overview are also extracted from the mobile layout.
17
+ - **Google parser: Shopping Ads** — new `shopping_ads` field on `SearchResults` containing product cards from Google Shopping units (mobile `wywECb`/`qvfQJe` layout). Each entry has `result_type="shopping_ad"` with `metadata["price"]` and `metadata["merchant"]`. Shopping ads are never included in the organic `results` list.
18
+ - New `result_type` value `"shopping_ad"` added to the `SearchResult` model's Literal type.
19
+ - Markdown formatter now renders a `## Shopping Ads` section when `shopping_ads` are present.
20
+ - 14 new unit tests covering mobile organic results (count, positions, URL decoding, titles), mobile PAA, mobile PASF, mobile AI Overview, and shopping ads (count, titles, prices, merchants, isolation).
21
+
22
+ ---
23
+
24
+ ## [0.4.0] - 2026-03-05
25
+
26
+ ### Added
27
+
28
+ - **Google parser: Jobs** — new `jobs` field on `SearchResults` containing job listings from the Google Jobs widget. Each entry has `result_type="job"` with `metadata["company"]` and `metadata["location"]` fields. Jobs are never included in the organic `results` list.
29
+ - **Google parser: Discussions and forums** — new `discussions` field on `SearchResults` containing entries from the "Discussions and forums" widget. Each entry has `result_type="discussion"` with a `description` excerpt and `metadata["source"]` (platform, forum, comment count, and date).
30
+ - New `result_type` values `"job"` and `"discussion"` added to the `SearchResult` model's Literal type.
31
+ - Markdown formatter now renders `## Jobs` and `## Discussions and Forums` sections when those fields are present.
32
+ - Unit tests for jobs and discussions parsing using the `supply-chain-director-jobs.html` fixture.
33
+
34
+ ---
35
+
12
36
  ## [0.3.0] - 2026-02-21
13
37
 
14
38
  ### Changed
@@ -16,6 +40,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
16
40
  - Renamed all stale `search_engine_parser` references to `search_parser` across config, docs, and CI workflows.
17
41
  - Fixed `pyproject.toml`: CLI entry point, hatch build target, and mypy `files` path all now point to `src/search_parser`.
18
42
  - Synced `__version__.py` to match `pyproject.toml`.
43
+ - Added author and publisher metadata (`linksc`, `hello@link.sc`) to `pyproject.toml`.
44
+ - Updated package description with explicit search engine names and output formats for better discoverability.
19
45
 
20
46
  ---
21
47
 
@@ -82,6 +108,8 @@ sponsored = results.sponsored # list[SearchResult]
82
108
  - CI/CD workflows for testing, linting, coverage, and publishing.
83
109
  - Pre-commit hooks for ruff and mypy.
84
110
 
85
- [Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.2.0...HEAD
111
+ [Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.4.0...HEAD
112
+ [0.4.0]: https://github.com/getlinksc/search-parser/compare/v0.3.0...v0.4.0
113
+ [0.3.0]: https://github.com/getlinksc/search-parser/compare/v0.2.0...v0.3.0
86
114
  [0.2.0]: https://github.com/getlinksc/search-parser/compare/v0.1.0...v0.2.0
87
115
  [0.1.0]: https://github.com/getlinksc/search-parser/releases/tag/v0.1.0
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: search-parser
3
- Version: 0.3.0
3
+ Version: 0.4.1
4
4
  Summary: Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or dict — with auto-detection
5
5
  Project-URL: Homepage, https://github.com/getlinksc/search-parser
6
6
  Project-URL: Documentation, https://getlinksc.github.io/search-parser
7
7
  Project-URL: Repository, https://github.com/getlinksc/search-parser
8
8
  Project-URL: Issues, https://github.com/getlinksc/search-parser/issues
9
9
  Project-URL: Changelog, https://github.com/getlinksc/search-parser/blob/main/CHANGELOG.md
10
- Author-email: Your Name <you@example.com>
10
+ Author-email: linksc <hello@link.sc>
11
11
  License: Apache-2.0
12
12
  License-File: LICENSE
13
13
  Keywords: bing,duckduckgo,google,parser,scraping,search
@@ -53,7 +53,7 @@ Description-Content-Type: text/markdown
53
53
 
54
54
  **Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or Python dict — with automatic search engine detection.**
55
55
 
56
- `search-parser` takes raw HTML from Google, Bing, and DuckDuckGo and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
56
+ `search-parser` takes raw HTML from Google, Bing, and DuckDuckGo — desktop or mobile — and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, shopping ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
57
57
 
58
58
  ---
59
59
 
@@ -120,6 +120,9 @@ pip install search-parser
120
120
  | What People Are Saying | `people_saying` | ✓ | — | — |
121
121
  | People Also Search For | `people_also_search` | ✓ | — | — |
122
122
  | Related Products & Services | `related_products` | ✓ | — | — |
123
+ | Jobs | `jobs` | ✓ | — | — |
124
+ | Discussions and forums | `discussions` | ✓ | — | — |
125
+ | Shopping ads | `shopping_ads` | ✓ | — | — |
123
126
 
124
127
  ---
125
128
 
@@ -161,6 +164,19 @@ for ad in data["sponsored"]:
161
164
  for product in data["related_products"]:
162
165
  print(product["title"])
163
166
 
167
+ # Jobs (title, metadata["company"], metadata["location"])
168
+ for job in data["jobs"]:
169
+ print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
170
+
171
+ # Discussions (title, url, description, metadata["source"])
172
+ for disc in data["discussions"]:
173
+ print(disc["title"], disc["url"])
174
+ print(disc["metadata"]["source"])
175
+
176
+ # Shopping ads (title, metadata["price"], metadata["merchant"])
177
+ for ad in data["shopping_ads"]:
178
+ print(ad["title"], ad["metadata"]["price"], ad["metadata"]["merchant"])
179
+
164
180
  # Metadata
165
181
  print(data["search_engine"]) # "google"
166
182
  print(data["query"]) # "python web scraping"
@@ -196,6 +212,9 @@ for q in results.people_also_ask:
196
212
  for post in results.people_saying:
197
213
  print(post.title, post.url)
198
214
 
215
+ for ad in results.shopping_ads:
216
+ print(ad.title, ad.metadata["price"], ad.metadata["merchant"])
217
+
199
218
  # Convert to JSON or Markdown directly on the model
200
219
  json_str = results.to_json()
201
220
  json_str = results.to_json(indent=4) # custom indent
@@ -244,6 +263,44 @@ md_str = results.to_markdown()
244
263
  "people_saying": [],
245
264
  "people_also_search": [],
246
265
  "related_products": [],
266
+ "jobs": [
267
+ {
268
+ "title": "Global Supply Chain Director",
269
+ "url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
270
+ "description": null,
271
+ "position": 0,
272
+ "result_type": "job",
273
+ "metadata": {
274
+ "company": "InterSources, Inc.",
275
+ "location": "San Jose, CA • via Ladders"
276
+ }
277
+ }
278
+ ],
279
+ "discussions": [
280
+ {
281
+ "title": "Being considered for Director of Supply Chain",
282
+ "url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
283
+ "description": "I work for a mid-sized company as a Procurement Manager...",
284
+ "position": 0,
285
+ "result_type": "discussion",
286
+ "metadata": {
287
+ "source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
288
+ }
289
+ }
290
+ ],
291
+ "shopping_ads": [
292
+ {
293
+ "title": "ALCON - Precision7 , 12 Pack",
294
+ "url": "http://www.google.com/aclk?sa=L&ai=...",
295
+ "description": null,
296
+ "position": 0,
297
+ "result_type": "shopping_ad",
298
+ "metadata": {
299
+ "price": "$51.19",
300
+ "merchant": "Contacts Direct"
301
+ }
302
+ }
303
+ ],
247
304
  "detection_confidence": 0.95,
248
305
  "parsed_at": "2026-02-21T00:00:00Z",
249
306
  "metadata": {}
@@ -276,6 +333,34 @@ Web scraping is the process of extracting data from websites...
276
333
  Learn how to scrape websites with Python...
277
334
 
278
335
  **URL:** https://realpython.com/python-web-scraping/
336
+
337
+ ---
338
+
339
+ ## Jobs
340
+
341
+ ### Global Supply Chain Director
342
+
343
+ **Company:** InterSources, Inc.
344
+ **Location:** San Jose, CA • via Ladders
345
+ **URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
346
+
347
+ ---
348
+
349
+ ## Discussions and Forums
350
+
351
+ ### Being considered for Director of Supply Chain
352
+
353
+ I work for a mid-sized company as a Procurement Manager...
354
+
355
+ **URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
356
+
357
+ ## Shopping Ads
358
+
359
+ ### ALCON - Precision7 , 12 Pack
360
+
361
+ **Price:** $51.19
362
+ **Merchant:** Contacts Direct
363
+ **URL:** http://www.google.com/aclk?sa=L&ai=...
279
364
  ```
280
365
 
281
366
  ---
@@ -10,7 +10,7 @@
10
10
 
11
11
  **Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or Python dict — with automatic search engine detection.**
12
12
 
13
- `search-parser` takes raw HTML from Google, Bing, and DuckDuckGo and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
13
+ `search-parser` takes raw HTML from Google, Bing, and DuckDuckGo — desktop or mobile — and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, shopping ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
14
14
 
15
15
  ---
16
16
 
@@ -77,6 +77,9 @@ pip install search-parser
77
77
  | What People Are Saying | `people_saying` | ✓ | — | — |
78
78
  | People Also Search For | `people_also_search` | ✓ | — | — |
79
79
  | Related Products & Services | `related_products` | ✓ | — | — |
80
+ | Jobs | `jobs` | ✓ | — | — |
81
+ | Discussions and forums | `discussions` | ✓ | — | — |
82
+ | Shopping ads | `shopping_ads` | ✓ | — | — |
80
83
 
81
84
  ---
82
85
 
@@ -118,6 +121,19 @@ for ad in data["sponsored"]:
118
121
  for product in data["related_products"]:
119
122
  print(product["title"])
120
123
 
124
+ # Jobs (title, metadata["company"], metadata["location"])
125
+ for job in data["jobs"]:
126
+ print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
127
+
128
+ # Discussions (title, url, description, metadata["source"])
129
+ for disc in data["discussions"]:
130
+ print(disc["title"], disc["url"])
131
+ print(disc["metadata"]["source"])
132
+
133
+ # Shopping ads (title, metadata["price"], metadata["merchant"])
134
+ for ad in data["shopping_ads"]:
135
+ print(ad["title"], ad["metadata"]["price"], ad["metadata"]["merchant"])
136
+
121
137
  # Metadata
122
138
  print(data["search_engine"]) # "google"
123
139
  print(data["query"]) # "python web scraping"
@@ -153,6 +169,9 @@ for q in results.people_also_ask:
153
169
  for post in results.people_saying:
154
170
  print(post.title, post.url)
155
171
 
172
+ for ad in results.shopping_ads:
173
+ print(ad.title, ad.metadata["price"], ad.metadata["merchant"])
174
+
156
175
  # Convert to JSON or Markdown directly on the model
157
176
  json_str = results.to_json()
158
177
  json_str = results.to_json(indent=4) # custom indent
@@ -201,6 +220,44 @@ md_str = results.to_markdown()
201
220
  "people_saying": [],
202
221
  "people_also_search": [],
203
222
  "related_products": [],
223
+ "jobs": [
224
+ {
225
+ "title": "Global Supply Chain Director",
226
+ "url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
227
+ "description": null,
228
+ "position": 0,
229
+ "result_type": "job",
230
+ "metadata": {
231
+ "company": "InterSources, Inc.",
232
+ "location": "San Jose, CA • via Ladders"
233
+ }
234
+ }
235
+ ],
236
+ "discussions": [
237
+ {
238
+ "title": "Being considered for Director of Supply Chain",
239
+ "url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
240
+ "description": "I work for a mid-sized company as a Procurement Manager...",
241
+ "position": 0,
242
+ "result_type": "discussion",
243
+ "metadata": {
244
+ "source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
245
+ }
246
+ }
247
+ ],
248
+ "shopping_ads": [
249
+ {
250
+ "title": "ALCON - Precision7 , 12 Pack",
251
+ "url": "http://www.google.com/aclk?sa=L&ai=...",
252
+ "description": null,
253
+ "position": 0,
254
+ "result_type": "shopping_ad",
255
+ "metadata": {
256
+ "price": "$51.19",
257
+ "merchant": "Contacts Direct"
258
+ }
259
+ }
260
+ ],
204
261
  "detection_confidence": 0.95,
205
262
  "parsed_at": "2026-02-21T00:00:00Z",
206
263
  "metadata": {}
@@ -233,6 +290,34 @@ Web scraping is the process of extracting data from websites...
233
290
  Learn how to scrape websites with Python...
234
291
 
235
292
  **URL:** https://realpython.com/python-web-scraping/
293
+
294
+ ---
295
+
296
+ ## Jobs
297
+
298
+ ### Global Supply Chain Director
299
+
300
+ **Company:** InterSources, Inc.
301
+ **Location:** San Jose, CA • via Ladders
302
+ **URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
303
+
304
+ ---
305
+
306
+ ## Discussions and Forums
307
+
308
+ ### Being considered for Director of Supply Chain
309
+
310
+ I work for a mid-sized company as a Procurement Manager...
311
+
312
+ **URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
313
+
314
+ ## Shopping Ads
315
+
316
+ ### ALCON - Precision7 , 12 Pack
317
+
318
+ **Price:** $51.19
319
+ **Merchant:** Contacts Direct
320
+ **URL:** http://www.google.com/aclk?sa=L&ai=...
236
321
  ```
237
322
 
238
323
  ---
@@ -87,6 +87,25 @@ for item in results.people_also_search:
87
87
  # Product / service suggestions
88
88
  for product in results.related_products:
89
89
  print(product.title, product.url)
90
+
91
+ # Jobs — metadata includes company, location, salary, employment_type, benefits
92
+ for job in results.jobs:
93
+ print(job.title)
94
+ print(f" {job.metadata['company']} — {job.metadata['location']}")
95
+ if job.metadata.get("salary"):
96
+ print(f" Salary: {job.metadata['salary']}")
97
+
98
+ # Discussions and forums — metadata["source"] has platform + comment count + age
99
+ for disc in results.discussions:
100
+ print(disc.title)
101
+ print(f" {disc.url}")
102
+ print(f" {disc.metadata['source']}")
103
+
104
+ # Shopping ads — metadata["price"] and metadata["merchant"]
105
+ for ad in results.shopping_ads:
106
+ print(ad.title)
107
+ print(f" {ad.metadata['price']} from {ad.metadata['merchant']}")
108
+ print(f" {ad.url}")
90
109
  ```
91
110
 
92
111
  ## Using to_json() and to_markdown()
@@ -79,6 +79,29 @@ for item in data["people_also_search"]:
79
79
  # Related Products & Services (Google only)
80
80
  for product in data["related_products"]:
81
81
  print(product["title"])
82
+
83
+ # Jobs (Google only)
84
+ for job in data["jobs"]:
85
+ print(job["title"], "at", job["metadata"]["company"])
86
+ print(" Location:", job["metadata"]["location"])
87
+ if job["metadata"].get("salary"):
88
+ print(" Salary:", job["metadata"]["salary"])
89
+ if job["metadata"].get("employment_type"):
90
+ print(" Type:", job["metadata"]["employment_type"])
91
+
92
+ # Discussions and forums (Google only)
93
+ for disc in data["discussions"]:
94
+ print(disc["title"])
95
+ print(" URL:", disc["url"])
96
+ print(" Source:", disc["metadata"]["source"])
97
+ if disc["description"]:
98
+ print(" Excerpt:", disc["description"][:100])
99
+
100
+ # Shopping ads (Google only)
101
+ for ad in data["shopping_ads"]:
102
+ print(ad["title"])
103
+ print(" Price:", ad["metadata"]["price"])
104
+ print(" Merchant:", ad["metadata"]["merchant"])
82
105
  ```
83
106
 
84
107
  ## Specifying the Engine
@@ -81,6 +81,30 @@ for ad in data["sponsored"]:
81
81
  for product in data["related_products"]:
82
82
  print("Product:", product["title"])
83
83
 
84
+ # Jobs (list) — Google only
85
+ # Each job has metadata: company, location, salary (optional), employment_type, benefits
86
+ for job in data["jobs"]:
87
+ print("Job:", job["title"])
88
+ print(" Company:", job["metadata"]["company"])
89
+ print(" Location:", job["metadata"]["location"])
90
+ print(" Salary:", job["metadata"].get("salary")) # None if not listed
91
+ print(" Type:", job["metadata"].get("employment_type"))
92
+ print(" Benefits:", job["metadata"].get("benefits", []))
93
+
94
+ # Discussions and forums (list) — Google only
95
+ # Each entry has a url, description excerpt, and metadata["source"]
96
+ for disc in data["discussions"]:
97
+ print("Discussion:", disc["title"])
98
+ print(" URL:", disc["url"])
99
+ print(" Source:", disc["metadata"]["source"])
100
+
101
+ # Shopping ads (list) — Google only
102
+ # Each entry has metadata["price"] and metadata["merchant"]
103
+ for ad in data["shopping_ads"]:
104
+ print("Ad:", ad["title"])
105
+ print(" Price:", ad["metadata"]["price"])
106
+ print(" Merchant:", ad["metadata"]["merchant"])
107
+
84
108
  # Metadata
85
109
  print(data["query"]) # "python web scraping"
86
110
  print(data["total_results"]) # 26200000 or None
@@ -112,6 +136,15 @@ if results.ai_overview:
112
136
  for q in results.people_also_ask:
113
137
  print(q.title)
114
138
 
139
+ for job in results.jobs:
140
+ print(job.title, job.metadata["company"], job.metadata.get("salary"))
141
+
142
+ for disc in results.discussions:
143
+ print(disc.title, disc.url)
144
+
145
+ for ad in results.shopping_ads:
146
+ print(ad.title, ad.metadata["price"], ad.metadata["merchant"])
147
+
115
148
  # Serialize without going through SearchParser
116
149
  json_str = results.to_json() # JSON string, indent=2 by default
117
150
  json_str = results.to_json(indent=4) # custom indent
@@ -4,8 +4,8 @@ Parse search engine HTML results into structured data (JSON, Markdown) with auto
4
4
 
5
5
  ## Features
6
6
 
7
- - **Auto-detection** of Google, Bing, and DuckDuckGo HTML
8
- - **Dedicated fields** for every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more
7
+ - **Auto-detection** of Google, Bing, and DuckDuckGo HTML — desktop and mobile layouts
8
+ - **Dedicated fields** for every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, shopping ads, and more
9
9
  - **Multiple output formats**: JSON, Markdown, Python dict
10
10
  - **Convenience methods**: `results.to_json()` and `results.to_markdown()` directly on the model
11
11
  - **Extensible** plugin architecture for adding new search engines
@@ -40,6 +40,10 @@ if data["ai_overview"]:
40
40
 
41
41
  for q in data["people_also_ask"]:
42
42
  print(q["title"])
43
+
44
+ # Shopping ads (Google only)
45
+ for ad in data["shopping_ads"]:
46
+ print(ad["title"], ad["metadata"]["price"], ad["metadata"]["merchant"])
43
47
  ```
44
48
 
45
49
  Or work directly with the typed model and use `to_json()` / `to_markdown()`:
@@ -69,3 +73,6 @@ md_str = results.to_markdown()
69
73
  | What People Are Saying | `people_saying` | ✓ | — | — |
70
74
  | People Also Search For | `people_also_search` | ✓ | — | — |
71
75
  | Related Products & Services | `related_products` | ✓ | — | — |
76
+ | Jobs | `jobs` | ✓ | — | — |
77
+ | Discussions and forums | `discussions` | ✓ | — | — |
78
+ | Shopping ads | `shopping_ads` | ✓ | — | — |
@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "search-parser"
7
- version = "0.3.0"
7
+ version = "0.4.1"
8
8
  description = "Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or dict — with auto-detection"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
11
11
  license = {text = "Apache-2.0"}
12
12
  authors = [
13
- {name = "Your Name", email = "you@example.com"}
13
+ {name = "linksc", email = "hello@link.sc"}
14
14
  ]
15
15
  keywords = ["search", "parser", "scraping", "google", "bing", "duckduckgo"]
16
16
  classifiers = [
@@ -1,3 +1,3 @@
1
1
  """Version information."""
2
2
 
3
- __version__ = "0.3.0"
3
+ __version__ = "0.4.1"
@@ -27,6 +27,9 @@ class SearchResult(BaseModel):
27
27
  "people_saying",
28
28
  "people_also_search",
29
29
  "related_products",
30
+ "job",
31
+ "discussion",
32
+ "shopping_ad",
30
33
  ] = "organic"
31
34
  metadata: dict[str, object] = Field(default_factory=dict)
32
35
 
@@ -53,6 +56,9 @@ class SearchResults(BaseModel):
53
56
  people_saying: list[SearchResult] = Field(default_factory=list)
54
57
  people_also_search: list[SearchResult] = Field(default_factory=list)
55
58
  related_products: list[SearchResult] = Field(default_factory=list)
59
+ jobs: list[SearchResult] = Field(default_factory=list)
60
+ discussions: list[SearchResult] = Field(default_factory=list)
61
+ shopping_ads: list[SearchResult] = Field(default_factory=list)
56
62
 
57
63
  detection_confidence: float = Field(ge=0.0, le=1.0)
58
64
  parsed_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
@@ -70,6 +70,24 @@ class MarkdownFormatter(BaseFormatter):
70
70
  for result in news:
71
71
  lines.extend(self._format_result(result))
72
72
 
73
+ if results.jobs:
74
+ lines.append("## Jobs")
75
+ lines.append("")
76
+ for result in results.jobs:
77
+ lines.extend(self._format_job(result))
78
+
79
+ if results.discussions:
80
+ lines.append("## Discussions and Forums")
81
+ lines.append("")
82
+ for result in results.discussions:
83
+ lines.extend(self._format_result(result))
84
+
85
+ if results.shopping_ads:
86
+ lines.append("## Shopping Ads")
87
+ lines.append("")
88
+ for result in results.shopping_ads:
89
+ lines.extend(self._format_shopping_ad(result))
90
+
73
91
  lines.append("---")
74
92
  lines.append("")
75
93
  lines.append(f"*Parsed with search-engine-parser v{__version__}*")
@@ -102,6 +120,38 @@ class MarkdownFormatter(BaseFormatter):
102
120
  lines.append("")
103
121
  return lines
104
122
 
123
+ def _format_job(self, result: SearchResult) -> list[str]:
124
+ """Format a job listing result."""
125
+ lines: list[str] = []
126
+ lines.append(f"### {result.title}")
127
+ lines.append("")
128
+ if result.metadata.get("company"):
129
+ lines.append(f"**Company:** {result.metadata['company']}")
130
+ if result.metadata.get("location"):
131
+ lines.append(f"**Location:** {result.metadata['location']}")
132
+ if result.metadata.get("salary"):
133
+ lines.append(f"**Salary:** {result.metadata['salary']}")
134
+ if result.metadata.get("employment_type"):
135
+ lines.append(f"**Type:** {result.metadata['employment_type']}")
136
+ if result.url:
137
+ lines.append(f"**URL:** {result.url}")
138
+ lines.append("")
139
+ return lines
140
+
141
+ def _format_shopping_ad(self, result: SearchResult) -> list[str]:
142
+ """Format a shopping ad card."""
143
+ lines: list[str] = []
144
+ lines.append(f"### {result.title}")
145
+ lines.append("")
146
+ if result.metadata.get("price"):
147
+ lines.append(f"**Price:** {result.metadata['price']}")
148
+ if result.metadata.get("merchant"):
149
+ lines.append(f"**Merchant:** {result.metadata['merchant']}")
150
+ if result.url:
151
+ lines.append(f"**URL:** {result.url}")
152
+ lines.append("")
153
+ return lines
154
+
105
155
  def _format_result(self, result: SearchResult) -> list[str]:
106
156
  """Format a generic result."""
107
157
  lines: list[str] = []