search-parser 0.3.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/codeql.yml +1 -1
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/publish.yml +1 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/test.yml +1 -1
- {search_parser-0.3.0 → search_parser-0.4.1}/.gitignore +2 -1
- {search_parser-0.3.0 → search_parser-0.4.1}/CHANGELOG.md +29 -1
- {search_parser-0.3.0 → search_parser-0.4.1}/PKG-INFO +88 -3
- {search_parser-0.3.0 → search_parser-0.4.1}/README.md +86 -1
- {search_parser-0.3.0 → search_parser-0.4.1}/docs/examples/advanced_usage.md +19 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/docs/examples/basic_usage.md +23 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/docs/getting_started.md +33 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/docs/index.md +9 -2
- {search_parser-0.3.0 → search_parser-0.4.1}/pyproject.toml +2 -2
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/__version__.py +1 -1
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/core/models.py +6 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/markdown_formatter.py +50 -0
- search_parser-0.4.1/src/search_parser/parsers/google.py +671 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/conftest.py +10 -0
- search_parser-0.4.1/tests/fixtures/google/supply-chain-director-jobs.html +92 -0
- search_parser-0.4.1/tests/fixtures/google/weekly_contacts_with_mobile.html +35 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/test_google_parser.py +235 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/uv.lock +1 -1
- search_parser-0.3.0/src/search_parser/parsers/google.py +0 -379
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/dependabot.yml +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/pull_request_template.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/coverage-badge.yml +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/.github/workflows/lint.yml +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/.pre-commit-config.yaml +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/CLAUDE.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/CODE_OF_CONDUCT.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/CONTRIBUTING.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/LICENSE +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/docs/CNAME +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/docs/adding_search_engine.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/docs/api_reference.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/docs/contributing.md +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/examples/basic_parsing.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/examples/batch_processing.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/examples/custom_formatter.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/mkdocs.yml +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/scripts/update_fixtures.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/__init__.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/cli.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/core/__init__.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/core/detector.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/core/parser.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/exceptions.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/__init__.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/base.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/json_formatter.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/parsers/__init__.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/parsers/base.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/parsers/bing.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/parsers/duckduckgo.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/utils.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/__init__.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/bing/organic_results.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/bing/search_github_repos.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/duckduckgo/organic_results.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/duckduckgo/search_github_repos.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/featured_snippet.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/knowledge_panel.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/need_turn_on_javascript.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/organic_results.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/search_best_employee_scheduling_app.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/search_github_repos.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/fixtures/google/search_python_web_scraping.html +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/test_cli.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/test_coverage_boost.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/test_formatters.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/__init__.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/test_bing_parser.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/test_detector.py +0 -0
- {search_parser-0.3.0 → search_parser-0.4.1}/tests/unit/test_duckduckgo_parser.py +0 -0
|
@@ -57,7 +57,7 @@ jobs:
|
|
|
57
57
|
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
|
58
58
|
steps:
|
|
59
59
|
- name: Checkout repository
|
|
60
|
-
uses: actions/checkout@
|
|
60
|
+
uses: actions/checkout@v6
|
|
61
61
|
|
|
62
62
|
# Add any setup steps before running the `github/codeql-action/init` action.
|
|
63
63
|
# This includes steps like installing compilers or runtimes (`actions/setup-node`
|
|
@@ -9,6 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
9
9
|
|
|
10
10
|
---
|
|
11
11
|
|
|
12
|
+
## [0.4.1] - 2026-03-05
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- **Google parser: Mobile HTML support** — organic results from mobile Google pages (using `div.xpd` / `egMi0` layout) are now parsed correctly. URLs are decoded from Google's `/url?q=...` redirect format to the actual destination. People Also Ask, People Also Search For, and AI Overview are also extracted from the mobile layout.
|
|
17
|
+
- **Google parser: Shopping Ads** — new `shopping_ads` field on `SearchResults` containing product cards from Google Shopping units (mobile `wywECb`/`qvfQJe` layout). Each entry has `result_type="shopping_ad"` with `metadata["price"]` and `metadata["merchant"]`. Shopping ads are never included in the organic `results` list.
|
|
18
|
+
- New `result_type` value `"shopping_ad"` added to the `SearchResult` model's Literal type.
|
|
19
|
+
- Markdown formatter now renders a `## Shopping Ads` section when `shopping_ads` are present.
|
|
20
|
+
- 14 new unit tests covering mobile organic results (count, positions, URL decoding, titles), mobile PAA, mobile PASF, mobile AI Overview, and shopping ads (count, titles, prices, merchants, isolation).
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## [0.4.0] - 2026-03-05
|
|
25
|
+
|
|
26
|
+
### Added
|
|
27
|
+
|
|
28
|
+
- **Google parser: Jobs** — new `jobs` field on `SearchResults` containing job listings from the Google Jobs widget. Each entry has `result_type="job"` with `metadata["company"]` and `metadata["location"]` fields. Jobs are never included in the organic `results` list.
|
|
29
|
+
- **Google parser: Discussions and forums** — new `discussions` field on `SearchResults` containing entries from the "Discussions and forums" widget. Each entry has `result_type="discussion"` with a `description` excerpt and `metadata["source"]` (platform, forum, comment count, and date).
|
|
30
|
+
- New `result_type` values `"job"` and `"discussion"` added to the `SearchResult` model's Literal type.
|
|
31
|
+
- Markdown formatter now renders `## Jobs` and `## Discussions and Forums` sections when those fields are present.
|
|
32
|
+
- Unit tests for jobs and discussions parsing using the `supply-chain-director-jobs.html` fixture.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
12
36
|
## [0.3.0] - 2026-02-21
|
|
13
37
|
|
|
14
38
|
### Changed
|
|
@@ -16,6 +40,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
16
40
|
- Renamed all stale `search_engine_parser` references to `search_parser` across config, docs, and CI workflows.
|
|
17
41
|
- Fixed `pyproject.toml`: CLI entry point, hatch build target, and mypy `files` path all now point to `src/search_parser`.
|
|
18
42
|
- Synced `__version__.py` to match `pyproject.toml`.
|
|
43
|
+
- Added author and publisher metadata (`linksc`, `hello@link.sc`) to `pyproject.toml`.
|
|
44
|
+
- Updated package description with explicit search engine names and output formats for better discoverability.
|
|
19
45
|
|
|
20
46
|
---
|
|
21
47
|
|
|
@@ -82,6 +108,8 @@ sponsored = results.sponsored # list[SearchResult]
|
|
|
82
108
|
- CI/CD workflows for testing, linting, coverage, and publishing.
|
|
83
109
|
- Pre-commit hooks for ruff and mypy.
|
|
84
110
|
|
|
85
|
-
[Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.
|
|
111
|
+
[Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.4.0...HEAD
|
|
112
|
+
[0.4.0]: https://github.com/getlinksc/search-parser/compare/v0.3.0...v0.4.0
|
|
113
|
+
[0.3.0]: https://github.com/getlinksc/search-parser/compare/v0.2.0...v0.3.0
|
|
86
114
|
[0.2.0]: https://github.com/getlinksc/search-parser/compare/v0.1.0...v0.2.0
|
|
87
115
|
[0.1.0]: https://github.com/getlinksc/search-parser/releases/tag/v0.1.0
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: search-parser
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or dict — with auto-detection
|
|
5
5
|
Project-URL: Homepage, https://github.com/getlinksc/search-parser
|
|
6
6
|
Project-URL: Documentation, https://getlinksc.github.io/search-parser
|
|
7
7
|
Project-URL: Repository, https://github.com/getlinksc/search-parser
|
|
8
8
|
Project-URL: Issues, https://github.com/getlinksc/search-parser/issues
|
|
9
9
|
Project-URL: Changelog, https://github.com/getlinksc/search-parser/blob/main/CHANGELOG.md
|
|
10
|
-
Author-email:
|
|
10
|
+
Author-email: linksc <hello@link.sc>
|
|
11
11
|
License: Apache-2.0
|
|
12
12
|
License-File: LICENSE
|
|
13
13
|
Keywords: bing,duckduckgo,google,parser,scraping,search
|
|
@@ -53,7 +53,7 @@ Description-Content-Type: text/markdown
|
|
|
53
53
|
|
|
54
54
|
**Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or Python dict — with automatic search engine detection.**
|
|
55
55
|
|
|
56
|
-
`search-parser` takes raw HTML from Google, Bing, and DuckDuckGo and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
|
|
56
|
+
`search-parser` takes raw HTML from Google, Bing, and DuckDuckGo — desktop or mobile — and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, shopping ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
|
|
57
57
|
|
|
58
58
|
---
|
|
59
59
|
|
|
@@ -120,6 +120,9 @@ pip install search-parser
|
|
|
120
120
|
| What People Are Saying | `people_saying` | ✓ | — | — |
|
|
121
121
|
| People Also Search For | `people_also_search` | ✓ | — | — |
|
|
122
122
|
| Related Products & Services | `related_products` | ✓ | — | — |
|
|
123
|
+
| Jobs | `jobs` | ✓ | — | — |
|
|
124
|
+
| Discussions and forums | `discussions` | ✓ | — | — |
|
|
125
|
+
| Shopping ads | `shopping_ads` | ✓ | — | — |
|
|
123
126
|
|
|
124
127
|
---
|
|
125
128
|
|
|
@@ -161,6 +164,19 @@ for ad in data["sponsored"]:
|
|
|
161
164
|
for product in data["related_products"]:
|
|
162
165
|
print(product["title"])
|
|
163
166
|
|
|
167
|
+
# Jobs (title, metadata["company"], metadata["location"])
|
|
168
|
+
for job in data["jobs"]:
|
|
169
|
+
print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
|
|
170
|
+
|
|
171
|
+
# Discussions (title, url, description, metadata["source"])
|
|
172
|
+
for disc in data["discussions"]:
|
|
173
|
+
print(disc["title"], disc["url"])
|
|
174
|
+
print(disc["metadata"]["source"])
|
|
175
|
+
|
|
176
|
+
# Shopping ads (title, metadata["price"], metadata["merchant"])
|
|
177
|
+
for ad in data["shopping_ads"]:
|
|
178
|
+
print(ad["title"], ad["metadata"]["price"], ad["metadata"]["merchant"])
|
|
179
|
+
|
|
164
180
|
# Metadata
|
|
165
181
|
print(data["search_engine"]) # "google"
|
|
166
182
|
print(data["query"]) # "python web scraping"
|
|
@@ -196,6 +212,9 @@ for q in results.people_also_ask:
|
|
|
196
212
|
for post in results.people_saying:
|
|
197
213
|
print(post.title, post.url)
|
|
198
214
|
|
|
215
|
+
for ad in results.shopping_ads:
|
|
216
|
+
print(ad.title, ad.metadata["price"], ad.metadata["merchant"])
|
|
217
|
+
|
|
199
218
|
# Convert to JSON or Markdown directly on the model
|
|
200
219
|
json_str = results.to_json()
|
|
201
220
|
json_str = results.to_json(indent=4) # custom indent
|
|
@@ -244,6 +263,44 @@ md_str = results.to_markdown()
|
|
|
244
263
|
"people_saying": [],
|
|
245
264
|
"people_also_search": [],
|
|
246
265
|
"related_products": [],
|
|
266
|
+
"jobs": [
|
|
267
|
+
{
|
|
268
|
+
"title": "Global Supply Chain Director",
|
|
269
|
+
"url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
|
|
270
|
+
"description": null,
|
|
271
|
+
"position": 0,
|
|
272
|
+
"result_type": "job",
|
|
273
|
+
"metadata": {
|
|
274
|
+
"company": "InterSources, Inc.",
|
|
275
|
+
"location": "San Jose, CA • via Ladders"
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
],
|
|
279
|
+
"discussions": [
|
|
280
|
+
{
|
|
281
|
+
"title": "Being considered for Director of Supply Chain",
|
|
282
|
+
"url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
|
|
283
|
+
"description": "I work for a mid-sized company as a Procurement Manager...",
|
|
284
|
+
"position": 0,
|
|
285
|
+
"result_type": "discussion",
|
|
286
|
+
"metadata": {
|
|
287
|
+
"source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
],
|
|
291
|
+
"shopping_ads": [
|
|
292
|
+
{
|
|
293
|
+
"title": "ALCON - Precision7 , 12 Pack",
|
|
294
|
+
"url": "http://www.google.com/aclk?sa=L&ai=...",
|
|
295
|
+
"description": null,
|
|
296
|
+
"position": 0,
|
|
297
|
+
"result_type": "shopping_ad",
|
|
298
|
+
"metadata": {
|
|
299
|
+
"price": "$51.19",
|
|
300
|
+
"merchant": "Contacts Direct"
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
],
|
|
247
304
|
"detection_confidence": 0.95,
|
|
248
305
|
"parsed_at": "2026-02-21T00:00:00Z",
|
|
249
306
|
"metadata": {}
|
|
@@ -276,6 +333,34 @@ Web scraping is the process of extracting data from websites...
|
|
|
276
333
|
Learn how to scrape websites with Python...
|
|
277
334
|
|
|
278
335
|
**URL:** https://realpython.com/python-web-scraping/
|
|
336
|
+
|
|
337
|
+
---
|
|
338
|
+
|
|
339
|
+
## Jobs
|
|
340
|
+
|
|
341
|
+
### Global Supply Chain Director
|
|
342
|
+
|
|
343
|
+
**Company:** InterSources, Inc.
|
|
344
|
+
**Location:** San Jose, CA • via Ladders
|
|
345
|
+
**URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
|
|
346
|
+
|
|
347
|
+
---
|
|
348
|
+
|
|
349
|
+
## Discussions and Forums
|
|
350
|
+
|
|
351
|
+
### Being considered for Director of Supply Chain
|
|
352
|
+
|
|
353
|
+
I work for a mid-sized company as a Procurement Manager...
|
|
354
|
+
|
|
355
|
+
**URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
|
|
356
|
+
|
|
357
|
+
## Shopping Ads
|
|
358
|
+
|
|
359
|
+
### ALCON - Precision7 , 12 Pack
|
|
360
|
+
|
|
361
|
+
**Price:** $51.19
|
|
362
|
+
**Merchant:** Contacts Direct
|
|
363
|
+
**URL:** http://www.google.com/aclk?sa=L&ai=...
|
|
279
364
|
```
|
|
280
365
|
|
|
281
366
|
---
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
**Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or Python dict — with automatic search engine detection.**
|
|
12
12
|
|
|
13
|
-
`search-parser` takes raw HTML from Google, Bing, and DuckDuckGo and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
|
|
13
|
+
`search-parser` takes raw HTML from Google, Bing, and DuckDuckGo — desktop or mobile — and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, shopping ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
|
|
14
14
|
|
|
15
15
|
---
|
|
16
16
|
|
|
@@ -77,6 +77,9 @@ pip install search-parser
|
|
|
77
77
|
| What People Are Saying | `people_saying` | ✓ | — | — |
|
|
78
78
|
| People Also Search For | `people_also_search` | ✓ | — | — |
|
|
79
79
|
| Related Products & Services | `related_products` | ✓ | — | — |
|
|
80
|
+
| Jobs | `jobs` | ✓ | — | — |
|
|
81
|
+
| Discussions and forums | `discussions` | ✓ | — | — |
|
|
82
|
+
| Shopping ads | `shopping_ads` | ✓ | — | — |
|
|
80
83
|
|
|
81
84
|
---
|
|
82
85
|
|
|
@@ -118,6 +121,19 @@ for ad in data["sponsored"]:
|
|
|
118
121
|
for product in data["related_products"]:
|
|
119
122
|
print(product["title"])
|
|
120
123
|
|
|
124
|
+
# Jobs (title, metadata["company"], metadata["location"])
|
|
125
|
+
for job in data["jobs"]:
|
|
126
|
+
print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
|
|
127
|
+
|
|
128
|
+
# Discussions (title, url, description, metadata["source"])
|
|
129
|
+
for disc in data["discussions"]:
|
|
130
|
+
print(disc["title"], disc["url"])
|
|
131
|
+
print(disc["metadata"]["source"])
|
|
132
|
+
|
|
133
|
+
# Shopping ads (title, metadata["price"], metadata["merchant"])
|
|
134
|
+
for ad in data["shopping_ads"]:
|
|
135
|
+
print(ad["title"], ad["metadata"]["price"], ad["metadata"]["merchant"])
|
|
136
|
+
|
|
121
137
|
# Metadata
|
|
122
138
|
print(data["search_engine"]) # "google"
|
|
123
139
|
print(data["query"]) # "python web scraping"
|
|
@@ -153,6 +169,9 @@ for q in results.people_also_ask:
|
|
|
153
169
|
for post in results.people_saying:
|
|
154
170
|
print(post.title, post.url)
|
|
155
171
|
|
|
172
|
+
for ad in results.shopping_ads:
|
|
173
|
+
print(ad.title, ad.metadata["price"], ad.metadata["merchant"])
|
|
174
|
+
|
|
156
175
|
# Convert to JSON or Markdown directly on the model
|
|
157
176
|
json_str = results.to_json()
|
|
158
177
|
json_str = results.to_json(indent=4) # custom indent
|
|
@@ -201,6 +220,44 @@ md_str = results.to_markdown()
|
|
|
201
220
|
"people_saying": [],
|
|
202
221
|
"people_also_search": [],
|
|
203
222
|
"related_products": [],
|
|
223
|
+
"jobs": [
|
|
224
|
+
{
|
|
225
|
+
"title": "Global Supply Chain Director",
|
|
226
|
+
"url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
|
|
227
|
+
"description": null,
|
|
228
|
+
"position": 0,
|
|
229
|
+
"result_type": "job",
|
|
230
|
+
"metadata": {
|
|
231
|
+
"company": "InterSources, Inc.",
|
|
232
|
+
"location": "San Jose, CA • via Ladders"
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
],
|
|
236
|
+
"discussions": [
|
|
237
|
+
{
|
|
238
|
+
"title": "Being considered for Director of Supply Chain",
|
|
239
|
+
"url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
|
|
240
|
+
"description": "I work for a mid-sized company as a Procurement Manager...",
|
|
241
|
+
"position": 0,
|
|
242
|
+
"result_type": "discussion",
|
|
243
|
+
"metadata": {
|
|
244
|
+
"source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
],
|
|
248
|
+
"shopping_ads": [
|
|
249
|
+
{
|
|
250
|
+
"title": "ALCON - Precision7 , 12 Pack",
|
|
251
|
+
"url": "http://www.google.com/aclk?sa=L&ai=...",
|
|
252
|
+
"description": null,
|
|
253
|
+
"position": 0,
|
|
254
|
+
"result_type": "shopping_ad",
|
|
255
|
+
"metadata": {
|
|
256
|
+
"price": "$51.19",
|
|
257
|
+
"merchant": "Contacts Direct"
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
],
|
|
204
261
|
"detection_confidence": 0.95,
|
|
205
262
|
"parsed_at": "2026-02-21T00:00:00Z",
|
|
206
263
|
"metadata": {}
|
|
@@ -233,6 +290,34 @@ Web scraping is the process of extracting data from websites...
|
|
|
233
290
|
Learn how to scrape websites with Python...
|
|
234
291
|
|
|
235
292
|
**URL:** https://realpython.com/python-web-scraping/
|
|
293
|
+
|
|
294
|
+
---
|
|
295
|
+
|
|
296
|
+
## Jobs
|
|
297
|
+
|
|
298
|
+
### Global Supply Chain Director
|
|
299
|
+
|
|
300
|
+
**Company:** InterSources, Inc.
|
|
301
|
+
**Location:** San Jose, CA • via Ladders
|
|
302
|
+
**URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
|
|
303
|
+
|
|
304
|
+
---
|
|
305
|
+
|
|
306
|
+
## Discussions and Forums
|
|
307
|
+
|
|
308
|
+
### Being considered for Director of Supply Chain
|
|
309
|
+
|
|
310
|
+
I work for a mid-sized company as a Procurement Manager...
|
|
311
|
+
|
|
312
|
+
**URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
|
|
313
|
+
|
|
314
|
+
## Shopping Ads
|
|
315
|
+
|
|
316
|
+
### ALCON - Precision7 , 12 Pack
|
|
317
|
+
|
|
318
|
+
**Price:** $51.19
|
|
319
|
+
**Merchant:** Contacts Direct
|
|
320
|
+
**URL:** http://www.google.com/aclk?sa=L&ai=...
|
|
236
321
|
```
|
|
237
322
|
|
|
238
323
|
---
|
|
@@ -87,6 +87,25 @@ for item in results.people_also_search:
|
|
|
87
87
|
# Product / service suggestions
|
|
88
88
|
for product in results.related_products:
|
|
89
89
|
print(product.title, product.url)
|
|
90
|
+
|
|
91
|
+
# Jobs — metadata includes company, location, salary, employment_type, benefits
|
|
92
|
+
for job in results.jobs:
|
|
93
|
+
print(job.title)
|
|
94
|
+
print(f" {job.metadata['company']} — {job.metadata['location']}")
|
|
95
|
+
if job.metadata.get("salary"):
|
|
96
|
+
print(f" Salary: {job.metadata['salary']}")
|
|
97
|
+
|
|
98
|
+
# Discussions and forums — metadata["source"] has platform + comment count + age
|
|
99
|
+
for disc in results.discussions:
|
|
100
|
+
print(disc.title)
|
|
101
|
+
print(f" {disc.url}")
|
|
102
|
+
print(f" {disc.metadata['source']}")
|
|
103
|
+
|
|
104
|
+
# Shopping ads — metadata["price"] and metadata["merchant"]
|
|
105
|
+
for ad in results.shopping_ads:
|
|
106
|
+
print(ad.title)
|
|
107
|
+
print(f" {ad.metadata['price']} from {ad.metadata['merchant']}")
|
|
108
|
+
print(f" {ad.url}")
|
|
90
109
|
```
|
|
91
110
|
|
|
92
111
|
## Using to_json() and to_markdown()
|
|
@@ -79,6 +79,29 @@ for item in data["people_also_search"]:
|
|
|
79
79
|
# Related Products & Services (Google only)
|
|
80
80
|
for product in data["related_products"]:
|
|
81
81
|
print(product["title"])
|
|
82
|
+
|
|
83
|
+
# Jobs (Google only)
|
|
84
|
+
for job in data["jobs"]:
|
|
85
|
+
print(job["title"], "at", job["metadata"]["company"])
|
|
86
|
+
print(" Location:", job["metadata"]["location"])
|
|
87
|
+
if job["metadata"].get("salary"):
|
|
88
|
+
print(" Salary:", job["metadata"]["salary"])
|
|
89
|
+
if job["metadata"].get("employment_type"):
|
|
90
|
+
print(" Type:", job["metadata"]["employment_type"])
|
|
91
|
+
|
|
92
|
+
# Discussions and forums (Google only)
|
|
93
|
+
for disc in data["discussions"]:
|
|
94
|
+
print(disc["title"])
|
|
95
|
+
print(" URL:", disc["url"])
|
|
96
|
+
print(" Source:", disc["metadata"]["source"])
|
|
97
|
+
if disc["description"]:
|
|
98
|
+
print(" Excerpt:", disc["description"][:100])
|
|
99
|
+
|
|
100
|
+
# Shopping ads (Google only)
|
|
101
|
+
for ad in data["shopping_ads"]:
|
|
102
|
+
print(ad["title"])
|
|
103
|
+
print(" Price:", ad["metadata"]["price"])
|
|
104
|
+
print(" Merchant:", ad["metadata"]["merchant"])
|
|
82
105
|
```
|
|
83
106
|
|
|
84
107
|
## Specifying the Engine
|
|
@@ -81,6 +81,30 @@ for ad in data["sponsored"]:
|
|
|
81
81
|
for product in data["related_products"]:
|
|
82
82
|
print("Product:", product["title"])
|
|
83
83
|
|
|
84
|
+
# Jobs (list) — Google only
|
|
85
|
+
# Each job has metadata: company, location, salary (optional), employment_type, benefits
|
|
86
|
+
for job in data["jobs"]:
|
|
87
|
+
print("Job:", job["title"])
|
|
88
|
+
print(" Company:", job["metadata"]["company"])
|
|
89
|
+
print(" Location:", job["metadata"]["location"])
|
|
90
|
+
print(" Salary:", job["metadata"].get("salary")) # None if not listed
|
|
91
|
+
print(" Type:", job["metadata"].get("employment_type"))
|
|
92
|
+
print(" Benefits:", job["metadata"].get("benefits", []))
|
|
93
|
+
|
|
94
|
+
# Discussions and forums (list) — Google only
|
|
95
|
+
# Each entry has a url, description excerpt, and metadata["source"]
|
|
96
|
+
for disc in data["discussions"]:
|
|
97
|
+
print("Discussion:", disc["title"])
|
|
98
|
+
print(" URL:", disc["url"])
|
|
99
|
+
print(" Source:", disc["metadata"]["source"])
|
|
100
|
+
|
|
101
|
+
# Shopping ads (list) — Google only
|
|
102
|
+
# Each entry has metadata["price"] and metadata["merchant"]
|
|
103
|
+
for ad in data["shopping_ads"]:
|
|
104
|
+
print("Ad:", ad["title"])
|
|
105
|
+
print(" Price:", ad["metadata"]["price"])
|
|
106
|
+
print(" Merchant:", ad["metadata"]["merchant"])
|
|
107
|
+
|
|
84
108
|
# Metadata
|
|
85
109
|
print(data["query"]) # "python web scraping"
|
|
86
110
|
print(data["total_results"]) # 26200000 or None
|
|
@@ -112,6 +136,15 @@ if results.ai_overview:
|
|
|
112
136
|
for q in results.people_also_ask:
|
|
113
137
|
print(q.title)
|
|
114
138
|
|
|
139
|
+
for job in results.jobs:
|
|
140
|
+
print(job.title, job.metadata["company"], job.metadata.get("salary"))
|
|
141
|
+
|
|
142
|
+
for disc in results.discussions:
|
|
143
|
+
print(disc.title, disc.url)
|
|
144
|
+
|
|
145
|
+
for ad in results.shopping_ads:
|
|
146
|
+
print(ad.title, ad.metadata["price"], ad.metadata["merchant"])
|
|
147
|
+
|
|
115
148
|
# Serialize without going through SearchParser
|
|
116
149
|
json_str = results.to_json() # JSON string, indent=2 by default
|
|
117
150
|
json_str = results.to_json(indent=4) # custom indent
|
|
@@ -4,8 +4,8 @@ Parse search engine HTML results into structured data (JSON, Markdown) with auto
|
|
|
4
4
|
|
|
5
5
|
## Features
|
|
6
6
|
|
|
7
|
-
- **Auto-detection** of Google, Bing, and DuckDuckGo HTML
|
|
8
|
-
- **Dedicated fields** for every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more
|
|
7
|
+
- **Auto-detection** of Google, Bing, and DuckDuckGo HTML — desktop and mobile layouts
|
|
8
|
+
- **Dedicated fields** for every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, shopping ads, and more
|
|
9
9
|
- **Multiple output formats**: JSON, Markdown, Python dict
|
|
10
10
|
- **Convenience methods**: `results.to_json()` and `results.to_markdown()` directly on the model
|
|
11
11
|
- **Extensible** plugin architecture for adding new search engines
|
|
@@ -40,6 +40,10 @@ if data["ai_overview"]:
|
|
|
40
40
|
|
|
41
41
|
for q in data["people_also_ask"]:
|
|
42
42
|
print(q["title"])
|
|
43
|
+
|
|
44
|
+
# Shopping ads (Google only)
|
|
45
|
+
for ad in data["shopping_ads"]:
|
|
46
|
+
print(ad["title"], ad["metadata"]["price"], ad["metadata"]["merchant"])
|
|
43
47
|
```
|
|
44
48
|
|
|
45
49
|
Or work directly with the typed model and use `to_json()` / `to_markdown()`:
|
|
@@ -69,3 +73,6 @@ md_str = results.to_markdown()
|
|
|
69
73
|
| What People Are Saying | `people_saying` | ✓ | — | — |
|
|
70
74
|
| People Also Search For | `people_also_search` | ✓ | — | — |
|
|
71
75
|
| Related Products & Services | `related_products` | ✓ | — | — |
|
|
76
|
+
| Jobs | `jobs` | ✓ | — | — |
|
|
77
|
+
| Discussions and forums | `discussions` | ✓ | — | — |
|
|
78
|
+
| Shopping ads | `shopping_ads` | ✓ | — | — |
|
|
@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "search-parser"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.1"
|
|
8
8
|
description = "Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or dict — with auto-detection"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
11
11
|
license = {text = "Apache-2.0"}
|
|
12
12
|
authors = [
|
|
13
|
-
{name = "
|
|
13
|
+
{name = "linksc", email = "hello@link.sc"}
|
|
14
14
|
]
|
|
15
15
|
keywords = ["search", "parser", "scraping", "google", "bing", "duckduckgo"]
|
|
16
16
|
classifiers = [
|
|
@@ -27,6 +27,9 @@ class SearchResult(BaseModel):
|
|
|
27
27
|
"people_saying",
|
|
28
28
|
"people_also_search",
|
|
29
29
|
"related_products",
|
|
30
|
+
"job",
|
|
31
|
+
"discussion",
|
|
32
|
+
"shopping_ad",
|
|
30
33
|
] = "organic"
|
|
31
34
|
metadata: dict[str, object] = Field(default_factory=dict)
|
|
32
35
|
|
|
@@ -53,6 +56,9 @@ class SearchResults(BaseModel):
|
|
|
53
56
|
people_saying: list[SearchResult] = Field(default_factory=list)
|
|
54
57
|
people_also_search: list[SearchResult] = Field(default_factory=list)
|
|
55
58
|
related_products: list[SearchResult] = Field(default_factory=list)
|
|
59
|
+
jobs: list[SearchResult] = Field(default_factory=list)
|
|
60
|
+
discussions: list[SearchResult] = Field(default_factory=list)
|
|
61
|
+
shopping_ads: list[SearchResult] = Field(default_factory=list)
|
|
56
62
|
|
|
57
63
|
detection_confidence: float = Field(ge=0.0, le=1.0)
|
|
58
64
|
parsed_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
{search_parser-0.3.0 → search_parser-0.4.1}/src/search_parser/formatters/markdown_formatter.py
RENAMED
|
@@ -70,6 +70,24 @@ class MarkdownFormatter(BaseFormatter):
|
|
|
70
70
|
for result in news:
|
|
71
71
|
lines.extend(self._format_result(result))
|
|
72
72
|
|
|
73
|
+
if results.jobs:
|
|
74
|
+
lines.append("## Jobs")
|
|
75
|
+
lines.append("")
|
|
76
|
+
for result in results.jobs:
|
|
77
|
+
lines.extend(self._format_job(result))
|
|
78
|
+
|
|
79
|
+
if results.discussions:
|
|
80
|
+
lines.append("## Discussions and Forums")
|
|
81
|
+
lines.append("")
|
|
82
|
+
for result in results.discussions:
|
|
83
|
+
lines.extend(self._format_result(result))
|
|
84
|
+
|
|
85
|
+
if results.shopping_ads:
|
|
86
|
+
lines.append("## Shopping Ads")
|
|
87
|
+
lines.append("")
|
|
88
|
+
for result in results.shopping_ads:
|
|
89
|
+
lines.extend(self._format_shopping_ad(result))
|
|
90
|
+
|
|
73
91
|
lines.append("---")
|
|
74
92
|
lines.append("")
|
|
75
93
|
lines.append(f"*Parsed with search-engine-parser v{__version__}*")
|
|
@@ -102,6 +120,38 @@ class MarkdownFormatter(BaseFormatter):
|
|
|
102
120
|
lines.append("")
|
|
103
121
|
return lines
|
|
104
122
|
|
|
123
|
+
def _format_job(self, result: SearchResult) -> list[str]:
|
|
124
|
+
"""Format a job listing result."""
|
|
125
|
+
lines: list[str] = []
|
|
126
|
+
lines.append(f"### {result.title}")
|
|
127
|
+
lines.append("")
|
|
128
|
+
if result.metadata.get("company"):
|
|
129
|
+
lines.append(f"**Company:** {result.metadata['company']}")
|
|
130
|
+
if result.metadata.get("location"):
|
|
131
|
+
lines.append(f"**Location:** {result.metadata['location']}")
|
|
132
|
+
if result.metadata.get("salary"):
|
|
133
|
+
lines.append(f"**Salary:** {result.metadata['salary']}")
|
|
134
|
+
if result.metadata.get("employment_type"):
|
|
135
|
+
lines.append(f"**Type:** {result.metadata['employment_type']}")
|
|
136
|
+
if result.url:
|
|
137
|
+
lines.append(f"**URL:** {result.url}")
|
|
138
|
+
lines.append("")
|
|
139
|
+
return lines
|
|
140
|
+
|
|
141
|
+
def _format_shopping_ad(self, result: SearchResult) -> list[str]:
|
|
142
|
+
"""Format a shopping ad card."""
|
|
143
|
+
lines: list[str] = []
|
|
144
|
+
lines.append(f"### {result.title}")
|
|
145
|
+
lines.append("")
|
|
146
|
+
if result.metadata.get("price"):
|
|
147
|
+
lines.append(f"**Price:** {result.metadata['price']}")
|
|
148
|
+
if result.metadata.get("merchant"):
|
|
149
|
+
lines.append(f"**Merchant:** {result.metadata['merchant']}")
|
|
150
|
+
if result.url:
|
|
151
|
+
lines.append(f"**URL:** {result.url}")
|
|
152
|
+
lines.append("")
|
|
153
|
+
return lines
|
|
154
|
+
|
|
105
155
|
def _format_result(self, result: SearchResult) -> list[str]:
|
|
106
156
|
"""Format a generic result."""
|
|
107
157
|
lines: list[str] = []
|