search-parser 0.0.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/pull_request_template.md +1 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/codeql.yml +1 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/coverage-badge.yml +1 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/lint.yml +1 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/publish.yml +1 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/test.yml +2 -2
- {search_parser-0.0.2 → search_parser-0.4.0}/.gitignore +2 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/CHANGELOG.md +38 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/CLAUDE.md +4 -4
- {search_parser-0.0.2 → search_parser-0.4.0}/CONTRIBUTING.md +6 -6
- {search_parser-0.0.2 → search_parser-0.4.0}/PKG-INFO +66 -10
- {search_parser-0.0.2 → search_parser-0.4.0}/README.md +63 -7
- {search_parser-0.0.2 → search_parser-0.4.0}/docs/adding_search_engine.md +8 -8
- {search_parser-0.0.2 → search_parser-0.4.0}/docs/api_reference.md +6 -6
- {search_parser-0.0.2 → search_parser-0.4.0}/docs/examples/advanced_usage.md +7 -7
- {search_parser-0.0.2 → search_parser-0.4.0}/docs/examples/basic_usage.md +4 -4
- {search_parser-0.0.2 → search_parser-0.4.0}/docs/getting_started.md +6 -6
- {search_parser-0.0.2 → search_parser-0.4.0}/docs/index.md +3 -3
- {search_parser-0.0.2 → search_parser-0.4.0}/examples/basic_parsing.py +2 -2
- {search_parser-0.0.2 → search_parser-0.4.0}/examples/batch_processing.py +1 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/examples/custom_formatter.py +3 -3
- {search_parser-0.0.2 → search_parser-0.4.0}/pyproject.toml +6 -6
- search_parser-0.4.0/src/search_parser/__init__.py +12 -0
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/__version__.py +1 -1
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/cli.py +2 -2
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/core/detector.py +1 -1
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/core/models.py +5 -1
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/core/parser.py +7 -7
- search_parser-0.4.0/src/search_parser/formatters/__init__.py +11 -0
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/formatters/base.py +1 -1
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/formatters/json_formatter.py +2 -2
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/formatters/markdown_formatter.py +33 -3
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/__init__.py +4 -4
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/base.py +1 -1
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/bing.py +3 -3
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/duckduckgo.py +3 -3
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/google.py +110 -3
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/conftest.py +5 -0
- search_parser-0.4.0/tests/fixtures/google/supply-chain-director-jobs.html +92 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/test_cli.py +1 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/test_coverage_boost.py +9 -9
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/test_formatters.py +3 -3
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/test_bing_parser.py +2 -2
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/test_detector.py +1 -1
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/test_duckduckgo_parser.py +2 -2
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/test_google_parser.py +83 -2
- search_parser-0.0.2/src/search_engine_parser/__init__.py +0 -12
- search_parser-0.0.2/src/search_engine_parser/formatters/__init__.py +0 -11
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/.github/dependabot.yml +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/.pre-commit-config.yaml +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/CODE_OF_CONDUCT.md +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/LICENSE +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/docs/CNAME +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/docs/contributing.md +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/mkdocs.yml +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/scripts/update_fixtures.py +0 -0
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/core/__init__.py +0 -0
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/exceptions.py +0 -0
- {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/utils.py +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/__init__.py +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/bing/organic_results.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/bing/search_github_repos.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/duckduckgo/organic_results.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/duckduckgo/search_github_repos.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/featured_snippet.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/knowledge_panel.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/need_turn_on_javascript.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/organic_results.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/search_best_employee_scheduling_app.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/search_github_repos.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/search_python_web_scraping.html +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/__init__.py +0 -0
- {search_parser-0.0.2 → search_parser-0.4.0}/uv.lock +0 -0
|
@@ -15,7 +15,7 @@ Brief description of the changes.
|
|
|
15
15
|
- [ ] I have added tests that prove my fix/feature works
|
|
16
16
|
- [ ] All new and existing tests pass (`uv run pytest`)
|
|
17
17
|
- [ ] Linting passes (`uv run ruff check .`)
|
|
18
|
-
- [ ] Type checking passes (`uv run mypy src/
|
|
18
|
+
- [ ] Type checking passes (`uv run mypy src/search_parser`)
|
|
19
19
|
- [ ] I have updated documentation as needed
|
|
20
20
|
|
|
21
21
|
## Test Plan
|
|
@@ -57,7 +57,7 @@ jobs:
|
|
|
57
57
|
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
|
58
58
|
steps:
|
|
59
59
|
- name: Checkout repository
|
|
60
|
-
uses: actions/checkout@
|
|
60
|
+
uses: actions/checkout@v6
|
|
61
61
|
|
|
62
62
|
# Add any setup steps before running the `github/codeql-action/init` action.
|
|
63
63
|
# This includes steps like installing compilers or runtimes (`actions/setup-node`
|
|
@@ -23,7 +23,7 @@ jobs:
|
|
|
23
23
|
|
|
24
24
|
- name: Run tests with coverage
|
|
25
25
|
run: |
|
|
26
|
-
uv run pytest --cov=
|
|
26
|
+
uv run pytest --cov=search_parser --cov-report=term --cov-report=json
|
|
27
27
|
COVERAGE=$(python -c "import json; print(int(float(json.load(open('coverage.json'))['totals']['percent_covered_display'])))")
|
|
28
28
|
echo "COVERAGE=${COVERAGE}%" >> $GITHUB_ENV
|
|
29
29
|
if [ "$COVERAGE" -ge 90 ]; then echo "COVERAGE_COLOR=brightgreen" >> $GITHUB_ENV;
|
|
@@ -30,7 +30,7 @@ jobs:
|
|
|
30
30
|
run: uv sync --all-extras
|
|
31
31
|
|
|
32
32
|
- name: Run tests with coverage
|
|
33
|
-
run: uv run pytest --cov=
|
|
33
|
+
run: uv run pytest --cov=search_parser --cov-report=xml --cov-report=term --cov-report=html
|
|
34
34
|
|
|
35
35
|
- name: Upload coverage to Codecov
|
|
36
36
|
uses: codecov/codecov-action@v5
|
|
@@ -41,7 +41,7 @@ jobs:
|
|
|
41
41
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
42
42
|
|
|
43
43
|
- name: Upload coverage HTML artifact
|
|
44
|
-
uses: actions/upload-artifact@
|
|
44
|
+
uses: actions/upload-artifact@v7
|
|
45
45
|
if: matrix.python-version == '3.11'
|
|
46
46
|
with:
|
|
47
47
|
name: coverage-report
|
|
@@ -7,6 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## [0.4.0] - 2026-03-05
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- **Google parser: Jobs** — new `jobs` field on `SearchResults` containing job listings from the Google Jobs widget. Each entry has `result_type="job"` with `metadata["company"]` and `metadata["location"]` fields. Jobs are never included in the organic `results` list.
|
|
17
|
+
- **Google parser: Discussions and forums** — new `discussions` field on `SearchResults` containing entries from the "Discussions and forums" widget. Each entry has `result_type="discussion"` with a `description` excerpt and `metadata["source"]` (platform, forum, comment count, and date).
|
|
18
|
+
- New `result_type` values `"job"` and `"discussion"` added to the `SearchResult` model's Literal type.
|
|
19
|
+
- Markdown formatter now renders `## Jobs` and `## Discussions and Forums` sections when those fields are present.
|
|
20
|
+
- Unit tests for jobs and discussions parsing using the `supply-chain-director-jobs.html` fixture.
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## [0.3.0] - 2026-02-21
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
|
|
28
|
+
- Renamed all stale `search_engine_parser` references to `search_parser` across config, docs, and CI workflows.
|
|
29
|
+
- Fixed `pyproject.toml`: CLI entry point, hatch build target, and mypy `files` path all now point to `src/search_parser`.
|
|
30
|
+
- Synced `__version__.py` to match `pyproject.toml`.
|
|
31
|
+
- Added author and publisher metadata (`linksc`, `hello@link.sc`) to `pyproject.toml`.
|
|
32
|
+
- Updated package description with explicit search engine names and output formats for better discoverability.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## [0.2.0] - 2026-02-20
|
|
37
|
+
|
|
10
38
|
### Added
|
|
11
39
|
|
|
12
40
|
- `SearchResults.to_json(indent=2)` — serialize results directly to a JSON string without going through `SearchParser`.
|
|
@@ -29,6 +57,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
29
57
|
- `SearchResults.people_also_search: list[SearchResult]` — related search items (new)
|
|
30
58
|
- `SearchResults.related_products: list[SearchResult]` — product ad suggestions (new)
|
|
31
59
|
|
|
60
|
+
### Fixed
|
|
61
|
+
|
|
62
|
+
- **Security:** `SearchEngineDetector._check_url_patterns` replaced substring `in href` checks with `urlparse` hostname parsing against an explicit allowlist (`_ALLOWED_HOSTS`), preventing false positives from URLs like `http://evil.com/google.com`.
|
|
63
|
+
- **Security:** `DuckDuckGoParser.can_parse` replaced `"duckduckgo.com" in href` substring check with `urlparse` hostname comparison (exact match or `.duckduckgo.com` subdomain).
|
|
64
|
+
- **CI:** Added explicit `permissions: contents: read` blocks to `test.yml`, `lint.yml`, and `coverage-badge.yml` workflows to follow least-privilege GITHUB_TOKEN scoping.
|
|
65
|
+
|
|
32
66
|
### Migration
|
|
33
67
|
|
|
34
68
|
If you were filtering `results.results` by `result_type`, update your code:
|
|
@@ -62,5 +96,8 @@ sponsored = results.sponsored # list[SearchResult]
|
|
|
62
96
|
- CI/CD workflows for testing, linting, coverage, and publishing.
|
|
63
97
|
- Pre-commit hooks for ruff and mypy.
|
|
64
98
|
|
|
65
|
-
[Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.
|
|
99
|
+
[Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.4.0...HEAD
|
|
100
|
+
[0.4.0]: https://github.com/getlinksc/search-parser/compare/v0.3.0...v0.4.0
|
|
101
|
+
[0.3.0]: https://github.com/getlinksc/search-parser/compare/v0.2.0...v0.3.0
|
|
102
|
+
[0.2.0]: https://github.com/getlinksc/search-parser/compare/v0.1.0...v0.2.0
|
|
66
103
|
[0.1.0]: https://github.com/getlinksc/search-parser/releases/tag/v0.1.0
|
|
@@ -15,16 +15,16 @@ Using `uv` for fast, modern Python package management.
|
|
|
15
15
|
```bash
|
|
16
16
|
uv sync --all-extras # Install/sync all dependencies
|
|
17
17
|
uv run pytest # Run tests
|
|
18
|
-
uv run pytest --cov=
|
|
18
|
+
uv run pytest --cov=search_parser --cov-report=term # Tests with coverage
|
|
19
19
|
uv run ruff check . # Lint code
|
|
20
20
|
uv run ruff format . # Format code
|
|
21
|
-
uv run mypy src/
|
|
21
|
+
uv run mypy src/search_parser # Type check
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
## Adding a New Search Engine Parser
|
|
25
|
-
1. Create `src/
|
|
25
|
+
1. Create `src/search_parser/parsers/engine_name.py`
|
|
26
26
|
2. Implement class extending `BaseParser` with `engine_name`, `parse()`, and `can_parse()` methods
|
|
27
|
-
3. Register in `src/
|
|
27
|
+
3. Register in `src/search_parser/parsers/__init__.py`
|
|
28
28
|
4. Add HTML test fixtures in `tests/fixtures/engine_name/`
|
|
29
29
|
5. Write unit tests in `tests/unit/test_engine_name_parser.py`
|
|
30
30
|
|
|
@@ -45,7 +45,7 @@ Adding support for a new search engine is straightforward. Follow these steps:
|
|
|
45
45
|
|
|
46
46
|
### Step 1: Create the parser module
|
|
47
47
|
|
|
48
|
-
Create a new file at `src/
|
|
48
|
+
Create a new file at `src/search_parser/parsers/<engine_name>.py`:
|
|
49
49
|
|
|
50
50
|
```python
|
|
51
51
|
"""Parser for <EngineName> search results."""
|
|
@@ -56,7 +56,7 @@ from typing import Any
|
|
|
56
56
|
|
|
57
57
|
from bs4 import BeautifulSoup, Tag
|
|
58
58
|
|
|
59
|
-
from
|
|
59
|
+
from search_parser.engines.base import BaseParser, SearchResult
|
|
60
60
|
|
|
61
61
|
|
|
62
62
|
class EnginNameParser(BaseParser):
|
|
@@ -84,10 +84,10 @@ class EnginNameParser(BaseParser):
|
|
|
84
84
|
|
|
85
85
|
### Step 2: Register the parser
|
|
86
86
|
|
|
87
|
-
Add your parser to the engine registry in `src/
|
|
87
|
+
Add your parser to the engine registry in `src/search_parser/parsers/__init__.py`:
|
|
88
88
|
|
|
89
89
|
```python
|
|
90
|
-
from
|
|
90
|
+
from search_parser.engines.engine_name import EngineNameParser
|
|
91
91
|
|
|
92
92
|
ENGINES: list[type[BaseParser]] = [
|
|
93
93
|
GoogleParser,
|
|
@@ -118,7 +118,7 @@ Create `tests/engines/test_<engine_name>.py`:
|
|
|
118
118
|
import json
|
|
119
119
|
from pathlib import Path
|
|
120
120
|
|
|
121
|
-
from
|
|
121
|
+
from search_parser.engines.engine_name import EngineNameParser
|
|
122
122
|
|
|
123
123
|
FIXTURES = Path(__file__).parent.parent / "fixtures" / "<engine_name>"
|
|
124
124
|
|
|
@@ -176,7 +176,7 @@ Key style rules:
|
|
|
176
176
|
uv run pytest
|
|
177
177
|
|
|
178
178
|
# Run with coverage
|
|
179
|
-
uv run pytest --cov=
|
|
179
|
+
uv run pytest --cov=search_parser --cov-report=term-missing
|
|
180
180
|
|
|
181
181
|
# Run a specific test file
|
|
182
182
|
uv run pytest tests/engines/test_google.py
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: search-parser
|
|
3
|
-
Version: 0.0
|
|
4
|
-
Summary: Parse
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or dict — with auto-detection
|
|
5
5
|
Project-URL: Homepage, https://github.com/getlinksc/search-parser
|
|
6
6
|
Project-URL: Documentation, https://getlinksc.github.io/search-parser
|
|
7
7
|
Project-URL: Repository, https://github.com/getlinksc/search-parser
|
|
8
8
|
Project-URL: Issues, https://github.com/getlinksc/search-parser/issues
|
|
9
9
|
Project-URL: Changelog, https://github.com/getlinksc/search-parser/blob/main/CHANGELOG.md
|
|
10
|
-
Author-email:
|
|
10
|
+
Author-email: linksc <hello@link.sc>
|
|
11
11
|
License: Apache-2.0
|
|
12
12
|
License-File: LICENSE
|
|
13
13
|
Keywords: bing,duckduckgo,google,parser,scraping,search
|
|
@@ -51,7 +51,7 @@ Description-Content-Type: text/markdown
|
|
|
51
51
|
[](https://github.com/astral-sh/ruff)
|
|
52
52
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
53
53
|
|
|
54
|
-
**Parse
|
|
54
|
+
**Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or Python dict — with automatic search engine detection.**
|
|
55
55
|
|
|
56
56
|
`search-parser` takes raw HTML from Google, Bing, and DuckDuckGo and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
|
|
57
57
|
|
|
@@ -60,7 +60,7 @@ Description-Content-Type: text/markdown
|
|
|
60
60
|
## Quick Start
|
|
61
61
|
|
|
62
62
|
```python
|
|
63
|
-
from
|
|
63
|
+
from search_parser import SearchParser
|
|
64
64
|
|
|
65
65
|
parser = SearchParser()
|
|
66
66
|
html = open("google_results.html").read()
|
|
@@ -120,6 +120,8 @@ pip install search-parser
|
|
|
120
120
|
| What People Are Saying | `people_saying` | ✓ | — | — |
|
|
121
121
|
| People Also Search For | `people_also_search` | ✓ | — | — |
|
|
122
122
|
| Related Products & Services | `related_products` | ✓ | — | — |
|
|
123
|
+
| Jobs | `jobs` | ✓ | — | — |
|
|
124
|
+
| Discussions and forums | `discussions` | ✓ | — | — |
|
|
123
125
|
|
|
124
126
|
---
|
|
125
127
|
|
|
@@ -161,6 +163,15 @@ for ad in data["sponsored"]:
|
|
|
161
163
|
for product in data["related_products"]:
|
|
162
164
|
print(product["title"])
|
|
163
165
|
|
|
166
|
+
# Jobs (title, metadata["company"], metadata["location"])
|
|
167
|
+
for job in data["jobs"]:
|
|
168
|
+
print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
|
|
169
|
+
|
|
170
|
+
# Discussions (title, url, description, metadata["source"])
|
|
171
|
+
for disc in data["discussions"]:
|
|
172
|
+
print(disc["title"], disc["url"])
|
|
173
|
+
print(disc["metadata"]["source"])
|
|
174
|
+
|
|
164
175
|
# Metadata
|
|
165
176
|
print(data["search_engine"]) # "google"
|
|
166
177
|
print(data["query"]) # "python web scraping"
|
|
@@ -173,7 +184,7 @@ print(data["detection_confidence"]) # 0.95
|
|
|
173
184
|
When you need the typed `SearchResults` object instead of a dict, call the engine parser directly. The model exposes `to_json()` and `to_markdown()` convenience methods:
|
|
174
185
|
|
|
175
186
|
```python
|
|
176
|
-
from
|
|
187
|
+
from search_parser.parsers.google import GoogleParser
|
|
177
188
|
|
|
178
189
|
parser = GoogleParser()
|
|
179
190
|
results = parser.parse(html) # returns SearchResults
|
|
@@ -181,7 +192,7 @@ results = parser.parse(html) # returns SearchResults
|
|
|
181
192
|
# Typed access — no dict key lookups
|
|
182
193
|
print(results.query)
|
|
183
194
|
print(results.total_results)
|
|
184
|
-
print(len(results.results))
|
|
195
|
+
print(len(results.results)) # organic count
|
|
185
196
|
|
|
186
197
|
if results.featured_snippet:
|
|
187
198
|
print(results.featured_snippet.title)
|
|
@@ -197,9 +208,9 @@ for post in results.people_saying:
|
|
|
197
208
|
print(post.title, post.url)
|
|
198
209
|
|
|
199
210
|
# Convert to JSON or Markdown directly on the model
|
|
200
|
-
json_str
|
|
201
|
-
json_str
|
|
202
|
-
md_str
|
|
211
|
+
json_str = results.to_json()
|
|
212
|
+
json_str = results.to_json(indent=4) # custom indent
|
|
213
|
+
md_str = results.to_markdown()
|
|
203
214
|
```
|
|
204
215
|
|
|
205
216
|
---
|
|
@@ -244,6 +255,31 @@ md_str = results.to_markdown()
|
|
|
244
255
|
"people_saying": [],
|
|
245
256
|
"people_also_search": [],
|
|
246
257
|
"related_products": [],
|
|
258
|
+
"jobs": [
|
|
259
|
+
{
|
|
260
|
+
"title": "Global Supply Chain Director",
|
|
261
|
+
"url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
|
|
262
|
+
"description": null,
|
|
263
|
+
"position": 0,
|
|
264
|
+
"result_type": "job",
|
|
265
|
+
"metadata": {
|
|
266
|
+
"company": "InterSources, Inc.",
|
|
267
|
+
"location": "San Jose, CA • via Ladders"
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
],
|
|
271
|
+
"discussions": [
|
|
272
|
+
{
|
|
273
|
+
"title": "Being considered for Director of Supply Chain",
|
|
274
|
+
"url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
|
|
275
|
+
"description": "I work for a mid-sized company as a Procurement Manager...",
|
|
276
|
+
"position": 0,
|
|
277
|
+
"result_type": "discussion",
|
|
278
|
+
"metadata": {
|
|
279
|
+
"source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
],
|
|
247
283
|
"detection_confidence": 0.95,
|
|
248
284
|
"parsed_at": "2026-02-21T00:00:00Z",
|
|
249
285
|
"metadata": {}
|
|
@@ -276,6 +312,26 @@ Web scraping is the process of extracting data from websites...
|
|
|
276
312
|
Learn how to scrape websites with Python...
|
|
277
313
|
|
|
278
314
|
**URL:** https://realpython.com/python-web-scraping/
|
|
315
|
+
|
|
316
|
+
---
|
|
317
|
+
|
|
318
|
+
## Jobs
|
|
319
|
+
|
|
320
|
+
### Global Supply Chain Director
|
|
321
|
+
|
|
322
|
+
**Company:** InterSources, Inc.
|
|
323
|
+
**Location:** San Jose, CA • via Ladders
|
|
324
|
+
**URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
|
|
325
|
+
|
|
326
|
+
---
|
|
327
|
+
|
|
328
|
+
## Discussions and Forums
|
|
329
|
+
|
|
330
|
+
### Being considered for Director of Supply Chain
|
|
331
|
+
|
|
332
|
+
I work for a mid-sized company as a Procurement Manager...
|
|
333
|
+
|
|
334
|
+
**URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
|
|
279
335
|
```
|
|
280
336
|
|
|
281
337
|
---
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
[](https://github.com/astral-sh/ruff)
|
|
9
9
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
10
10
|
|
|
11
|
-
**Parse
|
|
11
|
+
**Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or Python dict — with automatic search engine detection.**
|
|
12
12
|
|
|
13
13
|
`search-parser` takes raw HTML from Google, Bing, and DuckDuckGo and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
|
|
14
14
|
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
## Quick Start
|
|
18
18
|
|
|
19
19
|
```python
|
|
20
|
-
from
|
|
20
|
+
from search_parser import SearchParser
|
|
21
21
|
|
|
22
22
|
parser = SearchParser()
|
|
23
23
|
html = open("google_results.html").read()
|
|
@@ -77,6 +77,8 @@ pip install search-parser
|
|
|
77
77
|
| What People Are Saying | `people_saying` | ✓ | — | — |
|
|
78
78
|
| People Also Search For | `people_also_search` | ✓ | — | — |
|
|
79
79
|
| Related Products & Services | `related_products` | ✓ | — | — |
|
|
80
|
+
| Jobs | `jobs` | ✓ | — | — |
|
|
81
|
+
| Discussions and forums | `discussions` | ✓ | — | — |
|
|
80
82
|
|
|
81
83
|
---
|
|
82
84
|
|
|
@@ -118,6 +120,15 @@ for ad in data["sponsored"]:
|
|
|
118
120
|
for product in data["related_products"]:
|
|
119
121
|
print(product["title"])
|
|
120
122
|
|
|
123
|
+
# Jobs (title, metadata["company"], metadata["location"])
|
|
124
|
+
for job in data["jobs"]:
|
|
125
|
+
print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
|
|
126
|
+
|
|
127
|
+
# Discussions (title, url, description, metadata["source"])
|
|
128
|
+
for disc in data["discussions"]:
|
|
129
|
+
print(disc["title"], disc["url"])
|
|
130
|
+
print(disc["metadata"]["source"])
|
|
131
|
+
|
|
121
132
|
# Metadata
|
|
122
133
|
print(data["search_engine"]) # "google"
|
|
123
134
|
print(data["query"]) # "python web scraping"
|
|
@@ -130,7 +141,7 @@ print(data["detection_confidence"]) # 0.95
|
|
|
130
141
|
When you need the typed `SearchResults` object instead of a dict, call the engine parser directly. The model exposes `to_json()` and `to_markdown()` convenience methods:
|
|
131
142
|
|
|
132
143
|
```python
|
|
133
|
-
from
|
|
144
|
+
from search_parser.parsers.google import GoogleParser
|
|
134
145
|
|
|
135
146
|
parser = GoogleParser()
|
|
136
147
|
results = parser.parse(html) # returns SearchResults
|
|
@@ -138,7 +149,7 @@ results = parser.parse(html) # returns SearchResults
|
|
|
138
149
|
# Typed access — no dict key lookups
|
|
139
150
|
print(results.query)
|
|
140
151
|
print(results.total_results)
|
|
141
|
-
print(len(results.results))
|
|
152
|
+
print(len(results.results)) # organic count
|
|
142
153
|
|
|
143
154
|
if results.featured_snippet:
|
|
144
155
|
print(results.featured_snippet.title)
|
|
@@ -154,9 +165,9 @@ for post in results.people_saying:
|
|
|
154
165
|
print(post.title, post.url)
|
|
155
166
|
|
|
156
167
|
# Convert to JSON or Markdown directly on the model
|
|
157
|
-
json_str
|
|
158
|
-
json_str
|
|
159
|
-
md_str
|
|
168
|
+
json_str = results.to_json()
|
|
169
|
+
json_str = results.to_json(indent=4) # custom indent
|
|
170
|
+
md_str = results.to_markdown()
|
|
160
171
|
```
|
|
161
172
|
|
|
162
173
|
---
|
|
@@ -201,6 +212,31 @@ md_str = results.to_markdown()
|
|
|
201
212
|
"people_saying": [],
|
|
202
213
|
"people_also_search": [],
|
|
203
214
|
"related_products": [],
|
|
215
|
+
"jobs": [
|
|
216
|
+
{
|
|
217
|
+
"title": "Global Supply Chain Director",
|
|
218
|
+
"url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
|
|
219
|
+
"description": null,
|
|
220
|
+
"position": 0,
|
|
221
|
+
"result_type": "job",
|
|
222
|
+
"metadata": {
|
|
223
|
+
"company": "InterSources, Inc.",
|
|
224
|
+
"location": "San Jose, CA • via Ladders"
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
],
|
|
228
|
+
"discussions": [
|
|
229
|
+
{
|
|
230
|
+
"title": "Being considered for Director of Supply Chain",
|
|
231
|
+
"url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
|
|
232
|
+
"description": "I work for a mid-sized company as a Procurement Manager...",
|
|
233
|
+
"position": 0,
|
|
234
|
+
"result_type": "discussion",
|
|
235
|
+
"metadata": {
|
|
236
|
+
"source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
],
|
|
204
240
|
"detection_confidence": 0.95,
|
|
205
241
|
"parsed_at": "2026-02-21T00:00:00Z",
|
|
206
242
|
"metadata": {}
|
|
@@ -233,6 +269,26 @@ Web scraping is the process of extracting data from websites...
|
|
|
233
269
|
Learn how to scrape websites with Python...
|
|
234
270
|
|
|
235
271
|
**URL:** https://realpython.com/python-web-scraping/
|
|
272
|
+
|
|
273
|
+
---
|
|
274
|
+
|
|
275
|
+
## Jobs
|
|
276
|
+
|
|
277
|
+
### Global Supply Chain Director
|
|
278
|
+
|
|
279
|
+
**Company:** InterSources, Inc.
|
|
280
|
+
**Location:** San Jose, CA • via Ladders
|
|
281
|
+
**URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
|
|
282
|
+
|
|
283
|
+
---
|
|
284
|
+
|
|
285
|
+
## Discussions and Forums
|
|
286
|
+
|
|
287
|
+
### Being considered for Director of Supply Chain
|
|
288
|
+
|
|
289
|
+
I work for a mid-sized company as a Procurement Manager...
|
|
290
|
+
|
|
291
|
+
**URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
|
|
236
292
|
```
|
|
237
293
|
|
|
238
294
|
---
|
|
@@ -4,16 +4,16 @@ This guide walks you through adding support for a new search engine.
|
|
|
4
4
|
|
|
5
5
|
## Step 1: Create the Parser
|
|
6
6
|
|
|
7
|
-
Create `src/
|
|
7
|
+
Create `src/search_parser/parsers/myengine.py`:
|
|
8
8
|
|
|
9
9
|
```python
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
from bs4 import BeautifulSoup, Tag
|
|
13
13
|
|
|
14
|
-
from
|
|
15
|
-
from
|
|
16
|
-
from
|
|
14
|
+
from search_parser.core.models import SearchResult, SearchResults
|
|
15
|
+
from search_parser.parsers.base import BaseParser
|
|
16
|
+
from search_parser.utils import clean_text, make_soup
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class MyEngineParser(BaseParser):
|
|
@@ -58,14 +58,14 @@ class MyEngineParser(BaseParser):
|
|
|
58
58
|
|
|
59
59
|
- `results` must contain **organic results only**. Sponsored ads go in `sponsored`, featured snippets in `featured_snippet`, etc.
|
|
60
60
|
- Return an empty `SearchResults` (with `results=[]`) on malformed HTML — never raise.
|
|
61
|
-
- Use `clean_text()` from `
|
|
61
|
+
- Use `clean_text()` from `search_parser.utils` to normalize whitespace.
|
|
62
62
|
|
|
63
63
|
## Step 2: Register the Parser
|
|
64
64
|
|
|
65
|
-
Add to `src/
|
|
65
|
+
Add to `src/search_parser/parsers/__init__.py`:
|
|
66
66
|
|
|
67
67
|
```python
|
|
68
|
-
from
|
|
68
|
+
from search_parser.parsers.myengine import MyEngineParser
|
|
69
69
|
|
|
70
70
|
PARSER_REGISTRY: dict[str, type[BaseParser]] = {
|
|
71
71
|
"google": GoogleParser,
|
|
@@ -105,5 +105,5 @@ Run the full test suite and linting before submitting:
|
|
|
105
105
|
```bash
|
|
106
106
|
uv run pytest
|
|
107
107
|
uv run ruff check .
|
|
108
|
-
uv run mypy src/
|
|
108
|
+
uv run mypy src/search_parser
|
|
109
109
|
```
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
## SearchParser
|
|
4
4
|
|
|
5
|
-
:::
|
|
5
|
+
::: search_parser.SearchParser
|
|
6
6
|
options:
|
|
7
7
|
show_root_heading: true
|
|
8
8
|
members:
|
|
@@ -10,28 +10,28 @@
|
|
|
10
10
|
|
|
11
11
|
## Models
|
|
12
12
|
|
|
13
|
-
:::
|
|
13
|
+
::: search_parser.core.models.SearchResult
|
|
14
14
|
options:
|
|
15
15
|
show_root_heading: true
|
|
16
16
|
|
|
17
|
-
:::
|
|
17
|
+
::: search_parser.core.models.SearchResults
|
|
18
18
|
options:
|
|
19
19
|
show_root_heading: true
|
|
20
20
|
|
|
21
21
|
## Parsers
|
|
22
22
|
|
|
23
|
-
:::
|
|
23
|
+
::: search_parser.parsers.base.BaseParser
|
|
24
24
|
options:
|
|
25
25
|
show_root_heading: true
|
|
26
26
|
|
|
27
27
|
## Formatters
|
|
28
28
|
|
|
29
|
-
:::
|
|
29
|
+
::: search_parser.formatters.base.BaseFormatter
|
|
30
30
|
options:
|
|
31
31
|
show_root_heading: true
|
|
32
32
|
|
|
33
33
|
## Exceptions
|
|
34
34
|
|
|
35
|
-
:::
|
|
35
|
+
::: search_parser.exceptions
|
|
36
36
|
options:
|
|
37
37
|
show_root_heading: true
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
```python
|
|
6
6
|
import json
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from
|
|
8
|
+
from search_parser import SearchParser
|
|
9
9
|
|
|
10
10
|
parser = SearchParser()
|
|
11
11
|
|
|
@@ -22,8 +22,8 @@ for html_file in Path("html_files").glob("*.html"):
|
|
|
22
22
|
```python
|
|
23
23
|
import csv
|
|
24
24
|
import io
|
|
25
|
-
from
|
|
26
|
-
from
|
|
25
|
+
from search_parser.core.models import SearchResults
|
|
26
|
+
from search_parser.formatters.base import BaseFormatter
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
class CSVFormatter(BaseFormatter):
|
|
@@ -39,7 +39,7 @@ class CSVFormatter(BaseFormatter):
|
|
|
39
39
|
Use it with the engine parser directly (which returns a `SearchResults` object):
|
|
40
40
|
|
|
41
41
|
```python
|
|
42
|
-
from
|
|
42
|
+
from search_parser.parsers.google import GoogleParser
|
|
43
43
|
|
|
44
44
|
results = GoogleParser().parse(html)
|
|
45
45
|
csv_output = CSVFormatter().format(results)
|
|
@@ -50,7 +50,7 @@ csv_output = CSVFormatter().format(results)
|
|
|
50
50
|
Each result type has its own field on `SearchResults` — no filtering by `result_type` needed:
|
|
51
51
|
|
|
52
52
|
```python
|
|
53
|
-
from
|
|
53
|
+
from search_parser.parsers.google import GoogleParser
|
|
54
54
|
|
|
55
55
|
results = GoogleParser().parse(html)
|
|
56
56
|
|
|
@@ -110,8 +110,8 @@ prompt = f"Summarize these search results:\n\n{results.to_markdown()}"
|
|
|
110
110
|
## Error Handling
|
|
111
111
|
|
|
112
112
|
```python
|
|
113
|
-
from
|
|
114
|
-
from
|
|
113
|
+
from search_parser import SearchParser
|
|
114
|
+
from search_parser.exceptions import (
|
|
115
115
|
SearchEngineDetectionError,
|
|
116
116
|
ParserNotFoundError,
|
|
117
117
|
ParseError,
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
## Parsing Search Results
|
|
4
4
|
|
|
5
5
|
```python
|
|
6
|
-
from
|
|
6
|
+
from search_parser import SearchParser
|
|
7
7
|
|
|
8
8
|
parser = SearchParser()
|
|
9
9
|
|
|
@@ -30,13 +30,13 @@ When you use an engine parser directly, the returned `SearchResults` object has
|
|
|
30
30
|
`to_json()` and `to_markdown()` methods:
|
|
31
31
|
|
|
32
32
|
```python
|
|
33
|
-
from
|
|
33
|
+
from search_parser.parsers.google import GoogleParser
|
|
34
34
|
|
|
35
35
|
results = GoogleParser().parse(html)
|
|
36
36
|
|
|
37
|
-
json_str = results.to_json()
|
|
37
|
+
json_str = results.to_json() # same as JSONFormatter().format(results)
|
|
38
38
|
json_str = results.to_json(indent=4) # custom indentation
|
|
39
|
-
md_str
|
|
39
|
+
md_str = results.to_markdown() # same as MarkdownFormatter().format(results)
|
|
40
40
|
```
|
|
41
41
|
|
|
42
42
|
## Accessing Result Types
|