search-parser 0.0.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {search_parser-0.0.2 → search_parser-0.4.0}/.github/pull_request_template.md +1 -1
  2. {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/codeql.yml +1 -1
  3. {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/coverage-badge.yml +1 -1
  4. {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/lint.yml +1 -1
  5. {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/publish.yml +1 -0
  6. {search_parser-0.0.2 → search_parser-0.4.0}/.github/workflows/test.yml +2 -2
  7. {search_parser-0.0.2 → search_parser-0.4.0}/.gitignore +2 -1
  8. {search_parser-0.0.2 → search_parser-0.4.0}/CHANGELOG.md +38 -1
  9. {search_parser-0.0.2 → search_parser-0.4.0}/CLAUDE.md +4 -4
  10. {search_parser-0.0.2 → search_parser-0.4.0}/CONTRIBUTING.md +6 -6
  11. {search_parser-0.0.2 → search_parser-0.4.0}/PKG-INFO +66 -10
  12. {search_parser-0.0.2 → search_parser-0.4.0}/README.md +63 -7
  13. {search_parser-0.0.2 → search_parser-0.4.0}/docs/adding_search_engine.md +8 -8
  14. {search_parser-0.0.2 → search_parser-0.4.0}/docs/api_reference.md +6 -6
  15. {search_parser-0.0.2 → search_parser-0.4.0}/docs/examples/advanced_usage.md +7 -7
  16. {search_parser-0.0.2 → search_parser-0.4.0}/docs/examples/basic_usage.md +4 -4
  17. {search_parser-0.0.2 → search_parser-0.4.0}/docs/getting_started.md +6 -6
  18. {search_parser-0.0.2 → search_parser-0.4.0}/docs/index.md +3 -3
  19. {search_parser-0.0.2 → search_parser-0.4.0}/examples/basic_parsing.py +2 -2
  20. {search_parser-0.0.2 → search_parser-0.4.0}/examples/batch_processing.py +1 -1
  21. {search_parser-0.0.2 → search_parser-0.4.0}/examples/custom_formatter.py +3 -3
  22. {search_parser-0.0.2 → search_parser-0.4.0}/pyproject.toml +6 -6
  23. search_parser-0.4.0/src/search_parser/__init__.py +12 -0
  24. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/__version__.py +1 -1
  25. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/cli.py +2 -2
  26. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/core/detector.py +1 -1
  27. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/core/models.py +5 -1
  28. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/core/parser.py +7 -7
  29. search_parser-0.4.0/src/search_parser/formatters/__init__.py +11 -0
  30. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/formatters/base.py +1 -1
  31. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/formatters/json_formatter.py +2 -2
  32. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/formatters/markdown_formatter.py +33 -3
  33. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/__init__.py +4 -4
  34. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/base.py +1 -1
  35. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/bing.py +3 -3
  36. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/duckduckgo.py +3 -3
  37. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/parsers/google.py +110 -3
  38. {search_parser-0.0.2 → search_parser-0.4.0}/tests/conftest.py +5 -0
  39. search_parser-0.4.0/tests/fixtures/google/supply-chain-director-jobs.html +92 -0
  40. {search_parser-0.0.2 → search_parser-0.4.0}/tests/test_cli.py +1 -1
  41. {search_parser-0.0.2 → search_parser-0.4.0}/tests/test_coverage_boost.py +9 -9
  42. {search_parser-0.0.2 → search_parser-0.4.0}/tests/test_formatters.py +3 -3
  43. {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/test_bing_parser.py +2 -2
  44. {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/test_detector.py +1 -1
  45. {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/test_duckduckgo_parser.py +2 -2
  46. {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/test_google_parser.py +83 -2
  47. search_parser-0.0.2/src/search_engine_parser/__init__.py +0 -12
  48. search_parser-0.0.2/src/search_engine_parser/formatters/__init__.py +0 -11
  49. {search_parser-0.0.2 → search_parser-0.4.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  50. {search_parser-0.0.2 → search_parser-0.4.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  51. {search_parser-0.0.2 → search_parser-0.4.0}/.github/dependabot.yml +0 -0
  52. {search_parser-0.0.2 → search_parser-0.4.0}/.pre-commit-config.yaml +0 -0
  53. {search_parser-0.0.2 → search_parser-0.4.0}/CODE_OF_CONDUCT.md +0 -0
  54. {search_parser-0.0.2 → search_parser-0.4.0}/LICENSE +0 -0
  55. {search_parser-0.0.2 → search_parser-0.4.0}/docs/CNAME +0 -0
  56. {search_parser-0.0.2 → search_parser-0.4.0}/docs/contributing.md +0 -0
  57. {search_parser-0.0.2 → search_parser-0.4.0}/mkdocs.yml +0 -0
  58. {search_parser-0.0.2 → search_parser-0.4.0}/scripts/update_fixtures.py +0 -0
  59. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/core/__init__.py +0 -0
  60. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/exceptions.py +0 -0
  61. {search_parser-0.0.2/src/search_engine_parser → search_parser-0.4.0/src/search_parser}/utils.py +0 -0
  62. {search_parser-0.0.2 → search_parser-0.4.0}/tests/__init__.py +0 -0
  63. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/bing/organic_results.html +0 -0
  64. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/bing/search_github_repos.html +0 -0
  65. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/duckduckgo/organic_results.html +0 -0
  66. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/duckduckgo/search_github_repos.html +0 -0
  67. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/featured_snippet.html +0 -0
  68. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/knowledge_panel.html +0 -0
  69. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/need_turn_on_javascript.html +0 -0
  70. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/organic_results.html +0 -0
  71. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/search_best_employee_scheduling_app.html +0 -0
  72. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/search_github_repos.html +0 -0
  73. {search_parser-0.0.2 → search_parser-0.4.0}/tests/fixtures/google/search_python_web_scraping.html +0 -0
  74. {search_parser-0.0.2 → search_parser-0.4.0}/tests/unit/__init__.py +0 -0
  75. {search_parser-0.0.2 → search_parser-0.4.0}/uv.lock +0 -0
@@ -15,7 +15,7 @@ Brief description of the changes.
15
15
  - [ ] I have added tests that prove my fix/feature works
16
16
  - [ ] All new and existing tests pass (`uv run pytest`)
17
17
  - [ ] Linting passes (`uv run ruff check .`)
18
- - [ ] Type checking passes (`uv run mypy src/search_engine_parser`)
18
+ - [ ] Type checking passes (`uv run mypy src/search_parser`)
19
19
  - [ ] I have updated documentation as needed
20
20
 
21
21
  ## Test Plan
@@ -57,7 +57,7 @@ jobs:
57
57
  # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
58
58
  steps:
59
59
  - name: Checkout repository
60
- uses: actions/checkout@v4
60
+ uses: actions/checkout@v6
61
61
 
62
62
  # Add any setup steps before running the `github/codeql-action/init` action.
63
63
  # This includes steps like installing compilers or runtimes (`actions/setup-node`
@@ -23,7 +23,7 @@ jobs:
23
23
 
24
24
  - name: Run tests with coverage
25
25
  run: |
26
- uv run pytest --cov=search_engine_parser --cov-report=term --cov-report=json
26
+ uv run pytest --cov=search_parser --cov-report=term --cov-report=json
27
27
  COVERAGE=$(python -c "import json; print(int(float(json.load(open('coverage.json'))['totals']['percent_covered_display'])))")
28
28
  echo "COVERAGE=${COVERAGE}%" >> $GITHUB_ENV
29
29
  if [ "$COVERAGE" -ge 90 ]; then echo "COVERAGE_COLOR=brightgreen" >> $GITHUB_ENV;
@@ -30,4 +30,4 @@ jobs:
30
30
  run: uv run ruff format --check .
31
31
 
32
32
  - name: Run mypy
33
- run: uv run mypy src/search_engine_parser
33
+ run: uv run mypy src/search_parser
@@ -3,6 +3,7 @@ name: Publish to PyPI
3
3
  on:
4
4
  release:
5
5
  types: [published]
6
+ workflow_dispatch:
6
7
 
7
8
  jobs:
8
9
  deploy:
@@ -30,7 +30,7 @@ jobs:
30
30
  run: uv sync --all-extras
31
31
 
32
32
  - name: Run tests with coverage
33
- run: uv run pytest --cov=search_engine_parser --cov-report=xml --cov-report=term --cov-report=html
33
+ run: uv run pytest --cov=search_parser --cov-report=xml --cov-report=term --cov-report=html
34
34
 
35
35
  - name: Upload coverage to Codecov
36
36
  uses: codecov/codecov-action@v5
@@ -41,7 +41,7 @@ jobs:
41
41
  token: ${{ secrets.CODECOV_TOKEN }}
42
42
 
43
43
  - name: Upload coverage HTML artifact
44
- uses: actions/upload-artifact@v6
44
+ uses: actions/upload-artifact@v7
45
45
  if: matrix.python-version == '3.11'
46
46
  with:
47
47
  name: coverage-report
@@ -48,4 +48,5 @@ Thumbs.db
48
48
 
49
49
  # uv
50
50
 
51
- search_parser_prompt.md
51
+ search_parser_prompt.md
52
+
@@ -7,6 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ---
11
+
12
+ ## [0.4.0] - 2026-03-05
13
+
14
+ ### Added
15
+
16
+ - **Google parser: Jobs** — new `jobs` field on `SearchResults` containing job listings from the Google Jobs widget. Each entry has `result_type="job"` with `metadata["company"]` and `metadata["location"]` fields. Jobs are never included in the organic `results` list.
17
+ - **Google parser: Discussions and forums** — new `discussions` field on `SearchResults` containing entries from the "Discussions and forums" widget. Each entry has `result_type="discussion"` with a `description` excerpt and `metadata["source"]` (platform, forum, comment count, and date).
18
+ - New `result_type` values `"job"` and `"discussion"` added to the `SearchResult` model's Literal type.
19
+ - Markdown formatter now renders `## Jobs` and `## Discussions and Forums` sections when those fields are present.
20
+ - Unit tests for jobs and discussions parsing using the `supply-chain-director-jobs.html` fixture.
21
+
22
+ ---
23
+
24
+ ## [0.3.0] - 2026-02-21
25
+
26
+ ### Changed
27
+
28
+ - Renamed all stale `search_engine_parser` references to `search_parser` across config, docs, and CI workflows.
29
+ - Fixed `pyproject.toml`: CLI entry point, hatch build target, and mypy `files` path all now point to `src/search_parser`.
30
+ - Synced `__version__.py` to match `pyproject.toml`.
31
+ - Added author and publisher metadata (`linksc`, `hello@link.sc`) to `pyproject.toml`.
32
+ - Updated package description with explicit search engine names and output formats for better discoverability.
33
+
34
+ ---
35
+
36
+ ## [0.2.0] - 2026-02-20
37
+
10
38
  ### Added
11
39
 
12
40
  - `SearchResults.to_json(indent=2)` — serialize results directly to a JSON string without going through `SearchParser`.
@@ -29,6 +57,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
29
57
  - `SearchResults.people_also_search: list[SearchResult]` — related search items (new)
30
58
  - `SearchResults.related_products: list[SearchResult]` — product ad suggestions (new)
31
59
 
60
+ ### Fixed
61
+
62
+ - **Security:** `SearchEngineDetector._check_url_patterns` replaced substring `in href` checks with `urlparse` hostname parsing against an explicit allowlist (`_ALLOWED_HOSTS`), preventing false positives from URLs like `http://evil.com/google.com`.
63
+ - **Security:** `DuckDuckGoParser.can_parse` replaced `"duckduckgo.com" in href` substring check with `urlparse` hostname comparison (exact match or `.duckduckgo.com` subdomain).
64
+ - **CI:** Added explicit `permissions: contents: read` blocks to `test.yml`, `lint.yml`, and `coverage-badge.yml` workflows to follow least-privilege GITHUB_TOKEN scoping.
65
+
32
66
  ### Migration
33
67
 
34
68
  If you were filtering `results.results` by `result_type`, update your code:
@@ -62,5 +96,8 @@ sponsored = results.sponsored # list[SearchResult]
62
96
  - CI/CD workflows for testing, linting, coverage, and publishing.
63
97
  - Pre-commit hooks for ruff and mypy.
64
98
 
65
- [Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.1.0...HEAD
99
+ [Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.4.0...HEAD
100
+ [0.4.0]: https://github.com/getlinksc/search-parser/compare/v0.3.0...v0.4.0
101
+ [0.3.0]: https://github.com/getlinksc/search-parser/compare/v0.2.0...v0.3.0
102
+ [0.2.0]: https://github.com/getlinksc/search-parser/compare/v0.1.0...v0.2.0
66
103
  [0.1.0]: https://github.com/getlinksc/search-parser/releases/tag/v0.1.0
@@ -15,16 +15,16 @@ Using `uv` for fast, modern Python package management.
15
15
  ```bash
16
16
  uv sync --all-extras # Install/sync all dependencies
17
17
  uv run pytest # Run tests
18
- uv run pytest --cov=search_engine_parser --cov-report=term # Tests with coverage
18
+ uv run pytest --cov=search_parser --cov-report=term # Tests with coverage
19
19
  uv run ruff check . # Lint code
20
20
  uv run ruff format . # Format code
21
- uv run mypy src/search_engine_parser # Type check
21
+ uv run mypy src/search_parser # Type check
22
22
  ```
23
23
 
24
24
  ## Adding a New Search Engine Parser
25
- 1. Create `src/search_engine_parser/parsers/engine_name.py`
25
+ 1. Create `src/search_parser/parsers/engine_name.py`
26
26
  2. Implement class extending `BaseParser` with `engine_name`, `parse()`, and `can_parse()` methods
27
- 3. Register in `src/search_engine_parser/parsers/__init__.py`
27
+ 3. Register in `src/search_parser/parsers/__init__.py`
28
28
  4. Add HTML test fixtures in `tests/fixtures/engine_name/`
29
29
  5. Write unit tests in `tests/unit/test_engine_name_parser.py`
30
30
 
@@ -45,7 +45,7 @@ Adding support for a new search engine is straightforward. Follow these steps:
45
45
 
46
46
  ### Step 1: Create the parser module
47
47
 
48
- Create a new file at `src/search_engine_parser/engines/<engine_name>.py`:
48
+ Create a new file at `src/search_parser/parsers/<engine_name>.py`:
49
49
 
50
50
  ```python
51
51
  """Parser for <EngineName> search results."""
@@ -56,7 +56,7 @@ from typing import Any
56
56
 
57
57
  from bs4 import BeautifulSoup, Tag
58
58
 
59
- from search_engine_parser.engines.base import BaseParser, SearchResult
59
+ from search_parser.engines.base import BaseParser, SearchResult
60
60
 
61
61
 
62
62
  class EnginNameParser(BaseParser):
@@ -84,10 +84,10 @@ class EnginNameParser(BaseParser):
84
84
 
85
85
  ### Step 2: Register the parser
86
86
 
87
- Add your parser to the engine registry in `src/search_engine_parser/engines/__init__.py`:
87
+ Add your parser to the engine registry in `src/search_parser/parsers/__init__.py`:
88
88
 
89
89
  ```python
90
- from search_engine_parser.engines.engine_name import EngineNameParser
90
+ from search_parser.engines.engine_name import EngineNameParser
91
91
 
92
92
  ENGINES: list[type[BaseParser]] = [
93
93
  GoogleParser,
@@ -118,7 +118,7 @@ Create `tests/engines/test_<engine_name>.py`:
118
118
  import json
119
119
  from pathlib import Path
120
120
 
121
- from search_engine_parser.engines.engine_name import EngineNameParser
121
+ from search_parser.engines.engine_name import EngineNameParser
122
122
 
123
123
  FIXTURES = Path(__file__).parent.parent / "fixtures" / "<engine_name>"
124
124
 
@@ -176,7 +176,7 @@ Key style rules:
176
176
  uv run pytest
177
177
 
178
178
  # Run with coverage
179
- uv run pytest --cov=search_engine_parser --cov-report=term-missing
179
+ uv run pytest --cov=search_parser --cov-report=term-missing
180
180
 
181
181
  # Run a specific test file
182
182
  uv run pytest tests/engines/test_google.py
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: search-parser
3
- Version: 0.0.2
4
- Summary: Parse search engine HTML results into structured data
3
+ Version: 0.4.0
4
+ Summary: Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or dict — with auto-detection
5
5
  Project-URL: Homepage, https://github.com/getlinksc/search-parser
6
6
  Project-URL: Documentation, https://getlinksc.github.io/search-parser
7
7
  Project-URL: Repository, https://github.com/getlinksc/search-parser
8
8
  Project-URL: Issues, https://github.com/getlinksc/search-parser/issues
9
9
  Project-URL: Changelog, https://github.com/getlinksc/search-parser/blob/main/CHANGELOG.md
10
- Author-email: Your Name <you@example.com>
10
+ Author-email: linksc <hello@link.sc>
11
11
  License: Apache-2.0
12
12
  License-File: LICENSE
13
13
  Keywords: bing,duckduckgo,google,parser,scraping,search
@@ -51,7 +51,7 @@ Description-Content-Type: text/markdown
51
51
  [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
52
52
  [![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
53
53
 
54
- **Parse search engine HTML results into structured data (JSON, Markdown) with auto-detection.**
54
+ **Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or Python dict — with automatic search engine detection.**
55
55
 
56
56
  `search-parser` takes raw HTML from Google, Bing, and DuckDuckGo and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
57
57
 
@@ -60,7 +60,7 @@ Description-Content-Type: text/markdown
60
60
  ## Quick Start
61
61
 
62
62
  ```python
63
- from search_engine_parser import SearchParser
63
+ from search_parser import SearchParser
64
64
 
65
65
  parser = SearchParser()
66
66
  html = open("google_results.html").read()
@@ -120,6 +120,8 @@ pip install search-parser
120
120
  | What People Are Saying | `people_saying` | ✓ | — | — |
121
121
  | People Also Search For | `people_also_search` | ✓ | — | — |
122
122
  | Related Products & Services | `related_products` | ✓ | — | — |
123
+ | Jobs | `jobs` | ✓ | — | — |
124
+ | Discussions and forums | `discussions` | ✓ | — | — |
123
125
 
124
126
  ---
125
127
 
@@ -161,6 +163,15 @@ for ad in data["sponsored"]:
161
163
  for product in data["related_products"]:
162
164
  print(product["title"])
163
165
 
166
+ # Jobs (title, metadata["company"], metadata["location"])
167
+ for job in data["jobs"]:
168
+ print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
169
+
170
+ # Discussions (title, url, description, metadata["source"])
171
+ for disc in data["discussions"]:
172
+ print(disc["title"], disc["url"])
173
+ print(disc["metadata"]["source"])
174
+
164
175
  # Metadata
165
176
  print(data["search_engine"]) # "google"
166
177
  print(data["query"]) # "python web scraping"
@@ -173,7 +184,7 @@ print(data["detection_confidence"]) # 0.95
173
184
  When you need the typed `SearchResults` object instead of a dict, call the engine parser directly. The model exposes `to_json()` and `to_markdown()` convenience methods:
174
185
 
175
186
  ```python
176
- from search_engine_parser.parsers.google import GoogleParser
187
+ from search_parser.parsers.google import GoogleParser
177
188
 
178
189
  parser = GoogleParser()
179
190
  results = parser.parse(html) # returns SearchResults
@@ -181,7 +192,7 @@ results = parser.parse(html) # returns SearchResults
181
192
  # Typed access — no dict key lookups
182
193
  print(results.query)
183
194
  print(results.total_results)
184
- print(len(results.results)) # organic count
195
+ print(len(results.results)) # organic count
185
196
 
186
197
  if results.featured_snippet:
187
198
  print(results.featured_snippet.title)
@@ -197,9 +208,9 @@ for post in results.people_saying:
197
208
  print(post.title, post.url)
198
209
 
199
210
  # Convert to JSON or Markdown directly on the model
200
- json_str = results.to_json()
201
- json_str = results.to_json(indent=4) # custom indent
202
- md_str = results.to_markdown()
211
+ json_str = results.to_json()
212
+ json_str = results.to_json(indent=4) # custom indent
213
+ md_str = results.to_markdown()
203
214
  ```
204
215
 
205
216
  ---
@@ -244,6 +255,31 @@ md_str = results.to_markdown()
244
255
  "people_saying": [],
245
256
  "people_also_search": [],
246
257
  "related_products": [],
258
+ "jobs": [
259
+ {
260
+ "title": "Global Supply Chain Director",
261
+ "url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
262
+ "description": null,
263
+ "position": 0,
264
+ "result_type": "job",
265
+ "metadata": {
266
+ "company": "InterSources, Inc.",
267
+ "location": "San Jose, CA • via Ladders"
268
+ }
269
+ }
270
+ ],
271
+ "discussions": [
272
+ {
273
+ "title": "Being considered for Director of Supply Chain",
274
+ "url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
275
+ "description": "I work for a mid-sized company as a Procurement Manager...",
276
+ "position": 0,
277
+ "result_type": "discussion",
278
+ "metadata": {
279
+ "source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
280
+ }
281
+ }
282
+ ],
247
283
  "detection_confidence": 0.95,
248
284
  "parsed_at": "2026-02-21T00:00:00Z",
249
285
  "metadata": {}
@@ -276,6 +312,26 @@ Web scraping is the process of extracting data from websites...
276
312
  Learn how to scrape websites with Python...
277
313
 
278
314
  **URL:** https://realpython.com/python-web-scraping/
315
+
316
+ ---
317
+
318
+ ## Jobs
319
+
320
+ ### Global Supply Chain Director
321
+
322
+ **Company:** InterSources, Inc.
323
+ **Location:** San Jose, CA • via Ladders
324
+ **URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
325
+
326
+ ---
327
+
328
+ ## Discussions and Forums
329
+
330
+ ### Being considered for Director of Supply Chain
331
+
332
+ I work for a mid-sized company as a Procurement Manager...
333
+
334
+ **URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
279
335
  ```
280
336
 
281
337
  ---
@@ -8,7 +8,7 @@
8
8
  [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
9
9
  [![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
10
10
 
11
- **Parse search engine HTML results into structured data (JSON, Markdown) with auto-detection.**
11
+ **Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or Python dict — with automatic search engine detection.**
12
12
 
13
13
  `search-parser` takes raw HTML from Google, Bing, and DuckDuckGo and extracts every result type — organic results, featured snippets, AI Overviews, People Also Ask, sponsored ads, and more — into clean, typed Python objects. It auto-detects the search engine from the HTML, so you never have to specify which parser to use.
14
14
 
@@ -17,7 +17,7 @@
17
17
  ## Quick Start
18
18
 
19
19
  ```python
20
- from search_engine_parser import SearchParser
20
+ from search_parser import SearchParser
21
21
 
22
22
  parser = SearchParser()
23
23
  html = open("google_results.html").read()
@@ -77,6 +77,8 @@ pip install search-parser
77
77
  | What People Are Saying | `people_saying` | ✓ | — | — |
78
78
  | People Also Search For | `people_also_search` | ✓ | — | — |
79
79
  | Related Products & Services | `related_products` | ✓ | — | — |
80
+ | Jobs | `jobs` | ✓ | — | — |
81
+ | Discussions and forums | `discussions` | ✓ | — | — |
80
82
 
81
83
  ---
82
84
 
@@ -118,6 +120,15 @@ for ad in data["sponsored"]:
118
120
  for product in data["related_products"]:
119
121
  print(product["title"])
120
122
 
123
+ # Jobs (title, metadata["company"], metadata["location"])
124
+ for job in data["jobs"]:
125
+ print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
126
+
127
+ # Discussions (title, url, description, metadata["source"])
128
+ for disc in data["discussions"]:
129
+ print(disc["title"], disc["url"])
130
+ print(disc["metadata"]["source"])
131
+
121
132
  # Metadata
122
133
  print(data["search_engine"]) # "google"
123
134
  print(data["query"]) # "python web scraping"
@@ -130,7 +141,7 @@ print(data["detection_confidence"]) # 0.95
130
141
  When you need the typed `SearchResults` object instead of a dict, call the engine parser directly. The model exposes `to_json()` and `to_markdown()` convenience methods:
131
142
 
132
143
  ```python
133
- from search_engine_parser.parsers.google import GoogleParser
144
+ from search_parser.parsers.google import GoogleParser
134
145
 
135
146
  parser = GoogleParser()
136
147
  results = parser.parse(html) # returns SearchResults
@@ -138,7 +149,7 @@ results = parser.parse(html) # returns SearchResults
138
149
  # Typed access — no dict key lookups
139
150
  print(results.query)
140
151
  print(results.total_results)
141
- print(len(results.results)) # organic count
152
+ print(len(results.results)) # organic count
142
153
 
143
154
  if results.featured_snippet:
144
155
  print(results.featured_snippet.title)
@@ -154,9 +165,9 @@ for post in results.people_saying:
154
165
  print(post.title, post.url)
155
166
 
156
167
  # Convert to JSON or Markdown directly on the model
157
- json_str = results.to_json()
158
- json_str = results.to_json(indent=4) # custom indent
159
- md_str = results.to_markdown()
168
+ json_str = results.to_json()
169
+ json_str = results.to_json(indent=4) # custom indent
170
+ md_str = results.to_markdown()
160
171
  ```
161
172
 
162
173
  ---
@@ -201,6 +212,31 @@ md_str = results.to_markdown()
201
212
  "people_saying": [],
202
213
  "people_also_search": [],
203
214
  "related_products": [],
215
+ "jobs": [
216
+ {
217
+ "title": "Global Supply Chain Director",
218
+ "url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
219
+ "description": null,
220
+ "position": 0,
221
+ "result_type": "job",
222
+ "metadata": {
223
+ "company": "InterSources, Inc.",
224
+ "location": "San Jose, CA • via Ladders"
225
+ }
226
+ }
227
+ ],
228
+ "discussions": [
229
+ {
230
+ "title": "Being considered for Director of Supply Chain",
231
+ "url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
232
+ "description": "I work for a mid-sized company as a Procurement Manager...",
233
+ "position": 0,
234
+ "result_type": "discussion",
235
+ "metadata": {
236
+ "source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
237
+ }
238
+ }
239
+ ],
204
240
  "detection_confidence": 0.95,
205
241
  "parsed_at": "2026-02-21T00:00:00Z",
206
242
  "metadata": {}
@@ -233,6 +269,26 @@ Web scraping is the process of extracting data from websites...
233
269
  Learn how to scrape websites with Python...
234
270
 
235
271
  **URL:** https://realpython.com/python-web-scraping/
272
+
273
+ ---
274
+
275
+ ## Jobs
276
+
277
+ ### Global Supply Chain Director
278
+
279
+ **Company:** InterSources, Inc.
280
+ **Location:** San Jose, CA • via Ladders
281
+ **URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
282
+
283
+ ---
284
+
285
+ ## Discussions and Forums
286
+
287
+ ### Being considered for Director of Supply Chain
288
+
289
+ I work for a mid-sized company as a Procurement Manager...
290
+
291
+ **URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
236
292
  ```
237
293
 
238
294
  ---
@@ -4,16 +4,16 @@ This guide walks you through adding support for a new search engine.
4
4
 
5
5
  ## Step 1: Create the Parser
6
6
 
7
- Create `src/search_engine_parser/parsers/myengine.py`:
7
+ Create `src/search_parser/parsers/myengine.py`:
8
8
 
9
9
  ```python
10
10
  from __future__ import annotations
11
11
 
12
12
  from bs4 import BeautifulSoup, Tag
13
13
 
14
- from search_engine_parser.core.models import SearchResult, SearchResults
15
- from search_engine_parser.parsers.base import BaseParser
16
- from search_engine_parser.utils import clean_text, make_soup
14
+ from search_parser.core.models import SearchResult, SearchResults
15
+ from search_parser.parsers.base import BaseParser
16
+ from search_parser.utils import clean_text, make_soup
17
17
 
18
18
 
19
19
  class MyEngineParser(BaseParser):
@@ -58,14 +58,14 @@ class MyEngineParser(BaseParser):
58
58
 
59
59
  - `results` must contain **organic results only**. Sponsored ads go in `sponsored`, featured snippets in `featured_snippet`, etc.
60
60
  - Return an empty `SearchResults` (with `results=[]`) on malformed HTML — never raise.
61
- - Use `clean_text()` from `search_engine_parser.utils` to normalize whitespace.
61
+ - Use `clean_text()` from `search_parser.utils` to normalize whitespace.
62
62
 
63
63
  ## Step 2: Register the Parser
64
64
 
65
- Add to `src/search_engine_parser/parsers/__init__.py`:
65
+ Add to `src/search_parser/parsers/__init__.py`:
66
66
 
67
67
  ```python
68
- from search_engine_parser.parsers.myengine import MyEngineParser
68
+ from search_parser.parsers.myengine import MyEngineParser
69
69
 
70
70
  PARSER_REGISTRY: dict[str, type[BaseParser]] = {
71
71
  "google": GoogleParser,
@@ -105,5 +105,5 @@ Run the full test suite and linting before submitting:
105
105
  ```bash
106
106
  uv run pytest
107
107
  uv run ruff check .
108
- uv run mypy src/search_engine_parser
108
+ uv run mypy src/search_parser
109
109
  ```
@@ -2,7 +2,7 @@
2
2
 
3
3
  ## SearchParser
4
4
 
5
- ::: search_engine_parser.SearchParser
5
+ ::: search_parser.SearchParser
6
6
  options:
7
7
  show_root_heading: true
8
8
  members:
@@ -10,28 +10,28 @@
10
10
 
11
11
  ## Models
12
12
 
13
- ::: search_engine_parser.core.models.SearchResult
13
+ ::: search_parser.core.models.SearchResult
14
14
  options:
15
15
  show_root_heading: true
16
16
 
17
- ::: search_engine_parser.core.models.SearchResults
17
+ ::: search_parser.core.models.SearchResults
18
18
  options:
19
19
  show_root_heading: true
20
20
 
21
21
  ## Parsers
22
22
 
23
- ::: search_engine_parser.parsers.base.BaseParser
23
+ ::: search_parser.parsers.base.BaseParser
24
24
  options:
25
25
  show_root_heading: true
26
26
 
27
27
  ## Formatters
28
28
 
29
- ::: search_engine_parser.formatters.base.BaseFormatter
29
+ ::: search_parser.formatters.base.BaseFormatter
30
30
  options:
31
31
  show_root_heading: true
32
32
 
33
33
  ## Exceptions
34
34
 
35
- ::: search_engine_parser.exceptions
35
+ ::: search_parser.exceptions
36
36
  options:
37
37
  show_root_heading: true
@@ -5,7 +5,7 @@
5
5
  ```python
6
6
  import json
7
7
  from pathlib import Path
8
- from search_engine_parser import SearchParser
8
+ from search_parser import SearchParser
9
9
 
10
10
  parser = SearchParser()
11
11
 
@@ -22,8 +22,8 @@ for html_file in Path("html_files").glob("*.html"):
22
22
  ```python
23
23
  import csv
24
24
  import io
25
- from search_engine_parser.core.models import SearchResults
26
- from search_engine_parser.formatters.base import BaseFormatter
25
+ from search_parser.core.models import SearchResults
26
+ from search_parser.formatters.base import BaseFormatter
27
27
 
28
28
 
29
29
  class CSVFormatter(BaseFormatter):
@@ -39,7 +39,7 @@ class CSVFormatter(BaseFormatter):
39
39
  Use it with the engine parser directly (which returns a `SearchResults` object):
40
40
 
41
41
  ```python
42
- from search_engine_parser.parsers.google import GoogleParser
42
+ from search_parser.parsers.google import GoogleParser
43
43
 
44
44
  results = GoogleParser().parse(html)
45
45
  csv_output = CSVFormatter().format(results)
@@ -50,7 +50,7 @@ csv_output = CSVFormatter().format(results)
50
50
  Each result type has its own field on `SearchResults` — no filtering by `result_type` needed:
51
51
 
52
52
  ```python
53
- from search_engine_parser.parsers.google import GoogleParser
53
+ from search_parser.parsers.google import GoogleParser
54
54
 
55
55
  results = GoogleParser().parse(html)
56
56
 
@@ -110,8 +110,8 @@ prompt = f"Summarize these search results:\n\n{results.to_markdown()}"
110
110
  ## Error Handling
111
111
 
112
112
  ```python
113
- from search_engine_parser import SearchParser
114
- from search_engine_parser.exceptions import (
113
+ from search_parser import SearchParser
114
+ from search_parser.exceptions import (
115
115
  SearchEngineDetectionError,
116
116
  ParserNotFoundError,
117
117
  ParseError,
@@ -3,7 +3,7 @@
3
3
  ## Parsing Search Results
4
4
 
5
5
  ```python
6
- from search_engine_parser import SearchParser
6
+ from search_parser import SearchParser
7
7
 
8
8
  parser = SearchParser()
9
9
 
@@ -30,13 +30,13 @@ When you use an engine parser directly, the returned `SearchResults` object has
30
30
  `to_json()` and `to_markdown()` methods:
31
31
 
32
32
  ```python
33
- from search_engine_parser.parsers.google import GoogleParser
33
+ from search_parser.parsers.google import GoogleParser
34
34
 
35
35
  results = GoogleParser().parse(html)
36
36
 
37
- json_str = results.to_json() # same as JSONFormatter().format(results)
37
+ json_str = results.to_json() # same as JSONFormatter().format(results)
38
38
  json_str = results.to_json(indent=4) # custom indentation
39
- md_str = results.to_markdown() # same as MarkdownFormatter().format(results)
39
+ md_str = results.to_markdown() # same as MarkdownFormatter().format(results)
40
40
  ```
41
41
 
42
42
  ## Accessing Result Types