search-parser 0.3.0__tar.gz → 0.4.0__tar.gz

{search_parser-0.3.0 → search_parser-0.4.0}/.github/workflows/codeql.yml

@@ -57,7 +57,7 @@ jobs:
         # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     # Add any setup steps before running the `github/codeql-action/init` action.
     # This includes steps like installing compilers or runtimes (`actions/setup-node`

{search_parser-0.3.0 → search_parser-0.4.0}/.github/workflows/publish.yml

@@ -3,6 +3,7 @@ name: Publish to PyPI
 on:
   release:
     types: [published]
+  workflow_dispatch:
 
 jobs:
   deploy:

{search_parser-0.3.0 → search_parser-0.4.0}/.github/workflows/test.yml

@@ -41,7 +41,7 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Upload coverage HTML artifact
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         if: matrix.python-version == '3.11'
         with:
           name: coverage-report

{search_parser-0.3.0 → search_parser-0.4.0}/.gitignore

@@ -48,4 +48,5 @@ Thumbs.db
 
 # uv
 
-search_parser_prompt.md
+search_parser_prompt.md
+

{search_parser-0.3.0 → search_parser-0.4.0}/CHANGELOG.md

@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [0.4.0] - 2026-03-05
+
+### Added
+
+- **Google parser: Jobs** — new `jobs` field on `SearchResults` containing job listings from the Google Jobs widget. Each entry has `result_type="job"` with `metadata["company"]` and `metadata["location"]` fields. Jobs are never included in the organic `results` list.
+- **Google parser: Discussions and forums** — new `discussions` field on `SearchResults` containing entries from the "Discussions and forums" widget. Each entry has `result_type="discussion"` with a `description` excerpt and `metadata["source"]` (platform, forum, comment count, and date).
+- New `result_type` values `"job"` and `"discussion"` added to the `SearchResult` model's Literal type.
+- Markdown formatter now renders `## Jobs` and `## Discussions and Forums` sections when those fields are present.
+- Unit tests for jobs and discussions parsing using the `supply-chain-director-jobs.html` fixture.
+
+---
+
 ## [0.3.0] - 2026-02-21
 
 ### Changed
@@ -16,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Renamed all stale `search_engine_parser` references to `search_parser` across config, docs, and CI workflows.
 - Fixed `pyproject.toml`: CLI entry point, hatch build target, and mypy `files` path all now point to `src/search_parser`.
 - Synced `__version__.py` to match `pyproject.toml`.
+- Added author and publisher metadata (`linksc`, `hello@link.sc`) to `pyproject.toml`.
+- Updated package description with explicit search engine names and output formats for better discoverability.
 
 ---
 
@@ -82,6 +96,8 @@ sponsored = results.sponsored         # list[SearchResult]
 - CI/CD workflows for testing, linting, coverage, and publishing.
 - Pre-commit hooks for ruff and mypy.
 
-[Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.2.0...HEAD
+[Unreleased]: https://github.com/getlinksc/search-parser/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/getlinksc/search-parser/compare/v0.3.0...v0.4.0
+[0.3.0]: https://github.com/getlinksc/search-parser/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/getlinksc/search-parser/compare/v0.1.0...v0.2.0
 [0.1.0]: https://github.com/getlinksc/search-parser/releases/tag/v0.1.0

{search_parser-0.3.0 → search_parser-0.4.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: search-parser
-Version: 0.3.0
+Version: 0.4.0
 Summary: Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or dict — with auto-detection
 Project-URL: Homepage, https://github.com/getlinksc/search-parser
 Project-URL: Documentation, https://getlinksc.github.io/search-parser
 Project-URL: Repository, https://github.com/getlinksc/search-parser
 Project-URL: Issues, https://github.com/getlinksc/search-parser/issues
 Project-URL: Changelog, https://github.com/getlinksc/search-parser/blob/main/CHANGELOG.md
-Author-email: Your Name <you@example.com>
+Author-email: linksc <hello@link.sc>
 License: Apache-2.0
 License-File: LICENSE
 Keywords: bing,duckduckgo,google,parser,scraping,search
@@ -120,6 +120,8 @@ pip install search-parser
 | What People Are Saying | `people_saying` | ✓ | — | — |
 | People Also Search For | `people_also_search` | ✓ | — | — |
 | Related Products & Services | `related_products` | ✓ | — | — |
+| Jobs | `jobs` | ✓ | — | — |
+| Discussions and forums | `discussions` | ✓ | — | — |
 
 ---
 
@@ -161,6 +163,15 @@ for ad in data["sponsored"]:
 for product in data["related_products"]:
     print(product["title"])
 
+# Jobs (title, metadata["company"], metadata["location"])
+for job in data["jobs"]:
+    print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
+
+# Discussions (title, url, description, metadata["source"])
+for disc in data["discussions"]:
+    print(disc["title"], disc["url"])
+    print(disc["metadata"]["source"])
+
 # Metadata
 print(data["search_engine"])        # "google"
 print(data["query"])                # "python web scraping"
@@ -244,6 +255,31 @@ md_str = results.to_markdown()
   "people_saying": [],
   "people_also_search": [],
   "related_products": [],
+  "jobs": [
+    {
+      "title": "Global Supply Chain Director",
+      "url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
+      "description": null,
+      "position": 0,
+      "result_type": "job",
+      "metadata": {
+        "company": "InterSources, Inc.",
+        "location": "San Jose, CA  •  via Ladders"
+      }
+    }
+  ],
+  "discussions": [
+    {
+      "title": "Being considered for Director of Supply Chain",
+      "url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
+      "description": "I work for a mid-sized company as a Procurement Manager...",
+      "position": 0,
+      "result_type": "discussion",
+      "metadata": {
+        "source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
+      }
+    }
+  ],
   "detection_confidence": 0.95,
   "parsed_at": "2026-02-21T00:00:00Z",
   "metadata": {}
@@ -276,6 +312,26 @@ Web scraping is the process of extracting data from websites...
 Learn how to scrape websites with Python...
 
 **URL:** https://realpython.com/python-web-scraping/
+
+---
+
+## Jobs
+
+### Global Supply Chain Director
+
+**Company:** InterSources, Inc.
+**Location:** San Jose, CA  •  via Ladders
+**URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
+
+---
+
+## Discussions and Forums
+
+### Being considered for Director of Supply Chain
+
+I work for a mid-sized company as a Procurement Manager...
+
+**URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
 ```
 
 ---

{search_parser-0.3.0 → search_parser-0.4.0}/README.md

@@ -77,6 +77,8 @@ pip install search-parser
 | What People Are Saying | `people_saying` | ✓ | — | — |
 | People Also Search For | `people_also_search` | ✓ | — | — |
 | Related Products & Services | `related_products` | ✓ | — | — |
+| Jobs | `jobs` | ✓ | — | — |
+| Discussions and forums | `discussions` | ✓ | — | — |
 
 ---
 
@@ -118,6 +120,15 @@ for ad in data["sponsored"]:
 for product in data["related_products"]:
     print(product["title"])
 
+# Jobs (title, metadata["company"], metadata["location"])
+for job in data["jobs"]:
+    print(job["title"], job["metadata"]["company"], job["metadata"]["location"])
+
+# Discussions (title, url, description, metadata["source"])
+for disc in data["discussions"]:
+    print(disc["title"], disc["url"])
+    print(disc["metadata"]["source"])
+
 # Metadata
 print(data["search_engine"])        # "google"
 print(data["query"])                # "python web scraping"
@@ -201,6 +212,31 @@ md_str = results.to_markdown()
   "people_saying": [],
   "people_also_search": [],
   "related_products": [],
+  "jobs": [
+    {
+      "title": "Global Supply Chain Director",
+      "url": "https://www.google.com/search?q=%22Supply+Chain+Director&udm=8",
+      "description": null,
+      "position": 0,
+      "result_type": "job",
+      "metadata": {
+        "company": "InterSources, Inc.",
+        "location": "San Jose, CA  •  via Ladders"
+      }
+    }
+  ],
+  "discussions": [
+    {
+      "title": "Being considered for Director of Supply Chain",
+      "url": "https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/",
+      "description": "I work for a mid-sized company as a Procurement Manager...",
+      "position": 0,
+      "result_type": "discussion",
+      "metadata": {
+        "source": "Reddit · r/supplychain · 10+ comments · 1 year ago"
+      }
+    }
+  ],
   "detection_confidence": 0.95,
   "parsed_at": "2026-02-21T00:00:00Z",
   "metadata": {}
@@ -233,6 +269,26 @@ Web scraping is the process of extracting data from websites...
 Learn how to scrape websites with Python...
 
 **URL:** https://realpython.com/python-web-scraping/
+
+---
+
+## Jobs
+
+### Global Supply Chain Director
+
+**Company:** InterSources, Inc.
+**Location:** San Jose, CA  •  via Ladders
+**URL:** https://www.google.com/search?q=%22Supply+Chain+Director&udm=8
+
+---
+
+## Discussions and Forums
+
+### Being considered for Director of Supply Chain
+
+I work for a mid-sized company as a Procurement Manager...
+
+**URL:** https://www.reddit.com/r/supplychain/comments/1ib0c1a/being_considered_for_director_of_supply_chain/
 ```
 
 ---

{search_parser-0.3.0 → search_parser-0.4.0}/pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
 
 [project]
 name = "search-parser"
-version = "0.3.0"
+version = "0.4.0"
 description = "Parse Google, Bing, and DuckDuckGo HTML search results into JSON, Markdown, or dict — with auto-detection"
 readme = "README.md"
 requires-python = ">=3.10"
 license = {text = "Apache-2.0"}
 authors = [
-    {name = "Your Name", email = "you@example.com"}
+    {name = "linksc", email = "hello@link.sc"}
 ]
 keywords = ["search", "parser", "scraping", "google", "bing", "duckduckgo"]
 classifiers = [

{search_parser-0.3.0 → search_parser-0.4.0}/src/search_parser/__version__.py

@@ -1,3 +1,3 @@
 """Version information."""
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"

{search_parser-0.3.0 → search_parser-0.4.0}/src/search_parser/core/models.py

@@ -27,6 +27,8 @@ class SearchResult(BaseModel):
         "people_saying",
         "people_also_search",
         "related_products",
+        "job",
+        "discussion",
     ] = "organic"
     metadata: dict[str, object] = Field(default_factory=dict)
 
@@ -53,6 +55,8 @@ class SearchResults(BaseModel):
     people_saying: list[SearchResult] = Field(default_factory=list)
     people_also_search: list[SearchResult] = Field(default_factory=list)
     related_products: list[SearchResult] = Field(default_factory=list)
+    jobs: list[SearchResult] = Field(default_factory=list)
+    discussions: list[SearchResult] = Field(default_factory=list)
 
     detection_confidence: float = Field(ge=0.0, le=1.0)
     parsed_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))

{search_parser-0.3.0 → search_parser-0.4.0}/src/search_parser/formatters/markdown_formatter.py

@@ -70,6 +70,18 @@ class MarkdownFormatter(BaseFormatter):
             for result in news:
                 lines.extend(self._format_result(result))
 
+        if results.jobs:
+            lines.append("## Jobs")
+            lines.append("")
+            for result in results.jobs:
+                lines.extend(self._format_job(result))
+
+        if results.discussions:
+            lines.append("## Discussions and Forums")
+            lines.append("")
+            for result in results.discussions:
+                lines.extend(self._format_result(result))
+
         lines.append("---")
         lines.append("")
         lines.append(f"*Parsed with search-engine-parser v{__version__}*")
@@ -102,6 +114,24 @@ class MarkdownFormatter(BaseFormatter):
         lines.append("")
         return lines
 
+    def _format_job(self, result: SearchResult) -> list[str]:
+        """Format a job listing result."""
+        lines: list[str] = []
+        lines.append(f"### {result.title}")
+        lines.append("")
+        if result.metadata.get("company"):
+            lines.append(f"**Company:** {result.metadata['company']}")
+        if result.metadata.get("location"):
+            lines.append(f"**Location:** {result.metadata['location']}")
+        if result.metadata.get("salary"):
+            lines.append(f"**Salary:** {result.metadata['salary']}")
+        if result.metadata.get("employment_type"):
+            lines.append(f"**Type:** {result.metadata['employment_type']}")
+        if result.url:
+            lines.append(f"**URL:** {result.url}")
+        lines.append("")
+        return lines
+
     def _format_result(self, result: SearchResult) -> list[str]:
         """Format a generic result."""
         lines: list[str] = []

{search_parser-0.3.0 → search_parser-0.4.0}/src/search_parser/parsers/google.py

@@ -65,6 +65,8 @@ class GoogleParser(BaseParser):
             people_saying=self._extract_people_saying(soup),
             people_also_search=self._extract_people_also_search(soup),
             related_products=self._extract_related_products(soup),
+            jobs=self._extract_jobs(soup),
+            discussions=self._extract_discussions(soup),
             detection_confidence=confidence,
         )
 
@@ -344,6 +346,111 @@ class GoogleParser(BaseParser):
                 )
         return results
 
+    def _extract_jobs(self, soup: BeautifulSoup) -> list[SearchResult]:
+        """Extract Google Jobs widget listings.
+
+        Jobs are rendered inside a dedicated widget (``div.iYivne``) and are
+        intentionally kept out of the organic ``results`` list.
+        """
+        results: list[SearchResult] = []
+        widget = soup.find("div", class_="iYivne")
+        if not isinstance(widget, Tag):
+            return results
+        for item in widget.find_all("div", class_="EimVGf"):
+            if not isinstance(item, Tag):
+                continue
+            title_div = item.find("div", class_="tNxQIb")
+            company_div = item.find("div", class_="a3jPc")
+            location_div = item.find("div", class_="FqK3wc")
+            link = item.find("a", class_="MQUd2b")
+            title = clean_text(title_div.get_text()) if isinstance(title_div, Tag) else ""
+            if not title:
+                continue
+            url = str(link.get("href", "")) if isinstance(link, Tag) else ""
+            company = clean_text(company_div.get_text()) if isinstance(company_div, Tag) else None
+            location = clean_text(location_div.get_text()) if isinstance(location_div, Tag) else None
+
+            # Salary has the extra QZEeP class; employment type and other tags share K3eUK only
+            salary_div = item.find("div", class_="QZEeP")
+            salary = clean_text(salary_div.get_text()) if isinstance(salary_div, Tag) else None
+
+            # Employment type: K3eUK entries that are NOT the salary and NOT a date ("X days ago")
+            employment_type: str | None = None
+            for tag_div in item.find_all("div", class_="K3eUK"):
+                if not isinstance(tag_div, Tag):
+                    continue
+                if "QZEeP" in (tag_div.get("class") or []):
+                    continue  # skip salary
+                text = clean_text(tag_div.get_text())
+                if text and not re.search(r"\d+\s+\w+\s+ago", text):
+                    employment_type = text
+                    break
+
+            benefits = [
+                clean_text(b.get_text())
+                for b in item.find_all("div", class_="HvHIEc")
+                if isinstance(b, Tag) and clean_text(b.get_text())
+            ]
+
+            metadata: dict[str, object] = {}
+            if company:
+                metadata["company"] = company
+            if location:
+                metadata["location"] = location
+            if salary:
+                metadata["salary"] = salary
+            if employment_type:
+                metadata["employment_type"] = employment_type
+            if benefits:
+                metadata["benefits"] = benefits
+            results.append(
+                SearchResult(
+                    title=title,
+                    url=url,
+                    description=None,
+                    position=0,
+                    result_type="job",
+                    metadata=metadata,
+                )
+            )
+        return results
+
+    def _extract_discussions(self, soup: BeautifulSoup) -> list[SearchResult]:
+        """Extract 'Discussions and forums' widget entries."""
+        results: list[SearchResult] = []
+        header = soup.find("div", class_="DuxCpf")
+        if not isinstance(header, Tag):
+            return results
+        container = header.find_parent("div", class_="Ww4FFb")
+        if not isinstance(container, Tag):
+            return results
+        for item in container.find_all("div", class_="xYkm8c"):
+            if not isinstance(item, Tag):
+                continue
+            title_div = item.find("div", class_="j29v2b")
+            link = item.find("a", class_="KYg7td")
+            meta_div = item.find("div", class_="LbKnXb")
+            desc_div = item.find("div", class_="bCOlv")
+            title = clean_text(title_div.get_text()) if isinstance(title_div, Tag) else ""
+            if not title:
+                continue
+            url = str(link.get("href", "")) if isinstance(link, Tag) else ""
+            description = clean_text(desc_div.get_text()) if isinstance(desc_div, Tag) else None
+            metadata: dict[str, object] = {}
+            if isinstance(meta_div, Tag):
+                metadata["source"] = clean_text(meta_div.get_text())
+            results.append(
+                SearchResult(
+                    title=title,
+                    url=url,
+                    description=description,
+                    position=0,
+                    result_type="discussion",
+                    metadata=metadata,
+                )
+            )
+        return results
+
     def _extract_featured_snippet(self, soup: BeautifulSoup) -> SearchResult | None:
         """Extract featured snippet if present."""
         snippet_container = soup.find("div", class_="xpdopen")

{search_parser-0.3.0 → search_parser-0.4.0}/tests/conftest.py

@@ -62,3 +62,8 @@ def duckduckgo_github_repos_html() -> str:
 @pytest.fixture()
 def google_web_scraping_html() -> str:
     return (FIXTURES_DIR / "google" / "search_python_web_scraping.html").read_text()
+
+
+@pytest.fixture()
+def google_supply_chain_jobs_html() -> str:
+    return (FIXTURES_DIR / "google" / "supply-chain-director-jobs.html").read_text()