swe-scraper-icims 0.1.0__tar.gz

swe_scraper_icims-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Furkan Candar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

swe_scraper_icims-0.1.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: swe-scraper-icims
+Version: 0.1.0
+Summary: Experimental public iCIMS provider for swe-internship-scraper.
+Author: Furkan Candar
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: swe-internship-scraper<2,>=1.0.0rc1
+Dynamic: license-file
+
+# swe-scraper-icims
+
+Experimental public HTML provider for iCIMS job portals. It follows only links on
+the configured portal origin and stops at explicit `rel="next"` pagination. It
+does not authenticate, solve challenges, or bypass access controls.

swe_scraper_icims-0.1.0/README.md

@@ -0,0 +1,5 @@
+# swe-scraper-icims
+
+Experimental public HTML provider for iCIMS job portals. It follows only links on
+the configured portal origin and stops at explicit `rel="next"` pagination. It
+does not authenticate, solve challenges, or bypass access controls.

swe_scraper_icims-0.1.0/pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools>=77", "wheel>=0.43"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "swe-scraper-icims"
+version = "0.1.0"
+description = "Experimental public iCIMS provider for swe-internship-scraper."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{name = "Furkan Candar"}]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Programming Language :: Python :: 3",
+]
+dependencies = [
+  "swe-internship-scraper>=1.0.0rc1,<2",
+]
+
+[project.entry-points."swe_scraper.providers"]
+icims = "swe_scraper_icims:IcimsProvider"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

swe_scraper_icims-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

swe_scraper_icims-0.1.0/src/swe_scraper_icims/__init__.py

@@ -0,0 +1,246 @@
+"""Experimental, public-only iCIMS provider plugin."""
+
+from __future__ import annotations
+
+import html as html_module
+import json
+import re
+from collections.abc import Mapping
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import urljoin, urlsplit
+
+from swe_scraper.models import Job
+from swe_scraper.normalize import iso_datetime, normalize_locations
+from swe_scraper.providers.base import HttpClient, Target
+
+__version__ = "0.1.0"
+
+
+def _plain_text(value: object) -> str:
+    clean = re.sub(r"<[^>]+>", " ", str(value or ""))
+    return " ".join(html_module.unescape(clean).split())
+
+
+class _PortalParser(HTMLParser):
+    """Collect the small, explicit HTML surface used by public iCIMS pages."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.links: list[tuple[str, tuple[str, ...]]] = []
+        self.json_ld: list[str] = []
+        self._script_parts: list[str] | None = None
+        self._capture: str = ""
+        self._capture_depth = 0
+        self.text: dict[str, list[str]] = {
+            "title": [],
+            "description": [],
+            "location": [],
+        }
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        values = {key.casefold(): value or "" for key, value in attrs}
+        rel = tuple(values.get("rel", "").casefold().split())
+        if tag in {"a", "link"} and values.get("href"):
+            self.links.append((values["href"], rel))
+        if tag == "script" and values.get("type", "").casefold() == "application/ld+json":
+            self._script_parts = []
+
+        class_names = set(values.get("class", "").split())
+        itemprop = values.get("itemprop", "")
+        if tag == "h1" or class_names & {"iCIMS_Header", "header-text"}:
+            self._capture = "title"
+            self._capture_depth = 1
+        elif itemprop == "description" or class_names & {
+            "iCIMS_JobContent",
+            "job-description",
+        }:
+            self._capture = "description"
+            self._capture_depth = 1
+        elif itemprop == "jobLocation" or "iCIMS_JobHeaderField_location" in class_names:
+            self._capture = "location"
+            self._capture_depth = 1
+        elif self._capture:
+            self._capture_depth += 1
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag == "script" and self._script_parts is not None:
+            self.json_ld.append("".join(self._script_parts))
+            self._script_parts = None
+        if self._capture:
+            self._capture_depth -= 1
+            if self._capture_depth <= 0:
+                self._capture = ""
+
+    def handle_data(self, data: str) -> None:
+        if self._script_parts is not None:
+            self._script_parts.append(data)
+        if self._capture and data.strip():
+            self.text[self._capture].append(data)
+
+
+def _parse_html(value: str) -> _PortalParser:
+    parser = _PortalParser()
+    parser.feed(value)
+    parser.close()
+    return parser
+
+
+def _same_origin(left: str, right: str) -> bool:
+    first = urlsplit(left)
+    second = urlsplit(right)
+    return (
+        first.scheme.casefold(),
+        (first.hostname or "").casefold(),
+        first.port,
+    ) == (
+        second.scheme.casefold(),
+        (second.hostname or "").casefold(),
+        second.port,
+    )
+
+
+def _job_id(url: str) -> str:
+    match = re.search(r"/jobs/(\d+)(?:/|$)", urlsplit(url).path, re.IGNORECASE)
+    return match.group(1) if match else ""
+
+
+class IcimsProvider:
+    """Read public iCIMS HTML without attempting to bypass access controls."""
+
+    name = "icims"
+    required_options = ("search_url",)
+
+    def validate_target(self, target: Target) -> None:
+        search_url = str(target.options.get("search_url") or "").strip()
+        parsed = urlsplit(search_url)
+        if parsed.scheme != "https" or not parsed.hostname:
+            raise ValueError("iCIMS target requires an HTTPS search_url")
+        expected_host = target.slug.casefold().strip()
+        if (parsed.hostname or "").casefold() != expected_host:
+            raise ValueError("iCIMS search_url must use the same origin as the target slug")
+
+    def discover_page(self, html: str, current_url: str) -> tuple[tuple[str, ...], str]:
+        document = _parse_html(html)
+        links: set[str] = set()
+        for href, _rel in document.links:
+            candidate = urljoin(current_url, href)
+            if _same_origin(current_url, candidate) and _job_id(candidate):
+                links.add(candidate)
+
+        next_url = ""
+        for href, rel in document.links:
+            if "next" not in rel:
+                continue
+            candidate = urljoin(current_url, href)
+            if _same_origin(current_url, candidate):
+                next_url = candidate
+                break
+        return tuple(sorted(links)), next_url
+
+    @staticmethod
+    def _job_posting_json(html: str) -> Mapping[str, Any] | None:
+        document = _parse_html(html)
+        for script in document.json_ld:
+            try:
+                value = json.loads(script)
+            except (TypeError, json.JSONDecodeError):
+                continue
+            candidates = value if isinstance(value, list) else [value]
+            for candidate in candidates:
+                if not isinstance(candidate, Mapping):
+                    continue
+                graph = candidate.get("@graph")
+                nested = graph if isinstance(graph, list) else [candidate]
+                for item in nested:
+                    if isinstance(item, Mapping) and item.get("@type") == "JobPosting":
+                        return item
+        return None
+
+    def parse_job_page(self, html: str, url: str, target: Target) -> Job:
+        data = self._job_posting_json(html)
+        document = _parse_html(html)
+        if data is not None:
+            title = str(data.get("title") or "").strip()
+            company_raw = data.get("hiringOrganization")
+            company = (
+                str(company_raw.get("name") or "").strip()
+                if isinstance(company_raw, Mapping)
+                else target.name
+            )
+            identifier = data.get("identifier")
+            source_id = (
+                str(identifier.get("value") or "").strip()
+                if isinstance(identifier, Mapping)
+                else str(identifier or "").strip()
+            )
+            locations: list[str] = []
+            raw_locations = data.get("jobLocation")
+            for raw_location in (
+                raw_locations if isinstance(raw_locations, list) else [raw_locations]
+            ):
+                if not isinstance(raw_location, Mapping):
+                    continue
+                address = raw_location.get("address")
+                if not isinstance(address, Mapping):
+                    continue
+                parts = [
+                    str(address.get(key) or "").strip()
+                    for key in ("addressLocality", "addressRegion", "addressCountry")
+                ]
+                locations.append(", ".join(part for part in parts if part))
+            description = _plain_text(data.get("description"))
+            posted_at = iso_datetime(data.get("datePosted"))
+        else:
+            title = _plain_text(" ".join(document.text["title"]))
+            company = target.name
+            source_id = _job_id(url)
+            description = _plain_text(" ".join(document.text["description"]))
+            location = _plain_text(" ".join(document.text["location"]))
+            locations = [location] if location else []
+            posted_at = ""
+
+        source_id = source_id or _job_id(url)
+        if not title or not source_id:
+            raise ValueError(f"iCIMS job page is missing title or job id: {url}")
+        normalized_locations = normalize_locations(locations)
+        return Job(
+            id=f"icims:{target.slug}:{source_id}",
+            company=company or target.name,
+            title=title,
+            application_url=url,
+            provider=self.name,
+            source_job_id=source_id,
+            locations=normalized_locations,
+            posted_at=posted_at,
+            description=description,
+            remote="remote" in " ".join(normalized_locations).casefold(),
+            metadata={"portal": target.slug, "experimental": True},
+        )
+
+    def fetch(self, target: Target, client: HttpClient) -> list[Job]:
+        self.validate_target(target)
+        current_url = str(target.options["search_url"])
+        max_pages = int(target.options.get("max_pages", 10))
+        if not 1 <= max_pages <= 100:
+            raise ValueError("iCIMS max_pages must be between 1 and 100")
+        visited_pages: set[str] = set()
+        job_urls: set[str] = set()
+        for _ in range(max_pages):
+            if current_url in visited_pages:
+                raise RuntimeError("iCIMS pagination repeated a page")
+            visited_pages.add(current_url)
+            links, next_url = self.discover_page(client.get_text(current_url), current_url)
+            job_urls.update(links)
+            if not next_url:
+                break
+            current_url = next_url
+        else:
+            raise RuntimeError("iCIMS pagination exceeded max_pages")
+        return [
+            self.parse_job_page(client.get_text(url), url, target)
+            for url in sorted(job_urls)
+        ]
+
+
+__all__ = ["IcimsProvider", "__version__"]

swe_scraper_icims-0.1.0/src/swe_scraper_icims.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: swe-scraper-icims
+Version: 0.1.0
+Summary: Experimental public iCIMS provider for swe-internship-scraper.
+Author: Furkan Candar
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: swe-internship-scraper<2,>=1.0.0rc1
+Dynamic: license-file
+
+# swe-scraper-icims
+
+Experimental public HTML provider for iCIMS job portals. It follows only links on
+the configured portal origin and stops at explicit `rel="next"` pagination. It
+does not authenticate, solve challenges, or bypass access controls.

swe_scraper_icims-0.1.0/src/swe_scraper_icims.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+LICENSE
+README.md
+pyproject.toml
+src/swe_scraper_icims/__init__.py
+src/swe_scraper_icims.egg-info/PKG-INFO
+src/swe_scraper_icims.egg-info/SOURCES.txt
+src/swe_scraper_icims.egg-info/dependency_links.txt
+src/swe_scraper_icims.egg-info/entry_points.txt
+src/swe_scraper_icims.egg-info/requires.txt
+src/swe_scraper_icims.egg-info/top_level.txt
+tests/test_icims_provider.py

swe_scraper_icims-0.1.0/src/swe_scraper_icims.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

swe_scraper_icims-0.1.0/src/swe_scraper_icims.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[swe_scraper.providers]
+icims = swe_scraper_icims:IcimsProvider

swe_scraper_icims-0.1.0/src/swe_scraper_icims.egg-info/requires.txt

@@ -0,0 +1 @@
+swe-internship-scraper<2,>=1.0.0rc1

swe_scraper_icims-0.1.0/src/swe_scraper_icims.egg-info/top_level.txt

@@ -0,0 +1 @@
+swe_scraper_icims

swe_scraper_icims-0.1.0/tests/test_icims_provider.py

@@ -0,0 +1,186 @@
+import sys
+import unittest
+from pathlib import Path
+
+PLUGIN_ROOT = Path(__file__).resolve().parents[1]
+CORE_SRC = Path(__file__).resolve().parents[3] / "src"
+sys.path.insert(0, str(PLUGIN_ROOT / "src"))
+sys.path.insert(0, str(CORE_SRC))
+
+from swe_scraper_icims import IcimsProvider  # noqa: E402
+
+from swe_scraper.providers.base import Target  # noqa: E402
+
+
+class IcimsProviderTests(unittest.TestCase):
+    def test_discovers_same_origin_jobs_and_explicit_next_page(self):
+        html = """
+        <html><head><link rel="next" href="/jobs/search?pr=1"></head><body>
+          <a href="/jobs/123/software-engineer-intern/job">Role</a>
+          <a href="https://evil.example/jobs/999/job">External</a>
+        </body></html>
+        """
+        links, next_url = IcimsProvider().discover_page(
+            html, "https://careers-acme.icims.com/jobs/search"
+        )
+        self.assertEqual(
+            links,
+            ("https://careers-acme.icims.com/jobs/123/software-engineer-intern/job",),
+        )
+        self.assertEqual(next_url, "https://careers-acme.icims.com/jobs/search?pr=1")
+
+    def test_parses_json_ld_job_posting_before_dom_fallback(self):
+        html = """
+        <script type="application/ld+json">
+        {
+          "@type": "JobPosting",
+          "title": "Software Engineering Intern",
+          "description": "<p>Build reliable services.</p>",
+          "datePosted": "2026-09-01",
+          "jobLocation": {
+            "address": {"addressLocality": "New York", "addressRegion": "NY"}
+          },
+          "hiringOrganization": {"name": "Acme"},
+          "identifier": {"value": "123"}
+        }
+        </script>
+        """
+        target = Target(
+            "icims",
+            "Acme",
+            "careers-acme.icims.com",
+            {"search_url": "https://careers-acme.icims.com/jobs/search"},
+        )
+        parsed = IcimsProvider().parse_job_page(
+            html,
+            "https://careers-acme.icims.com/jobs/123/software-engineer-intern/job",
+            target,
+        )
+        self.assertEqual(parsed.title, "Software Engineering Intern")
+        self.assertEqual(parsed.source_job_id, "123")
+        self.assertEqual(parsed.locations, ("New York, NY",))
+        self.assertIn("Build reliable services", parsed.description)
+
+    def test_rejects_cross_origin_search_configuration(self):
+        target = Target(
+            "icims",
+            "Acme",
+            "careers-acme.icims.com",
+            {"search_url": "https://other.example/jobs/search"},
+        )
+        with self.assertRaisesRegex(ValueError, "same origin"):
+            IcimsProvider().validate_target(target)
+
+    def test_dom_fallback_and_nested_json_ld(self):
+        target = Target(
+            "icims",
+            "Acme",
+            "careers-acme.icims.com",
+            {"search_url": "https://careers-acme.icims.com/jobs/search"},
+        )
+        fallback = """
+        <h1>Platform Intern</h1>
+        <div itemprop="jobLocation">Remote - US</div>
+        <div itemprop="description"><p>Build platforms.</p></div>
+        """
+        job = IcimsProvider().parse_job_page(
+            fallback,
+            "https://careers-acme.icims.com/jobs/44/platform-intern/job",
+            target,
+        )
+        self.assertEqual(job.source_job_id, "44")
+        self.assertTrue(job.remote)
+        self.assertIn("Build platforms", job.description)
+
+        nested = """
+        <script type="application/ld+json">not-json</script>
+        <script type="application/ld+json">
+        {"@graph": [{"@type": "WebPage"}, {
+          "@type": "JobPosting", "title": "Security Intern",
+          "description": "Secure systems", "identifier": "45",
+          "jobLocation": [{"address": {
+            "addressLocality": "Austin", "addressRegion": "TX",
+            "addressCountry": "US"
+          }}]
+        }]}
+        </script>
+        """
+        job = IcimsProvider().parse_job_page(
+            nested,
+            "https://careers-acme.icims.com/jobs/45/security-intern/job",
+            target,
+        )
+        self.assertEqual(job.locations, ("Austin, TX, US",))
+
+    def test_fetch_follows_explicit_pages_and_rejects_incomplete_pagination(self):
+        search = "https://careers-acme.icims.com/jobs/search"
+        page_two = f"{search}?pr=1"
+        job_one = "https://careers-acme.icims.com/jobs/1/one/job"
+        job_two = "https://careers-acme.icims.com/jobs/2/two/job"
+        pages = {
+            search: f'<a href="{job_one}">One</a><a rel="next" href="{page_two}">Next</a>',
+            page_two: f'<a href="{job_two}">Two</a>',
+            job_one: (
+                '<script type="application/ld+json">'
+                '{"@type":"JobPosting","title":"Intern One","identifier":"1"}'
+                "</script>"
+            ),
+            job_two: (
+                '<script type="application/ld+json">'
+                '{"@type":"JobPosting","title":"Intern Two","identifier":"2"}'
+                "</script>"
+            ),
+        }
+
+        class Client:
+            def get_text(self, url, **kwargs):
+                return pages[url]
+
+        target = Target(
+            "icims",
+            "Acme",
+            "careers-acme.icims.com",
+            {"search_url": search},
+        )
+        jobs = IcimsProvider().fetch(target, Client())
+        self.assertEqual([job.source_job_id for job in jobs], ["1", "2"])
+
+        limited = Target(
+            "icims",
+            "Acme",
+            "careers-acme.icims.com",
+            {"search_url": search, "max_pages": 1},
+        )
+        with self.assertRaisesRegex(RuntimeError, "max_pages"):
+            IcimsProvider().fetch(limited, Client())
+
+    def test_validation_and_malformed_pages_fail_closed(self):
+        provider = IcimsProvider()
+        with self.assertRaisesRegex(ValueError, "HTTPS"):
+            provider.validate_target(
+                Target("icims", "Acme", "host", {"search_url": "http://host/jobs"})
+            )
+        with self.assertRaisesRegex(ValueError, "max_pages"):
+            provider.fetch(
+                Target(
+                    "icims",
+                    "Acme",
+                    "host.icims.com",
+                    {"search_url": "https://host.icims.com/jobs", "max_pages": 0},
+                ),
+                object(),
+            )
+        target = Target(
+            "icims",
+            "Acme",
+            "host.icims.com",
+            {"search_url": "https://host.icims.com/jobs"},
+        )
+        with self.assertRaisesRegex(ValueError, "missing title"):
+            provider.parse_job_page(
+                "<html></html>", "https://host.icims.com/jobs/1/missing/job", target
+            )
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)