ats-scrapers 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. ats_scrapers-0.1.0/.gitignore +130 -0
  2. ats_scrapers-0.1.0/LICENSE +21 -0
  3. ats_scrapers-0.1.0/PKG-INFO +198 -0
  4. ats_scrapers-0.1.0/README.md +126 -0
  5. ats_scrapers-0.1.0/pyproject.toml +139 -0
  6. ats_scrapers-0.1.0/src/ats_scrapers/__init__.py +50 -0
  7. ats_scrapers-0.1.0/src/ats_scrapers/_version.py +1 -0
  8. ats_scrapers-0.1.0/src/ats_scrapers/client.py +250 -0
  9. ats_scrapers-0.1.0/src/ats_scrapers/enrichment/__init__.py +13 -0
  10. ats_scrapers-0.1.0/src/ats_scrapers/enrichment/derived.py +120 -0
  11. ats_scrapers-0.1.0/src/ats_scrapers/exceptions.py +21 -0
  12. ats_scrapers-0.1.0/src/ats_scrapers/fetch.py +420 -0
  13. ats_scrapers-0.1.0/src/ats_scrapers/manifest.py +121 -0
  14. ats_scrapers-0.1.0/src/ats_scrapers/models.py +532 -0
  15. ats_scrapers-0.1.0/src/ats_scrapers/py.typed +0 -0
  16. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/__init__.py +119 -0
  17. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/_browserbase.py +123 -0
  18. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/_cloakbrowser.py +113 -0
  19. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/amazon.py +386 -0
  20. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/apple.py +498 -0
  21. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/arbetsformedlingen.py +421 -0
  22. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/ashby.py +172 -0
  23. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/avature.py +931 -0
  24. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/bamboohr.py +322 -0
  25. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/base.py +190 -0
  26. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/breezy.py +297 -0
  27. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/builtin.py +297 -0
  28. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/bundesagentur.py +623 -0
  29. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/cornerstone.py +309 -0
  30. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/eightfold.py +678 -0
  31. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/eures.py +744 -0
  32. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/gem.py +347 -0
  33. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/getonbrd.py +351 -0
  34. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/google.py +343 -0
  35. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/greenhouse.py +131 -0
  36. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/icims.py +445 -0
  37. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/infojobs_es.py +589 -0
  38. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/jazzhr.py +513 -0
  39. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/jobs_cz.py +406 -0
  40. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/jobsch.py +492 -0
  41. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/join_com.py +384 -0
  42. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/lever.py +186 -0
  43. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/manfred.py +279 -0
  44. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/mercor.py +215 -0
  45. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/meta.py +281 -0
  46. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/oracle.py +377 -0
  47. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/personio.py +263 -0
  48. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/phenom.py +376 -0
  49. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/pinpoint.py +268 -0
  50. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/programathor.py +475 -0
  51. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/recruitee.py +219 -0
  52. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/recruiterbox.py +222 -0
  53. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/remoteok.py +176 -0
  54. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/rippling.py +241 -0
  55. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/smartrecruiters.py +274 -0
  56. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/successfactors.py +258 -0
  57. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/taleo.py +291 -0
  58. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/teamtailor.py +161 -0
  59. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/tesla.py +346 -0
  60. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/thehub.py +270 -0
  61. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/tiktok.py +237 -0
  62. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/uber.py +196 -0
  63. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/usajobs.py +266 -0
  64. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/wanted.py +281 -0
  65. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/welcometothejungle.py +325 -0
  66. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/wellfound.py +521 -0
  67. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/weworkremotely.py +222 -0
  68. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/workable.py +222 -0
  69. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/workday.py +633 -0
  70. ats_scrapers-0.1.0/src/ats_scrapers/scrapers/ycombinator.py +435 -0
@@ -0,0 +1,130 @@
1
+ __pycache__/
2
+ *.pyc
3
+ *.pyo
4
+ *.egg-info/
5
+ build/
6
+ dist/
7
+ wheels/
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .mypy_cache/
11
+ .coverage
12
+ htmlcov/
13
+ .venv/
14
+ venv/
15
+ # Environment files — never commit these
16
+ .env
17
+ .env.*
18
+ **/.env
19
+ **/.env.*
20
+ .cursor
21
+ .vscode
22
+ .DS_Store
23
+ .claude/
24
+ *.parquet
25
+ *.log
26
+ logs/
27
+
28
+ # Library data fixtures (only keep tracked ones)
29
+ data/*.csv
30
+ !tests/fixtures/*.csv
31
+
32
+ # Operator-side artefacts (kept locally, not part of the library)
33
+ legacy/
34
+ # Top-level scripts/ stays private; .github/scripts/ ships with the repo.
35
+ /scripts/
36
+ viewer/
37
+ _legacy_discovery/
38
+ bloomberg/
39
+ map/
40
+ searxng-docker/
41
+ user_data/
42
+ yc/
43
+ z_searxng_instances.json
44
+ env.example
45
+ main.py
46
+ company_dictionary.txt
47
+ missing_locations.json
48
+ new_ai.csv
49
+ rm_ai.csv
50
+ ai*.csv
51
+ ai_companies.json
52
+ posted_jobs.json
53
+ hourly_thread.txt
54
+ daily_thread.txt
55
+ full_pipeline.sh
56
+ upload_to_cloudflare.py
57
+ ai.py
58
+ brute_dict_discovery.py
59
+ company_discovery.py
60
+ prune_dead_tenants.py
61
+ prune_workable_thorough.py
62
+ verify_migration.py
63
+ backfill_posted_at.py
64
+ fetch_job.py
65
+ gather_jobs.py
66
+ extract_salary_experience.py
67
+ export_utils.py
68
+ urls.md
69
+ CONTINUOUS_DISCOVERY_GUIDE.md
70
+ DISCOVERY_ENHANCEMENTS.md
71
+ XANALYSIS.md
72
+ /todo.md
73
+ /todo.*.md
74
+
75
+ # Per-ATS operator dirs (company CSVs + jobs.csv outputs)
76
+ amazon/
77
+ apple/
78
+ arbetsformedlingen/
79
+ ashby/
80
+ avature/
81
+ bamboohr/
82
+ breezy/
83
+ bundesagentur/
84
+ classifier/
85
+ cornerstone/
86
+ cursor/
87
+ eightfold/
88
+ gem/
89
+ google/
90
+ greenhouse/
91
+ icims/
92
+ jazzhr/
93
+ join_com/
94
+ lever/
95
+ mercor/
96
+ meta/
97
+ microsoft/
98
+ modelfiles/
99
+ models/
100
+ nvidia/
101
+ oracle/
102
+ personio/
103
+ phenom/
104
+ pinpoint/
105
+ recruitee/
106
+ recruiterbox/
107
+ rippling/
108
+ smartrecruiters/
109
+ successfactors/
110
+ taleo/
111
+ teamtailor/
112
+ tesla/
113
+ tiktok/
114
+ uber/
115
+ usajobs/
116
+ welcometothejungle/
117
+ workable/
118
+ workday/
119
+
120
+ # Discovery / audit outputs
121
+ **/*.bak
122
+ **/*.bak_*
123
+ workday_audit_*.csv
124
+ workday_discovery_*.csv
125
+ **/checkpoint.json
126
+ **/jobs.csv
127
+ **/jobs_diff_*.csv
128
+ **/companies/
129
+ **/processed_companies.txt
130
+ **/last_run.txt
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Kalil Bouzigues
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,198 @@
1
+ Metadata-Version: 2.4
2
+ Name: ats-scrapers
3
+ Version: 0.1.0
4
+ Summary: 4.2M+ live jobs from 63,000+ companies. An open dataset and Python toolkit for ATS and job-source data.
5
+ Project-URL: Homepage, https://github.com/kalil0321/ats-scrapers
6
+ Project-URL: Documentation, https://github.com/kalil0321/ats-scrapers#readme
7
+ Project-URL: Repository, https://github.com/kalil0321/ats-scrapers
8
+ Project-URL: Issues, https://github.com/kalil0321/ats-scrapers/issues
9
+ Project-URL: Changelog, https://github.com/kalil0321/ats-scrapers/blob/main/CHANGELOG.md
10
+ Project-URL: Dataset, https://storage.stapply.ai/jobhive/v1/manifest.json
11
+ Author-email: Kalil Bouzigues <kalil.bouzigues@gmail.com>
12
+ License: MIT
13
+ License-File: LICENSE
14
+ Keywords: ashby,ats,data,greenhouse,hiring,job-scraper,job-search,jobs,lever,recruiting,smartrecruiters,workable,workday
15
+ Classifier: Development Status :: 4 - Beta
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Intended Audience :: Science/Research
18
+ Classifier: License :: OSI Approved :: MIT License
19
+ Classifier: Operating System :: OS Independent
20
+ Classifier: Programming Language :: Python :: 3
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
25
+ Classifier: Topic :: Office/Business
26
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
27
+ Classifier: Typing :: Typed
28
+ Requires-Python: >=3.11
29
+ Requires-Dist: httpx>=0.27
30
+ Requires-Dist: pandas>=2.0
31
+ Requires-Dist: pydantic>=2.6
32
+ Provides-Extra: all
33
+ Requires-Dist: aiohttp>=3.9; extra == 'all'
34
+ Requires-Dist: beautifulsoup4>=4.12; extra == 'all'
35
+ Requires-Dist: cloakbrowser>=0.3; extra == 'all'
36
+ Requires-Dist: html2text>=2024.0; extra == 'all'
37
+ Requires-Dist: httpcloak>=1.6; extra == 'all'
38
+ Requires-Dist: pyarrow>=15.0; extra == 'all'
39
+ Provides-Extra: dev
40
+ Requires-Dist: aiohttp>=3.9; extra == 'dev'
41
+ Requires-Dist: beautifulsoup4>=4.12; extra == 'dev'
42
+ Requires-Dist: cloakbrowser>=0.3; extra == 'dev'
43
+ Requires-Dist: html2text>=2024.0; extra == 'dev'
44
+ Requires-Dist: httpcloak>=1.6; extra == 'dev'
45
+ Requires-Dist: mypy>=1.10; extra == 'dev'
46
+ Requires-Dist: pandas-stubs; extra == 'dev'
47
+ Requires-Dist: pyarrow>=15.0; extra == 'dev'
48
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
49
+ Requires-Dist: pytest-httpx>=0.30; extra == 'dev'
50
+ Requires-Dist: pytest>=8.0; extra == 'dev'
51
+ Requires-Dist: ruff>=0.5; extra == 'dev'
52
+ Requires-Dist: types-boto3; extra == 'dev'
53
+ Provides-Extra: parquet
54
+ Requires-Dist: pyarrow>=15.0; extra == 'parquet'
55
+ Provides-Extra: scrapers
56
+ Requires-Dist: aiohttp>=3.9; extra == 'scrapers'
57
+ Requires-Dist: beautifulsoup4>=4.12; extra == 'scrapers'
58
+ Requires-Dist: cloakbrowser>=0.3; extra == 'scrapers'
59
+ Requires-Dist: html2text>=2024.0; extra == 'scrapers'
60
+ Requires-Dist: httpcloak>=1.6; extra == 'scrapers'
61
+ Provides-Extra: test
62
+ Requires-Dist: aiohttp>=3.9; extra == 'test'
63
+ Requires-Dist: beautifulsoup4>=4.12; extra == 'test'
64
+ Requires-Dist: cloakbrowser>=0.3; extra == 'test'
65
+ Requires-Dist: html2text>=2024.0; extra == 'test'
66
+ Requires-Dist: httpcloak>=1.6; extra == 'test'
67
+ Requires-Dist: pyarrow>=15.0; extra == 'test'
68
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
69
+ Requires-Dist: pytest-httpx>=0.30; extra == 'test'
70
+ Requires-Dist: pytest>=8.0; extra == 'test'
71
+ Description-Content-Type: text/markdown
72
+
73
+ <p align="center">
74
+ <img src="https://raw.githubusercontent.com/kalil0321/ats-scrapers/main/assets/banner.jpeg" alt="ats-scrapers" />
75
+ </p>
76
+
77
+ # ats-scrapers
78
+
79
+ An open dataset and Python toolkit for job data from ATS platforms and public
80
+ sources.
81
+
82
+ [![PyPI](https://img.shields.io/pypi/v/ats-scrapers.svg?color=brightgreen)](https://pypi.org/project/ats-scrapers/)
83
+ [![Python](https://img.shields.io/pypi/pyversions/ats-scrapers.svg?color=brightgreen)](https://pypi.org/project/ats-scrapers/)
84
+ [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://github.com/kalil0321/ats-scrapers/blob/main/LICENSE)
85
+
86
+ `ats-scrapers` provides two layers:
87
+
88
+ - A free, hosted dataset with **4.2M+ live jobs** from **63,000+ companies**
89
+ across **49 sources**.
90
+ - More than 50 reusable scraper adapters, including Workday, Greenhouse, Lever,
91
+ Ashby, SmartRecruiters, and SuccessFactors.
92
+
93
+ Jobs are collected from ATS endpoints, company career sites, and public job
94
+ feeds, then normalized into one typed schema. Querying the hosted dataset
95
+ requires no API key or account.
96
+
97
+ ## Install
98
+
99
+ ```bash
100
+ pip install ats-scrapers
101
+ ```
102
+
103
+ The package is imported as `ats_scrapers`. Optional extras add only what you
104
+ need:
105
+
106
+ ```bash
107
+ pip install "ats-scrapers[parquet]" # query the full Parquet snapshot
108
+ pip install "ats-scrapers[scrapers]" # run the scraper library
109
+ pip install "ats-scrapers[all]" # install every runtime extra
110
+ ```
111
+
112
+ ## Query the public dataset
113
+
114
+ ```python
115
+ from ats_scrapers import search
116
+
117
+ # Per-source searches work with the base install.
118
+ jobs = search(
119
+ query="machine learning engineer",
120
+ location="Paris",
121
+ ats="greenhouse",
122
+ limit=100,
123
+ )
124
+
125
+ # The result is a pandas DataFrame.
126
+ print(jobs[["company", "title", "location", "apply_url"]])
127
+ ```
128
+
129
+ For practical full-dataset queries, install the `parquet` extra. The base
130
+ install is intended for smaller per-source CSV slices.
131
+
132
+ ```python
133
+ from ats_scrapers import search
134
+
135
+ jobs = search(query="data engineer", remote=True, salary_min=80_000)
136
+ ```
137
+
138
+ The [live manifest](https://storage.stapply.ai/jobhive/v1/manifest.json)
139
+ contains current row counts and artifact URLs. See the
140
+ [job schema](https://github.com/kalil0321/ats-scrapers/blob/main/JOB_SCHEMA.md)
141
+ for field definitions and normalization rules.
142
+
143
+ ## Scrape a company
144
+
145
+ ```python
146
+ from ats_scrapers.scrapers import get_scraper
147
+
148
+ scraper = get_scraper("ashby", "openai")
149
+ jobs = scraper.fetch()
150
+ ```
151
+
152
+ Scraper classes are also available directly:
153
+
154
+ ```python
155
+ from ats_scrapers.scrapers import GreenhouseScraper
156
+
157
+ jobs = GreenhouseScraper("anthropic").fetch()
158
+ ```
159
+
160
+ Scrapers are async-first — in async code (or for concurrency) use
161
+ `await scraper.afetch()` instead. The sync `fetch()` also works from
162
+ inside a running event loop (Jupyter, FastAPI): it transparently runs
163
+ on a worker thread.
164
+
165
+ Scraper adapters include:
166
+
167
+ - Major ATS platforms: Greenhouse, Lever, Ashby, Workday, SmartRecruiters,
168
+ SuccessFactors, Oracle, iCIMS, Workable, Personio, and more.
169
+ - First-party company APIs: Amazon, Apple, Google, TikTok, and Uber.
170
+ - Public and regional sources: EURES, Bundesagentur, Arbetsformedlingen,
171
+ Welcome to the Jungle, and others.
172
+
173
+ Run `python -c "from ats_scrapers import list_ats; print(*list_ats())"` for the
174
+ sources currently present in the hosted dataset.
175
+
176
+ ## Contributing
177
+
178
+ Contributions can add a source, improve an existing scraper, or add companies
179
+ to the CSV inventories in
180
+ [`ats-companies/`](https://github.com/kalil0321/ats-scrapers/tree/main/ats-companies).
181
+ The scraper API is intentionally tiny: subclass `BaseScraper`, set `ats`, and
182
+ implement `async def afetch()` using `self.make_fetcher()` for HTTP — retries,
183
+ backoff, and error mapping come for free. See
184
+ `src/ats_scrapers/scrapers/greenhouse.py` for a compact reference and the `Job`
185
+ model in `src/ats_scrapers/models.py` for the schema you populate.
186
+
187
+ ```bash
188
+ git clone https://github.com/kalil0321/ats-scrapers
189
+ cd ats-scrapers
190
+ uv sync --extra dev
191
+ uv pip install -r pipeline/requirements.txt # repo-only ops/publisher tests
192
+ uv run pytest
193
+ uv run ruff check .
194
+ ```
195
+
196
+ ## License
197
+
198
+ [MIT](https://github.com/kalil0321/ats-scrapers/blob/main/LICENSE)
@@ -0,0 +1,126 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/kalil0321/ats-scrapers/main/assets/banner.jpeg" alt="ats-scrapers" />
3
+ </p>
4
+
5
+ # ats-scrapers
6
+
7
+ An open dataset and Python toolkit for job data from ATS platforms and public
8
+ sources.
9
+
10
+ [![PyPI](https://img.shields.io/pypi/v/ats-scrapers.svg?color=brightgreen)](https://pypi.org/project/ats-scrapers/)
11
+ [![Python](https://img.shields.io/pypi/pyversions/ats-scrapers.svg?color=brightgreen)](https://pypi.org/project/ats-scrapers/)
12
+ [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://github.com/kalil0321/ats-scrapers/blob/main/LICENSE)
13
+
14
+ `ats-scrapers` provides two layers:
15
+
16
+ - A free, hosted dataset with **4.2M+ live jobs** from **63,000+ companies**
17
+ across **49 sources**.
18
+ - More than 50 reusable scraper adapters, including Workday, Greenhouse, Lever,
19
+ Ashby, SmartRecruiters, and SuccessFactors.
20
+
21
+ Jobs are collected from ATS endpoints, company career sites, and public job
22
+ feeds, then normalized into one typed schema. Querying the hosted dataset
23
+ requires no API key or account.
24
+
25
+ ## Install
26
+
27
+ ```bash
28
+ pip install ats-scrapers
29
+ ```
30
+
31
+ The package is imported as `ats_scrapers`. Optional extras add only what you
32
+ need:
33
+
34
+ ```bash
35
+ pip install "ats-scrapers[parquet]" # query the full Parquet snapshot
36
+ pip install "ats-scrapers[scrapers]" # run the scraper library
37
+ pip install "ats-scrapers[all]" # install every runtime extra
38
+ ```
39
+
40
+ ## Query the public dataset
41
+
42
+ ```python
43
+ from ats_scrapers import search
44
+
45
+ # Per-source searches work with the base install.
46
+ jobs = search(
47
+ query="machine learning engineer",
48
+ location="Paris",
49
+ ats="greenhouse",
50
+ limit=100,
51
+ )
52
+
53
+ # The result is a pandas DataFrame.
54
+ print(jobs[["company", "title", "location", "apply_url"]])
55
+ ```
56
+
57
+ For practical full-dataset queries, install the `parquet` extra. The base
58
+ install is intended for smaller per-source CSV slices.
59
+
60
+ ```python
61
+ from ats_scrapers import search
62
+
63
+ jobs = search(query="data engineer", remote=True, salary_min=80_000)
64
+ ```
65
+
66
+ The [live manifest](https://storage.stapply.ai/jobhive/v1/manifest.json)
67
+ contains current row counts and artifact URLs. See the
68
+ [job schema](https://github.com/kalil0321/ats-scrapers/blob/main/JOB_SCHEMA.md)
69
+ for field definitions and normalization rules.
70
+
71
+ ## Scrape a company
72
+
73
+ ```python
74
+ from ats_scrapers.scrapers import get_scraper
75
+
76
+ scraper = get_scraper("ashby", "openai")
77
+ jobs = scraper.fetch()
78
+ ```
79
+
80
+ Scraper classes are also available directly:
81
+
82
+ ```python
83
+ from ats_scrapers.scrapers import GreenhouseScraper
84
+
85
+ jobs = GreenhouseScraper("anthropic").fetch()
86
+ ```
87
+
88
+ Scrapers are async-first — in async code (or for concurrency) use
89
+ `await scraper.afetch()` instead. The sync `fetch()` also works from
90
+ inside a running event loop (Jupyter, FastAPI): it transparently runs
91
+ on a worker thread.
92
+
93
+ Scraper adapters include:
94
+
95
+ - Major ATS platforms: Greenhouse, Lever, Ashby, Workday, SmartRecruiters,
96
+ SuccessFactors, Oracle, iCIMS, Workable, Personio, and more.
97
+ - First-party company APIs: Amazon, Apple, Google, TikTok, and Uber.
98
+ - Public and regional sources: EURES, Bundesagentur, Arbetsformedlingen,
99
+ Welcome to the Jungle, and others.
100
+
101
+ Run `python -c "from ats_scrapers import list_ats; print(*list_ats())"` for the
102
+ sources currently present in the hosted dataset.
103
+
104
+ ## Contributing
105
+
106
+ Contributions can add a source, improve an existing scraper, or add companies
107
+ to the CSV inventories in
108
+ [`ats-companies/`](https://github.com/kalil0321/ats-scrapers/tree/main/ats-companies).
109
+ The scraper API is intentionally tiny: subclass `BaseScraper`, set `ats`, and
110
+ implement `async def afetch()` using `self.make_fetcher()` for HTTP — retries,
111
+ backoff, and error mapping come for free. See
112
+ `src/ats_scrapers/scrapers/greenhouse.py` for a compact reference and the `Job`
113
+ model in `src/ats_scrapers/models.py` for the schema you populate.
114
+
115
+ ```bash
116
+ git clone https://github.com/kalil0321/ats-scrapers
117
+ cd ats-scrapers
118
+ uv sync --extra dev
119
+ uv pip install -r pipeline/requirements.txt # repo-only ops/publisher tests
120
+ uv run pytest
121
+ uv run ruff check .
122
+ ```
123
+
124
+ ## License
125
+
126
+ [MIT](https://github.com/kalil0321/ats-scrapers/blob/main/LICENSE)
@@ -0,0 +1,139 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "ats-scrapers"
7
+ version = "0.1.0"
8
+ description = "4.2M+ live jobs from 63,000+ companies. An open dataset and Python toolkit for ATS and job-source data."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ authors = [
12
+ { name = "Kalil Bouzigues", email = "kalil.bouzigues@gmail.com" },
13
+ ]
14
+ keywords = [
15
+ "jobs",
16
+ "job-scraper",
17
+ "job-search",
18
+ "ats",
19
+ "greenhouse",
20
+ "lever",
21
+ "workday",
22
+ "ashby",
23
+ "workable",
24
+ "smartrecruiters",
25
+ "hiring",
26
+ "recruiting",
27
+ "data",
28
+ ]
29
+ classifiers = [
30
+ "Development Status :: 4 - Beta",
31
+ "Intended Audience :: Developers",
32
+ "Intended Audience :: Science/Research",
33
+ "License :: OSI Approved :: MIT License",
34
+ "Operating System :: OS Independent",
35
+ "Programming Language :: Python :: 3",
36
+ "Programming Language :: Python :: 3.11",
37
+ "Programming Language :: Python :: 3.12",
38
+ "Programming Language :: Python :: 3.13",
39
+ "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
40
+ "Topic :: Office/Business",
41
+ "Topic :: Scientific/Engineering :: Information Analysis",
42
+ "Typing :: Typed",
43
+ ]
44
+ requires-python = ">=3.11"
45
+ dependencies = [
46
+ "httpx>=0.27",
47
+ "pydantic>=2.6",
48
+ "pandas>=2.0",
49
+ ]
50
+
51
+ [project.optional-dependencies]
52
+ scrapers = [
53
+ "aiohttp>=3.9",
54
+ "beautifulsoup4>=4.12",
55
+ "html2text>=2024.0",
56
+ # TLS+h2 impersonation client used as the cheap fallback when an ATS
57
+ # load balancer 406-blocks plain httpx (Avature, JazzHR, Eightfold).
58
+ # 1.6.x ships fingerprint refreshes that fix recent breakages.
59
+ "httpcloak>=1.6",
60
+ # Stealth-patched Chromium used by Tesla / Meta (Akamai bot manager
61
+ # blocks every cheaper fingerprint — httpcloak, curl_cffi, even
62
+ # Browserbase Sessions). cloakbrowser ships its own binary and
63
+ # caches under ~/.cache/cloakbrowser, no separate playwright install.
64
+ "cloakbrowser>=0.3",
65
+ ]
66
+ parquet = [
67
+ "pyarrow>=15.0",
68
+ ]
69
+ all = [
70
+ "ats-scrapers[scrapers,parquet]",
71
+ ]
72
+ # Runs the library test suite. The ops pipeline under ``pipeline/``
73
+ # (publisher, R2 client) is repo-only — its tests additionally need
74
+ # ``pipeline/requirements.txt``, which CI installs alongside this
75
+ # extra.
76
+ test = [
77
+ "ats-scrapers[all]",
78
+ "pytest>=8.0",
79
+ "pytest-asyncio>=0.23",
80
+ "pytest-httpx>=0.30",
81
+ ]
82
+ dev = [
83
+ "ats-scrapers[test]",
84
+ "ruff>=0.5",
85
+ "mypy>=1.10",
86
+ "pandas-stubs",
87
+ "types-boto3",
88
+ ]
89
+
90
+ [project.urls]
91
+ Homepage = "https://github.com/kalil0321/ats-scrapers"
92
+ Documentation = "https://github.com/kalil0321/ats-scrapers#readme"
93
+ Repository = "https://github.com/kalil0321/ats-scrapers"
94
+ Issues = "https://github.com/kalil0321/ats-scrapers/issues"
95
+ Changelog = "https://github.com/kalil0321/ats-scrapers/blob/main/CHANGELOG.md"
96
+ Dataset = "https://storage.stapply.ai/jobhive/v1/manifest.json"
97
+
98
+ [tool.hatch.build.targets.wheel]
99
+ packages = ["src/ats_scrapers"]
100
+
101
+ # Tests are not shipped: the suite also covers the repo-only ops code
102
+ # under ``pipeline/``, so it only runs from a full checkout.
103
+ [tool.hatch.build.targets.sdist]
104
+ include = [
105
+ "/src",
106
+ "/README.md",
107
+ "/LICENSE",
108
+ ]
109
+
110
+ [tool.ruff]
111
+ line-length = 100
112
+ target-version = "py311"
113
+
114
+ [tool.ruff.lint]
115
+ select = ["E", "F", "I", "N", "UP", "B", "C4", "SIM", "RUF"]
116
+ # RUF001/2/3: tolerate Unicode dashes/quotes that real ATS payloads contain.
117
+ # RUF046: harmless `int(int_already)` calls happen in defensive casts.
118
+ ignore = ["E501", "RUF001", "RUF002", "RUF003", "RUF046"]
119
+
120
+ [tool.mypy]
121
+ python_version = "3.11"
122
+ strict = true
123
+ warn_return_any = true
124
+ warn_unused_configs = true
125
+
126
+ [tool.pytest.ini_options]
127
+ testpaths = ["tests"]
128
+ # Repo root on sys.path so tests can import the unpackaged ops code
129
+ # under ``pipeline/``.
130
+ pythonpath = ["."]
131
+ asyncio_mode = "auto"
132
+ # Live e2e tests (tests/e2e/) hit real ATS endpoints. They are doubly
133
+ # gated: deselected by default (`-m "not live"`) AND skipped unless
134
+ # ATS_SCRAPERS_LIVE_E2E=1. Run them with:
135
+ # ATS_SCRAPERS_LIVE_E2E=1 pytest -m live
136
+ addopts = "-ra --strict-markers -m 'not live'"
137
+ markers = [
138
+ "live: hits real ATS endpoints / the hosted dataset (opt-in)",
139
+ ]
@@ -0,0 +1,50 @@
1
+ """ats-scrapers — open dataset and toolkit for global job market data.
2
+
3
+ Two layers of progressive disclosure:
4
+
5
+ 1. Dataset client (zero config):
6
+ >>> from ats_scrapers import search
7
+ >>> df = search(query="ml engineer", location="Paris", ats="greenhouse")
8
+
9
+ 2. Per-ATS scrapers (BYO companies):
10
+ >>> from ats_scrapers.scrapers import GreenhouseScraper
11
+ >>> jobs = GreenhouseScraper("anthropic").fetch()
12
+
13
+ Publishing/orchestration code lives in the repo-only ``pipeline/``
14
+ directory and is not part of the installable package.
15
+ """
16
+
17
+ from ats_scrapers._version import __version__
18
+ from ats_scrapers.client import Client, list_ats, search
19
+ from ats_scrapers.exceptions import (
20
+ ATSScrapersError,
21
+ CompanyNotFoundError,
22
+ ManifestError,
23
+ ScraperError,
24
+ StorageError,
25
+ )
26
+ from ats_scrapers.fetch import Fetcher, FetchResponse, MalformedJSONError
27
+ from ats_scrapers.manifest import Manifest
28
+ from ats_scrapers.models import ATSType, Company, EmploymentType, Job, Salary, SalaryPeriod
29
+
30
+ __all__ = [
31
+ "ATSScrapersError",
32
+ "ATSType",
33
+ "Client",
34
+ "Company",
35
+ "CompanyNotFoundError",
36
+ "EmploymentType",
37
+ "FetchResponse",
38
+ "Fetcher",
39
+ "Job",
40
+ "MalformedJSONError",
41
+ "Manifest",
42
+ "ManifestError",
43
+ "Salary",
44
+ "SalaryPeriod",
45
+ "ScraperError",
46
+ "StorageError",
47
+ "__version__",
48
+ "list_ats",
49
+ "search",
50
+ ]
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"