ats-scrapers 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ats_scrapers-0.1.0/.gitignore +130 -0
- ats_scrapers-0.1.0/LICENSE +21 -0
- ats_scrapers-0.1.0/PKG-INFO +198 -0
- ats_scrapers-0.1.0/README.md +126 -0
- ats_scrapers-0.1.0/pyproject.toml +139 -0
- ats_scrapers-0.1.0/src/ats_scrapers/__init__.py +50 -0
- ats_scrapers-0.1.0/src/ats_scrapers/_version.py +1 -0
- ats_scrapers-0.1.0/src/ats_scrapers/client.py +250 -0
- ats_scrapers-0.1.0/src/ats_scrapers/enrichment/__init__.py +13 -0
- ats_scrapers-0.1.0/src/ats_scrapers/enrichment/derived.py +120 -0
- ats_scrapers-0.1.0/src/ats_scrapers/exceptions.py +21 -0
- ats_scrapers-0.1.0/src/ats_scrapers/fetch.py +420 -0
- ats_scrapers-0.1.0/src/ats_scrapers/manifest.py +121 -0
- ats_scrapers-0.1.0/src/ats_scrapers/models.py +532 -0
- ats_scrapers-0.1.0/src/ats_scrapers/py.typed +0 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/__init__.py +119 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/_browserbase.py +123 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/_cloakbrowser.py +113 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/amazon.py +386 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/apple.py +498 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/arbetsformedlingen.py +421 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/ashby.py +172 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/avature.py +931 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/bamboohr.py +322 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/base.py +190 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/breezy.py +297 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/builtin.py +297 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/bundesagentur.py +623 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/cornerstone.py +309 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/eightfold.py +678 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/eures.py +744 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/gem.py +347 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/getonbrd.py +351 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/google.py +343 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/greenhouse.py +131 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/icims.py +445 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/infojobs_es.py +589 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/jazzhr.py +513 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/jobs_cz.py +406 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/jobsch.py +492 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/join_com.py +384 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/lever.py +186 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/manfred.py +279 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/mercor.py +215 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/meta.py +281 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/oracle.py +377 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/personio.py +263 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/phenom.py +376 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/pinpoint.py +268 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/programathor.py +475 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/recruitee.py +219 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/recruiterbox.py +222 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/remoteok.py +176 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/rippling.py +241 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/smartrecruiters.py +274 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/successfactors.py +258 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/taleo.py +291 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/teamtailor.py +161 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/tesla.py +346 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/thehub.py +270 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/tiktok.py +237 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/uber.py +196 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/usajobs.py +266 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/wanted.py +281 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/welcometothejungle.py +325 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/wellfound.py +521 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/weworkremotely.py +222 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/workable.py +222 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/workday.py +633 -0
- ats_scrapers-0.1.0/src/ats_scrapers/scrapers/ycombinator.py +435 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.pyc
|
|
3
|
+
*.pyo
|
|
4
|
+
*.egg-info/
|
|
5
|
+
build/
|
|
6
|
+
dist/
|
|
7
|
+
wheels/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
.mypy_cache/
|
|
11
|
+
.coverage
|
|
12
|
+
htmlcov/
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
# Environment files — never commit these
|
|
16
|
+
.env
|
|
17
|
+
.env.*
|
|
18
|
+
**/.env
|
|
19
|
+
**/.env.*
|
|
20
|
+
.cursor
|
|
21
|
+
.vscode
|
|
22
|
+
.DS_Store
|
|
23
|
+
.claude/
|
|
24
|
+
*.parquet
|
|
25
|
+
*.log
|
|
26
|
+
logs/
|
|
27
|
+
|
|
28
|
+
# Library data fixtures (only keep tracked ones)
|
|
29
|
+
data/*.csv
|
|
30
|
+
!tests/fixtures/*.csv
|
|
31
|
+
|
|
32
|
+
# Operator-side artefacts (kept locally, not part of the library)
|
|
33
|
+
legacy/
|
|
34
|
+
# Top-level scripts/ stays private; .github/scripts/ ships with the repo.
|
|
35
|
+
/scripts/
|
|
36
|
+
viewer/
|
|
37
|
+
_legacy_discovery/
|
|
38
|
+
bloomberg/
|
|
39
|
+
map/
|
|
40
|
+
searxng-docker/
|
|
41
|
+
user_data/
|
|
42
|
+
yc/
|
|
43
|
+
z_searxng_instances.json
|
|
44
|
+
env.example
|
|
45
|
+
main.py
|
|
46
|
+
company_dictionary.txt
|
|
47
|
+
missing_locations.json
|
|
48
|
+
new_ai.csv
|
|
49
|
+
rm_ai.csv
|
|
50
|
+
ai*.csv
|
|
51
|
+
ai_companies.json
|
|
52
|
+
posted_jobs.json
|
|
53
|
+
hourly_thread.txt
|
|
54
|
+
daily_thread.txt
|
|
55
|
+
full_pipeline.sh
|
|
56
|
+
upload_to_cloudflare.py
|
|
57
|
+
ai.py
|
|
58
|
+
brute_dict_discovery.py
|
|
59
|
+
company_discovery.py
|
|
60
|
+
prune_dead_tenants.py
|
|
61
|
+
prune_workable_thorough.py
|
|
62
|
+
verify_migration.py
|
|
63
|
+
backfill_posted_at.py
|
|
64
|
+
fetch_job.py
|
|
65
|
+
gather_jobs.py
|
|
66
|
+
extract_salary_experience.py
|
|
67
|
+
export_utils.py
|
|
68
|
+
urls.md
|
|
69
|
+
CONTINUOUS_DISCOVERY_GUIDE.md
|
|
70
|
+
DISCOVERY_ENHANCEMENTS.md
|
|
71
|
+
XANALYSIS.md
|
|
72
|
+
/todo.md
|
|
73
|
+
/todo.*.md
|
|
74
|
+
|
|
75
|
+
# Per-ATS operator dirs (company CSVs + jobs.csv outputs)
|
|
76
|
+
amazon/
|
|
77
|
+
apple/
|
|
78
|
+
arbetsformedlingen/
|
|
79
|
+
ashby/
|
|
80
|
+
avature/
|
|
81
|
+
bamboohr/
|
|
82
|
+
breezy/
|
|
83
|
+
bundesagentur/
|
|
84
|
+
classifier/
|
|
85
|
+
cornerstone/
|
|
86
|
+
cursor/
|
|
87
|
+
eightfold/
|
|
88
|
+
gem/
|
|
89
|
+
google/
|
|
90
|
+
greenhouse/
|
|
91
|
+
icims/
|
|
92
|
+
jazzhr/
|
|
93
|
+
join_com/
|
|
94
|
+
lever/
|
|
95
|
+
mercor/
|
|
96
|
+
meta/
|
|
97
|
+
microsoft/
|
|
98
|
+
modelfiles/
|
|
99
|
+
models/
|
|
100
|
+
nvidia/
|
|
101
|
+
oracle/
|
|
102
|
+
personio/
|
|
103
|
+
phenom/
|
|
104
|
+
pinpoint/
|
|
105
|
+
recruitee/
|
|
106
|
+
recruiterbox/
|
|
107
|
+
rippling/
|
|
108
|
+
smartrecruiters/
|
|
109
|
+
successfactors/
|
|
110
|
+
taleo/
|
|
111
|
+
teamtailor/
|
|
112
|
+
tesla/
|
|
113
|
+
tiktok/
|
|
114
|
+
uber/
|
|
115
|
+
usajobs/
|
|
116
|
+
welcometothejungle/
|
|
117
|
+
workable/
|
|
118
|
+
workday/
|
|
119
|
+
|
|
120
|
+
# Discovery / audit outputs
|
|
121
|
+
**/*.bak
|
|
122
|
+
**/*.bak_*
|
|
123
|
+
workday_audit_*.csv
|
|
124
|
+
workday_discovery_*.csv
|
|
125
|
+
**/checkpoint.json
|
|
126
|
+
**/jobs.csv
|
|
127
|
+
**/jobs_diff_*.csv
|
|
128
|
+
**/companies/
|
|
129
|
+
**/processed_companies.txt
|
|
130
|
+
**/last_run.txt
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kalil Bouzigues
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ats-scrapers
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: 4.2M+ live jobs from 63,000+ companies. An open dataset and Python toolkit for ATS and job-source data.
|
|
5
|
+
Project-URL: Homepage, https://github.com/kalil0321/ats-scrapers
|
|
6
|
+
Project-URL: Documentation, https://github.com/kalil0321/ats-scrapers#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/kalil0321/ats-scrapers
|
|
8
|
+
Project-URL: Issues, https://github.com/kalil0321/ats-scrapers/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/kalil0321/ats-scrapers/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Dataset, https://storage.stapply.ai/jobhive/v1/manifest.json
|
|
11
|
+
Author-email: Kalil Bouzigues <kalil.bouzigues@gmail.com>
|
|
12
|
+
License: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: ashby,ats,data,greenhouse,hiring,job-scraper,job-search,jobs,lever,recruiting,smartrecruiters,workable,workday
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
20
|
+
Classifier: Programming Language :: Python :: 3
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
25
|
+
Classifier: Topic :: Office/Business
|
|
26
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.11
|
|
29
|
+
Requires-Dist: httpx>=0.27
|
|
30
|
+
Requires-Dist: pandas>=2.0
|
|
31
|
+
Requires-Dist: pydantic>=2.6
|
|
32
|
+
Provides-Extra: all
|
|
33
|
+
Requires-Dist: aiohttp>=3.9; extra == 'all'
|
|
34
|
+
Requires-Dist: beautifulsoup4>=4.12; extra == 'all'
|
|
35
|
+
Requires-Dist: cloakbrowser>=0.3; extra == 'all'
|
|
36
|
+
Requires-Dist: html2text>=2024.0; extra == 'all'
|
|
37
|
+
Requires-Dist: httpcloak>=1.6; extra == 'all'
|
|
38
|
+
Requires-Dist: pyarrow>=15.0; extra == 'all'
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Requires-Dist: aiohttp>=3.9; extra == 'dev'
|
|
41
|
+
Requires-Dist: beautifulsoup4>=4.12; extra == 'dev'
|
|
42
|
+
Requires-Dist: cloakbrowser>=0.3; extra == 'dev'
|
|
43
|
+
Requires-Dist: html2text>=2024.0; extra == 'dev'
|
|
44
|
+
Requires-Dist: httpcloak>=1.6; extra == 'dev'
|
|
45
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
46
|
+
Requires-Dist: pandas-stubs; extra == 'dev'
|
|
47
|
+
Requires-Dist: pyarrow>=15.0; extra == 'dev'
|
|
48
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
49
|
+
Requires-Dist: pytest-httpx>=0.30; extra == 'dev'
|
|
50
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
51
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
52
|
+
Requires-Dist: types-boto3; extra == 'dev'
|
|
53
|
+
Provides-Extra: parquet
|
|
54
|
+
Requires-Dist: pyarrow>=15.0; extra == 'parquet'
|
|
55
|
+
Provides-Extra: scrapers
|
|
56
|
+
Requires-Dist: aiohttp>=3.9; extra == 'scrapers'
|
|
57
|
+
Requires-Dist: beautifulsoup4>=4.12; extra == 'scrapers'
|
|
58
|
+
Requires-Dist: cloakbrowser>=0.3; extra == 'scrapers'
|
|
59
|
+
Requires-Dist: html2text>=2024.0; extra == 'scrapers'
|
|
60
|
+
Requires-Dist: httpcloak>=1.6; extra == 'scrapers'
|
|
61
|
+
Provides-Extra: test
|
|
62
|
+
Requires-Dist: aiohttp>=3.9; extra == 'test'
|
|
63
|
+
Requires-Dist: beautifulsoup4>=4.12; extra == 'test'
|
|
64
|
+
Requires-Dist: cloakbrowser>=0.3; extra == 'test'
|
|
65
|
+
Requires-Dist: html2text>=2024.0; extra == 'test'
|
|
66
|
+
Requires-Dist: httpcloak>=1.6; extra == 'test'
|
|
67
|
+
Requires-Dist: pyarrow>=15.0; extra == 'test'
|
|
68
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
|
|
69
|
+
Requires-Dist: pytest-httpx>=0.30; extra == 'test'
|
|
70
|
+
Requires-Dist: pytest>=8.0; extra == 'test'
|
|
71
|
+
Description-Content-Type: text/markdown
|
|
72
|
+
|
|
73
|
+
<p align="center">
|
|
74
|
+
<img src="https://raw.githubusercontent.com/kalil0321/ats-scrapers/main/assets/banner.jpeg" alt="ats-scrapers" />
|
|
75
|
+
</p>
|
|
76
|
+
|
|
77
|
+
# ats-scrapers
|
|
78
|
+
|
|
79
|
+
An open dataset and Python toolkit for job data from ATS platforms and public
|
|
80
|
+
sources.
|
|
81
|
+
|
|
82
|
+
[](https://pypi.org/project/ats-scrapers/)
|
|
83
|
+
[](https://pypi.org/project/ats-scrapers/)
|
|
84
|
+
[](https://github.com/kalil0321/ats-scrapers/blob/main/LICENSE)
|
|
85
|
+
|
|
86
|
+
`ats-scrapers` provides two layers:
|
|
87
|
+
|
|
88
|
+
- A free, hosted dataset with **4.2M+ live jobs** from **63,000+ companies**
|
|
89
|
+
across **49 sources**.
|
|
90
|
+
- More than 50 reusable scraper adapters, including Workday, Greenhouse, Lever,
|
|
91
|
+
Ashby, SmartRecruiters, and SuccessFactors.
|
|
92
|
+
|
|
93
|
+
Jobs are collected from ATS endpoints, company career sites, and public job
|
|
94
|
+
feeds, then normalized into one typed schema. Querying the hosted dataset
|
|
95
|
+
requires no API key or account.
|
|
96
|
+
|
|
97
|
+
## Install
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install ats-scrapers
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The package is imported as `ats_scrapers`. Optional extras add only what you
|
|
104
|
+
need:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
pip install "ats-scrapers[parquet]" # query the full Parquet snapshot
|
|
108
|
+
pip install "ats-scrapers[scrapers]" # run the scraper library
|
|
109
|
+
pip install "ats-scrapers[all]" # install every runtime extra
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Query the public dataset
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
from ats_scrapers import search
|
|
116
|
+
|
|
117
|
+
# Per-source searches work with the base install.
|
|
118
|
+
jobs = search(
|
|
119
|
+
query="machine learning engineer",
|
|
120
|
+
location="Paris",
|
|
121
|
+
ats="greenhouse",
|
|
122
|
+
limit=100,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
# The result is a pandas DataFrame.
|
|
126
|
+
print(jobs[["company", "title", "location", "apply_url"]])
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
For practical full-dataset queries, install the `parquet` extra. The base
|
|
130
|
+
install is intended for smaller per-source CSV slices.
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from ats_scrapers import search
|
|
134
|
+
|
|
135
|
+
jobs = search(query="data engineer", remote=True, salary_min=80_000)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The [live manifest](https://storage.stapply.ai/jobhive/v1/manifest.json)
|
|
139
|
+
contains current row counts and artifact URLs. See the
|
|
140
|
+
[job schema](https://github.com/kalil0321/ats-scrapers/blob/main/JOB_SCHEMA.md)
|
|
141
|
+
for field definitions and normalization rules.
|
|
142
|
+
|
|
143
|
+
## Scrape a company
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
from ats_scrapers.scrapers import get_scraper
|
|
147
|
+
|
|
148
|
+
scraper = get_scraper("ashby", "openai")
|
|
149
|
+
jobs = scraper.fetch()
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Scraper classes are also available directly:
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
from ats_scrapers.scrapers import GreenhouseScraper
|
|
156
|
+
|
|
157
|
+
jobs = GreenhouseScraper("anthropic").fetch()
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Scrapers are async-first — in async code (or for concurrency) use
|
|
161
|
+
`await scraper.afetch()` instead. The sync `fetch()` also works from
|
|
162
|
+
inside a running event loop (Jupyter, FastAPI): it transparently runs
|
|
163
|
+
on a worker thread.
|
|
164
|
+
|
|
165
|
+
Scraper adapters include:
|
|
166
|
+
|
|
167
|
+
- Major ATS platforms: Greenhouse, Lever, Ashby, Workday, SmartRecruiters,
|
|
168
|
+
SuccessFactors, Oracle, iCIMS, Workable, Personio, and more.
|
|
169
|
+
- First-party company APIs: Amazon, Apple, Google, TikTok, and Uber.
|
|
170
|
+
- Public and regional sources: EURES, Bundesagentur, Arbetsformedlingen,
|
|
171
|
+
Welcome to the Jungle, and others.
|
|
172
|
+
|
|
173
|
+
Run `python -c "from ats_scrapers import list_ats; print(*list_ats())"` for the
|
|
174
|
+
sources currently present in the hosted dataset.
|
|
175
|
+
|
|
176
|
+
## Contributing
|
|
177
|
+
|
|
178
|
+
Contributions can add a source, improve an existing scraper, or add companies
|
|
179
|
+
to the CSV inventories in
|
|
180
|
+
[`ats-companies/`](https://github.com/kalil0321/ats-scrapers/tree/main/ats-companies).
|
|
181
|
+
The scraper API is intentionally tiny: subclass `BaseScraper`, set `ats`, and
|
|
182
|
+
implement `async def afetch()` using `self.make_fetcher()` for HTTP — retries,
|
|
183
|
+
backoff, and error mapping come for free. See
|
|
184
|
+
`src/ats_scrapers/scrapers/greenhouse.py` for a compact reference and the `Job`
|
|
185
|
+
model in `src/ats_scrapers/models.py` for the schema you populate.
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
git clone https://github.com/kalil0321/ats-scrapers
|
|
189
|
+
cd ats-scrapers
|
|
190
|
+
uv sync --extra dev
|
|
191
|
+
uv pip install -r pipeline/requirements.txt # repo-only ops/publisher tests
|
|
192
|
+
uv run pytest
|
|
193
|
+
uv run ruff check .
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
## License
|
|
197
|
+
|
|
198
|
+
[MIT](https://github.com/kalil0321/ats-scrapers/blob/main/LICENSE)
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/kalil0321/ats-scrapers/main/assets/banner.jpeg" alt="ats-scrapers" />
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
# ats-scrapers
|
|
6
|
+
|
|
7
|
+
An open dataset and Python toolkit for job data from ATS platforms and public
|
|
8
|
+
sources.
|
|
9
|
+
|
|
10
|
+
[](https://pypi.org/project/ats-scrapers/)
|
|
11
|
+
[](https://pypi.org/project/ats-scrapers/)
|
|
12
|
+
[](https://github.com/kalil0321/ats-scrapers/blob/main/LICENSE)
|
|
13
|
+
|
|
14
|
+
`ats-scrapers` provides two layers:
|
|
15
|
+
|
|
16
|
+
- A free, hosted dataset with **4.2M+ live jobs** from **63,000+ companies**
|
|
17
|
+
across **49 sources**.
|
|
18
|
+
- More than 50 reusable scraper adapters, including Workday, Greenhouse, Lever,
|
|
19
|
+
Ashby, SmartRecruiters, and SuccessFactors.
|
|
20
|
+
|
|
21
|
+
Jobs are collected from ATS endpoints, company career sites, and public job
|
|
22
|
+
feeds, then normalized into one typed schema. Querying the hosted dataset
|
|
23
|
+
requires no API key or account.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install ats-scrapers
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The package is imported as `ats_scrapers`. Optional extras add only what you
|
|
32
|
+
need:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install "ats-scrapers[parquet]" # query the full Parquet snapshot
|
|
36
|
+
pip install "ats-scrapers[scrapers]" # run the scraper library
|
|
37
|
+
pip install "ats-scrapers[all]" # install every runtime extra
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Query the public dataset
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from ats_scrapers import search
|
|
44
|
+
|
|
45
|
+
# Per-source searches work with the base install.
|
|
46
|
+
jobs = search(
|
|
47
|
+
query="machine learning engineer",
|
|
48
|
+
location="Paris",
|
|
49
|
+
ats="greenhouse",
|
|
50
|
+
limit=100,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
# The result is a pandas DataFrame.
|
|
54
|
+
print(jobs[["company", "title", "location", "apply_url"]])
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
For practical full-dataset queries, install the `parquet` extra. The base
|
|
58
|
+
install is intended for smaller per-source CSV slices.
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from ats_scrapers import search
|
|
62
|
+
|
|
63
|
+
jobs = search(query="data engineer", remote=True, salary_min=80_000)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The [live manifest](https://storage.stapply.ai/jobhive/v1/manifest.json)
|
|
67
|
+
contains current row counts and artifact URLs. See the
|
|
68
|
+
[job schema](https://github.com/kalil0321/ats-scrapers/blob/main/JOB_SCHEMA.md)
|
|
69
|
+
for field definitions and normalization rules.
|
|
70
|
+
|
|
71
|
+
## Scrape a company
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from ats_scrapers.scrapers import get_scraper
|
|
75
|
+
|
|
76
|
+
scraper = get_scraper("ashby", "openai")
|
|
77
|
+
jobs = scraper.fetch()
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Scraper classes are also available directly:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from ats_scrapers.scrapers import GreenhouseScraper
|
|
84
|
+
|
|
85
|
+
jobs = GreenhouseScraper("anthropic").fetch()
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Scrapers are async-first — in async code (or for concurrency) use
|
|
89
|
+
`await scraper.afetch()` instead. The sync `fetch()` also works from
|
|
90
|
+
inside a running event loop (Jupyter, FastAPI): it transparently runs
|
|
91
|
+
on a worker thread.
|
|
92
|
+
|
|
93
|
+
Scraper adapters include:
|
|
94
|
+
|
|
95
|
+
- Major ATS platforms: Greenhouse, Lever, Ashby, Workday, SmartRecruiters,
|
|
96
|
+
SuccessFactors, Oracle, iCIMS, Workable, Personio, and more.
|
|
97
|
+
- First-party company APIs: Amazon, Apple, Google, TikTok, and Uber.
|
|
98
|
+
- Public and regional sources: EURES, Bundesagentur, Arbetsformedlingen,
|
|
99
|
+
Welcome to the Jungle, and others.
|
|
100
|
+
|
|
101
|
+
Run `python -c "from ats_scrapers import list_ats; print(*list_ats())"` for the
|
|
102
|
+
sources currently present in the hosted dataset.
|
|
103
|
+
|
|
104
|
+
## Contributing
|
|
105
|
+
|
|
106
|
+
Contributions can add a source, improve an existing scraper, or add companies
|
|
107
|
+
to the CSV inventories in
|
|
108
|
+
[`ats-companies/`](https://github.com/kalil0321/ats-scrapers/tree/main/ats-companies).
|
|
109
|
+
The scraper API is intentionally tiny: subclass `BaseScraper`, set `ats`, and
|
|
110
|
+
implement `async def afetch()` using `self.make_fetcher()` for HTTP — retries,
|
|
111
|
+
backoff, and error mapping come for free. See
|
|
112
|
+
`src/ats_scrapers/scrapers/greenhouse.py` for a compact reference and the `Job`
|
|
113
|
+
model in `src/ats_scrapers/models.py` for the schema you populate.
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
git clone https://github.com/kalil0321/ats-scrapers
|
|
117
|
+
cd ats-scrapers
|
|
118
|
+
uv sync --extra dev
|
|
119
|
+
uv pip install -r pipeline/requirements.txt # repo-only ops/publisher tests
|
|
120
|
+
uv run pytest
|
|
121
|
+
uv run ruff check .
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## License
|
|
125
|
+
|
|
126
|
+
[MIT](https://github.com/kalil0321/ats-scrapers/blob/main/LICENSE)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ats-scrapers"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "4.2M+ live jobs from 63,000+ companies. An open dataset and Python toolkit for ATS and job-source data."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "Kalil Bouzigues", email = "kalil.bouzigues@gmail.com" },
|
|
13
|
+
]
|
|
14
|
+
keywords = [
|
|
15
|
+
"jobs",
|
|
16
|
+
"job-scraper",
|
|
17
|
+
"job-search",
|
|
18
|
+
"ats",
|
|
19
|
+
"greenhouse",
|
|
20
|
+
"lever",
|
|
21
|
+
"workday",
|
|
22
|
+
"ashby",
|
|
23
|
+
"workable",
|
|
24
|
+
"smartrecruiters",
|
|
25
|
+
"hiring",
|
|
26
|
+
"recruiting",
|
|
27
|
+
"data",
|
|
28
|
+
]
|
|
29
|
+
classifiers = [
|
|
30
|
+
"Development Status :: 4 - Beta",
|
|
31
|
+
"Intended Audience :: Developers",
|
|
32
|
+
"Intended Audience :: Science/Research",
|
|
33
|
+
"License :: OSI Approved :: MIT License",
|
|
34
|
+
"Operating System :: OS Independent",
|
|
35
|
+
"Programming Language :: Python :: 3",
|
|
36
|
+
"Programming Language :: Python :: 3.11",
|
|
37
|
+
"Programming Language :: Python :: 3.12",
|
|
38
|
+
"Programming Language :: Python :: 3.13",
|
|
39
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
40
|
+
"Topic :: Office/Business",
|
|
41
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
42
|
+
"Typing :: Typed",
|
|
43
|
+
]
|
|
44
|
+
requires-python = ">=3.11"
|
|
45
|
+
dependencies = [
|
|
46
|
+
"httpx>=0.27",
|
|
47
|
+
"pydantic>=2.6",
|
|
48
|
+
"pandas>=2.0",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[project.optional-dependencies]
|
|
52
|
+
scrapers = [
|
|
53
|
+
"aiohttp>=3.9",
|
|
54
|
+
"beautifulsoup4>=4.12",
|
|
55
|
+
"html2text>=2024.0",
|
|
56
|
+
# TLS+h2 impersonation client used as the cheap fallback when an ATS
|
|
57
|
+
# load balancer 406-blocks plain httpx (Avature, JazzHR, Eightfold).
|
|
58
|
+
# 1.6.x ships fingerprint refreshes that fix recent breakages.
|
|
59
|
+
"httpcloak>=1.6",
|
|
60
|
+
# Stealth-patched Chromium used by Tesla / Meta (Akamai bot manager
|
|
61
|
+
# blocks every cheaper fingerprint — httpcloak, curl_cffi, even
|
|
62
|
+
# Browserbase Sessions). cloakbrowser ships its own binary and
|
|
63
|
+
# caches under ~/.cache/cloakbrowser, no separate playwright install.
|
|
64
|
+
"cloakbrowser>=0.3",
|
|
65
|
+
]
|
|
66
|
+
parquet = [
|
|
67
|
+
"pyarrow>=15.0",
|
|
68
|
+
]
|
|
69
|
+
all = [
|
|
70
|
+
"ats-scrapers[scrapers,parquet]",
|
|
71
|
+
]
|
|
72
|
+
# Runs the library test suite. The ops pipeline under ``pipeline/``
|
|
73
|
+
# (publisher, R2 client) is repo-only — its tests additionally need
|
|
74
|
+
# ``pipeline/requirements.txt``, which CI installs alongside this
|
|
75
|
+
# extra.
|
|
76
|
+
test = [
|
|
77
|
+
"ats-scrapers[all]",
|
|
78
|
+
"pytest>=8.0",
|
|
79
|
+
"pytest-asyncio>=0.23",
|
|
80
|
+
"pytest-httpx>=0.30",
|
|
81
|
+
]
|
|
82
|
+
dev = [
|
|
83
|
+
"ats-scrapers[test]",
|
|
84
|
+
"ruff>=0.5",
|
|
85
|
+
"mypy>=1.10",
|
|
86
|
+
"pandas-stubs",
|
|
87
|
+
"types-boto3",
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
[project.urls]
|
|
91
|
+
Homepage = "https://github.com/kalil0321/ats-scrapers"
|
|
92
|
+
Documentation = "https://github.com/kalil0321/ats-scrapers#readme"
|
|
93
|
+
Repository = "https://github.com/kalil0321/ats-scrapers"
|
|
94
|
+
Issues = "https://github.com/kalil0321/ats-scrapers/issues"
|
|
95
|
+
Changelog = "https://github.com/kalil0321/ats-scrapers/blob/main/CHANGELOG.md"
|
|
96
|
+
Dataset = "https://storage.stapply.ai/jobhive/v1/manifest.json"
|
|
97
|
+
|
|
98
|
+
[tool.hatch.build.targets.wheel]
|
|
99
|
+
packages = ["src/ats_scrapers"]
|
|
100
|
+
|
|
101
|
+
# Tests are not shipped: the suite also covers the repo-only ops code
|
|
102
|
+
# under ``pipeline/``, so it only runs from a full checkout.
|
|
103
|
+
[tool.hatch.build.targets.sdist]
|
|
104
|
+
include = [
|
|
105
|
+
"/src",
|
|
106
|
+
"/README.md",
|
|
107
|
+
"/LICENSE",
|
|
108
|
+
]
|
|
109
|
+
|
|
110
|
+
[tool.ruff]
|
|
111
|
+
line-length = 100
|
|
112
|
+
target-version = "py311"
|
|
113
|
+
|
|
114
|
+
[tool.ruff.lint]
|
|
115
|
+
select = ["E", "F", "I", "N", "UP", "B", "C4", "SIM", "RUF"]
|
|
116
|
+
# RUF001/2/3: tolerate Unicode dashes/quotes that real ATS payloads contain.
|
|
117
|
+
# RUF046: harmless `int(int_already)` calls happen in defensive casts.
|
|
118
|
+
ignore = ["E501", "RUF001", "RUF002", "RUF003", "RUF046"]
|
|
119
|
+
|
|
120
|
+
[tool.mypy]
|
|
121
|
+
python_version = "3.11"
|
|
122
|
+
strict = true
|
|
123
|
+
warn_return_any = true
|
|
124
|
+
warn_unused_configs = true
|
|
125
|
+
|
|
126
|
+
[tool.pytest.ini_options]
|
|
127
|
+
testpaths = ["tests"]
|
|
128
|
+
# Repo root on sys.path so tests can import the unpackaged ops code
|
|
129
|
+
# under ``pipeline/``.
|
|
130
|
+
pythonpath = ["."]
|
|
131
|
+
asyncio_mode = "auto"
|
|
132
|
+
# Live e2e tests (tests/e2e/) hit real ATS endpoints. They are doubly
|
|
133
|
+
# gated: deselected by default (`-m "not live"`) AND skipped unless
|
|
134
|
+
# ATS_SCRAPERS_LIVE_E2E=1. Run them with:
|
|
135
|
+
# ATS_SCRAPERS_LIVE_E2E=1 pytest -m live
|
|
136
|
+
addopts = "-ra --strict-markers -m 'not live'"
|
|
137
|
+
markers = [
|
|
138
|
+
"live: hits real ATS endpoints / the hosted dataset (opt-in)",
|
|
139
|
+
]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""ats-scrapers — open dataset and toolkit for global job market data.
|
|
2
|
+
|
|
3
|
+
Two layers of progressive disclosure:
|
|
4
|
+
|
|
5
|
+
1. Dataset client (zero config):
|
|
6
|
+
>>> from ats_scrapers import search
|
|
7
|
+
>>> df = search(query="ml engineer", location="Paris", ats="greenhouse")
|
|
8
|
+
|
|
9
|
+
2. Per-ATS scrapers (BYO companies):
|
|
10
|
+
>>> from ats_scrapers.scrapers import GreenhouseScraper
|
|
11
|
+
>>> jobs = GreenhouseScraper("anthropic").fetch()
|
|
12
|
+
|
|
13
|
+
Publishing/orchestration code lives in the repo-only ``pipeline/``
|
|
14
|
+
directory and is not part of the installable package.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from ats_scrapers._version import __version__
|
|
18
|
+
from ats_scrapers.client import Client, list_ats, search
|
|
19
|
+
from ats_scrapers.exceptions import (
|
|
20
|
+
ATSScrapersError,
|
|
21
|
+
CompanyNotFoundError,
|
|
22
|
+
ManifestError,
|
|
23
|
+
ScraperError,
|
|
24
|
+
StorageError,
|
|
25
|
+
)
|
|
26
|
+
from ats_scrapers.fetch import Fetcher, FetchResponse, MalformedJSONError
|
|
27
|
+
from ats_scrapers.manifest import Manifest
|
|
28
|
+
from ats_scrapers.models import ATSType, Company, EmploymentType, Job, Salary, SalaryPeriod
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"ATSScrapersError",
|
|
32
|
+
"ATSType",
|
|
33
|
+
"Client",
|
|
34
|
+
"Company",
|
|
35
|
+
"CompanyNotFoundError",
|
|
36
|
+
"EmploymentType",
|
|
37
|
+
"FetchResponse",
|
|
38
|
+
"Fetcher",
|
|
39
|
+
"Job",
|
|
40
|
+
"MalformedJSONError",
|
|
41
|
+
"Manifest",
|
|
42
|
+
"ManifestError",
|
|
43
|
+
"Salary",
|
|
44
|
+
"SalaryPeriod",
|
|
45
|
+
"ScraperError",
|
|
46
|
+
"StorageError",
|
|
47
|
+
"__version__",
|
|
48
|
+
"list_ats",
|
|
49
|
+
"search",
|
|
50
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|