swe-internship-scraper 1.0.0rc2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swe_internship_scraper-1.0.0rc2/LICENSE +21 -0
- swe_internship_scraper-1.0.0rc2/PKG-INFO +114 -0
- swe_internship_scraper-1.0.0rc2/README.md +85 -0
- swe_internship_scraper-1.0.0rc2/pyproject.toml +76 -0
- swe_internship_scraper-1.0.0rc2/setup.cfg +4 -0
- swe_internship_scraper-1.0.0rc2/src/swe_internship_scraper.egg-info/PKG-INFO +114 -0
- swe_internship_scraper-1.0.0rc2/src/swe_internship_scraper.egg-info/SOURCES.txt +49 -0
- swe_internship_scraper-1.0.0rc2/src/swe_internship_scraper.egg-info/dependency_links.txt +1 -0
- swe_internship_scraper-1.0.0rc2/src/swe_internship_scraper.egg-info/entry_points.txt +2 -0
- swe_internship_scraper-1.0.0rc2/src/swe_internship_scraper.egg-info/requires.txt +4 -0
- swe_internship_scraper-1.0.0rc2/src/swe_internship_scraper.egg-info/top_level.txt +1 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/__init__.py +13 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/__main__.py +4 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/cli.py +250 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/config.py +58 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/data/__init__.py +1 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/data/targets.json +9326 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/dedupe.py +283 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/exporters/__init__.py +6 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/exporters/csv_exporter.py +37 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/exporters/json_exporter.py +21 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/filters.py +54 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/health.py +238 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/models.py +204 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/normalize.py +101 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/notifications.py +37 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/__init__.py +13 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/ashby.py +184 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/base.py +51 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/greenhouse.py +66 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/http.py +105 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/lever.py +67 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/oracle.py +250 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/registry.py +118 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/smartrecruiters.py +172 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/providers/workday.py +107 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/py.typed +0 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/scanner.py +104 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/validation.py +51 -0
- swe_internship_scraper-1.0.0rc2/src/swe_scraper/watch.py +45 -0
- swe_internship_scraper-1.0.0rc2/tests/test_catalog_discovery.py +34 -0
- swe_internship_scraper-1.0.0rc2/tests/test_dedupe_benchmark.py +47 -0
- swe_internship_scraper-1.0.0rc2/tests/test_hardening.py +152 -0
- swe_internship_scraper-1.0.0rc2/tests/test_models_and_normalization.py +87 -0
- swe_internship_scraper-1.0.0rc2/tests/test_provider_contract_fixtures.py +81 -0
- swe_internship_scraper-1.0.0rc2/tests/test_provider_failures.py +322 -0
- swe_internship_scraper-1.0.0rc2/tests/test_providers.py +186 -0
- swe_internship_scraper-1.0.0rc2/tests/test_release_quality.py +324 -0
- swe_internship_scraper-1.0.0rc2/tests/test_scanner_cli.py +112 -0
- swe_internship_scraper-1.0.0rc2/tests/test_v1_contract.py +274 -0
- swe_internship_scraper-1.0.0rc2/tests/test_weekly_health.py +50 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Furkan Candar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: swe-internship-scraper
|
|
3
|
+
Version: 1.0.0rc2
|
|
4
|
+
Summary: Scrape official ATS job boards for software engineering internships.
|
|
5
|
+
Author: Furkan Candar
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/FurkanSource/swe-internship-scraper
|
|
8
|
+
Project-URL: Documentation, https://github.com/FurkanSource/swe-internship-scraper#readme
|
|
9
|
+
Project-URL: Issues, https://github.com/FurkanSource/swe-internship-scraper/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/FurkanSource/swe-internship-scraper/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: internships,jobs,scraper,ats,greenhouse,lever,ashby,workday,smartrecruiters,oracle
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: requests<3,>=2.32
|
|
26
|
+
Provides-Extra: browser
|
|
27
|
+
Requires-Dist: scrapling[fetchers]==0.4.15; extra == "browser"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# SWE Internship Scraper
|
|
31
|
+
|
|
32
|
+
[](https://github.com/FurkanSource/swe-internship-scraper/actions/workflows/ci.yml)
|
|
33
|
+
[](https://github.com/FurkanSource/swe-internship-scraper/actions/workflows/provider-health.yml)
|
|
34
|
+
[](https://pypi.org/project/swe-internship-scraper/)
|
|
35
|
+
|
|
36
|
+
A command-line tool for finding software engineering internships on official public job boards.
|
|
37
|
+
|
|
38
|
+
## Quick start
|
|
39
|
+
|
|
40
|
+
**You do not need to clone or download this repository to use the scraper.** Install
|
|
41
|
+
the package with Python 3.10–3.14, then run a small sample scan. The CSV file is
|
|
42
|
+
written to the folder where you run the command.
|
|
43
|
+
|
|
44
|
+
Windows PowerShell:
|
|
45
|
+
|
|
46
|
+
```powershell
|
|
47
|
+
py -m pip install --pre swe-internship-scraper
|
|
48
|
+
py -m swe_scraper scan --quick --output internships.csv
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
macOS or Linux:
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
python3 -m venv .venv
|
|
55
|
+
.venv/bin/python -m pip install --pre swe-internship-scraper
|
|
56
|
+
.venv/bin/python -m swe_scraper scan --quick --output internships.csv
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Open `internships.csv` in a spreadsheet. `--quick` checks the small set of monitored
|
|
60
|
+
boards; it is a sample, not the full catalog. Omit `--quick` for the default
|
|
61
|
+
priority boards, or use `--target-set all` for the entire catalog. The `--pre`
|
|
62
|
+
install flag is needed while the first release is a release candidate; it can be
|
|
63
|
+
removed for stable `1.0.0`.
|
|
64
|
+
|
|
65
|
+
Want to run the source code you downloaded from GitHub? Follow the
|
|
66
|
+
[source checkout guide](docs/getting-started.md). It has exact Windows and
|
|
67
|
+
macOS/Linux commands, without requiring PowerShell activation.
|
|
68
|
+
|
|
69
|
+
## What you install
|
|
70
|
+
|
|
71
|
+
`swe-internship-scraper` is the main package. It includes the command and six
|
|
72
|
+
providers: Greenhouse, Lever, Ashby, Workday, SmartRecruiters, and Oracle.
|
|
73
|
+
The separate `swe-scraper-icims` package is an **optional, experimental** plugin
|
|
74
|
+
for public iCIMS portals. Most users only need the main package. See the
|
|
75
|
+
[plugin guide](docs/icims-plugin.md) if you have an iCIMS portal to scan.
|
|
76
|
+
|
|
77
|
+
The repository contains scraper code and a public employer catalog. It does not
|
|
78
|
+
include an application tracker, personal data, or applicant ranking rules.
|
|
79
|
+
|
|
80
|
+
## Commands
|
|
81
|
+
|
|
82
|
+
```powershell
|
|
83
|
+
py -m swe_scraper scan --output internships.csv
|
|
84
|
+
py -m swe_scraper scan --location "New York" --output nyc-internships.csv
|
|
85
|
+
py -m swe_scraper scan --target-set all --output all-internships.csv
|
|
86
|
+
py -m swe_scraper providers
|
|
87
|
+
py -m swe_scraper --help
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The installed `swe-scraper` command is equivalent to `py -m swe_scraper` on
|
|
91
|
+
Windows or `python3 -m swe_scraper` in a Python environment on macOS/Linux.
|
|
92
|
+
For JSON output, use a `.json` filename. See the
|
|
93
|
+
[full command guide](docs/command-line.md) for custom targets, watching, health
|
|
94
|
+
checks, and deduplication reports.
|
|
95
|
+
|
|
96
|
+
## Output and deduplication
|
|
97
|
+
|
|
98
|
+
Schema v2 keeps the familiar primary job fields and adds:
|
|
99
|
+
|
|
100
|
+
- `sources`: every provider, source ID, and direct application URL represented by the record;
|
|
101
|
+
- `merge_evidence`: the rule and confidence for each merged source.
|
|
102
|
+
|
|
103
|
+
Exact canonical URLs and provider/source identities have confidence `1.0`. Semantic merging requires canonical company, equivalent title including season and year, and equivalent location. Uncertain pairs remain separate and can be written to the optional deduplication audit.
|
|
104
|
+
|
|
105
|
+
Primary record selection uses completeness and stable lexical tie breakers, so thread completion order does not change output. See [the schema reference](docs/output-schema.md) and [the v2 migration guide](docs/schema-v2-migration.md).
|
|
106
|
+
|
|
107
|
+
## Development
|
|
108
|
+
|
|
109
|
+
Read [CONTRIBUTING.md](CONTRIBUTING.md), [the provider guide](docs/provider-development.md),
|
|
110
|
+
and [the support policy](SUPPORT.md) before opening an issue or pull request.
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
MIT
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# SWE Internship Scraper
|
|
2
|
+
|
|
3
|
+
[](https://github.com/FurkanSource/swe-internship-scraper/actions/workflows/ci.yml)
|
|
4
|
+
[](https://github.com/FurkanSource/swe-internship-scraper/actions/workflows/provider-health.yml)
|
|
5
|
+
[](https://pypi.org/project/swe-internship-scraper/)
|
|
6
|
+
|
|
7
|
+
A command-line tool for finding software engineering internships on official public job boards.
|
|
8
|
+
|
|
9
|
+
## Quick start
|
|
10
|
+
|
|
11
|
+
**You do not need to clone or download this repository to use the scraper.** Install
|
|
12
|
+
the package with Python 3.10–3.14, then run a small sample scan. The CSV file is
|
|
13
|
+
written to the folder where you run the command.
|
|
14
|
+
|
|
15
|
+
Windows PowerShell:
|
|
16
|
+
|
|
17
|
+
```powershell
|
|
18
|
+
py -m pip install --pre swe-internship-scraper
|
|
19
|
+
py -m swe_scraper scan --quick --output internships.csv
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
macOS or Linux:
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
python3 -m venv .venv
|
|
26
|
+
.venv/bin/python -m pip install --pre swe-internship-scraper
|
|
27
|
+
.venv/bin/python -m swe_scraper scan --quick --output internships.csv
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Open `internships.csv` in a spreadsheet. `--quick` checks the small set of monitored
|
|
31
|
+
boards; it is a sample, not the full catalog. Omit `--quick` for the default
|
|
32
|
+
priority boards, or use `--target-set all` for the entire catalog. The `--pre`
|
|
33
|
+
install flag is needed while the first release is a release candidate; it can be
|
|
34
|
+
removed for stable `1.0.0`.
|
|
35
|
+
|
|
36
|
+
Want to run the source code you downloaded from GitHub? Follow the
|
|
37
|
+
[source checkout guide](docs/getting-started.md). It has exact Windows and
|
|
38
|
+
macOS/Linux commands, without requiring PowerShell activation.
|
|
39
|
+
|
|
40
|
+
## What you install
|
|
41
|
+
|
|
42
|
+
`swe-internship-scraper` is the main package. It includes the command and six
|
|
43
|
+
providers: Greenhouse, Lever, Ashby, Workday, SmartRecruiters, and Oracle.
|
|
44
|
+
The separate `swe-scraper-icims` package is an **optional, experimental** plugin
|
|
45
|
+
for public iCIMS portals. Most users only need the main package. See the
|
|
46
|
+
[plugin guide](docs/icims-plugin.md) if you have an iCIMS portal to scan.
|
|
47
|
+
|
|
48
|
+
The repository contains scraper code and a public employer catalog. It does not
|
|
49
|
+
include an application tracker, personal data, or applicant ranking rules.
|
|
50
|
+
|
|
51
|
+
## Commands
|
|
52
|
+
|
|
53
|
+
```powershell
|
|
54
|
+
py -m swe_scraper scan --output internships.csv
|
|
55
|
+
py -m swe_scraper scan --location "New York" --output nyc-internships.csv
|
|
56
|
+
py -m swe_scraper scan --target-set all --output all-internships.csv
|
|
57
|
+
py -m swe_scraper providers
|
|
58
|
+
py -m swe_scraper --help
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The installed `swe-scraper` command is equivalent to `py -m swe_scraper` on
|
|
62
|
+
Windows or `python3 -m swe_scraper` in a Python environment on macOS/Linux.
|
|
63
|
+
For JSON output, use a `.json` filename. See the
|
|
64
|
+
[full command guide](docs/command-line.md) for custom targets, watching, health
|
|
65
|
+
checks, and deduplication reports.
|
|
66
|
+
|
|
67
|
+
## Output and deduplication
|
|
68
|
+
|
|
69
|
+
Schema v2 keeps the familiar primary job fields and adds:
|
|
70
|
+
|
|
71
|
+
- `sources`: every provider, source ID, and direct application URL represented by the record;
|
|
72
|
+
- `merge_evidence`: the rule and confidence for each merged source.
|
|
73
|
+
|
|
74
|
+
Exact canonical URLs and provider/source identities have confidence `1.0`. Semantic merging requires canonical company, equivalent title including season and year, and equivalent location. Uncertain pairs remain separate and can be written to the optional deduplication audit.
|
|
75
|
+
|
|
76
|
+
Primary record selection uses completeness and stable lexical tie breakers, so thread completion order does not change output. See [the schema reference](docs/output-schema.md) and [the v2 migration guide](docs/schema-v2-migration.md).
|
|
77
|
+
|
|
78
|
+
## Development
|
|
79
|
+
|
|
80
|
+
Read [CONTRIBUTING.md](CONTRIBUTING.md), [the provider guide](docs/provider-development.md),
|
|
81
|
+
and [the support policy](SUPPORT.md) before opening an issue or pull request.
|
|
82
|
+
|
|
83
|
+
## License
|
|
84
|
+
|
|
85
|
+
MIT
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel>=0.43"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "swe-internship-scraper"
|
|
7
|
+
version = "1.0.0rc2"
|
|
8
|
+
description = "Scrape official ATS job boards for software engineering internships."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{name = "Furkan Candar"}]
|
|
14
|
+
keywords = ["internships", "jobs", "scraper", "ats", "greenhouse", "lever", "ashby", "workday", "smartrecruiters", "oracle"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
26
|
+
]
|
|
27
|
+
dependencies = [
|
|
28
|
+
"requests>=2.32,<3",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/FurkanSource/swe-internship-scraper"
|
|
33
|
+
Documentation = "https://github.com/FurkanSource/swe-internship-scraper#readme"
|
|
34
|
+
Issues = "https://github.com/FurkanSource/swe-internship-scraper/issues"
|
|
35
|
+
Changelog = "https://github.com/FurkanSource/swe-internship-scraper/blob/main/CHANGELOG.md"
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
browser = ["scrapling[fetchers]==0.4.15"]
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
swe-scraper = "swe_scraper.cli:main"
|
|
42
|
+
|
|
43
|
+
[tool.setuptools]
|
|
44
|
+
package-dir = {"" = "src"}
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.packages.find]
|
|
47
|
+
where = ["src"]
|
|
48
|
+
|
|
49
|
+
[tool.setuptools.package-data]
|
|
50
|
+
swe_scraper = ["data/*.json", "py.typed"]
|
|
51
|
+
|
|
52
|
+
[tool.ruff]
|
|
53
|
+
target-version = "py310"
|
|
54
|
+
line-length = 92
|
|
55
|
+
extend-exclude = ["tools", "server.py", "pipeline_store.py"]
|
|
56
|
+
|
|
57
|
+
[tool.ruff.lint]
|
|
58
|
+
select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
|
|
59
|
+
|
|
60
|
+
[tool.ruff.lint.per-file-ignores]
|
|
61
|
+
"tests/*.py" = ["S101"]
|
|
62
|
+
|
|
63
|
+
[tool.mypy]
|
|
64
|
+
python_version = "3.10"
|
|
65
|
+
strict = true
|
|
66
|
+
files = ["src/swe_scraper", "plugins/icims/src"]
|
|
67
|
+
warn_unreachable = true
|
|
68
|
+
|
|
69
|
+
[tool.coverage.run]
|
|
70
|
+
branch = true
|
|
71
|
+
source = ["swe_scraper", "swe_scraper_icims"]
|
|
72
|
+
|
|
73
|
+
[tool.coverage.report]
|
|
74
|
+
fail_under = 90
|
|
75
|
+
show_missing = true
|
|
76
|
+
skip_covered = true
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: swe-internship-scraper
|
|
3
|
+
Version: 1.0.0rc2
|
|
4
|
+
Summary: Scrape official ATS job boards for software engineering internships.
|
|
5
|
+
Author: Furkan Candar
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/FurkanSource/swe-internship-scraper
|
|
8
|
+
Project-URL: Documentation, https://github.com/FurkanSource/swe-internship-scraper#readme
|
|
9
|
+
Project-URL: Issues, https://github.com/FurkanSource/swe-internship-scraper/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/FurkanSource/swe-internship-scraper/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: internships,jobs,scraper,ats,greenhouse,lever,ashby,workday,smartrecruiters,oracle
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: requests<3,>=2.32
|
|
26
|
+
Provides-Extra: browser
|
|
27
|
+
Requires-Dist: scrapling[fetchers]==0.4.15; extra == "browser"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# SWE Internship Scraper
|
|
31
|
+
|
|
32
|
+
[](https://github.com/FurkanSource/swe-internship-scraper/actions/workflows/ci.yml)
|
|
33
|
+
[](https://github.com/FurkanSource/swe-internship-scraper/actions/workflows/provider-health.yml)
|
|
34
|
+
[](https://pypi.org/project/swe-internship-scraper/)
|
|
35
|
+
|
|
36
|
+
A command-line tool for finding software engineering internships on official public job boards.
|
|
37
|
+
|
|
38
|
+
## Quick start
|
|
39
|
+
|
|
40
|
+
**You do not need to clone or download this repository to use the scraper.** Install
|
|
41
|
+
the package with Python 3.10–3.14, then run a small sample scan. The CSV file is
|
|
42
|
+
written to the folder where you run the command.
|
|
43
|
+
|
|
44
|
+
Windows PowerShell:
|
|
45
|
+
|
|
46
|
+
```powershell
|
|
47
|
+
py -m pip install --pre swe-internship-scraper
|
|
48
|
+
py -m swe_scraper scan --quick --output internships.csv
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
macOS or Linux:
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
python3 -m venv .venv
|
|
55
|
+
.venv/bin/python -m pip install --pre swe-internship-scraper
|
|
56
|
+
.venv/bin/python -m swe_scraper scan --quick --output internships.csv
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Open `internships.csv` in a spreadsheet. `--quick` checks the small set of monitored
|
|
60
|
+
boards; it is a sample, not the full catalog. Omit `--quick` for the default
|
|
61
|
+
priority boards, or use `--target-set all` for the entire catalog. The `--pre`
|
|
62
|
+
install flag is needed while the first release is a release candidate; it can be
|
|
63
|
+
removed for stable `1.0.0`.
|
|
64
|
+
|
|
65
|
+
Want to run the source code you downloaded from GitHub? Follow the
|
|
66
|
+
[source checkout guide](docs/getting-started.md). It has exact Windows and
|
|
67
|
+
macOS/Linux commands, without requiring PowerShell activation.
|
|
68
|
+
|
|
69
|
+
## What you install
|
|
70
|
+
|
|
71
|
+
`swe-internship-scraper` is the main package. It includes the command and six
|
|
72
|
+
providers: Greenhouse, Lever, Ashby, Workday, SmartRecruiters, and Oracle.
|
|
73
|
+
The separate `swe-scraper-icims` package is an **optional, experimental** plugin
|
|
74
|
+
for public iCIMS portals. Most users only need the main package. See the
|
|
75
|
+
[plugin guide](docs/icims-plugin.md) if you have an iCIMS portal to scan.
|
|
76
|
+
|
|
77
|
+
The repository contains scraper code and a public employer catalog. It does not
|
|
78
|
+
include an application tracker, personal data, or applicant ranking rules.
|
|
79
|
+
|
|
80
|
+
## Commands
|
|
81
|
+
|
|
82
|
+
```powershell
|
|
83
|
+
py -m swe_scraper scan --output internships.csv
|
|
84
|
+
py -m swe_scraper scan --location "New York" --output nyc-internships.csv
|
|
85
|
+
py -m swe_scraper scan --target-set all --output all-internships.csv
|
|
86
|
+
py -m swe_scraper providers
|
|
87
|
+
py -m swe_scraper --help
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The installed `swe-scraper` command is equivalent to `py -m swe_scraper` on
|
|
91
|
+
Windows or `python3 -m swe_scraper` in a Python environment on macOS/Linux.
|
|
92
|
+
For JSON output, use a `.json` filename. See the
|
|
93
|
+
[full command guide](docs/command-line.md) for custom targets, watching, health
|
|
94
|
+
checks, and deduplication reports.
|
|
95
|
+
|
|
96
|
+
## Output and deduplication
|
|
97
|
+
|
|
98
|
+
Schema v2 keeps the familiar primary job fields and adds:
|
|
99
|
+
|
|
100
|
+
- `sources`: every provider, source ID, and direct application URL represented by the record;
|
|
101
|
+
- `merge_evidence`: the rule and confidence for each merged source.
|
|
102
|
+
|
|
103
|
+
Exact canonical URLs and provider/source identities have confidence `1.0`. Semantic merging requires canonical company, equivalent title including season and year, and equivalent location. Uncertain pairs remain separate and can be written to the optional deduplication audit.
|
|
104
|
+
|
|
105
|
+
Primary record selection uses completeness and stable lexical tie breakers, so thread completion order does not change output. See [the schema reference](docs/output-schema.md) and [the v2 migration guide](docs/schema-v2-migration.md).
|
|
106
|
+
|
|
107
|
+
## Development
|
|
108
|
+
|
|
109
|
+
Read [CONTRIBUTING.md](CONTRIBUTING.md), [the provider guide](docs/provider-development.md),
|
|
110
|
+
and [the support policy](SUPPORT.md) before opening an issue or pull request.
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
MIT
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/swe_internship_scraper.egg-info/PKG-INFO
|
|
5
|
+
src/swe_internship_scraper.egg-info/SOURCES.txt
|
|
6
|
+
src/swe_internship_scraper.egg-info/dependency_links.txt
|
|
7
|
+
src/swe_internship_scraper.egg-info/entry_points.txt
|
|
8
|
+
src/swe_internship_scraper.egg-info/requires.txt
|
|
9
|
+
src/swe_internship_scraper.egg-info/top_level.txt
|
|
10
|
+
src/swe_scraper/__init__.py
|
|
11
|
+
src/swe_scraper/__main__.py
|
|
12
|
+
src/swe_scraper/cli.py
|
|
13
|
+
src/swe_scraper/config.py
|
|
14
|
+
src/swe_scraper/dedupe.py
|
|
15
|
+
src/swe_scraper/filters.py
|
|
16
|
+
src/swe_scraper/health.py
|
|
17
|
+
src/swe_scraper/models.py
|
|
18
|
+
src/swe_scraper/normalize.py
|
|
19
|
+
src/swe_scraper/notifications.py
|
|
20
|
+
src/swe_scraper/py.typed
|
|
21
|
+
src/swe_scraper/scanner.py
|
|
22
|
+
src/swe_scraper/validation.py
|
|
23
|
+
src/swe_scraper/watch.py
|
|
24
|
+
src/swe_scraper/data/__init__.py
|
|
25
|
+
src/swe_scraper/data/targets.json
|
|
26
|
+
src/swe_scraper/exporters/__init__.py
|
|
27
|
+
src/swe_scraper/exporters/csv_exporter.py
|
|
28
|
+
src/swe_scraper/exporters/json_exporter.py
|
|
29
|
+
src/swe_scraper/providers/__init__.py
|
|
30
|
+
src/swe_scraper/providers/ashby.py
|
|
31
|
+
src/swe_scraper/providers/base.py
|
|
32
|
+
src/swe_scraper/providers/greenhouse.py
|
|
33
|
+
src/swe_scraper/providers/http.py
|
|
34
|
+
src/swe_scraper/providers/lever.py
|
|
35
|
+
src/swe_scraper/providers/oracle.py
|
|
36
|
+
src/swe_scraper/providers/registry.py
|
|
37
|
+
src/swe_scraper/providers/smartrecruiters.py
|
|
38
|
+
src/swe_scraper/providers/workday.py
|
|
39
|
+
tests/test_catalog_discovery.py
|
|
40
|
+
tests/test_dedupe_benchmark.py
|
|
41
|
+
tests/test_hardening.py
|
|
42
|
+
tests/test_models_and_normalization.py
|
|
43
|
+
tests/test_provider_contract_fixtures.py
|
|
44
|
+
tests/test_provider_failures.py
|
|
45
|
+
tests/test_providers.py
|
|
46
|
+
tests/test_release_quality.py
|
|
47
|
+
tests/test_scanner_cli.py
|
|
48
|
+
tests/test_v1_contract.py
|
|
49
|
+
tests/test_weekly_health.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
swe_scraper
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Reusable software engineering internship scraper."""
|
|
2
|
+
|
|
3
|
+
__version__ = "1.0.0rc2"
|
|
4
|
+
|
|
5
|
+
from .models import Job, JobSource, MatchEvidence, ProviderFailure, ScanResult
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"Job",
|
|
9
|
+
"JobSource",
|
|
10
|
+
"MatchEvidence",
|
|
11
|
+
"ProviderFailure",
|
|
12
|
+
"ScanResult",
|
|
13
|
+
]
|