captcha-detector 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- captcha_detector-1.0.0/LICENSE +21 -0
- captcha_detector-1.0.0/PKG-INFO +183 -0
- captcha_detector-1.0.0/README.md +157 -0
- captcha_detector-1.0.0/pyproject.toml +41 -0
- captcha_detector-1.0.0/setup.cfg +4 -0
- captcha_detector-1.0.0/src/captcha_detector/__init__.py +55 -0
- captcha_detector-1.0.0/src/captcha_detector/__main__.py +112 -0
- captcha_detector-1.0.0/src/captcha_detector/browser.py +231 -0
- captcha_detector-1.0.0/src/captcha_detector/captcha-detector.js +2152 -0
- captcha_detector-1.0.0/src/captcha_detector/engine.py +872 -0
- captcha_detector-1.0.0/src/captcha_detector/refine.py +387 -0
- captcha_detector-1.0.0/src/captcha_detector/response.py +156 -0
- captcha_detector-1.0.0/src/captcha_detector/signatures.json +3319 -0
- captcha_detector-1.0.0/src/captcha_detector.egg-info/PKG-INFO +183 -0
- captcha_detector-1.0.0/src/captcha_detector.egg-info/SOURCES.txt +21 -0
- captcha_detector-1.0.0/src/captcha_detector.egg-info/dependency_links.txt +1 -0
- captcha_detector-1.0.0/src/captcha_detector.egg-info/entry_points.txt +2 -0
- captcha_detector-1.0.0/src/captcha_detector.egg-info/requires.txt +6 -0
- captcha_detector-1.0.0/src/captcha_detector.egg-info/top_level.txt +1 -0
- captcha_detector-1.0.0/tests/test_api.py +269 -0
- captcha_detector-1.0.0/tests/test_browser.py +235 -0
- captcha_detector-1.0.0/tests/test_parity.py +45 -0
- captcha_detector-1.0.0/tests/test_selenium.py +82 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ScrapePass
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: captcha-detector
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Does this web page have a captcha? Detects 69 captcha and bot-check products (reCAPTCHA, hCaptcha, Turnstile, DataDome, PerimeterX...) in HTML, scraper API results, Playwright and Selenium. Zero dependencies.
|
|
5
|
+
Author: ScrapePass
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ScrapePass/captcha-detector
|
|
8
|
+
Project-URL: Source, https://github.com/ScrapePass/captcha-detector/tree/main/python
|
|
9
|
+
Project-URL: Issues, https://github.com/ScrapePass/captcha-detector/issues
|
|
10
|
+
Keywords: captcha,captcha-detection,recaptcha,hcaptcha,turnstile,cloudflare,bot-detection,web-scraping,playwright,selenium,datadome
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
18
|
+
Requires-Python: >=3.8
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Provides-Extra: playwright
|
|
22
|
+
Requires-Dist: playwright>=1.30; extra == "playwright"
|
|
23
|
+
Provides-Extra: selenium
|
|
24
|
+
Requires-Dist: selenium>=4; extra == "selenium"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# captcha-detector for Python
|
|
28
|
+
|
|
29
|
+
**Does this web page have a captcha?** The Python version of [captcha-detector](https://github.com/ScrapePass/captcha-detector): the same 69 named captcha and bot-check products plus catch-all rules, the same answers, zero dependencies.
|
|
30
|
+
|
|
31
|
+
- **Check what your scraper got back:** pass a `requests`, `httpx`, `aiohttp`, `curl_cffi`, Scrapy or urllib response, or the JSON a scraper API sent, and find out whether you got the real page or a captcha, a bot check or a block page.
|
|
32
|
+
- **Every kind:** checkbox widgets, invisible/score captchas, puzzles and sliders, proof-of-work, image captchas, math and text questions, full-page bot checks ("Just a moment…", "Press & Hold"), and access-denied pages.
|
|
33
|
+
- **Real browsers too:** Playwright (sync and async) and Selenium, cross-origin frames included, with `visible` telling you whether the captcha is on screen.
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install captcha-detector
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Check a response
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
import requests
|
|
43
|
+
from captcha_detector import detect_response
|
|
44
|
+
|
|
45
|
+
url = "https://shop.example/product/42"
|
|
46
|
+
res = requests.get(url)
|
|
47
|
+
r = detect_response(res)
|
|
48
|
+
|
|
49
|
+
if r.captcha or r.challenge_page or r.blocked:
|
|
50
|
+
print("Not the real page:", r.summary) # e.g. "Captcha found: DataDome captcha (slider) [visible]"
|
|
51
|
+
else:
|
|
52
|
+
parse(res.text)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`detect_response()` takes:
|
|
56
|
+
|
|
57
|
+
| Input | Example |
|
|
58
|
+
|---|---|
|
|
59
|
+
| `requests` / `httpx` / `curl_cffi` / `niquests` response | `detect_response(requests.get(url))` |
|
|
60
|
+
| Scrapy response | `detect_response(response)` inside `parse()` |
|
|
61
|
+
| `urllib.request` / `urllib3` response | `detect_response(urlopen(url))` |
|
|
62
|
+
| `aiohttp` response | `await detect_response_async(resp)` |
|
|
63
|
+
| HTML as `str` or `bytes` | `detect_response(html, url=url)` |
|
|
64
|
+
| A scraper API's JSON (`str`, `dict` or `list`) | `detect_response(api_res.json(), url=target)` |
|
|
65
|
+
|
|
66
|
+
### Results from scraper APIs
|
|
67
|
+
|
|
68
|
+
HTML wrapped in JSON (`{"html": …}`, `{"result": {"content": …}}` and the like) is found on its own, along with the status code, headers and URL next to it.
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
target = "https://shop.example/product/42"
|
|
72
|
+
res = requests.get("https://scraper-api.example/v1", params={"url": target, "key": KEY})
|
|
73
|
+
r = detect_response(res, url=target)
|
|
74
|
+
if r.captcha or r.challenge_page or r.blocked:
|
|
75
|
+
... # retry with another proxy or provider, don't parse or store this one
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Pass `url`, the page you asked the API to scrape, not the API's own address. Most scraper APIs answer `200` whatever the site did; that's fine, the page itself gives a captcha away. If your API reports the site's real status or headers somewhere else, pass them as `status=` and `headers=`.
|
|
79
|
+
|
|
80
|
+
## Check HTML
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from captcha_detector import detect_html
|
|
84
|
+
|
|
85
|
+
r = detect_html(html, url="https://shop.example/", status=403, headers={"server": "cloudflare"})
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## In a real browser
|
|
89
|
+
|
|
90
|
+
Raw HTML only shows what the server sent, and can't tell what is on screen. In a browser the detector sees what scripts added and whether the captcha is visible.
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from playwright.sync_api import sync_playwright
|
|
94
|
+
from captcha_detector import detect_in_page, wait_for_captcha
|
|
95
|
+
|
|
96
|
+
with sync_playwright() as p:
|
|
97
|
+
page = p.chromium.launch().new_page()
|
|
98
|
+
response = page.goto("https://www.google.com/recaptcha/api2/demo")
|
|
99
|
+
r = detect_in_page(page, response=response) # checks every frame, cross-origin ones too
|
|
100
|
+
print(r.summary) # Captcha found: reCAPTCHA (v2-checkbox) [visible]
|
|
101
|
+
|
|
102
|
+
r = wait_for_captcha(page, timeout=10, visible=True) # wait until one is on screen
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Async: `await detect_in_page_async(page)` and `await wait_for_captcha_async(page)`.
|
|
106
|
+
|
|
107
|
+
Selenium:
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from selenium import webdriver
|
|
111
|
+
from captcha_detector import detect_selenium
|
|
112
|
+
|
|
113
|
+
driver = webdriver.Chrome()
|
|
114
|
+
driver.get("https://www.google.com/recaptcha/api2/demo")
|
|
115
|
+
r = detect_selenium(driver) # switches into each frame and back
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Anything else that runs JavaScript can load `captcha_detector.BROWSER_SCRIPT` (it defines `window.CaptchaDetector`) or evaluate `captcha_detector.expression()`, then convert the result with `captcha_detector.from_js()`.
|
|
119
|
+
|
|
120
|
+
## The result
|
|
121
|
+
|
|
122
|
+
A `dict` (so `json.dumps` works) whose keys can also be read as attributes:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
{
|
|
126
|
+
"captcha": True, # a captcha or bot check is on the page (shown, invisible or not rendered yet)
|
|
127
|
+
"visible": None, # on screen right now: True/False in a browser, None for raw HTML
|
|
128
|
+
"challenge_page": False, # the whole page is a bot check
|
|
129
|
+
"blocked": False, # an access-denied page
|
|
130
|
+
"detections": [{
|
|
131
|
+
"id": "recaptcha", "name": "reCAPTCHA", "vendor": "Google",
|
|
132
|
+
"kind": "widget", # widget | invisible | image | question | challenge-page | block-page | protection
|
|
133
|
+
"subtype": "v2-checkbox", "confidence": 0.99, "visible": None,
|
|
134
|
+
"evidence": [{"type": "scripts", "value": "https://www.google.com/recaptcha/api.js"},
|
|
135
|
+
{"type": "elements", "value": ".g-recaptcha"}],
|
|
136
|
+
"site": "https://developers.google.com/recaptcha",
|
|
137
|
+
}],
|
|
138
|
+
"summary": "Captcha found: reCAPTCHA (v2-checkbox)",
|
|
139
|
+
"url": "https://shop.example/", "mode": "html", "version": "1.0.0",
|
|
140
|
+
}
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
`kind: "protection"` means a bot-protection service is loaded (DataDome, Akamai, PerimeterX…) but no captcha is shown; it does not set `captcha`. When the reCAPTCHA or hCaptcha image challenge is open, `details["challenge_open"]` is `True`.
|
|
144
|
+
|
|
145
|
+
Options for every function: `min_confidence` (0..1, default 0.5) and `signatures` (your own list; `SIGNATURES + mine` adds to the built-in ones).
|
|
146
|
+
|
|
147
|
+
## Command line
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
captcha-detector https://www.google.com/recaptcha/api2/demo
|
|
151
|
+
captcha-detector https://example.com --browser --json # needs: pip install playwright
|
|
152
|
+
curl -s https://example.com | captcha-detector -
|
|
153
|
+
curl -s "https://scraper-api.example/v1?url=..." | captcha-detector -
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Options: `--browser`, `--headful`, `--wait MS`, `--json`, `--fail` (exit code 1 when a captcha is found), `--min 0..1`. Also `python -m captcha_detector`.
|
|
157
|
+
|
|
158
|
+
## Add your own signature
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
from captcha_detector import SIGNATURES, detect_html
|
|
162
|
+
|
|
163
|
+
acme = {
|
|
164
|
+
"id": "acme-check", "name": "ACME bot check", "kind": "challenge-page",
|
|
165
|
+
"title": [(r"^acme says hold on", 0.9)], # regex, or (regex, weight)
|
|
166
|
+
"scripts": [r"cdn\.acme-shield\.example/"],
|
|
167
|
+
"elements": ["#acme-check", {"sel": "form", "attr": ["action"], "re": r"/acme/verify", "w": 0.9}],
|
|
168
|
+
"refine": lambda d, x: d if x.short else None, # optional: adjust or drop a match
|
|
169
|
+
}
|
|
170
|
+
r = detect_html(html, signatures=SIGNATURES + [acme])
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Matchers: `scripts`, `frames`, `elements` (CSS: tag, `#id`, `.class`, `[attr op "v" i]`, `:not()`), `inline` (inline script text), `urls`, `title`, `text` (short pages only, unless `(re, w, "any")`), `cookies`, `headers` (`(name, re, w)`), `status`. Matches combine into a confidence; the default threshold is 0.5.
|
|
174
|
+
|
|
175
|
+
To add a product for everyone, add it to `src/signatures.js` in the [repository](https://github.com/ScrapePass/captcha-detector): the JavaScript and Python packages share one signature database.
|
|
176
|
+
|
|
177
|
+
## How it stays in step with the JavaScript package
|
|
178
|
+
|
|
179
|
+
The signature database (`signatures.json`) is generated from the JavaScript source, and the test suite checks that the Python engine gives exactly the same result as the JavaScript engine on every test page.
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
MIT
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# captcha-detector for Python
|
|
2
|
+
|
|
3
|
+
**Does this web page have a captcha?** The Python version of [captcha-detector](https://github.com/ScrapePass/captcha-detector): the same 69 named captcha and bot-check products plus catch-all rules, the same answers, zero dependencies.
|
|
4
|
+
|
|
5
|
+
- **Check what your scraper got back:** pass a `requests`, `httpx`, `aiohttp`, `curl_cffi`, Scrapy or urllib response, or the JSON a scraper API sent, and find out whether you got the real page or a captcha, a bot check or a block page.
|
|
6
|
+
- **Every kind:** checkbox widgets, invisible/score captchas, puzzles and sliders, proof-of-work, image captchas, math and text questions, full-page bot checks ("Just a moment…", "Press & Hold"), and access-denied pages.
|
|
7
|
+
- **Real browsers too:** Playwright (sync and async) and Selenium, cross-origin frames included, with `visible` telling you whether the captcha is on screen.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install captcha-detector
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Check a response
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
import requests
|
|
17
|
+
from captcha_detector import detect_response
|
|
18
|
+
|
|
19
|
+
url = "https://shop.example/product/42"
|
|
20
|
+
res = requests.get(url)
|
|
21
|
+
r = detect_response(res)
|
|
22
|
+
|
|
23
|
+
if r.captcha or r.challenge_page or r.blocked:
|
|
24
|
+
print("Not the real page:", r.summary) # e.g. "Captcha found: DataDome captcha (slider) [visible]"
|
|
25
|
+
else:
|
|
26
|
+
parse(res.text)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`detect_response()` takes:
|
|
30
|
+
|
|
31
|
+
| Input | Example |
|
|
32
|
+
|---|---|
|
|
33
|
+
| `requests` / `httpx` / `curl_cffi` / `niquests` response | `detect_response(requests.get(url))` |
|
|
34
|
+
| Scrapy response | `detect_response(response)` inside `parse()` |
|
|
35
|
+
| `urllib.request` / `urllib3` response | `detect_response(urlopen(url))` |
|
|
36
|
+
| `aiohttp` response | `await detect_response_async(resp)` |
|
|
37
|
+
| HTML as `str` or `bytes` | `detect_response(html, url=url)` |
|
|
38
|
+
| A scraper API's JSON (`str`, `dict` or `list`) | `detect_response(api_res.json(), url=target)` |
|
|
39
|
+
|
|
40
|
+
### Results from scraper APIs
|
|
41
|
+
|
|
42
|
+
HTML wrapped in JSON (`{"html": …}`, `{"result": {"content": …}}` and the like) is found on its own, along with the status code, headers and URL next to it.
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
target = "https://shop.example/product/42"
|
|
46
|
+
res = requests.get("https://scraper-api.example/v1", params={"url": target, "key": KEY})
|
|
47
|
+
r = detect_response(res, url=target)
|
|
48
|
+
if r.captcha or r.challenge_page or r.blocked:
|
|
49
|
+
... # retry with another proxy or provider, don't parse or store this one
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Pass `url`, the page you asked the API to scrape, not the API's own address. Most scraper APIs answer `200` whatever the site did; that's fine, the page itself gives a captcha away. If your API reports the site's real status or headers somewhere else, pass them as `status=` and `headers=`.
|
|
53
|
+
|
|
54
|
+
## Check HTML
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from captcha_detector import detect_html
|
|
58
|
+
|
|
59
|
+
r = detect_html(html, url="https://shop.example/", status=403, headers={"server": "cloudflare"})
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## In a real browser
|
|
63
|
+
|
|
64
|
+
Raw HTML only shows what the server sent, and can't tell what is on screen. In a browser the detector sees what scripts added and whether the captcha is visible.
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from playwright.sync_api import sync_playwright
|
|
68
|
+
from captcha_detector import detect_in_page, wait_for_captcha
|
|
69
|
+
|
|
70
|
+
with sync_playwright() as p:
|
|
71
|
+
page = p.chromium.launch().new_page()
|
|
72
|
+
response = page.goto("https://www.google.com/recaptcha/api2/demo")
|
|
73
|
+
r = detect_in_page(page, response=response) # checks every frame, cross-origin ones too
|
|
74
|
+
print(r.summary) # Captcha found: reCAPTCHA (v2-checkbox) [visible]
|
|
75
|
+
|
|
76
|
+
r = wait_for_captcha(page, timeout=10, visible=True) # wait until one is on screen
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Async: `await detect_in_page_async(page)` and `await wait_for_captcha_async(page)`.
|
|
80
|
+
|
|
81
|
+
Selenium:
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from selenium import webdriver
|
|
85
|
+
from captcha_detector import detect_selenium
|
|
86
|
+
|
|
87
|
+
driver = webdriver.Chrome()
|
|
88
|
+
driver.get("https://www.google.com/recaptcha/api2/demo")
|
|
89
|
+
r = detect_selenium(driver) # switches into each frame and back
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Anything else that runs JavaScript can load `captcha_detector.BROWSER_SCRIPT` (it defines `window.CaptchaDetector`) or evaluate `captcha_detector.expression()`, then convert the result with `captcha_detector.from_js()`.
|
|
93
|
+
|
|
94
|
+
## The result
|
|
95
|
+
|
|
96
|
+
A `dict` (so `json.dumps` works) whose keys can also be read as attributes:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
{
|
|
100
|
+
"captcha": True, # a captcha or bot check is on the page (shown, invisible or not rendered yet)
|
|
101
|
+
"visible": None, # on screen right now: True/False in a browser, None for raw HTML
|
|
102
|
+
"challenge_page": False, # the whole page is a bot check
|
|
103
|
+
"blocked": False, # an access-denied page
|
|
104
|
+
"detections": [{
|
|
105
|
+
"id": "recaptcha", "name": "reCAPTCHA", "vendor": "Google",
|
|
106
|
+
"kind": "widget", # widget | invisible | image | question | challenge-page | block-page | protection
|
|
107
|
+
"subtype": "v2-checkbox", "confidence": 0.99, "visible": None,
|
|
108
|
+
"evidence": [{"type": "scripts", "value": "https://www.google.com/recaptcha/api.js"},
|
|
109
|
+
{"type": "elements", "value": ".g-recaptcha"}],
|
|
110
|
+
"site": "https://developers.google.com/recaptcha",
|
|
111
|
+
}],
|
|
112
|
+
"summary": "Captcha found: reCAPTCHA (v2-checkbox)",
|
|
113
|
+
"url": "https://shop.example/", "mode": "html", "version": "1.0.0",
|
|
114
|
+
}
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
`kind: "protection"` means a bot-protection service is loaded (DataDome, Akamai, PerimeterX…) but no captcha is shown; it does not set `captcha`. When the reCAPTCHA or hCaptcha image challenge is open, `details["challenge_open"]` is `True`.
|
|
118
|
+
|
|
119
|
+
Options for every function: `min_confidence` (0..1, default 0.5) and `signatures` (your own list; `SIGNATURES + mine` adds to the built-in ones).
|
|
120
|
+
|
|
121
|
+
## Command line
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
captcha-detector https://www.google.com/recaptcha/api2/demo
|
|
125
|
+
captcha-detector https://example.com --browser --json # needs: pip install playwright
|
|
126
|
+
curl -s https://example.com | captcha-detector -
|
|
127
|
+
curl -s "https://scraper-api.example/v1?url=..." | captcha-detector -
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Options: `--browser`, `--headful`, `--wait MS`, `--json`, `--fail` (exit code 1 when a captcha is found), `--min 0..1`. Also `python -m captcha_detector`.
|
|
131
|
+
|
|
132
|
+
## Add your own signature
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from captcha_detector import SIGNATURES, detect_html
|
|
136
|
+
|
|
137
|
+
acme = {
|
|
138
|
+
"id": "acme-check", "name": "ACME bot check", "kind": "challenge-page",
|
|
139
|
+
"title": [(r"^acme says hold on", 0.9)], # regex, or (regex, weight)
|
|
140
|
+
"scripts": [r"cdn\.acme-shield\.example/"],
|
|
141
|
+
"elements": ["#acme-check", {"sel": "form", "attr": ["action"], "re": r"/acme/verify", "w": 0.9}],
|
|
142
|
+
"refine": lambda d, x: d if x.short else None, # optional: adjust or drop a match
|
|
143
|
+
}
|
|
144
|
+
r = detect_html(html, signatures=SIGNATURES + [acme])
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Matchers: `scripts`, `frames`, `elements` (CSS: tag, `#id`, `.class`, `[attr op "v" i]`, `:not()`), `inline` (inline script text), `urls`, `title`, `text` (short pages only, unless `(re, w, "any")`), `cookies`, `headers` (`(name, re, w)`), `status`. Matches combine into a confidence; the default threshold is 0.5.
|
|
148
|
+
|
|
149
|
+
To add a product for everyone, add it to `src/signatures.js` in the [repository](https://github.com/ScrapePass/captcha-detector): the JavaScript and Python packages share one signature database.
|
|
150
|
+
|
|
151
|
+
## How it stays in step with the JavaScript package
|
|
152
|
+
|
|
153
|
+
The signature database (`signatures.json`) is generated from the JavaScript source, and the test suite checks that the Python engine gives exactly the same result as the JavaScript engine on every test page.
|
|
154
|
+
|
|
155
|
+
## License
|
|
156
|
+
|
|
157
|
+
MIT
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "captcha-detector"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Does this web page have a captcha? Detects 69 captcha and bot-check products (reCAPTCHA, hCaptcha, Turnstile, DataDome, PerimeterX...) in HTML, scraper API results, Playwright and Selenium. Zero dependencies."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
authors = [{ name = "ScrapePass" }]
|
|
12
|
+
requires-python = ">=3.8"
|
|
13
|
+
keywords = ["captcha", "captcha-detection", "recaptcha", "hcaptcha", "turnstile", "cloudflare", "bot-detection", "web-scraping", "playwright", "selenium", "datadome"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 5 - Production/Stable",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
21
|
+
"Topic :: Software Development :: Libraries",
|
|
22
|
+
]
|
|
23
|
+
dependencies = []
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
playwright = ["playwright>=1.30"]
|
|
27
|
+
selenium = ["selenium>=4"]
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Homepage = "https://github.com/ScrapePass/captcha-detector"
|
|
31
|
+
Source = "https://github.com/ScrapePass/captcha-detector/tree/main/python"
|
|
32
|
+
Issues = "https://github.com/ScrapePass/captcha-detector/issues"
|
|
33
|
+
|
|
34
|
+
[project.scripts]
|
|
35
|
+
captcha-detector = "captcha_detector.__main__:main"
|
|
36
|
+
|
|
37
|
+
[tool.setuptools.packages.find]
|
|
38
|
+
where = ["src"]
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.package-data]
|
|
41
|
+
captcha_detector = ["signatures.json", "captcha-detector.js"]
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""captcha-detector: does this web page have a captcha?
|
|
2
|
+
|
|
3
|
+
from captcha_detector import detect_html, detect_response
|
|
4
|
+
|
|
5
|
+
r = detect_response(requests.get(url), url=url)
|
|
6
|
+
if r.captcha or r.challenge_page or r.blocked:
|
|
7
|
+
print(r.summary)
|
|
8
|
+
|
|
9
|
+
Browser helpers (Playwright, Selenium) are in captcha_detector.browser and re-exported here.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from .engine import (
|
|
13
|
+
SIGNATURES,
|
|
14
|
+
VERSION,
|
|
15
|
+
Detection,
|
|
16
|
+
Result,
|
|
17
|
+
analyze,
|
|
18
|
+
detect_html,
|
|
19
|
+
from_js,
|
|
20
|
+
merge_results,
|
|
21
|
+
snapshot_from_html,
|
|
22
|
+
)
|
|
23
|
+
from .response import detect_response, detect_response_async
|
|
24
|
+
from .browser import (
|
|
25
|
+
BROWSER_SCRIPT,
|
|
26
|
+
detect_in_page,
|
|
27
|
+
detect_in_page_async,
|
|
28
|
+
detect_selenium,
|
|
29
|
+
expression,
|
|
30
|
+
wait_for_captcha,
|
|
31
|
+
wait_for_captcha_async,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__version__ = VERSION
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"VERSION",
|
|
38
|
+
"SIGNATURES",
|
|
39
|
+
"Detection",
|
|
40
|
+
"Result",
|
|
41
|
+
"detect_html",
|
|
42
|
+
"detect_response",
|
|
43
|
+
"detect_response_async",
|
|
44
|
+
"detect_in_page",
|
|
45
|
+
"detect_in_page_async",
|
|
46
|
+
"wait_for_captcha",
|
|
47
|
+
"wait_for_captcha_async",
|
|
48
|
+
"detect_selenium",
|
|
49
|
+
"merge_results",
|
|
50
|
+
"analyze",
|
|
51
|
+
"snapshot_from_html",
|
|
52
|
+
"from_js",
|
|
53
|
+
"expression",
|
|
54
|
+
"BROWSER_SCRIPT",
|
|
55
|
+
]
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Command line: python -m captcha_detector <url | file.html | -> [options] (also installed as `captcha-detector`)."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
import urllib.error
|
|
9
|
+
import urllib.request
|
|
10
|
+
|
|
11
|
+
from . import VERSION, detect_html, detect_response
|
|
12
|
+
|
|
13
|
+
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
|
|
14
|
+
|
|
15
|
+
EPILOG = """examples:
|
|
16
|
+
captcha-detector https://www.google.com/recaptcha/api2/demo
|
|
17
|
+
captcha-detector https://example.com --browser --json
|
|
18
|
+
curl -s https://example.com | captcha-detector -
|
|
19
|
+
curl -s "https://scraper-api.example/v1?url=..." | captcha-detector - # HTML or a scraper API's JSON
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def from_url(url, a):
|
|
24
|
+
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9"})
|
|
25
|
+
try:
|
|
26
|
+
res = urllib.request.urlopen(req, timeout=30)
|
|
27
|
+
except urllib.error.HTTPError as e: # 403/429/503 pages are what we want to see
|
|
28
|
+
res = e
|
|
29
|
+
body = res.read()
|
|
30
|
+
charset = res.headers.get_content_charset() or "utf-8"
|
|
31
|
+
return detect_html(body.decode(charset, "replace"), url=res.geturl(), status=res.getcode(), headers=list(res.headers.items()), min_confidence=a.min)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def from_browser(url, a):
|
|
35
|
+
try:
|
|
36
|
+
from playwright.sync_api import sync_playwright
|
|
37
|
+
except ImportError:
|
|
38
|
+
raise SystemExit("captcha-detector: --browser needs Playwright: pip install playwright && playwright install chromium")
|
|
39
|
+
from .browser import detect_in_page
|
|
40
|
+
|
|
41
|
+
with sync_playwright() as p:
|
|
42
|
+
browser = p.chromium.launch(headless=not a.headful)
|
|
43
|
+
try:
|
|
44
|
+
page = browser.new_context(viewport={"width": 1280, "height": 900}).new_page()
|
|
45
|
+
response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
|
46
|
+
try:
|
|
47
|
+
page.wait_for_load_state("load", timeout=15000)
|
|
48
|
+
except Exception:
|
|
49
|
+
pass
|
|
50
|
+
if a.wait > 0:
|
|
51
|
+
page.wait_for_timeout(a.wait)
|
|
52
|
+
return detect_in_page(page, response=response, min_confidence=a.min)
|
|
53
|
+
finally:
|
|
54
|
+
browser.close()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def show(r):
|
|
58
|
+
tty = sys.stdout.isatty()
|
|
59
|
+
|
|
60
|
+
def c(code, s):
|
|
61
|
+
return "\x1b[%sm%s\x1b[0m" % (code, s) if tty else s
|
|
62
|
+
|
|
63
|
+
head = c("33;1", "⚠ Captcha found") if r["captcha"] else c("31;1", "⛔ Blocked") if r["blocked"] else c("32;1", "✓ No captcha found")
|
|
64
|
+
print(head + (" " + c("2", r["url"]) if r["url"] else ""))
|
|
65
|
+
if r["captcha"]:
|
|
66
|
+
vis = "unknown (static HTML)" if r["visible"] is None else str(r["visible"]).lower()
|
|
67
|
+
print(c("2", " visible: %s challenge page: %s" % (vis, str(r["challenge_page"]).lower())))
|
|
68
|
+
for d in r["detections"]:
|
|
69
|
+
bits = [d["vendor"], d["kind"] + ("/" + d["subtype"] if d["subtype"] else ""), "visible" if d["visible"] is True else "not visible" if d["visible"] is False else None, "%d%%" % int(d["confidence"] * 100 + 0.5)]
|
|
70
|
+
print(" • %s %s" % (c("1", d["name"]), c("2", "— " + " · ".join(b for b in bits if b))))
|
|
71
|
+
for e in d["evidence"][:4]:
|
|
72
|
+
print(c("2", " %s: %s%s" % (e["type"], e["value"], " (in frame)" if e.get("frame") else "")))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def main(argv=None):
|
|
76
|
+
ap = argparse.ArgumentParser(prog="captcha-detector", description="captcha-detector %s: does this page have a captcha?" % VERSION, epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
77
|
+
ap.add_argument("input", help="a URL, an HTML (or scraper API JSON) file, or - for stdin")
|
|
78
|
+
ap.add_argument("--browser", action="store_true", help="open the page in a real browser (needs: pip install playwright) and check the live page and its frames")
|
|
79
|
+
ap.add_argument("--headful", action="store_true", help="show the browser window (with --browser)")
|
|
80
|
+
ap.add_argument("--wait", type=int, default=3000, metavar="MS", help="wait after the page loads, with --browser (default 3000)")
|
|
81
|
+
ap.add_argument("--json", action="store_true", help="print the full result as JSON")
|
|
82
|
+
ap.add_argument("--fail", action="store_true", help="exit with code 1 when a captcha is found")
|
|
83
|
+
ap.add_argument("--min", type=float, default=0.5, metavar="0..1", help="minimum confidence (default 0.5)")
|
|
84
|
+
ap.add_argument("--version", action="version", version=VERSION)
|
|
85
|
+
a = ap.parse_args(argv)
|
|
86
|
+
try:
|
|
87
|
+
if a.input == "-":
|
|
88
|
+
r = detect_response(sys.stdin.buffer.read(), min_confidence=a.min)
|
|
89
|
+
elif re.match(r"^https?://", a.input, re.I):
|
|
90
|
+
r = from_browser(a.input, a) if a.browser else from_url(a.input, a)
|
|
91
|
+
elif os.path.exists(a.input):
|
|
92
|
+
with open(a.input, "rb") as f:
|
|
93
|
+
r = detect_response(f.read(), min_confidence=a.min)
|
|
94
|
+
elif re.match(r"^[\w-]+(\.[\w-]+)+", a.input):
|
|
95
|
+
url = "https://" + a.input
|
|
96
|
+
r = from_browser(url, a) if a.browser else from_url(url, a)
|
|
97
|
+
else:
|
|
98
|
+
raise ValueError("Not a URL or a file: " + a.input)
|
|
99
|
+
except SystemExit:
|
|
100
|
+
raise
|
|
101
|
+
except Exception as e:
|
|
102
|
+
print("captcha-detector: %s" % e, file=sys.stderr)
|
|
103
|
+
return 2
|
|
104
|
+
if a.json:
|
|
105
|
+
print(json.dumps(r, indent=2, ensure_ascii=False))
|
|
106
|
+
else:
|
|
107
|
+
show(r)
|
|
108
|
+
return 1 if a.fail and r["captcha"] else 0
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if __name__ == "__main__":
|
|
112
|
+
sys.exit(main())
|