barebidi 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- barebidi-0.1.0/.claude/settings.local.json +8 -0
- barebidi-0.1.0/.dockerignore +12 -0
- barebidi-0.1.0/.gitignore +14 -0
- barebidi-0.1.0/Dockerfile +51 -0
- barebidi-0.1.0/LICENSE +21 -0
- barebidi-0.1.0/PKG-INFO +213 -0
- barebidi-0.1.0/README.md +203 -0
- barebidi-0.1.0/detection/behavior.py +24 -0
- barebidi-0.1.0/detection/browserscan.py +25 -0
- barebidi-0.1.0/detection/cloudflare.py +79 -0
- barebidi-0.1.0/detection/common.py +92 -0
- barebidi-0.1.0/detection/creepjs.py +21 -0
- barebidi-0.1.0/detection/deviceandbrowserinfo.py +18 -0
- barebidi-0.1.0/detection/run_all.py +22 -0
- barebidi-0.1.0/detection/sannysoft.py +31 -0
- barebidi-0.1.0/docker/desktop +11 -0
- barebidi-0.1.0/docker/jwmrc +15 -0
- barebidi-0.1.0/examples/camoufox.py +86 -0
- barebidi-0.1.0/examples/ny_courts.py +232 -0
- barebidi-0.1.0/examples/ny_courts_html.py +126 -0
- barebidi-0.1.0/pyproject.toml +24 -0
- barebidi-0.1.0/src/barebidi/__init__.py +33 -0
- barebidi-0.1.0/src/barebidi/browser.py +458 -0
- barebidi-0.1.0/src/barebidi/connection.py +123 -0
- barebidi-0.1.0/src/barebidi/context.py +96 -0
- barebidi-0.1.0/src/barebidi/dialogs.py +87 -0
- barebidi-0.1.0/src/barebidi/downloads.py +83 -0
- barebidi-0.1.0/src/barebidi/element.py +320 -0
- barebidi-0.1.0/src/barebidi/frame.py +291 -0
- barebidi-0.1.0/src/barebidi/human.py +150 -0
- barebidi-0.1.0/src/barebidi/network.py +354 -0
- barebidi-0.1.0/src/barebidi/protocol.py +1996 -0
- barebidi-0.1.0/src/barebidi/proxy.py +141 -0
- barebidi-0.1.0/src/barebidi/stealth.py +58 -0
- barebidi-0.1.0/src/barebidi/tab.py +116 -0
- barebidi-0.1.0/src/barebidi/values.py +76 -0
- barebidi-0.1.0/tests/conftest.py +149 -0
- barebidi-0.1.0/tests/pages/behavior.html +18 -0
- barebidi-0.1.0/tests/pages/data.json +1 -0
- barebidi-0.1.0/tests/pages/dialogs.html +11 -0
- barebidi-0.1.0/tests/pages/files.html +15 -0
- barebidi-0.1.0/tests/pages/frame_inner.html +17 -0
- barebidi-0.1.0/tests/pages/frames.html +36 -0
- barebidi-0.1.0/tests/pages/images.html +5 -0
- barebidi-0.1.0/tests/pages/index.html +21 -0
- barebidi-0.1.0/tests/pages/pixel.png +0 -0
- barebidi-0.1.0/tests/pages/second.html +5 -0
- barebidi-0.1.0/tests/pages/stealth.html +8 -0
- barebidi-0.1.0/tests/test_bidi_codegen.py +97 -0
- barebidi-0.1.0/tests/test_browser.py +128 -0
- barebidi-0.1.0/tests/test_chaining.py +51 -0
- barebidi-0.1.0/tests/test_dialogs.py +77 -0
- barebidi-0.1.0/tests/test_files.py +70 -0
- barebidi-0.1.0/tests/test_frames.py +104 -0
- barebidi-0.1.0/tests/test_human.py +113 -0
- barebidi-0.1.0/tests/test_identity.py +179 -0
- barebidi-0.1.0/tests/test_network.py +200 -0
- barebidi-0.1.0/tests/test_robustness.py +84 -0
- barebidi-0.1.0/tests/test_stealth.py +137 -0
- barebidi-0.1.0/tools/bidi_codegen.py +357 -0
- barebidi-0.1.0/tools/webdriver-bidi.cddl +2633 -0
- barebidi-0.1.0/uv.lock +244 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Ubuntu with Firefox that, seen from a web page, looks like an ordinary Ubuntu desktop: a screen with a top
|
|
2
|
+
# bar and a maximized window, a sound device, Ubuntu's fonts, video codecs and WebGL. For running
|
|
3
|
+
# barebidi (and its tests) on servers.
|
|
4
|
+
#
|
|
5
|
+
# docker build -t barebidi .
|
|
6
|
+
# docker run --rm --shm-size=2g barebidi # the test suite
|
|
7
|
+
# docker run --rm --shm-size=2g -e TZ=America/New_York barebidi python your_script.py
|
|
8
|
+
#
|
|
9
|
+
# Firefox needs more shared memory than Docker's default 64 MB, hence --shm-size. Set TZ (and the browser's
|
|
10
|
+
# locale) to match where your IP or proxy is. Use Browser(headless=False): on Linux, headless Firefox has no
|
|
11
|
+
# WebGL at all and detectors find it less like a desktop.
|
|
12
|
+
FROM ubuntu:24.04 AS system
|
|
13
|
+
ENV DEBIAN_FRONTEND=noninteractive
|
|
14
|
+
# Firefox from Mozilla's own repository: Ubuntu's package is only a wrapper around the snap.
|
|
15
|
+
RUN apt-get update \
|
|
16
|
+
&& apt-get install -y --no-install-recommends ca-certificates curl \
|
|
17
|
+
&& install -d -m 0755 /etc/apt/keyrings \
|
|
18
|
+
&& curl -fsSL https://packages.mozilla.org/apt/repo-signing-key.gpg -o /etc/apt/keyrings/packages.mozilla.org.asc \
|
|
19
|
+
&& echo "deb [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" > /etc/apt/sources.list.d/mozilla.list \
|
|
20
|
+
&& printf 'Package: *\nPin: origin packages.mozilla.org\nPin-Priority: 1000\n' > /etc/apt/preferences.d/mozilla \
|
|
21
|
+
&& apt-get update \
|
|
22
|
+
&& apt-get install -y --no-install-recommends \
|
|
23
|
+
firefox python3 tini tzdata \
|
|
24
|
+
xvfb xauth jwm pulseaudio \
|
|
25
|
+
libgl1-mesa-dri libegl1 libgl1 libavcodec60 \
|
|
26
|
+
fonts-ubuntu fonts-dejavu-core fonts-liberation2 fonts-noto-color-emoji \
|
|
27
|
+
&& apt-get purge -y curl && apt-get autoremove -y \
|
|
28
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
29
|
+
|
|
30
|
+
FROM system AS build
|
|
31
|
+
COPY --from=ghcr.io/astral-sh/uv:0.9.27 /uv /usr/local/bin/uv
|
|
32
|
+
ENV UV_PYTHON=/usr/bin/python3 UV_PYTHON_DOWNLOADS=never
|
|
33
|
+
WORKDIR /app
|
|
34
|
+
COPY pyproject.toml uv.lock ./
|
|
35
|
+
RUN uv sync --frozen --no-install-project
|
|
36
|
+
COPY . .
|
|
37
|
+
RUN uv sync --frozen
|
|
38
|
+
RUN uv run --with camoufox python -m camoufox fetch
|
|
39
|
+
|
|
40
|
+
FROM system
|
|
41
|
+
COPY docker/desktop /usr/local/bin/desktop
|
|
42
|
+
COPY --from=ghcr.io/astral-sh/uv:0.9.27 /uv /usr/local/bin/uv
|
|
43
|
+
RUN useradd --create-home scraper
|
|
44
|
+
COPY --chown=scraper docker/jwmrc /home/scraper/.jwmrc
|
|
45
|
+
COPY --from=build --chown=scraper /app /app
|
|
46
|
+
COPY --from=build --chown=scraper /root/.cache/camoufox /home/scraper/.cache/camoufox
|
|
47
|
+
USER scraper
|
|
48
|
+
WORKDIR /app
|
|
49
|
+
ENV PATH=/app/.venv/bin:$PATH UV_PYTHON=/usr/bin/python3 UV_PYTHON_DOWNLOADS=never
|
|
50
|
+
ENTRYPOINT ["tini", "--", "desktop"]
|
|
51
|
+
CMD ["python", "-m", "pytest", "-q", "-p", "no:cacheprovider"]
|
barebidi-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ernesto
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
barebidi-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: barebidi
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Async Firefox automation for Python, speaking WebDriver BiDi directly, no driver binaries.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Requires-Dist: websockets>=13
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
BAREBIDI
|
|
12
|
+
========
|
|
13
|
+
|
|
14
|
+
### Async Firefox automation for Python, speaking WebDriver BiDi straight to the browser
|
|
15
|
+
|
|
16
|
+
- **built for Firefox**: the regular Firefox you download from Mozilla, nothing patched
|
|
17
|
+
- no driver binaries: no geckodriver, no Selenium
|
|
18
|
+
- no Playwright-style browser builds: it drives the Firefox you already have
|
|
19
|
+
- one dependency: `websockets`
|
|
20
|
+
|
|
21
|
+
WebDriver BiDi is the W3C standard for browser automation, the successor of both classic WebDriver and the
|
|
22
|
+
Chrome DevTools Protocol. Firefox speaks it natively: barebidi opens a websocket to Firefox and talks to it
|
|
23
|
+
directly, so there is nothing in between to install, keep in sync or get detected.
|
|
24
|
+
|
|
25
|
+
Most automation tools are Chrome-first and treat Firefox as an afterthought. barebidi is the other way
|
|
26
|
+
around: every feature is built and tested against Firefox, including its quirks with cross-site iframes,
|
|
27
|
+
closed shadow roots and input routing. Chrome support may come later; Firefox is the point.
|
|
28
|
+
|
|
29
|
+
### Some features
|
|
30
|
+
|
|
31
|
+
- `tab.select(css)`, `tab.find(text)`, `tab.xpath(expr)`, all waiting until the element appears
|
|
32
|
+
- chainable queries: `await tab.select("form").select("button").click()`
|
|
33
|
+
- mouse and keyboard input with human timings and trajectories, on by default
|
|
34
|
+
- elements inside cross-site iframes and closed shadow roots, the ones scripts can't reach
|
|
35
|
+
- proxies with credentials (no password prompt), plus per-context locale and time zone
|
|
36
|
+
- isolated browser contexts: separate cookies, storage and proxy in one browser
|
|
37
|
+
- network: intercept, block or modify requests, wait for responses, extra headers
|
|
38
|
+
- dialogs (alert, confirm, prompt), file uploads and downloads
|
|
39
|
+
- cookies: read, set, clear
|
|
40
|
+
- every protocol type generated from the spec's CDDL, so your editor knows the shape of every message
|
|
41
|
+
|
|
42
|
+
# Installation
|
|
43
|
+
|
|
44
|
+
barebidi needs Firefox installed, from Mozilla (on Ubuntu, not the snap). It finds it in the usual places;
|
|
45
|
+
otherwise set `FIREFOX_PATH` or pass `executable_path=`. On a headless server, run Firefox under Xvfb (or use
|
|
46
|
+
`headless=True`).
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
pip install barebidi
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
To update:
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
pip install -U barebidi
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
# Usage examples
|
|
59
|
+
|
|
60
|
+
The browser starts with sensible defaults: a temporary profile, stealth and human input on. Everything is
|
|
61
|
+
async, and the browser closes itself when the `async with` block ends.
|
|
62
|
+
|
|
63
|
+
## Simple
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
import asyncio
|
|
67
|
+
import barebidi as bb
|
|
68
|
+
|
|
69
|
+
async def main():
|
|
70
|
+
async with bb.Browser() as browser:
|
|
71
|
+
tab = await browser.get("https://example.com")
|
|
72
|
+
print(await tab.title())
|
|
73
|
+
|
|
74
|
+
... further code ...
|
|
75
|
+
|
|
76
|
+
if __name__ == "__main__":
|
|
77
|
+
asyncio.run(main())
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Or without the block, nodriver style:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
browser = await bb.start()
|
|
84
|
+
tab = await browser.get("https://example.com")
|
|
85
|
+
...
|
|
86
|
+
await browser.stop()
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Custom starting options
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
browser = await bb.start(
|
|
93
|
+
headless=True,
|
|
94
|
+
screen=(1920, 1080), # headless only; a real window keeps the real screen
|
|
95
|
+
profile="/path/to/profile", # keep cookies and storage between runs
|
|
96
|
+
proxy="http://user:pass@host:8080", # credentials are handled for you
|
|
97
|
+
locale="en-US",
|
|
98
|
+
timezone="America/New_York",
|
|
99
|
+
executable_path="/path/to/firefox", # or set FIREFOX_PATH
|
|
100
|
+
args=["--some-firefox-arg"],
|
|
101
|
+
prefs={"some.pref": True}, # Firefox about:config prefs
|
|
102
|
+
downloads="/path/to/downloads",
|
|
103
|
+
humanize=True, # human-like mouse and keyboard
|
|
104
|
+
)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Finding and using elements
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
tab = await browser.get("https://example.com/login")
|
|
111
|
+
|
|
112
|
+
await tab.select("input[name=user]").type("me@example.com")
|
|
113
|
+
await tab.select("input[name=password]").type("secret")
|
|
114
|
+
await tab.find("Sign in").click()
|
|
115
|
+
|
|
116
|
+
items = await tab.select_all(".item")
|
|
117
|
+
for item in items:
|
|
118
|
+
print(await item.text(), await item.get_attribute("href"))
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Inside an iframe or a shadow root:
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
frame = await tab.frame("widget.example.org") # by URL
|
|
125
|
+
button = await frame.select("button")
|
|
126
|
+
|
|
127
|
+
root = await (await tab.select("my-component")).shadow_root()
|
|
128
|
+
await root.select("input").type("hello")
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Network
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
async with tab.expect_response("/api/items") as expected:
|
|
135
|
+
await tab.find("Load more").click()
|
|
136
|
+
items = await expected.response.json()
|
|
137
|
+
|
|
138
|
+
await tab.block(resource_types=["image", "font"])
|
|
139
|
+
await tab.set_extra_headers({"X-Debug": "1"})
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Dialogs, uploads and downloads
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
async def answer(dialog):
|
|
146
|
+
await dialog.accept("Barebidi")
|
|
147
|
+
|
|
148
|
+
await tab.on_dialog(answer)
|
|
149
|
+
|
|
150
|
+
await tab.select("input[type=file]").set_files("report.pdf")
|
|
151
|
+
|
|
152
|
+
async with tab.expect_download() as expected:
|
|
153
|
+
await tab.find("Export CSV").click()
|
|
154
|
+
path = await expected.download.path()
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Separate identities in one browser
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
us = await browser.new_context(proxy="http://user:pass@us-proxy:8080", locale="en-US", timezone="America/Chicago")
|
|
161
|
+
de = await browser.new_context(proxy="http://user:pass@de-proxy:8080", locale="de-DE", timezone="Europe/Berlin")
|
|
162
|
+
|
|
163
|
+
await us.get("https://example.com")
|
|
164
|
+
await de.get("https://example.com") # different cookies, storage, IP, language and clock
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## Raw protocol
|
|
168
|
+
|
|
169
|
+
Anything barebidi doesn't wrap is one call away:
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
result = await tab.send("browsingContext.captureScreenshot", {"context": tab.id, "origin": "document"})
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
# Note: driving Camoufox
|
|
176
|
+
|
|
177
|
+
[Camoufox](https://github.com/daijro/camoufox) is a patched Firefox build. Since it is still Firefox and
|
|
178
|
+
still speaks WebDriver BiDi, barebidi drives it like any other Firefox: point `executable_path` at its binary.
|
|
179
|
+
|
|
180
|
+
1. Download Camoufox with its own tool:
|
|
181
|
+
|
|
182
|
+
```
|
|
183
|
+
pip install camoufox
|
|
184
|
+
python -m camoufox fetch
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
2. Find the binary it downloaded:
|
|
188
|
+
|
|
189
|
+
- macOS: `~/Library/Caches/camoufox/browsers/official/<version>/Camoufox.app/Contents/MacOS/camoufox`
|
|
190
|
+
- Linux: `~/.cache/camoufox/browsers/official/<version>/camoufox`
|
|
191
|
+
|
|
192
|
+
3. Start barebidi with that path. Turn barebidi's own stealth off: Camoufox handles that part itself.
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
browser = await bb.start(executable_path="/path/to/camoufox", stealth=False)
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
4. Optionally, pass Camoufox its settings as JSON in the `CAMOU_CONFIG_1` environment variable through
|
|
199
|
+
`env=`. A config longer than 32767 characters goes in pieces: `CAMOU_CONFIG_1`, `CAMOU_CONFIG_2`, ...
|
|
200
|
+
See Camoufox's documentation for the available keys.
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
import json
|
|
204
|
+
|
|
205
|
+
config = {"navigator.hardwareConcurrency": 8}
|
|
206
|
+
browser = await bb.start(
|
|
207
|
+
executable_path="/path/to/camoufox",
|
|
208
|
+
stealth=False,
|
|
209
|
+
env={"CAMOU_CONFIG_1": json.dumps(config)},
|
|
210
|
+
)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
The `camoufox` package is only needed to download the browser; barebidi doesn't depend on it.
|
barebidi-0.1.0/README.md
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
BAREBIDI
|
|
2
|
+
========
|
|
3
|
+
|
|
4
|
+
### Async Firefox automation for Python, speaking WebDriver BiDi straight to the browser
|
|
5
|
+
|
|
6
|
+
- **built for Firefox**: the regular Firefox you download from Mozilla, nothing patched
|
|
7
|
+
- no driver binaries: no geckodriver, no Selenium
|
|
8
|
+
- no Playwright-style browser builds: it drives the Firefox you already have
|
|
9
|
+
- one dependency: `websockets`
|
|
10
|
+
|
|
11
|
+
WebDriver BiDi is the W3C standard for browser automation, the successor of both classic WebDriver and the
|
|
12
|
+
Chrome DevTools Protocol. Firefox speaks it natively: barebidi opens a websocket to Firefox and talks to it
|
|
13
|
+
directly, so there is nothing in between to install, keep in sync or get detected.
|
|
14
|
+
|
|
15
|
+
Most automation tools are Chrome-first and treat Firefox as an afterthought. barebidi is the other way
|
|
16
|
+
around: every feature is built and tested against Firefox, including its quirks with cross-site iframes,
|
|
17
|
+
closed shadow roots and input routing. Chrome support may come later; Firefox is the point.
|
|
18
|
+
|
|
19
|
+
### Some features
|
|
20
|
+
|
|
21
|
+
- `tab.select(css)`, `tab.find(text)`, `tab.xpath(expr)`, all waiting until the element appears
|
|
22
|
+
- chainable queries: `await tab.select("form").select("button").click()`
|
|
23
|
+
- mouse and keyboard input with human timings and trajectories, on by default
|
|
24
|
+
- elements inside cross-site iframes and closed shadow roots, the ones scripts can't reach
|
|
25
|
+
- proxies with credentials (no password prompt), plus per-context locale and time zone
|
|
26
|
+
- isolated browser contexts: separate cookies, storage and proxy in one browser
|
|
27
|
+
- network: intercept, block or modify requests, wait for responses, extra headers
|
|
28
|
+
- dialogs (alert, confirm, prompt), file uploads and downloads
|
|
29
|
+
- cookies: read, set, clear
|
|
30
|
+
- every protocol type generated from the spec's CDDL, so your editor knows the shape of every message
|
|
31
|
+
|
|
32
|
+
# Installation
|
|
33
|
+
|
|
34
|
+
barebidi needs Firefox installed, from Mozilla (on Ubuntu, not the snap). It finds it in the usual places;
|
|
35
|
+
otherwise set `FIREFOX_PATH` or pass `executable_path=`. On a headless server, run Firefox under Xvfb (or use
|
|
36
|
+
`headless=True`).
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
pip install barebidi
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
To update:
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
pip install -U barebidi
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
# Usage examples
|
|
49
|
+
|
|
50
|
+
The browser starts with sensible defaults: a temporary profile, stealth and human input on. Everything is
|
|
51
|
+
async, and the browser closes itself when the `async with` block ends.
|
|
52
|
+
|
|
53
|
+
## Simple
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import asyncio
|
|
57
|
+
import barebidi as bb
|
|
58
|
+
|
|
59
|
+
async def main():
|
|
60
|
+
async with bb.Browser() as browser:
|
|
61
|
+
tab = await browser.get("https://example.com")
|
|
62
|
+
print(await tab.title())
|
|
63
|
+
|
|
64
|
+
... further code ...
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
asyncio.run(main())
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Or without the block, nodriver style:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
browser = await bb.start()
|
|
74
|
+
tab = await browser.get("https://example.com")
|
|
75
|
+
...
|
|
76
|
+
await browser.stop()
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Custom starting options
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
browser = await bb.start(
|
|
83
|
+
headless=True,
|
|
84
|
+
screen=(1920, 1080), # headless only; a real window keeps the real screen
|
|
85
|
+
profile="/path/to/profile", # keep cookies and storage between runs
|
|
86
|
+
proxy="http://user:pass@host:8080", # credentials are handled for you
|
|
87
|
+
locale="en-US",
|
|
88
|
+
timezone="America/New_York",
|
|
89
|
+
executable_path="/path/to/firefox", # or set FIREFOX_PATH
|
|
90
|
+
args=["--some-firefox-arg"],
|
|
91
|
+
prefs={"some.pref": True}, # Firefox about:config prefs
|
|
92
|
+
downloads="/path/to/downloads",
|
|
93
|
+
humanize=True, # human-like mouse and keyboard
|
|
94
|
+
)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Finding and using elements
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
tab = await browser.get("https://example.com/login")
|
|
101
|
+
|
|
102
|
+
await tab.select("input[name=user]").type("me@example.com")
|
|
103
|
+
await tab.select("input[name=password]").type("secret")
|
|
104
|
+
await tab.find("Sign in").click()
|
|
105
|
+
|
|
106
|
+
items = await tab.select_all(".item")
|
|
107
|
+
for item in items:
|
|
108
|
+
print(await item.text(), await item.get_attribute("href"))
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Inside an iframe or a shadow root:
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
frame = await tab.frame("widget.example.org") # by URL
|
|
115
|
+
button = await frame.select("button")
|
|
116
|
+
|
|
117
|
+
root = await (await tab.select("my-component")).shadow_root()
|
|
118
|
+
await root.select("input").type("hello")
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## Network
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
async with tab.expect_response("/api/items") as expected:
|
|
125
|
+
await tab.find("Load more").click()
|
|
126
|
+
items = await expected.response.json()
|
|
127
|
+
|
|
128
|
+
await tab.block(resource_types=["image", "font"])
|
|
129
|
+
await tab.set_extra_headers({"X-Debug": "1"})
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Dialogs, uploads and downloads
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
async def answer(dialog):
|
|
136
|
+
await dialog.accept("Barebidi")
|
|
137
|
+
|
|
138
|
+
await tab.on_dialog(answer)
|
|
139
|
+
|
|
140
|
+
await tab.select("input[type=file]").set_files("report.pdf")
|
|
141
|
+
|
|
142
|
+
async with tab.expect_download() as expected:
|
|
143
|
+
await tab.find("Export CSV").click()
|
|
144
|
+
path = await expected.download.path()
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
## Separate identities in one browser
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
us = await browser.new_context(proxy="http://user:pass@us-proxy:8080", locale="en-US", timezone="America/Chicago")
|
|
151
|
+
de = await browser.new_context(proxy="http://user:pass@de-proxy:8080", locale="de-DE", timezone="Europe/Berlin")
|
|
152
|
+
|
|
153
|
+
await us.get("https://example.com")
|
|
154
|
+
await de.get("https://example.com") # different cookies, storage, IP, language and clock
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Raw protocol
|
|
158
|
+
|
|
159
|
+
Anything barebidi doesn't wrap is one call away:
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
result = await tab.send("browsingContext.captureScreenshot", {"context": tab.id, "origin": "document"})
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
# Note: driving Camoufox
|
|
166
|
+
|
|
167
|
+
[Camoufox](https://github.com/daijro/camoufox) is a patched Firefox build. Since it is still Firefox and
|
|
168
|
+
still speaks WebDriver BiDi, barebidi drives it like any other Firefox: point `executable_path` at its binary.
|
|
169
|
+
|
|
170
|
+
1. Download Camoufox with its own tool:
|
|
171
|
+
|
|
172
|
+
```
|
|
173
|
+
pip install camoufox
|
|
174
|
+
python -m camoufox fetch
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
2. Find the binary it downloaded:
|
|
178
|
+
|
|
179
|
+
- macOS: `~/Library/Caches/camoufox/browsers/official/<version>/Camoufox.app/Contents/MacOS/camoufox`
|
|
180
|
+
- Linux: `~/.cache/camoufox/browsers/official/<version>/camoufox`
|
|
181
|
+
|
|
182
|
+
3. Start barebidi with that path. Turn barebidi's own stealth off: Camoufox handles that part itself.
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
browser = await bb.start(executable_path="/path/to/camoufox", stealth=False)
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
4. Optionally, pass Camoufox its settings as JSON in the `CAMOU_CONFIG_1` environment variable through
|
|
189
|
+
`env=`. A config longer than 32767 characters goes in pieces: `CAMOU_CONFIG_1`, `CAMOU_CONFIG_2`, ...
|
|
190
|
+
See Camoufox's documentation for the available keys.
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
import json
|
|
194
|
+
|
|
195
|
+
config = {"navigator.hardwareConcurrency": 8}
|
|
196
|
+
browser = await bb.start(
|
|
197
|
+
executable_path="/path/to/camoufox",
|
|
198
|
+
stealth=False,
|
|
199
|
+
env={"CAMOU_CONFIG_1": json.dumps(config)},
|
|
200
|
+
)
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
The `camoufox` package is only needed to download the browser; barebidi doesn't depend on it.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""deviceandbrowserinfo.com interactions test: fills its login form like a person, then reads the verdict.
|
|
2
|
+
|
|
3
|
+
uv run python detection/behavior.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
|
|
8
|
+
from common import page_text, run, settle, verdict_and_flags
|
|
9
|
+
|
|
10
|
+
NAME = "deviceandbrowserinfo (comportamiento)"
|
|
11
|
+
URL = "https://deviceandbrowserinfo.com/are_you_a_bot_interactions"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
async def check(tab) -> str:
|
|
15
|
+
await settle(tab, URL, 2)
|
|
16
|
+
await tab.select("#email").type("maria.lopez@example.com")
|
|
17
|
+
await tab.select("#password").type("Secreta123!")
|
|
18
|
+
await tab.select("#loginForm button[type=submit]").click()
|
|
19
|
+
await asyncio.sleep(4)
|
|
20
|
+
return verdict_and_flags(await page_text(tab))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
if __name__ == "__main__":
|
|
24
|
+
run((NAME, check))
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""browserscan.net bot detection: an overall Normal/Robot verdict plus one per check.
|
|
2
|
+
|
|
3
|
+
uv run python detection/browserscan.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from common import page_text, run, settle
|
|
7
|
+
|
|
8
|
+
NAME = "browserscan.net"
|
|
9
|
+
URL = "https://www.browserscan.net/bot-detection"
|
|
10
|
+
VERDICTS = ("Normal", "Robot")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
async def check(tab) -> str:
|
|
14
|
+
await settle(tab, URL, 8)
|
|
15
|
+
lines = [line.strip() for line in (await page_text(tab)).splitlines() if line.strip()]
|
|
16
|
+
start = lines.index("Bot Detection") if "Bot Detection" in lines else 0
|
|
17
|
+
verdict = next((line for line in lines[start : start + 6] if line in VERDICTS), "sin veredicto")
|
|
18
|
+
robots = [lines[i - 1] for i, line in enumerate(lines) if line == "Robot" and i > start + 1]
|
|
19
|
+
for label in robots:
|
|
20
|
+
print(f" Robot: {label}")
|
|
21
|
+
return verdict + (f" (Robot en: {', '.join(robots)})" if robots else "")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
if __name__ == "__main__":
|
|
25
|
+
run((NAME, check))
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Cloudflare: scrapingcourse.com's bot challenge, and a click on nowsecure.nl's Turnstile checkbox.
|
|
2
|
+
|
|
3
|
+
nowsecure.nl has no challenge anymore (curl gets in too) and its Turnstile widget uses one of Cloudflare's
|
|
4
|
+
test sitekeys, which give every visitor the same answer. So that check measures our click, not detection:
|
|
5
|
+
a checkbox inside a cross-site iframe, behind two closed shadow roots, on a 3D cube that never stops moving.
|
|
6
|
+
|
|
7
|
+
uv run python detection/cloudflare.py
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from common import page_text, passes_cloudflare, run
|
|
14
|
+
|
|
15
|
+
SCRAPINGCOURSE = "https://www.scrapingcourse.com/antibot-challenge"
|
|
16
|
+
NOWSECURE = "https://nowsecure.nl/"
|
|
17
|
+
TEST_SITEKEY = re.compile(r"[123]x0{20}[A-Z]{2}") # e.g. 3x00000000000000000000FF: always asks for the click
|
|
18
|
+
ON_SCREEN = "function () { const r = this.getBoundingClientRect(); return r.width > 0 && r.bottom > 0 && r.top < innerHeight }"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
async def scrapingcourse(tab) -> str:
|
|
22
|
+
await asyncio.wait_for(tab.get(SCRAPINGCOURSE, wait="interactive"), 60)
|
|
23
|
+
how = await passes_cloudflare(tab)
|
|
24
|
+
if how is None:
|
|
25
|
+
return "NO superado: sigue en el challenge de Cloudflare"
|
|
26
|
+
await asyncio.sleep(2)
|
|
27
|
+
text = await page_text(tab)
|
|
28
|
+
return f"superado {how}" if "bypassed" in text.lower() else f"cargó, pero sin el mensaje de éxito: {text[:120]!r}"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async def nowsecure(tab) -> str:
|
|
32
|
+
await asyncio.wait_for(tab.get(NOWSECURE, wait="interactive"), 60)
|
|
33
|
+
if await passes_cloudflare(tab) is None:
|
|
34
|
+
return "NO superado: sigue en el challenge de Cloudflare"
|
|
35
|
+
widget = await turnstile_on_screen(tab)
|
|
36
|
+
await (await turnstile_checkbox(widget)).click()
|
|
37
|
+
token = await turnstile_token(widget)
|
|
38
|
+
if not token:
|
|
39
|
+
return "casilla pulsada, pero Turnstile no dio token"
|
|
40
|
+
sitekey = await widget.get_attribute("data-sitekey") or ""
|
|
41
|
+
note = f" (sitekey de prueba {sitekey}: mide el clic, no la detección)" if TEST_SITEKEY.fullmatch(sitekey) else ""
|
|
42
|
+
return f"casilla pulsada, token {token[:40]}{note}"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async def turnstile_on_screen(tab):
|
|
46
|
+
# nowsecure.nl has a second widget in its reflection, out of view.
|
|
47
|
+
for widget in await tab.select_all(".cf-turnstile", timeout=20):
|
|
48
|
+
if await widget.call(ON_SCREEN, isolated=True):
|
|
49
|
+
return widget
|
|
50
|
+
raise RuntimeError("no hay ningún Turnstile a la vista")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def turnstile_checkbox(widget, timeout: float = 30):
|
|
54
|
+
"""Turnstile hides its iframe in a closed shadow root, and the checkbox in another one inside the iframe."""
|
|
55
|
+
loop = asyncio.get_running_loop()
|
|
56
|
+
deadline = loop.time() + timeout
|
|
57
|
+
while loop.time() < deadline:
|
|
58
|
+
for element in await widget.select_all("div"):
|
|
59
|
+
if host_root := await element.shadow_root():
|
|
60
|
+
frame = await (await host_root.select("iframe")).content_frame()
|
|
61
|
+
if root := await (await frame.select("body", timeout=timeout)).shadow_root():
|
|
62
|
+
return await root.select("input[type=checkbox]", timeout=deadline - loop.time())
|
|
63
|
+
await asyncio.sleep(0.25) # the widget is still loading
|
|
64
|
+
raise TimeoutError(f"la casilla de Turnstile no apareció en {timeout}s")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
async def turnstile_token(widget, timeout: float = 15) -> str:
|
|
68
|
+
response = await widget.select("[name=cf-turnstile-response]")
|
|
69
|
+
for _ in range(int(timeout / 0.25)):
|
|
70
|
+
if token := await response.get_property("value"):
|
|
71
|
+
return token
|
|
72
|
+
await asyncio.sleep(0.25)
|
|
73
|
+
return ""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
CHECKS = [("Cloudflare: scrapingcourse.com", scrapingcourse), ("Cloudflare: nowsecure.nl", nowsecure)]
|
|
77
|
+
|
|
78
|
+
if __name__ == "__main__":
|
|
79
|
+
run(*CHECKS)
|