browserfetch 0.7.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {browserfetch-0.7.0 → browserfetch-0.10.0}/PKG-INFO +72 -71
- {browserfetch-0.7.0 → browserfetch-0.10.0}/README.rst +2 -1
- {browserfetch-0.7.0 → browserfetch-0.10.0}/browserfetch/__init__.py +93 -42
- browserfetch-0.10.0/browserfetch/__main__.py +36 -0
- browserfetch-0.10.0/browserfetch/browserfetch.js +147 -0
- browserfetch-0.10.0/pyproject.toml +62 -0
- browserfetch-0.7.0/browserfetch/__main__.py +0 -24
- browserfetch-0.7.0/browserfetch/browserfetch.js +0 -75
- browserfetch-0.7.0/browserfetch.egg-info/PKG-INFO +0 -71
- browserfetch-0.7.0/browserfetch.egg-info/SOURCES.txt +0 -12
- browserfetch-0.7.0/browserfetch.egg-info/dependency_links.txt +0 -1
- browserfetch-0.7.0/browserfetch.egg-info/not-zip-safe +0 -1
- browserfetch-0.7.0/browserfetch.egg-info/requires.txt +0 -2
- browserfetch-0.7.0/browserfetch.egg-info/top_level.txt +0 -1
- browserfetch-0.7.0/pyproject.toml +0 -50
- browserfetch-0.7.0/setup.cfg +0 -4
- {browserfetch-0.7.0 → browserfetch-0.10.0}/LICENSE +0 -0
|
@@ -1,71 +1,72 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
2
|
-
Name: browserfetch
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: fetch in Python using your browser!
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
9
|
-
Requires-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
loop
|
|
42
|
-
loop.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
.. _`browser_cookie3 stopped working on Chrome-based browsers`: https://github.com/borisbabic/browser_cookie3/issues/180
|
|
69
|
-
.. _tampermonkey: https://github.com/Tampermonkey/tampermonkey
|
|
70
|
-
.. _violentmonkey: https://github.com/violentmonkey/violentmonkey
|
|
71
|
-
.. _browserfetch.js: https://github.com/5j9/browserfetch/blob/master/browserfetch/browserfetch.js
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: browserfetch
|
|
3
|
+
Version: 0.10.0
|
|
4
|
+
Summary: fetch in Python using your browser!
|
|
5
|
+
Keywords: browser,fetch,python,cookies
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/x-rst
|
|
8
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
9
|
+
Requires-Dist: aiohttp
|
|
10
|
+
Requires-Dist: pyperclip
|
|
11
|
+
Requires-Dist: cyclopts
|
|
12
|
+
Project-URL: Homepage, https://github.com/5j9/browserfetch
|
|
13
|
+
|
|
14
|
+
Fetch using your browser.
|
|
15
|
+
|
|
16
|
+
Let the browser manage cookies for you.
|
|
17
|
+
|
|
18
|
+
⚠️ Incomplete. Not tested thoroughly. Consider using `Playwright`_, especially for more complex scenarios.
|
|
19
|
+
|
|
20
|
+
Usage
|
|
21
|
+
-----
|
|
22
|
+
1. You'll run a Python script containing some code like this:
|
|
23
|
+
|
|
24
|
+
.. code-block:: python
|
|
25
|
+
|
|
26
|
+
from asyncio import gather, new_event_loop
|
|
27
|
+
|
|
28
|
+
from browserfetch import fetch, get, post, run_server
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async def main():
|
|
32
|
+
response1, response2, reponse3 = await gather(
|
|
33
|
+
get('https://example.com/path1', params={'a': 1}),
|
|
34
|
+
fetch('https://example.com/image.png'),
|
|
35
|
+
post('https://example.com/path2', data={'a': 1}),
|
|
36
|
+
)
|
|
37
|
+
# do stuff with retrieved responses
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
loop = new_event_loop()
|
|
41
|
+
loop.create_task(start_server())
|
|
42
|
+
loop.run_until_complete(main())
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
2. Open your browser, goto http://example.com (perhaps solve a captcha and log in).
|
|
46
|
+
3. Copy the contents of `browserfetch.js`_ file and paste it in browser's console. (You can use a browser extensions like violentmonkey_/tampermonkey_ to do this step for you.)
|
|
47
|
+
|
|
48
|
+
That's it! Your Python script starts handling requests.
|
|
49
|
+
The browser tab should remain open of-coarse.
|
|
50
|
+
|
|
51
|
+
The server can handle multiple websocket connections from different websites simultaneously.
|
|
52
|
+
|
|
53
|
+
How it works
|
|
54
|
+
------------
|
|
55
|
+
``browserfetch`` communicates with your browser using a websocket. The ``fetch`` function just passes the request to browser and it is the browser that handles the actual request. Response data is sent back to Python using the same WebSocket connection.
|
|
56
|
+
|
|
57
|
+
Motivations
|
|
58
|
+
-----------
|
|
59
|
+
* `browser_cookie3 stopped working on Chrome-based browsers`_. There is a workaround: ShadowCopy, but it requires admin privilege.
|
|
60
|
+
* Another issue with browser_cookie's approach is that it retrieves cookies from cookie files, but these files are not updated instantly. Thus, you might have to wait or retry a few times before you can successfully access newly set cookies.
|
|
61
|
+
* ShadowCopying and File access are slow and inefficient operations.
|
|
62
|
+
|
|
63
|
+
Downsides
|
|
64
|
+
---------
|
|
65
|
+
* Setting up ``browserfetch`` is more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
|
|
66
|
+
|
|
67
|
+
.. _playwright: https://playwright.dev/python/docs/intro
|
|
68
|
+
.. _`browser_cookie3 stopped working on Chrome-based browsers`: https://github.com/borisbabic/browser_cookie3/issues/180
|
|
69
|
+
.. _tampermonkey: https://github.com/Tampermonkey/tampermonkey
|
|
70
|
+
.. _violentmonkey: https://github.com/violentmonkey/violentmonkey
|
|
71
|
+
.. _browserfetch.js: https://github.com/5j9/browserfetch/blob/master/browserfetch/browserfetch.js
|
|
72
|
+
|
|
@@ -2,7 +2,7 @@ Fetch using your browser.
|
|
|
2
2
|
|
|
3
3
|
Let the browser manage cookies for you.
|
|
4
4
|
|
|
5
|
-
⚠️
|
|
5
|
+
⚠️ Incomplete. Not tested thoroughly. Consider using `Playwright`_, especially for more complex scenarios.
|
|
6
6
|
|
|
7
7
|
Usage
|
|
8
8
|
-----
|
|
@@ -51,6 +51,7 @@ Downsides
|
|
|
51
51
|
---------
|
|
52
52
|
* Setting up ``browserfetch`` is more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
|
|
53
53
|
|
|
54
|
+
.. _playwright: https://playwright.dev/python/docs/intro
|
|
54
55
|
.. _`browser_cookie3 stopped working on Chrome-based browsers`: https://github.com/borisbabic/browser_cookie3/issues/180
|
|
55
56
|
.. _tampermonkey: https://github.com/Tampermonkey/tampermonkey
|
|
56
57
|
.. _violentmonkey: https://github.com/violentmonkey/violentmonkey
|
|
@@ -1,30 +1,36 @@
|
|
|
1
|
-
__version__ = '0.
|
|
2
|
-
|
|
1
|
+
__version__ = '0.10.0'
|
|
3
2
|
import atexit
|
|
4
3
|
from asyncio import (
|
|
5
4
|
AbstractEventLoop,
|
|
6
5
|
CancelledError,
|
|
7
6
|
Event,
|
|
7
|
+
Task,
|
|
8
8
|
get_running_loop,
|
|
9
9
|
wait_for,
|
|
10
|
-
Task,
|
|
11
10
|
)
|
|
12
11
|
from collections import defaultdict
|
|
13
12
|
from dataclasses import dataclass
|
|
13
|
+
from html import escape
|
|
14
14
|
from json import dumps, loads
|
|
15
15
|
from logging import getLogger
|
|
16
|
-
from typing import Any
|
|
17
16
|
from urllib.parse import urlencode
|
|
18
17
|
|
|
19
18
|
from aiohttp import ClientSession, ClientWebSocketResponse
|
|
20
|
-
from aiohttp.web import
|
|
19
|
+
from aiohttp.web import (
|
|
20
|
+
Application,
|
|
21
|
+
Request,
|
|
22
|
+
Response as _Response,
|
|
23
|
+
RouteTableDef,
|
|
24
|
+
WebSocketResponse,
|
|
25
|
+
)
|
|
21
26
|
from aiohttp.web_runner import AppRunner, TCPSite
|
|
22
27
|
|
|
23
28
|
logger = getLogger(__name__)
|
|
24
29
|
# maps host to its host_ready event or its websocket
|
|
25
|
-
hosts:
|
|
30
|
+
hosts: defaultdict[
|
|
26
31
|
str, Event | WebSocketResponse | ClientWebSocketResponse
|
|
27
32
|
] = defaultdict(Event)
|
|
33
|
+
|
|
28
34
|
# maps response event id to its response event or response dict
|
|
29
35
|
responses: dict[int, Event | dict] = {}
|
|
30
36
|
|
|
@@ -81,12 +87,15 @@ async def _request(
|
|
|
81
87
|
if body is not None:
|
|
82
88
|
bytes_ += b'\0' + body
|
|
83
89
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
+
value = hosts[host]
|
|
91
|
+
match value:
|
|
92
|
+
case Event():
|
|
93
|
+
# wait for the Event to be turned into a websocket response
|
|
94
|
+
await value.wait()
|
|
95
|
+
ws = hosts[host]
|
|
96
|
+
await ws.send_bytes(bytes_) # type: ignore
|
|
97
|
+
case WebSocketResponse() | ClientWebSocketResponse():
|
|
98
|
+
await value.send_bytes(bytes_)
|
|
90
99
|
|
|
91
100
|
try:
|
|
92
101
|
await wait_for(response_ready.wait(), data['timeout'])
|
|
@@ -94,7 +103,8 @@ async def _request(
|
|
|
94
103
|
responses.pop(event_id, None)
|
|
95
104
|
raise
|
|
96
105
|
|
|
97
|
-
return
|
|
106
|
+
# this must return a dict at this point, not an Event
|
|
107
|
+
return responses.pop(event_id) # type: ignore
|
|
98
108
|
|
|
99
109
|
|
|
100
110
|
async def receive_responses(ws: WebSocketResponse | ClientWebSocketResponse):
|
|
@@ -105,14 +115,17 @@ async def receive_responses(ws: WebSocketResponse | ClientWebSocketResponse):
|
|
|
105
115
|
j['body'] = body
|
|
106
116
|
event_id = j.pop('event_id')
|
|
107
117
|
try:
|
|
108
|
-
|
|
118
|
+
# We expect only one response to be recieved for each event,
|
|
119
|
+
# therefore this must be an Event, not a dict.
|
|
120
|
+
response_ready: Event = responses[event_id] # type: ignore
|
|
109
121
|
except KeyError: # lock has reached timeout already
|
|
110
122
|
continue
|
|
111
123
|
responses[event_id] = j
|
|
112
|
-
|
|
124
|
+
response_ready.set()
|
|
113
125
|
|
|
114
126
|
|
|
115
127
|
routes = RouteTableDef()
|
|
128
|
+
PROTOCOL = '3'
|
|
116
129
|
|
|
117
130
|
|
|
118
131
|
@routes.get('/ws')
|
|
@@ -120,7 +133,10 @@ async def _(request):
|
|
|
120
133
|
ws = WebSocketResponse()
|
|
121
134
|
await ws.prepare(request)
|
|
122
135
|
|
|
123
|
-
host = await ws.receive_str()
|
|
136
|
+
version, _, host = (await ws.receive_str()).partition(' ')
|
|
137
|
+
assert version == PROTOCOL, (
|
|
138
|
+
f'JavaScript protocol version: {version}, expected: {PROTOCOL}'
|
|
139
|
+
)
|
|
124
140
|
logger.info('registering host %s', host)
|
|
125
141
|
|
|
126
142
|
ws_or_e = hosts[host]
|
|
@@ -133,10 +149,11 @@ async def _(request):
|
|
|
133
149
|
except TypeError:
|
|
134
150
|
logger.info('host WebSocket was closed')
|
|
135
151
|
hosts[host] = Event()
|
|
152
|
+
return ws
|
|
136
153
|
|
|
137
154
|
|
|
138
155
|
@routes.get('/relay')
|
|
139
|
-
async def _(request):
|
|
156
|
+
async def _(request: Request) -> WebSocketResponse:
|
|
140
157
|
ws = WebSocketResponse()
|
|
141
158
|
await ws.prepare(request)
|
|
142
159
|
|
|
@@ -144,13 +161,15 @@ async def _(request):
|
|
|
144
161
|
try:
|
|
145
162
|
bytes_ = await ws.receive_bytes()
|
|
146
163
|
except TypeError: # ws closed
|
|
147
|
-
return
|
|
164
|
+
return ws
|
|
148
165
|
data, null, body = bytes_.partition(b'\0')
|
|
149
166
|
data = loads(data)
|
|
150
167
|
relay_event_id = data['event_id']
|
|
151
168
|
|
|
152
169
|
try:
|
|
153
|
-
r = await _request(
|
|
170
|
+
r: dict = await _request(
|
|
171
|
+
data.pop('host'), data, body if null else None
|
|
172
|
+
)
|
|
154
173
|
except TimeoutError:
|
|
155
174
|
r = {'error': 'TimeoutError in relay'}
|
|
156
175
|
|
|
@@ -159,6 +178,22 @@ async def _(request):
|
|
|
159
178
|
await ws.send_bytes(dumps(r).encode() + b'\0' + body)
|
|
160
179
|
|
|
161
180
|
|
|
181
|
+
@routes.get('/')
|
|
182
|
+
async def _(_) -> _Response:
|
|
183
|
+
hosts_html = '\n'.join(
|
|
184
|
+
[f'<li>{k}: {escape(str(v))}</li>' for k, v in hosts.items()]
|
|
185
|
+
)
|
|
186
|
+
responses_html = '\n'.join(
|
|
187
|
+
[f'<li>{k}: {escape(str(v))}</li>' for k, v in responses.items()]
|
|
188
|
+
)
|
|
189
|
+
return _Response(
|
|
190
|
+
body='<meta charset="utf-8">\n<title>browserfetch</title>\n'
|
|
191
|
+
f'Hosts:\n{hosts_html}\n'
|
|
192
|
+
f'Responses:\n{responses_html}',
|
|
193
|
+
content_type='text/html',
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
|
|
162
197
|
async def relay_client(server_host, server_port):
|
|
163
198
|
async with ClientSession() as session:
|
|
164
199
|
relay_url = f'ws://{server_host}:{server_port}/relay'
|
|
@@ -183,14 +218,26 @@ async def relay_client(server_host, server_port):
|
|
|
183
218
|
return
|
|
184
219
|
|
|
185
220
|
|
|
221
|
+
async def evaluate(
|
|
222
|
+
string: str,
|
|
223
|
+
host: str,
|
|
224
|
+
timeout: int | float = 95,
|
|
225
|
+
):
|
|
226
|
+
"""Evaluate string in browser context and return JSON.stringify(result)."""
|
|
227
|
+
d = await _request(
|
|
228
|
+
host, {'action': 'eval', 'string': string, 'timeout': timeout}, None
|
|
229
|
+
)
|
|
230
|
+
return d['result']
|
|
231
|
+
|
|
232
|
+
|
|
186
233
|
async def fetch(
|
|
187
234
|
url: str,
|
|
188
235
|
*,
|
|
189
|
-
params: dict = None,
|
|
190
|
-
body: bytes = None,
|
|
236
|
+
params: dict | None = None,
|
|
237
|
+
body: bytes | None = None,
|
|
191
238
|
timeout: int | float = 95,
|
|
192
|
-
options: dict = None,
|
|
193
|
-
host=None,
|
|
239
|
+
options: dict | None = None,
|
|
240
|
+
host: str | None = None,
|
|
194
241
|
) -> Response:
|
|
195
242
|
"""Fetch using browser fetch API available on host.
|
|
196
243
|
|
|
@@ -209,6 +256,7 @@ async def fetch(
|
|
|
209
256
|
d = await _request(
|
|
210
257
|
host,
|
|
211
258
|
{
|
|
259
|
+
'action': 'fetch',
|
|
212
260
|
'url': url,
|
|
213
261
|
'options': options,
|
|
214
262
|
'timeout': timeout,
|
|
@@ -225,9 +273,9 @@ async def fetch(
|
|
|
225
273
|
async def get(
|
|
226
274
|
url: str,
|
|
227
275
|
*,
|
|
228
|
-
params: dict = None,
|
|
229
|
-
options: dict = None,
|
|
230
|
-
host: str = None,
|
|
276
|
+
params: dict | None = None,
|
|
277
|
+
options: dict | None = None,
|
|
278
|
+
host: str | None = None,
|
|
231
279
|
timeout: int | float = 95,
|
|
232
280
|
) -> Response:
|
|
233
281
|
if options is None:
|
|
@@ -242,30 +290,33 @@ async def get(
|
|
|
242
290
|
async def post(
|
|
243
291
|
url: str,
|
|
244
292
|
*,
|
|
245
|
-
params: dict = None,
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
json=None,
|
|
293
|
+
params: dict | None = None,
|
|
294
|
+
data: bytes | dict | str | None = None,
|
|
295
|
+
form: dict | None = None,
|
|
249
296
|
timeout: int | float = 95,
|
|
250
|
-
options: dict = None,
|
|
251
|
-
host: str = None,
|
|
297
|
+
options: dict | None = None,
|
|
298
|
+
host: str | None = None,
|
|
252
299
|
) -> Response:
|
|
253
300
|
if options is None:
|
|
254
|
-
options
|
|
301
|
+
options = {'method': 'POST'}
|
|
255
302
|
else:
|
|
256
303
|
options['method'] = 'POST'
|
|
257
304
|
|
|
258
|
-
if json is not None:
|
|
259
|
-
assert body is None
|
|
260
|
-
body = dumps(json).encode()
|
|
261
|
-
headers = options.setdefault('headers', {})
|
|
262
|
-
headers['Content-Type'] = 'application/json'
|
|
263
|
-
|
|
264
305
|
if data is not None:
|
|
265
|
-
|
|
266
|
-
|
|
306
|
+
if isinstance(data, str):
|
|
307
|
+
body = data.encode()
|
|
308
|
+
elif isinstance(data, bytes):
|
|
309
|
+
body = data
|
|
310
|
+
else:
|
|
311
|
+
body = dumps(data).encode()
|
|
312
|
+
headers = options.setdefault('headers', {})
|
|
313
|
+
headers['Content-Type'] = 'application/json'
|
|
314
|
+
elif form is not None:
|
|
315
|
+
body = urlencode(form).encode()
|
|
267
316
|
headers = options.setdefault('headers', {})
|
|
268
317
|
headers['Content-Type'] = 'application/x-www-form-urlencoded'
|
|
318
|
+
else:
|
|
319
|
+
body = None
|
|
269
320
|
|
|
270
321
|
return await fetch(
|
|
271
322
|
url,
|
|
@@ -288,7 +339,7 @@ def _shutdown_server(loop: AbstractEventLoop):
|
|
|
288
339
|
|
|
289
340
|
|
|
290
341
|
def _cancel_relay_task(loop: AbstractEventLoop, task: Task):
|
|
291
|
-
logger.info(
|
|
342
|
+
logger.info('cancelling relay task')
|
|
292
343
|
task.cancel()
|
|
293
344
|
try:
|
|
294
345
|
loop.run_until_complete(task)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def read_js(host_name_generator: str | None):
|
|
5
|
+
js = (Path(__file__).parent / 'browserfetch.js').read_bytes().decode()
|
|
6
|
+
if host_name_generator is not None:
|
|
7
|
+
return js.replace(
|
|
8
|
+
'async function generateHostName() { return location.host };',
|
|
9
|
+
host_name_generator,
|
|
10
|
+
1,
|
|
11
|
+
)
|
|
12
|
+
return js
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def copyjs(*, host_name_generator: str | None = None):
|
|
16
|
+
"""Copy contents of browserfetch.js to clipboard.
|
|
17
|
+
|
|
18
|
+
`host_name_generator` should be a string containing
|
|
19
|
+
an async JavaScript function named `generateHostName()`. This file's
|
|
20
|
+
contents will be copied into the generated js script and will be used
|
|
21
|
+
to generate a host name for connections. The default
|
|
22
|
+
`generateHostName` function returns `location.host`.
|
|
23
|
+
"""
|
|
24
|
+
from pyperclip import copy
|
|
25
|
+
|
|
26
|
+
copy(read_js(host_name_generator))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
if __name__ == '__main__':
|
|
30
|
+
from cyclopts import App
|
|
31
|
+
|
|
32
|
+
from browserfetch import __version__
|
|
33
|
+
|
|
34
|
+
app = App(version=__version__)
|
|
35
|
+
app.command(copyjs)
|
|
36
|
+
app()
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// ==UserScript==
|
|
2
|
+
// @name browserfetch
|
|
3
|
+
// @namespace https://github.com/5j9/browserfetch
|
|
4
|
+
// @match https://example.com/
|
|
5
|
+
// @grant GM_registerMenuCommand
|
|
6
|
+
// ==/UserScript==
|
|
7
|
+
// @ts-check
|
|
8
|
+
(async () => {
|
|
9
|
+
/**
|
|
10
|
+
* @param {Uint8Array | null} body
|
|
11
|
+
* @param {Object} req
|
|
12
|
+
* @returns {Promise<Blob>}
|
|
13
|
+
*/
|
|
14
|
+
async function doFetch(req, body) {
|
|
15
|
+
var returnData, response;
|
|
16
|
+
var options = req['options'] || {};
|
|
17
|
+
|
|
18
|
+
if (req['timeout']) {
|
|
19
|
+
options.signal = AbortSignal.timeout(req['timeout'] * 1000);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
if (body !== null) {
|
|
23
|
+
options.body = body;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
var r = await fetch(req['url'], options);
|
|
28
|
+
returnData = {
|
|
29
|
+
'event_id': req['event_id'],
|
|
30
|
+
'headers': Object.fromEntries([...r.headers]),
|
|
31
|
+
'ok': r.ok,
|
|
32
|
+
'redirected': r.redirected,
|
|
33
|
+
'status': r.status,
|
|
34
|
+
'status_text': r.statusText,
|
|
35
|
+
'type': r.type,
|
|
36
|
+
'url': r.url
|
|
37
|
+
};
|
|
38
|
+
response = await r.blob();
|
|
39
|
+
} catch (err) {
|
|
40
|
+
returnData = {
|
|
41
|
+
'event_id': req['event_id'],
|
|
42
|
+
'error': err.toString()
|
|
43
|
+
};
|
|
44
|
+
response = "";
|
|
45
|
+
};
|
|
46
|
+
return new Blob([new TextEncoder().encode(JSON.stringify(returnData)), "\0", response]);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
*
|
|
51
|
+
* @param {Object} req
|
|
52
|
+
* @returns {Promise<Uint8Array>}
|
|
53
|
+
*/
|
|
54
|
+
async function doEval(req) {
|
|
55
|
+
var evalled, resp;
|
|
56
|
+
try {
|
|
57
|
+
evalled = eval(req['string']);
|
|
58
|
+
switch (evalled.constructor.name) {
|
|
59
|
+
case 'AsyncFunction':
|
|
60
|
+
evalled = await evalled;
|
|
61
|
+
break;
|
|
62
|
+
case 'Promise':
|
|
63
|
+
evalled = await evalled;
|
|
64
|
+
break;
|
|
65
|
+
case 'Function':
|
|
66
|
+
evalled = evalled();
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
resp = { 'result': evalled, 'event_id': req['event_id'] };
|
|
70
|
+
} catch (err) {
|
|
71
|
+
resp = { 'result': err.toString(), 'event_id': req['event_id'] };
|
|
72
|
+
}
|
|
73
|
+
return new TextEncoder().encode(JSON.stringify(resp));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
*
|
|
78
|
+
* @param {ArrayBuffer} d
|
|
79
|
+
* @returns {[Uint8Array | null, Object]}
|
|
80
|
+
*/
|
|
81
|
+
function parseData(d) {
|
|
82
|
+
var blob, jArray;
|
|
83
|
+
var dArray = new Uint8Array(d);
|
|
84
|
+
var nullIndex = dArray.indexOf(0);
|
|
85
|
+
if (nullIndex === -1) {
|
|
86
|
+
blob = null;
|
|
87
|
+
jArray = dArray;
|
|
88
|
+
} else {
|
|
89
|
+
blob = dArray.slice(nullIndex + 1);
|
|
90
|
+
jArray = dArray.slice(0, nullIndex)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return [blob, JSON.parse(new TextDecoder().decode(jArray))]
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function generateHostName() { return location.host };
|
|
97
|
+
var hostName;
|
|
98
|
+
|
|
99
|
+
function connect() {
|
|
100
|
+
var protocol = '3'
|
|
101
|
+
var ws = new WebSocket("ws://127.0.0.1:9404/ws");
|
|
102
|
+
ws.binaryType = "arraybuffer";
|
|
103
|
+
|
|
104
|
+
ws.onopen = async () => {
|
|
105
|
+
if (!hostName) {
|
|
106
|
+
hostName = await generateHostName();
|
|
107
|
+
}
|
|
108
|
+
ws.send(protocol + ' ' + hostName);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
ws.onclose = () => {
|
|
112
|
+
console.error('WebSocket was closed; will retry in 5 seconds');
|
|
113
|
+
setTimeout(connect, 5000);
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
ws.onmessage = async (evt) => {
|
|
117
|
+
var /**@type {Uint8Array | Blob} */ result, j, b;
|
|
118
|
+
[b, j] = parseData(evt.data);
|
|
119
|
+
switch (j['action']) {
|
|
120
|
+
case 'fetch':
|
|
121
|
+
result = await doFetch(j, b);
|
|
122
|
+
break;
|
|
123
|
+
case 'eval':
|
|
124
|
+
result = await doEval(j);
|
|
125
|
+
break;
|
|
126
|
+
default:
|
|
127
|
+
result = new TextEncoder().encode(JSON.stringify({
|
|
128
|
+
'event_id': j['event_id'],
|
|
129
|
+
'error': `Action ${j['action']} is not defined.`
|
|
130
|
+
}));
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
ws.send(result);
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
// @ts-ignore
|
|
138
|
+
if (window.GM_registerMenuCommand) {
|
|
139
|
+
// @ts-ignore
|
|
140
|
+
GM_registerMenuCommand(
|
|
141
|
+
'connect to browserfetch',
|
|
142
|
+
connect
|
|
143
|
+
);
|
|
144
|
+
} else {
|
|
145
|
+
connect();
|
|
146
|
+
}
|
|
147
|
+
})();
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ['flit_core >=3.2,<4']
|
|
3
|
+
build-backend = 'flit_core.buildapi'
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "browserfetch"
|
|
7
|
+
description = "fetch in Python using your browser! "
|
|
8
|
+
keywords = ["browser", "fetch", "python", "cookies"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
|
|
11
|
+
]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
dynamic = ["version"]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"aiohttp",
|
|
16
|
+
"pyperclip",
|
|
17
|
+
"cyclopts",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[[authors]]
|
|
21
|
+
name = "5j9"
|
|
22
|
+
email = "5j9@users.noreply.github.com"
|
|
23
|
+
|
|
24
|
+
[project.license]
|
|
25
|
+
text = "GNU General Public License v3 (GPLv3)"
|
|
26
|
+
|
|
27
|
+
[project.readme]
|
|
28
|
+
file = "README.rst"
|
|
29
|
+
content-type = "text/x-rst"
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/5j9/browserfetch"
|
|
33
|
+
|
|
34
|
+
[tool.ruff]
|
|
35
|
+
line-length = 79
|
|
36
|
+
format.quote-style = 'single'
|
|
37
|
+
lint.isort.combine-as-imports = true
|
|
38
|
+
lint.extend-select = [
|
|
39
|
+
'W605', # invalid-escape-sequence
|
|
40
|
+
'FA', # flake8-future-annotations
|
|
41
|
+
'I', # isort
|
|
42
|
+
'UP', # pyupgrade
|
|
43
|
+
]
|
|
44
|
+
lint.ignore = [
|
|
45
|
+
'E721', # Do not compare types, use `isinstance()`
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
[tool.pytest.ini_options]
|
|
49
|
+
addopts = '--quiet --tb=short'
|
|
50
|
+
|
|
51
|
+
[tool.pyright]
|
|
52
|
+
typeCheckingMode = 'standard'
|
|
53
|
+
reportDeprecated = "warning"
|
|
54
|
+
reportPropertyTypeMismatch = "warning"
|
|
55
|
+
reportUnnecessaryCast = "warning"
|
|
56
|
+
reportUnnecessaryContains = "warning"
|
|
57
|
+
reportUnnecessaryIsInstance = "warning"
|
|
58
|
+
reportUnnecessaryTypeIgnoreComment = true
|
|
59
|
+
reportInvalidStringEscapeSequence = false
|
|
60
|
+
reportConstantRedefinition = 'error'
|
|
61
|
+
reportTypeCommentUsage = 'warning'
|
|
62
|
+
reportUnnecessaryComparison = 'warning'
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
|
-
from pyperclip import copy
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
def main():
|
|
8
|
-
parser = ArgumentParser(
|
|
9
|
-
description='The command-line entry for browserfetch.',
|
|
10
|
-
formatter_class=ArgumentDefaultsHelpFormatter,
|
|
11
|
-
)
|
|
12
|
-
|
|
13
|
-
parser.add_argument(
|
|
14
|
-
'copyjs', help='copy contents of browserfetch.js to clipboard'
|
|
15
|
-
)
|
|
16
|
-
args = parser.parse_args()
|
|
17
|
-
|
|
18
|
-
if args.copyjs:
|
|
19
|
-
with (Path(__file__).parent / 'browserfetch.js').open() as f:
|
|
20
|
-
copy(f.read())
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
if __name__ == '__main__':
|
|
24
|
-
main()
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
// ==UserScript==
|
|
2
|
-
// @name browserfetch
|
|
3
|
-
// @namespace https://github.com/5j9/browserfetch
|
|
4
|
-
// @match https://example.com/
|
|
5
|
-
// @grant GM_registerMenuCommand
|
|
6
|
-
// ==/UserScript==
|
|
7
|
-
(() => {
|
|
8
|
-
function connect() {
|
|
9
|
-
var ws = new WebSocket("ws://127.0.0.1:9404/ws");
|
|
10
|
-
ws.binaryType = "arraybuffer";
|
|
11
|
-
|
|
12
|
-
ws.onopen = () => {
|
|
13
|
-
ws.send(location.host);
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
ws.onclose = function () {
|
|
17
|
-
console.error('browserfetch: WebSocket was closed; will retry in 5 seconds');
|
|
18
|
-
setTimeout(connect, 5000);
|
|
19
|
-
};
|
|
20
|
-
|
|
21
|
-
ws.onmessage = async (evt) => {
|
|
22
|
-
var returnData, responseBlob, body, jArray;
|
|
23
|
-
var requestArray = new Uint8Array(evt.data);
|
|
24
|
-
var nullIndex = requestArray.indexOf(0);
|
|
25
|
-
if (nullIndex === -1) {
|
|
26
|
-
body = null;
|
|
27
|
-
jArray = requestArray;
|
|
28
|
-
} else {
|
|
29
|
-
body = requestArray.slice(nullIndex + 1);
|
|
30
|
-
jArray = requestArray.slice(0, nullIndex)
|
|
31
|
-
}
|
|
32
|
-
var j = JSON.parse(new TextDecoder().decode(jArray));
|
|
33
|
-
var options = j['options'] || {};
|
|
34
|
-
|
|
35
|
-
if (j['timeout']) {
|
|
36
|
-
options.signal = AbortSignal.timeout(j['timeout'] * 1000);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
if (body !== null) {
|
|
40
|
-
options.body = body;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
try {
|
|
44
|
-
var r = await fetch(j['url'], options);
|
|
45
|
-
returnData = {
|
|
46
|
-
'event_id': j['event_id'],
|
|
47
|
-
'headers': Object.fromEntries([...r.headers]),
|
|
48
|
-
'ok': r.ok,
|
|
49
|
-
'redirected': r.redirected,
|
|
50
|
-
'status': r.status,
|
|
51
|
-
'status_text': r.statusText,
|
|
52
|
-
'type': r.type,
|
|
53
|
-
'url': r.url
|
|
54
|
-
};
|
|
55
|
-
responseBlob = await r.blob();
|
|
56
|
-
} catch (err) {
|
|
57
|
-
returnData = {
|
|
58
|
-
'event_id': j['event_id'],
|
|
59
|
-
'error': err.toString()
|
|
60
|
-
};
|
|
61
|
-
responseBlob = "";
|
|
62
|
-
};
|
|
63
|
-
ws.send(new Blob([new TextEncoder().encode(JSON.stringify(returnData)), "\0", responseBlob]));
|
|
64
|
-
}
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
if (window.GM_registerMenuCommand) {
|
|
68
|
-
GM_registerMenuCommand(
|
|
69
|
-
'connect to browserfetch',
|
|
70
|
-
connect
|
|
71
|
-
);
|
|
72
|
-
} else {
|
|
73
|
-
connect();
|
|
74
|
-
}
|
|
75
|
-
})();
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: browserfetch
|
|
3
|
-
Version: 0.7.0
|
|
4
|
-
Summary: fetch in Python using your browser!
|
|
5
|
-
License: GNU General Public License v3 (GPLv3)
|
|
6
|
-
Project-URL: Homepage, https://github.com/5j9/browserfetch
|
|
7
|
-
Keywords: browser,fetch,python,cookies
|
|
8
|
-
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
9
|
-
Requires-Python: >=3.11
|
|
10
|
-
Description-Content-Type: text/x-rst
|
|
11
|
-
License-File: LICENSE
|
|
12
|
-
Requires-Dist: aiohttp
|
|
13
|
-
Requires-Dist: pyperclip
|
|
14
|
-
|
|
15
|
-
Fetch using your browser.
|
|
16
|
-
|
|
17
|
-
Let the browser manage cookies for you.
|
|
18
|
-
|
|
19
|
-
⚠️ This project is a very simple implementation. Not tested thoroughly. Consider it a proof of concept.
|
|
20
|
-
|
|
21
|
-
Usage
|
|
22
|
-
-----
|
|
23
|
-
1. You'll run a Python script containing some code like this:
|
|
24
|
-
|
|
25
|
-
.. code-block:: python
|
|
26
|
-
|
|
27
|
-
from asyncio import gather, new_event_loop
|
|
28
|
-
|
|
29
|
-
from browserfetch import fetch, get, post, run_server
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
async def main():
|
|
33
|
-
response1, response2, reponse3 = await gather(
|
|
34
|
-
get('https://example.com/path1', params={'a': 1}),
|
|
35
|
-
fetch('https://example.com/image.png'),
|
|
36
|
-
post('https://example.com/path2', data={'a': 1}),
|
|
37
|
-
)
|
|
38
|
-
# do stuff with retrieved responses
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
loop = new_event_loop()
|
|
42
|
-
loop.create_task(start_server())
|
|
43
|
-
loop.run_until_complete(main())
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
2. Open your browser, goto http://example.com (perhaps solve a captcha and log in).
|
|
47
|
-
3. Copy the contents of `browserfetch.js`_ file and paste it in browser's console. (You can use a browser extensions like violentmonkey_/tampermonkey_ to do this step for you.)
|
|
48
|
-
|
|
49
|
-
That's it! Your Python script starts handling requests.
|
|
50
|
-
The browser tab should remain open of-coarse.
|
|
51
|
-
|
|
52
|
-
The server can handle multiple websocket connections from different websites simultaneously.
|
|
53
|
-
|
|
54
|
-
How it works
|
|
55
|
-
------------
|
|
56
|
-
``browserfetch`` communicates with your browser using a websocket. The ``fetch`` function just passes the request to browser and it is the browser that handles the actual request. Response data is sent back to Python using the same WebSocket connection.
|
|
57
|
-
|
|
58
|
-
Motivations
|
|
59
|
-
-----------
|
|
60
|
-
* `browser_cookie3 stopped working on Chrome-based browsers`_. There is a workaround: ShadowCopy, but it requires admin privilege.
|
|
61
|
-
* Another issue with browser_cookie's approach is that it retrieves cookies from cookie files, but these files are not updated instantly. Thus, you might have to wait or retry a few times before you can successfully access newly set cookies.
|
|
62
|
-
* ShadowCopying and File access are slow and inefficient operations.
|
|
63
|
-
|
|
64
|
-
Downsides
|
|
65
|
-
---------
|
|
66
|
-
* Setting up ``browserfetch`` is more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
|
|
67
|
-
|
|
68
|
-
.. _`browser_cookie3 stopped working on Chrome-based browsers`: https://github.com/borisbabic/browser_cookie3/issues/180
|
|
69
|
-
.. _tampermonkey: https://github.com/Tampermonkey/tampermonkey
|
|
70
|
-
.. _violentmonkey: https://github.com/violentmonkey/violentmonkey
|
|
71
|
-
.. _browserfetch.js: https://github.com/5j9/browserfetch/blob/master/browserfetch/browserfetch.js
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
README.rst
|
|
3
|
-
pyproject.toml
|
|
4
|
-
browserfetch/__init__.py
|
|
5
|
-
browserfetch/__main__.py
|
|
6
|
-
browserfetch/browserfetch.js
|
|
7
|
-
browserfetch.egg-info/PKG-INFO
|
|
8
|
-
browserfetch.egg-info/SOURCES.txt
|
|
9
|
-
browserfetch.egg-info/dependency_links.txt
|
|
10
|
-
browserfetch.egg-info/not-zip-safe
|
|
11
|
-
browserfetch.egg-info/requires.txt
|
|
12
|
-
browserfetch.egg-info/top_level.txt
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
browserfetch
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = [
|
|
3
|
-
"setuptools>=66.1.0",
|
|
4
|
-
"wheel",
|
|
5
|
-
]
|
|
6
|
-
build-backend = "setuptools.build_meta"
|
|
7
|
-
|
|
8
|
-
[project]
|
|
9
|
-
name = "browserfetch"
|
|
10
|
-
description = "fetch in Python using your browser! "
|
|
11
|
-
keywords = ["browser", "fetch", "python", "cookies"]
|
|
12
|
-
classifiers = [
|
|
13
|
-
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
|
|
14
|
-
]
|
|
15
|
-
requires-python = ">=3.11"
|
|
16
|
-
dynamic = ["version"]
|
|
17
|
-
dependencies = [
|
|
18
|
-
"aiohttp",
|
|
19
|
-
"pyperclip",
|
|
20
|
-
]
|
|
21
|
-
|
|
22
|
-
[[authors]]
|
|
23
|
-
name = "5j9"
|
|
24
|
-
email = "5j9@users.noreply.github.com"
|
|
25
|
-
|
|
26
|
-
[project.license]
|
|
27
|
-
text = "GNU General Public License v3 (GPLv3)"
|
|
28
|
-
|
|
29
|
-
[project.readme]
|
|
30
|
-
file = "README.rst"
|
|
31
|
-
content-type = "text/x-rst"
|
|
32
|
-
|
|
33
|
-
[project.urls]
|
|
34
|
-
Homepage = "https://github.com/5j9/browserfetch"
|
|
35
|
-
|
|
36
|
-
[tool.setuptools]
|
|
37
|
-
packages = ["browserfetch"]
|
|
38
|
-
zip-safe = false
|
|
39
|
-
include-package-data = false
|
|
40
|
-
|
|
41
|
-
[tool.setuptools.dynamic.version]
|
|
42
|
-
attr = "browserfetch.__version__"
|
|
43
|
-
|
|
44
|
-
[tool.setuptools.package-data]
|
|
45
|
-
browserfetch = ["*.js"]
|
|
46
|
-
|
|
47
|
-
[tool.ruff]
|
|
48
|
-
format.quote-style = 'single'
|
|
49
|
-
line-length = 79
|
|
50
|
-
isort.combine-as-imports = true
|
browserfetch-0.7.0/setup.cfg
DELETED
|
File without changes
|