browserfetch 0.2.0__tar.gz → 0.4.0__tar.gz

{browserfetch-0.2.0 → browserfetch-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: browserfetch
-Version: 0.2.0
+Version: 0.4.0
 Summary: fetch in Python using your browser! 
 License: GNU General Public License v3 (GPLv3)
 Project-URL: Homepage, https://github.com/5j9/browserfetch
@@ -26,21 +26,21 @@ Usage
 
     from asyncio import gather, get_event_loop
 
-    from browserfetch import fetch, run_server
+    from browserfetch import fetch, get, post, run_server
 
 
     async def main():
         response1, response2, reponse3 = await gather(
-            fetch('https://example.com/path1'),
+            get('https://example.com/path1', params={'a': 1}),
             fetch('https://example.com/image.png'),
-            fetch('https://example.com/path2'),
+            post('https://example.com/path2', data={'a': 1}),
         )
         # do stuff with retrieved responses
 
 
     loop = get_event_loop()
-    loop.create_task(main())
-    run_server(loop=loop)
+    loop.create_task(start_server())
+    loop.run_until_complete(main())
 
 
 2. Open your browser, goto http://example.com (perhaps solve a captcha and log in).
@@ -63,7 +63,8 @@ Motivations
 
 Downsides
 ---------
-* Setting up ``browserfetch`` is a more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
+* Setting up ``browserfetch`` is more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
+* To run more than one server at the same time, you have to change the port number in both `start_server(port=)` call and in the corresponding JavaScript for each instance of the program.
 
 .. _`browser_cookie3 stopped working on Chrome-based browsers`: https://github.com/borisbabic/browser_cookie3/issues/180
 .. _tampermonkey: https://github.com/Tampermonkey/tampermonkey

{browserfetch-0.2.0 → browserfetch-0.4.0}/README.rst

@@ -12,21 +12,21 @@ Usage
 
     from asyncio import gather, get_event_loop
 
-    from browserfetch import fetch, run_server
+    from browserfetch import fetch, get, post, run_server
 
 
     async def main():
         response1, response2, reponse3 = await gather(
-            fetch('https://example.com/path1'),
+            get('https://example.com/path1', params={'a': 1}),
             fetch('https://example.com/image.png'),
-            fetch('https://example.com/path2'),
+            post('https://example.com/path2', data={'a': 1}),
         )
         # do stuff with retrieved responses
 
 
     loop = get_event_loop()
-    loop.create_task(main())
-    run_server(loop=loop)
+    loop.create_task(start_server())
+    loop.run_until_complete(main())
 
 
 2. Open your browser, goto http://example.com (perhaps solve a captcha and log in).
@@ -49,7 +49,8 @@ Motivations
 
 Downsides
 ---------
-* Setting up ``browserfetch`` is a more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
+* Setting up ``browserfetch`` is more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
+* To run more than one server at the same time, you have to change the port number in both `start_server(port=)` call and in the corresponding JavaScript for each instance of the program.
 
 .. _`browser_cookie3 stopped working on Chrome-based browsers`: https://github.com/borisbabic/browser_cookie3/issues/180
 .. _tampermonkey: https://github.com/Tampermonkey/tampermonkey

browserfetch-0.4.0/browserfetch/__init__.py

@@ -0,0 +1,217 @@
+__version__ = '0.4.0'
+
+import atexit
+from asyncio import Lock, Queue, gather, get_event_loop, wait_for
+from dataclasses import dataclass
+from json import dumps, loads
+from logging import getLogger
+from typing import Any
+from urllib.parse import urlencode
+
+from aiohttp.web import Application, RouteTableDef, WebSocketResponse
+from aiohttp.web_runner import AppRunner, TCPSite
+
+logger = getLogger(__name__)
+# maps each domain the queue object of that domain
+queues: dict[str, Queue] = {}
+# maps each lock id to either the active lock or the resolved response
+locks: dict[int, Lock | dict] = {}
+
+
+class BrowserError(Exception):
+    pass
+
+
+@dataclass(slots=True, weakref_slot=True)
+class Response:
+    """
+    For the meaning of attributes see:
+    https://developer.mozilla.org/en-US/docs/Web/API/Response
+    """
+
+    body: bytes
+    ok: bool
+    redirected: bool
+    status: int
+    status_text: str
+    type: str
+    url: str
+    headers: dict
+
+    def text(self, encoding=None, errors='strict') -> str:
+        return self.body.decode(encoding or 'utf-8', errors)
+
+    def json(self, encoding=None, errors='strict'):
+        if encoding is None:
+            return loads(self.body)
+        return loads(self.text(encoding=encoding, errors=errors))
+
+
+def extract_host(url: str) -> str:
+    return url.partition('//')[2].partition('/')[0]
+
+
+async def send_requests(q, ws):
+    while True:
+        url, options, lock_id, timeout, body = await q.get()
+        request_blob = dumps(
+            {
+                'url': url,
+                'options': options,
+                'lock_id': lock_id,
+                'timeout': timeout,
+            }
+        ).encode()
+        if body is not None:
+            request_blob += b'\0' + body
+        await ws.send_bytes(request_blob)
+
+
+async def receive_responses(ws: WebSocketResponse):
+    while True:
+        blob = await ws.receive_bytes()
+        json_blob, _, body = blob.partition(b'\0')
+        j = loads(json_blob)
+        j['body'] = body
+        lock_id = j.pop('lock_id')
+        try:
+            lock = locks[lock_id]
+        except KeyError:  # lock has reached timeout already
+            continue
+        locks[lock_id] = j
+        lock.release()
+
+
+routes = RouteTableDef()
+
+
+@routes.get('/ws')
+async def _(request):
+    ws = WebSocketResponse()
+    await ws.prepare(request)
+
+    host = await ws.receive_str()
+    logger.info('registering host %s', host)
+    q = queues.get(host) or queues.setdefault(host, Queue())
+
+    await gather(send_requests(q, ws), receive_responses(ws))
+
+
+async def fetch(
+    url: str,
+    *,
+    params: dict = None,
+    body: bytes = None,
+    timeout: int | float = None,
+    options: dict = None,
+    host=None,
+) -> Response:
+    """Fetch using browser fetch API available on host.
+
+    :param url: the URL of the resource you want to fetch.
+    :param params: parameters to be url-encoded and added to url.
+    :param body: the body of the request (do not add to options).
+    :param timeout: timeout in seconds (do not add to options).
+    :param options: See https://developer.mozilla.org/en-US/docs/Web/API/fetch
+    :param host: `location.host` of the tab that is supposed to handle this
+        request.
+    :return: a dict of response values.
+    """
+    if params is not None:
+        url += urlencode(params)
+
+    if host is None:
+        host = extract_host(url)
+    q = queues.get(host) or queues.setdefault(host, Queue())
+    lock = Lock()
+    lock_id = id(lock)
+    locks[lock_id] = lock
+    await lock.acquire()
+
+    await q.put((url, options, lock_id, timeout, body))
+
+    try:
+        await wait_for(lock.acquire(), timeout)
+    except TimeoutError:
+        locks.pop(lock_id, None)
+        raise
+
+    j = locks.pop(lock_id)
+    if (err := j.get('error')) is not None:
+        raise BrowserError(err)
+
+    return Response(**j)
+
+
+async def get(
+    url: str,
+    *,
+    params: dict = None,
+    options: dict = None,
+    host: str = None,
+    timeout: int | float = None,
+) -> Response:
+    if options is None:
+        options = {'method': 'GET'}
+    else:
+        options['method'] = 'GET'
+    return await fetch(
+        url, options=options, host=host, timeout=timeout, params=params
+    )
+
+
+async def post(
+    url: str,
+    *,
+    params: dict = None,
+    body: bytes = None,
+    data: dict = None,
+    json=None,
+    timeout: int | float = None,
+    options: dict = None,
+    host: str = None,
+) -> Response:
+    if options is None:
+        options: dict[str, Any] = {'method': 'POST'}
+    else:
+        options['method'] = 'POST'
+
+    if json is not None:
+        assert body is None
+        body = dumps(json).encode()
+        headers = options.setdefault('headers', {})
+        headers['Content-Type'] = 'application/json'
+
+    if data is not None:
+        assert body is None
+        body = urlencode(data).encode()
+        headers = options.setdefault('headers', {})
+        headers['Content-Type'] = 'application/x-www-form-urlencoded'
+
+    return await fetch(
+        url,
+        options=options,
+        host=host,
+        timeout=timeout,
+        body=body,
+        params=params,
+    )
+
+
+app = Application()
+app.add_routes(routes)
+app_runner = AppRunner(app)
+
+
+@atexit.register
+def shutdown_server():
+    loop = get_event_loop()
+    logger.info('waiting for app_runner.cleanup()')
+    loop.run_until_complete(app_runner.cleanup())
+
+
+async def start_server(*, host='127.0.0.1', port=9404):
+    await app_runner.setup()
+    site = TCPSite(app_runner, host, port)
+    await site.start()
+    logger.info('server started at http://%s:%s', host, port)

{browserfetch-0.2.0 → browserfetch-0.4.0}/browserfetch/browserfetch.js

@@ -2,10 +2,12 @@
 // @name        browserfetch
 // @namespace   https://github.com/5j9/browserfetch
 // @match       https://example.com/
+// @grant       GM_registerMenuCommand
 // ==/UserScript==
 (() => {
     function connect() {
         var ws = new WebSocket("ws://127.0.0.1:9404/ws");
+        ws.binaryType = "arraybuffer";
 
         ws.onopen = () => {
             ws.send(location.host);
@@ -17,19 +19,30 @@
         };
 
         ws.onmessage = async (evt) => {
-            var returnData, responseBlob;
-            var j = JSON.parse(evt.data);
+            var returnData, responseBlob, body, jArray;
+            var requestArray = new Uint8Array(evt.data);
+            var nullIndex = requestArray.indexOf(0);
+            if (nullIndex === -1) {
+                body = null;
+                jArray = requestArray;
+            } else {
+                body = requestArray.slice(nullIndex + 1);
+                jArray = requestArray.slice(0, nullIndex)
+            }
+            var j = JSON.parse(new TextDecoder().decode(jArray));
+            var options = j['options'] || {};
+
             if (j['timeout']) {
-                var signal = AbortSignal.timeout(j['timeout'] * 1000);
-                if (j['options']) {
-                    j['options'].signal = signal;
-                } else {
-                    j['options'] = { 'signal': signal };
-                }
+                options.signal = AbortSignal.timeout(j['timeout'] * 1000);
+            }
+
+            if (body !== null) {
+                options.body = body;
             }
+
             try {
-                var r = await fetch(j['url'], j['options']);
-                var returnData = {
+                var r = await fetch(j['url'], options);
+                returnData = {
                     'lock_id': j['lock_id'],
                     'headers': Object.fromEntries([...r.headers]),
                     'ok': r.ok,
@@ -51,5 +64,12 @@
         }
     };
 
-    connect();
+    if (window.GM_registerMenuCommand) {
+        GM_registerMenuCommand(
+            'connect to browserfetch',
+            connect
+        );
+    } else {
+        connect();
+    }
 })();

{browserfetch-0.2.0 → browserfetch-0.4.0}/browserfetch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: browserfetch
-Version: 0.2.0
+Version: 0.4.0
 Summary: fetch in Python using your browser! 
 License: GNU General Public License v3 (GPLv3)
 Project-URL: Homepage, https://github.com/5j9/browserfetch
@@ -26,21 +26,21 @@ Usage
 
     from asyncio import gather, get_event_loop
 
-    from browserfetch import fetch, run_server
+    from browserfetch import fetch, get, post, run_server
 
 
     async def main():
         response1, response2, reponse3 = await gather(
-            fetch('https://example.com/path1'),
+            get('https://example.com/path1', params={'a': 1}),
             fetch('https://example.com/image.png'),
-            fetch('https://example.com/path2'),
+            post('https://example.com/path2', data={'a': 1}),
         )
         # do stuff with retrieved responses
 
 
     loop = get_event_loop()
-    loop.create_task(main())
-    run_server(loop=loop)
+    loop.create_task(start_server())
+    loop.run_until_complete(main())
 
 
 2. Open your browser, goto http://example.com (perhaps solve a captcha and log in).
@@ -63,7 +63,8 @@ Motivations
 
 Downsides
 ---------
-* Setting up ``browserfetch`` is a more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
+* Setting up ``browserfetch`` is more cumbersome since it requires running a Python server and also injecting a small script into the webpage. Using ``browser_cookie3`` might be a better choice if there are many websites that you need to communicate with.
+* To run more than one server at the same time, you have to change the port number in both `start_server(port=)` call and in the corresponding JavaScript for each instance of the program.
 
 .. _`browser_cookie3 stopped working on Chrome-based browsers`: https://github.com/borisbabic/browser_cookie3/issues/180
 .. _tampermonkey: https://github.com/Tampermonkey/tampermonkey

browserfetch-0.2.0/browserfetch/__init__.py

@@ -1,128 +0,0 @@
-__version__ = '0.2.0'
-
-from asyncio import Lock, Queue, gather, wait_for
-from dataclasses import dataclass
-from json import loads
-from logging import getLogger
-
-from aiohttp.web import Application, RouteTableDef, WebSocketResponse, run_app
-
-logger = getLogger(__name__)
-# maps each domain the queue object of that domain
-queues: dict[str, Queue] = {}
-# maps each lock id to either the active lock or the resolved response
-locks: dict[int, Lock | dict] = {}
-
-
-class BrowserError(Exception):
-    pass
-
-
-@dataclass(slots=True, weakref_slot=True)
-class FetchResponse:
-    body: bytes
-    ok: bool
-    redirected: bool
-    status: int
-    status_text: str
-    type: str
-    url: str
-    headers: dict
-
-    def text(self, encoding=None, errors='strict') -> str:
-        return self.body.decode(encoding or 'utf-8', errors)
-
-    def json(self, encoding=None, errors='strict'):
-        if encoding is None:
-            return loads(self.body, errors=errors)
-        return loads(self.text(encoding=encoding, errors=errors))
-
-
-def extract_host(url: str) -> str:
-    return url.partition('//')[2].partition('/')[0]
-
-
-async def send_requests(q, ws):
-    while True:
-        url, options, lock, timeout = await q.get()
-        await ws.send_json(
-            {
-                'url': url,
-                'options': options,
-                'lock_id': id(lock),
-                'timeout': timeout,
-            }
-        )
-
-
-async def receive_responses(ws: WebSocketResponse):
-    while True:
-        blob = await ws.receive_bytes()
-        json_blob, _, body = blob.partition(b'\0')
-        j = loads(json_blob)
-        j['body'] = body
-        lock_id = j.pop('lock_id')
-        try:
-            lock = locks[lock_id]
-        except KeyError:  # lock has reached timeout already
-            continue
-        locks[lock_id] = j
-        lock.release()
-
-
-routes = RouteTableDef()
-
-
-@routes.get('/ws')
-async def _(request):
-    ws = WebSocketResponse()
-    await ws.prepare(request)
-
-    host = await ws.receive_str()
-    logger.info('registering host %s', host)
-    q = queues.get(host) or queues.setdefault(host, Queue())
-
-    await gather(send_requests(q, ws), receive_responses(ws))
-
-
-async def fetch(
-    url: str, options: dict = None, *, host=None, timeout: int | float = None
-) -> FetchResponse:
-    """fetch using browser fetch API available on host.
-
-    :param url: the URL of the resource you want to fetch.
-    :param options: See https://developer.mozilla.org/en-US/docs/Web/API/fetch
-    :param host: `location.host` of the tab that is supposed to handle this
-        request.
-    :param timeout: timeout in seconds.
-    :return: a dict of response values.
-    """
-    if host is None:
-        host = extract_host(url)
-    q = queues.get(host) or queues.setdefault(host, Queue())
-    lock = Lock()
-    lock_id = id(lock)
-    locks[lock_id] = lock
-    await lock.acquire()
-
-    await q.put((url, options, lock, timeout))
-
-    try:
-        await wait_for(lock.acquire(), timeout)
-    except TimeoutError:
-        locks.pop(lock_id, None)
-        raise
-
-    j = locks.pop(lock_id)
-    if (err := j.get('error')) is not None:
-        raise BrowserError(err)
-
-    return FetchResponse(**j)
-
-
-app = Application()
-app.add_routes(routes)
-
-
-def run_server(*, host='127.0.0.1', port=9404, loop=None):
-    run_app(app, host=host, port=port, loop=loop)