QuLab 2.10.10__cp313-cp313-win_amd64.whl

qulab/sys/net/kcp.py

@@ -0,0 +1,192 @@
+import asyncio
+import time
+
+from . import _kcp
+
+
+class KCPConnection():
+
+    def __init__(self,
+                 address: tuple[str, int],
+                 transport: asyncio.DatagramTransport,
+                 conv: int,
+                 nodelay: bool = True,
+                 interval: int = 20,
+                 resend: int = 2,
+                 nc: bool = True):
+        self._kcp = _kcp.kcp_create(conv, self)
+        _kcp.kcp_nodelay(self._kcp, int(nodelay), interval, resend, int(nc))
+        self.interval = interval
+        self.address = address
+        self.transport = transport
+        self.last_active = time.time()
+        self.send_queue = asyncio.Queue()
+        self.recv_queue = asyncio.Queue()
+        self.fut = asyncio.create_task(self.run())
+
+    def __del__(self):
+        try:
+            self.fut.cancel()
+        except:
+            pass
+        _kcp.kcp_release(self._kcp)
+
+    def output(self, msg):
+        self.transport.sendto(msg, self.address)
+
+    def send(self, buff):
+        self.last_active = time.time()
+        self.send_queue.put_nowait(buff)
+
+    def _send(self, buff):
+        return _kcp.kcp_send(self._kcp, buff)
+
+    async def recv(self):
+        return await self.recv_queue.get()
+
+    def recv_nowait(self):
+        self.flush()
+        try:
+            return self.recv_queue.get_nowait()
+        except:
+            return None
+
+    def _recv(self):
+        return _kcp.kcp_recv(self._kcp)
+
+    def _update(self):
+        _kcp.kcp_update(self._kcp, self.clock())
+
+    def check(self):
+        current = self.clock()
+        return _kcp.kcp_check(self._kcp, current) - current
+
+    def clock(self):
+        return (time.monotonic_ns() // 1000_000) & 0x7fffffff
+
+    def input(self, buff):
+        self.last_active = time.time()
+        return _kcp.kcp_input(self._kcp, buff)
+
+    async def update(self):
+        while True:
+            await asyncio.sleep(self.check() / 1000)
+            self._update()
+
+    async def sync(self):
+        while True:
+            await asyncio.sleep(self.interval / 1000)
+            self.flush()
+
+    async def run(self):
+        await asyncio.gather(self.update(), self.sync())
+
+    def flush(self):
+        while True:
+            buff = self._recv()
+            if isinstance(buff, int):
+                break
+            self.recv_queue.put_nowait(buff)
+        while True:
+            try:
+                buff = self.send_queue.get_nowait()
+                if self._send(buff) < 0:
+                    self.send_queue.put_nowait(buff)
+                    break
+            except asyncio.QueueEmpty:
+                break
+
+        return True
+
+
+class KCPProtocol(asyncio.DatagramProtocol):
+
+    def __init__(self,
+                 conv: int,
+                 ttl: int = 3600,
+                 nodelay: bool = True,
+                 interval: int = 20,
+                 resend: int = 2,
+                 nc: bool = True):
+        self.ttl = ttl
+        self.conv = conv
+        self.nodelay = nodelay
+        self.interval = interval
+        self.resend = resend
+        self.nc = nc
+        self.transport = None
+        self.connections = {}
+        self.autoclean_loop: asyncio.Future = None
+        self.clean()
+
+    def connection_lost(self, exc):
+        self.autoclean_loop.cancel()
+        for address in self.connections:
+            del self.connections[address]
+
+    def connection_made(self, transport):
+        self.transport = transport
+
+    def get_connection(self, address):
+        if address not in self.connections:
+            self.connections[address] = KCPConnection(address, self.transport,
+                                                      self.conv, self.nodelay,
+                                                      self.interval,
+                                                      self.resend, self.nc)
+        self.connections[address].last_recv = time.time()
+        return self.connections[address]
+
+    def datagram_received(self, datagram, address):
+        self.get_connection(address).input(datagram)
+
+    def sendto(self, msg, address):
+        self.get_connection(address).send(msg)
+
+    def recvfrom(self):
+        for address, conn in list(self.connections.items()):
+            buff = conn.recv_nowait()
+            if buff:
+                return buff, address
+
+    def clean(self):
+        current = time.time()
+        dead = []
+        for address, conn in list(self.connections.items()):
+            if current - conn.last_active > self.ttl:
+                dead.append(address)
+        for address in dead:
+            del self.connections[address]
+        loop = asyncio.get_running_loop()
+        self.autoclean_loop = loop.call_later(10, self.clean)
+
+
+async def listen(address: tuple[str, int],
+                 conv: int = 1234,
+                 ttl: int = 3600,
+                 nodelay: bool = True,
+                 interval: int = 20,
+                 resend: int = 2,
+                 nc: bool = True):
+    """
+    Listen on a UDP address and return a KCPProtocol instance.
+
+    Args:
+        ttl (int): The connection TTL (in second).
+        address (ip, port): The address to listen on.
+        conv (int): The conversation ID.
+        nodelay (bool): Whether to enable nodelay mode.
+        interval (int): The internal update interval (in millisecond).
+        resend (int): The number of times to resend unacknowledged packets.
+        nc (bool): Whether to enable congestion control.
+
+    Returns:
+        A KCPProtocol instance.
+    """
+
+    def protocol_factory():
+        return KCPProtocol(conv, ttl, nodelay, interval, resend, nc)
+
+    loop = asyncio.get_running_loop()
+    transport, protocol = await loop.create_datagram_endpoint(
+        protocol_factory, local_addr=address)
+    return protocol

qulab/sys/net/nginx.py

@@ -0,0 +1,194 @@
+import asyncio
+import os
+from urllib.parse import quote_plus
+
+from fastapi import FastAPI, Request
+from fastapi.responses import Response
+from pydantic import BaseModel
+
+NGINX_DIR = 'C:/Users/Administrator/Desktop/nginx/nginx-1.20.2'
+HOST_URL = 'https://systemq.baqis.ac.cn'
+
+db = {
+    "n01": {
+        "notebook_port": 1000,
+        "codeserver_port": 1001,
+        "ip": "127.0.0.1",
+        "token": "ed747047-1d9d-47b0-8fc2-b4914f8a9bd9",
+    },
+    "n02": {
+        "notebook_port": 1002,
+        "codeserver_port": 1003,
+        "ip": "127.0.0.1",
+        "token": "ed423dd1-05fb-48d2-a720-dbebfb2ea3e1",
+    },
+    "n03": {
+        "notebook_port": 1004,
+        "codeserver_port": 1005,
+        "ip": "127.0.0.1",
+        "token": "e3b0c442-98fc-11e4-8dfc-aa07a5b093d3",
+    },
+}
+
+
+def fmt_config(ip, notebook_port, codeserver_port, local_notebook_port,
+               local_codeserver_port, page):
+    return """
+server {""" + f"""
+    listen {notebook_port} ssl;
+    listen [::]:{notebook_port} ssl;
+""" + """
+    include ssl/ssl.conf;
+
+    server_name  systemq.baqis.ac.cn;
+
+    client_max_body_size 100M;
+
+    # notebook
+    location / {
+        include auth/auth.conf;
+""" + f"""
+        proxy_pass http://{ip}:{local_notebook_port}/;""" + """
+        proxy_set_header X-Real_IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-NginX-Proxy true;
+        proxy_ssl_session_reuse off;
+        proxy_set_header Host $http_host;
+
+        proxy_redirect off;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+
+    location /static/ {
+        alias  html/_jupyter_notebook/static/;
+    }
+
+    location /login {
+        include auth/auth.conf;
+        default_type text/html;
+        add_header Content-Type "text/html; charset=UTF-8";
+""" + f"""
+        return 200 "{page}";
+""" + """
+    }
+
+    include auth/location.conf;
+}
+
+server {""" + f"""
+    listen  {codeserver_port} ssl;
+    listen [::]:{codeserver_port} ssl;
+""" + """
+    include ssl/ssl.conf;
+
+    server_name  systemq.baqis.ac.cn;
+
+    client_max_body_size 100M;
+
+    # vscode
+    location / {
+        include auth/auth.conf;
+""" + f"""
+        proxy_pass http://{ip}:{local_codeserver_port}/;""" + """
+        proxy_set_header X-Real_IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-NginX-Proxy true;
+        proxy_ssl_session_reuse off;
+        proxy_set_header Host $http_host;
+
+        proxy_redirect off;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+
+    location /oss-dev/static/out/ {
+        root   html;
+    }
+
+    include auth/location.conf;
+}
+"""
+
+
+def fmt_page(notebook_url, codeserver_url):
+    return """
+<!DOCTYPE html>
+<html>
+<head>
+<title>Welcome to systemq!</title>
+<style>
+body {
+    width: 35em;
+    margin: 0 auto;
+    font-family: Tahoma, Verdana, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<h1>Welcome to systemq!</h1>""" + f"""
+<a href=\\"{notebook_url}\\">notebook</a>.<br/>
+<a href=\\"{codeserver_url}\\">codeserver</a>.<br/>
+</body>
+</html>
+"""
+
+
+class Node(BaseModel):
+    token: str
+    name: str
+    notebook_port: int = 8888
+    codeserver_port: int = 14263
+    codeserver_path: str = ""
+
+
+app = FastAPI()
+
+
+@app.post("/auth")
+async def auth(request: Request, node: Node):
+    try:
+        assert node.name in db
+        assert node.token == db[node.name]['token']
+        notebook_port = db[node.name]['notebook_port']
+        codeserver_port = db[node.name]['codeserver_port']
+
+        ip = request.client.host
+        if ip == db[node.name]['ip']:
+            return Response(status_code=200)
+        else:
+            print("    CHANGE IP", db[node.name]['ip'], "===>", ip)
+            db[node.name]['ip'] = ip
+
+        #ip = request.headers.get('x-real-ip')
+        local_notebook_port = node.notebook_port
+        local_codeserver_port = node.codeserver_port
+        codeserver_path = quote_plus(node.codeserver_path)
+
+        codeserver_url = f"{HOST_URL}:{codeserver_port}?tkn=lZBAQISFF532&folder={codeserver_path}"
+        notebook_url = f"{HOST_URL}:{notebook_port}/tree"
+
+        page = fmt_page(notebook_url, codeserver_url)
+        page = ''.join(page.splitlines())
+
+        config = fmt_config(ip, notebook_port, codeserver_port,
+                            local_notebook_port, local_codeserver_port, page)
+
+        with open(f"{NGINX_DIR}/conf/servers/{node.name}.conf", "w") as f:
+            f.write(config)
+
+        await asyncio.sleep(0.1)
+
+        cwd = os.getcwd()
+
+        os.chdir(NGINX_DIR)
+
+        os.system(f'{NGINX_DIR}/nginx.exe -s reload')
+
+        os.chdir(cwd)
+
+        return Response(status_code=200)
+    except:
+        return Response(status_code=400)

qulab/sys/progress.py

@@ -0,0 +1,190 @@
+import asyncio
+from collections import deque
+from datetime import timedelta
+from time import monotonic
+
+from blinker import Signal
+
+try:
+    import ipywidgets as widgets
+    from IPython.display import display
+except:
+    pass
+
+
+class Progress:
+    sma_window = 10  # Simple Moving Average window
+
+    def __init__(self, *, max=10):
+        self.start_ts = monotonic()
+        self._ts = self.start_ts
+        self._xput = deque(maxlen=self.sma_window)
+        self._max = max
+        self.index = 0
+        self.updated = Signal()
+        self.finished = Signal()
+        self.status = "running"
+
+    @property
+    def progress(self):
+        return min(1.0, self.index / self.max)
+
+    @property
+    def remaining(self):
+        return max(self.max - self.index, 0)
+
+    @property
+    def percent(self):
+        return self.progress * 100
+
+    @property
+    def elapsed(self):
+        if self.status != "running":
+            return self._ts - self.start_ts
+        return monotonic() - self.start_ts
+
+    @property
+    def eta(self):
+        if self.status != "running":
+            return 0
+        avg = sum(self._xput) / len(self._xput)
+        dt = monotonic() - self._ts
+        if dt > avg:
+            avg = 0.9 * avg + 0.1 * dt
+            return avg * self.remaining
+        else:
+            return avg * self.remaining - dt
+
+    @property
+    def max(self):
+        return self._max
+
+    @max.setter
+    def max(self, x):
+        self._max = x
+        self.updated.send(self)
+
+    def next(self, n=1):
+        assert n > 0
+        now = monotonic()
+        dt = now - self._ts
+        self._xput.append(dt / n)
+        self._ts = now
+        self.index = self.index + n
+        self.updated.send(self)
+
+    def goto(self, index):
+        incr = index - self.index
+        self.next(incr)
+
+    def finish(self, success=True):
+        self._ts = monotonic()
+        if success:
+            self.status = "finished"
+        else:
+            self.status = "failure"
+        self.finished.send(self, success=success)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is None:
+            self.finish(True)
+        else:
+            self.finish(False)
+
+    def iter(self, it, max=None):
+        if max is None:
+            try:
+                self.max = len(it)
+            except TypeError:
+                self.max = 0
+        else:
+            self.max = max
+
+        with self:
+            for x in it:
+                yield x
+                self.next()
+
+    def __repr__(self):
+        if self._ts == self.start_ts:
+            eta_td = '--:--:--'
+        else:
+            eta_td = timedelta(seconds=round(self.eta))
+        return (
+            f"({self.index}/{self.max}) {self.percent:.0f}%"
+            f" Used time: {timedelta(seconds=round(self.elapsed))} Remaining time: {eta_td}"
+            f" {self.status}")
+
+
+class ProgressBar():
+
+    def listen(self, progress: Progress):
+        self.progress = progress
+        self.progress.updated.connect(self.update)
+        self.progress.finished.connect(self.finish)
+
+    def update(self, sender: Progress):
+        raise NotImplementedError()
+
+    def finish(self, sender: Progress, success: bool):
+        self.progress.updated.disconnect(self.update)
+        self.progress.finished.disconnect(self.finish)
+
+
+class JupyterProgressBar(ProgressBar):
+
+    def __init__(self, *, description='Progressing', hiden=False):
+        self.description = description
+        self.hiden = hiden
+
+    def display(self):
+        if self.hiden:
+            return
+        self.progress_ui = widgets.FloatProgress(value=0,
+                                                 min=0,
+                                                 max=100.0,
+                                                 step=1,
+                                                 description=self.description,
+                                                 bar_style='')
+
+        self.elapsed_ui = widgets.Label(value='Used time: 00:00:00')
+        self.eta_ui = widgets.Label(value='Remaining time: --:--:--')
+        self.ui = widgets.HBox(
+            [self.progress_ui, self.elapsed_ui, self.eta_ui])
+        display(self.ui)
+
+    def listen(self, progress: Progress):
+        super().listen(progress)
+        self.update_regularly()
+
+    def update_regularly(self, frequency=1):
+        try:
+            self.update(self.progress)
+        except:
+            pass
+        asyncio.get_running_loop().call_later(frequency, self.update_regularly)
+
+    def update(self, sender: Progress):
+        if self.hiden:
+            return
+        self.progress_ui.value = sender.percent
+        self.elapsed_ui.value = (
+            f'({sender.index}/{sender.max}) '
+            f'Used time: {timedelta(seconds=round(sender.elapsed))}')
+        if sender.eta == sender.start_ts:
+            self.eta_ui.value = f'Remaining time: --:--:--'
+        else:
+            self.eta_ui.value = f'Remaining time: {timedelta(seconds=round(sender.eta))}'
+
+    def finish(self, sender: Progress, success: bool = True):
+        if self.hiden:
+            return
+        if success:
+            self.progress_ui.bar_style = 'success'
+            self.progress_ui.value = 100.0
+        else:
+            self.progress_ui.bar_style = 'danger'
+        super().finish(sender, success)

qulab/sys/rpc/exceptions.py

@@ -0,0 +1,96 @@
+import inspect
+from collections import deque
+
+
+def _parse_frame(frame):
+    ret = {}
+    ret['source'] = inspect.getsource(frame)
+    ret['name'] = frame.f_code.co_name
+    ret['firstlineno'] = 1
+    if ret['name'] != '<module>':
+        argnames = frame.f_code.co_varnames[:frame.f_code.co_argcount +
+                                            frame.f_code.co_kwonlyargcount]
+        ret['name'] += '(' + ', '.join(argnames) + ')'
+        ret['firstlineno'] = frame.f_code.co_firstlineno
+    ret['filename'] = frame.f_code.co_filename
+    return ret
+
+
+def _parse_traceback(err):
+    ret = []
+    tb = err.__traceback__
+    while tb is not None:
+        frame = _parse_frame(tb.tb_frame)
+        frame['lineno'] = tb.tb_lineno
+        ret.append(frame)
+        tb = tb.tb_next
+    return ret
+
+
+def _format_frame(frame):
+    post_lines = -1
+    lines = deque(maxlen=16)
+    lines.append(f"{frame['filename']} in {frame['name']}")
+    for n, line in enumerate(frame['source'].split('\n')):
+        lno = n + frame['firstlineno']
+        if lno == frame['lineno']:
+            lines.append(f"->{lno:3d} {line}")
+            post_lines = 0
+        else:
+            lines.append(f"  {lno:3d} {line}")
+            if post_lines >= 0:
+                post_lines += 1
+        if post_lines >= 7:
+            break
+    return '\n'.join(lines)
+
+
+def _format_traceback(err):
+    frame_text = []
+    for frame in _parse_traceback(err):
+        frame_text.append(_format_frame(frame))
+    traceback_text = '\n'.join(frame_text)
+    args = list(err.args)
+    args.append(traceback_text)
+    err.args = tuple(args)
+    return err
+
+
+###############################################################
+# RPC Exceptions
+###############################################################
+
+
+class RPCError(Exception):
+    """
+    RPC base exception.
+    """
+
+
+class RPCServerError(RPCError):
+    """
+    Server side error.
+    """
+
+    @classmethod
+    def make(cls, exce):
+        exce = _format_traceback(exce)
+        args = [exce.__class__.__name__]
+        args.extend(list(exce.args))
+        return cls(*args)
+
+    def _repr_markdown_(self):
+        return '\n'.join([
+            '```python',
+            '---------------------------------------------------------------------------',
+            f'RPCServerError({self.args[0]})               Server raise:{self.args[1]}',
+            f'{self.args[2]}',
+            '---------------------------------------------------------------------------',
+            '```',
+        ])
+
+
+class RPCTimeout(RPCError):
+    """
+    Timeout.
+    """