cloudoll 2.4.0__py3-none-any.whl

cloudoll/utils/patch_loop.py

@@ -0,0 +1,222 @@
+"""
+from https://github.com/erdewit/nest_asyncio/blob/master/nest_asyncio.py
+"""
+"""Patch asyncio to allow nested event loops."""
+
+import asyncio
+import asyncio.events as events
+import os
+import sys
+import threading
+from contextlib import contextmanager, suppress
+from heapq import heappop
+
+
+def apply(loop=None):
+    """Patch asyncio to make its event loop reentrant."""
+    _patch_asyncio()
+    _patch_policy()
+    _patch_tornado()
+
+    loop = loop or asyncio.get_event_loop()
+    _patch_loop(loop)
+
+
+def _patch_asyncio():
+    """Patch asyncio module to use pure Python tasks and futures."""
+
+    def run(main, *, debug=False):
+        loop = asyncio.get_event_loop()
+        loop.set_debug(debug)
+        task = asyncio.ensure_future(main)
+        try:
+            return loop.run_until_complete(task)
+        finally:
+            if not task.done():
+                task.cancel()
+                with suppress(asyncio.CancelledError):
+                    loop.run_until_complete(task)
+
+    def _get_event_loop(stacklevel=3):
+        loop = events._get_running_loop()
+        if loop is None:
+            loop = events.get_event_loop_policy().get_event_loop()
+        return loop
+
+    # Use module level _current_tasks, all_tasks and patch run method.
+    if hasattr(asyncio, '_nest_patched'):
+        return
+    if sys.version_info >= (3, 6, 0):
+        asyncio.Task = asyncio.tasks._CTask = asyncio.tasks.Task = \
+            asyncio.tasks._PyTask
+        asyncio.Future = asyncio.futures._CFuture = asyncio.futures.Future = \
+            asyncio.futures._PyFuture
+    if sys.version_info < (3, 7, 0):
+        asyncio.tasks._current_tasks = asyncio.tasks.Task._current_tasks
+        asyncio.all_tasks = asyncio.tasks.Task.all_tasks
+    if sys.version_info >= (3, 9, 0):
+        events._get_event_loop = events.get_event_loop = \
+            asyncio.get_event_loop = _get_event_loop
+    asyncio.run = run
+    asyncio._nest_patched = True
+
+
+def _patch_policy():
+    """Patch the policy to always return a patched loop."""
+
+    def get_event_loop(self):
+        if self._local._loop is None:
+            loop = self.new_event_loop()
+            _patch_loop(loop)
+            self.set_event_loop(loop)
+        return self._local._loop
+
+    policy = events.get_event_loop_policy()
+    policy.__class__.get_event_loop = get_event_loop
+
+
+def _patch_loop(loop):
+    """Patch loop to make it reentrant."""
+
+    def run_forever(self):
+        with manage_run(self), manage_asyncgens(self):
+            while True:
+                self._run_once()
+                if self._stopping:
+                    break
+        self._stopping = False
+
+    def run_until_complete(self, future):
+        with manage_run(self):
+            f = asyncio.ensure_future(future, loop=self)
+            if f is not future:
+                f._log_destroy_pending = False
+            while not f.done():
+                self._run_once()
+                if self._stopping:
+                    break
+            if not f.done():
+                raise RuntimeError(
+                    'Event loop stopped before Future completed.')
+            return f.result()
+
+    def _run_once(self):
+        """
+        Simplified re-implementation of asyncio's _run_once that
+        runs handles as they become ready.
+        """
+        ready = self._ready
+        scheduled = self._scheduled
+        while scheduled and scheduled[0]._cancelled:
+            heappop(scheduled)
+
+        timeout = (
+            0 if ready or self._stopping
+            else min(max(
+                scheduled[0]._when - self.time(), 0), 86400) if scheduled
+            else None)
+        event_list = self._selector.select(timeout)
+        self._process_events(event_list)
+
+        end_time = self.time() + self._clock_resolution
+        while scheduled and scheduled[0]._when < end_time:
+            handle = heappop(scheduled)
+            ready.append(handle)
+
+        for _ in range(len(ready)):
+            if not ready:
+                break
+            handle = ready.popleft()
+            if not handle._cancelled:
+                # preempt the current task so that that checks in
+                # Task.__step do not raise
+                curr_task = curr_tasks.pop(self, None)
+
+                try:
+                    handle._run()
+                finally:
+                    # restore the current task
+                    if curr_task is not None:
+                        curr_tasks[self] = curr_task
+
+        handle = None
+
+    @contextmanager
+    def manage_run(self):
+        """Set up the loop for running."""
+        self._check_closed()
+        old_thread_id = self._thread_id
+        old_running_loop = events._get_running_loop()
+        try:
+            self._thread_id = threading.get_ident()
+            events._set_running_loop(self)
+            self._num_runs_pending += 1
+            if self._is_proactorloop:
+                if self._self_reading_future is None:
+                    self.call_soon(self._loop_self_reading)
+            yield
+        finally:
+            self._thread_id = old_thread_id
+            events._set_running_loop(old_running_loop)
+            self._num_runs_pending -= 1
+            if self._is_proactorloop:
+                if (self._num_runs_pending == 0
+                        and self._self_reading_future is not None):
+                    ov = self._self_reading_future._ov
+                    self._self_reading_future.cancel()
+                    if ov is not None:
+                        self._proactor._unregister(ov)
+                    self._self_reading_future = None
+
+    @contextmanager
+    def manage_asyncgens(self):
+        if not hasattr(sys, 'get_asyncgen_hooks'):
+            # Python version is too old.
+            return
+        old_agen_hooks = sys.get_asyncgen_hooks()
+        try:
+            self._set_coroutine_origin_tracking(self._debug)
+            if self._asyncgens is not None:
+                sys.set_asyncgen_hooks(
+                    firstiter=self._asyncgen_firstiter_hook,
+                    finalizer=self._asyncgen_finalizer_hook)
+            yield
+        finally:
+            self._set_coroutine_origin_tracking(False)
+            if self._asyncgens is not None:
+                sys.set_asyncgen_hooks(*old_agen_hooks)
+
+    def _check_running(self):
+        """Do not throw exception if loop is already running."""
+        pass
+
+    if hasattr(loop, '_nest_patched'):
+        return
+    if not isinstance(loop, asyncio.BaseEventLoop):
+        raise ValueError('Can\'t patch loop of type %s' % type(loop))
+    cls = loop.__class__
+    cls.run_forever = run_forever
+    cls.run_until_complete = run_until_complete
+    cls._run_once = _run_once
+    cls._check_running = _check_running
+    cls._check_runnung = _check_running  # typo in Python 3.7 source
+    cls._num_runs_pending = 1 if loop.is_running() else 0
+    cls._is_proactorloop = (
+        os.name == 'nt' and issubclass(cls, asyncio.ProactorEventLoop))
+    if sys.version_info < (3, 7, 0):
+        cls._set_coroutine_origin_tracking = cls._set_coroutine_wrapper
+    curr_tasks = asyncio.tasks._current_tasks \
+        if sys.version_info >= (3, 7, 0) else asyncio.Task._current_tasks
+    cls._nest_patched = True
+
+
+def _patch_tornado():
+    """
+    If tornado is imported before nest_asyncio, make tornado aware of
+    the pure-Python asyncio Future.
+    """
+    if 'tornado' in sys.modules:
+        import tornado.concurrent as tc  # type: ignore
+        tc.Future = asyncio.Future
+        if asyncio.Future not in tc.FUTURES:
+            tc.FUTURES += (asyncio.Future,)

cloudoll/utils/snowflake.py

@@ -0,0 +1,86 @@
+
+import time
+import threading
+
+
+"""
+雪花算法（全局唯一id）
+
+# 最早是Twitter公司在其内部用于分布式环境下生成唯一ID
+
+# Twitter's Snowflake algorithm implementation which is used to generate distributed IDs.
+
+# https://github.com/twitter-archive/snowflake/blob/snowflake-2010/src/main/scala/com/twitter/service/snowflake/IdWorker.scala
+"""
+
+
+class IdWorker:
+    def __init__(self, worker_id, datacenter_id, sequence=0):
+         # 位移值
+        self.worker_id_bits = 5
+        self.datacenter_id_bits = 5
+        self.sequence_bits = 12
+        
+        self.max_worker_id = -1 ^ (-1 << self.worker_id_bits)
+        self.max_datacenter_id = -1 ^ (-1 << self.datacenter_id_bits)
+
+        # 参数校验
+        if worker_id > self.max_worker_id or worker_id < 0:
+            raise ValueError(f'worker_id must be between 0 and {self.max_worker_id}')
+        if datacenter_id > self.max_datacenter_id or datacenter_id < 0:
+            raise ValueError(f'datacenter_id must be between 0 and {self.max_datacenter_id}')
+
+        self.worker_id = worker_id
+        self.datacenter_id = datacenter_id
+        self.sequence = sequence
+        self.lock = threading.Lock()
+
+        # Twitter Epoch (2010-11-04 01:42:54 UTC)
+        self.twepoch = 1288834974657
+
+       
+
+       
+        self.worker_id_shift = self.sequence_bits
+        self.datacenter_id_shift = self.sequence_bits + self.worker_id_bits
+        self.timestamp_left_shift = self.sequence_bits + self.worker_id_bits + self.datacenter_id_bits
+        self.sequence_mask = -1 ^ (-1 << self.sequence_bits)
+
+        self.last_timestamp = -1
+
+    def _time_gen(self):
+        return int(time.time() * 1000)
+
+    def _til_next_millis(self, last_timestamp):
+        timestamp = self._time_gen()
+        while timestamp <= last_timestamp:
+            timestamp = self._time_gen()
+        return timestamp
+
+    def next_id(self):
+        with self.lock:
+            timestamp = self._time_gen()
+
+            if timestamp < self.last_timestamp:
+                raise Exception(f'Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds')
+
+            if self.last_timestamp == timestamp:
+                self.sequence = (self.sequence + 1) & self.sequence_mask
+                if self.sequence == 0:
+                    timestamp = self._til_next_millis(self.last_timestamp)
+            else:
+                self.sequence = 0
+
+            self.last_timestamp = timestamp
+
+            id = ((timestamp - self.twepoch) << self.timestamp_left_shift) | \
+                (self.datacenter_id << self.datacenter_id_shift) | \
+                (self.worker_id << self.worker_id_shift) | \
+                self.sequence
+
+            return id
+
+
+# eg:
+# worker = IdWorker(worker_id=1, datacenter_id=1)
+# print(worker.next_id())

cloudoll/utils/watch.py

@@ -0,0 +1,194 @@
+from typing import AsyncIterator, Optional, Union
+from pathlib import Path
+from ..web import Application, app
+import asyncio
+import os
+import signal
+import sys
+from contextlib import suppress
+from multiprocessing import Process
+from watchfiles import awatch, DefaultFilter
+from ..logging import info, warning, error
+import contextlib
+from typing import Iterator, Optional
+from typing import Optional, Union
+from aiohttp import web
+import traceback
+from .common import check_port_open
+
+class CloudollFilter(DefaultFilter):
+    def __init__(self, ignore_dirs: tuple = None) -> None:
+        self.ignore_dirs = self.ignore_dirs + tuple("logs")
+        if ignore_dirs:
+            self.ignore_dirs = self.ignore_dirs + ignore_dirs
+
+        super().__init__()
+
+
+class WatchTask:
+    _app: Application
+    _task: "asyncio.Task[None]"
+
+    def __init__(self, path: Union[Path, str]):
+        self._path = path
+
+    async def start(self, app: Application) -> None:
+        self._app = app
+        self.stopper = asyncio.Event()
+        ignore_dirs = self._config["server"].get("ignore_dirs", [])
+        self._awatch = awatch(
+            self._path,
+            stop_event=self.stopper,
+            watch_filter=CloudollFilter(tuple(ignore_dirs)),
+        )
+        self._task = asyncio.create_task(self._run())
+
+    async def _run(self) -> None:
+        raise NotImplementedError()
+
+    async def close(self, *args) -> None:
+        if self._task:
+            self.stopper.set()
+            self._task.cancel()
+            with suppress(asyncio.CancelledError):
+                await self._task
+
+    async def cleanup_ctx(self, app: Application) -> AsyncIterator[None]:
+        await self.start(app)
+        yield
+        await self.close()
+
+
+@contextlib.contextmanager
+def set_tty(tty_path: Optional[str]) -> Iterator[None]:
+    try:
+        if not tty_path:
+            # to match OSError from open
+            raise OSError()
+        with open(tty_path) as tty:
+            sys.stdin = tty
+            yield
+    except OSError:
+        # either tty_path is None (windows) or opening it fails (eg. on pycharm)
+        yield
+
+
+def mian_app(tty_path, config, entry, env):
+    with set_tty(tty_path):
+        if sys.version_info >= (3, 11):
+            with asyncio.Runner() as runner:
+                app_runner = runner.run(create_main_app(config, entry, env))
+                try:
+                    runner.run(start_main_app(app_runner, config["server"]["port"]))
+                    runner.get_loop().run_forever()
+                except KeyboardInterrupt:
+                    pass
+                finally:
+                    with suppress(asyncio.TimeoutError, KeyboardInterrupt):
+                        runner.run(app_runner.cleanup())
+        else:
+            loop = asyncio.new_event_loop()
+            runner = loop.run_until_complete(create_main_app(config, entry, env))
+            try:
+                loop.run_until_complete(start_main_app(runner, config["server"]["port"]))
+                loop.run_forever()
+            except KeyboardInterrupt:
+                pass
+            finally:
+                with suppress(asyncio.TimeoutError, KeyboardInterrupt):
+                    loop.run_until_complete(runner.cleanup())
+
+
+async def create_main_app(config, entry, env):
+    await check_port_open(config["server"]["port"])
+    App = app.create(env=env, config=config, entry_model=entry)
+    return web.AppRunner(App.app, shutdown_timeout=0.1)
+
+
+async def start_main_app(runner, port):
+    await runner.setup()
+    site = web.TCPSite(runner, host="0.0.0.0", port=port)
+    info(f"Server running on http://0.0.0.0:{port}")
+    info("(Press CTRL+C to quit)")
+    await site.start()
+
+
+class AppTask(WatchTask):
+    def __init__(self, watch_path: str, config, entry: None, env: str):
+        self._config = config
+        self._entry = entry
+        self._env = env
+        self._reloads = 0
+        assert watch_path
+        super().__init__(watch_path)
+
+    async def _run(self) -> None:
+        assert self._app is not None
+        self._start_dev_server()
+        try:
+            async for changes in self._awatch:
+                self._reloads += 1
+                if any(f.endswith(".py") for _, f in changes):
+                    info(f"{len(changes)} changes, restarting server")
+                    await self._stop_dev_server()
+                    self._start_dev_server()
+                    await asyncio.sleep(1)
+        except Exception as exc:
+            traceback.print_exc()
+
+            raise Exception("error running dev server")
+
+    def _start_dev_server(self) -> None:
+        act = "Start" if self._reloads == 0 else "Restart"
+        info(f"{act}ing dev server")
+
+        try:
+            tty_path = os.ttyname(sys.stdin.fileno())
+        except OSError:  # pragma: no branch
+            # fileno() always fails with pytest
+            tty_path = "/dev/tty"
+        except AttributeError:
+            # on windows, without a windows machine I've no idea what else to do here
+            tty_path = None
+
+        # for key, value in self._config.items():
+        #     try:
+        #         pickle.dumps(value)
+        #     except Exception as e:
+        #         print(f"Serialization error on key '{key}': {e} {value}")
+
+        # import pickle
+        # try:
+        #     pickle.dumps(b)  # 检查是否可序列化
+        # except Exception as e:
+        #     print("Serialization error:", e)
+        try:
+            self._process = Process(
+                target=mian_app, args=(tty_path, self._config, self._entry, self._env)
+            )
+            self._process.start()
+        except Exception as e:
+            error(f"Error starting server: {e}")
+
+    async def _stop_dev_server(self) -> None:
+        if self._process.is_alive():
+            info("stopping server process...")
+            if self._process.pid:
+                info("sending SIGINT")
+                os.kill(self._process.pid, signal.SIGINT)
+            self._process.join(5)
+            if self._process.exitcode is None:
+                warning("process has not terminated, sending SIGKILL")
+                self._process.kill()
+                self._process.join(1)
+            else:
+                info("process stopped")
+        else:
+            warning(
+                "server process already dead, exit code: %s", self._process.exitcode
+            )
+
+    async def close(self) -> None:
+        self.stopper.set()
+        await self._stop_dev_server()
+        await asyncio.gather(super().close())

cloudoll/web/__init__.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+__author__ = "chuchur/chuchur.com"
+from typing import Tuple
+
+from .settings import get_config
+from . import jwt
+from .core import (
+    Application,
+    app,
+    Response,
+    WebSocketResponse,
+    WebSocket,
+    WebStream,
+    WSMsgType,
+    View,
+    routes,
+    get,
+    put,
+    delete,
+    post,
+    render_json,
+    render_error,
+    render_view,
+    render,
+    middleware,
+    redirect,
+)
+
+__all__: Tuple[str, ...] = (
+    "Application",
+    "app",
+    "WebSocket",
+    "Response",
+    "WebSocketResponse",
+    "WebStream",
+    "WSMsgType",
+    "View",
+    "routes",
+    "get",
+    "put",
+    "delete",
+    "post",
+    "render_json",
+    "render_error",
+    "render_view",
+    "render",
+    "middleware",
+    "redirect",
+    "jwt",
+    "get_config",
+)