cloudoll 3.0.6__tar.gz → 3.0.7__tar.gz

{cloudoll-3.0.6 → cloudoll-3.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudoll
-Version: 3.0.6
+Version: 3.0.7
 Summary: Assist in quickly creating distributable microservices based on aiohttp, with a focus on simplicity and ease of use.
 Home-page: https://github.com/smallerqiu/cloudoll-py
 Author: Qiu

cloudoll-3.0.7/cloudoll/__init__.py

@@ -0,0 +1 @@
+__version__ = "3.0.7"

cloudoll-3.0.7/cloudoll/orm/__init__.py

@@ -0,0 +1,44 @@
+from redis import asyncio as aioredis
+from cloudoll.orm.parse import parse_coon
+import aiopg as pg
+from .mysql import Mysql
+from .postgres import Postgres
+from .awspostgres import AwsPostgres
+from .awsmysql import AwsMysql
+
+__all__ = ["create_engine"]
+
+
+async def create_engine(**kw):
+    url = kw.get("url")
+    driver = None
+    configs = {}
+    query = {}
+
+    if url is not None:
+        configs, query = parse_coon(url)
+        driver = configs["type"]
+    else:
+        driver = kw.get("type")
+        configs = kw
+
+    # info("DB Config:", configs, query)
+
+    if driver == "mysql":
+        return await Mysql().create_engine(**configs, **query)
+    elif driver == "aws-mysql":
+        return await AwsMysql().create_engine(**configs, **query)
+    elif driver in ["aws-postgres", "aws-postgressql"]:
+        return await AwsPostgres().create_engine(**configs, **query)
+    elif driver in ["postgres", "postgressql"]:
+        return await Postgres().create_engine(**configs, **query)
+    elif driver in ["redis", "rediss"]:
+        """
+        redis://[[username]:[password]]@localhost:6379/0
+        rediss://[[username]:[password]]@localhost:6379/0
+        """
+        if url is None:
+            url = f"{driver}://{configs['username']}:{configs['password']}@{configs['host']}:{configs['port']}/{configs['db']}"
+        return await aioredis.from_url(url, **query)
+    else:
+        raise ValueError("Not support this database type.")

cloudoll-3.0.7/cloudoll/orm/awsmysql.py

@@ -0,0 +1,79 @@
+from mysql.connector import Connect
+
+from aws_advanced_python_wrapper import AwsWrapperConnection
+from aws_advanced_python_wrapper.connection_provider import ConnectionProviderManager
+from aws_advanced_python_wrapper.sql_alchemy_connection_provider import (
+    SqlAlchemyPooledConnectionProvider,
+)
+
+from cloudoll.orm.base import MeteBase, QueryTypes
+from cloudoll.logging import info, error
+
+
+class AwsMysql(MeteBase):
+    def __init__(self):
+        self.driver = "aws-mysql"
+        provider = SqlAlchemyPooledConnectionProvider()
+        ConnectionProviderManager.set_connection_provider(provider)
+
+    async def close(self):
+        ConnectionProviderManager.release_resources()
+
+    async def create_engine(self, **kw):
+        self._dsn = f"host={kw.get("host")} database={kw.get("db")} user={kw.get("username")} password={kw.get("password","")}"
+        self._params = {
+            "plugins": kw.get("plugins", ""),
+            "wrapper_dialect": kw.get("wrapper_dialect", ""),
+            "autocommit": True,
+        }
+        return self
+
+    async def query(self, sql, params=None, query_type=QueryTypes.ONE, size=10):
+        sql = sql.replace("?", "%s")
+        try:
+            with AwsWrapperConnection.connect(
+                Connect, self._dsn, **self._params
+            ) as conn:
+                with conn.cursor() as cursor:
+                    if (
+                        query_type == QueryTypes.CREATEBATCH
+                        or query_type == QueryTypes.UPDATEBATCH
+                    ):
+                        cursor.executemany(sql, params)
+                    else:
+                        cursor.execute(sql, params)
+
+                    if query_type == QueryTypes.ALL:
+                        return cursor.fetchall()
+                    elif query_type == QueryTypes.ONE:
+                        return cursor.fetchone()
+                    elif query_type == QueryTypes.MANY:
+                        return cursor.fetchmany(size)
+                    elif query_type == QueryTypes.COUNT:
+                        rows = cursor.fetchone()
+                        count = 0
+                        if rows is None:
+                            return count
+                        for row in rows:
+                            count = rows[row]
+                        return count
+                    elif query_type == QueryTypes.GROUP_COUNT:
+                        result = cursor.fetchall()
+                        return 0 if not result else len(result)
+                    elif query_type == QueryTypes.CREATE:
+                        result = cursor.rowcount > 0
+                        id = cursor.target_cursor.lastrowid
+                        return result, id
+                    elif query_type == QueryTypes.CREATEBATCH:
+                        count = cursor.rowcount
+                        id = cursor.target_cursor.lastrowid
+                        return count, id
+                    elif query_type == QueryTypes.UPDATE:
+                        return cursor.rowcount > 0
+                    elif query_type == QueryTypes.UPDATEBATCH:
+                        return cursor.rowcount
+                    elif query_type == QueryTypes.DELETE:
+                        return cursor.rowcount > 0
+        except Exception as e:
+            error(f"[AWS PG] query error: {e}, SQL: {sql}")
+            raise e

cloudoll-3.0.7/cloudoll/orm/awspostgres.py

@@ -0,0 +1,93 @@
+import psycopg
+from aws_advanced_python_wrapper import AwsWrapperConnection
+from aws_advanced_python_wrapper.connection_provider import ConnectionProviderManager
+from aws_advanced_python_wrapper.sql_alchemy_connection_provider import (
+    SqlAlchemyPooledConnectionProvider,
+)
+
+from cloudoll.orm.base import MeteBase, QueryTypes
+from cloudoll.logging import info, error
+
+
+class AwsPostgres(MeteBase):
+    def __init__(self):
+        self.driver = "aws-postgres"
+        provider = SqlAlchemyPooledConnectionProvider()
+        ConnectionProviderManager.set_connection_provider(provider)
+
+    async def close(self):
+        ConnectionProviderManager.release_resources()
+
+    async def create_engine(self, **kw):
+        self._dsn = f"host={kw.get("host")} dbname={kw.get("db")} user={kw.get("username")} password={kw.get("username")}"
+        self._params = {
+            "plugins": kw.get("plugins", ""),
+            "wrapper_dialect": kw.get("wrapper_dialect"),
+            "autocommit": True,
+        }
+        return self
+
+    async def query(self, sql, params=None, query_type=QueryTypes.ONE, size=10):
+        sql = sql.replace("?", "%s").replace("`", '"')
+        try:
+            with AwsWrapperConnection.connect(
+                psycopg.Connection.connect, self._dsn, **self._params
+            ) as conn:
+                with conn.cursor() as cursor:
+                    if (
+                        query_type == QueryTypes.CREATEBATCH
+                        or query_type == QueryTypes.UPDATEBATCH
+                    ):
+                        cursor.executemany(sql, params)
+                    else:
+                        cursor.execute(sql, params)
+
+                    result = None
+
+                    if query_type == QueryTypes.ALL and cursor.description is not None:
+                        columns = [desc[0] for desc in cursor.target_cursor.description]
+                        rows = cursor.fetchall()
+                        result = [dict(zip(columns, row)) for row in rows]
+                        return result
+                    elif (
+                        query_type == QueryTypes.ONE and cursor.description is not None
+                    ):
+                        columns = [desc[0] for desc in cursor.target_cursor.description]
+                        row = cursor.fetchone()
+                        result = dict(zip(columns, row)) if row else {}
+                        return result
+                    elif (
+                        query_type == QueryTypes.MANY and cursor.description is not None
+                    ):
+                        columns = [desc[0] for desc in cursor.target_cursor.description]
+                        rows = cursor.fetchmany(size)
+                        result = [dict(zip(columns, row)) for row in rows]
+                        return result
+                    elif query_type == QueryTypes.COUNT:
+                        result = cursor.fetchone()
+                        count = 0
+                        if result is None:
+                            return count
+                        for value in result:
+                            count = value
+                        return count
+                    elif query_type == QueryTypes.GROUP_COUNT:
+                        result = cursor.fetchall()
+                        return 0 if not result else len(result)
+                    elif query_type == QueryTypes.CREATE:
+                        result = cursor.rowcount > 0
+                        id = cursor.target_cursor.lastrowid
+                        return result, id
+                    elif query_type == QueryTypes.CREATEBATCH:
+                        count = cursor.rowcount
+                        id = cursor.target_cursor.lastrowid
+                        return count, id
+                    elif query_type == QueryTypes.UPDATE:
+                        return cursor.rowcount > 0
+                    elif query_type == QueryTypes.UPDATEBATCH:
+                        return cursor.rowcount
+                    elif query_type == QueryTypes.DELETE:
+                        return cursor.rowcount > 0
+        except Exception as e:
+            error(f"[AWS PG] query error: {e}, SQL: {sql}")
+            raise e

{cloudoll-3.0.6 → cloudoll-3.0.7}/cloudoll/orm/base.py

@@ -19,8 +19,8 @@ class QueryTypes(Enum):
 
 
 class MeteBase:
-    def __init__(self):
-        self.pool: Optional[MyPool | PGPool] = None
+    # def __init__(self):
+        # self.pool: Optional[MyPool | PGPool] = None
         # self.cursor: Optional[Cursor] = None
         # self.conn: Optional[Connection] = None
 
@@ -28,9 +28,7 @@ class MeteBase:
         await self.close()
 
     async def close(self):
-        if self.pool:
-            self.pool.close()
-            await self.pool.wait_closed()
+        ...
 
     # async def begin_transaction(self):
     #     conn = await self._set_conn()

{cloudoll-3.0.6 → cloudoll-3.0.7}/cloudoll/orm/model.py

@@ -150,14 +150,18 @@ class Model(metaclass=ModelMetaclass):
     @classmethod
     def use(cls, pool=None):
         cls.__pool__ = pool
-        cls.__ispg = pool.driver == "postgres" if pool else False
+        cls.__is_pg = (
+            pool.driver in ["postgres", "postgressql", "aws-postgres"]
+            if pool
+            else False
+        )
         return cls()
 
     def select(cls, *args):
         cols = []
         for col in args:
             if isinstance(col, Field):
-                cols.append(col.name if cls.__ispg else col.full_name)
+                cols.append(col.name if cls.__is_pg else col.full_name)
             elif isinstance(col, Function):
                 q, p = col.sql()
                 cols.append(q)
@@ -200,7 +204,7 @@ class Model(metaclass=ModelMetaclass):
             if isinstance(f, str):
                 by.append(f)
             else:
-                by.append(f"{f.lhs.name if self.__ispg else f.lhs.full_name} {f.op}")
+                by.append(f"{f.lhs.name if self.__is_pg else f.lhs.full_name} {f.op}")
         if self.__order_by__ is not None:
             by = self.__order_by__ + by
         self.__order_by__ = by
@@ -209,7 +213,7 @@ class Model(metaclass=ModelMetaclass):
     def group_by(self, *args):
         by = []
         for f in args:
-            by.append(f.name if self.__ispg else f.full_name)
+            by.append(f.name if self.__is_pg else f.full_name)
         if self.__group_by__ is not None:
             by = self.__group_by__ + by
         self.__group_by__ = by
@@ -365,7 +369,7 @@ class Model(metaclass=ModelMetaclass):
         return await self.__pool__.all(sql, args)
 
     def _exchange_sql(self, sql: str):
-        if self.__ispg:
+        if self.__is_pg:
             sql = sql.replace("CURDATE()", "CURRENT_DATE")
             sql = sql.replace("NOW()", "CURRENT_TIMESTAMP")
 

cloudoll-3.0.7/cloudoll/orm/mysql.py

@@ -0,0 +1,112 @@
+from typing import Any, Optional
+import aiomysql
+from cloudoll.logging import error
+from cloudoll.orm.base import MeteBase, QueryTypes
+from cloudoll.logging import info, error
+
+
+class AttrDict(dict):
+    def __getattr__(self, name):
+        try:
+            return self[name]
+        except KeyError:
+            return None
+
+
+class AttrDictCursor(aiomysql.DictCursor):
+    dict_type = AttrDict
+
+
+class Mysql(MeteBase):
+    def __init__(self):
+        self.pool: aiomysql.Pool
+        self.driver = "mysql"
+
+    def __call__(self, *args: Any, **kwds: Any) -> Any:
+        self.__init__(*args, **kwds)
+        return self
+
+    async def close(self):
+        if self.pool:
+            self.pool.close()
+            await self.pool.wait_closed()
+
+    async def query(
+        self, sql, params=None, query_type: QueryTypes = QueryTypes.ONE, size: int = 10
+    ):
+        sql = sql.replace("?", "%s")
+        if not self.pool:
+            raise ValueError("must be create_engine first.")
+        if self.pool._closing or self.pool._closed:
+            return None
+        async with self.pool.acquire() as conn:
+            if conn.echo:
+                info("sql: %s , %s", sql, params)
+
+            async with conn.cursor() as cursor:
+                cursor: aiomysql.Cursor = cursor
+                if (
+                    query_type == QueryTypes.CREATEBATCH
+                    or query_type == QueryTypes.UPDATEBATCH
+                ):
+                    await cursor.executemany(sql, params)
+                else:
+                    await cursor.execute(sql, params)
+
+                await conn.commit()
+
+                if query_type == QueryTypes.ALL:
+                    return await cursor.fetchall()
+                elif query_type == QueryTypes.ONE:
+                    return await cursor.fetchone()
+                elif query_type == QueryTypes.MANY:
+                    return await cursor.fetchmany(size)
+                elif query_type == QueryTypes.COUNT:
+                    rows = await cursor.fetchone()
+                    count = 0
+                    if rows is None:
+                        return count
+                    for row in rows:
+                        count = rows[row]
+                    return count
+                elif query_type == QueryTypes.GROUP_COUNT:
+                    result = await cursor.fetchall()
+                    return 0 if not result else len(result)
+                elif query_type == QueryTypes.CREATE:
+                    result = cursor.rowcount > 0
+                    id = cursor.lastrowid
+                    return result, id
+                elif query_type == QueryTypes.CREATEBATCH:
+                    count = cursor.rowcount
+                    id = cursor.lastrowid
+                    return count, id
+                elif query_type == QueryTypes.UPDATE:
+                    return cursor.rowcount > 0
+                elif query_type == QueryTypes.UPDATEBATCH:
+                    return cursor.rowcount
+                elif query_type == QueryTypes.DELETE:
+                    return cursor.rowcount > 0
+
+        self.pool.release(conn)
+
+    async def create_engine(self, loop=None, **kw):
+        try:
+            self.pool = await aiomysql.create_pool(
+                host=kw.get("host", "localhost"),
+                port=kw.get("port", 3306),
+                user=kw.get("username"),
+                password=str(kw.get("password", "")),
+                db=kw.get("db"),
+                echo=kw.get("echo", False),
+                charset=kw.get("charset", "utf8"),
+                autocommit=False,  # kw.get("autocommit", False),
+                maxsize=kw.get("maxsize", 10),
+                minsize=kw.get("minsize", 5),
+                cursorclass=AttrDictCursor,
+                loop=loop,
+            )
+            info(f"Database connection successfully for mysql/{kw.get('db')}.")
+        except Exception as e:
+            # print(traceback.format_exc())
+            error(f"Database connection failed,the instance : mysql/{kw.get('db')}")
+        return self

cloudoll-3.0.7/cloudoll/orm/postgres.py

@@ -0,0 +1,116 @@
+import aiopg
+from cloudoll.logging import error
+from cloudoll.orm.base import MeteBase, QueryTypes
+from cloudoll.logging import info, error
+from typing import Any, Optional
+
+class Postgres(MeteBase):
+    def __init__(self):
+        self.pool: Optional[aiopg.Pool] = None
+        self.driver = "postgres"
+
+    def __call__(self, *args: Any, **kwds: Any) -> Any:
+        self.__init__(*args, **kwds)
+        return self
+
+    async def close(self):
+        if self.pool:
+            self.pool.close()
+            await self.pool.wait_closed()
+
+    async def query(
+        self,
+        sql,
+        params=None,
+        query_type: QueryTypes = QueryTypes.COUNT,
+        size: int = 10,
+    ):
+        sql = sql.replace("?", "%s").replace("`", '"')
+        if not self.pool:
+            raise ValueError("must be create_engine first.")
+        if self.pool._closing or self.pool._closed:
+            return None
+
+        async with self.pool.acquire() as conn:
+            if conn.echo:
+                info("sql: %s ,%s", sql, params)
+
+            async with conn.cursor() as cursor:
+                # current_cursor = getattr(cursor, 'lastrowid', None)
+                if (
+                    query_type == QueryTypes.CREATEBATCH
+                    or query_type == QueryTypes.UPDATEBATCH
+                ):
+                    await cursor.executemany(sql, params)
+                else:
+                    await cursor.execute(sql, params)
+
+                # await conn.commit()
+                result = None
+
+                if query_type == QueryTypes.ALL and cursor.description is not None:
+                    columns = [desc[0] for desc in cursor.description]
+                    rows = await cursor.fetchall()
+                    result = [dict(zip(columns, row)) for row in rows]
+                    return result
+                elif query_type == QueryTypes.ONE and cursor.description is not None:
+                    columns = [desc[0] for desc in cursor.description]
+                    row = await cursor.fetchone()
+                    result = dict(zip(columns, row)) if row else {}
+                    return result
+                elif query_type == QueryTypes.MANY and cursor.description is not None:
+                    columns = [desc[0] for desc in cursor.description]
+                    rows = await cursor.fetchmany(size)
+                    result = [dict(zip(columns, row)) for row in rows]
+                    return result
+                elif query_type == QueryTypes.COUNT:
+                    result = await cursor.fetchone()
+                    count = 0
+                    if result is None:
+                        return count
+                    for value in result:
+                        count = value
+                    return count
+                elif query_type == QueryTypes.GROUP_COUNT:
+                    result = await cursor.fetchall()
+                    return 0 if not result else len(result)
+                elif query_type == QueryTypes.CREATE:
+                    result = cursor.rowcount > 0
+                    id = cursor.lastrowid
+                    return result, id
+                elif query_type == QueryTypes.CREATEBATCH:
+                    count = cursor.rowcount
+                    id = cursor.lastrowid
+                    return count, id
+                elif query_type == QueryTypes.UPDATE:
+                    return cursor.rowcount > 0
+                elif query_type == QueryTypes.UPDATEBATCH:
+                    return cursor.rowcount
+                elif query_type == QueryTypes.DELETE:
+                    return cursor.rowcount > 0
+
+    async def create_engine(self, **kw):
+        try:
+            host = (kw.get("host", "localhost"),)
+            port = (kw.get("port", 5432),)
+            user = (kw.get("username"),)
+            password = (str(kw.get("password", "")),)
+            db = (kw.get("db"),)
+            dsn = f"dbname={db[0]} user={user[0]} password={password[0]} host={host[0]} port={port[0]}"  # aiopg
+            # dsn = f"postgres://{user[0]}:{password[0]}@{host[0]}:{port[0]}/{db[0]}" # asyncpg
+            self.pool = await aiopg.create_pool(
+                dsn=dsn,
+                timeout=float(kw.get("timeout", 60)),
+                echo=kw.get("echo", False),  # aiopg
+                # max_size=kw.get("maxsize", 10), # asyncpg
+                # min_size=kw.get("minsize", 5), # asyncpg
+                maxsize=kw.get("maxsize", 10),  # aiopg
+                minsize=kw.get("minsize", 5),  # aiopg
+            )
+            info(f"Database connection successfully for postgres/{kw.get('db')}")
+        except Exception as e:
+            error(e)
+            # print(traceback.format_exc())
+            error(f"Database connection failed,the instance : postgres/{kw.get('db')}")
+
+        return self

{cloudoll-3.0.6 → cloudoll-3.0.7}/cloudoll/orm/test_sql.py

@@ -42,7 +42,7 @@ class B(Model):
 
 class C(Model):
 
-    __table__ = "i_network_speed_result"
+    __table__ = "c"
 
     id = models.BigIntegerField(primary_key=True, max_length=64, not_null=True)
     pack_id = models.BigIntegerField(max_length=64, default="0", not_null=True)

{cloudoll-3.0.6 → cloudoll-3.0.7}/cloudoll/web/core.py

@@ -16,7 +16,7 @@ import time
 from urllib import parse
 from aiohttp import web, hdrs
 from aiohttp.web import Response
-from aiohttp.web_ws import WebSocketResponse,WSMsgType
+from aiohttp.web_ws import WebSocketResponse, WSMsgType
 from aiohttp.web_response import StreamResponse
 from aiohttp.web_request import Request
 from aiohttp.typedefs import LooseHeaders
@@ -28,8 +28,6 @@ from aiohttp_session import (
     cookie_storage,
 )
 from cloudoll.web.settings import get_config
-import aiomcache
-from redis import asyncio as aioredis
 from cloudoll.logging import info
 from cloudoll.orm.model import Model
 from cloudoll.web import jwt
@@ -62,6 +60,7 @@ async def _set_session_route(request: Request):
 
 
 def _auto_reg_module(module_dir: str):
+    info(f"Auto-registration {module_dir}")
     base_path = Path(module_dir).resolve()
     if str(base_path.parent) not in sys.path:
         sys.path.insert(0, str(base_path.parent))
@@ -129,8 +128,8 @@ async def _render_result(request: Request, func):
 
 def _sa_ignore_hash(method, path):
     md5 = hashlib.md5()
-    hashstr = f"{method}{path}"
-    md5.update(hashstr.encode("utf-8"))
+    hash_str = f"{method}{path}"
+    md5.update(hash_str.encode("utf-8"))
     return md5.hexdigest()
 
 
@@ -140,8 +139,8 @@ def _parse_int(num):
 
 def _sa_ignore_middleware():
     async def set_ignore(ctx, handler):
-        hashstr = _sa_ignore_hash(ctx.method, ctx.path)
-        ctx.is_sa_ignore = hashstr in ctx.app.ignore_paths
+        hash_str = _sa_ignore_hash(ctx.method, ctx.path)
+        ctx.is_sa_ignore = hash_str in ctx.app.ignore_paths
         start_time = time.time()
         response = await handler(ctx)
         end_time = time.time()
@@ -162,13 +161,18 @@ class Application(object):
         self.config = {}
         self.clean_up = False
 
-    def _load_life_cycle(self, entry_model=None):
+    def _load_life_cycle(self, entry_model=None, func_name=None):
         try:
             if not entry_model:
                 return
 
             entry = importlib.import_module(entry_model, ".")
 
+            if func_name:
+                func = getattr(entry, func_name)
+                func(self)
+                return
+
             life_cycle = ["on_startup", "on_shutdown", "on_cleanup", "on_task"]
             for cycle in life_cycle:
                 if hasattr(entry, cycle):
@@ -199,15 +203,17 @@ class Application(object):
             config = get_config(env or "local")
         self.config = config
 
+        # try to load func and override configuration
+        self._load_life_cycle(entry_model, func_name="on_create")
+
         sa_ignore_mid = _sa_ignore_middleware()
         sa_ignore_mid.__middleware_version__ = 1
         self._middleware.append(sa_ignore_mid)
 
         # middlewares
         _auto_reg_module("middlewares")
-        info("Auto-registration middleware")
 
-        conf_server = config.get("server", {})
+        conf_server = self.config.get("server", {})
         client_max_size = 1024**2 * 2
         if conf_server is not None:
             client_max_size = conf_server.get("client_max_size", client_max_size)
@@ -218,12 +224,16 @@ class Application(object):
             client_max_size=_parse_int(client_max_size),
         )
 
+        # load life
+        # entry = conf_server.get("entry", entry_model)
+        # self._load_life_cycle(entry)
+
         # database
         self.app.ignore_paths = set()
         self.app.db = Object()
         self.app.on_startup.append(self._init_database)
         self.app.on_cleanup.append(self._close_database)
-        self.app.config = config
+        self.app.config = self.config
         self.app.env = env
         self.app.jwt_encode = self.jwt_encode
         self.app.jwt_decode = self.jwt_decode
@@ -231,14 +241,9 @@ class Application(object):
         self.app.on_startup.append(self._init_session)
         # router:
         _auto_reg_module("controllers")
-        info("Auto-registration controller")
 
         self.app.add_routes(self._route_table)
 
-        # load life
-        entry = conf_server.get("entry", entry_model)
-        self._load_life_cycle(entry)
-
         # static
         if conf_server is not None:
             conf_st = conf_server.get("static", {})
@@ -283,7 +288,7 @@ class Application(object):
         if conf_db:
             # for db_key in conf_db:
             #     apps.db[db_key] = await create_engine(**conf_db[db_key])
-            
+
             tasks = [create_engine(**conf_db[db_key]) for db_key in conf_db]
             engines = await asyncio.gather(*tasks)
             for db_key, engine in zip(conf_db.keys(), engines):
@@ -309,6 +314,8 @@ class Application(object):
                 cfg, qs = parse_coon(redis_url)
                 redis_url = f"{cfg['type']}://{cfg['username']}:{cfg['password']}@{cfg['host']}:{cfg['port']}/{cfg['db']}"
 
+            from redis import asyncio as aioredis
+
             redis = await aioredis.from_url(redis_url, **qs)
             apps.redis = redis
             storage = redis_storage.RedisStorage(
@@ -324,6 +331,8 @@ class Application(object):
             host = mcache_conf.get("host")
             port = mcache_conf.get("port", 11211)
 
+            import aiomcache
+
             mc = aiomcache.Client(host, port)
             apps.memcached = mc
             storage = memcached_storage.MemcachedStorage(
@@ -382,12 +391,14 @@ class Application(object):
     def add_router(self, path, method, name, sa_ignore):
         def inner(handler):
             handler = RequestHandler(handler)
-            self.router.add_route(method, path, handler, name=name)
+            if self.router is not None:
+                self.router.add_route(method, path, handler, name=name)
             return handler
 
         if sa_ignore:
-            hashstr = _sa_ignore_hash(method, path)
-            self.app.ignore_paths.add(hashstr)
+            if self.app is not None:
+                hash_str = _sa_ignore_hash(method, path)
+                self.app.ignore_paths.add(hash_str)
         return inner
 
     def add_middleware(self, func):
@@ -414,27 +425,39 @@ class Application(object):
 
     @property
     def router(self):
-        return self.app.router
+        if self.app is not None:
+            return self.app.router
+        return None
 
     @property
     def middlewares(self):
-        return self.app.middlewares
+        if self.app is not None:
+            return self.app.middlewares
+        return None
 
     @property
     def on_startup(self):
-        return self.app.on_startup
+        if self.app is not None:
+            return self.app.on_startup
+        return None
 
     @property
     def on_shutdown(self):
-        return self.app.on_shutdown
+        if self.app is not None:
+            return self.app.on_shutdown
+        return None
 
     @property
     def on_cleanup(self):
-        return self.app.on_cleanup
+        if self.app is not None:
+            return self.app.on_cleanup
+        return None
 
     @property
     def on_task(self):
-        return self.app.cleanup_ctx
+        if self.app is not None:
+            return self.app.cleanup_ctx
+        return None
 
 
 class View(web.View):
@@ -525,8 +548,8 @@ def delete(path: str, name=None, sa_ignore=False):
 def routes(path: str, sa_ignore=False):
     if sa_ignore:
         for method in hdrs.METH_ALL:
-            hashstr = _sa_ignore_hash(method, path)
-            app.app.ignore_paths.add(hashstr)
+            hash_str = _sa_ignore_hash(method, path)
+            app.app.ignore_paths.add(hash_str)
     return app.route_table.view(path)
 
 

{cloudoll-3.0.6 → cloudoll-3.0.7}/cloudoll.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudoll
-Version: 3.0.6
+Version: 3.0.7
 Summary: Assist in quickly creating distributable microservices based on aiohttp, with a focus on simplicity and ease of use.
 Home-page: https://github.com/smallerqiu/cloudoll-py
 Author: Qiu

{cloudoll-3.0.6 → cloudoll-3.0.7}/cloudoll.egg-info/SOURCES.txt

@@ -22,6 +22,8 @@ cloudoll/logging/__init__.py
 cloudoll/mail/__init__.py
 cloudoll/mail/smtp.py
 cloudoll/orm/__init__.py
+cloudoll/orm/awsmysql.py
+cloudoll/orm/awspostgres.py
 cloudoll/orm/base.py
 cloudoll/orm/err.py
 cloudoll/orm/field.py

{cloudoll-3.0.6 → cloudoll-3.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cloudoll"
-version = "3.0.6" 
+version = "3.0.7" 
 description = "Assist in quickly creating distributable microservices based on aiohttp, with a focus on simplicity and ease of use."
 authors = [
   {name = "Qiu", email = "smallerqiu@gmail.com"},

cloudoll-3.0.6/cloudoll/__init__.py

@@ -1 +0,0 @@
-__version__ = "3.0.6"

cloudoll-3.0.6/cloudoll/orm/__init__.py

@@ -1,247 +0,0 @@
-from redis import asyncio as aioredis
-from cloudoll.orm.parse import parse_coon
-from cloudoll.orm.base import MeteBase, QueryTypes
-from typing import Any
-from cloudoll.logging import info, error
-import traceback
-import aiomysql
-from aiomysql import Pool as MYPool
-
-import aiopg as pg
-from aiopg.pool import Pool as PGPool
-from typing import Optional
-
-__all__ = ["create_engine"]
-
-
-async def create_engine(**kw):
-    url = kw.get("url")
-    driver = None
-    configs = {}
-    query = {}
-
-    if url is not None:
-        configs, query = parse_coon(url)
-        driver = configs["type"]
-    else:
-        driver = kw.get("type")
-        configs = kw
-
-    # info("DB Config:", configs, query)
-
-    if driver == "mysql":
-        return await Mysql().create_engine(**configs, **query)
-    elif driver == "postgres" or driver == "postgressql":
-        return await Postgres().create_engine(**configs, **query)
-    elif driver == "redis" or driver == "rediss":
-        """
-        redis://[[username]:[password]]@localhost:6379/0
-        rediss://[[username]:[password]]@localhost:6379/0
-        """
-        if url is None:
-            url = f"{driver}://{configs['username']}:{configs['password']}@{configs['host']}:{configs['port']}/{configs['db']}"
-        return await aioredis.from_url(url, **query)
-    else:
-        raise ValueError("Not support this database type.")
-
-
-class Postgres(MeteBase):
-    def __init__(self, pool=None):
-        # self.pool: Optional[PGPool] = pool
-        self.driver = "postgres"
-
-    def __call__(self, *args: Any, **kwds: Any) -> Any:
-        self.__init__(*args, **kwds)
-
-    async def query(self, sql, params=None, query_type: QueryTypes = QueryTypes.COUNT, size: int = 10):
-        sql = sql.replace("?", "%s").replace("`", '"')
-        if not self.pool:
-            raise ValueError("must be create_engine first.")
-        if self.pool._closing or self.pool._closed:
-            return None
-
-        async with self.pool.acquire() as conn:
-            if conn.echo:
-                info("sql: %s ,%s", sql, params)
-
-            async with conn.cursor() as cursor:
-                # current_cursor = getattr(cursor, 'lastrowid', None)
-                if (
-                    query_type == QueryTypes.CREATEBATCH
-                    or query_type == QueryTypes.UPDATEBATCH
-                ):
-                    # aiopg don't support executemany
-                    raise RuntimeError("postgres don't support executemany")
-                else:
-                    await cursor.execute(sql, params)
-
-                result = None
-    
-                if query_type == QueryTypes.ALL and cursor.description is not None:
-                    columns = [desc[0] for desc in cursor.description]
-                    rows = await cursor.fetchall()
-                    result = [dict(zip(columns, row)) for row in rows]
-                    return result
-                elif query_type == QueryTypes.ONE and cursor.description is not None:
-                    columns = [desc[0] for desc in cursor.description]
-                    row = await cursor.fetchone()
-                    result = dict(zip(columns, row)) if row else {}
-                    return result
-                elif query_type == QueryTypes.MANY and cursor.description is not None:
-                    columns = [desc[0] for desc in cursor.description]
-                    rows = await cursor.fetchmany(size)
-                    result = [dict(zip(columns, row)) for row in rows]
-                    return result
-                elif query_type == QueryTypes.COUNT:
-                    result = await cursor.fetchone()
-                    count = 0
-                    if result is None:
-                        return count
-                    for value in result:
-                        count = value
-                    return count
-                elif query_type == QueryTypes.GROUP_COUNT:
-                    result = await cursor.fetchall()
-                    return 0 if not result else len(result)
-                elif query_type == QueryTypes.CREATE:
-                    result = cursor.rowcount > 0
-                    id = cursor.lastrowid
-                    return result, id
-                elif query_type == QueryTypes.CREATEBATCH:
-                    count = cursor.rowcount
-                    id = cursor.lastrowid
-                    return count, id
-                elif query_type == QueryTypes.UPDATE:
-                    return cursor.rowcount > 0
-                elif query_type == QueryTypes.UPDATEBATCH:
-                    return cursor.rowcount
-                elif query_type == QueryTypes.DELETE:
-                    return cursor.rowcount > 0
-
-    async def create_engine(self, **kw):
-        try:
-            host = (kw.get("host", "localhost"),)
-            port = (kw.get("port", 5432),)
-            user = (kw.get("username"),)
-            password = (str(kw.get("password", "")),)
-            db = (kw.get("db"),)
-            dsn = f"dbname={db[0]} user={user[0]} password={password[0]} host={host[0]} port={port[0]}"  # aiopg
-            # dsn = f"postgres://{user[0]}:{password[0]}@{host[0]}:{port[0]}/{db[0]}" # asyncpg
-            self.pool = await pg.create_pool(
-                dsn=dsn,
-                timeout=float(kw.get("timeout", 60)),
-                echo=kw.get("echo", False),  # aiopg
-                # max_size=kw.get("maxsize", 10), # asyncpg
-                # min_size=kw.get("minsize", 5), # asyncpg
-                maxsize=kw.get("maxsize", 10),  # aiopg
-                minsize=kw.get("minsize", 5),  # aiopg
-            )
-            info(f"Database connection successfuly for postgres/{kw.get('db')}")
-        except Exception as e:
-            error(e)
-            # print(traceback.format_exc())
-            error(f"Database connection failed,the instance : postgres/{kw.get('db')}")
-
-        return self
-
-
-class AttrDict(dict):
-    """Dict that can get attribute by dot, and doesn't raise KeyError"""
-
-    def __getattr__(self, name):
-        try:
-            return self[name]
-        except KeyError:
-            return None
-
-
-class AttrDictCursor(aiomysql.DictCursor):
-    dict_type = AttrDict
-
-
-class Mysql(MeteBase):
-    def __init__(self, pool=None):
-        # self.pool: Optional[MYPool] = pool
-        self.driver = "mysql"
-
-    def __call__(self, *args: Any, **kwds: Any) -> Any:
-        self.__init__(*args, **kwds)
-        return self
-
-    async def query(self, sql, params=None, query_type: QueryTypes = QueryTypes.ONE, size: int = 10):
-        sql = sql.replace("?", "%s")
-        if not self.pool:
-            raise ValueError("must be create_engine first.")
-        if self.pool._closing or self.pool._closed:
-            return None
-        async with self.pool.acquire() as conn:
-            if conn.echo:
-                info("sql: %s , %s", sql, params)
-
-            async with conn.cursor() as cursor:
-                # current_cursor = getattr(cursor, 'lastrowid', None)
-                if (
-                    query_type == QueryTypes.CREATEBATCH
-                    or query_type == QueryTypes.UPDATEBATCH
-                ):
-                    await cursor.executemany(sql, params)
-                else:
-                    await cursor.execute(sql, params)
-
-                await conn.commit()
-
-                if query_type == QueryTypes.ALL:
-                    return await cursor.fetchall()
-                elif query_type == QueryTypes.ONE:
-                    return await cursor.fetchone()
-                elif query_type == QueryTypes.MANY:
-                    return await cursor.fetchmany(size)
-                elif query_type == QueryTypes.COUNT:
-                    rows = await cursor.fetchone()
-                    count = 0
-                    if rows is None:
-                        return count
-                    for row in rows:
-                        count = rows[row]
-                    return count
-                elif query_type == QueryTypes.GROUP_COUNT:
-                    result = await cursor.fetchall()
-                    return 0 if not result else len(result)
-                elif query_type == QueryTypes.CREATE:
-                    result = cursor.rowcount > 0
-                    id = cursor.lastrowid
-                    return result, id
-                elif query_type == QueryTypes.CREATEBATCH:
-                    count = cursor.rowcount
-                    id = cursor.lastrowid
-                    return count, id
-                elif query_type == QueryTypes.UPDATE:
-                    return cursor.rowcount > 0
-                elif query_type == QueryTypes.UPDATEBATCH:
-                    return cursor.rowcount
-                elif query_type == QueryTypes.DELETE:
-                    return cursor.rowcount > 0
-
-        self.pool.release(conn)
-
-    async def create_engine(self, loop=None, **kw):
-        try:
-            self.pool = await aiomysql.create_pool(
-                host=kw.get("host", "localhost"),
-                port=kw.get("port", 3306),
-                user=kw.get("username"),
-                password=str(kw.get("password", "")),
-                db=kw.get("db"),
-                echo=kw.get("echo", False),
-                charset=kw.get("charset", "utf8"),
-                autocommit=False,  # kw.get("autocommit", False),
-                maxsize=kw.get("maxsize", 10),
-                minsize=kw.get("minsize", 5),
-                cursorclass=AttrDictCursor,
-                loop=loop,
-            )
-            info(f"Database connection successfuly for mysql/{kw.get('db')}.")
-        except Exception as e:
-            # print(traceback.format_exc())
-            error(f"Database connection failed,the instance : mysql/{kw.get('db')}")
-        return self

cloudoll-3.0.6/cloudoll/orm/mysql.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-from cloudoll.orm import Mysql
-
-# for config folder conf.{env}.yaml
-
-database:
-  mysql:
-    type: mysql  # important ,default mysql
-    host: 127.0.0.1
-    port: 3306
-    user: root
-    password: 123456
-    db: test
-    charset:utf8mb4
-    pool_size:5
-
-# to get yaml config
-config = ....(database.mysql)
-
-pool = await Mysql().create_engine(**config)
-result = await pool.query("sql")
-"""
-__author__ = "Qiu / smallerqiu@gmail.com"
-
-from aiomysql import create_pool, DictCursor
-
-# from aiomysql.cursors import Cursor
-# from aiomysql.connection import Connection
-from cloudoll.logging import error
-
-# from inspect import isclass, isfunction
-from cloudoll.orm.base import MeteBase
-
-
-class AttrDict(dict):
-    """Dict that can get attribute by dot, and doesn't raise KeyError"""
-
-    def __getattr__(self, name):
-        try:
-            return self[name]
-        except KeyError:
-            return None
-
-
-class AttrDictCursor(DictCursor):
-    dict_type = AttrDict
-
-
-class Mysql(MeteBase):
-    # def __init__(self):
-        # self.pool: Optional[Pool] = None
-        # self.cursor: Cursor = None
-        # self.conn: Connection = None
-        # self.__MODELS__ = []
-
-    # def __call__(cls, *args: Any, **kwds: Any) -> Any:
-    #     cls.__init__(*args, **kwds)
-    #     return cls()
-
-    # @classmethod
-    async def create_engine(self, loop=None, **kw):
-        try:
-            self.pool = await create_pool(
-                host=kw.get("host", "localhost"),
-                port=kw.get("port", 3306),
-                user=kw.get("user"),
-                password=str(kw.get("password", "")),
-                db=kw.get("db"),
-                echo=kw.get("echo", False),
-                charset=kw.get("charset", "utf8"),
-                autocommit=False,  # kw.get("autocommit", False),
-                maxsize=kw.get("maxsize", 10),
-                minsize=kw.get("pool_size", 5),
-                cursorclass=AttrDictCursor,
-                connect_timeout=kw.get("connect_timeout", 10),
-                loop=loop,
-            )
-        except Exception as e:
-            error(f"Database connection failed,the instance : {kw.get('db')}")
-        return self

cloudoll-3.0.6/cloudoll/orm/postgres.py

@@ -1,68 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-from cloudoll.orm.postgres import Postgres
-
-# for config folder conf.{env}.yaml
-
-database:
-  pg:
-    type: postgres  # important ,default mysql
-    host: 127.0.0.1
-    port: 3306
-    user: root
-    password: 123456
-    db: test
-    echo: True
-    timeout: 60
-    pool_size:5
-
-# to get yaml config
-config = ....(database.postgres)
-
-pool = await Postgres().create_engine(**config)
-result = await pool.query("sql")
-"""
-__author__ = "Qiu / smallerqiu@gmail.com"
-
-from aiopg import create_pool
-from cloudoll.logging import error
-from cloudoll.orm.base import MeteBase
-
-
-class Postgres(MeteBase):
-    # def __init__(self):
-    #     self.pool: Optional[Pool] = None
-        # self.cursor: Optional[Cursor] = None
-        # self.conn: Optional[Connection] = None
-        # self.__MODELS__ = []
-
-    # def __call__(cls, *args: Any, **kwds: Any) -> Any:
-    #     cls.__init__(*args, **kwds)
-    #     return cls()
-
-    @classmethod
-    async def create_engine(cls, **kw):
-        try:
-            host = (kw.get("host", "localhost"),)
-            port = (kw.get("port", 5432),)
-            user = (kw.get("user"),)
-            password = (str(kw.get("password", "")),)
-            db = (kw.get("db"),)
-            dsn = f"dbname={db[0]} user={user[0]} password={password[0]} host={host[0]} port={port[0]}"
-            cls.pool = await create_pool(
-                dsn=dsn,
-                timeout=kw.get("timeout", 10.0),
-                enable_json=kw.get("enable_json", False),
-                enable_hstore=kw.get("enable_hstore", False),
-                enable_uuid=kw.get("enable_uuid", False),
-                pool_recycle=kw.get("pool_recycle", 10),
-                echo=kw.get("echo", False),
-                minsize=kw.get("pool_size", 5),
-                maxsize=kw.get("pool_size", 10),
-            )
-        except Exception as e:
-            # error(e)
-            error(f"Database connection failed,the instance : {kw.get('db')}")
-
-        return cls