cloudoll 2.0.24__tar.gz → 2.2.0__tar.gz

{cloudoll-2.0.24 → cloudoll-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudoll
-Version: 2.0.24
+Version: 2.2.0
 Summary: 辅助快速创建可分布的微服务。
 Home-page: https://gitee.com/chuchur/cloudoll-py
 Author: chuchur
@@ -350,20 +350,36 @@ server:
     show_index: true
     append_version: true
     follow_symlinks: true
-mysql:
-  host: 127.0.0.1
-  port: 3306
-  user: root
-  password: abcd
-  db: blog
-  charset: utf8mb4
+database:
+	mysql:
+		host: 127.0.0.1
+		port: 3306
+		user: root
+		password: abcd
+		db: blog
+		charset: utf8mb4
 ```
 
 默认开发会使用默认的`local`作为配置。 启动时 通过 `env` 加载对应的配置。 如 `python3 app.py --env=prod` 会加载 `conf.prod.yaml`
 
+# cli 
 
-# 开发模式
+## 生成模型
 
+从数据库导出 `users` 表模型
 ```sh
-adev runserver app.py
+cloudoll gen -t users 
+```
+更多参数 :
+- -p (--path) 导出的模型路径
+- -c (--create) 值 model(默认) 生成模型, 值table 建表
+- -t (--table) 要生的模型或要建的表名,以`,`分开, `ALL` 所有表
+- -env (--environment) 读取配置名 ,值 local(默认) / test / prod 
+- -db (--database) 数据库实例名,取决于配置文件,如果有多个数据库
+- -h (--help) 帮助
+
+## 开发调试
+
+```sh
+cloudoll start
 ```

{cloudoll-2.0.24 → cloudoll-2.2.0}/README.md

@@ -331,20 +331,36 @@ server:
     show_index: true
     append_version: true
     follow_symlinks: true
-mysql:
-  host: 127.0.0.1
-  port: 3306
-  user: root
-  password: abcd
-  db: blog
-  charset: utf8mb4
+database:
+	mysql:
+		host: 127.0.0.1
+		port: 3306
+		user: root
+		password: abcd
+		db: blog
+		charset: utf8mb4
 ```
 
 默认开发会使用默认的`local`作为配置。 启动时 通过 `env` 加载对应的配置。 如 `python3 app.py --env=prod` 会加载 `conf.prod.yaml`
 
+# cli 
 
-# 开发模式
+## 生成模型
 
+从数据库导出 `users` 表模型
 ```sh
-adev runserver app.py
+cloudoll gen -t users 
+```
+更多参数 :
+- -p (--path) 导出的模型路径
+- -c (--create) 值 model(默认) 生成模型, 值table 建表
+- -t (--table) 要生的模型或要建的表名,以`,`分开, `ALL` 所有表
+- -env (--environment) 读取配置名 ,值 local(默认) / test / prod 
+- -db (--database) 数据库实例名,取决于配置文件,如果有多个数据库
+- -h (--help) 帮助
+
+## 开发调试
+
+```sh
+cloudoll start
 ```

cloudoll-2.2.0/cloudoll/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.2.0"

{cloudoll-2.0.24 → cloudoll-2.2.0}/cloudoll/cli.py

@@ -1,7 +1,6 @@
 import click
 from .web.settings import get_config
-from .orm.mysql import Mysql
-from .orm.postgres import Postgres
+from .orm import create_engine
 from .utils.m2d import create_models, create_tables
 import asyncio
 import os
@@ -50,7 +49,7 @@ def gen(path, create, table, environment, database) -> None:
             if db_config is None:
                 raise ValueError(
                     f"Can't find the database config key ->{database} in conf.{environment}.yaml")
-            sa = await Mysql().create_engine(**db_config)
+            sa = await create_engine(**db_config)
             if sa.pool is None:
                 return
             model_path = os.path.join(os.path.abspath("."), path)

cloudoll-2.2.0/cloudoll/orm/__init__.py

@@ -0,0 +1,193 @@
+from redis import asyncio as aioredis
+from .parse import parse_coon
+import aiopg, aiomysql
+from .base import MeteBase, QueryTypes
+from typing import Any
+from ..logging import error
+import traceback
+
+
+async def create_engine(**kw):
+    url = kw.get("url")
+    driver = None
+    configs = {}
+    query = {}
+
+    if url is not None:
+        configs, query = parse_coon(url)
+        driver = configs["type"]
+    else:
+        driver = kw.get("type")
+        configs = kw
+
+    if driver == "mysql":
+        return await Mysql().create_engine(**configs, **query)
+    elif driver == "postgres":
+        return await Postgres().create_engine(**configs, **query)
+    elif driver == "redis" or driver == "rediss":
+        """
+        redis://[[username]:[password]]@localhost:6379/0
+        rediss://[[username]:[password]]@localhost:6379/0
+        """
+        if url is None:
+            url = f"{driver}://{configs['username']}:{configs['password']}@{configs['host']}:{configs['port']}/{configs['db']}"
+        return await aioredis.from_url(url, **query)
+    else:
+        error("Not suport this database type.")
+
+
+class Postgres(MeteBase):
+    def __init__(self, pool=None):
+        self.pool = pool
+        self.driver = "postgres"
+
+    def __call__(self, *args: Any, **kwds: Any) -> Any:
+        self.__init__(*args, **kwds)
+
+    async def query(self, sql, params=None, query_type: QueryTypes = 2, size: int = 10):
+        sql = sql.replace("?", "%s").replace("`", "")
+        print("sql", sql, params)
+        if not self.pool:
+            raise ValueError("must be create_engine first.")
+        async with self.pool.acquire() as conn:
+            async with conn.cursor() as cursor:
+                # current_cursor = getattr(cursor, 'lastrowid', None)
+                await cursor.execute(sql, params)
+
+                # if query_type.value > 4:
+                # if self.driver == "mysql":
+                # await conn.commit()
+                # elif self.driver == "postgres":
+                columns = [desc[0] for desc in cursor.description]
+                result = None
+
+                if query_type == QueryTypes.ALL:
+                    rows = await cursor.fetchall()
+                    result = [dict(zip(columns, row)) for row in rows]
+                    return result
+                elif query_type == QueryTypes.ONE:
+                    rows = await cursor.fetchone()
+                    result = dict(zip(columns, rows))
+                    return result
+                elif query_type == QueryTypes.MANY:
+                    rows = await cursor.fetchmany(size)
+                    result = [dict(zip(columns, row)) for row in rows]
+                    return result
+                elif query_type == QueryTypes.COUNT:
+                    rs = await cursor.fetchone()
+                    return rs[0]
+                elif query_type == QueryTypes.CREATE:
+                    result = cursor.rowcount > 0
+                    id = cursor.lastrowid
+                    return result, id
+                elif query_type == QueryTypes.UPDATE:
+                    return cursor.rowcount > 0
+                elif query_type == QueryTypes.DELETE:
+                    return cursor.rowcount > 0
+
+    async def create_engine(self, **kw):
+        try:
+            host = (kw.get("host", "localhost"),)
+            port = (kw.get("port", 5432),)
+            user = (kw.get("username"),)
+            password = (str(kw.get("password", "")),)
+            db = (kw.get("db"),)
+            dsn = f"dbname={db[0]} user={user[0]} password={password[0]} host={host[0]} port={port[0]}"
+            self.pool = await aiopg.create_pool(
+                dsn=dsn,
+                timeout=kw.get("timeout"),
+                echo=kw.get("echo", False),
+                maxsize=kw.get("maxsize", 10),
+                minsize=kw.get("pool_size", 5),
+            )
+            print(f"Database connection successfuly for postgres/{kw.get('db')}")
+        except Exception as e:
+            error(e)
+            # print(traceback.format_exc())
+            error(f"Database connection failed,the instance : postgres/{kw.get('db')}")
+
+        return self
+
+
+class AttrDict(dict):
+    """Dict that can get attribute by dot, and doesn't raise KeyError"""
+
+    def __getattr__(self, name):
+        try:
+            return self[name]
+        except KeyError:
+            return None
+
+
+class AttrDictCursor(aiomysql.DictCursor):
+    dict_type = AttrDict
+
+
+class Mysql(MeteBase):
+    def __init__(self, pool=None):
+        self.pool = pool
+        self.driver = "mysql"
+
+    def __call__(self, *args: Any, **kwds: Any) -> Any:
+        self.__init__(*args, **kwds)
+        return self
+
+    async def query(self, sql, params=None, query_type: QueryTypes = 2, size: int = 10):
+        sql = sql.replace("?", "%s")
+        print("sql", sql, params)
+        if not self.pool:
+            raise ValueError("must be create_engine first.")
+        async with self.pool.acquire() as conn:
+            async with conn.cursor() as cursor:
+                # current_cursor = getattr(cursor, 'lastrowid', None)
+                await cursor.execute(sql, params)
+
+                # if query_type.value > 4:
+                # if self.driver == "mysql":
+                await conn.commit()
+                # elif self.driver == "postgres":
+                # columns = [desc[0] for desc in cursor.description]
+
+                if query_type == QueryTypes.ALL:
+                    return await cursor.fetchall()
+                elif query_type == QueryTypes.ONE:
+                    return await cursor.fetchone()
+                elif query_type == QueryTypes.MANY:
+                    return await cursor.fetchmany(size)
+                elif query_type == QueryTypes.COUNT:
+                    rs = await cursor.fetchone()
+                    value = 0
+                    for r in rs:
+                        value = rs[r]
+                    return value
+                elif query_type == QueryTypes.CREATE:
+                    result = cursor.rowcount > 0
+                    id = cursor.lastrowid
+                    return result, id
+                elif query_type == QueryTypes.UPDATE:
+                    return cursor.rowcount > 0
+                elif query_type == QueryTypes.DELETE:
+                    return cursor.rowcount > 0
+
+    async def create_engine(self, loop=None, **kw):
+        try:
+            self.pool = await aiomysql.create_pool(
+                host=kw.get("host", "localhost"),
+                port=kw.get("port", 3306),
+                user=kw.get("username"),
+                password=str(kw.get("password", "")),
+                db=kw.get("db"),
+                echo=kw.get("echo", False),
+                charset=kw.get("charset", "utf8"),
+                autocommit=False,  # kw.get("autocommit", False),
+                maxsize=kw.get("maxsize", 10),
+                minsize=kw.get("pool_size", 5),
+                cursorclass=AttrDictCursor,
+                loop=loop,
+            )
+            print(f"Database connection successfuly for mysql/{kw.get('db')}.")
+        except Exception as e:
+            print(e)
+            # print(traceback.format_exc())
+            error(f"Database connection failed,the instance : mysql/{kw.get('db')}")
+        return self

cloudoll-2.2.0/cloudoll/orm/base.py

@@ -0,0 +1,68 @@
+from enum import Enum
+
+
+class QueryTypes(Enum):
+    ALL = 1
+    ONE = 2
+    MANY = 3
+    COUNT = 4
+    CREATE = 5
+    UPDATE = 6
+    DELETE = 7
+
+
+class MeteBase:
+
+    async def __aexit__(self):
+        await self.close()
+
+    async def close(self):
+        if self.pool:
+            self.pool.close()
+            await self.pool.wait_closed()
+
+    # async def begin_transaction(self):
+    #     conn = await self._set_conn()
+    #     await conn.begin()
+    #     return conn
+
+    # async def begin_transaction_scope(self, fun):
+    #     conn = await self._set_conn()
+    #     await conn.begin()
+    #     cursor = None
+    #     if isfunction(fun):
+    #         try:
+    #             cursor = await conn.cursor()
+    #             await fun(cursor)
+    #             await conn.commit()
+    #         except Exception as e:
+    #             error(e)
+    #             await conn.rollback()
+    #         finally:
+    #             if cursor:
+    #                 await cursor.close()
+    #         self.pool.release(conn)
+    async def query(
+        self, sql, params=None, query_type: QueryTypes = 2, size: int = 10
+    ): ...
+   
+    async def all(self, sql, params):
+        return await self.query(sql, params, QueryTypes.ALL)
+
+    async def one(self, sql, params):
+        return await self.query(sql, params, QueryTypes.ONE)
+
+    async def many(self, sql, params, size: int):
+        return await self.query(sql, params, QueryTypes.MANY, size)
+
+    async def count(self, sql, params):
+        return await self.query(sql, params, QueryTypes.COUNT)
+
+    async def update(self, sql, params):
+        return await self.query(sql, params, QueryTypes.UPDATE)
+
+    async def delete(self, sql, params):
+        return await self.query(sql, params, QueryTypes.DELETE)
+
+    async def create(self, sql, params):
+        return await self.query(sql, params, QueryTypes.CREATE)

{cloudoll-2.0.24 → cloudoll-2.2.0}/cloudoll/orm/field.py

@@ -7,45 +7,50 @@ class objdict(dict):
 
 
 OP = objdict(
-    AND='AND',
-    OR='OR',
-    ADD='+',
-    SUB='-',
-    MUL='*',
-    DIV='/',
-    BIN_AND='&',
-    BIN_OR='|',
-    XOR='#',
-    MOD='%',
-    EQ='=',
-    LT='<',
-    LTE='<=',
-    GT='>',
-    GTE='>=',
-    NE='!=',
-    IN='IN',
-    NOT_IN='NOT IN',
-    IS='IS',
-    DESC='DESC',
-    ASC='ASC',
-    AVG='AVG',
-    AS='AS',
-    SUM='SUM',
-    COUNT='COUNT',
-    IS_NOT='IS NOT',
-    IS_NOT_NULL='IS NOT NULL',
-    IS_NULL='IS NULL',
-    LIKE='LIKE',
-    NOT_LIKE='NOT LIKE',
-    ILIKE='ILIKE',
-    BETWEEN='BETWEEN',
-    CONTAINS='CONTAINS',
-    NOT_BETWEEN='NOT BETWEEN',
-    REGEXP='REGEXP',
-    IREGEXP='IREGEXP',
-    DISTINCT='DISTINCT',
-    CONCAT='||',
-    BITWISE_NEGATION='~')
+    AND="AND",
+    OR="OR",
+    ADD="+",
+    SUB="-",
+    MUL="*",
+    DIV="/",
+    BIN_AND="&",
+    BIN_OR="|",
+    XOR="#",
+    MOD="%",
+    EQ="=",
+    LT="<",
+    LTE="<=",
+    GT=">",
+    GTE=">=",
+    NE="!=",
+    IN="IN",
+    NOT_IN="NOT IN",
+    IS="IS",
+    DESC="DESC",
+    ASC="ASC",
+    AVG="AVG",
+    AS="AS",
+    SUM="SUM",
+    DATE_FORMAT="DATE_FORMAT",
+    JSON_CONTAINS_OBJECT="JSON_CONTAINS_OBJECT",
+    JSON_CONTAINS_ARRAY="JSON_CONTAINS_ARRAY",
+    COUNT="COUNT",
+    COUNT_WHEN="COUNT_WHEN",
+    IS_NOT="IS NOT",
+    IS_NOT_NULL="IS NOT NULL",
+    IS_NULL="IS NULL",
+    LIKE="LIKE",
+    NOT_LIKE="NOT LIKE",
+    ILIKE="ILIKE",
+    BETWEEN="BETWEEN",
+    CONTAINS="CONTAINS",
+    NOT_BETWEEN="NOT BETWEEN",
+    REGEXP="REGEXP",
+    IREGEXP="IREGEXP",
+    DISTINCT="DISTINCT",
+    CONCAT="||",
+    BITWISE_NEGATION="~",
+)
 
 
 class FieldBase:
@@ -54,6 +59,7 @@ class FieldBase:
             if inverse:
                 return Expression(rhs, op, self)
             return Expression(self, op, rhs)
+
         return inner
 
     def __eq__(self, rhs):
@@ -73,7 +79,7 @@ class FieldBase:
     __add__ = _op(OP.ADD)  # +
     __radd__ = _op(OP.ADD, True)  # +=
     __sub__ = _op(OP.SUB)  # -
-    __sub__ = _op(OP.SUB, True)  # -=
+    __rsub__ = _op(OP.SUB, True)  # -=
     __mul__ = _op(OP.MUL)  # *
     __rmul__ = _op(OP.MUL, True)  # *=
     __div__ = _op(OP.DIV)  # /
@@ -97,12 +103,24 @@ class FieldBase:
     def count(self):
         return Function(self, OP.COUNT)
 
+    def count_when(self, value):
+        return Function(self, OP.COUNT_WHEN, value)
+
     def sum(self):
         return Function(self, OP.SUM)
 
     def avg(self):
         return Function(self, OP.AVG)
 
+    def date_format(self, args):
+        return Function(self, OP.DATE_FORMAT, args)
+
+    def json_contains_object(self, key, value):
+        return Function(self, OP.JSON_CONTAINS_OBJECT, (key, value))
+
+    def json_contains_array(self, args):
+        return Function(self, OP.JSON_CONTAINS_ARRAY, args)
+
     def contains(self, args):
         return Function(self, OP.CONTAINS, args)
         # return AO(f"contains({self.full_name},?)", args)
@@ -137,16 +155,16 @@ class ExpList(FieldBase):
         q = []
         if isinstance(lpt, Exception):
             _q, _p = lpt.sql()
-            q = q+_q
-            p = p+_p
+            q = q + _q
+            p = p + _p
         else:
             q.append("?")
             p.append(lpt)
         q.append(f" {self.op} ")
         if isinstance(rpt, Exception):
             _q, _p = lpt.sql()
-            q = q+_q
-            p = p+_p
+            q = q + _q
+            p = p + _p
         else:
             q.append("?")
             p.append(lpt)
@@ -162,8 +180,27 @@ class Function(FieldBase):
         self.rpt = rpt
 
     def sql(self):
+        op = self.op
+        col_name = self.col.full_name
+        is_field = isinstance(self.rpt, Field)
         # todo:
-        return f"{self.op}({self.col.full_name} {','+str(self.rpt) if self.rpt else ''})"
+        if op == OP.DATE_FORMAT:
+            return f"{op}({col_name},?)", [self.rpt]
+        elif op == OP.JSON_CONTAINS_OBJECT:
+            _k, _v = self.rpt
+            key = _k.full_name if isinstance(_k, Field) else f"'{_k}'"
+            return f"json_contains({col_name},json_object({key},?))", [_v]
+        elif op == OP.JSON_CONTAINS_ARRAY:
+            if is_field:
+                return (
+                    f"json_contains({col_name},json_array({self.rpt.full_name}))",
+                    None,
+                )
+            return f"json_contains({col_name},json_array(?))", [self.rpt]
+        elif op == OP.COUNT:
+            return f"COUNT({col_name})", None
+        elif op == OP.COUNT_WHEN:
+            return f"COUNT(CASE WHEN {col_name} = ? THEN 1 END)", [self.rpt]
 
 
 class Expression(FieldBase):
@@ -171,32 +208,75 @@ class Expression(FieldBase):
         self.lhs = lhs
         self.op = op
         self.rhs = rhs
+        # self.__value = ""
+        # if op in "+-*/|&~" and lhs._value:
+        #     cal = f"{lhs._value}{op}"
+        #     if isinstance(rhs, FieldBase):
+        #         self.__value = str(eval(f"{cal}{rhs._value}"))
+        #     else:
+        #         self.__value = str(eval(f"{cal}{rhs}"))
 
     def sql(self):
         l = self.lhs
         r = self.rhs
+        is_fun = isinstance(l, Function)
         p = []
-        q = ["("]
+        q = []
+        if not is_fun:
+            q.append("(")
         if isinstance(l, Field):
             q.append(l.full_name)
         elif isinstance(l, Expression):
             _q, _p = l.sql()
             q.append(_q)
-            p = p+_p
+            p = p + _p
+        elif is_fun:
+            _q, _p = l.sql()
+            q.append(_q)
+            if _p:
+                p = p + _p
+            # return l.sql()
+
         q.append(f" {self.op} ")
 
         if isinstance(r, Field):
-            q.append(r.full_name)
+            if r._value is not None:
+                # _value = r._value if r._value is not None else r.full_name
+                q.append("?")
+                p.append(r._value)
+            else:
+                q.append(r.full_name)
+            # q.append(str(r._value) if r._value is not None else r.full_name)
         elif isinstance(r, FieldBase):
             _q, _p = r.sql()
             q.append(_q)
-            p = p+_p
-        else:
-            q.append('?')
+            p = p + _p
+        elif self.op == OP.AS:
+            # is_fun or (isinstance(r, str) and (self.op == OP.AND or self.op == OP.OR)):
+            q.append(r)
+        elif r is not None:
+            q.append("?")
             p.append(r)
-        q.append(')')
+        if not is_fun:
+            q.append(")")
+        # q.append(")")
+        q = [str(num) for num in q]
+        # print("".join(q))
         return "".join(q), p
-        # return f"({l.full_name if isinstance(l,Field) else l.sql()} {self.op} {r.full_name if isinstance(r,Field) else (r.sql() if isinstance(r,FieldBase) else str(r))})"
+
+    # def __str__(self):
+    #     return self.__value
+    #     cal = f"{self.lhs._value}{self.op}"
+    #     if isinstance(self.rhs, FieldBase):
+    #         return str(eval(f"{cal}{self.rhs._value}"))
+    #     return str(eval(f"{cal}{self.rhs}"))
+
+    def __repr__(self):
+        # return self.__value
+        cal = f"{self.lhs.value}{self.op}"
+        if isinstance(self.rhs, FieldBase):
+            return str(eval(f"{cal}{self.rhs.value}"))
+        return str(eval(f"{cal}{self.rhs}"))
 
 
 class Field(FieldBase):
@@ -237,11 +317,15 @@ class Field(FieldBase):
         # "<%s, %s:%s>" % (self.__class__.__name__, self.column_type, self.name)
         # return self.name
 
+    def __repr__(self):
+        return str(self._value)
+
     # def __getattr__(self, item):
     #     return self[item]
-
-    def set_value(self, value):
-        self._value = value
-
-    def get_value(self):
+    @property
+    def value(self):
         return self._value
+
+    @value.setter
+    def value(self, value):
+        self._value = value