cloudoll 2.2.23__tar.gz → 2.2.24__tar.gz

{cloudoll-2.2.23 → cloudoll-2.2.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudoll
-Version: 2.2.23
+Version: 2.2.24
 Summary: 辅助快速创建可分布的微服务。
 Home-page: https://gitee.com/chuchur/cloudoll-py
 Author: chuchur

cloudoll-2.2.24/cloudoll/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.2.24"

{cloudoll-2.2.23 → cloudoll-2.2.24}/cloudoll/orm/__init__.py

@@ -1,13 +1,20 @@
 from redis import asyncio as aioredis
 from .parse import parse_coon
-import aiopg, aiomysql
 from .base import MeteBase, QueryTypes
 from typing import Any
 from ..logging import print_info, print_error
 import traceback
+import aiomysql
 from aiomysql import Pool as MYPool
+
+import aiopg as pg
 from aiopg.pool import Pool as PGPool
 
+# import asyncpg as pg
+# from asyncpg.pool import Pool as PGPool
+
+__ALL__ = "create_engine"
+
 
 async def create_engine(**kw):
     url = kw.get("url")
@@ -48,6 +55,65 @@ class Postgres(MeteBase):
     def __call__(self, *args: Any, **kwds: Any) -> Any:
         self.__init__(*args, **kwds)
 
+    # async def query(self, sql, params=None, query_type: QueryTypes = 2, size: int = 10):
+    #     sql = sql.replace("?", "%s").replace("`", "")
+    #     print_info("sql", sql, params)
+    #     if not self.pool:
+    #         raise ValueError("must be create_engine first.")
+    #     if self.pool._closing or self.pool._closed:
+    #         return None
+
+    #     # async with self.pool.acquire() as conn:
+    #     async with self.pool.acquire() as conn:
+    #         # current_cursor = getattr(cursor, 'lastrowid', None)
+    #         if (
+    #             query_type == QueryTypes.CREATEBATCH
+    #             or query_type == QueryTypes.UPDATEBATCH
+    #         ):
+    #             result = await conn.executemany(sql, params)  # aiopg don't support executemany
+    #         else:
+    #             result = await conn.execute(sql, params)
+
+    #         # result = None
+
+    #         return result
+
+    #         if query_type == QueryTypes.ALL:
+    #             columns = [desc[0] for desc in cursor.description]
+    #             rows = await cursor.fetchall()
+    #             result = [dict(zip(columns, row)) for row in rows]
+    #             return result
+    #         elif query_type == QueryTypes.ONE:
+    #             columns = [desc[0] for desc in cursor.description]
+    #             row = await cursor.fetchone()
+    #             # print("rows_reuslt", rows)
+    #             result = dict(zip(columns, row)) if row else {}
+    #             return result
+    #         elif query_type == QueryTypes.MANY:
+    #             columns = [desc[0] for desc in cursor.description]
+    #             rows = await cursor.fetchmany(size)
+    #             result = [dict(zip(columns, row)) for row in rows]
+    #             return result
+    #         elif query_type == QueryTypes.COUNT:
+    #             rs = await cursor.fetchone()
+    #             return rs[0] if row else 0
+    #         elif query_type == QueryTypes.CREATE:
+    #             result = cursor.rowcount > 0
+    #             id = cursor.lastrowid
+    #             return result, id
+    #         elif query_type == QueryTypes.CREATEBATCH:
+    #             count = cursor.rowcount
+    #             id = cursor.lastrowid
+    #             return count, id
+    #         elif query_type == QueryTypes.UPDATE:
+    #             return cursor.rowcount > 0
+    #         elif query_type == QueryTypes.UPDATEBATCH:
+    #             return cursor.rowcount
+    #         elif query_type == QueryTypes.DELETE:
+    #             return cursor.rowcount > 0
+
+    #         self.pool.release(cursor)
+
     async def query(self, sql, params=None, query_type: QueryTypes = 2, size: int = 10):
         sql = sql.replace("?", "%s").replace("`", "")
         print_info("sql", sql, params)
@@ -55,16 +121,19 @@ class Postgres(MeteBase):
             raise ValueError("must be create_engine first.")
         if self.pool._closing or self.pool._closed:
             return None
+
         async with self.pool.acquire() as conn:
             async with conn.cursor() as cursor:
                 # current_cursor = getattr(cursor, 'lastrowid', None)
-                await cursor.execute(sql, params)
+                if (
+                    query_type == QueryTypes.CREATEBATCH
+                    or query_type == QueryTypes.UPDATEBATCH
+                ):
+                    # aiopg don't support executemany
+                    await cursor.executemany(sql, params)
+                else:
+                    await cursor.execute(sql, params)
 
-                # if query_type.value > 4:
-                # if self.driver == "mysql":
-                # print(columns)
-                # await conn.commit()
-                # elif self.driver == "postgres":
                 result = None
 
                 if query_type == QueryTypes.ALL:
@@ -90,12 +159,18 @@ class Postgres(MeteBase):
                     result = cursor.rowcount > 0
                     id = cursor.lastrowid
                     return result, id
+                elif query_type == QueryTypes.CREATEBATCH:
+                    count = cursor.rowcount
+                    id = cursor.lastrowid
+                    return count, id
                 elif query_type == QueryTypes.UPDATE:
                     return cursor.rowcount > 0
+                elif query_type == QueryTypes.UPDATEBATCH:
+                    return cursor.rowcount
                 elif query_type == QueryTypes.DELETE:
                     return cursor.rowcount > 0
 
-        self.pool.release(conn)
+        self.pool.release(cursor)
 
     async def create_engine(self, **kw):
         try:
@@ -104,17 +179,20 @@ class Postgres(MeteBase):
             user = (kw.get("username"),)
             password = (str(kw.get("password", "")),)
             db = (kw.get("db"),)
-            dsn = f"dbname={db[0]} user={user[0]} password={password[0]} host={host[0]} port={port[0]}"
-            self.pool = await aiopg.create_pool(
+            dsn = f"dbname={db[0]} user={user[0]} password={password[0]} host={host[0]} port={port[0]}"  # aiopg
+            # dsn = f"postgres://{user[0]}:{password[0]}@{host[0]}:{port[0]}/{db[0]}" #asyncpg
+            self.pool = await pg.create_pool(
                 dsn=dsn,
                 timeout=kw.get("timeout"),
-                echo=kw.get("echo", False),
-                maxsize=kw.get("maxsize", 10),
-                minsize=kw.get("pool_size", 5),
+                echo=kw.get("echo", False),  # aiopg
+                # max_size=kw.get("maxsize", 10), #asyncpg
+                # min_size=kw.get("minsize", 5), #asyncpg
+                maxsize=kw.get("maxsize", 10),  # aiopg
+                minsize=kw.get("minsize", 5),  # aiopg
             )
             print_info(f"Database connection successfuly for postgres/{kw.get('db')}")
         except Exception as e:
-            # print_error(e)
+            print_error(e)
             # print(traceback.format_exc())
             print_error(
                 f"Database connection failed,the instance : postgres/{kw.get('db')}"
@@ -148,7 +226,7 @@ class Mysql(MeteBase):
 
     async def query(self, sql, params=None, query_type: QueryTypes = 2, size: int = 10):
         sql = sql.replace("?", "%s")
-        print_info("sql", sql, params)
+        # print_info("sql", sql, params)
         if not self.pool:
             raise ValueError("must be create_engine first.")
         if self.pool._closing or self.pool._closed:
@@ -156,7 +234,13 @@ class Mysql(MeteBase):
         async with self.pool.acquire() as conn:
             async with conn.cursor() as cursor:
                 # current_cursor = getattr(cursor, 'lastrowid', None)
-                await cursor.execute(sql, params)
+                if (
+                    query_type == QueryTypes.CREATEBATCH
+                    or query_type == QueryTypes.UPDATEBATCH
+                ):
+                    await cursor.executemany(sql, params)
+                else:
+                    await cursor.execute(sql, params)
 
                 # if query_type.value > 4:
                 # if self.driver == "mysql":
@@ -180,8 +264,14 @@ class Mysql(MeteBase):
                     result = cursor.rowcount > 0
                     id = cursor.lastrowid
                     return result, id
+                elif query_type == QueryTypes.CREATEBATCH:
+                    count = cursor.rowcount
+                    id = cursor.lastrowid
+                    return count, id
                 elif query_type == QueryTypes.UPDATE:
                     return cursor.rowcount > 0
+                elif query_type == QueryTypes.UPDATEBATCH:
+                    return cursor.rowcount
                 elif query_type == QueryTypes.DELETE:
                     return cursor.rowcount > 0
 
@@ -199,7 +289,7 @@ class Mysql(MeteBase):
                 charset=kw.get("charset", "utf8"),
                 autocommit=False,  # kw.get("autocommit", False),
                 maxsize=kw.get("maxsize", 10),
-                minsize=kw.get("pool_size", 5),
+                minsize=kw.get("minsize", 5),
                 cursorclass=AttrDictCursor,
                 loop=loop,
             )

{cloudoll-2.2.23 → cloudoll-2.2.24}/cloudoll/orm/base.py

@@ -9,6 +9,8 @@ class QueryTypes(Enum):
     CREATE = 5
     UPDATE = 6
     DELETE = 7
+    CREATEBATCH = 8
+    UPDATEBATCH = 9
 
 
 class MeteBase:
@@ -60,9 +62,15 @@ class MeteBase:
 
     async def update(self, sql, params):
         return await self.query(sql, params, QueryTypes.UPDATE)
+    
+    async def update_batch(self, sql, params):
+        return await self.query(sql, params, QueryTypes.UPDATEBATCH)
 
     async def delete(self, sql, params):
         return await self.query(sql, params, QueryTypes.DELETE)
 
     async def create(self, sql, params):
         return await self.query(sql, params, QueryTypes.CREATE)
+    
+    async def create_batch(self, sql, params):
+        return await self.query(sql, params, QueryTypes.CREATEBATCH)

{cloudoll-2.2.23 → cloudoll-2.2.24}/cloudoll/orm/model.py

@@ -7,6 +7,8 @@ import re
 import datetime
 from ..utils.common import Object
 
+__ALL__ = ("models", "Model")
+
 
 class ModelMetaclass(type):
     # def __init__(self, **kw):
@@ -197,9 +199,7 @@ class Model(metaclass=ModelMetaclass):
             if isinstance(f, str):
                 by.append(f)
             else:
-                by.append(
-                    f"{f.lhs.name if self.__ispg else f.lhs.full_name} {f.op}"
-                )
+                by.append(f"{f.lhs.name if self.__ispg else f.lhs.full_name} {f.op}")
         if self.__order_by__ is not None:
             by = self.__order_by__ + by
         self.__order_by__ = by
@@ -214,9 +214,8 @@ class Model(metaclass=ModelMetaclass):
         self.__group_by__ = by
         return self
 
-    def _get_key_args(self, action, args):
+    def _format_data(self, action, args):
         data = dict()
-        md = None
         if args is None or not args:
             for k in self.__fields__:
                 f = getattr(self, k)
@@ -226,15 +225,19 @@ class Model(metaclass=ModelMetaclass):
             data = args
         else:
             for item in args:
-                md = item
                 if isinstance(item, Model):
                     for k in item.__dict__:
-                        if (action == 'u' and item[k].value) or action == 'i':
+                        if (action == "u" and item[k].value) or action == "i":
                             data[k] = item[k]
 
                 else:  # for object
                     for k, v in item.items():
                         data[k] = v
+
+        return data
+
+    def _get_update_key_args(self, action, args):
+        data = self._format_data(action, args)
         keys = []
         params = []
         for k, v in data.items():
@@ -251,7 +254,48 @@ class Model(metaclass=ModelMetaclass):
             elif v is not None:  # fix sql format %s
                 keys.append("`%s`=?" % k)
                 params.append(v)
-        return ",".join(keys), params, md
+        return ",".join(keys), params
+
+    def _get_insert_key_args(self, action, args):
+        data = self._format_data(action, args)
+        keys = []
+        params = []
+        for k, v in data.items():
+            if isinstance(v, Field):
+                value = v.value
+                if value is None:
+                    if v.created_generated == True or v.update_generated == True:
+                        value = datetime.datetime.now()
+                    else:
+                        value = v.default
+                if value is not None:
+                    keys.append(k)
+                    params.append(value)
+            elif v is not None:  # fix sql format %s
+                keys.append(k)
+                params.append(v)
+        return keys, params
+
+    def _get_batch_keys_values(self, items: list):
+        keys = []
+        values = []
+        item = items[0]
+        if isinstance(item, Model):
+            keys = item.__dict__.keys()
+        elif isinstance(item, dict):
+            keys = item.keys()
+
+        for item in items:
+            value = []
+            if isinstance(item, Model):
+                for k in item.__dict__:
+                    value.append(item[k].value)
+            elif isinstance(item, dict):  # for object
+                for k, v in item.items():
+                    value.append(v)
+            value = tuple(key for key in value)
+            values.append(value)
+        return keys, values
 
     def _merge_params(self, p):
         if p is not None:
@@ -343,7 +387,7 @@ class Model(metaclass=ModelMetaclass):
         # where = self.__where__
         where = self._literal("WHERE", self.__where__)
 
-        keys, params, md = self._get_key_args('u', args or kw)
+        keys, params = self._get_update_key_args("u", args or kw)
 
         sql = f"update `{table}` set {keys} {where}"
 
@@ -389,11 +433,21 @@ class Model(metaclass=ModelMetaclass):
 
     async def insert(self, *args, **kw):
         table = self.__table__
-        keys, params, md = self._get_key_args('i', args or kw)
-        sql = f"insert into `{table}` set {keys}"
+        keys, params = self._get_insert_key_args("i", args or kw)
+        sql = f"insert into `{table}` ({','.join(keys)}) values ({','.join(['?' for k in keys])})"
         self._reset()
+        params = tuple(key for key in params)
         return await self.__pool__.create(sql, params)
 
+    async def insert_batch(self, items: list):
+        table = self.__table__
+        if len(items) == 0:
+            return 0
+        keys, params = self._get_batch_keys_values(items)
+        sql = f"insert into `{table}` ({','.join(keys)}) values ({','.join(['?' for k in keys])})"
+        self._reset()
+        return await self.__pool__.create_batch(sql, params)
+
     async def count(self):
         __where__ = copy.copy(self.__where__)
         __join__ = copy.copy(self.__join__)

{cloudoll-2.2.23 → cloudoll-2.2.24}/cloudoll/robot/feishu.py

@@ -5,7 +5,8 @@ __author__ = "chuchur/chuchur.com"
 
 import hashlib, hmac, time, base64
 from curl_cffi import requests as http
-from ..logging import print_error,print_info
+from ..logging import print_error, print_info
+
 
 class Client(object):
     def __init__(self, **kw):

{cloudoll-2.2.23 → cloudoll-2.2.24}/cloudoll/utils/m2d.py

@@ -253,7 +253,8 @@ def get_col_sql(field):
     if field.primary_key:
         sql += " PRIMARY KEY"
     if field.auto_increment:
-        sql += " AUTO_INCREMENT"
+        # todo: postgres SERIAL
+        sql += " AUTO_INCREMENT" 
     if field.NOT_NULL:
         sql += " NOT NULL"
     if field.default:

cloudoll-2.2.24/cloudoll/utils/snowflake.py

@@ -0,0 +1,86 @@
+
+import time
+import threading
+
+
+"""
+雪花算法（全局唯一id）
+
+# 最早是Twitter公司在其内部用于分布式环境下生成唯一ID
+
+# Twitter's Snowflake algorithm implementation which is used to generate distributed IDs.
+
+# https://github.com/twitter-archive/snowflake/blob/snowflake-2010/src/main/scala/com/twitter/service/snowflake/IdWorker.scala
+"""
+
+
+class IdWorker:
+    def __init__(self, worker_id, datacenter_id, sequence=0):
+         # 位移值
+        self.worker_id_bits = 5
+        self.datacenter_id_bits = 5
+        self.sequence_bits = 12
+        
+        self.max_worker_id = -1 ^ (-1 << self.worker_id_bits)
+        self.max_datacenter_id = -1 ^ (-1 << self.datacenter_id_bits)
+
+        # 参数校验
+        if worker_id > self.max_worker_id or worker_id < 0:
+            raise ValueError(f'worker_id must be between 0 and {self.max_worker_id}')
+        if datacenter_id > self.max_datacenter_id or datacenter_id < 0:
+            raise ValueError(f'datacenter_id must be between 0 and {self.max_datacenter_id}')
+
+        self.worker_id = worker_id
+        self.datacenter_id = datacenter_id
+        self.sequence = sequence
+        self.lock = threading.Lock()
+
+        # Twitter Epoch (2010-11-04 01:42:54 UTC)
+        self.twepoch = 1288834974657
+
+       
+
+       
+        self.worker_id_shift = self.sequence_bits
+        self.datacenter_id_shift = self.sequence_bits + self.worker_id_bits
+        self.timestamp_left_shift = self.sequence_bits + self.worker_id_bits + self.datacenter_id_bits
+        self.sequence_mask = -1 ^ (-1 << self.sequence_bits)
+
+        self.last_timestamp = -1
+
+    def _time_gen(self):
+        return int(time.time() * 1000)
+
+    def _til_next_millis(self, last_timestamp):
+        timestamp = self._time_gen()
+        while timestamp <= last_timestamp:
+            timestamp = self._time_gen()
+        return timestamp
+
+    def next_id(self):
+        with self.lock:
+            timestamp = self._time_gen()
+
+            if timestamp < self.last_timestamp:
+                raise Exception(f'Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds')
+
+            if self.last_timestamp == timestamp:
+                self.sequence = (self.sequence + 1) & self.sequence_mask
+                if self.sequence == 0:
+                    timestamp = self._til_next_millis(self.last_timestamp)
+            else:
+                self.sequence = 0
+
+            self.last_timestamp = timestamp
+
+            id = ((timestamp - self.twepoch) << self.timestamp_left_shift) | \
+                (self.datacenter_id << self.datacenter_id_shift) | \
+                (self.worker_id << self.worker_id_shift) | \
+                self.sequence
+
+            return id
+
+
+# 示例使用
+worker = IdWorker(worker_id=1, datacenter_id=1)
+print(worker.next_id())

{cloudoll-2.2.23 → cloudoll-2.2.24}/cloudoll.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudoll
-Version: 2.2.23
+Version: 2.2.24
 Summary: 辅助快速创建可分布的微服务。
 Home-page: https://gitee.com/chuchur/cloudoll-py
 Author: chuchur

{cloudoll-2.2.23 → cloudoll-2.2.24}/cloudoll.egg-info/SOURCES.txt

@@ -31,6 +31,7 @@ cloudoll/utils/__init__.py
 cloudoll/utils/common.py
 cloudoll/utils/m2d.py
 cloudoll/utils/patch_loop.py
+cloudoll/utils/snowflake.py
 cloudoll/utils/watch.py
 cloudoll/web/__init__.py
 cloudoll/web/exception.py

cloudoll-2.2.23/cloudoll/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.2.23"