cloudoll 3.0.4__tar.gz → 3.0.6__tar.gz

{cloudoll-3.0.4 → cloudoll-3.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudoll
-Version: 3.0.4
+Version: 3.0.6
 Summary: Assist in quickly creating distributable microservices based on aiohttp, with a focus on simplicity and ease of use.
 Home-page: https://github.com/smallerqiu/cloudoll-py
 Author: Qiu

cloudoll-3.0.6/cloudoll/__init__.py

@@ -0,0 +1 @@
+__version__ = "3.0.6"

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll/orm/__init__.py

@@ -9,6 +9,7 @@ from aiomysql import Pool as MYPool
 
 import aiopg as pg
 from aiopg.pool import Pool as PGPool
+from typing import Optional
 
 __all__ = ["create_engine"]
 
@@ -46,7 +47,7 @@ async def create_engine(**kw):
 
 class Postgres(MeteBase):
     def __init__(self, pool=None):
-        self.pool: PGPool = pool
+        # self.pool: Optional[PGPool] = pool
         self.driver = "postgres"
 
     def __call__(self, *args: Any, **kwds: Any) -> Any:
@@ -61,7 +62,7 @@ class Postgres(MeteBase):
 
         async with self.pool.acquire() as conn:
             if conn.echo:
-                info("sql", sql, params)
+                info("sql: %s ,%s", sql, params)
 
             async with conn.cursor() as cursor:
                 # current_cursor = getattr(cursor, 'lastrowid', None)
@@ -75,18 +76,18 @@ class Postgres(MeteBase):
                     await cursor.execute(sql, params)
 
                 result = None
-
-                if query_type == QueryTypes.ALL:
+    
+                if query_type == QueryTypes.ALL and cursor.description is not None:
                     columns = [desc[0] for desc in cursor.description]
                     rows = await cursor.fetchall()
                     result = [dict(zip(columns, row)) for row in rows]
                     return result
-                elif query_type == QueryTypes.ONE:
+                elif query_type == QueryTypes.ONE and cursor.description is not None:
                     columns = [desc[0] for desc in cursor.description]
                     row = await cursor.fetchone()
                     result = dict(zip(columns, row)) if row else {}
                     return result
-                elif query_type == QueryTypes.MANY:
+                elif query_type == QueryTypes.MANY and cursor.description is not None:
                     columns = [desc[0] for desc in cursor.description]
                     rows = await cursor.fetchmany(size)
                     result = [dict(zip(columns, row)) for row in rows]
@@ -160,14 +161,14 @@ class AttrDictCursor(aiomysql.DictCursor):
 
 class Mysql(MeteBase):
     def __init__(self, pool=None):
-        self.pool: MYPool = pool
+        # self.pool: Optional[MYPool] = pool
         self.driver = "mysql"
 
     def __call__(self, *args: Any, **kwds: Any) -> Any:
         self.__init__(*args, **kwds)
         return self
 
-    async def query(self, sql, params=None, query_type: QueryTypes = 2, size: int = 10):
+    async def query(self, sql, params=None, query_type: QueryTypes = QueryTypes.ONE, size: int = 10):
         sql = sql.replace("?", "%s")
         if not self.pool:
             raise ValueError("must be create_engine first.")
@@ -175,7 +176,7 @@ class Mysql(MeteBase):
             return None
         async with self.pool.acquire() as conn:
             if conn.echo:
-                info("sql", sql, params)
+                info("sql: %s , %s", sql, params)
 
             async with conn.cursor() as cursor:
                 # current_cursor = getattr(cursor, 'lastrowid', None)

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll/orm/base.py

@@ -1,5 +1,8 @@
+from abc import abstractmethod
 from enum import Enum
-from typing import Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
+from aiomysql.pool import Pool as MyPool
+from aiopg import Pool as PGPool
 
 
 class QueryTypes(Enum):
@@ -16,6 +19,10 @@ class QueryTypes(Enum):
 
 
 class MeteBase:
+    def __init__(self):
+        self.pool: Optional[MyPool | PGPool] = None
+        # self.cursor: Optional[Cursor] = None
+        # self.conn: Optional[Connection] = None
 
     async def __aexit__(self):
         await self.close()
@@ -46,9 +53,10 @@ class MeteBase:
     #             if cursor:
     #                 await cursor.close()
     #         self.pool.release(conn)
+    @abstractmethod
     async def query(
-        self, sql, params=None, query_type: QueryTypes = 2, size: int = 10
-    ): ...
+        self, sql, params=None, query_type: QueryTypes = QueryTypes.ONE, size: int = 10
+    ) -> Union[None, int, Tuple[Any, Any], Dict, List[Dict]]: ...
 
     async def all(self, sql, params):
         return await self.query(sql, params, QueryTypes.ALL)
@@ -61,6 +69,7 @@ class MeteBase:
 
     async def count(self, sql, params):
         return await self.query(sql, params, QueryTypes.COUNT)
+
     async def group_count(self, sql, params):
         return await self.query(sql, params, QueryTypes.GROUP_COUNT)
 
@@ -73,7 +82,7 @@ class MeteBase:
     async def delete(self, sql, params):
         return await self.query(sql, params, QueryTypes.DELETE)
 
-    async def create(self, sql, params) -> Tuple[bool, int]:
+    async def create(self, sql, params):
         return await self.query(sql, params, QueryTypes.CREATE)
 
     async def create_batch(self, sql, params):

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll/orm/field.py

@@ -1,3 +1,6 @@
+from typing import Optional
+
+
 class objdict(dict):
     def __getattr__(self, attr):
         try:
@@ -141,7 +144,7 @@ class FieldBase(object):
 
     def max_when(self, *arg):
         """
-        input: max_when(A.field > 10 ,A.field) \n 
+        input: max_when(A.field > 10 ,A.field) \n
         output: max(case when A.field > 10 then A.field end)
         """
         return Function(self, OP.MAX_WHEN, arg)
@@ -151,7 +154,7 @@ class FieldBase(object):
 
     def min_when(self, *arg):
         """
-        input: min_when(A.field > 10 ,A.field) \n 
+        input: min_when(A.field > 10 ,A.field) \n
         output: min(case when A.field > 10 then A.field end)
         """
         return Function(self, OP.MIN_WHEN, arg)
@@ -288,14 +291,14 @@ class ExpList(FieldBase):
 def deconstruct(args):
     f = args[0] if len(args) > 0 else None
     v1 = args[1] if len(args) > 1 else 1
-    v2 = args[2] if len(args) > 2 and args[2] is not None else 'NULL'
+    v2 = args[2] if len(args) > 2 and args[2] is not None else "NULL"
     c = None
     v = None
     if isinstance(f, FieldBase):
         c = f"{f.lhs.full_name} {f.op} ?"
         v = f.rhs
     if isinstance(v1, FieldBase):
-        v1 = v1.name
+        v1 = v1.full_name
     return c, v, v1, v2
 
 
@@ -355,32 +358,46 @@ class Function(FieldBase):
             return f"json_contains({col_name},json_object({key},?))", [_v]
         elif op == OP.JSON_CONTAINS_ARRAY:
             if is_field:
-                return f"json_contains({col_name},json_array({self.rpt.full_name}))", None
+                return (
+                    f"json_contains({col_name},json_array({self.rpt.full_name}))",
+                    None,
+                )
             return f"json_contains({col_name},json_array(?))", [self.rpt]
         elif op == OP.COUNT:
             return f"COUNT({col_name})", None
         elif op == OP.COUNT_WHEN:
             c, v, v1, v2 = deconstruct(self.rpt)
+            if v2 == "NULL":
+                return f"COUNT(CASE WHEN {c} THEN {v1} END)", [v]
             return f"COUNT(CASE WHEN {c} THEN {v1} ELSE {v2} END)", [v]
         elif op == OP.SUM:
             return f"SUM({col_name})", None
         elif op == OP.SUM_WHEN:
             c, v, v1, v2 = deconstruct(self.rpt)
+            if v2 == "NULL":
+                return f"SUM(CASE WHEN {c} THEN {v1} END)", [v]
             return f"SUM(CASE WHEN {c} THEN {v1} ELSE {v2} END)", [v]
         elif op == OP.AVG:
             return f"AVG({col_name})", None
         elif op == OP.AVG_WHEN:
             c, v, v1, v2 = deconstruct(self.rpt)
+            print("avg", v2)
+            if v2 == "NULL":
+                return f"AVG(CASE WHEN {c} THEN {v1} END)", [v]
             return f"AVG(CASE WHEN {c} THEN {v1} ELSE {v2} END)", [v]
         elif op == OP.MAX:
             return f"MAX({col_name})", None
         elif op == OP.MAX_WHEN:
             c, v, v1, v2 = deconstruct(self.rpt)
+            if v2 == "NULL":
+                return f"MAX(CASE WHEN {c} THEN {v1} END)", [v]
             return f"MAX(CASE WHEN {c} THEN {v1} ELSE {v2} END)", [v]
         elif op == OP.MIN:
             return f"MIN({col_name})", None
         elif op == OP.MIN_WHEN:
             c, v, v1, v2 = deconstruct(self.rpt)
+            if v2 == "NULL":
+                return f"MIN(CASE WHEN {c} THEN {v1} END)", [v]
             return f"MIN(CASE WHEN {c} THEN {v1} ELSE {v2} END)", [v]
         elif op == OP.GROUP_CONCAT:
             arg = self.rpt
@@ -488,7 +505,7 @@ class Field(FieldBase):
         **kwargs,
     ):
         super().__init__()
-        self.full_name = None
+        self.full_name: Optional[str] = None
         self._value = None
         self.name = name
         self.column_type = column_type

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll/orm/model.py

@@ -150,7 +150,7 @@ class Model(metaclass=ModelMetaclass):
     @classmethod
     def use(cls, pool=None):
         cls.__pool__ = pool
-        cls.__ispg = cls.__pool__.driver == "postgres" if pool else False
+        cls.__ispg = pool.driver == "postgres" if pool else False
         return cls()
 
     def select(cls, *args):
@@ -339,7 +339,7 @@ class Model(metaclass=ModelMetaclass):
         return self
 
     def test(self):
-        return self._sql()
+        return self._sql(), self.__params__
 
     async def one(self):
         self.limit(1)

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll/orm/mysql.py

@@ -24,9 +24,7 @@ result = await pool.query("sql")
 """
 __author__ = "Qiu / smallerqiu@gmail.com"
 
-from typing import Any
 from aiomysql import create_pool, DictCursor
-from aiomysql.pool import Pool
 
 # from aiomysql.cursors import Cursor
 # from aiomysql.connection import Connection
@@ -51,8 +49,8 @@ class AttrDictCursor(DictCursor):
 
 
 class Mysql(MeteBase):
-    def __init__(self):
-        self.pool: Pool = None
+    # def __init__(self):
+        # self.pool: Optional[Pool] = None
         # self.cursor: Cursor = None
         # self.conn: Connection = None
         # self.__MODELS__ = []

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll/orm/postgres.py

@@ -25,17 +25,16 @@ result = await pool.query("sql")
 """
 __author__ = "Qiu / smallerqiu@gmail.com"
 
-from aiopg import create_pool, Pool, Cursor, Connection
+from aiopg import create_pool
 from cloudoll.logging import error
 from cloudoll.orm.base import MeteBase
-from typing import Any, Optional
 
 
 class Postgres(MeteBase):
-    def __init__(self):
-        self.pool: Optional[Pool] = None
-        self.cursor: Optional[Cursor] = None
-        self.conn: Optional[Connection] = None
+    # def __init__(self):
+    #     self.pool: Optional[Pool] = None
+        # self.cursor: Optional[Cursor] = None
+        # self.conn: Optional[Connection] = None
         # self.__MODELS__ = []
 
     # def __call__(cls, *args: Any, **kwds: Any) -> Any:

cloudoll-3.0.6/cloudoll/orm/test_sql.py

@@ -0,0 +1,89 @@
+from model import models, Model
+
+
+class A(Model):
+
+    __table__ = "a"
+
+    id = models.BigIntegerField(
+        primary_key=True, max_length=64, not_null=True, comment="ID"
+    )
+    source = models.IntegerField(max_length=16, not_null=True, comment="状态")
+    param = models.TextField(not_null=True)
+    gmt_create = models.TimestampField(
+        max_length=3, default="CURRENT_TIMESTAMP(3)", not_null=True, comment="创建时间"
+    )
+
+
+class B(Model):
+
+    __table__ = "b"
+
+    id = models.BigIntegerField(primary_key=True, max_length=64, not_null=True)
+    task_id = models.BigIntegerField(max_length=64, default="0", not_null=True)
+    employee_id = models.BigIntegerField(max_length=64, default="0", not_null=True)
+    agency_id = models.BigIntegerField(max_length=64, default="0", not_null=True)
+    proxy_str = models.VarCharField(max_length=512, default="", not_null=True)
+    proxy_scheme = models.VarCharField(max_length=16, default="", not_null=True)
+    proxy_ip = models.VarCharField(max_length=64, default="", not_null=True)
+    proxy_region = models.VarCharField(max_length=32, default="", not_null=True)
+    proxy_order_no = models.VarCharField(max_length=64, default="", not_null=True)
+    proxy_merchant = models.VarCharField(max_length=32, default="", not_null=True)
+    proxy_subnet = models.VarCharField(max_length=128, default="", not_null=True)
+    channel_type = models.IntegerField(max_length=16, default="0", not_null=True)
+    channel = models.VarCharField(max_length=512, default="", not_null=True)
+    type = models.IntegerField(max_length=16, default="0", not_null=True)
+    url = models.VarCharField(max_length=512, default="", not_null=True)
+    gmt_create = models.TimestampField(
+        max_length=3, default="CURRENT_TIMESTAMP(3)", not_null=True
+    )
+    timeout = models.IntegerField(max_length=32, default="0", not_null=True)
+
+
+class C(Model):
+
+    __table__ = "i_network_speed_result"
+
+    id = models.BigIntegerField(primary_key=True, max_length=64, not_null=True)
+    pack_id = models.BigIntegerField(max_length=64, default="0", not_null=True)
+    delay = models.IntegerField(max_length=32, default="-1", not_null=True)
+    finish_load = models.IntegerField(max_length=32, default="-1", not_null=True)
+    ip = models.VarCharField(max_length=64, default="", not_null=True)
+    region = models.VarCharField(max_length=32, default="", not_null=True)
+    gmt_create = models.TimestampField(
+        max_length=3, default="CURRENT_TIMESTAMP(3)", not_null=True
+    )
+    speed = models.NumericField(
+        max_length=12, scale_length=2, default="0", not_null=True
+    )
+    packet_loss_rate = models.NumericField(
+        max_length=12, scale_length=2, default="0", not_null=True
+    )
+    is_failed = models.BooleanField(default="true", not_null=True)
+    is_timeout = models.BooleanField(default="true", not_null=True)
+
+
+if __name__ == "__main__":
+    sql, arg = (
+        B.use()
+        .select(
+            B.proxy_order_no.As("host_no"),
+            B.proxy_order_no.count().As("total_count"),
+            C.region.As("region"),
+            C.is_failed.avg_when(C.is_failed == False, C.finish_load).As("avg_val"),
+            C.is_failed.min_when(C.is_failed == False, C.finish_load).As("min_val"),
+            C.is_failed.max_when(C.is_failed == False, C.finish_load).As("max_val"),
+            C.is_failed.sum_when(C.is_failed == False, 1, 0).As("success_count"),
+            C.is_failed.sum_when(C.is_failed == True, 1, 0).As("failure_count"),
+        )
+        .join(C, C.pack_id == B.id)
+        .where(
+            B.channel_type == 1,
+            B.type == int(1),
+            B.proxy_order_no.In(tuple(["aa"])),
+        )
+        .group_by(B.proxy_order_no, C.region)
+        .test()
+    )
+    # sql = B()._exchange_sql(sql)
+    print(sql, arg)

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll/utils/snowflake.py

@@ -1,34 +1,30 @@
-
 import time
 import threading
 
 
 """
-雪花算法（全局唯一id）
-
-# 最早是Twitter公司在其内部用于分布式环境下生成唯一ID
-
 # Twitter's Snowflake algorithm implementation which is used to generate distributed IDs.
-
 # https://github.com/twitter-archive/snowflake/blob/snowflake-2010/src/main/scala/com/twitter/service/snowflake/IdWorker.scala
 """
 
 
 class IdWorker:
     def __init__(self, worker_id, datacenter_id, sequence=0):
-         # 位移值
+        # 位移值
         self.worker_id_bits = 5
         self.datacenter_id_bits = 5
         self.sequence_bits = 12
-        
+
         self.max_worker_id = -1 ^ (-1 << self.worker_id_bits)
         self.max_datacenter_id = -1 ^ (-1 << self.datacenter_id_bits)
 
         # 参数校验
         if worker_id > self.max_worker_id or worker_id < 0:
-            raise ValueError(f'worker_id must be between 0 and {self.max_worker_id}')
+            raise ValueError(f"worker_id must be between 0 and {self.max_worker_id}")
         if datacenter_id > self.max_datacenter_id or datacenter_id < 0:
-            raise ValueError(f'datacenter_id must be between 0 and {self.max_datacenter_id}')
+            raise ValueError(
+                f"datacenter_id must be between 0 and {self.max_datacenter_id}"
+            )
 
         self.worker_id = worker_id
         self.datacenter_id = datacenter_id
@@ -38,12 +34,11 @@ class IdWorker:
         # Twitter Epoch (2010-11-04 01:42:54 UTC)
         self.twepoch = 1288834974657
 
-       
-
-       
         self.worker_id_shift = self.sequence_bits
         self.datacenter_id_shift = self.sequence_bits + self.worker_id_bits
-        self.timestamp_left_shift = self.sequence_bits + self.worker_id_bits + self.datacenter_id_bits
+        self.timestamp_left_shift = (
+            self.sequence_bits + self.worker_id_bits + self.datacenter_id_bits
+        )
         self.sequence_mask = -1 ^ (-1 << self.sequence_bits)
 
         self.last_timestamp = -1
@@ -62,7 +57,9 @@ class IdWorker:
             timestamp = self._time_gen()
 
             if timestamp < self.last_timestamp:
-                raise Exception(f'Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds')
+                raise Exception(
+                    f"Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds"
+                )
 
             if self.last_timestamp == timestamp:
                 self.sequence = (self.sequence + 1) & self.sequence_mask
@@ -73,10 +70,12 @@ class IdWorker:
 
             self.last_timestamp = timestamp
 
-            id = ((timestamp - self.twepoch) << self.timestamp_left_shift) | \
-                (self.datacenter_id << self.datacenter_id_shift) | \
-                (self.worker_id << self.worker_id_shift) | \
-                self.sequence
+            id = (
+                ((timestamp - self.twepoch) << self.timestamp_left_shift)
+                | (self.datacenter_id << self.datacenter_id_shift)
+                | (self.worker_id << self.worker_id_shift)
+                | self.sequence
+            )
 
             return id
 

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll/web/core.py

@@ -6,7 +6,6 @@ __author__ = "Qiu / smallerqiu@gmail.com"
 import argparse
 import asyncio
 import hashlib
-import os
 import base64
 import importlib.util
 import inspect
@@ -17,7 +16,7 @@ import time
 from urllib import parse
 from aiohttp import web, hdrs
 from aiohttp.web import Response
-from aiohttp.web_ws import WebSocketResponse, WSMsgType
+from aiohttp.web_ws import WebSocketResponse,WSMsgType
 from aiohttp.web_response import StreamResponse
 from aiohttp.web_request import Request
 from aiohttp.typedefs import LooseHeaders
@@ -157,7 +156,7 @@ class Application(object):
     def __init__(self):
         self._loop = None
         self.env = None
-        self.app: web.Application = None
+        self.app: Optional[web.Application] = None
         self._route_table = web.RouteTableDef()
         self._middleware = []
         self.config = {}
@@ -282,8 +281,13 @@ class Application(object):
     async def _init_database(self, apps):
         conf_db = self.config.get("database")
         if conf_db:
-            for db_key in conf_db:
-                apps.db[db_key] = await create_engine(**conf_db[db_key])
+            # for db_key in conf_db:
+            #     apps.db[db_key] = await create_engine(**conf_db[db_key])
+            
+            tasks = [create_engine(**conf_db[db_key]) for db_key in conf_db]
+            engines = await asyncio.gather(*tasks)
+            for db_key, engine in zip(conf_db.keys(), engines):
+                apps.db[db_key] = engine
 
     async def _init_session(self, apps):
         config = self.config or {}

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudoll
-Version: 3.0.4
+Version: 3.0.6
 Summary: Assist in quickly creating distributable microservices based on aiohttp, with a focus on simplicity and ease of use.
 Home-page: https://github.com/smallerqiu/cloudoll-py
 Author: Qiu

{cloudoll-3.0.4 → cloudoll-3.0.6}/cloudoll.egg-info/SOURCES.txt

@@ -29,6 +29,7 @@ cloudoll/orm/model.py
 cloudoll/orm/mysql.py
 cloudoll/orm/parse.py
 cloudoll/orm/postgres.py
+cloudoll/orm/test_sql.py
 cloudoll/template/__init__.py
 cloudoll/utils/__init__.py
 cloudoll/utils/common.py

{cloudoll-3.0.4 → cloudoll-3.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cloudoll"
-version = "3.0.4" 
+version = "3.0.6" 
 description = "Assist in quickly creating distributable microservices based on aiohttp, with a focus on simplicity and ease of use."
 authors = [
   {name = "Qiu", email = "smallerqiu@gmail.com"},

cloudoll-3.0.4/cloudoll/__init__.py

@@ -1 +0,0 @@
-__version__ = "3.0.4"