cloudoll 3.0.5__tar.gz → 3.0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloudoll-3.0.5 → cloudoll-3.0.7}/PKG-INFO +1 -1
- cloudoll-3.0.7/cloudoll/__init__.py +1 -0
- cloudoll-3.0.7/cloudoll/orm/__init__.py +44 -0
- cloudoll-3.0.7/cloudoll/orm/awsmysql.py +79 -0
- cloudoll-3.0.7/cloudoll/orm/awspostgres.py +93 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/orm/base.py +14 -7
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/orm/field.py +4 -1
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/orm/model.py +9 -5
- cloudoll-3.0.7/cloudoll/orm/mysql.py +112 -0
- cloudoll-3.0.7/cloudoll/orm/postgres.py +116 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/orm/test_sql.py +1 -1
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/utils/snowflake.py +18 -19
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/web/core.py +51 -29
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll.egg-info/PKG-INFO +1 -1
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll.egg-info/SOURCES.txt +2 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/pyproject.toml +1 -1
- cloudoll-3.0.5/cloudoll/__init__.py +0 -1
- cloudoll-3.0.5/cloudoll/orm/__init__.py +0 -246
- cloudoll-3.0.5/cloudoll/orm/mysql.py +0 -84
- cloudoll-3.0.5/cloudoll/orm/postgres.py +0 -69
- {cloudoll-3.0.5 → cloudoll-3.0.7}/LICENSE +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/README.md +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/__main__.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/cli.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/clitool/__init__.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/clitool/cli_main.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/clitool/m2d.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/clitool/process.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/clitool/spinner.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/clitool/watch.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/logging/__init__.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/mail/__init__.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/mail/smtp.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/orm/err.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/orm/parse.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/template/__init__.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/utils/__init__.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/utils/common.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/web/__init__.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/web/exception.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/web/html.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/web/jwt.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/web/requests.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll/web/settings.py +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll.egg-info/dependency_links.txt +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll.egg-info/entry_points.txt +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll.egg-info/requires.txt +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/cloudoll.egg-info/top_level.txt +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/setup.cfg +0 -0
- {cloudoll-3.0.5 → cloudoll-3.0.7}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloudoll
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.7
|
|
4
4
|
Summary: Assist in quickly creating distributable microservices based on aiohttp, with a focus on simplicity and ease of use.
|
|
5
5
|
Home-page: https://github.com/smallerqiu/cloudoll-py
|
|
6
6
|
Author: Qiu
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.0.7"
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from redis import asyncio as aioredis
|
|
2
|
+
from cloudoll.orm.parse import parse_coon
|
|
3
|
+
import aiopg as pg
|
|
4
|
+
from .mysql import Mysql
|
|
5
|
+
from .postgres import Postgres
|
|
6
|
+
from .awspostgres import AwsPostgres
|
|
7
|
+
from .awsmysql import AwsMysql
|
|
8
|
+
|
|
9
|
+
__all__ = ["create_engine"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
async def create_engine(**kw):
|
|
13
|
+
url = kw.get("url")
|
|
14
|
+
driver = None
|
|
15
|
+
configs = {}
|
|
16
|
+
query = {}
|
|
17
|
+
|
|
18
|
+
if url is not None:
|
|
19
|
+
configs, query = parse_coon(url)
|
|
20
|
+
driver = configs["type"]
|
|
21
|
+
else:
|
|
22
|
+
driver = kw.get("type")
|
|
23
|
+
configs = kw
|
|
24
|
+
|
|
25
|
+
# info("DB Config:", configs, query)
|
|
26
|
+
|
|
27
|
+
if driver == "mysql":
|
|
28
|
+
return await Mysql().create_engine(**configs, **query)
|
|
29
|
+
elif driver == "aws-mysql":
|
|
30
|
+
return await AwsMysql().create_engine(**configs, **query)
|
|
31
|
+
elif driver in ["aws-postgres", "aws-postgressql"]:
|
|
32
|
+
return await AwsPostgres().create_engine(**configs, **query)
|
|
33
|
+
elif driver in ["postgres", "postgressql"]:
|
|
34
|
+
return await Postgres().create_engine(**configs, **query)
|
|
35
|
+
elif driver in ["redis", "rediss"]:
|
|
36
|
+
"""
|
|
37
|
+
redis://[[username]:[password]]@localhost:6379/0
|
|
38
|
+
rediss://[[username]:[password]]@localhost:6379/0
|
|
39
|
+
"""
|
|
40
|
+
if url is None:
|
|
41
|
+
url = f"{driver}://{configs['username']}:{configs['password']}@{configs['host']}:{configs['port']}/{configs['db']}"
|
|
42
|
+
return await aioredis.from_url(url, **query)
|
|
43
|
+
else:
|
|
44
|
+
raise ValueError("Not support this database type.")
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from mysql.connector import Connect
|
|
2
|
+
|
|
3
|
+
from aws_advanced_python_wrapper import AwsWrapperConnection
|
|
4
|
+
from aws_advanced_python_wrapper.connection_provider import ConnectionProviderManager
|
|
5
|
+
from aws_advanced_python_wrapper.sql_alchemy_connection_provider import (
|
|
6
|
+
SqlAlchemyPooledConnectionProvider,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
from cloudoll.orm.base import MeteBase, QueryTypes
|
|
10
|
+
from cloudoll.logging import info, error
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AwsMysql(MeteBase):
|
|
14
|
+
def __init__(self):
|
|
15
|
+
self.driver = "aws-mysql"
|
|
16
|
+
provider = SqlAlchemyPooledConnectionProvider()
|
|
17
|
+
ConnectionProviderManager.set_connection_provider(provider)
|
|
18
|
+
|
|
19
|
+
async def close(self):
|
|
20
|
+
ConnectionProviderManager.release_resources()
|
|
21
|
+
|
|
22
|
+
async def create_engine(self, **kw):
|
|
23
|
+
self._dsn = f"host={kw.get("host")} database={kw.get("db")} user={kw.get("username")} password={kw.get("password","")}"
|
|
24
|
+
self._params = {
|
|
25
|
+
"plugins": kw.get("plugins", ""),
|
|
26
|
+
"wrapper_dialect": kw.get("wrapper_dialect", ""),
|
|
27
|
+
"autocommit": True,
|
|
28
|
+
}
|
|
29
|
+
return self
|
|
30
|
+
|
|
31
|
+
async def query(self, sql, params=None, query_type=QueryTypes.ONE, size=10):
|
|
32
|
+
sql = sql.replace("?", "%s")
|
|
33
|
+
try:
|
|
34
|
+
with AwsWrapperConnection.connect(
|
|
35
|
+
Connect, self._dsn, **self._params
|
|
36
|
+
) as conn:
|
|
37
|
+
with conn.cursor() as cursor:
|
|
38
|
+
if (
|
|
39
|
+
query_type == QueryTypes.CREATEBATCH
|
|
40
|
+
or query_type == QueryTypes.UPDATEBATCH
|
|
41
|
+
):
|
|
42
|
+
cursor.executemany(sql, params)
|
|
43
|
+
else:
|
|
44
|
+
cursor.execute(sql, params)
|
|
45
|
+
|
|
46
|
+
if query_type == QueryTypes.ALL:
|
|
47
|
+
return cursor.fetchall()
|
|
48
|
+
elif query_type == QueryTypes.ONE:
|
|
49
|
+
return cursor.fetchone()
|
|
50
|
+
elif query_type == QueryTypes.MANY:
|
|
51
|
+
return cursor.fetchmany(size)
|
|
52
|
+
elif query_type == QueryTypes.COUNT:
|
|
53
|
+
rows = cursor.fetchone()
|
|
54
|
+
count = 0
|
|
55
|
+
if rows is None:
|
|
56
|
+
return count
|
|
57
|
+
for row in rows:
|
|
58
|
+
count = rows[row]
|
|
59
|
+
return count
|
|
60
|
+
elif query_type == QueryTypes.GROUP_COUNT:
|
|
61
|
+
result = cursor.fetchall()
|
|
62
|
+
return 0 if not result else len(result)
|
|
63
|
+
elif query_type == QueryTypes.CREATE:
|
|
64
|
+
result = cursor.rowcount > 0
|
|
65
|
+
id = cursor.target_cursor.lastrowid
|
|
66
|
+
return result, id
|
|
67
|
+
elif query_type == QueryTypes.CREATEBATCH:
|
|
68
|
+
count = cursor.rowcount
|
|
69
|
+
id = cursor.target_cursor.lastrowid
|
|
70
|
+
return count, id
|
|
71
|
+
elif query_type == QueryTypes.UPDATE:
|
|
72
|
+
return cursor.rowcount > 0
|
|
73
|
+
elif query_type == QueryTypes.UPDATEBATCH:
|
|
74
|
+
return cursor.rowcount
|
|
75
|
+
elif query_type == QueryTypes.DELETE:
|
|
76
|
+
return cursor.rowcount > 0
|
|
77
|
+
except Exception as e:
|
|
78
|
+
error(f"[AWS PG] query error: {e}, SQL: {sql}")
|
|
79
|
+
raise e
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import psycopg
|
|
2
|
+
from aws_advanced_python_wrapper import AwsWrapperConnection
|
|
3
|
+
from aws_advanced_python_wrapper.connection_provider import ConnectionProviderManager
|
|
4
|
+
from aws_advanced_python_wrapper.sql_alchemy_connection_provider import (
|
|
5
|
+
SqlAlchemyPooledConnectionProvider,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
from cloudoll.orm.base import MeteBase, QueryTypes
|
|
9
|
+
from cloudoll.logging import info, error
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AwsPostgres(MeteBase):
|
|
13
|
+
def __init__(self):
|
|
14
|
+
self.driver = "aws-postgres"
|
|
15
|
+
provider = SqlAlchemyPooledConnectionProvider()
|
|
16
|
+
ConnectionProviderManager.set_connection_provider(provider)
|
|
17
|
+
|
|
18
|
+
async def close(self):
|
|
19
|
+
ConnectionProviderManager.release_resources()
|
|
20
|
+
|
|
21
|
+
async def create_engine(self, **kw):
|
|
22
|
+
self._dsn = f"host={kw.get("host")} dbname={kw.get("db")} user={kw.get("username")} password={kw.get("username")}"
|
|
23
|
+
self._params = {
|
|
24
|
+
"plugins": kw.get("plugins", ""),
|
|
25
|
+
"wrapper_dialect": kw.get("wrapper_dialect"),
|
|
26
|
+
"autocommit": True,
|
|
27
|
+
}
|
|
28
|
+
return self
|
|
29
|
+
|
|
30
|
+
async def query(self, sql, params=None, query_type=QueryTypes.ONE, size=10):
|
|
31
|
+
sql = sql.replace("?", "%s").replace("`", '"')
|
|
32
|
+
try:
|
|
33
|
+
with AwsWrapperConnection.connect(
|
|
34
|
+
psycopg.Connection.connect, self._dsn, **self._params
|
|
35
|
+
) as conn:
|
|
36
|
+
with conn.cursor() as cursor:
|
|
37
|
+
if (
|
|
38
|
+
query_type == QueryTypes.CREATEBATCH
|
|
39
|
+
or query_type == QueryTypes.UPDATEBATCH
|
|
40
|
+
):
|
|
41
|
+
cursor.executemany(sql, params)
|
|
42
|
+
else:
|
|
43
|
+
cursor.execute(sql, params)
|
|
44
|
+
|
|
45
|
+
result = None
|
|
46
|
+
|
|
47
|
+
if query_type == QueryTypes.ALL and cursor.description is not None:
|
|
48
|
+
columns = [desc[0] for desc in cursor.target_cursor.description]
|
|
49
|
+
rows = cursor.fetchall()
|
|
50
|
+
result = [dict(zip(columns, row)) for row in rows]
|
|
51
|
+
return result
|
|
52
|
+
elif (
|
|
53
|
+
query_type == QueryTypes.ONE and cursor.description is not None
|
|
54
|
+
):
|
|
55
|
+
columns = [desc[0] for desc in cursor.target_cursor.description]
|
|
56
|
+
row = cursor.fetchone()
|
|
57
|
+
result = dict(zip(columns, row)) if row else {}
|
|
58
|
+
return result
|
|
59
|
+
elif (
|
|
60
|
+
query_type == QueryTypes.MANY and cursor.description is not None
|
|
61
|
+
):
|
|
62
|
+
columns = [desc[0] for desc in cursor.target_cursor.description]
|
|
63
|
+
rows = cursor.fetchmany(size)
|
|
64
|
+
result = [dict(zip(columns, row)) for row in rows]
|
|
65
|
+
return result
|
|
66
|
+
elif query_type == QueryTypes.COUNT:
|
|
67
|
+
result = cursor.fetchone()
|
|
68
|
+
count = 0
|
|
69
|
+
if result is None:
|
|
70
|
+
return count
|
|
71
|
+
for value in result:
|
|
72
|
+
count = value
|
|
73
|
+
return count
|
|
74
|
+
elif query_type == QueryTypes.GROUP_COUNT:
|
|
75
|
+
result = cursor.fetchall()
|
|
76
|
+
return 0 if not result else len(result)
|
|
77
|
+
elif query_type == QueryTypes.CREATE:
|
|
78
|
+
result = cursor.rowcount > 0
|
|
79
|
+
id = cursor.target_cursor.lastrowid
|
|
80
|
+
return result, id
|
|
81
|
+
elif query_type == QueryTypes.CREATEBATCH:
|
|
82
|
+
count = cursor.rowcount
|
|
83
|
+
id = cursor.target_cursor.lastrowid
|
|
84
|
+
return count, id
|
|
85
|
+
elif query_type == QueryTypes.UPDATE:
|
|
86
|
+
return cursor.rowcount > 0
|
|
87
|
+
elif query_type == QueryTypes.UPDATEBATCH:
|
|
88
|
+
return cursor.rowcount
|
|
89
|
+
elif query_type == QueryTypes.DELETE:
|
|
90
|
+
return cursor.rowcount > 0
|
|
91
|
+
except Exception as e:
|
|
92
|
+
error(f"[AWS PG] query error: {e}, SQL: {sql}")
|
|
93
|
+
raise e
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
from abc import abstractmethod
|
|
1
2
|
from enum import Enum
|
|
2
|
-
from typing import Tuple
|
|
3
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
4
|
+
from aiomysql.pool import Pool as MyPool
|
|
5
|
+
from aiopg import Pool as PGPool
|
|
3
6
|
|
|
4
7
|
|
|
5
8
|
class QueryTypes(Enum):
|
|
@@ -16,14 +19,16 @@ class QueryTypes(Enum):
|
|
|
16
19
|
|
|
17
20
|
|
|
18
21
|
class MeteBase:
|
|
22
|
+
# def __init__(self):
|
|
23
|
+
# self.pool: Optional[MyPool | PGPool] = None
|
|
24
|
+
# self.cursor: Optional[Cursor] = None
|
|
25
|
+
# self.conn: Optional[Connection] = None
|
|
19
26
|
|
|
20
27
|
async def __aexit__(self):
|
|
21
28
|
await self.close()
|
|
22
29
|
|
|
23
30
|
async def close(self):
|
|
24
|
-
|
|
25
|
-
self.pool.close()
|
|
26
|
-
await self.pool.wait_closed()
|
|
31
|
+
...
|
|
27
32
|
|
|
28
33
|
# async def begin_transaction(self):
|
|
29
34
|
# conn = await self._set_conn()
|
|
@@ -46,9 +51,10 @@ class MeteBase:
|
|
|
46
51
|
# if cursor:
|
|
47
52
|
# await cursor.close()
|
|
48
53
|
# self.pool.release(conn)
|
|
54
|
+
@abstractmethod
|
|
49
55
|
async def query(
|
|
50
|
-
self, sql, params=None, query_type: QueryTypes =
|
|
51
|
-
): ...
|
|
56
|
+
self, sql, params=None, query_type: QueryTypes = QueryTypes.ONE, size: int = 10
|
|
57
|
+
) -> Union[None, int, Tuple[Any, Any], Dict, List[Dict]]: ...
|
|
52
58
|
|
|
53
59
|
async def all(self, sql, params):
|
|
54
60
|
return await self.query(sql, params, QueryTypes.ALL)
|
|
@@ -61,6 +67,7 @@ class MeteBase:
|
|
|
61
67
|
|
|
62
68
|
async def count(self, sql, params):
|
|
63
69
|
return await self.query(sql, params, QueryTypes.COUNT)
|
|
70
|
+
|
|
64
71
|
async def group_count(self, sql, params):
|
|
65
72
|
return await self.query(sql, params, QueryTypes.GROUP_COUNT)
|
|
66
73
|
|
|
@@ -73,7 +80,7 @@ class MeteBase:
|
|
|
73
80
|
async def delete(self, sql, params):
|
|
74
81
|
return await self.query(sql, params, QueryTypes.DELETE)
|
|
75
82
|
|
|
76
|
-
async def create(self, sql, params)
|
|
83
|
+
async def create(self, sql, params):
|
|
77
84
|
return await self.query(sql, params, QueryTypes.CREATE)
|
|
78
85
|
|
|
79
86
|
async def create_batch(self, sql, params):
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
|
|
1
4
|
class objdict(dict):
|
|
2
5
|
def __getattr__(self, attr):
|
|
3
6
|
try:
|
|
@@ -502,7 +505,7 @@ class Field(FieldBase):
|
|
|
502
505
|
**kwargs,
|
|
503
506
|
):
|
|
504
507
|
super().__init__()
|
|
505
|
-
self.full_name = None
|
|
508
|
+
self.full_name: Optional[str] = None
|
|
506
509
|
self._value = None
|
|
507
510
|
self.name = name
|
|
508
511
|
self.column_type = column_type
|
|
@@ -150,14 +150,18 @@ class Model(metaclass=ModelMetaclass):
|
|
|
150
150
|
@classmethod
|
|
151
151
|
def use(cls, pool=None):
|
|
152
152
|
cls.__pool__ = pool
|
|
153
|
-
cls.
|
|
153
|
+
cls.__is_pg = (
|
|
154
|
+
pool.driver in ["postgres", "postgressql", "aws-postgres"]
|
|
155
|
+
if pool
|
|
156
|
+
else False
|
|
157
|
+
)
|
|
154
158
|
return cls()
|
|
155
159
|
|
|
156
160
|
def select(cls, *args):
|
|
157
161
|
cols = []
|
|
158
162
|
for col in args:
|
|
159
163
|
if isinstance(col, Field):
|
|
160
|
-
cols.append(col.name if cls.
|
|
164
|
+
cols.append(col.name if cls.__is_pg else col.full_name)
|
|
161
165
|
elif isinstance(col, Function):
|
|
162
166
|
q, p = col.sql()
|
|
163
167
|
cols.append(q)
|
|
@@ -200,7 +204,7 @@ class Model(metaclass=ModelMetaclass):
|
|
|
200
204
|
if isinstance(f, str):
|
|
201
205
|
by.append(f)
|
|
202
206
|
else:
|
|
203
|
-
by.append(f"{f.lhs.name if self.
|
|
207
|
+
by.append(f"{f.lhs.name if self.__is_pg else f.lhs.full_name} {f.op}")
|
|
204
208
|
if self.__order_by__ is not None:
|
|
205
209
|
by = self.__order_by__ + by
|
|
206
210
|
self.__order_by__ = by
|
|
@@ -209,7 +213,7 @@ class Model(metaclass=ModelMetaclass):
|
|
|
209
213
|
def group_by(self, *args):
|
|
210
214
|
by = []
|
|
211
215
|
for f in args:
|
|
212
|
-
by.append(f.name if self.
|
|
216
|
+
by.append(f.name if self.__is_pg else f.full_name)
|
|
213
217
|
if self.__group_by__ is not None:
|
|
214
218
|
by = self.__group_by__ + by
|
|
215
219
|
self.__group_by__ = by
|
|
@@ -365,7 +369,7 @@ class Model(metaclass=ModelMetaclass):
|
|
|
365
369
|
return await self.__pool__.all(sql, args)
|
|
366
370
|
|
|
367
371
|
def _exchange_sql(self, sql: str):
|
|
368
|
-
if self.
|
|
372
|
+
if self.__is_pg:
|
|
369
373
|
sql = sql.replace("CURDATE()", "CURRENT_DATE")
|
|
370
374
|
sql = sql.replace("NOW()", "CURRENT_TIMESTAMP")
|
|
371
375
|
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
from typing import Any, Optional
|
|
2
|
+
import aiomysql
|
|
3
|
+
from cloudoll.logging import error
|
|
4
|
+
from cloudoll.orm.base import MeteBase, QueryTypes
|
|
5
|
+
from cloudoll.logging import info, error
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AttrDict(dict):
|
|
9
|
+
def __getattr__(self, name):
|
|
10
|
+
try:
|
|
11
|
+
return self[name]
|
|
12
|
+
except KeyError:
|
|
13
|
+
return None
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class AttrDictCursor(aiomysql.DictCursor):
|
|
17
|
+
dict_type = AttrDict
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Mysql(MeteBase):
|
|
21
|
+
def __init__(self):
|
|
22
|
+
self.pool: aiomysql.Pool
|
|
23
|
+
self.driver = "mysql"
|
|
24
|
+
|
|
25
|
+
def __call__(self, *args: Any, **kwds: Any) -> Any:
|
|
26
|
+
self.__init__(*args, **kwds)
|
|
27
|
+
return self
|
|
28
|
+
|
|
29
|
+
async def close(self):
|
|
30
|
+
if self.pool:
|
|
31
|
+
self.pool.close()
|
|
32
|
+
await self.pool.wait_closed()
|
|
33
|
+
|
|
34
|
+
async def query(
|
|
35
|
+
self, sql, params=None, query_type: QueryTypes = QueryTypes.ONE, size: int = 10
|
|
36
|
+
):
|
|
37
|
+
sql = sql.replace("?", "%s")
|
|
38
|
+
if not self.pool:
|
|
39
|
+
raise ValueError("must be create_engine first.")
|
|
40
|
+
if self.pool._closing or self.pool._closed:
|
|
41
|
+
return None
|
|
42
|
+
async with self.pool.acquire() as conn:
|
|
43
|
+
if conn.echo:
|
|
44
|
+
info("sql: %s , %s", sql, params)
|
|
45
|
+
|
|
46
|
+
async with conn.cursor() as cursor:
|
|
47
|
+
cursor: aiomysql.Cursor = cursor
|
|
48
|
+
if (
|
|
49
|
+
query_type == QueryTypes.CREATEBATCH
|
|
50
|
+
or query_type == QueryTypes.UPDATEBATCH
|
|
51
|
+
):
|
|
52
|
+
await cursor.executemany(sql, params)
|
|
53
|
+
else:
|
|
54
|
+
await cursor.execute(sql, params)
|
|
55
|
+
|
|
56
|
+
await conn.commit()
|
|
57
|
+
|
|
58
|
+
if query_type == QueryTypes.ALL:
|
|
59
|
+
return await cursor.fetchall()
|
|
60
|
+
elif query_type == QueryTypes.ONE:
|
|
61
|
+
return await cursor.fetchone()
|
|
62
|
+
elif query_type == QueryTypes.MANY:
|
|
63
|
+
return await cursor.fetchmany(size)
|
|
64
|
+
elif query_type == QueryTypes.COUNT:
|
|
65
|
+
rows = await cursor.fetchone()
|
|
66
|
+
count = 0
|
|
67
|
+
if rows is None:
|
|
68
|
+
return count
|
|
69
|
+
for row in rows:
|
|
70
|
+
count = rows[row]
|
|
71
|
+
return count
|
|
72
|
+
elif query_type == QueryTypes.GROUP_COUNT:
|
|
73
|
+
result = await cursor.fetchall()
|
|
74
|
+
return 0 if not result else len(result)
|
|
75
|
+
elif query_type == QueryTypes.CREATE:
|
|
76
|
+
result = cursor.rowcount > 0
|
|
77
|
+
id = cursor.lastrowid
|
|
78
|
+
return result, id
|
|
79
|
+
elif query_type == QueryTypes.CREATEBATCH:
|
|
80
|
+
count = cursor.rowcount
|
|
81
|
+
id = cursor.lastrowid
|
|
82
|
+
return count, id
|
|
83
|
+
elif query_type == QueryTypes.UPDATE:
|
|
84
|
+
return cursor.rowcount > 0
|
|
85
|
+
elif query_type == QueryTypes.UPDATEBATCH:
|
|
86
|
+
return cursor.rowcount
|
|
87
|
+
elif query_type == QueryTypes.DELETE:
|
|
88
|
+
return cursor.rowcount > 0
|
|
89
|
+
|
|
90
|
+
self.pool.release(conn)
|
|
91
|
+
|
|
92
|
+
async def create_engine(self, loop=None, **kw):
|
|
93
|
+
try:
|
|
94
|
+
self.pool = await aiomysql.create_pool(
|
|
95
|
+
host=kw.get("host", "localhost"),
|
|
96
|
+
port=kw.get("port", 3306),
|
|
97
|
+
user=kw.get("username"),
|
|
98
|
+
password=str(kw.get("password", "")),
|
|
99
|
+
db=kw.get("db"),
|
|
100
|
+
echo=kw.get("echo", False),
|
|
101
|
+
charset=kw.get("charset", "utf8"),
|
|
102
|
+
autocommit=False, # kw.get("autocommit", False),
|
|
103
|
+
maxsize=kw.get("maxsize", 10),
|
|
104
|
+
minsize=kw.get("minsize", 5),
|
|
105
|
+
cursorclass=AttrDictCursor,
|
|
106
|
+
loop=loop,
|
|
107
|
+
)
|
|
108
|
+
info(f"Database connection successfully for mysql/{kw.get('db')}.")
|
|
109
|
+
except Exception as e:
|
|
110
|
+
# print(traceback.format_exc())
|
|
111
|
+
error(f"Database connection failed,the instance : mysql/{kw.get('db')}")
|
|
112
|
+
return self
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import aiopg
|
|
2
|
+
from cloudoll.logging import error
|
|
3
|
+
from cloudoll.orm.base import MeteBase, QueryTypes
|
|
4
|
+
from cloudoll.logging import info, error
|
|
5
|
+
from typing import Any, Optional
|
|
6
|
+
|
|
7
|
+
class Postgres(MeteBase):
|
|
8
|
+
def __init__(self):
|
|
9
|
+
self.pool: Optional[aiopg.Pool] = None
|
|
10
|
+
self.driver = "postgres"
|
|
11
|
+
|
|
12
|
+
def __call__(self, *args: Any, **kwds: Any) -> Any:
|
|
13
|
+
self.__init__(*args, **kwds)
|
|
14
|
+
return self
|
|
15
|
+
|
|
16
|
+
async def close(self):
|
|
17
|
+
if self.pool:
|
|
18
|
+
self.pool.close()
|
|
19
|
+
await self.pool.wait_closed()
|
|
20
|
+
|
|
21
|
+
async def query(
|
|
22
|
+
self,
|
|
23
|
+
sql,
|
|
24
|
+
params=None,
|
|
25
|
+
query_type: QueryTypes = QueryTypes.COUNT,
|
|
26
|
+
size: int = 10,
|
|
27
|
+
):
|
|
28
|
+
sql = sql.replace("?", "%s").replace("`", '"')
|
|
29
|
+
if not self.pool:
|
|
30
|
+
raise ValueError("must be create_engine first.")
|
|
31
|
+
if self.pool._closing or self.pool._closed:
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
async with self.pool.acquire() as conn:
|
|
35
|
+
if conn.echo:
|
|
36
|
+
info("sql: %s ,%s", sql, params)
|
|
37
|
+
|
|
38
|
+
async with conn.cursor() as cursor:
|
|
39
|
+
# current_cursor = getattr(cursor, 'lastrowid', None)
|
|
40
|
+
if (
|
|
41
|
+
query_type == QueryTypes.CREATEBATCH
|
|
42
|
+
or query_type == QueryTypes.UPDATEBATCH
|
|
43
|
+
):
|
|
44
|
+
await cursor.executemany(sql, params)
|
|
45
|
+
else:
|
|
46
|
+
await cursor.execute(sql, params)
|
|
47
|
+
|
|
48
|
+
# await conn.commit()
|
|
49
|
+
result = None
|
|
50
|
+
|
|
51
|
+
if query_type == QueryTypes.ALL and cursor.description is not None:
|
|
52
|
+
columns = [desc[0] for desc in cursor.description]
|
|
53
|
+
rows = await cursor.fetchall()
|
|
54
|
+
result = [dict(zip(columns, row)) for row in rows]
|
|
55
|
+
return result
|
|
56
|
+
elif query_type == QueryTypes.ONE and cursor.description is not None:
|
|
57
|
+
columns = [desc[0] for desc in cursor.description]
|
|
58
|
+
row = await cursor.fetchone()
|
|
59
|
+
result = dict(zip(columns, row)) if row else {}
|
|
60
|
+
return result
|
|
61
|
+
elif query_type == QueryTypes.MANY and cursor.description is not None:
|
|
62
|
+
columns = [desc[0] for desc in cursor.description]
|
|
63
|
+
rows = await cursor.fetchmany(size)
|
|
64
|
+
result = [dict(zip(columns, row)) for row in rows]
|
|
65
|
+
return result
|
|
66
|
+
elif query_type == QueryTypes.COUNT:
|
|
67
|
+
result = await cursor.fetchone()
|
|
68
|
+
count = 0
|
|
69
|
+
if result is None:
|
|
70
|
+
return count
|
|
71
|
+
for value in result:
|
|
72
|
+
count = value
|
|
73
|
+
return count
|
|
74
|
+
elif query_type == QueryTypes.GROUP_COUNT:
|
|
75
|
+
result = await cursor.fetchall()
|
|
76
|
+
return 0 if not result else len(result)
|
|
77
|
+
elif query_type == QueryTypes.CREATE:
|
|
78
|
+
result = cursor.rowcount > 0
|
|
79
|
+
id = cursor.lastrowid
|
|
80
|
+
return result, id
|
|
81
|
+
elif query_type == QueryTypes.CREATEBATCH:
|
|
82
|
+
count = cursor.rowcount
|
|
83
|
+
id = cursor.lastrowid
|
|
84
|
+
return count, id
|
|
85
|
+
elif query_type == QueryTypes.UPDATE:
|
|
86
|
+
return cursor.rowcount > 0
|
|
87
|
+
elif query_type == QueryTypes.UPDATEBATCH:
|
|
88
|
+
return cursor.rowcount
|
|
89
|
+
elif query_type == QueryTypes.DELETE:
|
|
90
|
+
return cursor.rowcount > 0
|
|
91
|
+
|
|
92
|
+
async def create_engine(self, **kw):
|
|
93
|
+
try:
|
|
94
|
+
host = (kw.get("host", "localhost"),)
|
|
95
|
+
port = (kw.get("port", 5432),)
|
|
96
|
+
user = (kw.get("username"),)
|
|
97
|
+
password = (str(kw.get("password", "")),)
|
|
98
|
+
db = (kw.get("db"),)
|
|
99
|
+
dsn = f"dbname={db[0]} user={user[0]} password={password[0]} host={host[0]} port={port[0]}" # aiopg
|
|
100
|
+
# dsn = f"postgres://{user[0]}:{password[0]}@{host[0]}:{port[0]}/{db[0]}" # asyncpg
|
|
101
|
+
self.pool = await aiopg.create_pool(
|
|
102
|
+
dsn=dsn,
|
|
103
|
+
timeout=float(kw.get("timeout", 60)),
|
|
104
|
+
echo=kw.get("echo", False), # aiopg
|
|
105
|
+
# max_size=kw.get("maxsize", 10), # asyncpg
|
|
106
|
+
# min_size=kw.get("minsize", 5), # asyncpg
|
|
107
|
+
maxsize=kw.get("maxsize", 10), # aiopg
|
|
108
|
+
minsize=kw.get("minsize", 5), # aiopg
|
|
109
|
+
)
|
|
110
|
+
info(f"Database connection successfully for postgres/{kw.get('db')}")
|
|
111
|
+
except Exception as e:
|
|
112
|
+
error(e)
|
|
113
|
+
# print(traceback.format_exc())
|
|
114
|
+
error(f"Database connection failed,the instance : postgres/{kw.get('db')}")
|
|
115
|
+
|
|
116
|
+
return self
|
|
@@ -42,7 +42,7 @@ class B(Model):
|
|
|
42
42
|
|
|
43
43
|
class C(Model):
|
|
44
44
|
|
|
45
|
-
__table__ = "
|
|
45
|
+
__table__ = "c"
|
|
46
46
|
|
|
47
47
|
id = models.BigIntegerField(primary_key=True, max_length=64, not_null=True)
|
|
48
48
|
pack_id = models.BigIntegerField(max_length=64, default="0", not_null=True)
|
|
@@ -1,34 +1,30 @@
|
|
|
1
|
-
|
|
2
1
|
import time
|
|
3
2
|
import threading
|
|
4
3
|
|
|
5
4
|
|
|
6
5
|
"""
|
|
7
|
-
雪花算法(全局唯一id)
|
|
8
|
-
|
|
9
|
-
# 最早是Twitter公司在其内部用于分布式环境下生成唯一ID
|
|
10
|
-
|
|
11
6
|
# Twitter's Snowflake algorithm implementation which is used to generate distributed IDs.
|
|
12
|
-
|
|
13
7
|
# https://github.com/twitter-archive/snowflake/blob/snowflake-2010/src/main/scala/com/twitter/service/snowflake/IdWorker.scala
|
|
14
8
|
"""
|
|
15
9
|
|
|
16
10
|
|
|
17
11
|
class IdWorker:
|
|
18
12
|
def __init__(self, worker_id, datacenter_id, sequence=0):
|
|
19
|
-
|
|
13
|
+
# 位移值
|
|
20
14
|
self.worker_id_bits = 5
|
|
21
15
|
self.datacenter_id_bits = 5
|
|
22
16
|
self.sequence_bits = 12
|
|
23
|
-
|
|
17
|
+
|
|
24
18
|
self.max_worker_id = -1 ^ (-1 << self.worker_id_bits)
|
|
25
19
|
self.max_datacenter_id = -1 ^ (-1 << self.datacenter_id_bits)
|
|
26
20
|
|
|
27
21
|
# 参数校验
|
|
28
22
|
if worker_id > self.max_worker_id or worker_id < 0:
|
|
29
|
-
raise ValueError(f
|
|
23
|
+
raise ValueError(f"worker_id must be between 0 and {self.max_worker_id}")
|
|
30
24
|
if datacenter_id > self.max_datacenter_id or datacenter_id < 0:
|
|
31
|
-
raise ValueError(
|
|
25
|
+
raise ValueError(
|
|
26
|
+
f"datacenter_id must be between 0 and {self.max_datacenter_id}"
|
|
27
|
+
)
|
|
32
28
|
|
|
33
29
|
self.worker_id = worker_id
|
|
34
30
|
self.datacenter_id = datacenter_id
|
|
@@ -38,12 +34,11 @@ class IdWorker:
|
|
|
38
34
|
# Twitter Epoch (2010-11-04 01:42:54 UTC)
|
|
39
35
|
self.twepoch = 1288834974657
|
|
40
36
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
37
|
self.worker_id_shift = self.sequence_bits
|
|
45
38
|
self.datacenter_id_shift = self.sequence_bits + self.worker_id_bits
|
|
46
|
-
self.timestamp_left_shift =
|
|
39
|
+
self.timestamp_left_shift = (
|
|
40
|
+
self.sequence_bits + self.worker_id_bits + self.datacenter_id_bits
|
|
41
|
+
)
|
|
47
42
|
self.sequence_mask = -1 ^ (-1 << self.sequence_bits)
|
|
48
43
|
|
|
49
44
|
self.last_timestamp = -1
|
|
@@ -62,7 +57,9 @@ class IdWorker:
|
|
|
62
57
|
timestamp = self._time_gen()
|
|
63
58
|
|
|
64
59
|
if timestamp < self.last_timestamp:
|
|
65
|
-
raise Exception(
|
|
60
|
+
raise Exception(
|
|
61
|
+
f"Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds"
|
|
62
|
+
)
|
|
66
63
|
|
|
67
64
|
if self.last_timestamp == timestamp:
|
|
68
65
|
self.sequence = (self.sequence + 1) & self.sequence_mask
|
|
@@ -73,10 +70,12 @@ class IdWorker:
|
|
|
73
70
|
|
|
74
71
|
self.last_timestamp = timestamp
|
|
75
72
|
|
|
76
|
-
id = (
|
|
77
|
-
(self.
|
|
78
|
-
(self.
|
|
79
|
-
self.
|
|
73
|
+
id = (
|
|
74
|
+
((timestamp - self.twepoch) << self.timestamp_left_shift)
|
|
75
|
+
| (self.datacenter_id << self.datacenter_id_shift)
|
|
76
|
+
| (self.worker_id << self.worker_id_shift)
|
|
77
|
+
| self.sequence
|
|
78
|
+
)
|
|
80
79
|
|
|
81
80
|
return id
|
|
82
81
|
|