pymecli 0.5.6__tar.gz → 0.5.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pymecli-0.5.6 → pymecli-0.5.7}/PKG-INFO +12 -1
- {pymecli-0.5.6 → pymecli-0.5.7}/README.md +9 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/bitget.py +15 -8
- pymecli-0.5.7/cli/etcd_mysql.py +162 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/gate.py +12 -2
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/okx.py +11 -2
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/redis_csv.py +19 -10
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/util.py +4 -1
- {pymecli-0.5.6 → pymecli-0.5.7}/crypto/bitget.py +14 -7
- {pymecli-0.5.6 → pymecli-0.5.7}/crypto/gate.py +4 -2
- {pymecli-0.5.6 → pymecli-0.5.7}/crypto/okx.py +4 -2
- {pymecli-0.5.6 → pymecli-0.5.7}/pyproject.toml +4 -1
- pymecli-0.5.7/utils/etcd.py +39 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/mysql.py +63 -51
- {pymecli-0.5.6 → pymecli-0.5.7}/.gitignore +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/api/__init__.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/api/v1/__init__.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/api/v1/clash.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/api/v1/redis.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/__init__.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/dst.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/example.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/fast.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/cli/tele.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/core/__init__.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/core/clash.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/core/config.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/core/redis_client.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/crypto/__init__.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/data/__init__.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/data/dou_dict.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/data/dou_list.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/data/main.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/data/template.yaml +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/models/__init__.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/models/douzero_model.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/models/ocr_model.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/models/response.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/__init__.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/elapsed.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/helper.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/logger.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/path.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/pd.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/pyredis.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/sleep.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/text.py +0 -0
- {pymecli-0.5.6 → pymecli-0.5.7}/utils/toml.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: pymecli
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.7
|
|
4
4
|
Summary: My CLI
|
|
5
5
|
Project-URL: Homepage, https://pypi.org/project/pymecli
|
|
6
6
|
Project-URL: Repository, https://github.com/meme2046/pymecli
|
|
@@ -13,11 +13,13 @@ Requires-Dist: arrow>=1.4.0
|
|
|
13
13
|
Requires-Dist: babel>=2.17.0
|
|
14
14
|
Requires-Dist: cowsay>=6.1
|
|
15
15
|
Requires-Dist: dotenv>=0.9.9
|
|
16
|
+
Requires-Dist: etcd3>=0.12.0
|
|
16
17
|
Requires-Dist: fastapi>=0.127.0
|
|
17
18
|
Requires-Dist: ifaddr>=0.2.0
|
|
18
19
|
Requires-Dist: netifaces-plus>=0.12.5
|
|
19
20
|
Requires-Dist: numpy<2.0.0
|
|
20
21
|
Requires-Dist: pandas>=2.3.3
|
|
22
|
+
Requires-Dist: protobuf<4
|
|
21
23
|
Requires-Dist: pydantic-settings>=2.14.2
|
|
22
24
|
Requires-Dist: pymysql>=1.1.2
|
|
23
25
|
Requires-Dist: pytz>=2025.2
|
|
@@ -61,3 +63,12 @@ uv run dst cmu d:/.backups/dontstarvetogether/modoverrides.lua -o c:/.dst/save/C
|
|
|
61
63
|
|
|
62
64
|
uv run dst cml d:/.backups/dontstarvetogether/modoverrides.lua -m 3486375086
|
|
63
65
|
```
|
|
66
|
+
|
|
67
|
+
# etcdsync
|
|
68
|
+
|
|
69
|
+
```shell
|
|
70
|
+
etcdsync etcd2mysql /test -e d:/.env/pymecli.env -t kvs
|
|
71
|
+
etcdsync etcd2mysql "" -e d:/.env/pymecli.env -t kvs # 全量
|
|
72
|
+
etcdsync mysql2etcd /test -e d:/.env/pymecli.env -t kvs
|
|
73
|
+
etcdsync mysql2etcd -e d:/.env/pymecli.env -t kvs # 全量
|
|
74
|
+
```
|
|
@@ -24,3 +24,12 @@ uv run dst cmu d:/.backups/dontstarvetogether/modoverrides.lua -o c:/.dst/save/C
|
|
|
24
24
|
|
|
25
25
|
uv run dst cml d:/.backups/dontstarvetogether/modoverrides.lua -m 3486375086
|
|
26
26
|
```
|
|
27
|
+
|
|
28
|
+
# etcdsync
|
|
29
|
+
|
|
30
|
+
```shell
|
|
31
|
+
etcdsync etcd2mysql /test -e d:/.env/pymecli.env -t kvs
|
|
32
|
+
etcdsync etcd2mysql "" -e d:/.env/pymecli.env -t kvs # 全量
|
|
33
|
+
etcdsync mysql2etcd /test -e d:/.env/pymecli.env -t kvs
|
|
34
|
+
etcdsync mysql2etcd -e d:/.env/pymecli.env -t kvs # 全量
|
|
35
|
+
```
|
|
@@ -14,6 +14,7 @@ from crypto.bitget import (
|
|
|
14
14
|
spot_tickers,
|
|
15
15
|
)
|
|
16
16
|
from utils.mysql import get_database_engine
|
|
17
|
+
from utils.pyredis import get_redis_client
|
|
17
18
|
|
|
18
19
|
app = typer.Typer()
|
|
19
20
|
|
|
@@ -27,16 +28,22 @@ def sync(
|
|
|
27
28
|
):
|
|
28
29
|
"""同步mysql中grid数据到redis"""
|
|
29
30
|
engine = get_database_engine(env_path)
|
|
31
|
+
redis = get_redis_client()
|
|
32
|
+
try:
|
|
30
33
|
|
|
31
|
-
|
|
32
|
-
|
|
34
|
+
async def _run():
|
|
35
|
+
await grid_open(engine, redis)
|
|
36
|
+
await grid_close(engine, redis)
|
|
37
|
+
await bitget_sf_open(engine, redis)
|
|
38
|
+
await bitget_sf_close(engine, redis)
|
|
39
|
+
await bitget_ff_open(engine, redis)
|
|
40
|
+
await bitget_ff_pending(engine, redis)
|
|
41
|
+
await bitget_ff_close(engine, redis)
|
|
33
42
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
asyncio.run(bitget_ff_pending(engine))
|
|
39
|
-
asyncio.run(bitget_ff_close(engine))
|
|
43
|
+
asyncio.run(_run())
|
|
44
|
+
finally:
|
|
45
|
+
engine.dispose()
|
|
46
|
+
asyncio.run(redis.close())
|
|
40
47
|
|
|
41
48
|
|
|
42
49
|
@app.command()
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MySQL <-> Etcd 数据同步 CLI。
|
|
3
|
+
|
|
4
|
+
两个子命令:
|
|
5
|
+
mysql2etcd 将 MySQL kvs 表的 k, v 写入 etcd
|
|
6
|
+
etcd2mysql 将 etcd 指定前缀的 key/value 写入 MySQL kvs 表
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
|
|
12
|
+
import typer
|
|
13
|
+
from sqlalchemy import text
|
|
14
|
+
|
|
15
|
+
from utils.etcd import get_etcd_client
|
|
16
|
+
from utils.logger import get_logger
|
|
17
|
+
from utils.mysql import get_database_engine
|
|
18
|
+
|
|
19
|
+
app = typer.Typer(help="MySQL <-> Etcd 数据同步工具")
|
|
20
|
+
logger = get_logger(__name__, level=logging.INFO)
|
|
21
|
+
|
|
22
|
+
DEFAULT_TABLE = "kvs"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@app.command("mysql2etcd")
|
|
26
|
+
def mysql_to_etcd(
|
|
27
|
+
prefix: str = typer.Argument(
|
|
28
|
+
"",
|
|
29
|
+
help="MySQL k 列的前缀过滤(空字符串表示全量)",
|
|
30
|
+
),
|
|
31
|
+
env_path: str = typer.Option(
|
|
32
|
+
".env",
|
|
33
|
+
"--env",
|
|
34
|
+
"-e",
|
|
35
|
+
help="dotenv 文件路径",
|
|
36
|
+
),
|
|
37
|
+
table: str = typer.Option(
|
|
38
|
+
DEFAULT_TABLE,
|
|
39
|
+
"--table",
|
|
40
|
+
"-t",
|
|
41
|
+
help="MySQL 表名,包含 k, v 字段",
|
|
42
|
+
),
|
|
43
|
+
key_col: str = typer.Option(
|
|
44
|
+
"k",
|
|
45
|
+
"--key-col",
|
|
46
|
+
help="MySQL 中作为 etcd key 的列名",
|
|
47
|
+
),
|
|
48
|
+
val_col: str = typer.Option(
|
|
49
|
+
"v",
|
|
50
|
+
"--val-col",
|
|
51
|
+
help="MySQL 中作为 etcd value 的列名",
|
|
52
|
+
),
|
|
53
|
+
):
|
|
54
|
+
"""
|
|
55
|
+
将 MySQL 表中的 k, v 全量同步到 etcd。
|
|
56
|
+
|
|
57
|
+
每行记录会以 k 为 key、v 为 value 写入 etcd(覆盖已有值)。
|
|
58
|
+
"""
|
|
59
|
+
engine = get_database_engine(env_path)
|
|
60
|
+
etcd = get_etcd_client(env_path)
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
count = 0
|
|
64
|
+
with engine.connect() as conn:
|
|
65
|
+
if prefix:
|
|
66
|
+
query = text(
|
|
67
|
+
f"SELECT {key_col}, {val_col} FROM {table} WHERE {key_col} LIKE :p"
|
|
68
|
+
)
|
|
69
|
+
rows = conn.execute(query, {"p": f"{prefix}%"}).fetchall()
|
|
70
|
+
else:
|
|
71
|
+
query = text(f"SELECT {key_col}, {val_col} FROM {table}")
|
|
72
|
+
rows = conn.execute(query).fetchall()
|
|
73
|
+
|
|
74
|
+
for row in rows:
|
|
75
|
+
k, v = str(row[0]), str(row[1])
|
|
76
|
+
etcd.put(k, v)
|
|
77
|
+
logger.debug(f"put: {k} -> {v}")
|
|
78
|
+
count += 1
|
|
79
|
+
|
|
80
|
+
logger.info(f"✅ 已写入 {count} 条到 etcd")
|
|
81
|
+
finally:
|
|
82
|
+
engine.dispose()
|
|
83
|
+
etcd.close()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@app.command("etcd2mysql")
|
|
87
|
+
def etcd_to_mysql(
|
|
88
|
+
prefix: str = typer.Argument(
|
|
89
|
+
...,
|
|
90
|
+
help="etcd key 前缀(空字符串表示全量)",
|
|
91
|
+
),
|
|
92
|
+
env_path: str = typer.Option(
|
|
93
|
+
".env",
|
|
94
|
+
"--env",
|
|
95
|
+
"-e",
|
|
96
|
+
help="dotenv 文件路径",
|
|
97
|
+
),
|
|
98
|
+
table: str = typer.Option(
|
|
99
|
+
DEFAULT_TABLE,
|
|
100
|
+
"--table",
|
|
101
|
+
"-t",
|
|
102
|
+
help="MySQL 表名,包含 k, v 字段",
|
|
103
|
+
),
|
|
104
|
+
key_col: str = typer.Option(
|
|
105
|
+
"k",
|
|
106
|
+
"--key-col",
|
|
107
|
+
help="MySQL 中作为 key 的列名",
|
|
108
|
+
),
|
|
109
|
+
val_col: str = typer.Option(
|
|
110
|
+
"v",
|
|
111
|
+
"--val-col",
|
|
112
|
+
help="MySQL 中作为 value 的列名",
|
|
113
|
+
),
|
|
114
|
+
):
|
|
115
|
+
"""
|
|
116
|
+
将 etcd 中指定前缀的 key/value 同步到 MySQL。
|
|
117
|
+
|
|
118
|
+
如果 k 已存在则更新 v, 不存在则新增。
|
|
119
|
+
"""
|
|
120
|
+
engine = get_database_engine(env_path)
|
|
121
|
+
etcd = get_etcd_client(env_path)
|
|
122
|
+
|
|
123
|
+
try:
|
|
124
|
+
# get_prefix 不支持空前缀,全量时用 get_all
|
|
125
|
+
count = 0
|
|
126
|
+
with engine.connect() as conn:
|
|
127
|
+
iterator = etcd.get_all() if not prefix else etcd.get_prefix(prefix)
|
|
128
|
+
for value, meta in iterator:
|
|
129
|
+
k = meta.key.decode("utf-8")
|
|
130
|
+
v = value.decode("utf-8") if isinstance(value, bytes) else value
|
|
131
|
+
|
|
132
|
+
# 检查是否存在
|
|
133
|
+
check_sql = text(
|
|
134
|
+
f"SELECT id FROM {table} WHERE {key_col} = :k LIMIT 1"
|
|
135
|
+
)
|
|
136
|
+
existing = conn.execute(check_sql, {"k": k}).fetchone()
|
|
137
|
+
|
|
138
|
+
if existing:
|
|
139
|
+
update_sql = text(
|
|
140
|
+
f"UPDATE {table} SET {val_col} = :v WHERE {key_col} = :k"
|
|
141
|
+
)
|
|
142
|
+
conn.execute(update_sql, {"v": v, "k": k})
|
|
143
|
+
logger.debug(f"update: {k}")
|
|
144
|
+
else:
|
|
145
|
+
insert_sql = text(
|
|
146
|
+
f"INSERT INTO {table} ({key_col}, {val_col}) VALUES (:k, :v)"
|
|
147
|
+
)
|
|
148
|
+
conn.execute(insert_sql, {"k": k, "v": v})
|
|
149
|
+
logger.debug(f"insert: {k}")
|
|
150
|
+
|
|
151
|
+
count += 1
|
|
152
|
+
|
|
153
|
+
conn.commit()
|
|
154
|
+
|
|
155
|
+
logger.info(f"✅ 已同步 {count} 条到 MySQL")
|
|
156
|
+
finally:
|
|
157
|
+
engine.dispose()
|
|
158
|
+
etcd.close()
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
if __name__ == "__main__":
|
|
162
|
+
app()
|
|
@@ -4,6 +4,7 @@ import typer
|
|
|
4
4
|
|
|
5
5
|
from crypto.gate import grid_close, grid_open
|
|
6
6
|
from utils.mysql import get_database_engine
|
|
7
|
+
from utils.pyredis import get_redis_client
|
|
7
8
|
|
|
8
9
|
app = typer.Typer()
|
|
9
10
|
|
|
@@ -16,5 +17,14 @@ def rsync(
|
|
|
16
17
|
):
|
|
17
18
|
"""同步mysql中grid数据到redis"""
|
|
18
19
|
engine = get_database_engine(env_path)
|
|
19
|
-
|
|
20
|
-
|
|
20
|
+
redis = get_redis_client()
|
|
21
|
+
try:
|
|
22
|
+
|
|
23
|
+
async def _run():
|
|
24
|
+
await grid_open(engine, redis)
|
|
25
|
+
await grid_close(engine, redis)
|
|
26
|
+
|
|
27
|
+
asyncio.run(_run())
|
|
28
|
+
finally:
|
|
29
|
+
engine.dispose()
|
|
30
|
+
asyncio.run(redis.close())
|
|
@@ -4,6 +4,7 @@ import typer
|
|
|
4
4
|
|
|
5
5
|
from crypto.okx import grid_close, grid_open
|
|
6
6
|
from utils.mysql import get_database_engine
|
|
7
|
+
from utils.pyredis import get_redis_client
|
|
7
8
|
|
|
8
9
|
app = typer.Typer()
|
|
9
10
|
|
|
@@ -17,6 +18,14 @@ def sync(
|
|
|
17
18
|
):
|
|
18
19
|
"""同步mysql中grid数据到redis"""
|
|
19
20
|
engine = get_database_engine(env_path)
|
|
21
|
+
redis = get_redis_client()
|
|
22
|
+
try:
|
|
20
23
|
|
|
21
|
-
|
|
22
|
-
|
|
24
|
+
async def _run():
|
|
25
|
+
await grid_open(engine, redis)
|
|
26
|
+
await grid_close(engine, redis)
|
|
27
|
+
|
|
28
|
+
asyncio.run(_run())
|
|
29
|
+
finally:
|
|
30
|
+
engine.dispose()
|
|
31
|
+
asyncio.run(redis.close())
|
|
@@ -86,7 +86,6 @@ async def csv_pd_redis(
|
|
|
86
86
|
# df[col] = pd.to_datetime(df[col], format="mixed")
|
|
87
87
|
# df[col] = dt_to_timestamp(df[col])
|
|
88
88
|
|
|
89
|
-
r = get_redis_client()
|
|
90
89
|
pipe = r.pipeline() # 启用 pipeline
|
|
91
90
|
count = 0
|
|
92
91
|
|
|
@@ -141,7 +140,13 @@ def count(
|
|
|
141
140
|
:type key_prefix: str
|
|
142
141
|
|
|
143
142
|
"""
|
|
144
|
-
|
|
143
|
+
async def _cmd():
|
|
144
|
+
try:
|
|
145
|
+
await count_async(key_prefix)
|
|
146
|
+
finally:
|
|
147
|
+
await r.close()
|
|
148
|
+
|
|
149
|
+
asyncio.run(_cmd())
|
|
145
150
|
|
|
146
151
|
|
|
147
152
|
@app.command()
|
|
@@ -173,14 +178,18 @@ def csv2redis(
|
|
|
173
178
|
:param fp: 说明
|
|
174
179
|
:type fp: csv文件路径
|
|
175
180
|
"""
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
181
|
+
async def _cmd():
|
|
182
|
+
try:
|
|
183
|
+
await csv_pd_redis(
|
|
184
|
+
id1,
|
|
185
|
+
id2,
|
|
186
|
+
kp,
|
|
187
|
+
fp,
|
|
188
|
+
)
|
|
189
|
+
finally:
|
|
190
|
+
await r.close()
|
|
191
|
+
|
|
192
|
+
asyncio.run(_cmd())
|
|
184
193
|
|
|
185
194
|
|
|
186
195
|
@app.command()
|
|
@@ -47,7 +47,7 @@ def spot_tickers(symbols: list, proxy: str | None = None):
|
|
|
47
47
|
tickers(url, symbols, proxy)
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
async def bitget_sf_open(engine: Engine):
|
|
50
|
+
async def bitget_sf_open(engine: Engine, redis_client=None):
|
|
51
51
|
query = "select * from bitget_sf where spot_open_usdt is not null and futures_open_usdt is not null and pnl is null and up_status = 0 and deleted_at is null;"
|
|
52
52
|
key_prefix = "bitget_sf"
|
|
53
53
|
table = "bitget_sf"
|
|
@@ -70,12 +70,13 @@ async def bitget_sf_open(engine: Engine):
|
|
|
70
70
|
"spot_close_at": "datetime64[ns]",
|
|
71
71
|
"futures_close_at": "datetime64[ns]",
|
|
72
72
|
},
|
|
73
|
+
redis_client=redis_client,
|
|
73
74
|
)
|
|
74
75
|
|
|
75
76
|
logger.info(f"🚀 bitget SF open count:({row_count})")
|
|
76
77
|
|
|
77
78
|
|
|
78
|
-
async def bitget_ff_open(engine: Engine):
|
|
79
|
+
async def bitget_ff_open(engine: Engine, redis_client=None):
|
|
79
80
|
key_prefix = "bitget_ff"
|
|
80
81
|
table = "bitget_ff"
|
|
81
82
|
query = f"select * from {table} where long_open_usdt is not null and short_open_usdt is not null and pnl is null and up_status = 0 and deleted_at is null;"
|
|
@@ -98,12 +99,13 @@ async def bitget_ff_open(engine: Engine):
|
|
|
98
99
|
"long_close_at": "datetime64[ns]",
|
|
99
100
|
"short_close_at": "datetime64[ns]",
|
|
100
101
|
},
|
|
102
|
+
redis_client=redis_client,
|
|
101
103
|
)
|
|
102
104
|
|
|
103
105
|
logger.info(f"💰 bitget FF open count:({row_count})")
|
|
104
106
|
|
|
105
107
|
|
|
106
|
-
async def bitget_sf_close(engine: Engine):
|
|
108
|
+
async def bitget_sf_close(engine: Engine, redis_client=None):
|
|
107
109
|
query = "select * from bitget_sf where pnl is not null and up_status in (0,1);"
|
|
108
110
|
key_prefix = "bitget_sf"
|
|
109
111
|
table = "bitget_sf"
|
|
@@ -126,12 +128,13 @@ async def bitget_sf_close(engine: Engine):
|
|
|
126
128
|
"spot_close_at": "datetime64[ns]",
|
|
127
129
|
"futures_close_at": "datetime64[ns]",
|
|
128
130
|
},
|
|
131
|
+
redis_client=redis_client,
|
|
129
132
|
)
|
|
130
133
|
|
|
131
134
|
logger.info(f"🚀 bitget SF close count:({row_count})")
|
|
132
135
|
|
|
133
136
|
|
|
134
|
-
async def bitget_ff_pending(engine: Engine):
|
|
137
|
+
async def bitget_ff_pending(engine: Engine, redis_client=None):
|
|
135
138
|
key_prefix = "bitget_ff"
|
|
136
139
|
table = "bitget_ff"
|
|
137
140
|
query = f"select * from {table} where pnl is null and (long_close_at is not null or short_close_at is not null) and up_status in (1);"
|
|
@@ -154,12 +157,13 @@ async def bitget_ff_pending(engine: Engine):
|
|
|
154
157
|
"long_close_at": "datetime64[ns]",
|
|
155
158
|
"short_close_at": "datetime64[ns]",
|
|
156
159
|
},
|
|
160
|
+
redis_client=redis_client,
|
|
157
161
|
)
|
|
158
162
|
|
|
159
163
|
logger.info(f"💰 bitget FF pending count:({row_count})")
|
|
160
164
|
|
|
161
165
|
|
|
162
|
-
async def bitget_ff_close(engine: Engine):
|
|
166
|
+
async def bitget_ff_close(engine: Engine, redis_client=None):
|
|
163
167
|
key_prefix = "bitget_ff"
|
|
164
168
|
table = "bitget_ff"
|
|
165
169
|
query = f"select * from {table} where pnl is not null and up_status in (0,1,2);"
|
|
@@ -182,12 +186,13 @@ async def bitget_ff_close(engine: Engine):
|
|
|
182
186
|
"long_close_at": "datetime64[ns]",
|
|
183
187
|
"short_close_at": "datetime64[ns]",
|
|
184
188
|
},
|
|
189
|
+
redis_client=redis_client,
|
|
185
190
|
)
|
|
186
191
|
|
|
187
192
|
logger.info(f"💰 bitget FF close count:({row_count})")
|
|
188
193
|
|
|
189
194
|
|
|
190
|
-
async def grid_open(engine: Engine):
|
|
195
|
+
async def grid_open(engine: Engine, redis_client=None):
|
|
191
196
|
query = "select * from bitget where ((cost is not null or benefit is not null) and profit is null) and up_status = 0 and order_id is not null and deleted_at is null;"
|
|
192
197
|
key_prefix = "bitget_grid"
|
|
193
198
|
table = "bitget"
|
|
@@ -206,12 +211,13 @@ async def grid_open(engine: Engine):
|
|
|
206
211
|
"open_at": "datetime64[ns]",
|
|
207
212
|
"close_at": "datetime64[ns]",
|
|
208
213
|
},
|
|
214
|
+
redis_client=redis_client,
|
|
209
215
|
)
|
|
210
216
|
|
|
211
217
|
logger.info(f"🧮 bitget grid open count:({row_count})")
|
|
212
218
|
|
|
213
219
|
|
|
214
|
-
async def grid_close(engine: Engine):
|
|
220
|
+
async def grid_close(engine: Engine, redis_client=None):
|
|
215
221
|
query = "select * from bitget where profit is not null and up_status in (0,1) and deleted_at is null;"
|
|
216
222
|
key_prefix = "bitget_grid"
|
|
217
223
|
table = "bitget"
|
|
@@ -230,6 +236,7 @@ async def grid_close(engine: Engine):
|
|
|
230
236
|
"open_at": "datetime64[ns]",
|
|
231
237
|
"close_at": "datetime64[ns]",
|
|
232
238
|
},
|
|
239
|
+
redis_client=redis_client,
|
|
233
240
|
)
|
|
234
241
|
|
|
235
242
|
logger.info(f"🧮 bitget grid close count:({row_count})")
|
|
@@ -4,7 +4,7 @@ from utils import logger
|
|
|
4
4
|
from utils.mysql import mysql_to_csv, mysql_to_redis
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
async def grid_open(engine: Engine):
|
|
7
|
+
async def grid_open(engine: Engine, redis_client=None):
|
|
8
8
|
query = "select * from gate where ((cost is not null or benefit is not null) and profit is null) and up_status = 0 and order_id is not null and deleted_at is null;"
|
|
9
9
|
key_prefix = "gate_grid"
|
|
10
10
|
table = "gate"
|
|
@@ -23,12 +23,13 @@ async def grid_open(engine: Engine):
|
|
|
23
23
|
"open_at": "datetime64[ns]",
|
|
24
24
|
"close_at": "datetime64[ns]",
|
|
25
25
|
},
|
|
26
|
+
redis_client=redis_client,
|
|
26
27
|
)
|
|
27
28
|
|
|
28
29
|
logger.info(f"🧮 gate open count:({row_count})")
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
async def grid_close(engine: Engine):
|
|
32
|
+
async def grid_close(engine: Engine, redis_client=None):
|
|
32
33
|
query = "select * from gate where profit is not null and up_status in (0,1) and deleted_at is null;"
|
|
33
34
|
key_prefix = "gate_grid"
|
|
34
35
|
table = "gate"
|
|
@@ -47,6 +48,7 @@ async def grid_close(engine: Engine):
|
|
|
47
48
|
"open_at": "datetime64[ns]",
|
|
48
49
|
"close_at": "datetime64[ns]",
|
|
49
50
|
},
|
|
51
|
+
redis_client=redis_client,
|
|
50
52
|
)
|
|
51
53
|
logger.info(f"🧮 gate close count:({row_count})")
|
|
52
54
|
|
|
@@ -4,7 +4,7 @@ from utils import logger
|
|
|
4
4
|
from utils.mysql import mysql_to_redis
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
async def grid_open(engine: Engine):
|
|
7
|
+
async def grid_open(engine: Engine, redis_client=None):
|
|
8
8
|
query = "select * from okx_spot where ((cost is not null or benefit is not null) and profit is null) and up_status = 0 and order_id is not null and deleted_at is null;"
|
|
9
9
|
key_prefix = "okx_grid"
|
|
10
10
|
table = "okx_spot"
|
|
@@ -23,12 +23,13 @@ async def grid_open(engine: Engine):
|
|
|
23
23
|
"open_at": "datetime64[ns]",
|
|
24
24
|
"close_at": "datetime64[ns]",
|
|
25
25
|
},
|
|
26
|
+
redis_client=redis_client,
|
|
26
27
|
)
|
|
27
28
|
|
|
28
29
|
logger.info(f"🧮 okx grid open count:({row_count})")
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
async def grid_close(engine: Engine):
|
|
32
|
+
async def grid_close(engine: Engine, redis_client=None):
|
|
32
33
|
query = "select * from okx_spot where profit is not null and up_status in (0,1) and deleted_at is null;"
|
|
33
34
|
key_prefix = "okx_grid"
|
|
34
35
|
table = "okx_spot"
|
|
@@ -47,6 +48,7 @@ async def grid_close(engine: Engine):
|
|
|
47
48
|
"open_at": "datetime64[ns]",
|
|
48
49
|
"close_at": "datetime64[ns]",
|
|
49
50
|
},
|
|
51
|
+
redis_client=redis_client,
|
|
50
52
|
)
|
|
51
53
|
|
|
52
54
|
logger.info(f"🧮 okx grid close count:({row_count})")
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
"babel>=2.17.0",
|
|
10
10
|
"cowsay>=6.1",
|
|
11
11
|
"dotenv>=0.9.9",
|
|
12
|
+
"etcd3>=0.12.0",
|
|
12
13
|
"fastapi>=0.127.0",
|
|
13
14
|
"SLPP>=1.2.3",
|
|
14
15
|
"numpy<2.0.0",
|
|
@@ -30,13 +31,14 @@
|
|
|
30
31
|
"netifaces-plus>=0.12.5",
|
|
31
32
|
"ifaddr>=0.2.0",
|
|
32
33
|
"telethon>=1.44.0",
|
|
34
|
+
"protobuf<4",
|
|
33
35
|
]
|
|
34
36
|
description = "My CLI"
|
|
35
37
|
license = { text = "MIT" }
|
|
36
38
|
name = "pymecli"
|
|
37
39
|
readme = "README.md"
|
|
38
40
|
requires-python = ">=3.10"
|
|
39
|
-
version = "0.5.
|
|
41
|
+
version = "0.5.7"
|
|
40
42
|
|
|
41
43
|
[project.urls]
|
|
42
44
|
Homepage = "https://pypi.org/project/pymecli"
|
|
@@ -46,6 +48,7 @@
|
|
|
46
48
|
# baidu = "cli.baidu:app"
|
|
47
49
|
bitget = "cli.bitget:app"
|
|
48
50
|
dst = "cli.dst:app"
|
|
51
|
+
etcdsync = "cli.etcd_mysql:app"
|
|
49
52
|
example = "cli.example:app"
|
|
50
53
|
fast = "cli.fast:typer_app"
|
|
51
54
|
gate = "cli.gate:app"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from urllib.parse import urlsplit
|
|
3
|
+
|
|
4
|
+
from dotenv import load_dotenv
|
|
5
|
+
|
|
6
|
+
import etcd3
|
|
7
|
+
|
|
8
|
+
DEFAULT_ETCD_URL = "http://localhost:2379"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _parse_etcd_url(url: str) -> tuple[str, int]:
|
|
12
|
+
"""
|
|
13
|
+
解析 ETCD_URL 为 (host, port)。
|
|
14
|
+
|
|
15
|
+
支持格式:
|
|
16
|
+
http://localhost:2379
|
|
17
|
+
https://127.0.0.1:2379
|
|
18
|
+
localhost:2379
|
|
19
|
+
127.0.0.1
|
|
20
|
+
"""
|
|
21
|
+
if "://" not in url:
|
|
22
|
+
url = f"http://{url}"
|
|
23
|
+
parts = urlsplit(url)
|
|
24
|
+
host = parts.hostname or "localhost"
|
|
25
|
+
port = parts.port or 2379
|
|
26
|
+
return host, port
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def get_etcd_client(env_path: str = ".env") -> "etcd3.Etcd3Client":
|
|
30
|
+
"""
|
|
31
|
+
创建 etcd v3 客户端。
|
|
32
|
+
|
|
33
|
+
从环境变量 ETCD_URL 读取连接地址,默认 http://localhost:2379。
|
|
34
|
+
"""
|
|
35
|
+
load_dotenv(env_path)
|
|
36
|
+
etcd_url = os.getenv("ETCD_URL", DEFAULT_ETCD_URL)
|
|
37
|
+
host, port = _parse_etcd_url(etcd_url)
|
|
38
|
+
|
|
39
|
+
return etcd3.client(host=host, port=port)
|
|
@@ -121,6 +121,7 @@ async def mysql_to_redis_and_csv(
|
|
|
121
121
|
d_column_names: list[str],
|
|
122
122
|
pd_dtype: dict | None = None,
|
|
123
123
|
del_column_names: list[str] = ["id", "created_at", "updated_at", "deleted_at"],
|
|
124
|
+
redis_client=None,
|
|
124
125
|
) -> int:
|
|
125
126
|
# 查询数据
|
|
126
127
|
df = pd.read_sql(query, engine, dtype=pd_dtype)
|
|
@@ -149,33 +150,38 @@ async def mysql_to_redis_and_csv(
|
|
|
149
150
|
# df[col] = dt_to_timestamp(pd.to_datetime(df[col], errors="coerce"))
|
|
150
151
|
|
|
151
152
|
# 数据写入redis
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
score =
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
153
|
+
own_client = redis_client is None
|
|
154
|
+
r = redis_client or get_redis_client()
|
|
155
|
+
try:
|
|
156
|
+
pipe = r.pipeline() # 启用 pipeline
|
|
157
|
+
count = 0
|
|
158
|
+
n1, n2 = d_column_names
|
|
159
|
+
|
|
160
|
+
for _, row in df.iterrows():
|
|
161
|
+
idx1 = row[n1]
|
|
162
|
+
idx2 = row[n2]
|
|
163
|
+
if not idx1 or not idx2:
|
|
164
|
+
raise ValueError("ERR:id行无效")
|
|
165
|
+
id = f"{idx1}_{idx2}"
|
|
166
|
+
key = f"{key_prefix}:{id}"
|
|
167
|
+
|
|
168
|
+
# 转换行数据为字典(处理 NaN 为 None 或空字符串)
|
|
169
|
+
row_dict = row.where(pd.notna(row), "").to_dict()
|
|
170
|
+
|
|
171
|
+
# 1. 写入完整数据到 Hash(自动覆盖)
|
|
172
|
+
pipe.hset(key, mapping=row_dict)
|
|
173
|
+
# 2. 写入 ZSet 索引:score = 开仓时间戳(空值用 0 兜底)
|
|
174
|
+
score = row["open_at"]
|
|
175
|
+
if score is None or pd.isna(score):
|
|
176
|
+
score = 0
|
|
177
|
+
pipe.zadd(f"by_time:{key_prefix}", {id: score})
|
|
178
|
+
count += 1
|
|
179
|
+
|
|
180
|
+
await pipe.execute()
|
|
181
|
+
logger.debug(f"🧱 to redis: {count}")
|
|
182
|
+
finally:
|
|
183
|
+
if own_client:
|
|
184
|
+
await r.close()
|
|
179
185
|
|
|
180
186
|
df.to_csv(
|
|
181
187
|
csv_fp,
|
|
@@ -223,6 +229,7 @@ async def mysql_to_redis(
|
|
|
223
229
|
d_column_names: list[str],
|
|
224
230
|
pd_dtype: dict | None = None,
|
|
225
231
|
del_column_names: list[str] = ["id", "created_at", "updated_at", "deleted_at"],
|
|
232
|
+
redis_client=None,
|
|
226
233
|
) -> int:
|
|
227
234
|
df = pd.read_sql(query, engine, dtype=pd_dtype)
|
|
228
235
|
df["open_at"] = df["open_at"].fillna(df["created_at"])
|
|
@@ -246,30 +253,35 @@ async def mysql_to_redis(
|
|
|
246
253
|
if col in df.columns:
|
|
247
254
|
df[col] = dt_to_timestamp(df[col])
|
|
248
255
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
score =
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
256
|
+
own_client = redis_client is None
|
|
257
|
+
r = redis_client or get_redis_client()
|
|
258
|
+
try:
|
|
259
|
+
pipe = r.pipeline()
|
|
260
|
+
count = 0
|
|
261
|
+
n1, n2 = d_column_names
|
|
262
|
+
|
|
263
|
+
for _, row in df.iterrows():
|
|
264
|
+
idx1 = row[n1]
|
|
265
|
+
idx2 = row[n2]
|
|
266
|
+
if not idx1 or not idx2:
|
|
267
|
+
raise ValueError("ERR:id行无效")
|
|
268
|
+
id = f"{idx1}_{idx2}"
|
|
269
|
+
key = f"{key_prefix}:{id}"
|
|
270
|
+
|
|
271
|
+
row_dict = row.where(pd.notna(row), "").to_dict()
|
|
272
|
+
|
|
273
|
+
pipe.hset(key, mapping=row_dict)
|
|
274
|
+
score = row["open_at"]
|
|
275
|
+
if score is None or pd.isna(score):
|
|
276
|
+
score = 0
|
|
277
|
+
pipe.zadd(f"by_time:{key_prefix}", {id: score})
|
|
278
|
+
count += 1
|
|
279
|
+
|
|
280
|
+
await pipe.execute()
|
|
281
|
+
logger.debug(f"🧱 to redis: {count}")
|
|
282
|
+
finally:
|
|
283
|
+
if own_client:
|
|
284
|
+
await r.close()
|
|
273
285
|
|
|
274
286
|
return _update_status(engine, table, ids, update_status)
|
|
275
287
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|