nlpertools 1.0.8__py3-none-any.whl → 1.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nlpertools/__init__.py +1 -1
- nlpertools/cli.py +60 -4
- nlpertools/draw/draw.py +5 -7
- nlpertools/io/dir.py +2 -2
- nlpertools/io/file.py +2 -2
- nlpertools/ml.py +22 -0
- nlpertools/other.py +17 -0
- {nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/METADATA +8 -6
- {nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/RECORD +13 -13
- {nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/WHEEL +1 -1
- {nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/LICENSE +0 -0
- {nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/entry_points.txt +0 -0
- {nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/top_level.txt +0 -0
nlpertools/__init__.py
CHANGED
nlpertools/cli.py
CHANGED
@@ -3,8 +3,6 @@ import os
|
|
3
3
|
import uuid
|
4
4
|
import sys
|
5
5
|
|
6
|
-
import pyotp
|
7
|
-
|
8
6
|
"""
|
9
7
|
如何Debug cli.py
|
10
8
|
"""
|
@@ -49,6 +47,7 @@ def get_mac_address():
|
|
49
47
|
|
50
48
|
|
51
49
|
def get_2af_value(key):
|
50
|
+
import pyotp
|
52
51
|
"""
|
53
52
|
key应该是7位的
|
54
53
|
"""
|
@@ -57,14 +56,67 @@ def get_2af_value(key):
|
|
57
56
|
print(totp.now())
|
58
57
|
|
59
58
|
|
59
|
+
def start_gpu_usage_notify_server():
|
60
|
+
from flask import Flask
|
61
|
+
|
62
|
+
app = Flask(__name__)
|
63
|
+
|
64
|
+
@app.route("/notify", methods=["GET"])
|
65
|
+
def notify():
|
66
|
+
# 这里可以根据需要动态生成通知内容
|
67
|
+
usage = os.popen("nvidia-smi --query-gpu=memory.used --format=csv").read().split("\n")[1:]
|
68
|
+
res = 0
|
69
|
+
for edx, each in enumerate(usage):
|
70
|
+
if each.startswith("0"):
|
71
|
+
res += 1
|
72
|
+
print(res)
|
73
|
+
return str(res), 200
|
74
|
+
|
75
|
+
app.run(host="0.0.0.0", port=5000)
|
76
|
+
|
77
|
+
|
78
|
+
def start_gpu_usage_notify_client():
|
79
|
+
import requests
|
80
|
+
from plyer import notification
|
81
|
+
import time
|
82
|
+
|
83
|
+
SERVER_URL = 'http://127.0.0.1:5000/notify' # 服务器的 API 地址
|
84
|
+
|
85
|
+
def notify(text):
|
86
|
+
# 使用 plyer 发送通知
|
87
|
+
notification.notify(
|
88
|
+
title='远程通知',
|
89
|
+
message=text,
|
90
|
+
timeout=10 # 10秒的通知显示时间
|
91
|
+
)
|
92
|
+
|
93
|
+
"""定时轮询服务器获取通知"""
|
94
|
+
while True:
|
95
|
+
try:
|
96
|
+
response = requests.get(SERVER_URL)
|
97
|
+
if response.status_code == 200:
|
98
|
+
num = int(response.text)
|
99
|
+
if num > 0:
|
100
|
+
notify(f"服务器有{num}张卡")
|
101
|
+
print(f"服务器有{num}张卡")
|
102
|
+
else:
|
103
|
+
print("服务器没有新通知")
|
104
|
+
except Exception as e:
|
105
|
+
print(f"与服务器连接失败: {e}")
|
106
|
+
|
107
|
+
time.sleep(1)
|
108
|
+
|
109
|
+
|
60
110
|
def main():
|
61
111
|
parser = argparse.ArgumentParser(description="CLI tool for git operations and getting MAC address.")
|
62
112
|
parser.add_argument('--gitpush', action='store_true', help='Perform git push operation.')
|
63
|
-
parser.add_argument('--gitpull', action='store_true', help='Perform git
|
113
|
+
parser.add_argument('--gitpull', action='store_true', help='Perform git pull operation.')
|
64
114
|
parser.add_argument('--mac_address', action='store_true', help='Get the MAC address.')
|
65
115
|
|
66
116
|
parser.add_argument('--get_2fa', action='store_true', help='Get the 2fa value.')
|
67
117
|
parser.add_argument('--get_2fa_key', type=str, help='Get the 2fa value.')
|
118
|
+
parser.add_argument('--monitor_gpu_cli', action='store_true', help='Get the 2fa value.')
|
119
|
+
parser.add_argument('--monitor_gpu_ser', action='store_true', help='Get the 2fa value.')
|
68
120
|
|
69
121
|
args = parser.parse_args()
|
70
122
|
|
@@ -74,13 +126,17 @@ def main():
|
|
74
126
|
git_pull()
|
75
127
|
elif args.mac_address:
|
76
128
|
get_mac_address()
|
129
|
+
elif args.monitor_gpu_cli:
|
130
|
+
start_gpu_usage_notify_client()
|
131
|
+
elif args.monitor_gpu_ser:
|
132
|
+
start_gpu_usage_notify_server()
|
77
133
|
elif args.get_2fa:
|
78
134
|
if args.get_2fa_key:
|
79
135
|
get_2af_value(args.get_2fa_key)
|
80
136
|
else:
|
81
137
|
print("Please provide a key as an argument.")
|
82
138
|
else:
|
83
|
-
print("No operation specified.
|
139
|
+
print("No operation specified.")
|
84
140
|
|
85
141
|
|
86
142
|
if __name__ == '__main__':
|
nlpertools/draw/draw.py
CHANGED
@@ -20,7 +20,7 @@ def confused_matrix(confuse_matrix):
|
|
20
20
|
f.savefig('tmp.jpg', bbox_inches='tight')
|
21
21
|
|
22
22
|
|
23
|
-
def plot_histogram(data, bin_size):
|
23
|
+
def plot_histogram(data, bin_size, max_bin):
|
24
24
|
"""
|
25
25
|
画直方图,超过1000的统一按1000算
|
26
26
|
:param data:
|
@@ -33,15 +33,15 @@ def plot_histogram(data, bin_size):
|
|
33
33
|
from matplotlib.ticker import MaxNLocator
|
34
34
|
# 将超过1000的值改为1000
|
35
35
|
def process_lengths(data):
|
36
|
-
return [length if length <=
|
36
|
+
return [length if length <= max_bin else max_bin + 3 for length in data]
|
37
37
|
|
38
38
|
# 前闭后开
|
39
|
-
min_num, max_num = 0, 1000
|
39
|
+
# min_num, max_num = 0, 1000
|
40
40
|
# min_num, max_num = min(data), max(data)
|
41
41
|
|
42
42
|
plt.figure(figsize=(12, 8))
|
43
43
|
processed_data = process_lengths(data)
|
44
|
-
bins = np.arange(0,
|
44
|
+
bins = np.arange(0, max_bin + 2 * bin_size, bin_size)
|
45
45
|
# 绘制直方图
|
46
46
|
n, new_bins, patches = plt.hist(processed_data, bins=bins, edgecolor='black', color='skyblue', alpha=0.7,
|
47
47
|
linewidth=0)
|
@@ -60,10 +60,8 @@ def plot_histogram(data, bin_size):
|
|
60
60
|
plt.xlabel('module line number', fontsize=14)
|
61
61
|
plt.ylabel('frequency', fontsize=14)
|
62
62
|
|
63
|
-
# 添加网格
|
64
63
|
plt.grid(True, linestyle='--', alpha=0.6)
|
65
64
|
|
66
|
-
# 美化x轴和y轴的刻度
|
67
65
|
plt.xticks(fontsize=12)
|
68
66
|
plt.yticks(fontsize=12)
|
69
67
|
|
@@ -80,4 +78,4 @@ if __name__ == '__main__':
|
|
80
78
|
# 调整区间大小
|
81
79
|
bin_size = 50
|
82
80
|
# 示例模块长度数据
|
83
|
-
plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size)
|
81
|
+
plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size, max_bin=1000)
|
nlpertools/io/dir.py
CHANGED
@@ -46,7 +46,7 @@ def get_filename(path, suffix=True) -> str:
|
|
46
46
|
return filename
|
47
47
|
|
48
48
|
|
49
|
-
def
|
49
|
+
def listdir(dir_name, including_dir=True):
|
50
50
|
filenames = os.listdir(dir_name)
|
51
51
|
if including_dir:
|
52
52
|
return [os.path.join(dir_name, filename) for filename in filenames]
|
@@ -54,7 +54,7 @@ def j_listdir(dir_name, including_dir=True):
|
|
54
54
|
return list(filenames)
|
55
55
|
|
56
56
|
|
57
|
-
def
|
57
|
+
def listdir_yield(dir_name, including_dir=True):
|
58
58
|
filenames = os.listdir(dir_name)
|
59
59
|
for filename in filenames:
|
60
60
|
if including_dir:
|
nlpertools/io/file.py
CHANGED
@@ -241,12 +241,12 @@ def load_from_jsonl(path):
|
|
241
241
|
return corpus
|
242
242
|
|
243
243
|
|
244
|
-
def
|
244
|
+
def save_pkl(data, path):
|
245
245
|
with open(path, 'wb') as f:
|
246
246
|
pickle.dump(data, f)
|
247
247
|
|
248
248
|
|
249
|
-
def
|
249
|
+
def load_pkl(path):
|
250
250
|
with open(path, 'rb') as f:
|
251
251
|
data = pickle.load(f)
|
252
252
|
return data
|
nlpertools/ml.py
CHANGED
@@ -17,6 +17,28 @@ from .io.file import readtxt_list_all_strip, writetxt_w_list, save_to_csv
|
|
17
17
|
from .utils.package import *
|
18
18
|
|
19
19
|
|
20
|
+
def estimate_pass_at_k(num_samples:list, num_correct:list, k):
|
21
|
+
"""
|
22
|
+
copy from https://huggingface.co/spaces/evaluate-metric/code_eval/blob/main/code_eval.py
|
23
|
+
num_samples: list
|
24
|
+
"""
|
25
|
+
"""Estimates pass@k of each problem and returns them in an array."""
|
26
|
+
|
27
|
+
def estimator(n: int, c: int, k: int) -> float:
|
28
|
+
"""Calculates 1 - comb(n - c, k) / comb(n, k)."""
|
29
|
+
if n - c < k:
|
30
|
+
return 1.0
|
31
|
+
return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
|
32
|
+
|
33
|
+
if isinstance(num_samples, int):
|
34
|
+
num_samples_it = itertools.repeat(num_samples, len(num_correct))
|
35
|
+
else:
|
36
|
+
assert len(num_samples) == len(num_correct)
|
37
|
+
num_samples_it = iter(num_samples)
|
38
|
+
|
39
|
+
return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
|
40
|
+
|
41
|
+
|
20
42
|
def calc_llm_train_activation_memory(
|
21
43
|
model_name, sequence_length, batch_size, hidden_dim, lay_number, attention_heads_num, gpu_num=1
|
22
44
|
):
|
nlpertools/other.py
CHANGED
@@ -30,6 +30,21 @@ ENGLISH_PUNCTUATION = list(',.;:\'"!?<>()')
|
|
30
30
|
OTHER_PUNCTUATION = list('!@#$%^&*')
|
31
31
|
|
32
32
|
|
33
|
+
def setup_logging(log_file):
|
34
|
+
"""
|
35
|
+
Set up logging configuration.
|
36
|
+
|
37
|
+
Args:
|
38
|
+
log_file (str): Path to the log file.
|
39
|
+
"""
|
40
|
+
logging.basicConfig(
|
41
|
+
filename=log_file,
|
42
|
+
level=logging.INFO,
|
43
|
+
format='%(asctime)s - %(levelname)s - %(message)s',
|
44
|
+
datefmt='%Y-%m-%d %H:%M:%S'
|
45
|
+
)
|
46
|
+
|
47
|
+
|
33
48
|
def get_diff_parts(str1, str2):
|
34
49
|
# 创建一个 SequenceMatcher 对象
|
35
50
|
matcher = difflib.SequenceMatcher(None, str1, str2)
|
@@ -361,10 +376,12 @@ def unsqueeze_list(flatten_list, each_element_len):
|
|
361
376
|
range(len(flatten_list) // each_element_len)]
|
362
377
|
return two_dim_list
|
363
378
|
|
379
|
+
|
364
380
|
def split_list(input_list, chunk_size):
|
365
381
|
# 使用列表推导式将列表分割成二维数组
|
366
382
|
return [input_list[i:i + chunk_size] for i in range(0, len(input_list), chunk_size)]
|
367
383
|
|
384
|
+
|
368
385
|
def auto_close():
|
369
386
|
"""
|
370
387
|
针对企业微信15分钟会显示离开的机制,假装自己还在上班
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.2
|
2
2
|
Name: nlpertools
|
3
|
-
Version: 1.0.
|
3
|
+
Version: 1.0.9
|
4
4
|
Summary: A small package about small basic IO operation when coding
|
5
5
|
Home-page: https://github.com/lvzii/nlpertools
|
6
6
|
Author: youshuJi
|
@@ -17,6 +17,8 @@ Requires-Dist: pandas
|
|
17
17
|
Requires-Dist: psutil
|
18
18
|
Provides-Extra: torch
|
19
19
|
Requires-Dist: torch; extra == "torch"
|
20
|
+
Dynamic: provides-extra
|
21
|
+
Dynamic: requires-dist
|
20
22
|
|
21
23
|
<div align="center">
|
22
24
|
<h4 align="center">
|
@@ -35,7 +37,7 @@ Requires-Dist: torch; extra == "torch"
|
|
35
37
|
|
36
38
|
它解决了什么问题:
|
37
39
|
|
38
|
-
- 很多函数是记不住的,
|
40
|
+
- 很多函数是记不住的, 每次写都要~~搜~~问大模型 ,例如pandas排序
|
39
41
|
- 刷题的时候,树结构的题目很难调试
|
40
42
|
|
41
43
|
|
@@ -111,11 +113,11 @@ b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
|
|
111
113
|
|
112
114
|
```bash
|
113
115
|
# 生成pypi双因素认证的实时密钥(需要提供key)
|
114
|
-
python -m nlpertools.get_2fa your_key
|
116
|
+
python -m nlpertools.cli --get_2fa --get_2fa_key your_key
|
115
117
|
|
116
118
|
## git
|
117
|
-
python nlpertools.cli --git_push
|
118
|
-
python nlpertools.cli --git_pull
|
119
|
+
python -m nlpertools.cli --git_push
|
120
|
+
python -m nlpertools.cli --git_pull
|
119
121
|
|
120
122
|
# 以下功能被nvitop替代,不推荐使用
|
121
123
|
## 监控gpu显存
|
@@ -1,14 +1,14 @@
|
|
1
|
-
nlpertools/__init__.py,sha256=
|
2
|
-
nlpertools/cli.py,sha256=
|
1
|
+
nlpertools/__init__.py,sha256=5ka-NeGW2AUDJ4YZ12DD64xcxuxf9PlQUurxDp5DHbQ,483
|
2
|
+
nlpertools/cli.py,sha256=4Ik1NyFaoZpZLsYLAFRLk6xuYQk0IvexPr1Ieq08viE,3932
|
3
3
|
nlpertools/data_client.py,sha256=esX8lUQrTui4uVkqPfhpHVok7Eq6ywpuemKjLeqoglc,14674
|
4
4
|
nlpertools/dataprocess.py,sha256=v1mobuYN7I3dT6xIKlNOHVtcg31YtjF6FwNPTxeBFFY,23153
|
5
5
|
nlpertools/default_db_config.yml,sha256=E1K9k_xzXVlsf-HJQh8kyHXHYuvTpD12jD4Hfe5rUk8,606
|
6
6
|
nlpertools/get_2fa.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
7
|
-
nlpertools/ml.py,sha256=
|
7
|
+
nlpertools/ml.py,sha256=qhUBCLuHfcFy8g5ZHNGYq4eH2vYWiGetyKucv8n60-A,18523
|
8
8
|
nlpertools/movie.py,sha256=rkyOnAXdsbWfMSbi1sE1VNRT7f66Hp9BnZsN_58Afmw,897
|
9
9
|
nlpertools/nlpertools_config.yml,sha256=ksXejxFs7pxR47tNAsrN88_4gvq9PCA2ZMO07H-dJXY,26
|
10
10
|
nlpertools/open_api.py,sha256=uyTY00OUlM57Cn0Wm0yZXcIS8vAszy9rKnDMBEWfWJM,1744
|
11
|
-
nlpertools/other.py,sha256=
|
11
|
+
nlpertools/other.py,sha256=JWJiXHRI8mhiUV3k4CZ4kQQS9QN3mw67SmGgTqZFtjs,15026
|
12
12
|
nlpertools/pic.py,sha256=13aaFJh3USGYGs4Y9tAKTvWjmdQR4YDjl3LlIhJheOA,9906
|
13
13
|
nlpertools/plugin.py,sha256=LB7j9GdoQi6TITddH-6EglHlOa0WIHLUT7X5vb_aIZY,1168
|
14
14
|
nlpertools/reminder.py,sha256=wiXwZQmxMck5vY3EvG8_oakP3FAdjGTikAIOiTPUQrs,2977
|
@@ -25,11 +25,11 @@ nlpertools/algo/union.py,sha256=0l7lGZbw1qIfW1z5TE8Oo3tybL1bKIP5rzpa5ZT-vLQ,249
|
|
25
25
|
nlpertools/data_structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
26
|
nlpertools/data_structure/base_structure.py,sha256=gVUvJZ5jsCAswRETTpMwcEjLKoageWiTuCKNEwIWKWk,2641
|
27
27
|
nlpertools/draw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
28
|
-
nlpertools/draw/draw.py,sha256=
|
28
|
+
nlpertools/draw/draw.py,sha256=19dskkr0wrgczxPJnphEszliwYshEh5SjD8Zz07nlk0,2615
|
29
29
|
nlpertools/draw/math_func.py,sha256=0NQ22Dfi9DFG6Bg_hXnCT27w65-dqpOOIgZX7oUIW-Q,881
|
30
30
|
nlpertools/io/__init__.py,sha256=YMuKtC2Ddh5dL5MvXjyUKYOOuqzFYUhBPFaP2kyFG9I,68
|
31
|
-
nlpertools/io/dir.py,sha256=
|
32
|
-
nlpertools/io/file.py,sha256=
|
31
|
+
nlpertools/io/dir.py,sha256=FPY62COQN8Ji72pk0dYRoXkrORYaUlybKNcL4474uUI,2263
|
32
|
+
nlpertools/io/file.py,sha256=mLWl09IEi0rWPN4tTq3LwdYMvAjj4e_QsjEMhufuPPo,7192
|
33
33
|
nlpertools/monitor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
34
34
|
nlpertools/monitor/gpu.py,sha256=M59O6i0hlew7AzXZlaVZqbZA5IR93OhBY2WI0-T_HtY,531
|
35
35
|
nlpertools/monitor/memory.py,sha256=9t6q9BC8VVx4o3G4sBCn7IoQRx272zMPjSnL3yvTBAQ,657
|
@@ -41,9 +41,9 @@ nlpertools/utils/package.py,sha256=wLg_M8j7Y6ReRjWHWCWoZJHrzEwuAr9TyG2jvb7OQCo,3
|
|
41
41
|
nlpertools/utils/package_v1.py,sha256=sqgFb-zbTdMd5ziJLY6YUPqR49qUNZjxBH35DnyR5Wg,3542
|
42
42
|
nlpertools/utils/package_v2.py,sha256=WOcsguWfUd4XSAfmPgCtL8HtUbqJ6GRSMHb0OsB47r0,3932
|
43
43
|
nlpertools_helper/__init__.py,sha256=obxRUdZDctvcvK_iA1Dx2HmQFMlMzJto-xDPryq1lJ0,198
|
44
|
-
nlpertools-1.0.
|
45
|
-
nlpertools-1.0.
|
46
|
-
nlpertools-1.0.
|
47
|
-
nlpertools-1.0.
|
48
|
-
nlpertools-1.0.
|
49
|
-
nlpertools-1.0.
|
44
|
+
nlpertools-1.0.9.dist-info/LICENSE,sha256=SBcMozykvTbZJ--MqSiKUmHLLROdnr25V70xCQgEwqw,11331
|
45
|
+
nlpertools-1.0.9.dist-info/METADATA,sha256=lcKmxc7_mtYH47mPj8UHOM8-5T5YtrDwhHWVZkfHZXU,3330
|
46
|
+
nlpertools-1.0.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
47
|
+
nlpertools-1.0.9.dist-info/entry_points.txt,sha256=XEazQ4vUwJMoMAgAwk1Lq4PRQGklPkPBaFkiP0zN_JE,45
|
48
|
+
nlpertools-1.0.9.dist-info/top_level.txt,sha256=_4q4MIFvMr4cAUbhWKWYdRXIXsF4PJDg4BUsZvgk94s,29
|
49
|
+
nlpertools-1.0.9.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|