nlpertools 1.0.8__py3-none-any.whl → 1.0.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
nlpertools/__init__.py CHANGED
@@ -20,4 +20,4 @@ from .cli import *
20
20
 
21
21
 
22
22
 
23
- __version__ = '1.0.8'
23
+ __version__ = '1.0.9'
nlpertools/cli.py CHANGED
@@ -3,8 +3,6 @@ import os
3
3
  import uuid
4
4
  import sys
5
5
 
6
- import pyotp
7
-
8
6
  """
9
7
  如何Debug cli.py
10
8
  """
@@ -49,6 +47,7 @@ def get_mac_address():
49
47
 
50
48
 
51
49
  def get_2af_value(key):
50
+ import pyotp
52
51
  """
53
52
  key应该是7位的
54
53
  """
@@ -57,14 +56,67 @@ def get_2af_value(key):
57
56
  print(totp.now())
58
57
 
59
58
 
59
+ def start_gpu_usage_notify_server():
60
+ from flask import Flask
61
+
62
+ app = Flask(__name__)
63
+
64
+ @app.route("/notify", methods=["GET"])
65
+ def notify():
66
+ # 这里可以根据需要动态生成通知内容
67
+ usage = os.popen("nvidia-smi --query-gpu=memory.used --format=csv").read().split("\n")[1:]
68
+ res = 0
69
+ for edx, each in enumerate(usage):
70
+ if each.startswith("0"):
71
+ res += 1
72
+ print(res)
73
+ return str(res), 200
74
+
75
+ app.run(host="0.0.0.0", port=5000)
76
+
77
+
78
+ def start_gpu_usage_notify_client():
79
+ import requests
80
+ from plyer import notification
81
+ import time
82
+
83
+ SERVER_URL = 'http://127.0.0.1:5000/notify' # 服务器的 API 地址
84
+
85
+ def notify(text):
86
+ # 使用 plyer 发送通知
87
+ notification.notify(
88
+ title='远程通知',
89
+ message=text,
90
+ timeout=10 # 10秒的通知显示时间
91
+ )
92
+
93
+ """定时轮询服务器获取通知"""
94
+ while True:
95
+ try:
96
+ response = requests.get(SERVER_URL)
97
+ if response.status_code == 200:
98
+ num = int(response.text)
99
+ if num > 0:
100
+ notify(f"服务器有{num}张卡")
101
+ print(f"服务器有{num}张卡")
102
+ else:
103
+ print("服务器没有新通知")
104
+ except Exception as e:
105
+ print(f"与服务器连接失败: {e}")
106
+
107
+ time.sleep(1)
108
+
109
+
60
110
  def main():
61
111
  parser = argparse.ArgumentParser(description="CLI tool for git operations and getting MAC address.")
62
112
  parser.add_argument('--gitpush', action='store_true', help='Perform git push operation.')
63
- parser.add_argument('--gitpull', action='store_true', help='Perform git push operation.')
113
+ parser.add_argument('--gitpull', action='store_true', help='Perform git pull operation.')
64
114
  parser.add_argument('--mac_address', action='store_true', help='Get the MAC address.')
65
115
 
66
116
  parser.add_argument('--get_2fa', action='store_true', help='Get the 2fa value.')
67
117
  parser.add_argument('--get_2fa_key', type=str, help='Get the 2fa value.')
118
+ parser.add_argument('--monitor_gpu_cli', action='store_true', help='Get the 2fa value.')
119
+ parser.add_argument('--monitor_gpu_ser', action='store_true', help='Get the 2fa value.')
68
120
 
69
121
  args = parser.parse_args()
70
122
 
@@ -74,13 +126,17 @@ def main():
74
126
  git_pull()
75
127
  elif args.mac_address:
76
128
  get_mac_address()
129
+ elif args.monitor_gpu_cli:
130
+ start_gpu_usage_notify_client()
131
+ elif args.monitor_gpu_ser:
132
+ start_gpu_usage_notify_server()
77
133
  elif args.get_2fa:
78
134
  if args.get_2fa_key:
79
135
  get_2af_value(args.get_2fa_key)
80
136
  else:
81
137
  print("Please provide a key as an argument.")
82
138
  else:
83
- print("No operation specified. Use --gitpush or --get_mac_address.")
139
+ print("No operation specified.")
84
140
 
85
141
 
86
142
  if __name__ == '__main__':
nlpertools/draw/draw.py CHANGED
@@ -20,7 +20,7 @@ def confused_matrix(confuse_matrix):
20
20
  f.savefig('tmp.jpg', bbox_inches='tight')
21
21
 
22
22
 
23
- def plot_histogram(data, bin_size):
23
+ def plot_histogram(data, bin_size, max_bin):
24
24
  """
25
25
  画直方图,超过1000的统一按1000算
26
26
  :param data:
@@ -33,15 +33,15 @@ def plot_histogram(data, bin_size):
33
33
  from matplotlib.ticker import MaxNLocator
34
34
  # 将超过1000的值改为1000
35
35
  def process_lengths(data):
36
- return [length if length <= 1000 else 1003 for length in data]
36
+ return [length if length <= max_bin else max_bin + 3 for length in data]
37
37
 
38
38
  # 前闭后开
39
- min_num, max_num = 0, 1000
39
+ # min_num, max_num = 0, 1000
40
40
  # min_num, max_num = min(data), max(data)
41
41
 
42
42
  plt.figure(figsize=(12, 8))
43
43
  processed_data = process_lengths(data)
44
- bins = np.arange(0, 1000 + 2 * bin_size, bin_size)
44
+ bins = np.arange(0, max_bin + 2 * bin_size, bin_size)
45
45
  # 绘制直方图
46
46
  n, new_bins, patches = plt.hist(processed_data, bins=bins, edgecolor='black', color='skyblue', alpha=0.7,
47
47
  linewidth=0)
@@ -60,10 +60,8 @@ def plot_histogram(data, bin_size):
60
60
  plt.xlabel('module line number', fontsize=14)
61
61
  plt.ylabel('frequency', fontsize=14)
62
62
 
63
- # 添加网格
64
63
  plt.grid(True, linestyle='--', alpha=0.6)
65
64
 
66
- # 美化x轴和y轴的刻度
67
65
  plt.xticks(fontsize=12)
68
66
  plt.yticks(fontsize=12)
69
67
 
@@ -80,4 +78,4 @@ if __name__ == '__main__':
80
78
  # 调整区间大小
81
79
  bin_size = 50
82
80
  # 示例模块长度数据
83
- plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size)
81
+ plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size, max_bin=1000)
nlpertools/io/dir.py CHANGED
@@ -46,7 +46,7 @@ def get_filename(path, suffix=True) -> str:
46
46
  return filename
47
47
 
48
48
 
49
- def j_listdir(dir_name, including_dir=True):
49
+ def listdir(dir_name, including_dir=True):
50
50
  filenames = os.listdir(dir_name)
51
51
  if including_dir:
52
52
  return [os.path.join(dir_name, filename) for filename in filenames]
@@ -54,7 +54,7 @@ def j_listdir(dir_name, including_dir=True):
54
54
  return list(filenames)
55
55
 
56
56
 
57
- def j_listdir_yield(dir_name, including_dir=True):
57
+ def listdir_yield(dir_name, including_dir=True):
58
58
  filenames = os.listdir(dir_name)
59
59
  for filename in filenames:
60
60
  if including_dir:
nlpertools/io/file.py CHANGED
@@ -241,12 +241,12 @@ def load_from_jsonl(path):
241
241
  return corpus
242
242
 
243
243
 
244
- def pickle_save(data, path):
244
+ def save_pkl(data, path):
245
245
  with open(path, 'wb') as f:
246
246
  pickle.dump(data, f)
247
247
 
248
248
 
249
- def pickle_load(path):
249
+ def load_pkl(path):
250
250
  with open(path, 'rb') as f:
251
251
  data = pickle.load(f)
252
252
  return data
nlpertools/ml.py CHANGED
@@ -17,6 +17,28 @@ from .io.file import readtxt_list_all_strip, writetxt_w_list, save_to_csv
17
17
  from .utils.package import *
18
18
 
19
19
 
20
+ def estimate_pass_at_k(num_samples:list, num_correct:list, k):
21
+ """
22
+ copy from https://huggingface.co/spaces/evaluate-metric/code_eval/blob/main/code_eval.py
23
+ num_samples: list
24
+ """
25
+ """Estimates pass@k of each problem and returns them in an array."""
26
+
27
+ def estimator(n: int, c: int, k: int) -> float:
28
+ """Calculates 1 - comb(n - c, k) / comb(n, k)."""
29
+ if n - c < k:
30
+ return 1.0
31
+ return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
32
+
33
+ if isinstance(num_samples, int):
34
+ num_samples_it = itertools.repeat(num_samples, len(num_correct))
35
+ else:
36
+ assert len(num_samples) == len(num_correct)
37
+ num_samples_it = iter(num_samples)
38
+
39
+ return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
40
+
41
+
20
42
  def calc_llm_train_activation_memory(
21
43
  model_name, sequence_length, batch_size, hidden_dim, lay_number, attention_heads_num, gpu_num=1
22
44
  ):
nlpertools/other.py CHANGED
@@ -30,6 +30,21 @@ ENGLISH_PUNCTUATION = list(',.;:\'"!?<>()')
30
30
  OTHER_PUNCTUATION = list('!@#$%^&*')
31
31
 
32
32
 
33
+ def setup_logging(log_file):
34
+ """
35
+ Set up logging configuration.
36
+
37
+ Args:
38
+ log_file (str): Path to the log file.
39
+ """
40
+ logging.basicConfig(
41
+ filename=log_file,
42
+ level=logging.INFO,
43
+ format='%(asctime)s - %(levelname)s - %(message)s',
44
+ datefmt='%Y-%m-%d %H:%M:%S'
45
+ )
46
+
47
+
33
48
  def get_diff_parts(str1, str2):
34
49
  # 创建一个 SequenceMatcher 对象
35
50
  matcher = difflib.SequenceMatcher(None, str1, str2)
@@ -361,10 +376,12 @@ def unsqueeze_list(flatten_list, each_element_len):
361
376
  range(len(flatten_list) // each_element_len)]
362
377
  return two_dim_list
363
378
 
379
+
364
380
  def split_list(input_list, chunk_size):
365
381
  # 使用列表推导式将列表分割成二维数组
366
382
  return [input_list[i:i + chunk_size] for i in range(0, len(input_list), chunk_size)]
367
383
 
384
+
368
385
  def auto_close():
369
386
  """
370
387
  针对企业微信15分钟会显示离开的机制,假装自己还在上班
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: nlpertools
3
- Version: 1.0.8
3
+ Version: 1.0.9
4
4
  Summary: A small package about small basic IO operation when coding
5
5
  Home-page: https://github.com/lvzii/nlpertools
6
6
  Author: youshuJi
@@ -17,6 +17,8 @@ Requires-Dist: pandas
17
17
  Requires-Dist: psutil
18
18
  Provides-Extra: torch
19
19
  Requires-Dist: torch; extra == "torch"
20
+ Dynamic: provides-extra
21
+ Dynamic: requires-dist
20
22
 
21
23
  <div align="center">
22
24
  <h4 align="center">
@@ -35,7 +37,7 @@ Requires-Dist: torch; extra == "torch"
35
37
 
36
38
  它解决了什么问题:
37
39
 
38
- - 很多函数是记不住的, ~~每次写每次都要搜~~ 每次都要问大模型 ,例如pandas排序
40
+ - 很多函数是记不住的, 每次写都要~~搜~~问大模型 ,例如pandas排序
39
41
  - 刷题的时候,树结构的题目很难调试
40
42
 
41
43
 
@@ -111,11 +113,11 @@ b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
111
113
 
112
114
  ```bash
113
115
  # 生成pypi双因素认证的实时密钥(需要提供key)
114
- python -m nlpertools.get_2fa your_key
116
+ python -m nlpertools.cli --get_2fa --get_2fa_key your_key
115
117
 
116
118
  ## git
117
- python nlpertools.cli --git_push
118
- python nlpertools.cli --git_pull
119
+ python -m nlpertools.cli --git_push
120
+ python -m nlpertools.cli --git_pull
119
121
 
120
122
  # 以下功能被nvitop替代,不推荐使用
121
123
  ## 监控gpu显存
@@ -1,14 +1,14 @@
1
- nlpertools/__init__.py,sha256=h7JJEN_JRn3iKcqIcaFgYtAjP90XiT1KILrm8utoHvQ,483
2
- nlpertools/cli.py,sha256=xDl_tWl9pfqQ3PUdd7oesvgM2FVqnaw8dFFliEX5c4Y,2203
1
+ nlpertools/__init__.py,sha256=5ka-NeGW2AUDJ4YZ12DD64xcxuxf9PlQUurxDp5DHbQ,483
2
+ nlpertools/cli.py,sha256=4Ik1NyFaoZpZLsYLAFRLk6xuYQk0IvexPr1Ieq08viE,3932
3
3
  nlpertools/data_client.py,sha256=esX8lUQrTui4uVkqPfhpHVok7Eq6ywpuemKjLeqoglc,14674
4
4
  nlpertools/dataprocess.py,sha256=v1mobuYN7I3dT6xIKlNOHVtcg31YtjF6FwNPTxeBFFY,23153
5
5
  nlpertools/default_db_config.yml,sha256=E1K9k_xzXVlsf-HJQh8kyHXHYuvTpD12jD4Hfe5rUk8,606
6
6
  nlpertools/get_2fa.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
- nlpertools/ml.py,sha256=z-0ep9svAyzcS2n7Lsyqo65VEQRGzWKFMLdZofCv1LQ,17716
7
+ nlpertools/ml.py,sha256=qhUBCLuHfcFy8g5ZHNGYq4eH2vYWiGetyKucv8n60-A,18523
8
8
  nlpertools/movie.py,sha256=rkyOnAXdsbWfMSbi1sE1VNRT7f66Hp9BnZsN_58Afmw,897
9
9
  nlpertools/nlpertools_config.yml,sha256=ksXejxFs7pxR47tNAsrN88_4gvq9PCA2ZMO07H-dJXY,26
10
10
  nlpertools/open_api.py,sha256=uyTY00OUlM57Cn0Wm0yZXcIS8vAszy9rKnDMBEWfWJM,1744
11
- nlpertools/other.py,sha256=CeUea17Oe5MV_r-CmeYdAhdj5kWLvmxoDDgRc56o7bE,14704
11
+ nlpertools/other.py,sha256=JWJiXHRI8mhiUV3k4CZ4kQQS9QN3mw67SmGgTqZFtjs,15026
12
12
  nlpertools/pic.py,sha256=13aaFJh3USGYGs4Y9tAKTvWjmdQR4YDjl3LlIhJheOA,9906
13
13
  nlpertools/plugin.py,sha256=LB7j9GdoQi6TITddH-6EglHlOa0WIHLUT7X5vb_aIZY,1168
14
14
  nlpertools/reminder.py,sha256=wiXwZQmxMck5vY3EvG8_oakP3FAdjGTikAIOiTPUQrs,2977
@@ -25,11 +25,11 @@ nlpertools/algo/union.py,sha256=0l7lGZbw1qIfW1z5TE8Oo3tybL1bKIP5rzpa5ZT-vLQ,249
25
25
  nlpertools/data_structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
26
26
  nlpertools/data_structure/base_structure.py,sha256=gVUvJZ5jsCAswRETTpMwcEjLKoageWiTuCKNEwIWKWk,2641
27
27
  nlpertools/draw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
28
- nlpertools/draw/draw.py,sha256=PgdG7unpCtbbQdYISODTYMV7p10GwWDh9czeURkG0x4,2629
28
+ nlpertools/draw/draw.py,sha256=19dskkr0wrgczxPJnphEszliwYshEh5SjD8Zz07nlk0,2615
29
29
  nlpertools/draw/math_func.py,sha256=0NQ22Dfi9DFG6Bg_hXnCT27w65-dqpOOIgZX7oUIW-Q,881
30
30
  nlpertools/io/__init__.py,sha256=YMuKtC2Ddh5dL5MvXjyUKYOOuqzFYUhBPFaP2kyFG9I,68
31
- nlpertools/io/dir.py,sha256=p7J34qUxYCqKSO5DQMhL8FxFcHDrwn_1lIxNl0klasU,2267
32
- nlpertools/io/file.py,sha256=CsFdluEczuz3fonbeZi9dHPasL1Hm18JL3Aux2ziQMU,7198
31
+ nlpertools/io/dir.py,sha256=FPY62COQN8Ji72pk0dYRoXkrORYaUlybKNcL4474uUI,2263
32
+ nlpertools/io/file.py,sha256=mLWl09IEi0rWPN4tTq3LwdYMvAjj4e_QsjEMhufuPPo,7192
33
33
  nlpertools/monitor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
34
  nlpertools/monitor/gpu.py,sha256=M59O6i0hlew7AzXZlaVZqbZA5IR93OhBY2WI0-T_HtY,531
35
35
  nlpertools/monitor/memory.py,sha256=9t6q9BC8VVx4o3G4sBCn7IoQRx272zMPjSnL3yvTBAQ,657
@@ -41,9 +41,9 @@ nlpertools/utils/package.py,sha256=wLg_M8j7Y6ReRjWHWCWoZJHrzEwuAr9TyG2jvb7OQCo,3
41
41
  nlpertools/utils/package_v1.py,sha256=sqgFb-zbTdMd5ziJLY6YUPqR49qUNZjxBH35DnyR5Wg,3542
42
42
  nlpertools/utils/package_v2.py,sha256=WOcsguWfUd4XSAfmPgCtL8HtUbqJ6GRSMHb0OsB47r0,3932
43
43
  nlpertools_helper/__init__.py,sha256=obxRUdZDctvcvK_iA1Dx2HmQFMlMzJto-xDPryq1lJ0,198
44
- nlpertools-1.0.8.dist-info/LICENSE,sha256=SBcMozykvTbZJ--MqSiKUmHLLROdnr25V70xCQgEwqw,11331
45
- nlpertools-1.0.8.dist-info/METADATA,sha256=v2doRda1amZbXXfIYuzo-rFPvTICt3ByDCKVr6gsUw0,3276
46
- nlpertools-1.0.8.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
47
- nlpertools-1.0.8.dist-info/entry_points.txt,sha256=XEazQ4vUwJMoMAgAwk1Lq4PRQGklPkPBaFkiP0zN_JE,45
48
- nlpertools-1.0.8.dist-info/top_level.txt,sha256=_4q4MIFvMr4cAUbhWKWYdRXIXsF4PJDg4BUsZvgk94s,29
49
- nlpertools-1.0.8.dist-info/RECORD,,
44
+ nlpertools-1.0.9.dist-info/LICENSE,sha256=SBcMozykvTbZJ--MqSiKUmHLLROdnr25V70xCQgEwqw,11331
45
+ nlpertools-1.0.9.dist-info/METADATA,sha256=lcKmxc7_mtYH47mPj8UHOM8-5T5YtrDwhHWVZkfHZXU,3330
46
+ nlpertools-1.0.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
47
+ nlpertools-1.0.9.dist-info/entry_points.txt,sha256=XEazQ4vUwJMoMAgAwk1Lq4PRQGklPkPBaFkiP0zN_JE,45
48
+ nlpertools-1.0.9.dist-info/top_level.txt,sha256=_4q4MIFvMr4cAUbhWKWYdRXIXsF4PJDg4BUsZvgk94s,29
49
+ nlpertools-1.0.9.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.6.0)
2
+ Generator: setuptools (75.8.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5