nlpertools 1.0.8__py3-none-any.whl → 1.0.9__py3-none-any.whl

nlpertools/__init__.py

@@ -20,4 +20,4 @@ from .cli import *
 
 
 
-__version__ = '1.0.8'
+__version__ = '1.0.9'

nlpertools/cli.py

@@ -3,8 +3,6 @@ import os
 import uuid
 import sys
 
-import pyotp
-
 """
 如何Debug cli.py
 """
@@ -49,6 +47,7 @@ def get_mac_address():
 
 
 def get_2af_value(key):
+    import pyotp
     """
     key应该是7位的
     """
@@ -57,14 +56,67 @@ def get_2af_value(key):
     print(totp.now())
 
 
+def start_gpu_usage_notify_server():
+    from flask import Flask
+
+    app = Flask(__name__)
+
+    @app.route("/notify", methods=["GET"])
+    def notify():
+        # 这里可以根据需要动态生成通知内容
+        usage = os.popen("nvidia-smi --query-gpu=memory.used --format=csv").read().split("\n")[1:]
+        res = 0
+        for edx, each in enumerate(usage):
+            if each.startswith("0"):
+                res += 1
+        print(res)
+        return str(res), 200
+
+    app.run(host="0.0.0.0", port=5000)
+
+
+def start_gpu_usage_notify_client():
+    import requests
+    from plyer import notification
+    import time
+
+    SERVER_URL = 'http://127.0.0.1:5000/notify'  # 服务器的 API 地址
+
+    def notify(text):
+        # 使用 plyer 发送通知
+        notification.notify(
+            title='远程通知',
+            message=text,
+            timeout=10  # 10秒的通知显示时间
+        )
+
+    """定时轮询服务器获取通知"""
+    while True:
+        try:
+            response = requests.get(SERVER_URL)
+            if response.status_code == 200:
+                num = int(response.text)
+                if num > 0:
+                    notify(f"服务器有{num}张卡")
+                print(f"服务器有{num}张卡")
+            else:
+                print("服务器没有新通知")
+        except Exception as e:
+            print(f"与服务器连接失败: {e}")
+
+        time.sleep(1)
+
+
 def main():
     parser = argparse.ArgumentParser(description="CLI tool for git operations and getting MAC address.")
     parser.add_argument('--gitpush', action='store_true', help='Perform git push operation.')
-    parser.add_argument('--gitpull', action='store_true', help='Perform git push operation.')
+    parser.add_argument('--gitpull', action='store_true', help='Perform git pull operation.')
     parser.add_argument('--mac_address', action='store_true', help='Get the MAC address.')
 
     parser.add_argument('--get_2fa', action='store_true', help='Get the 2fa value.')
     parser.add_argument('--get_2fa_key', type=str, help='Get the 2fa value.')
+    parser.add_argument('--monitor_gpu_cli', action='store_true', help='Get the 2fa value.')
+    parser.add_argument('--monitor_gpu_ser', action='store_true', help='Get the 2fa value.')
 
     args = parser.parse_args()
 
@@ -74,13 +126,17 @@ def main():
         git_pull()
     elif args.mac_address:
         get_mac_address()
+    elif args.monitor_gpu_cli:
+        start_gpu_usage_notify_client()
+    elif args.monitor_gpu_ser:
+        start_gpu_usage_notify_server()
     elif args.get_2fa:
         if args.get_2fa_key:
             get_2af_value(args.get_2fa_key)
         else:
             print("Please provide a key as an argument.")
     else:
-        print("No operation specified. Use --gitpush or --get_mac_address.")
+        print("No operation specified.")
 
 
 if __name__ == '__main__':

nlpertools/draw/draw.py

@@ -20,7 +20,7 @@ def confused_matrix(confuse_matrix):
     f.savefig('tmp.jpg', bbox_inches='tight')
 
 
-def plot_histogram(data, bin_size):
+def plot_histogram(data, bin_size, max_bin):
     """
     画直方图，超过1000的统一按1000算
     :param data:
@@ -33,15 +33,15 @@ def plot_histogram(data, bin_size):
     from matplotlib.ticker import MaxNLocator
     # 将超过1000的值改为1000
     def process_lengths(data):
-        return [length if length <= 1000 else 1003 for length in data]
+        return [length if length <= max_bin else max_bin + 3 for length in data]
 
     # 前闭后开
-    min_num, max_num = 0, 1000
+    # min_num, max_num = 0, 1000
     # min_num, max_num = min(data), max(data)
 
     plt.figure(figsize=(12, 8))
     processed_data = process_lengths(data)
-    bins = np.arange(0, 1000 + 2 * bin_size, bin_size)
+    bins = np.arange(0, max_bin + 2 * bin_size, bin_size)
     # 绘制直方图
     n, new_bins, patches = plt.hist(processed_data, bins=bins, edgecolor='black', color='skyblue', alpha=0.7,
                                     linewidth=0)
@@ -60,10 +60,8 @@ def plot_histogram(data, bin_size):
     plt.xlabel('module line number', fontsize=14)
     plt.ylabel('frequency', fontsize=14)
 
-    # 添加网格
     plt.grid(True, linestyle='--', alpha=0.6)
 
-    # 美化x轴和y轴的刻度
     plt.xticks(fontsize=12)
     plt.yticks(fontsize=12)
 
@@ -80,4 +78,4 @@ if __name__ == '__main__':
     # 调整区间大小
     bin_size = 50
     # 示例模块长度数据
-    plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size)
+    plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size, max_bin=1000)

nlpertools/io/dir.py

@@ -46,7 +46,7 @@ def get_filename(path, suffix=True) -> str:
     return filename
 
 
-def j_listdir(dir_name, including_dir=True):
+def listdir(dir_name, including_dir=True):
     filenames = os.listdir(dir_name)
     if including_dir:
         return [os.path.join(dir_name, filename) for filename in filenames]
@@ -54,7 +54,7 @@ def j_listdir(dir_name, including_dir=True):
         return list(filenames)
 
 
-def j_listdir_yield(dir_name, including_dir=True):
+def listdir_yield(dir_name, including_dir=True):
     filenames = os.listdir(dir_name)
     for filename in filenames:
         if including_dir:

nlpertools/io/file.py

@@ -241,12 +241,12 @@ def load_from_jsonl(path):
         return corpus
 
 
-def pickle_save(data, path):
+def save_pkl(data, path):
     with open(path, 'wb') as f:
         pickle.dump(data, f)
 
 
-def pickle_load(path):
+def load_pkl(path):
     with open(path, 'rb') as f:
         data = pickle.load(f)
     return data

nlpertools/ml.py

@@ -17,6 +17,28 @@ from .io.file import readtxt_list_all_strip, writetxt_w_list, save_to_csv
 from .utils.package import *
 
 
+def estimate_pass_at_k(num_samples:list, num_correct:list, k):
+    """
+    copy from https://huggingface.co/spaces/evaluate-metric/code_eval/blob/main/code_eval.py
+    num_samples: list
+    """
+    """Estimates pass@k of each problem and returns them in an array."""
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """Calculates 1 - comb(n - c, k) / comb(n, k)."""
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+
 def calc_llm_train_activation_memory(
         model_name, sequence_length, batch_size, hidden_dim, lay_number, attention_heads_num, gpu_num=1
 ):

nlpertools/other.py

@@ -30,6 +30,21 @@ ENGLISH_PUNCTUATION = list(',.;:\'"!?<>()')
 OTHER_PUNCTUATION = list('!@#$%^&*')
 
 
+def setup_logging(log_file):
+    """
+    Set up logging configuration.
+
+    Args:
+        log_file (str): Path to the log file.
+    """
+    logging.basicConfig(
+        filename=log_file,
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+
+
 def get_diff_parts(str1, str2):
     # 创建一个 SequenceMatcher 对象
     matcher = difflib.SequenceMatcher(None, str1, str2)
@@ -361,10 +376,12 @@ def unsqueeze_list(flatten_list, each_element_len):
                     range(len(flatten_list) // each_element_len)]
     return two_dim_list
 
+
 def split_list(input_list, chunk_size):
     # 使用列表推导式将列表分割成二维数组
     return [input_list[i:i + chunk_size] for i in range(0, len(input_list), chunk_size)]
 
+
 def auto_close():
     """
     针对企业微信15分钟会显示离开的机制，假装自己还在上班

{nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: nlpertools
-Version: 1.0.8
+Version: 1.0.9
 Summary: A small package about small basic IO operation when coding
 Home-page: https://github.com/lvzii/nlpertools
 Author: youshuJi
@@ -17,6 +17,8 @@ Requires-Dist: pandas
 Requires-Dist: psutil
 Provides-Extra: torch
 Requires-Dist: torch; extra == "torch"
+Dynamic: provides-extra
+Dynamic: requires-dist
 
 <div align="center">
   <h4 align="center">
@@ -35,7 +37,7 @@ Requires-Dist: torch; extra == "torch"
 
 它解决了什么问题：
 
-- 很多函数是记不住的， ~~每次写每次都要搜~~ 每次都要问大模型 ，例如pandas排序
+- 很多函数是记不住的， 每次写都要~~搜~~问大模型 ，例如pandas排序
 - 刷题的时候，树结构的题目很难调试
 
 
@@ -111,11 +113,11 @@ b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
 
 ```bash
 # 生成pypi双因素认证的实时密钥(需要提供key)
-python -m nlpertools.get_2fa your_key
+python -m nlpertools.cli --get_2fa --get_2fa_key your_key
 
 ## git
-python nlpertools.cli --git_push
-python nlpertools.cli --git_pull
+python -m nlpertools.cli --git_push
+python -m nlpertools.cli --git_pull
 
 # 以下功能被nvitop替代，不推荐使用
 ## 监控gpu显存

{nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/RECORD

@@ -1,14 +1,14 @@
-nlpertools/__init__.py,sha256=h7JJEN_JRn3iKcqIcaFgYtAjP90XiT1KILrm8utoHvQ,483
-nlpertools/cli.py,sha256=xDl_tWl9pfqQ3PUdd7oesvgM2FVqnaw8dFFliEX5c4Y,2203
+nlpertools/__init__.py,sha256=5ka-NeGW2AUDJ4YZ12DD64xcxuxf9PlQUurxDp5DHbQ,483
+nlpertools/cli.py,sha256=4Ik1NyFaoZpZLsYLAFRLk6xuYQk0IvexPr1Ieq08viE,3932
 nlpertools/data_client.py,sha256=esX8lUQrTui4uVkqPfhpHVok7Eq6ywpuemKjLeqoglc,14674
 nlpertools/dataprocess.py,sha256=v1mobuYN7I3dT6xIKlNOHVtcg31YtjF6FwNPTxeBFFY,23153
 nlpertools/default_db_config.yml,sha256=E1K9k_xzXVlsf-HJQh8kyHXHYuvTpD12jD4Hfe5rUk8,606
 nlpertools/get_2fa.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nlpertools/ml.py,sha256=z-0ep9svAyzcS2n7Lsyqo65VEQRGzWKFMLdZofCv1LQ,17716
+nlpertools/ml.py,sha256=qhUBCLuHfcFy8g5ZHNGYq4eH2vYWiGetyKucv8n60-A,18523
 nlpertools/movie.py,sha256=rkyOnAXdsbWfMSbi1sE1VNRT7f66Hp9BnZsN_58Afmw,897
 nlpertools/nlpertools_config.yml,sha256=ksXejxFs7pxR47tNAsrN88_4gvq9PCA2ZMO07H-dJXY,26
 nlpertools/open_api.py,sha256=uyTY00OUlM57Cn0Wm0yZXcIS8vAszy9rKnDMBEWfWJM,1744
-nlpertools/other.py,sha256=CeUea17Oe5MV_r-CmeYdAhdj5kWLvmxoDDgRc56o7bE,14704
+nlpertools/other.py,sha256=JWJiXHRI8mhiUV3k4CZ4kQQS9QN3mw67SmGgTqZFtjs,15026
 nlpertools/pic.py,sha256=13aaFJh3USGYGs4Y9tAKTvWjmdQR4YDjl3LlIhJheOA,9906
 nlpertools/plugin.py,sha256=LB7j9GdoQi6TITddH-6EglHlOa0WIHLUT7X5vb_aIZY,1168
 nlpertools/reminder.py,sha256=wiXwZQmxMck5vY3EvG8_oakP3FAdjGTikAIOiTPUQrs,2977
@@ -25,11 +25,11 @@ nlpertools/algo/union.py,sha256=0l7lGZbw1qIfW1z5TE8Oo3tybL1bKIP5rzpa5ZT-vLQ,249
 nlpertools/data_structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nlpertools/data_structure/base_structure.py,sha256=gVUvJZ5jsCAswRETTpMwcEjLKoageWiTuCKNEwIWKWk,2641
 nlpertools/draw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nlpertools/draw/draw.py,sha256=PgdG7unpCtbbQdYISODTYMV7p10GwWDh9czeURkG0x4,2629
+nlpertools/draw/draw.py,sha256=19dskkr0wrgczxPJnphEszliwYshEh5SjD8Zz07nlk0,2615
 nlpertools/draw/math_func.py,sha256=0NQ22Dfi9DFG6Bg_hXnCT27w65-dqpOOIgZX7oUIW-Q,881
 nlpertools/io/__init__.py,sha256=YMuKtC2Ddh5dL5MvXjyUKYOOuqzFYUhBPFaP2kyFG9I,68
-nlpertools/io/dir.py,sha256=p7J34qUxYCqKSO5DQMhL8FxFcHDrwn_1lIxNl0klasU,2267
-nlpertools/io/file.py,sha256=CsFdluEczuz3fonbeZi9dHPasL1Hm18JL3Aux2ziQMU,7198
+nlpertools/io/dir.py,sha256=FPY62COQN8Ji72pk0dYRoXkrORYaUlybKNcL4474uUI,2263
+nlpertools/io/file.py,sha256=mLWl09IEi0rWPN4tTq3LwdYMvAjj4e_QsjEMhufuPPo,7192
 nlpertools/monitor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nlpertools/monitor/gpu.py,sha256=M59O6i0hlew7AzXZlaVZqbZA5IR93OhBY2WI0-T_HtY,531
 nlpertools/monitor/memory.py,sha256=9t6q9BC8VVx4o3G4sBCn7IoQRx272zMPjSnL3yvTBAQ,657
@@ -41,9 +41,9 @@ nlpertools/utils/package.py,sha256=wLg_M8j7Y6ReRjWHWCWoZJHrzEwuAr9TyG2jvb7OQCo,3
 nlpertools/utils/package_v1.py,sha256=sqgFb-zbTdMd5ziJLY6YUPqR49qUNZjxBH35DnyR5Wg,3542
 nlpertools/utils/package_v2.py,sha256=WOcsguWfUd4XSAfmPgCtL8HtUbqJ6GRSMHb0OsB47r0,3932
 nlpertools_helper/__init__.py,sha256=obxRUdZDctvcvK_iA1Dx2HmQFMlMzJto-xDPryq1lJ0,198
-nlpertools-1.0.8.dist-info/LICENSE,sha256=SBcMozykvTbZJ--MqSiKUmHLLROdnr25V70xCQgEwqw,11331
-nlpertools-1.0.8.dist-info/METADATA,sha256=v2doRda1amZbXXfIYuzo-rFPvTICt3ByDCKVr6gsUw0,3276
-nlpertools-1.0.8.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-nlpertools-1.0.8.dist-info/entry_points.txt,sha256=XEazQ4vUwJMoMAgAwk1Lq4PRQGklPkPBaFkiP0zN_JE,45
-nlpertools-1.0.8.dist-info/top_level.txt,sha256=_4q4MIFvMr4cAUbhWKWYdRXIXsF4PJDg4BUsZvgk94s,29
-nlpertools-1.0.8.dist-info/RECORD,,
+nlpertools-1.0.9.dist-info/LICENSE,sha256=SBcMozykvTbZJ--MqSiKUmHLLROdnr25V70xCQgEwqw,11331
+nlpertools-1.0.9.dist-info/METADATA,sha256=lcKmxc7_mtYH47mPj8UHOM8-5T5YtrDwhHWVZkfHZXU,3330
+nlpertools-1.0.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nlpertools-1.0.9.dist-info/entry_points.txt,sha256=XEazQ4vUwJMoMAgAwk1Lq4PRQGklPkPBaFkiP0zN_JE,45
+nlpertools-1.0.9.dist-info/top_level.txt,sha256=_4q4MIFvMr4cAUbhWKWYdRXIXsF4PJDg4BUsZvgk94s,29
+nlpertools-1.0.9.dist-info/RECORD,,

{nlpertools-1.0.8.dist-info → nlpertools-1.0.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any