nlpertools 1.0.8__tar.gz → 1.0.9__tar.gz

{nlpertools-1.0.8/src/nlpertools.egg-info → nlpertools-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: nlpertools
-Version: 1.0.8
+Version: 1.0.9
 Summary: A small package about small basic IO operation when coding
 Home-page: https://github.com/lvzii/nlpertools
 Author: youshuJi
@@ -17,6 +17,8 @@ Requires-Dist: pandas
 Requires-Dist: psutil
 Provides-Extra: torch
 Requires-Dist: torch; extra == "torch"
+Dynamic: provides-extra
+Dynamic: requires-dist
 
 <div align="center">
   <h4 align="center">
@@ -35,7 +37,7 @@ Requires-Dist: torch; extra == "torch"
 
 它解决了什么问题：
 
-- 很多函数是记不住的， ~~每次写每次都要搜~~ 每次都要问大模型 ，例如pandas排序
+- 很多函数是记不住的， 每次写都要~~搜~~问大模型 ，例如pandas排序
 - 刷题的时候，树结构的题目很难调试
 
 
@@ -111,11 +113,11 @@ b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
 
 ```bash
 # 生成pypi双因素认证的实时密钥(需要提供key)
-python -m nlpertools.get_2fa your_key
+python -m nlpertools.cli --get_2fa --get_2fa_key your_key
 
 ## git
-python nlpertools.cli --git_push
-python nlpertools.cli --git_pull
+python -m nlpertools.cli --git_push
+python -m nlpertools.cli --git_pull
 
 # 以下功能被nvitop替代，不推荐使用
 ## 监控gpu显存

{nlpertools-1.0.8 → nlpertools-1.0.9}/README.md

@@ -15,7 +15,7 @@
 
 它解决了什么问题：
 
-- 很多函数是记不住的， ~~每次写每次都要搜~~ 每次都要问大模型 ，例如pandas排序
+- 很多函数是记不住的， 每次写都要~~搜~~问大模型 ，例如pandas排序
 - 刷题的时候，树结构的题目很难调试
 
 
@@ -91,11 +91,11 @@ b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
 
 ```bash
 # 生成pypi双因素认证的实时密钥(需要提供key)
-python -m nlpertools.get_2fa your_key
+python -m nlpertools.cli --get_2fa --get_2fa_key your_key
 
 ## git
-python nlpertools.cli --git_push
-python nlpertools.cli --git_pull
+python -m nlpertools.cli --git_push
+python -m nlpertools.cli --git_pull
 
 # 以下功能被nvitop替代，不推荐使用
 ## 监控gpu显存

{nlpertools-1.0.8 → nlpertools-1.0.9}/src/nlpertools/__init__.py

@@ -20,4 +20,4 @@ from .cli import *
 
 
 
-__version__ = '1.0.8'
+__version__ = '1.0.9'

{nlpertools-1.0.8 → nlpertools-1.0.9}/src/nlpertools/cli.py

@@ -3,8 +3,6 @@ import os
 import uuid
 import sys
 
-import pyotp
-
 """
 如何Debug cli.py
 """
@@ -49,6 +47,7 @@ def get_mac_address():
 
 
 def get_2af_value(key):
+    import pyotp
     """
     key应该是7位的
     """
@@ -57,14 +56,67 @@ def get_2af_value(key):
     print(totp.now())
 
 
+def start_gpu_usage_notify_server():
+    from flask import Flask
+
+    app = Flask(__name__)
+
+    @app.route("/notify", methods=["GET"])
+    def notify():
+        # 这里可以根据需要动态生成通知内容
+        usage = os.popen("nvidia-smi --query-gpu=memory.used --format=csv").read().split("\n")[1:]
+        res = 0
+        for edx, each in enumerate(usage):
+            if each.startswith("0"):
+                res += 1
+        print(res)
+        return str(res), 200
+
+    app.run(host="0.0.0.0", port=5000)
+
+
+def start_gpu_usage_notify_client():
+    import requests
+    from plyer import notification
+    import time
+
+    SERVER_URL = 'http://127.0.0.1:5000/notify'  # 服务器的 API 地址
+
+    def notify(text):
+        # 使用 plyer 发送通知
+        notification.notify(
+            title='远程通知',
+            message=text,
+            timeout=10  # 10秒的通知显示时间
+        )
+
+    """定时轮询服务器获取通知"""
+    while True:
+        try:
+            response = requests.get(SERVER_URL)
+            if response.status_code == 200:
+                num = int(response.text)
+                if num > 0:
+                    notify(f"服务器有{num}张卡")
+                print(f"服务器有{num}张卡")
+            else:
+                print("服务器没有新通知")
+        except Exception as e:
+            print(f"与服务器连接失败: {e}")
+
+        time.sleep(1)
+
+
 def main():
     parser = argparse.ArgumentParser(description="CLI tool for git operations and getting MAC address.")
     parser.add_argument('--gitpush', action='store_true', help='Perform git push operation.')
-    parser.add_argument('--gitpull', action='store_true', help='Perform git push operation.')
+    parser.add_argument('--gitpull', action='store_true', help='Perform git pull operation.')
     parser.add_argument('--mac_address', action='store_true', help='Get the MAC address.')
 
     parser.add_argument('--get_2fa', action='store_true', help='Get the 2fa value.')
     parser.add_argument('--get_2fa_key', type=str, help='Get the 2fa value.')
+    parser.add_argument('--monitor_gpu_cli', action='store_true', help='Get the 2fa value.')
+    parser.add_argument('--monitor_gpu_ser', action='store_true', help='Get the 2fa value.')
 
     args = parser.parse_args()
 
@@ -74,13 +126,17 @@ def main():
         git_pull()
     elif args.mac_address:
         get_mac_address()
+    elif args.monitor_gpu_cli:
+        start_gpu_usage_notify_client()
+    elif args.monitor_gpu_ser:
+        start_gpu_usage_notify_server()
     elif args.get_2fa:
         if args.get_2fa_key:
             get_2af_value(args.get_2fa_key)
         else:
             print("Please provide a key as an argument.")
     else:
-        print("No operation specified. Use --gitpush or --get_mac_address.")
+        print("No operation specified.")
 
 
 if __name__ == '__main__':

{nlpertools-1.0.8 → nlpertools-1.0.9}/src/nlpertools/draw/draw.py

@@ -20,7 +20,7 @@ def confused_matrix(confuse_matrix):
     f.savefig('tmp.jpg', bbox_inches='tight')
 
 
-def plot_histogram(data, bin_size):
+def plot_histogram(data, bin_size, max_bin):
     """
     画直方图，超过1000的统一按1000算
     :param data:
@@ -33,15 +33,15 @@ def plot_histogram(data, bin_size):
     from matplotlib.ticker import MaxNLocator
     # 将超过1000的值改为1000
     def process_lengths(data):
-        return [length if length <= 1000 else 1003 for length in data]
+        return [length if length <= max_bin else max_bin + 3 for length in data]
 
     # 前闭后开
-    min_num, max_num = 0, 1000
+    # min_num, max_num = 0, 1000
     # min_num, max_num = min(data), max(data)
 
     plt.figure(figsize=(12, 8))
     processed_data = process_lengths(data)
-    bins = np.arange(0, 1000 + 2 * bin_size, bin_size)
+    bins = np.arange(0, max_bin + 2 * bin_size, bin_size)
     # 绘制直方图
     n, new_bins, patches = plt.hist(processed_data, bins=bins, edgecolor='black', color='skyblue', alpha=0.7,
                                     linewidth=0)
@@ -60,10 +60,8 @@ def plot_histogram(data, bin_size):
     plt.xlabel('module line number', fontsize=14)
     plt.ylabel('frequency', fontsize=14)
 
-    # 添加网格
     plt.grid(True, linestyle='--', alpha=0.6)
 
-    # 美化x轴和y轴的刻度
     plt.xticks(fontsize=12)
     plt.yticks(fontsize=12)
 
@@ -80,4 +78,4 @@ if __name__ == '__main__':
     # 调整区间大小
     bin_size = 50
     # 示例模块长度数据
-    plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size)
+    plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size, max_bin=1000)

{nlpertools-1.0.8 → nlpertools-1.0.9}/src/nlpertools/io/dir.py

@@ -46,7 +46,7 @@ def get_filename(path, suffix=True) -> str:
     return filename
 
 
-def j_listdir(dir_name, including_dir=True):
+def listdir(dir_name, including_dir=True):
     filenames = os.listdir(dir_name)
     if including_dir:
         return [os.path.join(dir_name, filename) for filename in filenames]
@@ -54,7 +54,7 @@ def j_listdir(dir_name, including_dir=True):
         return list(filenames)
 
 
-def j_listdir_yield(dir_name, including_dir=True):
+def listdir_yield(dir_name, including_dir=True):
     filenames = os.listdir(dir_name)
     for filename in filenames:
         if including_dir:

{nlpertools-1.0.8 → nlpertools-1.0.9}/src/nlpertools/io/file.py

@@ -241,12 +241,12 @@ def load_from_jsonl(path):
         return corpus
 
 
-def pickle_save(data, path):
+def save_pkl(data, path):
     with open(path, 'wb') as f:
         pickle.dump(data, f)
 
 
-def pickle_load(path):
+def load_pkl(path):
     with open(path, 'rb') as f:
         data = pickle.load(f)
     return data

{nlpertools-1.0.8 → nlpertools-1.0.9}/src/nlpertools/ml.py

@@ -17,6 +17,28 @@ from .io.file import readtxt_list_all_strip, writetxt_w_list, save_to_csv
 from .utils.package import *
 
 
+def estimate_pass_at_k(num_samples:list, num_correct:list, k):
+    """
+    copy from https://huggingface.co/spaces/evaluate-metric/code_eval/blob/main/code_eval.py
+    num_samples: list
+    """
+    """Estimates pass@k of each problem and returns them in an array."""
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """Calculates 1 - comb(n - c, k) / comb(n, k)."""
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+
 def calc_llm_train_activation_memory(
         model_name, sequence_length, batch_size, hidden_dim, lay_number, attention_heads_num, gpu_num=1
 ):

{nlpertools-1.0.8 → nlpertools-1.0.9}/src/nlpertools/other.py

@@ -30,6 +30,21 @@ ENGLISH_PUNCTUATION = list(',.;:\'"!?<>()')
 OTHER_PUNCTUATION = list('!@#$%^&*')
 
 
+def setup_logging(log_file):
+    """
+    Set up logging configuration.
+
+    Args:
+        log_file (str): Path to the log file.
+    """
+    logging.basicConfig(
+        filename=log_file,
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+
+
 def get_diff_parts(str1, str2):
     # 创建一个 SequenceMatcher 对象
     matcher = difflib.SequenceMatcher(None, str1, str2)
@@ -361,10 +376,12 @@ def unsqueeze_list(flatten_list, each_element_len):
                     range(len(flatten_list) // each_element_len)]
     return two_dim_list
 
+
 def split_list(input_list, chunk_size):
     # 使用列表推导式将列表分割成二维数组
     return [input_list[i:i + chunk_size] for i in range(0, len(input_list), chunk_size)]
 
+
 def auto_close():
     """
     针对企业微信15分钟会显示离开的机制，假装自己还在上班

{nlpertools-1.0.8 → nlpertools-1.0.9/src/nlpertools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: nlpertools
-Version: 1.0.8
+Version: 1.0.9
 Summary: A small package about small basic IO operation when coding
 Home-page: https://github.com/lvzii/nlpertools
 Author: youshuJi
@@ -17,6 +17,8 @@ Requires-Dist: pandas
 Requires-Dist: psutil
 Provides-Extra: torch
 Requires-Dist: torch; extra == "torch"
+Dynamic: provides-extra
+Dynamic: requires-dist
 
 <div align="center">
   <h4 align="center">
@@ -35,7 +37,7 @@ Requires-Dist: torch; extra == "torch"
 
 它解决了什么问题：
 
-- 很多函数是记不住的， ~~每次写每次都要搜~~ 每次都要问大模型 ，例如pandas排序
+- 很多函数是记不住的， 每次写都要~~搜~~问大模型 ，例如pandas排序
 - 刷题的时候，树结构的题目很难调试
 
 
@@ -111,11 +113,11 @@ b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
 
 ```bash
 # 生成pypi双因素认证的实时密钥(需要提供key)
-python -m nlpertools.get_2fa your_key
+python -m nlpertools.cli --get_2fa --get_2fa_key your_key
 
 ## git
-python nlpertools.cli --git_push
-python nlpertools.cli --git_pull
+python -m nlpertools.cli --git_push
+python -m nlpertools.cli --git_pull
 
 # 以下功能被nvitop替代，不推荐使用
 ## 监控gpu显存