nlpertools 1.0.9__tar.gz → 1.0.11__tar.gz

{nlpertools-1.0.9/src/nlpertools.egg-info → nlpertools-1.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: nlpertools
-Version: 1.0.9
+Version: 1.0.11
 Summary: A small package about small basic IO operation when coding
 Home-page: https://github.com/lvzii/nlpertools
 Author: youshuJi
@@ -15,8 +15,10 @@ License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: psutil
+Requires-Dist: openai
 Provides-Extra: torch
 Requires-Dist: torch; extra == "torch"
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 
@@ -50,6 +52,25 @@ nlpertools
 
 ```
 
+# 最常用/喜欢的功能（使用示例）
+```python
+# 读txt, json文件
+import nlpertools
+
+txt_data = nlpertools.readtxt_list_all_strip('res.txt')
+json_data = nlpertools.load_from_json('res.json')
+```
+
+```bash
+## git, 连接github不稳定的时候非常有用
+ncli git pull
+## 带有参数时，加上--以避免-u被解析
+ncli -- git push -u origin main
+
+# 生成pypi双因素认证的实时密钥(需要提供key)
+ncli --get_2fa --get_2fa_key your_key
+```
+
 # 安装
 
 Install the latest release version
@@ -101,30 +122,7 @@ https://nlpertools.readthedocs.io/en/latest/
 
 一些可能需要配置才能用的函数，写上示例
 
-## 使用示例
-
-```python
-import nlpertools
-
-a = nlpertools.readtxt_list_all_strip('res.txt')
-# 或
-b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
-```
 
-```bash
-# 生成pypi双因素认证的实时密钥(需要提供key)
-python -m nlpertools.cli --get_2fa --get_2fa_key your_key
-
-## git
-python -m nlpertools.cli --git_push
-python -m nlpertools.cli --git_pull
-
-# 以下功能被nvitop替代，不推荐使用
-## 监控gpu显存
-python -m nlpertools.monitor.gpu
-## 监控cpu
-python -m  nlpertools.monitor.memory
-```
 
 ## 一些常用项目
 
@@ -132,3 +130,7 @@ nvitop
 
 ydata-profiling
 
+## 贡献
+
+https://github.com/bigscience-workshop/data-preparation
+

{nlpertools-1.0.9 → nlpertools-1.0.11}/README.md

@@ -28,6 +28,25 @@ nlpertools
 
 ```
 
+# 最常用/喜欢的功能（使用示例）
+```python
+# 读txt, json文件
+import nlpertools
+
+txt_data = nlpertools.readtxt_list_all_strip('res.txt')
+json_data = nlpertools.load_from_json('res.json')
+```
+
+```bash
+## git, 连接github不稳定的时候非常有用
+ncli git pull
+## 带有参数时，加上--以避免-u被解析
+ncli -- git push -u origin main
+
+# 生成pypi双因素认证的实时密钥(需要提供key)
+ncli --get_2fa --get_2fa_key your_key
+```
+
 # 安装
 
 Install the latest release version
@@ -79,30 +98,7 @@ https://nlpertools.readthedocs.io/en/latest/
 
 一些可能需要配置才能用的函数，写上示例
 
-## 使用示例
-
-```python
-import nlpertools
-
-a = nlpertools.readtxt_list_all_strip('res.txt')
-# 或
-b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
-```
 
-```bash
-# 生成pypi双因素认证的实时密钥(需要提供key)
-python -m nlpertools.cli --get_2fa --get_2fa_key your_key
-
-## git
-python -m nlpertools.cli --git_push
-python -m nlpertools.cli --git_pull
-
-# 以下功能被nvitop替代，不推荐使用
-## 监控gpu显存
-python -m nlpertools.monitor.gpu
-## 监控cpu
-python -m  nlpertools.monitor.memory
-```
 
 ## 一些常用项目
 
@@ -110,3 +106,7 @@ nvitop
 
 ydata-profiling
 
+## 贡献
+
+https://github.com/bigscience-workshop/data-preparation
+

{nlpertools-1.0.9 → nlpertools-1.0.11}/setup.py

@@ -7,19 +7,17 @@ from setuptools import setup
 def get_version():
     with open(os.path.join("src", "nlpertools", "__init__.py"), "r", encoding="utf-8") as f:
         file_content = f.read()
-        pattern = r"{}\W*=\W*\'([^\"]+)\'".format("__version__")
-        (version,) = re.findall(pattern, file_content)
-        return version
+        pattern = r"{}\W*=\W*[\"']([^\"']+)[\"']".format("__version__")
+        matches = re.findall(pattern, file_content)
+        if not matches:
+            raise ValueError(f"Could not find __version__ in __init__.py")
+        return matches[0]
 
 
 def main():
     setup(
         # https://juejin.cn/post/7369349560421040128
-        install_requires=[
-            "numpy",
-            "pandas",
-            "psutil"
-        ],
+        install_requires=["numpy", "pandas", "psutil", "openai"],
         extras_require={
             "torch": ["torch"],
         },
@@ -28,9 +26,9 @@ def main():
             "console_scripts": [
                 "ncli=nlpertools.cli:main",
             ]
-        }
+        },
     )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

{nlpertools-1.0.9 → nlpertools-1.0.11}/src/nlpertools/__init__.py

@@ -4,6 +4,7 @@
 from .algo.kmp import *
 from .data_structure.base_structure import *
 from .draw import *
+from .dataprocess.dp_main import *
 from .dataprocess import *
 from .io.dir import *
 from .io.file import *
@@ -17,7 +18,7 @@ from .utils_for_nlpertools import *
 from .wrapper import *
 from .monitor import *
 from .cli import *
+from .llm import *
 
 
-
-__version__ = '1.0.9'
+__version__ = "1.0.11"

{nlpertools-1.0.9 → nlpertools-1.0.11}/src/nlpertools/cli.py

@@ -2,45 +2,30 @@ import argparse
 import os
 import uuid
 import sys
+from .dataprocess.dp_main import startwith
 
-"""
-如何Debug cli.py
-"""
 
-
-def git_push():
-    """
-    针对国内提交github经常失败，自动提交
-    """
-    num = -1
-    while 1:
-        num += 1
-        print("retry num: {}".format(num))
-        info = os.system("git push --set-upstream origin main")
-        print(str(info))
-        if not str(info).startswith("fatal"):
-            print("scucess")
-            break
-
-
-def git_pull():
+def run_git_command(command):
     """
-    针对国内提交github经常失败，自动提交
+    循环执行git命令，直到成功
     """
+    print(command)
     num = -1
-    while 1:
+    while True:
         num += 1
-        print("retry num: {}".format(num))
-        info = os.system("git pull")
+        print(f"retry num: {num}")
+        info = os.system(command)
         print(str(info))
-        if not str(info).startswith("fatal") and not str(info).startswith("error"):
-            print("scucess")
+        # 检查命令执行结果，若未出现错误则认为执行成功
+        if (not startwith(str(info), ["fatal", "error", "128", "1"])) and "fatal" not in str(info):
+            print("success")
+            print(f"success info : ##{info}##")
             break
 
 
 def get_mac_address():
     mac = uuid.UUID(int=uuid.getnode()).hex[-12:]
-    mac_address = ":".join([mac[e:e + 2] for e in range(0, 11, 2)])
+    mac_address = ":".join([mac[e : e + 2] for e in range(0, 11, 2)])
     print("mac address 不一定准确")
     print(mac_address)
     return mac_address
@@ -48,6 +33,7 @@ def get_mac_address():
 
 def get_2af_value(key):
     import pyotp
+
     """
     key应该是7位的
     """
@@ -80,15 +66,11 @@ def start_gpu_usage_notify_client():
     from plyer import notification
     import time
 
-    SERVER_URL = 'http://127.0.0.1:5000/notify'  # 服务器的 API 地址
+    SERVER_URL = "http://127.0.0.1:5000/notify"  # 服务器的 API 地址
 
     def notify(text):
         # 使用 plyer 发送通知
-        notification.notify(
-            title='远程通知',
-            message=text,
-            timeout=10  # 10秒的通知显示时间
-        )
+        notification.notify(title="远程通知", message=text, timeout=10)  # 10秒的通知显示时间
 
     """定时轮询服务器获取通知"""
     while True:
@@ -108,22 +90,19 @@ def start_gpu_usage_notify_client():
 
 
 def main():
-    parser = argparse.ArgumentParser(description="CLI tool for git operations and getting MAC address.")
-    parser.add_argument('--gitpush', action='store_true', help='Perform git push operation.')
-    parser.add_argument('--gitpull', action='store_true', help='Perform git pull operation.')
-    parser.add_argument('--mac_address', action='store_true', help='Get the MAC address.')
-
-    parser.add_argument('--get_2fa', action='store_true', help='Get the 2fa value.')
-    parser.add_argument('--get_2fa_key', type=str, help='Get the 2fa value.')
-    parser.add_argument('--monitor_gpu_cli', action='store_true', help='Get the 2fa value.')
-    parser.add_argument('--monitor_gpu_ser', action='store_true', help='Get the 2fa value.')
+    parser = argparse.ArgumentParser(description="CLI tool for git operations and other functions.")
+    parser.add_argument("git_command", nargs="*", help="Any git command (e.g., push, pull)")
+    parser.add_argument("--mac_address", action="store_true", help="Get the MAC address.")
+    parser.add_argument("--get_2fa", action="store_true", help="Get the 2fa value.")
+    parser.add_argument("--get_2fa_key", type=str, help="Get the 2fa value.")
+    parser.add_argument("--monitor_gpu_cli", action="store_true", help="monitor gpu cli")
+    parser.add_argument("--monitor_gpu_ser", action="store_true", help="monitor gpu ser")
 
     args = parser.parse_args()
 
-    if args.gitpush:
-        git_push()
-    elif args.gitpull:
-        git_pull()
+    if args.git_command:
+        git_cmd = " ".join(args.git_command)
+        run_git_command(git_cmd)
     elif args.mac_address:
         get_mac_address()
     elif args.monitor_gpu_cli:
@@ -139,5 +118,5 @@ def main():
         print("No operation specified.")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

nlpertools-1.0.11/src/nlpertools/dataprocess/__init__.py

@@ -0,0 +1 @@
+from .dedupl import *

nlpertools-1.0.11/src/nlpertools/dataprocess/dedupl.py

@@ -0,0 +1,9 @@
+# 根据字段对一个元素为dict的list去重
+def deduplicate_dict_list(dict_list: list, key: str) -> list:
+    seen = set()
+    result = []
+    for d in dict_list:
+        if key in d and d[key] not in seen:
+            seen.add(d[key])
+            result.append(d)
+    return result

nlpertools-1.0.9/src/nlpertools/dataprocess.py → nlpertools-1.0.11/src/nlpertools/dataprocess/dp_main.py

@@ -8,7 +8,7 @@ from typing import List
 import numpy as np
 
 # from . import DB_CONFIG_FILE # cannot import name 'DB_CONFIG_FILE' from partially initialized module 'nlpertools'
-from .utils.package import *
+from ..utils.package import *
 
 main_special_characters = string.punctuation + string.digits + string.whitespace
 other_special_characters = (
@@ -19,6 +19,18 @@ other_special_characters = (
     "」﴾》"
 )
 
+def startwith(text: str, pattern_list: list) -> bool:
+    """
+    判断text是否以pattern_list中的某个pattern开头
+    :param text:
+    :param pattern_list:
+    :return:
+    """
+    for pattern in pattern_list:
+        if text.startswith(pattern):
+            return True
+    return False
+
 
 class Pattern:
     """

{nlpertools-1.0.9 → nlpertools-1.0.11}/src/nlpertools/io/dir.py

@@ -3,6 +3,7 @@
 # @Author  : youshu.Ji
 import os
 from pathlib import Path
+from typing import overload,Literal,Union
 
 
 # dir ----------------------------------------------------------------------
@@ -45,15 +46,34 @@ def get_filename(path, suffix=True) -> str:
         filename = filename.split('.')[0]
     return filename
 
-
-def listdir(dir_name, including_dir=True):
-    filenames = os.listdir(dir_name)
+"""
+因为os.listdir无法支持Path类型，虽然是bytelikepath,但是传入Path后只会返回字符串
+且无法只返回文件名
+故重新实现
+"""
+@overload
+def listdir(dir_name: Path, including_dir: Literal[True]) -> list[Path]: ...
+@overload
+def listdir(dir_name: str, including_dir: Literal[True]) -> list[str]: ...
+@overload
+def listdir(dir_name: Path, including_dir: Literal[False] = False) -> list[str]: ...
+@overload
+def listdir(dir_name: str, including_dir: Literal[False] = False) -> list[str]: ...
+
+def listdir(dir_name: Union[Path, str], including_dir: bool = False) -> list[Path] | list[str]:
+    """
+    including_dir=True -> list[Path] or list[str]
+    including_dir=False -> list[str]
+    """
+    filenames = os.listdir(str(dir_name))
     if including_dir:
-        return [os.path.join(dir_name, filename) for filename in filenames]
+        if isinstance(dir_name, Path):
+            return [dir_name / filename for filename in filenames]
+        else:
+            return [os.path.join(dir_name, filename) for filename in filenames]
     else:
         return list(filenames)
 
-
 def listdir_yield(dir_name, including_dir=True):
     filenames = os.listdir(dir_name)
     for filename in filenames: