nlpertools 1.0.6.dev0__py3-none-any.whl → 1.0.8__py3-none-any.whl

nlpertools/__init__.py

@@ -3,6 +3,7 @@
 # @Author  : youshu.Ji
 from .algo.kmp import *
 from .data_structure.base_structure import *
+from .draw import *
 from .dataprocess import *
 from .io.dir import *
 from .io.file import *
@@ -15,10 +16,8 @@ from .reminder import *
 from .utils_for_nlpertools import *
 from .wrapper import *
 from .monitor import *
+from .cli import *
 
-import os
 
 
-DB_CONFIG_FILE = os.path.join(os.path.dirname(__file__),"default_db_config.yml")
-
-__version__ = '1.0.5'
+__version__ = '1.0.8'

nlpertools/cli.py

@@ -0,0 +1,87 @@
+import argparse
+import os
+import uuid
+import sys
+
+import pyotp
+
+"""
+如何Debug cli.py
+"""
+
+
+def git_push():
+    """
+    针对国内提交github经常失败，自动提交
+    """
+    num = -1
+    while 1:
+        num += 1
+        print("retry num: {}".format(num))
+        info = os.system("git push --set-upstream origin main")
+        print(str(info))
+        if not str(info).startswith("fatal"):
+            print("scucess")
+            break
+
+
+def git_pull():
+    """
+    针对国内提交github经常失败，自动提交
+    """
+    num = -1
+    while 1:
+        num += 1
+        print("retry num: {}".format(num))
+        info = os.system("git pull")
+        print(str(info))
+        if not str(info).startswith("fatal") and not str(info).startswith("error"):
+            print("scucess")
+            break
+
+
+def get_mac_address():
+    mac = uuid.UUID(int=uuid.getnode()).hex[-12:]
+    mac_address = ":".join([mac[e:e + 2] for e in range(0, 11, 2)])
+    print("mac address 不一定准确")
+    print(mac_address)
+    return mac_address
+
+
+def get_2af_value(key):
+    """
+    key应该是7位的
+    """
+    print(key)
+    totp = pyotp.TOTP(key)
+    print(totp.now())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="CLI tool for git operations and getting MAC address.")
+    parser.add_argument('--gitpush', action='store_true', help='Perform git push operation.')
+    parser.add_argument('--gitpull', action='store_true', help='Perform git push operation.')
+    parser.add_argument('--mac_address', action='store_true', help='Get the MAC address.')
+
+    parser.add_argument('--get_2fa', action='store_true', help='Get the 2fa value.')
+    parser.add_argument('--get_2fa_key', type=str, help='Get the 2fa value.')
+
+    args = parser.parse_args()
+
+    if args.gitpush:
+        git_push()
+    elif args.gitpull:
+        git_pull()
+    elif args.mac_address:
+        get_mac_address()
+    elif args.get_2fa:
+        if args.get_2fa_key:
+            get_2af_value(args.get_2fa_key)
+        else:
+            print("Please provide a key as an argument.")
+    else:
+        print("No operation specified. Use --gitpush or --get_mac_address.")
+
+
+if __name__ == '__main__':
+    main()

nlpertools/data_client.py

@@ -1,3 +1,4 @@
+#encoding=utf-8
 # !/usr/bin/python3.8
 # -*- coding: utf-8 -*-
 # @Author  : youshu.Ji
@@ -5,9 +6,11 @@ import datetime
 import json
 import logging
 
-from . import DB_CONFIG_FILE
 from .io.file import read_yaml
 from .utils.package import *
+import os
+
+DB_CONFIG_FILE = os.path.join(os.path.dirname(__file__), "default_db_config.yml")
 
 # import aioredis
 # import happybase
@@ -28,21 +31,24 @@ class Neo4jOps(object):
     NEO4J_TIMEOUT = 0.3
     pass
 
+
 class SqliteOps(object):
-    import sqlite3
-    database_path = r'xx.db'
-    conn = sqlite3.connect(database_path)
-    c = conn.cursor()
-    sql = "select name from sqlite_master where type='table' order by name"
-    c.execute(sql)
-    print(c.fetchall())
-    sql = "select * from typecho_contents"
-    c.execute(sql)
-    res = c.fetchall()
-    print(res[3])
-
-    conn.commit()
-    conn.close()
+    pass
+    # import sqlite3
+    # database_path = r'xx.db'
+    # conn = sqlite3.connect(database_path)
+    # c = conn.cursor()
+    # sql = "select name from sqlite_master where type='table' order by name"
+    # c.execute(sql)
+    # print(c.fetchall())
+    # sql = "select * from typecho_contents"
+    # c.execute(sql)
+    # res = c.fetchall()
+    # print(res[3])
+    #
+    # conn.commit()
+    # conn.close()
+
 
 class MysqlOps(object):
     import pandas as pd
@@ -116,6 +122,41 @@ class EsOps(object):
         print(f"批量保存数据： {_res}")
 
 
+class MongoDB_BETA:
+    def __init__(self, host='localhost', port=27017, db_name=None, collection_name=None):
+        self.host = host
+        self.port = port
+        self.db_name = db_name
+        self.collection_name = collection_name
+        self.client = None
+        self.db = None
+        self.collection = None
+
+    def connect(self):
+        self.client = MongoClient(self.host, self.port)
+        self.db = self.client[self.db_name]
+        self.collection = self.db[self.collection_name]
+
+    def close(self):
+        if self.client:
+            self.client.close()
+
+    def insert_data(self, data):
+        if isinstance(data, list):
+            self.collection.insert_many(data)
+        else:
+            self.collection.insert_one(data)
+
+    def check_data_exists(self, query):
+        """
+        检查某个数据是否存在于数据库中
+        :param query: 查询条件
+        :return: 布尔值，表示数据是否存在
+        """
+        return self.collection.count_documents(query) > 0
+
+
+
 class MongoOps(object):
     from pymongo import MongoClient
     def __init__(self, config=global_db_config["mongo"]):
@@ -348,8 +389,6 @@ class KafkaOps(object):
             print(recv)
 
 
-
-
 class MilvusOps(object):
     def __init__(self, config=global_db_config.milvus):
         from pymilvus import connections, Collection

nlpertools/dataprocess.py

@@ -55,9 +55,9 @@ class Pattern:
     # 中文人名
     chinese_name_pattern = "(?:[\u4e00-\u9fa5·]{2,3})"
     # 英文人名
-    english_name_pattern = "(^[a-zA-Z][a-zA-Z\s]{0,20}[a-zA-Z]$)"
+    english_name_pattern = r"(^[a-zA-Z][a-zA-Z\s]{0,20}[a-zA-Z]$)"
     # 纯数字
-    pure_num_pattern = "\d+"
+    pure_num_pattern = r"\d+"
     # xxxx图/表 之类的表述
     pic_table_descript_pattern = ".{1,15}图"
 
@@ -66,20 +66,20 @@ class Pattern:
     hlink_pattern = (
         r"(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]"
     )
-    http_pattern = "(http|https):\/\/([\w.]+\/?)\S*/\S*"
+    http_pattern = r"(http|https):\/\/([\w.]+\/?)\S*/\S*"
     # 邮箱
-    email_pattern = "[A-Za-z0-9\u4e00-\u9fa5]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+"
+    email_pattern = r"[A-Za-z0-9\u4e00-\u9fa5]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+"
     # html 可能过于严格了
-    html_pattern = "<[\s\S]*?>"
+    html_pattern = r"<[\s\S]*?>"
     # 重复 “asdasdasdasd”
     repeat_pattern = "(.)\1+"
     # 日期
-    day_time_pattern = "\d{1,4}(-)(1[0-2]|0?[1-9])\1(0?[1-9]|[1-2]\d|30|31)"
+    day_time_pattern = r"\d{1,4}(-)(1[0-2]|0?[1-9])\1(0?[1-9]|[1-2]\d|30|31)"
     # 小时
-    hour_time_pattern = "(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d"
+    hour_time_pattern = r"(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d"
     # 股票
     stock_pattern = (
-        "(s[hz]|S[HZ])(000[\d]{3}|002[\d]{3}|300[\d]{3}|600[\d]{3}|60[\d]{4})"
+        r"(s[hz]|S[HZ])(000[\d]{3}|002[\d]{3}|300[\d]{3}|600[\d]{3}|60[\d]{4})"
     )
 
     # 一般是需要替换的
@@ -91,7 +91,7 @@ class Pattern:
     # 微博视频等
     weibo_pattern = r"([\s]\w+(的微博视频)|#|【|】|转发微博)"
     # @
-    at_pattern = "@\w+"
+    at_pattern = r"@\w+"
 
     # from https://github.com/bigscience-workshop/data-preparation pii
     year_patterns = [
@@ -116,7 +116,7 @@ class Pattern:
     ipv4_pattern = r'(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}'
     ipv6_pattern = r'(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])'
     ip_pattern = r"(?:^|[\b\s@?,!;:\'\")(.\p{Han}])(" + r"|".join(
-        [ipv4_pattern, ipv6_pattern]) + ")(?:$|[\s@,?!;:\'\"(.\p{Han}])"
+        [ipv4_pattern, ipv6_pattern]) + r")(?:$|[\s@,?!;:\'\"(.\p{Han}])"
 
     # https://regex101.com/r/EpA5B7/1
     email_line_pattern = r'''
@@ -466,7 +466,7 @@ class TextProcess(object):
         p = re.compile(pattern, re.S)
         text = p.sub("", text)
 
-        dr = re.compile("@\w+", re.S)
+        dr = re.compile(r"@\w+", re.S)
         text = dr.sub("", text)
 
         return text
@@ -527,7 +527,7 @@ class TextProcess(object):
             text = re.sub(pattern, replace, text)
         return text
 
-    def calc_proportion_zh(self,text):
+    def calc_proportion_zh(self, text):
         text = text.strip()
         # 如果是中国英文的情况，并且英文有空格分开
         if " " in text:
@@ -538,6 +538,8 @@ class TextProcess(object):
                 chinese_count += 1
             else:
                 pass
+
+
 class CopyFunc():
     # from https://github.com/lemon234071/clean-dialog
     def is_chinese_char(cp):
@@ -597,6 +599,20 @@ def convert_basic2fullwidth(sentence):
         new_sentence += char
     return new_sentence
 
+
+def clean_illegal_chars_for_excel(df):
+    # openpyxl 库写入 Excel 文件时，有一些非法字符，需要删除
+    # 定义一个函数来移除字符串中的非法字符
+    def remove_illegal_chars(s):
+        if isinstance(s, str):
+            # 移除 ASCII 码在非法范围内的字符
+            return re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F]', '', s)
+        return s
+
+    # 应用清理函数到数据框的每个元素
+    return df.map(remove_illegal_chars)
+
+
 if __name__ == "__main__":
     pattern_for_filter = [
         Pattern.redundancy_space_pattern,

nlpertools/draw/draw.py

@@ -0,0 +1,83 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+from ..utils.package import plt
+
+
+def confused_matrix(confuse_matrix):
+    import seaborn as sns
+    sns.set()
+    f, ax = plt.subplots()
+    ticklabels = ["l1", "l2", "l31"]
+    sns.heatmap(confuse_matrix, annot=True, fmt=".3g", ax=ax, cmap='rainbow',
+                xticklabels=ticklabels, yticklabels=ticklabels)  # 画热力图
+
+    ax.set_title('confusion matrix')  # 标题
+    ax.set_xlabel('predict')  # x轴
+    ax.set_ylabel('true')  # y轴
+    plt.show()
+
+    f.savefig('tmp.jpg', bbox_inches='tight')
+
+
+def plot_histogram(data, bin_size):
+    """
+    画直方图，超过1000的统一按1000算
+    :param data:
+    :param bin_size:
+    :return:
+    """
+    import matplotlib.pyplot as plt
+    import numpy as np
+    import pandas as pd
+    from matplotlib.ticker import MaxNLocator
+    # 将超过1000的值改为1000
+    def process_lengths(data):
+        return [length if length <= 1000 else 1003 for length in data]
+
+    # 前闭后开
+    min_num, max_num = 0, 1000
+    # min_num, max_num = min(data), max(data)
+
+    plt.figure(figsize=(12, 8))
+    processed_data = process_lengths(data)
+    bins = np.arange(0, 1000 + 2 * bin_size, bin_size)
+    # 绘制直方图
+    n, new_bins, patches = plt.hist(processed_data, bins=bins, edgecolor='black', color='skyblue', alpha=0.7,
+                                    linewidth=0)
+
+    # 添加"∞"的标签
+    # bins会改变
+    plt.gca().set_xticks(bins)
+    plt.gca().set_xticklabels([str(i) for i in plt.xticks()[0][:-1]] + ["∞"])
+
+    mean_val = np.mean(data)
+    plt.axvline(mean_val, color='red', linestyle='dashed', linewidth=1)
+    plt.text(mean_val + bin_size / 10, max(n) * 0.9, f'Mean: {mean_val:.2f}', color='red')
+
+    # 添加标题和标签
+    plt.title('Module Line Number Distribution', fontsize=16, fontweight='bold')
+    plt.xlabel('module line number', fontsize=14)
+    plt.ylabel('frequency', fontsize=14)
+
+    # 添加网格
+    plt.grid(True, linestyle='--', alpha=0.6)
+
+    # 美化x轴和y轴的刻度
+    plt.xticks(fontsize=12)
+    plt.yticks(fontsize=12)
+
+    # 在每个柱状图上显示数值
+    for i in range(len(patches)):
+        plt.text(patches[i].get_x() + patches[i].get_width() / 2, patches[i].get_height(),
+                 str(int(n[i])), ha='center', va='bottom', fontsize=12)
+    plt.gca().yaxis.set_major_locator(MaxNLocator(integer=True))
+    # 显示图表
+    plt.show()
+
+
+if __name__ == '__main__':
+    # 调整区间大小
+    bin_size = 50
+    # 示例模块长度数据
+    plot_histogram([1, 100, 999, 1000, 1002, 1100, 1150], bin_size)

nlpertools/draw/math_func.py

@@ -0,0 +1,33 @@
+# 数学函数
+def draw_log():
+    import matplotlib.pyplot as plt
+    import numpy as np
+    from matplotlib.ticker import MultipleLocator, FormatStrFormatter
+
+    # 生成一些数据
+    x = np.linspace(0.1, 10, 100)
+    # 默认log指的时loge
+    y = np.log(x)
+
+    # 创建一个新的图形和轴
+    fig, ax = plt.subplots()
+
+    # 绘制log图像
+    ax.plot(x, y)
+
+    # 设置图像标题和轴标签
+    ax.set_title("Logarithmic Function")
+    ax.set_xlabel("x")
+    ax.set_ylabel("log(x)")
+    # 设置横坐标的刻度间隔为1
+    ax.xaxis.set_major_locator(MultipleLocator(1))
+
+    # 设置横坐标的刻度格式
+    ax.xaxis.set_major_formatter(FormatStrFormatter("%.1f"))
+    # 添加x=1的虚线
+    ax.axvline(x=1, linestyle="--", color="gray")
+    # 添加y=1的虚线
+    ax.axhline(y=0, linestyle="--", color="gray")
+
+    # 显示图像
+    plt.show()

nlpertools/io/dir.py

@@ -10,7 +10,30 @@ def j_mkdir(name):
     os.makedirs(name, exist_ok=True)
 
 
-def get_filename(path) -> str:
+def j_walk(name, suffix=None):
+    paths = []
+    for root, dirs, files in os.walk(name):
+        for file in files:
+            path = os.path.join(root, file)
+            if not (suffix and not path.endswith(suffix)):
+                paths.append(path)
+    return paths
+
+
+def windows_to_wsl_path(windows_path):
+    # 转换驱动器号
+    if windows_path[1:3] == ':\\':
+        drive_letter = windows_path[0].lower()
+        path = windows_path[2:].replace('\\', '/')
+        wsl_path = f'/mnt/{drive_letter}{path}'
+    else:
+        # 如果路径不是以驱动器号开头，则直接替换路径分隔符
+        wsl_path = windows_path.replace('\\', '/').replace("'", "\'")
+
+    return wsl_path
+
+
+def get_filename(path, suffix=True) -> str:
     """
     返回路径最后的文件名
     :param path:
@@ -18,11 +41,20 @@ def get_filename(path) -> str:
     """
     # path = r'***/**/***.txt'
     filename = os.path.split(path)[-1]
+    if not suffix:
+        filename = filename.split('.')[0]
     return filename
 
 
 def j_listdir(dir_name, including_dir=True):
-    #  yield
+    filenames = os.listdir(dir_name)
+    if including_dir:
+        return [os.path.join(dir_name, filename) for filename in filenames]
+    else:
+        return list(filenames)
+
+
+def j_listdir_yield(dir_name, including_dir=True):
     filenames = os.listdir(dir_name)
     for filename in filenames:
         if including_dir:

nlpertools/io/file.py

@@ -5,7 +5,6 @@ import codecs
 import json
 import pickle
 import random
-import time
 from itertools import (takewhile, repeat)
 import pandas as pd
 # import omegaconf
@@ -15,10 +14,16 @@ from ..utils.package import *
 LARGE_FILE_THRESHOLD = 1e5
 
 
+def safe_filename(filename: str) -> str:
+    for char in ['\\', '/', ':', '*', '?', '"', '<', '>', '|']:
+        filename = filename.replace(char, '_')
+    return filename
+
+
 def read_yaml(path, omega=False):
     if omega:
         return omegaconf.OmegaConf.load(path)
-    return yaml.load(codecs.open(path), Loader=yaml.FullLoader)
+    return yaml.load(codecs.open(path, encoding='utf-8'), Loader=yaml.FullLoader)
 
 
 def _merge_file(filelist, save_filename, shuffle=False):
@@ -52,7 +57,7 @@ load_from_json
 
 
 # 读txt文件 一次全读完 返回list 去换行
-def readtxt_list_all_strip(path, encoding='utf-8'):
+def readtxt_list_all_strip(path, encoding='utf-8') -> list:
     file_line_num = iter_count(path)
     lines = []
     with codecs.open(path, 'r', encoding) as r:
@@ -67,7 +72,7 @@ def readtxt_list_all_strip(path, encoding='utf-8'):
 
 
 # 读txt 一次读一行 最后返回list
-def readtxt_list_each(path):
+def readtxt_list_each(path) -> list:
     lines = []
     with codecs.open(path, 'r', 'utf-8') as r:
         line = r.readline()
@@ -77,7 +82,7 @@ def readtxt_list_each(path):
     return lines
 
 
-def readtxt_list_each_strip(path):
+def readtxt_list_each_strip(path) -> list:
     """
     yield方法
     """
@@ -89,14 +94,14 @@ def readtxt_list_each_strip(path):
 
 
 # 读txt文件 一次全读完 返回list
-def readtxt_list_all(path):
+def readtxt_list_all(path) -> list:
     with codecs.open(path, 'r', 'utf-8') as r:
         lines = r.readlines()
         return lines
 
 
 # 读byte文件 读成一条string
-def readtxt_byte(path, encoding="utf-8"):
+def readtxt_byte(path, encoding="utf-8") -> str:
     with codecs.open(path, 'rb') as r:
         lines = r.read()
         lines = lines.decode(encoding)
@@ -104,7 +109,7 @@ def readtxt_byte(path, encoding="utf-8"):
 
 
 # 读txt文件 读成一条string
-def readtxt_string(path, encoding="utf-8"):
+def readtxt_string(path, encoding="utf-8") -> str:
     with codecs.open(path, 'r', encoding) as r:
         lines = r.read()
         return lines.replace('\r', '')
@@ -261,6 +266,7 @@ def save_to_mongo():
     """
     pass
 
+
 def load_from_mongo():
     pass
 
@@ -274,4 +280,4 @@ def unmerge_cells_df(df) -> pd.DataFrame:
             else:
                 values.append(i)
         df[column] = values
-    return df
+    return df

nlpertools/ml.py

@@ -18,9 +18,8 @@ from .utils.package import *
 
 
 def calc_llm_train_activation_memory(
-    model_name, sequence_length, batch_size, hidden_dim, lay_number, attention_heads_num, gpu_num=1
+        model_name, sequence_length, batch_size, hidden_dim, lay_number, attention_heads_num, gpu_num=1
 ):
-
     """
     return bytes
 
@@ -33,18 +32,18 @@ def calc_llm_train_activation_memory(
     # FFN
     # Layer Norm
     r1 = (
-        sequence_length
-        * batch_size
-        * hidden_dim
-        * lay_number
-        * (34 + 5 * attention_heads_num * sequence_length / hidden_dim)
+            sequence_length
+            * batch_size
+            * hidden_dim
+            * lay_number
+            * (34 + 5 * attention_heads_num * sequence_length / hidden_dim)
     )
     # reference2
     r2 = (
-        lay_number*(2 * sequence_length * attention_heads_num + 16 * hidden_dim)
-        * sequence_length
-        * batch_size
-        / gpu_num
+            lay_number * (2 * sequence_length * attention_heads_num + 16 * hidden_dim)
+            * sequence_length
+            * batch_size
+            / gpu_num
     )
     print(r1)
     print(r2)
@@ -80,7 +79,7 @@ class DataStructure:
     }
     ner_input_example = "这句话一共有两个实体分别为大象和老鼠。"
     ner_label_example = (
-        list("OOOOOOOOOOOOO") + ["B-s", "I-s"] + ["O"] + ["B-o", "I-o"] + ["O"]
+            list("OOOOOOOOOOOOO") + ["B-s", "I-s"] + ["O"] + ["B-o", "I-o"] + ["O"]
     )
 
 
@@ -135,7 +134,7 @@ class STEM(object):
             if each_srl:
                 args = []
                 for arg in each_srl:
-                    args.extend(seg[arg[1] : arg[2] + 1])
+                    args.extend(seg[arg[1]: arg[2] + 1])
                 # 添加上谓词
                 args.insert(each_srl[0][2] - each_srl[0][1] + 1, seg[wdx])
                 events.append(args)
@@ -174,7 +173,7 @@ def subject_object_labeling(spo_list, text):
         q_list_length = len(q_list)
         k_list_length = len(k_list)
         for idx in range(k_list_length - q_list_length + 1):
-            t = [q == k for q, k in zip(q_list, k_list[idx : idx + q_list_length])]
+            t = [q == k for q, k in zip(q_list, k_list[idx: idx + q_list_length])]
             # print(idx, t)
             if all(t):
                 # print(idx)
@@ -187,8 +186,8 @@ def subject_object_labeling(spo_list, text):
         if len(spo) == 2:
             labeling_list[idx_start + 1] = "I-" + spo_type
         elif len(spo) >= 3:
-            labeling_list[idx_start + 1 : idx_start + len(spo)] = ["I-" + spo_type] * (
-                len(spo) - 1
+            labeling_list[idx_start + 1: idx_start + len(spo)] = ["I-" + spo_type] * (
+                    len(spo) - 1
             )
         else:
             pass
@@ -239,12 +238,12 @@ def convert_crf_format_10_fold(corpus, objdir_path):
     split_position = int(len(corpus) / 10)
     for k in range(0, 10):
         if k == 9:
-            dev_set = corpus[k * split_position :]
+            dev_set = corpus[k * split_position:]
             train_set = corpus[: k * split_position]
         else:
-            dev_set = corpus[k * split_position : (k + 1) * split_position]
+            dev_set = corpus[k * split_position: (k + 1) * split_position]
             train_set = (
-                corpus[: k * split_position] + corpus[(k + 1) * split_position :]
+                    corpus[: k * split_position] + corpus[(k + 1) * split_position:]
             )
         writetxt_w_list(
             train_set, os.path.join(objdir_path, "train{}.txt".format(k + 1))
@@ -292,12 +291,41 @@ def kfold_txt(corpus, path, k=9, is_shuffle=True):
     if is_shuffle:
         random.shuffle(corpus)
     split_position = int(len(corpus) / 10)
-    train_set, dev_set = corpus[: k * split_position], corpus[k * split_position :]
+    train_set, dev_set = corpus[: k * split_position], corpus[k * split_position:]
     writetxt_w_list(train_set, os.path.join(path, "train.tsv"), num_lf=1)
     writetxt_w_list(dev_set, os.path.join(path, "test.tsv"), num_lf=1)
     writetxt_w_list(dev_set, os.path.join(path, "dev.tsv"), num_lf=1)
 
 
+def sample():
+    import pandas as pd
+    from sklearn.model_selection import StratifiedShuffleSplit
+
+    # 假设 df 是你的 DataFrame
+
+    df = pd.DataFrame({
+        "count_line": [i for i in range(100)],
+        "x": [i for i in range(100)],
+        "y": [i // 10 for i in range(100)],
+    })
+    print(df)
+    # count_line 是用于分层抽样的字段
+
+    # 创建 StratifiedShuffleSplit 对象，设置测试集比例为 0.1
+    split = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=42)
+
+    # 获取训练集和测试集的索引
+    train_index, test_index = next(split.split(df, df['y']))
+
+    # 根据索引划分训练集和测试集
+    train_df = df.loc[train_index]
+    test_df = df.loc[test_index]
+
+    # 打印训练集和测试集的行数
+    print("训练集行数：", len(train_df))
+    print("测试集行数：", len(test_df))
+
+
 def kfold_df(df, save_dir=None):
     """
     划分train test val集， 写为windows可读的csv。
@@ -389,7 +417,7 @@ def split_sentence(sentence, language="chinese", cross_line=True):
     for idx, char in enumerate(sentence):
         if idx == len(sentence) - 1:
             if char in split_signs:
-                sentences.append(sentence[start_idx : idx + 1].strip())
+                sentences.append(sentence[start_idx: idx + 1].strip())
                 start_idx = idx + 1
             else:
                 sentences.append(sentence[start_idx:].strip())
@@ -399,10 +427,10 @@ def split_sentence(sentence, language="chinese", cross_line=True):
                     if idx < len(sentence) - 2:
                         # 处理。”。
                         if sentence[idx + 2] not in split_signs:
-                            sentences.append(sentence[start_idx : idx + 2].strip())
+                            sentences.append(sentence[start_idx: idx + 2].strip())
                             start_idx = idx + 2
                 elif sentence[idx + 1] not in split_signs:
-                    sentences.append(sentence[start_idx : idx + 1].strip())
+                    sentences.append(sentence[start_idx: idx + 1].strip())
                     start_idx = idx + 1
     return sentences
 
@@ -480,4 +508,4 @@ if __name__ == "__main__":
         attention_heads_num=32,
         gpu_num=1
     )
-    print(res, "G")
+    print(res, "G")

nlpertools/other.py

@@ -5,10 +5,13 @@ import itertools
 import os
 import re
 import string
+import subprocess
+import threading
 from concurrent.futures import ThreadPoolExecutor
 from functools import reduce
 import math
 import datetime
+import difflib
 import psutil
 from .io.file import writetxt_w_list, writetxt_a
 # import numpy as np
@@ -27,6 +30,134 @@ ENGLISH_PUNCTUATION = list(',.;:\'"!?<>()')
 OTHER_PUNCTUATION = list('!@#$%^&*')
 
 
+def get_diff_parts(str1, str2):
+    # 创建一个 SequenceMatcher 对象
+    matcher = difflib.SequenceMatcher(None, str1, str2)
+
+    # 获取差异部分
+    diff_parts = []
+    for tag, i1, i2, j1, j2 in matcher.get_opcodes():
+        if tag == 'replace' or tag == 'delete' or tag == 'insert':
+            diff_parts.append((tag, str1[i1:i2], str2[j1:j2]))
+
+    return diff_parts
+
+
+def run_cmd_with_timeout(cmd, timeout):
+    """
+    https://juejin.cn/post/7391703459803086848
+    """
+    process = subprocess.Popen(cmd, shell=True, encoding="utf-8", errors="ignore", stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    res = [None]
+
+    def target():
+        try:
+            ans = process.communicate()
+            res[0] = ans
+        except subprocess.TimeoutExpired:
+            process.kill()
+            process.communicate()
+
+    thread = threading.Thread(target=target)
+    thread.start()
+    thread.join(timeout)
+    if thread.is_alive():
+        print(f"Terminating {cmd}")
+        process.terminate()
+        thread.join()
+        print("Terminated successfully")
+        return False, f"{cmd} is running over {timeout}s"
+    if process.returncode == 0:
+        # res[0][0] 是output
+        return True, res[0][0]
+    else:
+        return False, res[0][0]
+
+
+def print_three_line_table(df):
+    # TODO 这里需要添加可以支持excel里变红的功能
+    import webbrowser
+
+    # import pandas as pd
+    # data = {'from_pc': ['valid_data', 'illegal_char', 'more_data'],
+    #         'rom_pc': ['another_valid_data', 'illegal_char', 'data']}
+    # df = pd.DataFrame(data)
+
+    # 将 DataFrame 转换为 HTML 表格
+    html_table = df.to_html(index=False)
+    html_table = html_table.replace('border="1"', 'border="0"')
+
+    first_line_px = str(2)
+    second_line_px = str(1)
+    third_line_px = str(2)
+    # 定义三线表的 CSS 样式
+    # // thead 表头
+    # // tr 行
+    # // td 单元格
+    head = """<!DOCTYPE html>
+    <html lang="zh">
+    <head>
+        <meta charset="UTF-8">
+        <title>页面标题</title>
+    </head>"""
+    style = """
+    <style>
+
+        table {
+            border-collapse: collapse;
+        }
+
+        tr, td, th {
+            text-align: center; /* 水平居中文本 */
+            vertical-align: middle; /* 垂直居中文本 */
+        }
+        thead tr {
+            border-top: (first_line_px)px solid black;
+            border-bottom: (second_line_px)px solid black;
+        }
+
+        thead th {
+            border-bottom: (second_line_px)px solid black;
+        }
+
+        tbody tr td {
+            border-bottom: 0px solid black;
+        }
+
+        tbody tr:last-child td {
+            border-bottom: (third_line_px)px solid black;
+        }
+    </style>"""
+    style = style.replace("(first_line_px)", first_line_px).replace("(second_line_px)", second_line_px).replace(
+        "(third_line_px)", third_line_px)
+    # 将 CSS 样式和 HTML 表格结合起来
+    html = f"{style}{html_table}"
+    print(html)
+    temp_file_path = "temp.html"
+    # 将 HTML 保存到文件中
+    with open(temp_file_path, "w") as f:
+        f.write(html)
+    webbrowser.open('file://' + os.path.realpath(temp_file_path))
+
+
+def jprint(obj, depth=0):
+    if isinstance(obj, dict):
+        sep = "-" * (10 - depth * 3)
+        for k, v in obj.items():
+            print(depth * "|", sep, k, sep)
+            jprint(v)
+    elif isinstance(obj, list):
+        for v in obj:
+            jprint(v, depth + 1)
+    else:
+        print(obj)
+
+
+def print_split(sign="=", num=20):
+    print(sign * num)
+
+
 def seed_everything():
     import torch
     # seed everything
@@ -82,21 +213,6 @@ def convert_np_to_py(obj):
         return obj
 
 
-def git_push():
-    """
-    针对国内提交github经常失败，自动提交
-    """
-    num = -1
-    while 1:
-        num += 1
-        print("retry num: {}".format(num))
-        info = os.system("git push --set-upstream origin main")
-        print(str(info))
-        if not str(info).startswith("fatal"):
-            print("scucess")
-            break
-
-
 def snake_to_camel(s: str) -> str:
     """
     author: u
@@ -235,24 +351,19 @@ def stress_test(func, ipts):
     return results
 
 
-def get_substring_loc(text, subtext):
-    res = re.finditer(
-        subtext.replace('\\', '\\\\').replace('?', '\?').replace('(', '\(').replace(')', '\)').replace(']',
-                                                                                                       '\]').replace(
-            '[', '\[').replace('+', '\+'), text)
-    l, r = [i for i in res][0].regs[0]
-    return l, r
-
-
 def squeeze_list(high_dim_list):
     return list(itertools.chain.from_iterable(high_dim_list))
 
 
 def unsqueeze_list(flatten_list, each_element_len):
+    # 该函数是错的，被split_list替代了
     two_dim_list = [flatten_list[i * each_element_len:(i + 1) * each_element_len] for i in
                     range(len(flatten_list) // each_element_len)]
     return two_dim_list
 
+def split_list(input_list, chunk_size):
+    # 使用列表推导式将列表分割成二维数组
+    return [input_list[i:i + chunk_size] for i in range(0, len(input_list), chunk_size)]
 
 def auto_close():
     """

{nlpertools-1.0.6.dev0.dist-info → nlpertools-1.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nlpertools
-Version: 1.0.6.dev0
+Version: 1.0.8
 Summary: A small package about small basic IO operation when coding
 Home-page: https://github.com/lvzii/nlpertools
 Author: youshuJi
@@ -12,6 +12,11 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: psutil
+Provides-Extra: torch
+Requires-Dist: torch; extra == "torch"
 
 <div align="center">
   <h4 align="center">
@@ -23,9 +28,6 @@ License-File: LICENSE
 </div>
 
 
-# 当前版本
-
-1.0.5
 
 # 说明
 
@@ -75,9 +77,9 @@ https://nlpertools.readthedocs.io/en/latest/
       def __init__(self, IPT_MODEL_PATH):
           self.ltp = LTP(IPT_MODEL_PATH)
   ```
-  通过pyinstrument判断，超过1s的包即采用这种方式
+  通过`pyinstrument`判断，超过1s的包即采用这种方式
   - 2s+ happybase、seaborn、torch、jieba
-  - 1s+
+  - 1s+ /
   - 0.5s+ pandas elasticsearch transformers xgboost nltk mongo
 
 
@@ -85,6 +87,8 @@ https://nlpertools.readthedocs.io/en/latest/
 
 - [readthedoc 检查文档构建状况](https://readthedocs.org/projects/nlpertools/builds)
 
+- [打包发布指南](https://juejin.cn/post/7369413136224878644)
+
 - 发布版本需要加tag
 
 ## 开发哲学
@@ -106,6 +110,23 @@ b = nlpertools.io.file.readtxt_list_all_strip('res.txt')
 ```
 
 ```bash
-# 监控gpu显存
-python -m nlpertools
+# 生成pypi双因素认证的实时密钥(需要提供key)
+python -m nlpertools.get_2fa your_key
+
+## git
+python nlpertools.cli --git_push
+python nlpertools.cli --git_pull
+
+# 以下功能被nvitop替代，不推荐使用
+## 监控gpu显存
+python -m nlpertools.monitor.gpu
+## 监控cpu
+python -m  nlpertools.monitor.memory
 ```
+
+## 一些常用项目
+
+nvitop
+
+ydata-profiling
+

{nlpertools-1.0.6.dev0.dist-info → nlpertools-1.0.8.dist-info}/RECORD

@@ -1,12 +1,14 @@
-nlpertools/__init__.py,sha256=MTicDqOofy0loriBCpRcUdy8yQ9j5c3dFrwCVWWa8ic,536
-nlpertools/data_client.py,sha256=83jv7r7CsL8FYt7fJy_8ZKNG0XfAYiU8UPTYUURx4m8,13534
-nlpertools/dataprocess.py,sha256=z3nLWvWbGVi8N2mmOm70itgUhb_vhQYGVWWpVMwziNg,22658
+nlpertools/__init__.py,sha256=h7JJEN_JRn3iKcqIcaFgYtAjP90XiT1KILrm8utoHvQ,483
+nlpertools/cli.py,sha256=xDl_tWl9pfqQ3PUdd7oesvgM2FVqnaw8dFFliEX5c4Y,2203
+nlpertools/data_client.py,sha256=esX8lUQrTui4uVkqPfhpHVok7Eq6ywpuemKjLeqoglc,14674
+nlpertools/dataprocess.py,sha256=v1mobuYN7I3dT6xIKlNOHVtcg31YtjF6FwNPTxeBFFY,23153
 nlpertools/default_db_config.yml,sha256=E1K9k_xzXVlsf-HJQh8kyHXHYuvTpD12jD4Hfe5rUk8,606
-nlpertools/ml.py,sha256=n_WZMCAuD4KaaeYixq0RRiHiU1RuYLMV3dkGV7k4OaE,16798
+nlpertools/get_2fa.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nlpertools/ml.py,sha256=z-0ep9svAyzcS2n7Lsyqo65VEQRGzWKFMLdZofCv1LQ,17716
 nlpertools/movie.py,sha256=rkyOnAXdsbWfMSbi1sE1VNRT7f66Hp9BnZsN_58Afmw,897
 nlpertools/nlpertools_config.yml,sha256=ksXejxFs7pxR47tNAsrN88_4gvq9PCA2ZMO07H-dJXY,26
 nlpertools/open_api.py,sha256=uyTY00OUlM57Cn0Wm0yZXcIS8vAszy9rKnDMBEWfWJM,1744
-nlpertools/other.py,sha256=TuMx0B4qL_0kIJu469k94gE5NsncCE-IEo3ejt3gH9A,11583
+nlpertools/other.py,sha256=CeUea17Oe5MV_r-CmeYdAhdj5kWLvmxoDDgRc56o7bE,14704
 nlpertools/pic.py,sha256=13aaFJh3USGYGs4Y9tAKTvWjmdQR4YDjl3LlIhJheOA,9906
 nlpertools/plugin.py,sha256=LB7j9GdoQi6TITddH-6EglHlOa0WIHLUT7X5vb_aIZY,1168
 nlpertools/reminder.py,sha256=wiXwZQmxMck5vY3EvG8_oakP3FAdjGTikAIOiTPUQrs,2977
@@ -22,9 +24,12 @@ nlpertools/algo/template.py,sha256=9vsHr4g3jZZ5KVU_2I9i97o8asRXq-8pSaCXIv0sHeM,2
 nlpertools/algo/union.py,sha256=0l7lGZbw1qIfW1z5TE8Oo3tybL1bKIP5rzpa5ZT-vLQ,249
 nlpertools/data_structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nlpertools/data_structure/base_structure.py,sha256=gVUvJZ5jsCAswRETTpMwcEjLKoageWiTuCKNEwIWKWk,2641
+nlpertools/draw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nlpertools/draw/draw.py,sha256=PgdG7unpCtbbQdYISODTYMV7p10GwWDh9czeURkG0x4,2629
+nlpertools/draw/math_func.py,sha256=0NQ22Dfi9DFG6Bg_hXnCT27w65-dqpOOIgZX7oUIW-Q,881
 nlpertools/io/__init__.py,sha256=YMuKtC2Ddh5dL5MvXjyUKYOOuqzFYUhBPFaP2kyFG9I,68
-nlpertools/io/dir.py,sha256=cK65qSZ9Tu5HrNrDiNyx0PEPrPldRSq34LpCWLz9WHc,1272
-nlpertools/io/file.py,sha256=tALfmzFRWztYpsmmBNvz-U6DXBe6cxef9j3_KexWdRM,6970
+nlpertools/io/dir.py,sha256=p7J34qUxYCqKSO5DQMhL8FxFcHDrwn_1lIxNl0klasU,2267
+nlpertools/io/file.py,sha256=CsFdluEczuz3fonbeZi9dHPasL1Hm18JL3Aux2ziQMU,7198
 nlpertools/monitor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nlpertools/monitor/gpu.py,sha256=M59O6i0hlew7AzXZlaVZqbZA5IR93OhBY2WI0-T_HtY,531
 nlpertools/monitor/memory.py,sha256=9t6q9BC8VVx4o3G4sBCn7IoQRx272zMPjSnL3yvTBAQ,657
@@ -36,8 +41,9 @@ nlpertools/utils/package.py,sha256=wLg_M8j7Y6ReRjWHWCWoZJHrzEwuAr9TyG2jvb7OQCo,3
 nlpertools/utils/package_v1.py,sha256=sqgFb-zbTdMd5ziJLY6YUPqR49qUNZjxBH35DnyR5Wg,3542
 nlpertools/utils/package_v2.py,sha256=WOcsguWfUd4XSAfmPgCtL8HtUbqJ6GRSMHb0OsB47r0,3932
 nlpertools_helper/__init__.py,sha256=obxRUdZDctvcvK_iA1Dx2HmQFMlMzJto-xDPryq1lJ0,198
-nlpertools-1.0.6.dev0.dist-info/LICENSE,sha256=SBcMozykvTbZJ--MqSiKUmHLLROdnr25V70xCQgEwqw,11331
-nlpertools-1.0.6.dev0.dist-info/METADATA,sha256=VLFgFPh2o0YeWXEEoPIinO_rn6--mhUFU4vBASPPoNc,2772
-nlpertools-1.0.6.dev0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-nlpertools-1.0.6.dev0.dist-info/top_level.txt,sha256=_4q4MIFvMr4cAUbhWKWYdRXIXsF4PJDg4BUsZvgk94s,29
-nlpertools-1.0.6.dev0.dist-info/RECORD,,
+nlpertools-1.0.8.dist-info/LICENSE,sha256=SBcMozykvTbZJ--MqSiKUmHLLROdnr25V70xCQgEwqw,11331
+nlpertools-1.0.8.dist-info/METADATA,sha256=v2doRda1amZbXXfIYuzo-rFPvTICt3ByDCKVr6gsUw0,3276
+nlpertools-1.0.8.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+nlpertools-1.0.8.dist-info/entry_points.txt,sha256=XEazQ4vUwJMoMAgAwk1Lq4PRQGklPkPBaFkiP0zN_JE,45
+nlpertools-1.0.8.dist-info/top_level.txt,sha256=_4q4MIFvMr4cAUbhWKWYdRXIXsF4PJDg4BUsZvgk94s,29
+nlpertools-1.0.8.dist-info/RECORD,,

{nlpertools-1.0.6.dev0.dist-info → nlpertools-1.0.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

nlpertools-1.0.8.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+ncli = nlpertools.cli:main