neverlib 0.1.8__py3-none-any.whl → 0.2.0__py3-none-any.whl

neverlib/utils/audio_split.py

@@ -0,0 +1,266 @@
+'''
+Author: 凌逆战 | Never
+Date: 2025-04-10 18:07:03
+Description: 音频切割
+'''
+import os
+import random
+import subprocess
+from tqdm import tqdm
+import soundfile as sf
+import numpy as np
+from utils import get_path_list
+from pydub import AudioSegment
+
+
+def audio_split_ffmpeg(source_path, target_path, sr, channel_num, duration, endwith="*.pcm"):
+    """ 切割音频切不准，会留点尾巴0.016s
+    使用ffmpeg分割音频, 分割为短音频(单位:秒), 似乎无法非常准确的分割到指定长度
+    :param source_path: 源音频路径
+    :param target_path: 目标音频路径
+    :param sr: 源音频采样率
+    :param channel_num: 源音频声道数
+    :param duration: 分割为时长(短音频)(单位:秒)
+    :param endwith: 音频格式(支持pcm和wav)
+    """
+    wav_path_list = get_path_list(source_path, end=endwith)
+    print("待分割的音频数: ", len(wav_path_list))
+    for wav_path in wav_path_list:
+        wav_folder = wav_path[:-4].replace(source_path, target_path)
+        os.makedirs(wav_folder, exist_ok=True)
+
+        if endwith == "*.pcm":
+            # 将pcm文件切割成30s的语音, 有括号会报错
+            # ffmpeg -f s16le -ar 16000 -ac 6 -i ./NO.1_A3035_2.pcm -f segment -segment_time 30 -c copy NO.1_A3035_2/%03d.wav
+            command = ["ffmpeg", "-f", "s16le", "-ar", f"{sr}", "-ac", str(channel_num),
+                       "-i", wav_path, "-f", "segment", "-segment_time",
+                       f"{duration}", "-c", "copy", f"{wav_folder}/%03d.wav"]
+            subprocess.run(command, check=True)
+        elif endwith == "*.wav":
+            # ffmpeg -i ./NO.1_A3035_2.wav -f segment -segment_time 30 -c copy NO.1_A3035_2/%03d.wav
+            command = ["ffmpeg", "-i", wav_path, "-f", "segment", "-segment_time",
+                       f"{duration}", "-c", "copy", f"{wav_folder}/%03d.wav"]
+            subprocess.run(command, check=True)
+        else:
+            assert False, "不支持的音频格式"
+    print("分割完毕: done!")
+
+
+def audio_split_sox(source_path, target_path, duration, endwith="*.wav"):
+    """
+    使用sox分割音频, 分割为短音频(单位:秒), 可以非常准确的分割到指定长度
+    :param source_path: 源音频路径
+    :param target_path: 目标音频路径
+    :param duration: 分割为时长(短音频)(单位:秒)
+    :param endwith: 音频格式(只支持wav)
+    """
+    wav_path_list = get_path_list(source_path, end=endwith)
+
+    for wav_path in wav_path_list:
+        wav_folder = wav_path[:-4].replace(source_path, target_path)
+        os.makedirs(wav_folder, exist_ok=True)
+
+        output_pattern = f"{wav_folder}/%.wav"
+
+        if endwith == "*.wav":
+            # 对 WAV 文件直接进行分割
+            os.system(f"sox {wav_path} {output_pattern} trim 0 {str(duration)} : newfile : restart")
+        else:
+            assert False, "不支持的音频格式"
+
+    print("分割完毕: done!")
+
+
+def audio_split_np(source_path, target_path, sr, channel_num, duration, endwith="*.pcm"):
+    """
+    使用numpy读取pcm文件并切割保存为wav文件, 保持通道数一致, 保存不足30秒的最后一段音频
+    :param source_path: 源音频路径
+    :param target_path: 目标音频路径
+    :param sr: 采样率
+    :param channel_num: 声道数
+    :param duration: 分割的时长 (秒)
+    :param endwith: 音频格式 (支持 pcm)
+    """
+    wav_path_list = get_path_list(source_path, end=endwith)  # 获取音频文件列表
+    print("待分割的音频数: ", len(wav_path_list))
+
+    segment_length_samples = duration * sr  # 每个切片音频的采样点数
+
+    for wav_path in wav_path_list:
+        print("正在分割: ", wav_path)
+        wav_folder = wav_path[:-4].replace(source_path, target_path)
+        os.makedirs(wav_folder, exist_ok=True)
+
+        # 注意读取时使用正确的dtype(例如int16表示16位PCM)
+        pcm_data = np.fromfile(wav_path, dtype=np.int16)
+        pcm_data = pcm_data.reshape(-1, channel_num)
+
+        # 计算分割的数量
+        num_segments = len(pcm_data) // segment_length_samples
+
+        # 切割并保存每段音频
+        for i in tqdm(range(num_segments)):
+            start_idx = i * segment_length_samples
+            end_idx = (i + 1) * segment_length_samples
+            segment = pcm_data[start_idx:end_idx]
+            segment_filename = os.path.join(wav_folder, f"{i + 1:03d}.wav")   # 保存为wav文件
+            sf.write(segment_filename, segment, sr, subtype='PCM_16')
+
+        # 如果剩余部分少于30秒, 保存最后一段不足30秒的音频
+        remaining_samples = len(pcm_data) % segment_length_samples
+        if remaining_samples > 0:
+            segment = pcm_data[-remaining_samples:]
+            # 保存剩余部分
+            remaining_filename = os.path.join(wav_folder, f"{num_segments + 1:03d}.wav")
+            sf.write(remaining_filename, segment, sr, subtype='PCM_16')
+
+    print("分割完毕: done!")
+
+
+def audio_split_pydub(source_path, target_path, sr, channel_num, duration, endwith="*.pcm", sample_width=2):
+    """
+    使用pydub分割音频, 进行精确的分割
+    :param source_path: 源音频路径
+    :param target_path: 目标音频路径
+    :param sr: 源音频采样率
+    :param channel_num: 源音频声道数
+    :param duration: 分割为时长(短音频)(单位:秒), 必须是1s的整数倍
+    :param endwith: 音频格式(支持pcm和wav)
+    :param sample_width: 音频的样本宽度(字节数), 默认为2, 表示16位音频
+    """
+    assert duration % 1 == 0, "duration必须是1s的整数倍"
+    wav_path_list = get_path_list(source_path, end=endwith)  # 获取音频文件列表
+    print("待分割的音频数: ", len(wav_path_list))
+
+    for wav_path in wav_path_list:
+        print("正在分割: ", wav_path)
+        wav_folder = wav_path[:-4].replace(source_path, target_path)  # 设置目标文件夹
+        os.makedirs(wav_folder, exist_ok=True)
+
+        # 使用pydub加载音频
+        if endwith == "*.pcm":
+            # 读取pcm文件, 指定采样率、声道数和样本宽度
+            audio = AudioSegment.from_file(wav_path, format="raw", channels=channel_num, frame_rate=sr, sample_width=sample_width)
+        elif endwith == "*.wav":
+            # 读取wav文件
+            audio = AudioSegment.from_wav(wav_path)
+        else:
+            assert False, "不支持的音频格式"
+
+        # 计算每段的时长(以毫秒为单位)
+        segment_length = duration * 1000  # 转换为毫秒
+
+        # 切割音频并保存为多个文件
+        segment_number = 1
+        for i in tqdm(range(0, len(audio), segment_length)):
+            segment = audio[i:i + segment_length]
+            segment_filename = os.path.join(wav_folder, f"{segment_number:03d}.wav")
+            segment.export(segment_filename, format="wav")
+            segment_number += 1
+
+    print("分割完毕: done!")
+
+
+def audio_split_random(source_dir, target_dir, min_duration=3, max_duration=10, sr=16000):
+    """
+    将音频切割成 3 到 10 秒的多个片段并保存。
+    参数:
+    - input_audio_path: 输入音频文件路径
+    - output_dir: 输出音频文件夹路径
+    - min_duration: 最短切割片段长度 (秒), 默认3秒
+    - max_duration: 最长切割片段长度 (秒), 默认10秒
+    - sample_rate: 采样率, 默认16000
+    """
+    wav_path_list = get_path_list(source_dir, "*.wav")
+    for wav_path in wav_path_list:
+        output_dir = wav_path[:-4].replace(source_dir, target_dir)
+        os.makedirs(output_dir, exist_ok=True)
+
+        wav, wav_sr = sf.read(wav_path, always_2d=True)
+        assert wav_sr == sr, f"音频采样率不匹配: {wav_sr} != {sr}"
+        count = 0
+        while len(wav) > min_duration * sr:
+            segment_len = random.randint(min_duration * sr, max_duration * sr)
+            segment = wav[0: segment_len]
+            wav = wav[segment_len:]
+            count += 1
+            sf.write(os.path.join(output_dir, f"{count}.wav"), segment, sr)
+        sf.write(os.path.join(output_dir, f"{count + 1}.wav"), wav, sr)
+
+
+def audio_split_VADfunasr(source_dir, target_dir, sr=16000):
+    """
+    使用funasr的vad模型将音频中的语音分割成短句
+    """
+    from filter import HPFilter
+    from audio_aug import volume_norm
+    from funasr import AutoModel
+    model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
+
+    wav_path_list = get_path_list(source_dir, "*.wav")
+    for wav_path in wav_path_list:
+        wav_folder = wav_path[:-4].replace(source_dir, target_dir)
+        os.makedirs(wav_folder, exist_ok=True)
+
+        wav_orig, wav_sr = sf.read(wav_path, always_2d=True)
+        assert wav_sr == sr, f"音频采样率为{wav_sr}, 期望为{sr}"
+
+        wav = HPFilter(wav_orig[:, 0], sr=sr, order=6, cutoff=100)
+        wav = volume_norm(wav)
+
+        res_list = model.generate(input=wav)
+
+        for res in res_list:
+            for i, value_item in enumerate(res["value"]):
+                start, end = value_item
+                start, end = int(start * wav_sr / 1000), int(end * wav_sr / 1000)
+
+                # short_wav = wav_orig[start - int(0.5 * sr):end + int(0.5 * sr)]
+                # duration = (end - start) / sr
+                # assert len(short_wav) > sr * 3, f"{end/sr:.2f}-{start/sr:.2f}={duration:.2f}"
+                sf.write(os.path.join(wav_folder, f"{i}.wav"), wav_orig[start:end], sr)
+        # break
+
+
+def audio_split_VADsilero(source_dir, target_dir, sr, threshold=0.4,
+                          min_speech_duration_ms=400, min_silence_duration_ms=400,
+                          window_size_samples=512, speech_pad_ms=500):
+    """
+    使用silero的vad模型将音频中的语音分割成短句
+    source_dir: 音频文件目录
+    target_dir: 分割后的音频文件目录
+    sr: 音频采样率
+    threshold: 阈值
+    min_speech_duration_ms: 语音块的最小持续时间 ms
+    min_silence_duration_ms: 语音块之间的最小静音时间 ms
+    window_size_samples: 512\1024\1536
+    """
+    import torch
+    from filter import HPFilter
+    from audio_aug import volume_norm
+    model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad', force_reload=False, onnx=True)
+    (get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks) = utils
+
+    wav_path_list = get_path_list(source_dir, "*.wav")
+    for wav_path in wav_path_list:
+        wav_folder = wav_path[:-4].replace(source_dir, target_dir)
+        os.makedirs(wav_folder, exist_ok=True)
+
+        wav_orig, wav_sr = sf.read(wav_path, always_2d=True)
+        assert wav_sr == sr, f"音频采样率为{wav_sr}, 期望为{sr}"
+
+        wav = HPFilter(wav_orig[:, 0], sr=sr, order=6, cutoff=100)
+        wav = volume_norm(wav)
+
+        speech_timestamps = get_speech_timestamps(wav, model,
+                                                  sampling_rate=sr,
+                                                  threshold=threshold,
+                                                  min_speech_duration_ms=min_speech_duration_ms,  # 语音块的最小持续时间 ms
+                                                  min_silence_duration_ms=min_silence_duration_ms,  # 语音块之间的最小静音时间 ms
+                                                  window_size_samples=window_size_samples,  # 512\1024\1536
+                                                  speech_pad_ms=speech_pad_ms,  # 最后的语音块由两侧的speech_pad_ms填充
+                                                  )
+        for i, timestamp in enumerate(speech_timestamps):
+            wav_vad = wav_orig[timestamp["start"]:timestamp["end"]]
+
+            sf.write(os.path.join(wav_folder, f"{i}.wav"), wav_vad, sr)

neverlib/utils/message.py

@@ -1,19 +1,24 @@
+'''
+Author: 凌逆战 | Never
+Date: 2025-02-13 20:06:07
+Description: 请填写文件描述
+'''
 # -*- coding:utf-8 -*-
 # Author:凌逆战 | Never
 # Date: 2024/2/1
 """
 通过邮件发送通知
 """
+import os
 import smtplib
 from email.header import Header
 from email.mime.text import MIMEText
+from email.mime.image import MIMEImage
 from email.utils import formataddr
-import numpy as np
+from email.mime.multipart import MIMEMultipart
 
-np.set_printoptions(precision=1)  # 设置numpy的打印精度
 
-
-def send_QQEmail(title, content, from_email, from_password, to_email):
+def send_QQEmail(title, content, from_name, from_email, from_password, to_email):
     """
     Args:
         title: 邮件标题
@@ -28,8 +33,8 @@ def send_QQEmail(title, content, from_email, from_password, to_email):
     # 设置邮件正文
     message = MIMEText(content, "plain", "utf-8")
     message["Subject"] = Header(title, charset="utf-8")
-    message["From"] = formataddr(("never", from_email))
-    message["From"] = Header(from_email)
+    message["From"] = formataddr((from_name, from_email))
+    # message["From"] = Header(from_email)
     message["To"] = Header(to_email)
 
     try:
@@ -44,6 +49,70 @@ def send_QQEmail(title, content, from_email, from_password, to_email):
         print(f"邮件发送失败：{e}")
 
 
+def send_QQEmail_with_images(title, content, from_name, from_email, from_password, to_email, image_paths):
+    """
+    发送包含多张PNG图片附件的QQ邮件
+    :param title: 邮件标题
+    :param content: 邮件正文内容（支持HTML格式）
+    :param from_email: 发件人邮箱
+    :param from_password: 发件人邮箱SMTP授权码
+    :param to_email: 收件人邮箱
+    :param image_paths: 图片文件路径列表，应为PNG格式
+    """
+    # 设置邮箱的域名
+    HOST = "smtp.qq.com"
+    # 创建一个MIMEMultipart对象来包含邮件的各个部分
+    message = MIMEMultipart()
+    message["Subject"] = Header(title, charset="utf-8")
+    message["From"] = formataddr((from_name, from_email))
+    message["To"] = Header(to_email)
+
+    # 准备HTML内容
+    html_content = content + "<br>"
+
+    # 循环处理每张图片
+    for index, image_path in enumerate(image_paths):
+        try:
+            # 生成图片的Content-ID
+            image_id = f'image{index + 1}'
+            cid = f"<{image_id}>"
+
+            # 在HTML内容中添加图片引用
+            html_content += f'<br><img src="cid:{image_id}" style="max-width:100%; max-height:100%;"><br>'
+
+            # 打开图片文件并将其添加到邮件中
+            with open(image_path, 'rb') as img_file:
+                img_data = img_file.read()
+
+            # 使用 MIMEImage 添加图片
+            image_part = MIMEImage(img_data)
+
+            # 设置Content-ID，以便在正文中引用图片
+            image_part.add_header('Content-ID', cid)
+
+            # 设置为 inline 显示，避免附件处理
+            image_part.add_header('Content-Disposition', 'inline', filename=os.path.basename(image_path))
+
+            # 添加图片到邮件
+            message.attach(image_part)
+        except Exception as e:
+            print(f"添加图片 {image_path} 失败: {e}")
+
+    # 添加邮件正文
+    text_part = MIMEText(html_content, "html", "utf-8")
+    message.attach(text_part)
+
+    try:
+        # 使用SSL连接
+        server = smtplib.SMTP_SSL(HOST)
+        server.connect(HOST, 465)
+        server.login(from_email, from_password)  # 登录邮箱
+        server.sendmail(from_email, to_email, message.as_string())  # 发送邮件
+        server.quit()  # 关闭SMTP服务器
+        print("邮件发送成功")
+    except Exception as e:
+        print(f"邮件发送失败：{e}")
+
 
 if __name__ == "__main__":
     send_QQEmail("实验跑完", "实验跑完了，快去看看吧！",

neverlib/utils/utils.py

@@ -8,11 +8,10 @@ import os
 import random
 import shutil
 import fnmatch
-import subprocess
 from tqdm import tqdm
-import multiprocessing
 from datetime import datetime
-from joblib import Parallel, delayed
+import soundfile as sf
+import numpy as np
 
 
 def get_path_list(source_path, end="*.wav", shuffle=False):
@@ -47,111 +46,6 @@ def rename_files_and_folders(directory, replace='_-', replacement='_'):
                 print(f'Renamed folder: {old_path} -> {new_path}')
 
 
-def audio_split_ffmpeg(source_path, target_path, sr, channel_num, duration, endwith="*.pcm"):
-    """
-    使用ffmpeg分割音频, 分割为短音频(单位:秒), 似乎无法非常准确的分割到指定长度
-    :param source_path: 源音频路径
-    :param target_path: 目标音频路径
-    :param sr: 源音频采样率
-    :param channel_num: 源音频声道数
-    :param duration: 分割为时长(短音频)(单位:秒)
-    :param endwith: 音频格式(支持pcm和wav)
-    """
-    wav_path_list = get_path_list(source_path, end=endwith)
-    print("待分割的音频数: ", len(wav_path_list))
-    for wav_path in wav_path_list:
-        wav_folder = wav_path[:-4].replace(source_path, target_path)
-        if not os.path.exists(wav_folder):
-            os.makedirs(wav_folder)
-
-        if endwith == "*.pcm":
-            # 将pcm文件切割成30s的语音, 有括号会报错
-            # ffmpeg -f s16le -ar 16000 -ac 6 -i ./NO.1_A3035_2.pcm -f segment -segment_time 30 -c copy NO.1_A3035_2/%03d.wav
-            command = ["ffmpeg", "-f", "s16le", "-ar", f"{sr}", "-ac", str(channel_num),
-                       "-i", wav_path, "-f", "segment", "-segment_time",
-                       f"{duration}", "-c", "copy", f"{wav_folder}/%03d.wav"]
-            subprocess.run(command, check=True)
-        elif endwith == "*.wav":
-            # ffmpeg -i ./NO.1_A3035_2.wav -f segment -segment_time 30 -c copy NO.1_A3035_2/%03d.wav
-            command = ["ffmpeg", "-i", wav_path, "-f", "segment", "-segment_time",
-                       f"{duration}", "-c", "copy", f"{wav_folder}/%03d.wav"]
-            subprocess.run(command, check=True)
-        else:
-            assert False, "不支持的音频格式"
-    print("分割完毕: done!")
-
-
-def audio_split_sox(source_path, target_path, duration, endwith="*.wav"):
-    """
-    使用sox分割音频, 分割为短音频(单位:秒), 可以非常准确的分割到指定长度
-    :param source_path: 源音频路径
-    :param target_path: 目标音频路径
-    :param duration: 分割为时长(短音频)(单位:秒)
-    :param endwith: 音频格式(只支持wav)
-    """
-    wav_path_list = get_path_list(source_path, end=endwith)
-
-    for wav_path in wav_path_list:
-        wav_folder = wav_path[:-4].replace(source_path, target_path)
-        if not os.path.exists(wav_folder):
-            os.makedirs(wav_folder)
-
-        output_pattern = f"{wav_folder}/%.wav"
-
-        if endwith == "*.wav":
-            # 对 WAV 文件直接进行分割
-            os.system(f"sox {wav_path} {output_pattern} trim 0 {str(duration)} : newfile : restart")
-        else:
-            assert False, "不支持的音频格式"
-
-    print("分割完毕: done!")
-
-
-def audio_split_worker(wav_path, target_path, sr, channel_num, duration, endwith="*.pcm"):
-    wav_name = os.path.basename(wav_path)[:-4]
-    wav_folder = os.path.join(target_path, wav_name)
-    if not os.path.exists(wav_folder):
-        os.makedirs(wav_folder)
-
-    if endwith == "*.pcm":
-        # 将pcm文件切割成30s的语音, 有括号会报错
-        # os.system(r"ffmpeg -f s16le -ar {} -ac {} -i {} -f segment -segment_time {} -c copy {}/%03d.wav".format(
-        #           sr, channel_num, wav_path, duration, wav_folder))
-        command = ["ffmpeg", "-f", "s16le", "-ar", f"{sr}", "-ac", str(channel_num),
-                   "-i", wav_path, "-f", "segment", "-segment_time",
-                   f"{duration}", "-c", "copy", f"{wav_folder}/%03d.wav"]
-        subprocess.run(command, check=True)
-        # 调用库
-        # input_audio = ffmpeg.input(wav_path, format='s16le', ar=16000, ac=5)
-        # output_audio = ffmpeg.output(input_audio, f='segment', segment_time=30, c='copy', path=f'{wav_folder}/%03d.wav')
-        # ffmpeg.run(output_audio)
-    elif endwith == "*.wav":
-        # ffmpeg -i ./NO.1_A3035_2.wav -f segment -segment_time 30 -c copy NO.1_A3035_2/%03d.wav
-        command = ["ffmpeg", "-i", wav_path, "-f", "segment", "-segment_time",
-                   f"{duration}", "-c", "copy", f"{wav_folder}/%03d.wav"]
-        subprocess.run(command, check=True)
-    else:
-        assert False, "不支持的音频格式"
-
-
-def audio_split_multiprocessing(source_path, target_path, sr, channel_num, duration, endwith="*.pcm"):
-    wav_path_list = get_path_list(source_path, end=endwith)
-    print("待分割的音频数: ", len(wav_path_list))
-
-    # 创建进程池
-    pool = multiprocessing.Pool(processes=5)
-
-    # 使用进程池处理音频文件分割任务
-    for wav_path in wav_path_list:
-        pool.apply_async(audio_split_worker, args=(wav_path, target_path, sr, channel_num, duration, endwith))
-
-    # 关闭进程池，等待所有进程完成
-    pool.close()
-    pool.join()
-
-    print("分割完毕: done!")
-
-
 def get_file_time(file_path):
     # 获取最后修改时间
     mod_time = os.path.getmtime(file_path)
@@ -192,3 +86,225 @@ def TrainValSplit(dataset_dir, train_dir, val_dir, percentage=0.9):
         shutil.copy(val_wavpath, target_path)
 
     print("Done!")
+
+
+def TrainValTestSplit(dataset_dir, train_dir, val_dir, test_dir, percentage=[0.8, 0.1, 0.1]):
+    """ 分割数据集为训练集、验证集和测试集
+    :param dataset_dir: 源数据集地址
+    :param train_dir: 训练集地址
+    :param val_dir: 验证集地址
+    :param test_dir: 测试集地址
+    :param percentage: 分割百分比
+    """
+    assert sum(percentage) == 1.0, "百分比总和必须等于1.0"
+    
+    wav_path_list = sorted(get_path_list(dataset_dir, end="*.wav"))
+    random.seed(10086)
+    random.shuffle(wav_path_list)  # 打乱列表的顺序
+    total_wav_num = len(wav_path_list)
+    
+    # 计算训练集、验证集和测试集的分割点
+    train_split_idx = int(total_wav_num * percentage[0])
+    val_split_idx = train_split_idx + int(total_wav_num * percentage[1])
+    
+    train_path_list = wav_path_list[:train_split_idx]
+    val_path_list = wav_path_list[train_split_idx:val_split_idx]
+    test_path_list = wav_path_list[val_split_idx:]
+    
+    for train_wavpath in tqdm(train_path_list, desc="复制训练集音频"):
+        target_path = train_wavpath.replace(dataset_dir, train_dir)
+        if not os.path.exists(os.path.split(target_path)[0]):
+            os.makedirs(os.path.split(target_path)[0])
+        shutil.copy(train_wavpath, target_path)
+    
+    for val_wavpath in tqdm(val_path_list, desc="复制验证集音频"):
+        target_path = val_wavpath.replace(dataset_dir, val_dir)
+        if not os.path.exists(os.path.split(target_path)[0]):
+            os.makedirs(os.path.split(target_path)[0])
+        shutil.copy(val_wavpath, target_path)
+    
+    for test_wavpath in tqdm(test_path_list, desc="复制测试集音频"):
+        target_path = test_wavpath.replace(dataset_dir, test_dir)
+        if not os.path.exists(os.path.split(target_path)[0]):
+            os.makedirs(os.path.split(target_path)[0])
+        shutil.copy(test_wavpath, target_path)
+    
+    print(f"完成! 训练集: {len(train_path_list)}个文件, 验证集: {len(val_path_list)}个文件, 测试集: {len(test_path_list)}个文件")
+
+
+def get_leaf_folders(directory):
+    # 获取最底层的文件夹路径
+    leaf_folders = []
+    for root, dirs, _ in os.walk(directory):
+        if not dirs:  # 如果当前文件夹没有子文件夹
+            leaf_folders.append(root)
+    return leaf_folders
+
+
+def DatasetSubfloderSplit(source_dir, split_dirs, percentage=None):
+    """
+    将一个数据集按照子文件夹数量分割成train/val/test数据集
+    Args:
+        source_dir (str): 源数据集目录
+        split_dirs (list): 目标目录列表，如 [train_dir, val_dir] 或 [train_dir, val_dir, test_dir]
+        percentage (list, optional): 分割比例，如 [0.9, 0.1] 或 [0.8, 0.1, 0.1]。默认为 None，此时：
+            - 如果是两路分割，默认为 [0.9, 0.1]
+            - 如果是三路分割，默认为 [0.8, 0.1, 0.1]
+    Example:
+        # 两路分割示例
+        DatasetSplit(
+            source_dir=source_dataset_path,
+            split_dirs=[target_train_path, target_val_path],
+            percentage=[0.9, 0.1]
+        )
+
+        # 三路分割示例
+        DatasetSplit(
+            source_dir=source_dataset_path,
+            split_dirs=[target_train_path, target_val_path, target_test_path],
+            percentage=[0.8, 0.1, 0.1]
+        )
+
+        # 使用默认比例的两路分割
+        DatasetSplit(
+            source_dir=source_dataset_path,
+            split_dirs=[target_train_path, target_val_path]
+        )
+    """
+    if percentage is None:
+        percentage = [0.9, 0.1] if len(split_dirs) == 2 else [0.8, 0.1, 0.1]
+
+    # 验证输入参数
+    if len(split_dirs) not in [2, 3]:
+        raise ValueError("只支持2路或3路分割（训练集/验证集 或 训练集/验证集/测试集）")
+    if len(percentage) != len(split_dirs):
+        raise ValueError("分割比例数量必须与目标目录数量相同")
+    if sum(percentage) != 1.0:
+        raise ValueError("分割比例之和必须等于1.0")
+
+    # 获取并打乱文件夹列表
+    leaf_folder_list = sorted(get_leaf_folders(source_dir))
+    random.seed(10086)
+    random.shuffle(leaf_folder_list)
+    total_folder_num = len(leaf_folder_list)
+
+    # 计算分割点
+    split_indices = []
+    acc_percentage = 0
+    for p in percentage[:-1]:  # 最后一个比例不需要计算
+        acc_percentage += p
+        split_indices.append(int(total_folder_num * acc_percentage))
+
+    # 分割文件夹列表
+    split_folder_lists = []
+    start_idx = 0
+    for end_idx in split_indices:
+        split_folder_lists.append(leaf_folder_list[start_idx:end_idx])
+        start_idx = end_idx
+    split_folder_lists.append(leaf_folder_list[start_idx:])  # 添加最后一部分
+
+    # 复制文件夹
+    split_names = ['train', 'val', 'test']
+    for folders, target_dir, split_name in zip(split_folder_lists, split_dirs, split_names[:len(split_dirs)]):
+        for folder in tqdm(folders, desc=f"Copying {split_name} folders"):
+            target_folder = folder.replace(source_dir, target_dir)
+            os.makedirs(os.path.dirname(target_folder), exist_ok=True)
+            shutil.copytree(folder, target_folder)
+
+    # 打印统计信息
+    print(f"Total folders: {total_folder_num}")
+    for folders, split_name in zip(split_folder_lists, split_names[:len(split_dirs)]):
+        print(f"{split_name.capitalize()} folders: {len(folders)}")
+
+
+def pcm2wav(pcm_path, wav_path, sr=16000, channels=1, subtype='PCM_16'):
+    """
+    将pcm文件转换为wav文件
+    :param pcm_path: pcm文件路径
+    :param wav_path: wav文件路径
+    :param sr: 采样率
+    :param channels: 声道数
+    :param subtype: 子类型
+    """
+    pcm_data = np.fromfile(pcm_path, dtype=np.int16)
+    pcm_data = pcm_data.reshape(-1, channels)  # 支持多通道
+    sf.write(wav_path, pcm_data, sr, subtype=subtype)
+
+
+def wav2pcm(wav_path, pcm_path):
+    """
+    将wav文件转换为pcm文件
+    :param wav_path: wav文件路径
+    :param pcm_path: pcm文件路径
+    """
+    data, _ = sf.read(wav_path, dtype='int16')
+    data.tofile(pcm_path)
+
+
+def save_weight_histogram(model, save_dir, mode=["params", "buffers"], ignore_name=["scale", "bias"], bins=100):
+    """
+    保存模型权重分布直方图
+    Args:
+        model: PyTorch模型
+        save_dir: 保存路径
+        mode: 保存模式，可选值为["params", "buffers"]
+        bins: 直方图bin数量
+    """
+    import matplotlib.pyplot as plt
+    # 如果路径存在，则删除
+    if os.path.exists(save_dir):
+        shutil.rmtree(save_dir)
+
+    if "params" in mode:
+        os.makedirs(os.path.join(save_dir, "param"), exist_ok=True)
+        for name, param in model.named_parameters():
+            if any(ignore in name for ignore in ignore_name):
+                continue
+            param = param.cpu().data.flatten().numpy()
+            param_min = param.min()
+            param_max = param.max()
+            param_mean = param.mean()
+            param_std = param.std()
+
+            # 保存模型参数到地址
+            # 绘制直方图
+            plt.title(name)
+            plt.xlabel("value")
+            plt.ylabel("count")
+            plt.grid(alpha=0.5)
+            # 在右上角添加统计信息
+            plt.text(1, 1, f"max: {param_max:.2f}\n \
+                            min: {param_min:.2f}\n \
+                            mean: {param_mean:.2f}\n \
+                            std: {param_std:.2f}",
+                     ha='right', va='top', transform=plt.gca().transAxes)
+            plt.hist(param, bins=bins)
+            plt.savefig(os.path.join(save_dir, "param", f"{name}.png"))
+            plt.close()
+    if "buffers" in mode:
+        os.makedirs(os.path.join(save_dir, "buffer"), exist_ok=True)
+        for name, buffer in model.named_buffers():
+            if "running_mean" not in name and "running_var" not in name:
+                continue
+            buffer = buffer.cpu().data.flatten().numpy()
+
+            # 计算统计数据
+            buffer_min = buffer.min()
+            buffer_max = buffer.max()
+            buffer_mean = buffer.mean()
+            buffer_std = buffer.std()
+
+            # 绘制直方图
+            plt.title(name)
+            plt.xlabel("value")
+            plt.ylabel("count")
+            plt.grid(alpha=0.5)
+            # 在右上角添加统计信息
+            plt.text(1, 1, f"max: {buffer_max:.2f}\n \
+                            min: {buffer_min:.2f}\n \
+                            mean: {buffer_mean:.2f}\n \
+                            std: {buffer_std:.2f}",
+                     ha='right', va='top', transform=plt.gca().transAxes)
+            plt.hist(buffer, bins=bins)
+            plt.savefig(os.path.join(save_dir, "buffer", f"{name}.png"))
+            plt.close()

neverlib/vad/utils.py

@@ -7,20 +7,18 @@
 import numpy as np
 
 
-def find_active_segments(vad_array):
+def from_vadArray_to_vadEndpoint(vad_array):
     # 计算活动段的起始点和结束点
     # vad_array = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1])
     # 返回 [(2320, 8079), (8400, 8719), (8880, 10959), (11600, 25039), (25840, 27439), (29040, 29359), (29520, 31759), (32240, 32399)]
-    starts = np.where((vad_array[:-1] == 0) & (vad_array[1:] == 1))[0] + 1
-    ends = np.where((vad_array[:-1] == 1) & (vad_array[1:] == 0))[0]
+    starts = np.where((vad_array[:-1] == 0) & (vad_array[1:] == 1))[0] + 1    # +1是因为提前了一个点
+    ends = np.where((vad_array[:-1] == 1) & (vad_array[1:] == 0))[0] + 1  # + 1是因为不取最后一个点
 
-    # 如果活动段以1开始但没有以0结束，则需要手动添加结束点
-    if vad_array[-1] == 1: ends = np.append(ends, len(vad_array) - 1)
-    # 如果活动段以0开始但没有以1结束，则需要手动添加起始点
+    # 如果最后一个点还是1，则需要手动添加结束点
+    if vad_array[-1] == 1: ends = np.append(ends, len(vad_array))
+    # 如果第一个点就是1, 则需要手动添加起始点
     if vad_array[0] == 1: starts = np.insert(starts, 0, 0)
-
-    # 处理可能存在的错位情况
-    if len(starts) > len(ends): starts = starts[:-1]
+    assert len(starts) == len(ends), "starts and ends must have the same length"
 
     Timestamps = [{"start": int(start), "end": int(end)} for start, end in zip(starts, ends)]
 

{neverlib-0.1.8.dist-info → neverlib-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: neverlib
-Version: 0.1.8
+Version: 0.2.0
 Summary: A successful sign for python setup
 Author-email: "Never.Ling" <1786088386@qq.com>
 License: MIT
@@ -41,6 +41,7 @@ Requires-Dist: funasr; extra == "all"
 Requires-Dist: openai-whisper; extra == "all"
 Requires-Dist: transformers; extra == "all"
 Requires-Dist: GPUtil; extra == "all"
+Dynamic: license-file
 
 # NeverLib
 

{neverlib-0.1.8.dist-info → neverlib-0.2.0.dist-info}/RECORD

@@ -14,9 +14,10 @@ neverlib/filter/common.py,sha256=9MGVpnW2D6wYxO0-5Kru4SwmB7CriPZqf5FacTt6b7Y,103
 neverlib/tests/__init__.py,sha256=PITL4btOVji1rexFP35A20v2gcLqc6vmGWiF-_5ywnI,27
 neverlib/tests/test_preprocess.py,sha256=GHj2PvlG8chJlV-zpRfy8IYlTXzGkn5heO9ipy6Xe20,1003
 neverlib/utils/__init__.py,sha256=kUUhBstCxeXNg3WMmHyrNcx6aDNhUs_LucyygnOF4Ro,137
+neverlib/utils/audio_split.py,sha256=pRh2DqOg8Rg4o6QBv_xuQKWlZ4WI-RDqtCoP_-YTE6o,12014
 neverlib/utils/checkGPU.py,sha256=Wye2UovtVK6Fk8W4wZ3dqaYrdZc30WyW5iGa-6Ouxf0,4075
-neverlib/utils/message.py,sha256=niZ0nZwW3YdyafhlP06Qxla2FHH4rgF1l8suz764_g8,1628
-neverlib/utils/utils.py,sha256=57t1lPUXFiVnNf2L1u6c3U5zQPnHTsGw6laRB5-7gfo,8216
+neverlib/utils/message.py,sha256=xeBwvHXJ0GTJ-4Mt0KrrfJNk3JY22_fxYwV7TTuLjWk,4235
+neverlib/utils/utils.py,sha256=A3zqRVIoZeskuDMI_Dnx7g46yyN8ZT1k1g0x0prK_6A,12336
 neverlib/utils/waveform_analyzer.py,sha256=H4lgcms2Obkeg8uxazLvA7GPwa5dsMaIaP_GXF6fi5w,1264
 neverlib/vad/PreProcess.py,sha256=KlJ3LlJjv4dgVMqpC49BbKLoaueRvCC6OGnUkR9fdAs,2078
 neverlib/vad/README.md,sha256=179PS6jXe9FfzNmg1yhTshIPqLsczWRP8FoSd5dGLBY,1502
@@ -30,10 +31,10 @@ neverlib/vad/VAD_whisper.py,sha256=Hmk6l8quenr4cnnWre_3DNGJE9XHhwzUFpM-q27ZSCI,1
 neverlib/vad/__init__.py,sha256=RePuPF-0sCRxxQk4l74A66CnA8fE7jRxWRhC0uiEzX4,557
 neverlib/vad/class_get_speech.py,sha256=yIfGqvtsLCkDsOwS8HOGf9jZnt7WN5yMt_vgTAGrpnA,2078
 neverlib/vad/class_vad.py,sha256=u5szyOHjDrLt3-OA4TKlGZu8R8nQ6t1znatNlYhPwsI,5216
-neverlib/vad/utils.py,sha256=MdKqXWRPRoy_9lJYuSBXje8nTCz9dnv1ahz5k_-RpFU,1122
+neverlib/vad/utils.py,sha256=oHYIDnG_rCEJWbgvCBDQ2QQtuaKOPCcaF4AdwOZJkqI,1151
 neverlib/wav_data/000_short.wav,sha256=3pVeI6yGfjiD2q1_A9F_mIEYQnUhp9AMjy-gR4TAqK8,210430
-neverlib-0.1.8.dist-info/LICENSE,sha256=h-U7vhFdQhieyaNCiA_utrRNMk1rjfrBAUJVodgkfsw,1096
-neverlib-0.1.8.dist-info/METADATA,sha256=V9B_TEIJFs0D1Cgvgtb1xmT3eVaiFNpfk_zm3yxpS70,2601
-neverlib-0.1.8.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-neverlib-0.1.8.dist-info/top_level.txt,sha256=QqwYFuDiY_iFTz0Kx8av6zKCP_s4M5NiMTNsIXOy8Po,9
-neverlib-0.1.8.dist-info/RECORD,,
+neverlib-0.2.0.dist-info/licenses/LICENSE,sha256=h-U7vhFdQhieyaNCiA_utrRNMk1rjfrBAUJVodgkfsw,1096
+neverlib-0.2.0.dist-info/METADATA,sha256=tismKBKfAteFiKX8IzZLuTROG3plY0EXBCc9NhRZTQ8,2624
+neverlib-0.2.0.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
+neverlib-0.2.0.dist-info/top_level.txt,sha256=QqwYFuDiY_iFTz0Kx8av6zKCP_s4M5NiMTNsIXOy8Po,9
+neverlib-0.2.0.dist-info/RECORD,,

{neverlib-0.1.8.dist-info → neverlib-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.1.0)
+Generator: setuptools (78.1.1)
 Root-Is-Purelib: true
 Tag: py3-none-any