neverlib 0.2.6__py3-none-any.whl → 0.2.7__py3-none-any.whl

neverlib/data_analyze/dataset_analyzer.py

@@ -4,25 +4,20 @@ Dataset Analyzer Module
 
 提供音频数据集批量分析和报告生成功能
 """
-
-import numpy as np
-import librosa
 import os
 import json
+import librosa
+import numpy as np
 from pathlib import Path
-from typing import List, Dict, Tuple, Optional, Union, Any
+from typing import List, Dict, Optional, Union, Any
 from dataclasses import dataclass, asdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import multiprocessing
 from tqdm import tqdm
-import warnings
-
-from .statistics import AudioStatistics
 from .quality_metrics import QualityAnalyzer, audio_health_check
-from .spectral_analysis import SpectralAnalyzer, compute_spectral_features
-from .temporal_features import TemporalAnalyzer, compute_temporal_features
-from .visualization import AudioVisualizer
-from .utils import rms_amplitude, dB
+from .spectral_analysis import SpectralAnalyzer
+from .temporal_features import TemporalAnalyzer
+from ..utils import rms_amplitude, dB
 
 
 @dataclass
@@ -35,26 +30,26 @@ class AudioFileInfo:
     channels: int
     bit_depth: Optional[int]
     format: str
-    
+
     # 基础统计
     max_amplitude: float
     rms_amplitude: float
     mean_amplitude: float
     std_amplitude: float
-    
+
     # 质量指标
     dynamic_range: float
     snr_estimate: Optional[float]
     has_clipping: bool
     is_silent: bool
     dc_offset: float
-    
+
     # 特征摘要
     spectral_centroid_mean: float
     spectral_rolloff_mean: float
     zero_crossing_rate_mean: float
     tempo: Optional[float]
-    
+
     # 健康状态
     health_score: float  # 0-100
     issues: List[str]
@@ -63,52 +58,52 @@ class AudioFileInfo:
 
 class DatasetAnalyzer:
     """数据集分析器类"""
-    
+
     def __init__(self, sr: int = 22050, n_jobs: int = None):
         """
         初始化数据集分析器
-        
+
         Args:
             sr: 目标采样率
             n_jobs: 并行作业数量, None表示使用CPU核心数
         """
         self.sr = sr
         self.n_jobs = n_jobs or min(multiprocessing.cpu_count(), 8)
-        
+
         # 初始化分析器
         self.quality_analyzer = QualityAnalyzer(sr=sr)
         self.spectral_analyzer = SpectralAnalyzer(sr=sr)
         self.temporal_analyzer = TemporalAnalyzer(sr=sr)
-        
+
         # 分析结果
         self.file_infos: List[AudioFileInfo] = []
         self.dataset_summary: Dict = {}
         self.analysis_complete = False
-    
+
     def analyze_single_file(self, file_path: str) -> Optional[AudioFileInfo]:
         """
         分析单个音频文件
-        
+
         Args:
             file_path: 音频文件路径
-            
+
         Returns:
             音频文件信息对象
         """
         try:
             # 加载音频
             audio, original_sr = librosa.load(file_path, sr=None)
-            
+
             # 如果需要重采样
             if self.sr != original_sr:
                 audio_resampled = librosa.resample(audio, orig_sr=original_sr, target_sr=self.sr)
             else:
                 audio_resampled = audio
-            
+
             # 获取文件基本信息
             file_size = os.path.getsize(file_path)
             duration = len(audio) / original_sr
-            
+
             # 检测音频格式信息
             try:
                 import soundfile as sf
@@ -120,21 +115,21 @@ class DatasetAnalyzer:
                 channels = 1 if len(audio.shape) == 1 else audio.shape[1]
                 bit_depth = None
                 format_info = Path(file_path).suffix.lower()
-            
+
             # 基础统计
             max_amplitude = float(np.max(np.abs(audio_resampled)))
             rms_amp = float(rms_amplitude(audio_resampled))
             mean_amplitude = float(np.mean(np.abs(audio_resampled)))
             std_amplitude = float(np.std(audio_resampled))
-            
+
             # 质量分析
             dynamic_range = self.quality_analyzer.dynamic_range(audio_resampled)
             dc_offset = float(np.mean(audio_resampled))
-            
+
             # 检测问题
             has_clipping = max_amplitude >= 0.99
             is_silent = mean_amplitude < 1e-6
-            
+
             # SNR估计（基于信号强度和噪声层）
             snr_estimate = None
             try:
@@ -150,7 +145,7 @@ class DatasetAnalyzer:
                             snr_estimate = 20 * np.log10(rms_amp / noise_rms)
             except:
                 pass
-            
+
             # 频域特征
             try:
                 spectral_centroid = self.spectral_analyzer.spectral_centroid(audio_resampled)
@@ -160,29 +155,29 @@ class DatasetAnalyzer:
             except:
                 spectral_centroid_mean = 0.0
                 spectral_rolloff_mean = 0.0
-            
+
             # 时域特征
             try:
                 zcr = self.temporal_analyzer.zero_crossing_rate(audio_resampled)
                 zcr_mean = float(np.mean(zcr))
-                
+
                 # 节拍检测
                 tempo, _ = self.temporal_analyzer.tempo_estimation(audio_resampled)
                 tempo = float(tempo) if tempo > 0 else None
             except:
                 zcr_mean = 0.0
                 tempo = None
-            
+
             # 健康检查
             health_check = audio_health_check(audio_resampled, self.sr)
             issues = health_check['issues']
             warnings_list = health_check['warnings']
-            
+
             # 计算健康分数 (0-100)
             health_score = 100.0
             health_score -= len(issues) * 20  # 每个严重问题扣20分
             health_score -= len(warnings_list) * 5  # 每个警告扣5分
-            
+
             if has_clipping:
                 health_score -= 15
             if is_silent:
@@ -191,9 +186,9 @@ class DatasetAnalyzer:
                 health_score -= 10
             if dynamic_range < 6:
                 health_score -= 10
-            
+
             health_score = max(0.0, min(100.0, health_score))
-            
+
             # 创建文件信息对象
             file_info = AudioFileInfo(
                 file_path=file_path,
@@ -203,109 +198,109 @@ class DatasetAnalyzer:
                 channels=channels,
                 bit_depth=bit_depth,
                 format=format_info,
-                
+
                 max_amplitude=max_amplitude,
                 rms_amplitude=rms_amp,
                 mean_amplitude=mean_amplitude,
                 std_amplitude=std_amplitude,
-                
+
                 dynamic_range=dynamic_range,
                 snr_estimate=snr_estimate,
                 has_clipping=has_clipping,
                 is_silent=is_silent,
                 dc_offset=dc_offset,
-                
+
                 spectral_centroid_mean=spectral_centroid_mean,
                 spectral_rolloff_mean=spectral_rolloff_mean,
                 zero_crossing_rate_mean=zcr_mean,
                 tempo=tempo,
-                
+
                 health_score=health_score,
                 issues=issues,
                 warnings=warnings_list
             )
-            
+
             return file_info
-            
+
         except Exception as e:
             print(f"Error analyzing {file_path}: {str(e)}")
             return None
-    
+
     def analyze_dataset(self, file_paths: List[str], show_progress: bool = True) -> Dict[str, Any]:
         """
         批量分析数据集
-        
+
         Args:
             file_paths: 音频文件路径列表
             show_progress: 是否显示进度条
-            
+
         Returns:
             分析结果摘要
         """
         self.file_infos = []
-        
+
         # 并行处理文件
         with ThreadPoolExecutor(max_workers=self.n_jobs) as executor:
             # 提交所有任务
             future_to_path = {
-                executor.submit(self.analyze_single_file, path): path 
+                executor.submit(self.analyze_single_file, path): path
                 for path in file_paths
             }
-            
+
             # 收集结果
             if show_progress:
-                futures = tqdm(as_completed(future_to_path), total=len(file_paths), 
-                             desc="分析音频文件")
+                futures = tqdm(as_completed(future_to_path), total=len(file_paths),
+                               desc="分析音频文件")
             else:
                 futures = as_completed(future_to_path)
-            
+
             for future in futures:
                 result = future.result()
                 if result is not None:
                     self.file_infos.append(result)
-        
+
         # 生成数据集摘要
         self.dataset_summary = self._generate_dataset_summary()
         self.analysis_complete = True
-        
+
         return self.dataset_summary
-    
+
     def _generate_dataset_summary(self) -> Dict[str, Any]:
         """
         生成数据集摘要统计
-        
+
         Returns:
             数据集摘要字典
         """
         if not self.file_infos:
             return {}
-        
+
         # 基础统计
         total_files = len(self.file_infos)
         total_duration = sum(info.duration for info in self.file_infos)
         total_size = sum(info.file_size for info in self.file_infos)
-        
+
         # 格式统计
         formats = {}
         sample_rates = {}
         channels_count = {}
-        
+
         for info in self.file_infos:
             formats[info.format] = formats.get(info.format, 0) + 1
             sample_rates[info.sample_rate] = sample_rates.get(info.sample_rate, 0) + 1
             channels_count[info.channels] = channels_count.get(info.channels, 0) + 1
-        
+
         # 质量统计
         health_scores = [info.health_score for info in self.file_infos]
         problematic_files = [info for info in self.file_infos if info.health_score < 80]
         silent_files = [info for info in self.file_infos if info.is_silent]
         clipped_files = [info for info in self.file_infos if info.has_clipping]
-        
+
         # 音频特征统计
         durations = [info.duration for info in self.file_infos]
         rms_values = [info.rms_amplitude for info in self.file_infos]
         dynamic_ranges = [info.dynamic_range for info in self.file_infos]
-        
+
         # 生成摘要
         summary = {
             'overview': {
@@ -315,13 +310,13 @@ class DatasetAnalyzer:
                 'average_file_duration': np.mean(durations),
                 'analysis_target_sr': self.sr
             },
-            
+
             'format_distribution': {
                 'formats': formats,
                 'sample_rates': sample_rates,
                 'channels': channels_count
             },
-            
+
             'duration_statistics': {
                 'mean': np.mean(durations),
                 'median': np.median(durations),
@@ -335,7 +330,7 @@ class DatasetAnalyzer:
                     '95th': np.percentile(durations, 95)
                 }
             },
-            
+
             'quality_assessment': {
                 'average_health_score': np.mean(health_scores),
                 'problematic_files_count': len(problematic_files),
@@ -350,7 +345,7 @@ class DatasetAnalyzer:
                     'bad (<60)': len([s for s in health_scores if s < 60])
                 }
             },
-            
+
             'audio_characteristics': {
                 'rms_statistics': {
                     'mean_linear': np.mean(rms_values),
@@ -367,91 +362,91 @@ class DatasetAnalyzer:
                     'max': np.max(dynamic_ranges)
                 }
             },
-            
+
             'recommendations': self._generate_recommendations()
         }
-        
+
         return summary
-    
+
     def _generate_recommendations(self) -> List[str]:
         """
         基于分析结果生成改进建议
-        
+
         Returns:
             建议列表
         """
         recommendations = []
-        
+
         if not self.file_infos:
             return recommendations
-        
+
         # 检查质量问题
         problematic_count = len([info for info in self.file_infos if info.health_score < 80])
         if problematic_count > 0:
             recommendations.append(f"发现 {problematic_count} 个文件存在质量问题, 建议进行质量检查和修复")
-        
+
         # 检查削波
         clipped_count = len([info for info in self.file_infos if info.has_clipping])
         if clipped_count > 0:
             recommendations.append(f"发现 {clipped_count} 个文件存在削波, 建议重新录制或降低增益")
-        
+
         # 检查静音文件
         silent_count = len([info for info in self.file_infos if info.is_silent])
         if silent_count > 0:
             recommendations.append(f"发现 {silent_count} 个静音文件, 建议移除或重新录制")
-        
+
         # 检查采样率一致性
         sample_rates = set(info.sample_rate for info in self.file_infos)
         if len(sample_rates) > 1:
             recommendations.append(f"数据集包含多种采样率 {sample_rates}, 建议统一采样率")
-        
+
         # 检查动态范围
         low_dr_count = len([info for info in self.file_infos if info.dynamic_range < 20])
         if low_dr_count > len(self.file_infos) * 0.2:  # 超过20%的文件动态范围过低
             recommendations.append("大量文件动态范围过低, 可能影响音频质量")
-        
+
         # 检查时长分布
         durations = [info.duration for info in self.file_infos]
         duration_std = np.std(durations)
         duration_mean = np.mean(durations)
         if duration_std / duration_mean > 0.5:  # 变异系数大于0.5
             recommendations.append("文件时长分布不均匀, 可能影响训练效果")
-        
+
         return recommendations
-    
+
     def get_problematic_files(self, min_health_score: float = 80) -> List[AudioFileInfo]:
         """
         获取有问题的文件列表
-        
+
         Args:
             min_health_score: 最低健康分数阈值
-            
+
         Returns:
             问题文件列表
         """
         return [info for info in self.file_infos if info.health_score < min_health_score]
-    
+
     def export_results(self, output_dir: str):
         """
         导出分析结果
-        
+
         Args:
             output_dir: 输出目录
         """
         output_path = Path(output_dir)
         output_path.mkdir(parents=True, exist_ok=True)
-        
+
         # 导出摘要
         summary_path = output_path / 'dataset_summary.json'
         with open(summary_path, 'w', encoding='utf-8') as f:
             json.dump(self.dataset_summary, f, indent=2, ensure_ascii=False, default=str)
-        
+
         # 导出详细文件信息
         details_path = output_path / 'file_details.json'
         file_details = [asdict(info) for info in self.file_infos]
         with open(details_path, 'w', encoding='utf-8') as f:
             json.dump(file_details, f, indent=2, ensure_ascii=False, default=str)
-        
+
         # 导出问题文件列表
         problematic_files = self.get_problematic_files()
         if problematic_files:
@@ -459,35 +454,35 @@ class DatasetAnalyzer:
             problems_data = [asdict(info) for info in problematic_files]
             with open(problems_path, 'w', encoding='utf-8') as f:
                 json.dump(problems_data, f, indent=2, ensure_ascii=False, default=str)
-        
+
         print(f"分析结果已导出到: {output_path}")
-    
+
     def create_analysis_report(self, output_path: str):
         """
         创建HTML分析报告
-        
+
         Args:
             output_path: 输出HTML文件路径
         """
         if not self.analysis_complete:
             raise ValueError("请先完成数据集分析")
-        
+
         html_content = self._generate_html_report()
-        
+
         with open(output_path, 'w', encoding='utf-8') as f:
             f.write(html_content)
-        
+
         print(f"HTML报告已生成: {output_path}")
-    
+
     def _generate_html_report(self) -> str:
         """
         生成HTML格式的分析报告
-        
+
         Returns:
             HTML内容字符串
         """
         summary = self.dataset_summary
-        
+
         html = f"""
         <!DOCTYPE html>
         <html lang="zh-CN">
@@ -511,7 +506,7 @@ class DatasetAnalyzer:
                 <h1>音频数据集分析报告</h1>
                 <p>生成时间: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S') if 'pd' in globals() else 'N/A'}</p>
             </div>
-            
+
             <div class="section">
                 <h2>数据集概览</h2>
                 <div class="metric">文件总数: {summary['overview']['total_files']}</div>
@@ -519,7 +514,7 @@ class DatasetAnalyzer:
                 <div class="metric">总大小: {summary['overview']['total_size_mb']:.2f} MB</div>
                 <div class="metric">平均文件时长: {summary['overview']['average_file_duration']:.2f} 秒</div>
             </div>
-            
+
             <div class="section">
                 <h2>质量评估</h2>
                 <div class="metric">平均健康分数: {summary['quality_assessment']['average_health_score']:.1f}/100</div>
@@ -528,63 +523,63 @@ class DatasetAnalyzer:
                 <div class="metric">静音文件: {summary['quality_assessment']['silent_files_count']}</div>
                 <div class="metric">削波文件: {summary['quality_assessment']['clipped_files_count']}</div>
             </div>
-            
+
             <div class="section">
                 <h2>改进建议</h2>
         """
-        
+
         for rec in summary['recommendations']:
             html += f'<div class="recommendation">• {rec}</div>'
-        
+
         html += """
             </div>
         </body>
         </html>
         """
-        
+
         return html
 
 
-def analyze_audio_dataset(directory: str, output_dir: str = None, 
-                         extensions: List[str] = None, sr: int = 22050,
-                         n_jobs: int = None) -> Dict[str, Any]:
+def analyze_audio_dataset(directory: str, output_dir: str = None,
+                          extensions: List[str] = None, sr: int = 22050,
+                          n_jobs: int = None) -> Dict[str, Any]:
     """
     快速分析音频数据集
-    
+
     Args:
         directory: 音频文件目录
         output_dir: 输出目录（可选）
         extensions: 支持的文件扩展名
         sr: 目标采样率
         n_jobs: 并行作业数
-        
+
     Returns:
         分析结果摘要
     """
     if extensions is None:
         extensions = ['.wav', '.mp3', '.flac', '.m4a', '.aac']
-    
+
     # 收集文件
     directory_path = Path(directory)
     file_paths = []
     for ext in extensions:
         file_paths.extend(list(directory_path.glob(f'**/*{ext}')))
-    
+
     file_paths = [str(p) for p in file_paths]
-    
+
     if not file_paths:
         raise ValueError(f"在目录 {directory} 中未找到音频文件")
-    
+
     # 分析数据集
     analyzer = DatasetAnalyzer(sr=sr, n_jobs=n_jobs)
     results = analyzer.analyze_dataset(file_paths)
-    
+
     # 导出结果
     if output_dir:
         analyzer.export_results(output_dir)
-        
+
         # 生成HTML报告
         html_path = Path(output_dir) / 'analysis_report.html'
         analyzer.create_analysis_report(str(html_path))
-    
-    return results
+
+    return results