webmainbench 0.1.0__py3-none-any.whl

webmainbench/__init__.py

@@ -0,0 +1,33 @@
+"""
+WebMainBench: A comprehensive benchmark for web main content extraction.
+
+This package provides a standardized evaluation framework for comparing
+different web content extraction tools and methods.
+"""
+
+__version__ = "0.1.0"
+__author__ = "WebMainBench Team"
+
+from .data import DataLoader, DataSaver, BenchmarkDataset, DataSample
+from .extractors import BaseExtractor, ExtractorFactory, ExtractionResult
+from .metrics import BaseMetric, MetricCalculator, MetricResult
+from .evaluator import Evaluator, EvaluationResult, MainHTMLEvaluator
+from .utils import setup_logging, format_results
+
+__all__ = [
+    "DataLoader",
+    "DataSaver", 
+    "BenchmarkDataset",
+    "DataSample",
+    "BaseExtractor",
+    "ExtractorFactory",
+    "ExtractionResult",
+    "BaseMetric",
+    "MetricCalculator",
+    "MetricResult",
+    "Evaluator",
+    "EvaluationResult",
+    "setup_logging",
+    "format_results",
+    "MainHTMLEvaluator"
+] 

webmainbench/config.py

@@ -0,0 +1,11 @@
+"""
+全局配置文件
+"""
+
+# LLM配置，用于修正抽取工具的抽取结果
+LLM_CONFIG = {
+    'llm_base_url': '',
+    'llm_api_key': '',
+    'llm_model': 'deepseek-chat',
+    'use_llm': True
+}

webmainbench/data/__init__.py

@@ -0,0 +1,16 @@
+"""
+Data module for WebMainBench.
+
+This module handles loading, saving and managing benchmark datasets.
+"""
+
+from .dataset import BenchmarkDataset, DataSample
+from .loader import DataLoader
+from .saver import DataSaver
+
+__all__ = [
+    "BenchmarkDataset",
+    "DataSample", 
+    "DataLoader",
+    "DataSaver",
+] 

webmainbench/data/dataset.py

@@ -0,0 +1,161 @@
+"""
+Dataset classes for WebMainBench.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Any, Union
+import json
+from pathlib import Path
+
+
+@dataclass
+class DataSample:
+    """Single data sample in the benchmark dataset."""
+    
+    # Required fields
+    id: str
+    html: str  # HTML with cc-select=true annotations
+    groundtruth_content: str  # Groundtruth markdown content
+    # Optional fields with defaults
+    groundtruth_content_list: Optional[List[Dict[str, Any]]] = None  # Groundtruth content_list from llm-webkit
+    content_list: List[Dict[str, Any]] = None # Content_list from llm-webkit
+    content: str = None  # Content from llm-webkit
+    # Optional metadata
+    url: Optional[str] = None
+    domain: Optional[str] = None
+    language: Optional[str] = None
+    content_type: Optional[str] = None  # article, forum, blog, etc.
+    difficulty: Optional[str] = None  # easy, medium, hard
+    tags: Optional[List[str]] = None
+    llm_webkit_md: Optional[str] = None
+    llm_webkit_html: Optional[str] = None  # 预处理HTML字段
+    main_html: Optional[str] = None  # 主要HTML内容字段
+    
+    # Extracted results (populated during evaluation)
+    extracted_results: Optional[Dict[str, Any]] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary format."""
+        return {
+            "id": self.id,
+            "html": self.html,
+            "groundtruth_content": self.groundtruth_content,
+            "groundtruth_content_list": self.groundtruth_content_list,
+            "content_list": self.content_list,
+            "content": self.content,
+            "llm_webkit_md": self.llm_webkit_md,
+            "llm_webkit_html": self.llm_webkit_html,
+            "main_html": self.main_html,
+            "url": self.url,
+            "domain": self.domain,
+            "language": self.language,
+            "content_type": self.content_type,
+            "difficulty": self.difficulty,
+            "tags": self.tags,
+            "extracted_results": self.extracted_results,
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "DataSample":
+        """Create from dictionary, ignoring unknown fields and supporting field mapping."""
+        # 获取类的所有字段名
+        import dataclasses
+        field_names = {f.name for f in dataclasses.fields(cls)}
+        
+        # 定义字段名映射（外部字段名 -> 内部字段名）
+        field_mapping = {
+            "track_id": "id",  # track_id 映射到 id
+            "content": "groundtruth_content",  # content 映射到 groundtruth_content
+            "convert_main_content": "groundtruth_content",  # convert_main_content 映射到 groundtruth_content
+            "content_list": "groundtruth_content_list",  # content_list 映射到 groundtruth_content_list
+        }
+        
+        # 只提取定义的字段，忽略其他字段
+        filtered_data = {}
+        for key, value in data.items():
+            # 首先检查是否需要字段映射
+            mapped_key = field_mapping.get(key, key)
+            
+            # 如果映射后的字段名在类字段中，则添加
+            if mapped_key in field_names:
+                filtered_data[mapped_key] = value
+            # 忽略未定义的字段，如 layout_id、max_layer_n 等
+        
+        return cls(**filtered_data)
+
+
+class BenchmarkDataset:
+    """Main dataset class for WebMainBench."""
+    
+    def __init__(self, name: str, description: str = ""):
+        self.name = name
+        self.description = description
+        self.samples: List[DataSample] = []
+        self._metadata: Dict[str, Any] = {}
+    
+    def add_sample(self, sample: DataSample) -> None:
+        """Add a data sample to the dataset."""
+        self.samples.append(sample)
+    
+    def get_sample(self, sample_id: str) -> Optional[DataSample]:
+        """Get a sample by ID."""
+        for sample in self.samples:
+            if sample.id == sample_id:
+                return sample
+        return None
+    
+    def filter_by_criteria(self, **kwargs) -> List[DataSample]:
+        """Filter samples by criteria (e.g., language='en', difficulty='hard')."""
+        filtered = self.samples
+        for key, value in kwargs.items():
+            filtered = [s for s in filtered if getattr(s, key, None) == value]
+        return filtered
+    
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get dataset statistics."""
+        stats = {
+            "total_samples": len(self.samples),
+            "languages": {},
+            "content_types": {},
+            "difficulties": {},
+            "domains": {},
+        }
+        
+        for sample in self.samples:
+            # Count languages
+            lang = sample.language or "unknown"
+            stats["languages"][lang] = stats["languages"].get(lang, 0) + 1
+            
+            # Count content types
+            ctype = sample.content_type or "unknown"
+            stats["content_types"][ctype] = stats["content_types"].get(ctype, 0) + 1
+            
+            # Count difficulties
+            diff = sample.difficulty or "unknown"
+            stats["difficulties"][diff] = stats["difficulties"].get(diff, 0) + 1
+            
+            # Count domains
+            domain = sample.domain or "unknown"
+            stats["domains"][domain] = stats["domains"].get(domain, 0) + 1
+        
+        return stats
+    
+    def set_metadata(self, key: str, value: Any) -> None:
+        """Set dataset metadata."""
+        self._metadata[key] = value
+    
+    def get_metadata(self, key: str = None) -> Union[Any, Dict[str, Any]]:
+        """Get dataset metadata."""
+        if key:
+            return self._metadata.get(key)
+        return self._metadata.copy()
+    
+    def __len__(self) -> int:
+        return len(self.samples)
+    
+    def __iter__(self):
+        return iter(self.samples)
+    
+    def __getitem__(self, index: int) -> DataSample:
+        return self.samples[index] 

webmainbench/data/loader.py

@@ -0,0 +1,231 @@
+"""
+Data loader for WebMainBench.
+"""
+
+import json
+import jsonlines
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Union, Iterator
+from .dataset import BenchmarkDataset, DataSample
+
+
+class DataLoader:
+    """Data loader for various input formats."""
+    
+    @staticmethod
+    def load_jsonl(file_path: Union[str, Path], **kwargs) -> BenchmarkDataset:
+        """
+        Load dataset from JSONL file.
+        
+        Args:
+            file_path: Path to the JSONL file
+            **kwargs: Additional parameters for dataset creation
+        
+        Returns:
+            BenchmarkDataset instance
+        """
+        file_path = Path(file_path)
+        dataset_name = kwargs.get('name', file_path.stem)
+        dataset = BenchmarkDataset(name=dataset_name)
+        
+        with jsonlines.open(file_path, 'r') as reader:
+            for idx, line in enumerate(reader):
+                try:
+                    # 使用DataSample.from_dict()来正确处理字段映射和过滤
+                    sample = DataSample.from_dict(line)
+                    dataset.add_sample(sample)
+                    
+                except Exception as e:
+                    print(f"Warning: Failed to load sample at line {idx}: {e}")
+                    continue
+        
+        return dataset
+    
+    @staticmethod
+    def load_json(file_path: Union[str, Path], **kwargs) -> BenchmarkDataset:
+        """
+        Load dataset from JSON file.
+        
+        Args:
+            file_path: Path to the JSON file
+            **kwargs: Additional parameters for dataset creation
+        
+        Returns:
+            BenchmarkDataset instance
+        """
+        file_path = Path(file_path)
+        dataset_name = kwargs.get('name', file_path.stem)
+        dataset = BenchmarkDataset(name=dataset_name)
+        
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        
+        # Handle different JSON structures
+        if isinstance(data, list):
+            # Array of samples
+            samples_data = data
+        elif isinstance(data, dict):
+            if 'samples' in data:
+                # Structured format with metadata
+                samples_data = data['samples']
+                # Load metadata if available
+                if 'metadata' in data:
+                    for key, value in data['metadata'].items():
+                        dataset.set_metadata(key, value)
+            else:
+                # Single sample in dict format
+                samples_data = [data]
+        else:
+            raise ValueError(f"Unsupported JSON structure in {file_path}")
+        
+        for idx, sample_data in enumerate(samples_data):
+            try:
+                sample = DataSample.from_dict(sample_data)
+                if not sample.id:
+                    sample.id = f"sample_{idx}"
+                dataset.add_sample(sample)
+            except Exception as e:
+                print(f"Warning: Failed to load sample {idx}: {e}")
+                continue
+        
+        return dataset
+    
+    @staticmethod
+    def load_from_directory(dir_path: Union[str, Path], 
+                          pattern: str = "*.jsonl", 
+                          **kwargs) -> Dict[str, BenchmarkDataset]:
+        """
+        Load multiple datasets from a directory.
+        
+        Args:
+            dir_path: Directory containing dataset files
+            pattern: File pattern to match (default: "*.jsonl")
+            **kwargs: Additional parameters for dataset creation
+        
+        Returns:
+            Dictionary mapping filenames to BenchmarkDataset instances
+        """
+        dir_path = Path(dir_path)
+        datasets = {}
+        
+        for file_path in dir_path.glob(pattern):
+            try:
+                if file_path.suffix == '.jsonl':
+                    dataset = DataLoader.load_jsonl(file_path, **kwargs)
+                elif file_path.suffix == '.json':
+                    dataset = DataLoader.load_json(file_path, **kwargs)
+                else:
+                    print(f"Warning: Unsupported file format: {file_path}")
+                    continue
+                
+                datasets[file_path.stem] = dataset
+                
+            except Exception as e:
+                print(f"Error loading {file_path}: {e}")
+                continue
+        
+        return datasets
+    
+    @staticmethod
+    def merge_datasets(datasets: List[BenchmarkDataset], 
+                      name: str = "merged_dataset") -> BenchmarkDataset:
+        """
+        Merge multiple datasets into one.
+        
+        Args:
+            datasets: List of BenchmarkDataset instances to merge
+            name: Name for the merged dataset
+        
+        Returns:
+            Merged BenchmarkDataset instance
+        """
+        merged = BenchmarkDataset(name=name)
+        
+        for dataset in datasets:
+            for sample in dataset.samples:
+                # Ensure unique IDs
+                original_id = sample.id
+                counter = 1
+                while merged.get_sample(sample.id) is not None:
+                    sample.id = f"{original_id}_{counter}"
+                    counter += 1
+                
+                merged.add_sample(sample)
+        
+        return merged
+    
+    @staticmethod
+    def stream_jsonl(file_path: Union[str, Path],
+                    categories: Optional[List[str]] = None,
+                    max_samples: Optional[int] = None) -> Iterator[DataSample]:
+        """
+        流式读取JSONL文件，逐个返回DataSample，减少内存使用。
+        
+        Args:
+            file_path: JSONL文件路径
+            categories: 类别过滤列表
+            max_samples: 最大样本数限制
+            
+        Yields:
+            DataSample: 逐个生成的数据样本
+        """
+        file_path = Path(file_path)
+        
+        sample_count = 0
+        with jsonlines.open(file_path, 'r') as reader:
+            for line_idx, line in enumerate(reader):
+                try:
+                    # 创建样本
+                    sample = DataSample.from_dict(line)
+                    
+                    # 类别过滤
+                    if categories and sample.content_type not in categories:
+                        continue
+                    
+                    # 返回样本
+                    yield sample
+                    sample_count += 1
+                    
+                    # 检查样本数限制
+                    if max_samples and sample_count >= max_samples:
+                        break
+                        
+                except Exception as e:
+                    print(f"Warning: Failed to load sample at line {line_idx}: {e}")
+                    continue
+    
+    @staticmethod
+    def stream_jsonl_batched(file_path: Union[str, Path],
+                           batch_size: int = 50,
+                           categories: Optional[List[str]] = None,
+                           max_samples: Optional[int] = None) -> Iterator[List[DataSample]]:
+        """
+        流式读取JSONL文件，按批次返回DataSample列表。
+        
+        Args:
+            file_path: JSONL文件路径
+            batch_size: 批次大小
+            categories: 类别过滤列表
+            max_samples: 最大样本数限制
+            
+        Yields:
+            List[DataSample]: 批次数据样本列表
+        """
+        batch = []
+        sample_count = 0
+        
+        for sample in DataLoader.stream_jsonl(file_path, categories, max_samples):
+            batch.append(sample)
+            sample_count += 1
+            
+            # 达到批次大小或样本数限制时返回批次
+            if len(batch) >= batch_size or (max_samples and sample_count >= max_samples):
+                yield batch
+                batch = []
+                
+                if max_samples and sample_count >= max_samples:
+                    break
+        
+        # 返回最后一批（如果有）
+        if batch:
+            yield batch