cleanbookmarks 4.0.0__py3-none-any.whl

cleanbookmarks/__init__.py

@@ -0,0 +1,3 @@
+"""CleanBookmarks - 离线书签清理与分类 CLI"""
+
+__version__ = "4.0.0"

cleanbookmarks/cache.py

@@ -0,0 +1,102 @@
+"""线程安全的 LRU 缓存"""
+
+from collections import OrderedDict
+from typing import Callable, Generic, Hashable, Optional, TypeVar
+import threading
+
+T = TypeVar("T")
+
+
+class CacheManager(Generic[T]):
+    """LRU 缓存管理器"""
+
+    def __init__(self, max_size: int = 1000, strategy: str = "lru", thread_safe: bool = True):
+        if strategy != "lru":
+            raise ValueError(f"不支持的淘汰策略: {strategy}，目前只支持 'lru'")
+        self.max_size = max_size
+        self.strategy = strategy
+        self._cache: OrderedDict[Hashable, T] = OrderedDict()
+        self._lock = threading.Lock() if thread_safe else None
+        self._stats = {"hits": 0, "misses": 0, "evictions": 0, "put_count": 0, "get_count": 0}
+
+    def get(self, key: Hashable) -> Optional[T]:
+        if self._lock:
+            with self._lock:
+                return self._get_unsafe(key)
+        return self._get_unsafe(key)
+
+    def _get_unsafe(self, key: Hashable) -> Optional[T]:
+        self._stats["get_count"] += 1
+        if key in self._cache:
+            self._cache.move_to_end(key)
+            self._stats["hits"] += 1
+            return self._cache[key]
+        self._stats["misses"] += 1
+        return None
+
+    def get_or_compute(self, key: Hashable, factory: Callable[[], T]) -> T:
+        if self._lock:
+            with self._lock:
+                value = self._get_unsafe(key)
+                if value is not None:
+                    return value
+                value = factory()
+                self._put_unsafe(key, value)
+                return value
+        value = self._get_unsafe(key)
+        if value is not None:
+            return value
+        value = factory()
+        self._put_unsafe(key, value)
+        return value
+
+    def put(self, key: Hashable, value: T) -> None:
+        if self._lock:
+            with self._lock:
+                self._put_unsafe(key, value)
+        else:
+            self._put_unsafe(key, value)
+
+    def _put_unsafe(self, key: Hashable, value: T) -> None:
+        self._stats["put_count"] += 1
+        if key in self._cache:
+            self._cache.move_to_end(key)
+            self._cache[key] = value
+        else:
+            self._cache[key] = value
+            if len(self._cache) > self.max_size:
+                self._cache.popitem(last=False)
+                self._stats["evictions"] += 1
+
+    def invalidate(self, key: Hashable) -> bool:
+        if self._lock:
+            with self._lock:
+                if key in self._cache:
+                    del self._cache[key]
+                    return True
+                return False
+        if key in self._cache:
+            del self._cache[key]
+            return True
+        return False
+
+    def clear(self) -> None:
+        if self._lock:
+            with self._lock:
+                self._cache.clear()
+        else:
+            self._cache.clear()
+
+    def get_stats(self) -> dict:
+        total = self._stats["hits"] + self._stats["misses"]
+        hit_rate = self._stats["hits"] / total if total > 0 else 0.0
+        return {**self._stats, "size": len(self._cache), "max_size": self.max_size, "hit_rate": hit_rate}
+
+    def __len__(self) -> int:
+        return len(self._cache)
+
+    def __contains__(self, key: Hashable) -> bool:
+        if self._lock:
+            with self._lock:
+                return key in self._cache
+        return key in self._cache

cleanbookmarks/classifier.py

@@ -0,0 +1,282 @@
+"""书签分类器 - 规则优先 + LLM(可选) 两级级联"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import threading
+from datetime import datetime
+from typing import Dict, Optional
+
+from cleanbookmarks.cache import CacheManager
+from cleanbookmarks.config import load_json_config, resolve_config_path
+from cleanbookmarks.models import BookmarkFeatures, ClassificationResult
+from cleanbookmarks.rules import RuleEngine
+from cleanbookmarks.text_utils import (
+    detect_language,
+    is_video_url,
+    normalize_category_config,
+)
+
+try:
+    from cleanbookmarks.llm import LLMClassifier
+except ImportError:
+    LLMClassifier = None  # type: ignore[assignment,misc]
+
+
+class BookmarkClassifier:
+    """书签分类器
+
+    两级级联：规则引擎给出确定性主分类，LLM(可选)在规则未命中时兜底、命中时补充子分类。
+    """
+
+    def __init__(
+        self,
+        config_path: Optional[str] = None,
+        config: Optional[Dict] = None,
+    ):
+        resolved_path, _ = resolve_config_path(config_path)
+        self.config_path = str(resolved_path)
+        self.logger = logging.getLogger(__name__)
+
+        if isinstance(config, dict):
+            normalized = normalize_category_config(config)
+            if not isinstance(normalized.get("category_rules"), dict) or not normalized.get("category_rules"):
+                raise ValueError("传入的 config 缺少有效的 category_rules")
+            self._config = normalized
+        else:
+            self._config = None
+        self._rule_engine: Optional[RuleEngine] = None
+        self._llm_classifier = None
+
+        # 缓存大小来自配置（默认 10000），分类结果缓存减半以省内存
+        cache_size = 10000
+        if isinstance(config, dict):
+            try:
+                cache_size = int((config.get("ai_settings") or {}).get("cache_size", 10000))
+            except (TypeError, ValueError):
+                cache_size = 10000
+        self.feature_cache: CacheManager[BookmarkFeatures] = CacheManager(max_size=cache_size, strategy="lru")
+        self.classification_cache: CacheManager[ClassificationResult] = CacheManager(max_size=max(cache_size // 2, 100), strategy="lru")
+
+        # stats 由多线程（_classify_batch）并发更新，需要锁保护
+        self._stats_lock = threading.Lock()
+
+        self.stats = {
+            "total_classified": 0,
+            "rule_engine": 0,
+            "fallback": 0,
+            "cache_hits": 0,
+            "average_confidence": 0.0,
+            "llm": 0,
+        }
+
+    @property
+    def config(self) -> Dict:
+        if self._config is None:
+            self._config = self._load_config()
+        return self._config
+
+    @property
+    def rule_engine(self) -> RuleEngine:
+        if self._rule_engine is None:
+            self._rule_engine = RuleEngine(self.config)
+        return self._rule_engine
+
+    @property
+    def llm_classifier(self):
+        if self._llm_classifier is None and LLMClassifier is not None:
+            try:
+                self._llm_classifier = LLMClassifier(self.config_path)
+            except Exception as e:
+                self.logger.warning(f"LLM 分类器初始化失败: {e}")
+        return self._llm_classifier
+
+    def _load_config(self) -> Dict:
+        config, _, _ = load_json_config(self.config_path)
+        normalized = normalize_category_config(config)
+        if not isinstance(normalized.get("category_rules"), dict) or not normalized.get("category_rules"):
+            raise ValueError(f"配置缺少有效的 category_rules: {self.config_path}")
+        return normalized
+
+    def extract_features(self, url: str, title: str) -> BookmarkFeatures:
+        cache_key = f"{url}::{title}"
+
+        def _extract():
+            content_type = self._detect_content_type(url, title)
+            language = detect_language(title)
+            return BookmarkFeatures.from_url_title(url, title, content_type, language)
+
+        return self.feature_cache.get_or_compute(cache_key, _extract)
+
+    def classify(self, url: str, title: str) -> ClassificationResult:
+        start_time = datetime.now()
+        cache_key = hashlib.md5(f"{url}::{title}".encode()).hexdigest()
+        cached = self.classification_cache.get(cache_key)
+        if cached is not None:
+            with self._stats_lock:
+                self.stats["cache_hits"] += 1
+            cached.processing_time = (datetime.now() - start_time).total_seconds()
+            return cached
+
+        features = self.extract_features(url, title)
+
+        # 1) 规则引擎 - 确定性优先
+        rule_result = self.rule_engine.classify(features)
+
+        # 2) LLM（可选）- 规则未命中时兜底，命中时补充子分类
+        llm_result = None
+        if self.llm_classifier and self.llm_classifier.enabled():
+            try:
+                llm_result = self.llm_classifier.classify(
+                    url, title,
+                    context={"domain": features.domain, "content_type": features.content_type, "language": features.language},
+                )
+            except Exception as e:
+                self.logger.warning(f"LLM 分类调用失败: {e}")
+
+        confidence_threshold = self.config.get("ai_settings", {}).get("confidence_threshold", 0.7)
+        final_result = self._cascade_fuse(
+            rule_result=rule_result,
+            llm_result=llm_result,
+            features=features,
+            confidence_threshold=float(confidence_threshold),
+        )
+
+        with self._stats_lock:
+            if "rule_engine" in final_result.method:
+                self.stats["rule_engine"] += 1
+            if "llm" in final_result.method:
+                self.stats["llm"] += 1
+            if final_result.method == "fallback":
+                self.stats["fallback"] += 1
+
+        final_result.processing_time = (datetime.now() - start_time).total_seconds()
+        self._update_stats(final_result)
+        self.classification_cache.put(cache_key, final_result)
+        return final_result
+
+    def _cascade_fuse(
+        self,
+        rule_result,
+        llm_result,
+        features: BookmarkFeatures,
+        confidence_threshold: float,
+    ) -> ClassificationResult:
+        """级联决策：规则命中即采用规则主分类，LLM 补子分类/facets；规则未命中才走 LLM"""
+        if rule_result is not None:
+            result = self._to_classification_result(rule_result)
+            # 1) 配置的 category_hierarchy 标题匹配
+            if result.subcategory is None:
+                result.subcategory = self._determine_subcategory(result.category, features)
+            # 2) LLM 补充子分类/facets/理由
+            if llm_result is not None:
+                llm = self._to_classification_result(llm_result)
+                if result.subcategory is None and llm.subcategory:
+                    result.subcategory = llm.subcategory
+                # LLM 输出不可控，facets 可能是非 dict（如列表/字符串），防御性处理
+                llm_facets = llm.facets if isinstance(llm.facets, dict) else {}
+                for k, v in llm_facets.items():
+                    if v and k not in (result.facets or {}):
+                        result.facets[k] = v
+                result.reasoning.extend(llm.reasoning or [])
+        elif llm_result is not None:
+            result = self._to_classification_result(llm_result)
+        else:
+            return ClassificationResult(
+                category="未分类", confidence=0.0,
+                reasoning=["没有匹配到任何分类规则"], method="fallback",
+            )
+
+        if result.category != "未分类" and result.confidence < confidence_threshold:
+            result.reasoning.append(
+                f"最终置信度 {result.confidence:.2f} 低于阈值 {confidence_threshold:.2f}，标记为未分类"
+            )
+            return ClassificationResult(
+                category="未分类", subcategory=None,
+                confidence=result.confidence,
+                reasoning=result.reasoning,
+                alternatives=result.alternatives[:3],
+                method=result.method, facets=result.facets,
+            )
+        return result
+
+    @staticmethod
+    def _to_classification_result(raw) -> ClassificationResult:
+        if isinstance(raw, ClassificationResult):
+            return raw
+        if isinstance(raw, dict):
+            # LLM 输出不可控，facets 可能是非 dict（如列表/字符串），统一防御
+            facets = raw.get("facets", {})
+            if not isinstance(facets, dict):
+                facets = {}
+            return ClassificationResult(
+                category=raw.get("category", "未分类"),
+                confidence=float(raw.get("confidence", 0.0)),
+                subcategory=raw.get("subcategory"),
+                reasoning=raw.get("reasoning", []),
+                alternatives=raw.get("alternatives", []),
+                processing_time=float(raw.get("processing_time", 0.0)),
+                method=raw.get("method", "unknown"),
+                facets=facets,
+            )
+        raise TypeError(f"Unexpected classification result type: {type(raw)}")
+
+    def _determine_subcategory(self, category: str, features: BookmarkFeatures) -> Optional[str]:
+        hierarchy = self.config.get("category_hierarchy", {})
+        if not isinstance(hierarchy, dict):
+            return None
+        # 规则引擎的 category 可能是 '主类/子类' 格式，按主类查 hierarchy
+        main = category.split("/", 1)[0].strip()
+        subs = hierarchy.get(category) or hierarchy.get(main)
+        if not isinstance(subs, list):
+            return None
+        title_lower = features.title.lower()
+        for sub in subs:
+            if str(sub).lower() in title_lower:
+                return sub
+        return None
+
+    def _detect_content_type(self, url: str, title: str) -> str:
+        url_lower = url.lower()
+        title_lower = title.lower()
+        if is_video_url(url):
+            return "video"
+        if any(d in url_lower for d in ["github.com", "gitlab.com"]):
+            return "code_repository"
+        if any(p in url_lower for p in ["docs.", "documentation", "wiki"]):
+            return "documentation"
+        if any(d in url_lower for d in ["arxiv.org", "acm.org", "ieee.org"]):
+            return "academic_paper"
+        if any(k in title_lower for k in ["news", "新闻", "breaking"]):
+            return "news"
+        if any(k in title_lower for k in ["tool", "工具", "online", "generator"]):
+            return "online_tool"
+        return "webpage"
+
+    def _update_stats(self, result: ClassificationResult):
+        with self._stats_lock:
+            self.stats["total_classified"] += 1
+            total = self.stats["total_classified"]
+            old_avg = self.stats["average_confidence"]
+            self.stats["average_confidence"] = (old_avg * (total - 1) + result.confidence) / total
+
+    def get_statistics(self) -> Dict:
+        total_predictions = (
+            self.stats["rule_engine"] + self.stats["llm"] + self.stats["fallback"]
+        )
+        # total_classified 只在缓存未命中时 +1，分母 = 命中 + 未命中 = 总尝试数
+        total_attempts = self.stats["cache_hits"] + self.stats["total_classified"]
+        return {
+            "total_classified": self.stats["total_classified"],
+            "cache_hits": self.stats["cache_hits"],
+            "cache_hit_rate": self.stats["cache_hits"] / max(total_attempts, 1),
+            "average_confidence": self.stats["average_confidence"],
+            "classification_methods": {
+                "rule_engine": self.stats["rule_engine"],
+                "llm": self.stats["llm"],
+                "unclassified (fallback)": self.stats["fallback"],
+                "total": total_predictions,
+            },
+            "llm_enabled": self.llm_classifier is not None and self.llm_classifier.enabled(),
+        }

cleanbookmarks/cli.py

@@ -0,0 +1,219 @@
+"""CleanBookmarks CLI 入口"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import logging
+import os
+import sys
+from pathlib import Path
+
+from cleanbookmarks import __version__
+from cleanbookmarks.config import ResourceResolutionError, resolve_config_path
+from cleanbookmarks.processor import BookmarkProcessor
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging(log_level: str = "INFO", use_file: bool = False):
+    handlers: list[logging.Handler] = [logging.StreamHandler()]
+    if use_file:
+        os.makedirs("logs", exist_ok=True)
+        handlers.insert(0, logging.FileHandler("logs/cleanbookmarks.log", encoding="utf-8"))
+    logging.basicConfig(
+        level=getattr(logging, log_level.upper()),
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=handlers,
+        force=True,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=f"CleanBookmarks v{__version__} - 书签清理与分类",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+示例:
+  %(prog)s -i bookmarks.html -o output/
+  %(prog)s -i examples/demo_bookmarks.html
+  %(prog)s --health-check
+        """,
+    )
+    parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}")
+    parser.add_argument("-i", "--input", nargs="+", help="输入的HTML书签文件")
+    parser.add_argument("-o", "--output", default="output", help="输出目录")
+    parser.add_argument("-c", "--config", default=None, help="配置文件路径")
+    parser.add_argument("--health-check", action="store_true", help="运行健康检查")
+    parser.add_argument("--workers", type=int, default=4, help="并行处理线程数")
+    parser.add_argument("--threshold", type=float, default=None, help="分类置信度阈值（默认使用配置文件中的值）")
+    parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
+    parser.add_argument("--limit", type=int, default=0, help="限制处理的书签数量（调试用）")
+    parser.add_argument("--eval", metavar="FILE", help="评估分类效果，传入标注数据 JSON 文件")
+
+    args = parser.parse_args()
+
+    try:
+        config_path, _ = resolve_config_path(args.config)
+        use_file_logging = bool(args.input)
+        setup_logging(args.log_level, use_file=use_file_logging)
+
+        if args.health_check:
+            from cleanbookmarks.health import run_health_check
+            ok = run_health_check(str(config_path))
+            sys.exit(0 if ok else 1)
+
+        if args.input:
+            input_files = []
+            for pattern in args.input:
+                if "*" in pattern or "?" in pattern:
+                    expanded = glob.glob(pattern)
+                    if expanded:
+                        input_files.extend(expanded)
+                    else:
+                        logger.warning(f"没有找到匹配模式的文件: {pattern}")
+                else:
+                    if Path(pattern).is_file():
+                        input_files.append(pattern)
+                    else:
+                        logger.warning(f"文件不存在: {pattern}")
+
+            if not input_files:
+                logger.error("没有找到有效的输入文件")
+                sys.exit(1)
+
+            logger.info(f"将处理 {len(input_files)} 个文件: {input_files}")
+
+            processor = BookmarkProcessor(
+                config_path=str(config_path),
+                max_workers=args.workers,
+                confidence_threshold=args.threshold,
+            )
+
+            results = processor.process_files(
+                input_files=input_files,
+                output_dir=args.output,
+                limit=args.limit if args.limit and args.limit > 0 else 0,
+            )
+
+            logger.info(f"处理完成: {results['processed_bookmarks']} 个书签已分类")
+            return
+
+        if args.eval:
+            return run_eval(args)
+
+        parser.print_help()
+
+    except KeyboardInterrupt:
+        logger.info("程序被用户中断")
+        sys.exit(1)
+    except (FileNotFoundError, ValueError, ResourceResolutionError) as e:
+        logger.error(f"配置或资源错误: {e}")
+        if args.log_level == "DEBUG":
+            raise
+        sys.exit(2)
+    except ImportError as e:
+        logger.error(f"依赖缺失: {e}")
+        if args.log_level == "DEBUG":
+            raise
+        sys.exit(3)
+    except Exception as e:
+        logger.error(f"程序执行失败: {e}")
+        if args.log_level == "DEBUG":
+            raise
+        sys.exit(1)
+
+
+def run_eval(args):
+    """评估分类效果：加载标注数据，逐条分类，输出准确率"""
+    import json
+    from cleanbookmarks.text_utils import normalize_category_string
+
+    eval_path = Path(args.eval)
+    if not eval_path.is_file():
+        logger.error(f"标注文件不存在: {eval_path}")
+        sys.exit(1)
+
+    try:
+        with open(eval_path, "r", encoding="utf-8") as f:
+            labeled = json.load(f)
+    except json.JSONDecodeError as e:
+        logger.error(f"标注文件不是合法 JSON: {e}")
+        sys.exit(1)
+    if not isinstance(labeled, list):
+        logger.error("标注文件顶层必须是 JSON 数组")
+        sys.exit(1)
+
+    processor = BookmarkProcessor(
+        config_path=str(resolve_config_path(args.config)[0]),
+        max_workers=args.workers,
+        confidence_threshold=args.threshold,
+    )
+
+    correct = 0
+    evaluated = 0
+    skipped = 0
+    mismatches = []
+    category_stats = {}
+
+    for item in labeled:
+        if not isinstance(item, dict):
+            logger.warning(f"标注数据格式非法，跳过: {item}")
+            skipped += 1
+            continue
+        url = item.get("url", "")
+        title = item.get("title", "")
+        expected = normalize_category_string(item.get("expected", ""))
+        if not url or not title or not expected:
+            logger.warning(f"标注数据缺字段，跳过: {item}")
+            skipped += 1
+            continue
+        result = processor.classifier.classify(url, title)
+        predicted = normalize_category_string(result.category)
+        evaluated += 1
+
+        top_expected = expected.split("/")[0]
+        top_predicted = predicted.split("/")[0]
+
+        category_stats.setdefault(top_expected, {"correct": 0, "total": 0})
+        category_stats[top_expected]["total"] += 1
+        if top_predicted == top_expected:
+            correct += 1
+            category_stats[top_expected]["correct"] += 1
+        else:
+            mismatches.append({
+                "url": url,
+                "title": title,
+                "expected": top_expected,
+                "predicted": top_predicted,
+                "confidence": round(result.confidence, 3),
+                "method": result.method,
+            })
+
+    total = evaluated
+    accuracy = correct / total if total > 0 else 0
+    print(f"\n{'='*50}")
+    print(f"评估结果: {correct}/{total} 正确 ({accuracy:.1%})")
+    if skipped:
+        print(f"（跳过 {skipped} 条缺字段的标注）")
+    print("分类方法: 规则引擎（LLM 可选）")
+    print(f"{'='*50}")
+
+    print(f"\n各分类准确率:")
+    for cat in sorted(category_stats.keys()):
+        s = category_stats[cat]
+        rate = s["correct"] / s["total"] if s["total"] > 0 else 0
+        print(f"  {cat}: {s['correct']}/{s['total']} ({rate:.0%})")
+
+    if mismatches:
+        print(f"\n分类错误 ({len(mismatches)} 条):")
+        for m in mismatches:
+            print(f"  [{m['expected']} -> {m['predicted']}] {m['title'][:50]}")
+            print(f"    url: {m['url'][:80]}")
+            print(f"    method={m['method']}, confidence={m['confidence']}")
+
+    return
+
+
+if __name__ == "__main__":
+    main()