novel-downloader 1.1.0__py3-none-any.whl

novel_downloader/utils/hash_store.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.hash_store
+---------------------------------
+
+Manage a small collection of image perceptual hashes and their labels.
+Supports loading/saving to .json or .npy, and basic CRUD + search.
+"""
+
+import heapq
+import json
+import logging
+from pathlib import Path
+from typing import Callable, Dict, List, Optional, Set, Tuple, Union
+
+import numpy as np
+from PIL import Image
+
+from .constants import HASH_STORE_FILE
+from .hash_utils import HASH_DISTANCE_THRESHOLD, fast_hamming_distance, phash
+
+logger = logging.getLogger(__name__)
+
+
+class _BKNode:
+    """
+    A node in a Burkhard-Keller tree (BK-Tree) for distance search.
+    Stores one value and a dict of children keyed by distance.
+    """
+
+    __slots__ = ("value", "children")
+
+    def __init__(self, value: int):
+        self.value = value
+        self.children: Dict[int, _BKNode] = {}
+
+    def add(self, h: int, dist_fn: Callable[[int, int], int]) -> None:
+        d = dist_fn(h, self.value)
+        child = self.children.get(d)
+        if child is not None:
+            child.add(h, dist_fn)
+        else:
+            self.children[d] = _BKNode(h)
+
+    def query(
+        self,
+        target: int,
+        threshold: int,
+        dist_fn: Callable[[int, int], int],
+    ) -> List[Tuple[int, int]]:
+        """
+        Recursively collect (value, dist) pairs within threshold.
+        """
+        d0 = dist_fn(target, self.value)
+        matches: List[Tuple[int, int]] = []
+        if d0 <= threshold:
+            matches.append((self.value, d0))
+        # Only children whose edge-dist \in [d0-threshold, d0+threshold]
+        lower, upper = d0 - threshold, d0 + threshold
+        for edge, child in self.children.items():
+            if lower <= edge <= upper:
+                matches.extend(child.query(target, threshold, dist_fn))
+        return matches
+
+
+class ImageHashStore:
+    """
+    Store and manage image hashes grouped by label, with a BK-Tree index.
+
+    :param path: file path for persistence (".json" or ".npy")
+    :param auto_save: if True, every modification automatically calls save()
+    :param hash_func: function to compute hash from PIL.Image
+    :param ham_dist: function to compute Hamming distance between two hashes
+    """
+
+    def __init__(
+        self,
+        path: Union[str, Path] = HASH_STORE_FILE,
+        auto_save: bool = False,
+        hash_func: Callable[[Image.Image], int] = phash,
+        ham_dist: Callable[[int, int], int] = fast_hamming_distance,
+        threshold: int = HASH_DISTANCE_THRESHOLD,
+    ) -> None:
+        self._path = Path(path)
+        self._auto = auto_save
+        self._hf = hash_func
+        self._hd = ham_dist
+        self._th = threshold
+
+        # label -> set of hashes
+        self._hash: Dict[str, Set[int]] = {}
+        # hash -> list of labels (for reverse lookup)
+        self._hash_to_labels: Dict[int, List[str]] = {}
+        # root of BK-Tree (or None if empty)
+        self._bk_root: Optional[_BKNode] = None
+
+        self.load()
+
+    def load(self) -> None:
+        """Load store from disk and rebuild BK-Tree index."""
+        if not self._path.exists():
+            self._hash.clear()
+            logger.info(
+                "[ImageHashStore] No file found at %s, starting empty.", self._path
+            )
+            return
+        else:
+            if self._path.suffix == ".npy":
+                arr = np.load(self._path, allow_pickle=True).item()
+                self._hash = {lbl: set(v) for lbl, v in arr.items()}
+            else:
+                txt = self._path.read_text(encoding="utf-8")
+                obj = json.loads(txt) or {}
+                self._hash = {lbl: set(obj.get(lbl, [])) for lbl in obj}
+
+        # rebuild reverse map and BK-Tree
+        self._hash_to_labels.clear()
+        for lbl, hs in self._hash.items():
+            for h in hs:
+                self._hash_to_labels.setdefault(h, []).append(lbl)
+        logger.info(
+            "[ImageHashStore] Loaded hash store from %s with %d hashes",
+            self._path,
+            sum(len(v) for v in self._hash.values()),
+        )
+
+        self._build_index()
+
+    def _build_index(self) -> None:
+        """Construct a BK-Tree over all stored hashes."""
+        self._bk_root = None
+        for h in self._hash_to_labels:
+            if self._bk_root is None:
+                self._bk_root = _BKNode(h)
+            else:
+                self._bk_root.add(h, self._hd)
+        logger.info(
+            "[ImageHashStore] BK-tree index built with %d unique hashes",
+            len(self._hash_to_labels),
+        )
+
+    def save(self) -> None:
+        """Persist current store to disk."""
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        data = {lbl: list(s) for lbl, s in self._hash.items()}
+        if self._path.suffix == ".npy":
+            np.save(self._path, data)
+        else:
+            txt = json.dumps(data, ensure_ascii=False, indent=2)
+            self._path.write_text(txt, encoding="utf-8")
+        logger.info("[ImageHashStore] Saved hash store to %s", self._path)
+
+    def _maybe_save(self) -> None:
+        if self._auto:
+            self.save()
+
+    def add_image(self, img_path: Union[str, Path], label: str) -> int:
+        """
+        Compute hash for the given image and add it under `label`.
+        Updates BK-Tree index incrementally.
+        """
+        img = Image.open(img_path).convert("L")
+        h = self._hf(img)
+        self._hash.setdefault(label, set()).add(h)
+        self._hash_to_labels.setdefault(h, []).append(label)
+        # insert into BK-Tree
+        if self._bk_root is None:
+            self._bk_root = _BKNode(h)
+        else:
+            self._bk_root.add(h, self._hd)
+        logger.debug("[ImageHashStore] Added hash %d under label '%s'", h, label)
+        self._maybe_save()
+        return h
+
+    def add_from_map(self, map_path: Union[str, Path]) -> None:
+        """
+        Load a JSON file of the form { "image_path": "label", ... }
+        and add each entry.
+        """
+        map_path = Path(map_path)
+        text = map_path.read_text(encoding="utf-8")
+        mapping = json.loads(text)
+        for rel_img_path, lbl in mapping.items():
+            img_path = (map_path.parent / rel_img_path).resolve()
+            try:
+                self.add_image(img_path, lbl)
+            except Exception as e:
+                logger.warning(
+                    "[ImageHashStore] Failed to add image '%s': %s", img_path, str(e)
+                )
+                continue
+
+    def labels(self) -> List[str]:
+        """Return a sorted list of all labels in the store."""
+        return sorted(self._hash.keys())
+
+    def hashes(self, label: str) -> Set[int]:
+        """Return the set of hashes for a given `label` (empty set if none)."""
+        return set(self._hash.get(label, ()))
+
+    def remove_label(self, label: str) -> None:
+        """Remove all hashes associated with `label`."""
+        if label in self._hash:
+            del self._hash[label]
+            logger.debug("[ImageHashStore] Removed label '%s'", label)
+            self._maybe_save()
+
+    def remove_hash(self, label: str, this: Union[int, str, Path]) -> bool:
+        """
+        Remove a specific hash under `label`.
+        `this` can be:
+         - an integer hash
+         - a Path (image file) -> will compute its hash then remove
+        Returns True if something was removed.
+        """
+        if label not in self._hash:
+            return False
+
+        h = None
+        if isinstance(this, (str, Path)):
+            try:
+                img = Image.open(this).convert("L")
+                h = self._hf(img)
+            except Exception as e:
+                logger.warning(
+                    "[ImageHashStore] Could not open image '%s': %s", this, str(e)
+                )
+                return False
+        else:
+            h = int(this)
+
+        if h in self._hash[label]:
+            self._hash[label].remove(h)
+            logger.debug("[ImageHashStore] Removed hash %d from label '%s'", h, label)
+            self._maybe_save()
+            return True
+        return False
+
+    def query(
+        self,
+        target: Union[int, str, Path, Image.Image],
+        k: int = 1,
+        threshold: Optional[int] = None,
+    ) -> List[Tuple[str, float]]:
+        """
+        Find up to `k` distinct labels whose stored hashes are most similar
+        to `target` within `threshold`. Returns a list of (label, score),
+        sorted by descending score. Each label appears at most once.
+
+        :param target: Image path / int hash / PIL.Image
+        :param k: number of labels to return (default=1)
+        :param threshold: Hamming distance cutoff (default=self._th)
+        """
+        if threshold is None:
+            threshold = self._th
+
+        # compute target hash
+        if isinstance(target, Image.Image):
+            img = target.convert("L")
+            thash = self._hf(img)
+        elif isinstance(target, (str, Path)):
+            img = Image.open(target).convert("L")
+            thash = self._hf(img)
+        else:
+            thash = int(target)
+
+        if self._bk_root is None:
+            return []
+
+        # find all (hash,dist) within threshold
+        matches = self._bk_root.query(thash, threshold, self._hd)
+
+        # collapse to one best dist per label
+        best_per_label: Dict[str, float] = {}
+        h2l = self._hash_to_labels
+        for h, dist in matches:
+            for lbl in h2l.get(h, ()):
+                score = 1.0 - dist / threshold
+                prev = best_per_label.get(lbl)
+                if prev is None or score > prev:
+                    best_per_label[lbl] = score
+
+        top_k = heapq.nsmallest(k, best_per_label.items(), key=lambda x: x[1])
+        return top_k
+
+
+img_hash_store = ImageHashStore()

novel_downloader/utils/hash_utils.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.hash_utils
+---------------------------------
+
+Utilities for image perceptual hashing and comparison.
+
+Implements a perceptual hash (pHash) based on DCT, following the method
+described in:
+https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html
+
+Provides:
+- pHash computation via DCT and median thresholding
+- Integer hash representation
+- Fast Hamming distance between hashes
+"""
+
+import numpy as np
+from PIL import Image
+from scipy.fft import dct as dct_1d
+
+ANTIALIAS = Image.Resampling.LANCZOS
+HASH_SIZE = 10  # default is 8
+HASH_DISTANCE_THRESHOLD = 5
+
+
+def hash_to_int(hash_array: np.ndarray) -> int:
+    """
+    Convert a boolean hash array to an integer.
+
+    :param hash_array: A binary array (dtype=bool) from a hash function.
+    :type hash_array: np.ndarray
+    :return: Integer representation of the binary hash.
+    :rtype: int
+    """
+    result = 0
+    for bit in hash_array:
+        result = (result << 1) | int(bit)
+    return result
+
+
+def fast_hamming_distance(hash_1: int, hash_2: int) -> int:
+    """
+    Compute the Hamming distance between two integer-based image hashes.
+
+    Uses bitwise XOR and bit count for fast comparison.
+
+    :param hash_1: First image hash (as integer).
+    :type hash_1: int
+    :param hash_2: Second image hash (as integer).
+    :type hash_2: int
+    :return: Number of differing bits between the two hashes.
+    :rtype: int
+    """
+    x = hash_1 ^ hash_2
+    count = 0
+    while x:
+        x &= x - 1
+        count += 1
+    return count
+
+
+def _threshold_and_pack(dct_low: np.ndarray) -> int:
+    """
+    Convert a low-frequency DCT matrix into a binary hash.
+
+    Compares each element to the median, builds a boolean mask,
+    then packs it into an integer.
+    """
+    med = np.median(dct_low)
+    diff = dct_low > med
+    return hash_to_int(diff.flatten())
+
+
+def phash(
+    image: Image.Image, hash_size: int = HASH_SIZE, highfreq_factor: int = 4
+) -> int:
+    """
+    Compute the perceptual hash (pHash) of an image.
+
+    This method applies a Discrete Cosine Transform (DCT) to extract
+    low-frequency features, then compares them to the median to create
+    a binary fingerprint of the image.
+
+    :param image: The input image.
+    :type image: PIL.Image.Image
+    :param hash_size: Size of the resulting hash (NxN).
+    :type hash_size: int
+    :param highfreq_factor: Multiplier for the image resize to preserve detail.
+    :type highfreq_factor: int
+    :return: Integer representation of the perceptual hash.
+    :rtype: int
+    """
+    if hash_size < 2:
+        raise ValueError("Hash size must be greater than or equal to 2")
+
+    img_size = hash_size * highfreq_factor
+    image = image.convert("L").resize((img_size, img_size), ANTIALIAS)
+    pixels = np.asarray(image)
+    dct = dct_1d(dct_1d(pixels, axis=0, norm="ortho"), axis=1, norm="ortho")
+    dctlowfreq = dct[:hash_size, :hash_size]
+    return _threshold_and_pack(dctlowfreq)

novel_downloader/utils/i18n.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.i18n
+---------------------------
+
+Multilingual text dictionary and utility for CLI and interactive mode.
+"""
+
+import json
+from typing import Any, Dict
+
+from novel_downloader.utils.constants import LOCALES_DIR
+from novel_downloader.utils.state import state_mgr
+
+_TRANSLATIONS: Dict[str, Dict[str, str]] = {}
+
+for locale_path in LOCALES_DIR.glob("*.json"):
+    lang = locale_path.stem
+    try:
+        with open(locale_path, "r", encoding="utf-8") as f:
+            _TRANSLATIONS[lang] = json.load(f)
+    except Exception:
+        continue
+
+
+def t(key: str, **kwargs: Any) -> str:
+    """
+    Retrieve a localized string by key and language.
+
+    :param key: The string key.
+    :param kwargs: Optional formatting arguments.
+    :return: The translated string, or the key if not found.
+    """
+    lang = state_mgr.get_language() or "zh"
+    txt = (
+        _TRANSLATIONS.get(lang, {}).get(key)
+        or _TRANSLATIONS.get("en", {}).get(key)
+        or key
+    )
+    return txt.format(**kwargs)

novel_downloader/utils/logger.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.logger
+-----------------------------
+
+Provides a configurable logging setup for Python applications.
+Log files are rotated daily and named with the given logger name and current date.
+"""
+
+import logging
+from datetime import datetime
+from logging.handlers import TimedRotatingFileHandler
+from pathlib import Path
+from typing import Dict, Literal, Optional, Union
+
+from .constants import LOGGER_DIR, LOGGER_NAME
+
+LogLevel = Literal["DEBUG", "INFO", "WARNING", "ERROR"]
+
+LOG_LEVELS: Dict[LogLevel, int] = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+}
+
+
+def setup_logging(
+    log_filename_prefix: Optional[str] = None,
+    log_level: Optional[LogLevel] = None,
+    log_dir: Optional[Union[str, Path]] = None,
+) -> logging.Logger:
+    """
+    Create and configure a logger for both console and rotating file output.
+
+    :param log_filename_prefix: Prefix for the log file name.
+                                If None, will use the last part of `logger_name`.
+    :param log_level: Minimum log level to show in console:
+                      "DEBUG", "INFO", "WARNING", or "ERROR".
+                      Defaults to "INFO" if not specified.
+    :param log_dir: Directory where log files will be saved.
+                    Defaults to "./logs" if not specified.
+    :return: A fully configured logger instance.
+    """
+    ft_logger = logging.getLogger("fontTools.ttLib.tables._p_o_s_t")
+    ft_logger.setLevel(logging.ERROR)
+    ft_logger.propagate = False
+
+    # Determine console level (default INFO)
+    level_str: LogLevel = log_level or "INFO"
+    console_level = LOG_LEVELS.get(level_str)
+    if console_level is None:
+        raise ValueError(
+            f"Invalid log level: {level_str}. Must be one of {list(LOG_LEVELS.keys())}"
+        )
+
+    # Resolve log file path
+    log_path = Path(log_dir) if log_dir else LOGGER_DIR
+    log_path.mkdir(parents=True, exist_ok=True)
+
+    # Resolve log file name
+    if not log_filename_prefix:
+        log_filename_prefix = LOGGER_NAME
+    date_str = datetime.now().strftime("%Y-%m-%d")
+    log_filename = log_path / f"{log_filename_prefix}_{date_str}.log"
+
+    # Create or retrieve logger
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)  # Capture everything, filter by handlers
+
+    # Clear existing handlers to avoid duplicate logs
+    if logger.hasHandlers():
+        logger.handlers.clear()
+
+    # File handler: rotates at midnight, keeps 7 days of logs
+    file_handler = TimedRotatingFileHandler(
+        filename=str(log_filename),
+        when="midnight",
+        interval=1,
+        backupCount=7,
+        encoding="utf-8",
+        utc=False,
+    )
+    file_formatter = logging.Formatter(
+        fmt="%(asctime)s [%(levelname)s] %(name)s.%(funcName)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    file_handler.setFormatter(file_formatter)
+    file_handler.setLevel(logging.DEBUG)
+    logger.addHandler(file_handler)
+
+    # Console handler
+    console_handler = logging.StreamHandler()
+    console_formatter = logging.Formatter(
+        fmt="%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S"
+    )
+    console_handler.setFormatter(console_formatter)
+    console_handler.setLevel(console_level)
+    logger.addHandler(console_handler)
+
+    print(f"Logging to {log_path}")
+
+    return logger

novel_downloader/utils/model_loader.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.model_loader
+-----------------------------------
+
+Utility functions for managing pre-trained model downloads.
+
+Currently supports:
+- Character recognition model for single Chinese character inference
+"""
+
+from pathlib import Path
+
+from huggingface_hub import hf_hub_download
+from huggingface_hub.utils import LocalEntryNotFoundError
+
+from novel_downloader.utils.constants import (
+    MODEL_CACHE_DIR,
+    REC_CHAR_MODEL_FILES,
+    REC_CHAR_MODEL_REPO,
+    REC_CHAR_VECTOR_FILES,
+)
+
+
+def get_rec_chinese_char_model_dir(version: str = "v1.0") -> Path:
+    """
+    Ensure model files are downloaded, return the directory path.
+    """
+    model_dir = MODEL_CACHE_DIR / "rec_chinese_char"
+
+    model_dir.mkdir(parents=True, exist_ok=True)
+
+    for fname in REC_CHAR_MODEL_FILES:
+        try:
+            hf_hub_download(
+                repo_id=REC_CHAR_MODEL_REPO,
+                filename=fname,
+                revision=version,
+                local_dir=model_dir,
+                local_dir_use_symlinks=False,
+            )
+        except LocalEntryNotFoundError:
+            raise RuntimeError(
+                f"[model] Missing model file '{fname}' and no internet connection."
+            )
+    return model_dir
+
+
+def get_rec_char_vector_dir(version: str = "v1.0") -> Path:
+    """
+    Ensure vector files are downloaded into a 'vector' subfolder under model directory.
+    Return the directory path.
+    """
+    vector_dir = MODEL_CACHE_DIR / "rec_chinese_char"
+    vector_dir.mkdir(parents=True, exist_ok=True)
+
+    for fname in REC_CHAR_VECTOR_FILES:
+        try:
+            hf_hub_download(
+                repo_id=REC_CHAR_MODEL_REPO,
+                filename=fname,
+                revision=version,
+                local_dir=vector_dir,
+                local_dir_use_symlinks=False,
+            )
+        except LocalEntryNotFoundError:
+            raise RuntimeError(
+                f"[vector] Missing vector file '{fname}' and no internet connection."
+            )
+
+    return vector_dir