novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/utils/hash_store.py

@@ -1,279 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.utils.hash_store
----------------------------------
-
-Manage a small collection of image perceptual hashes and their labels.
-Supports loading/saving to .json or .npy, and basic CRUD + search.
-"""
-
-import heapq
-import json
-import logging
-from collections.abc import Callable
-from pathlib import Path
-
-from PIL import Image
-
-from .constants import HASH_STORE_FILE
-from .hash_utils import HASH_DISTANCE_THRESHOLD, fast_hamming_distance, phash
-
-logger = logging.getLogger(__name__)
-
-
-class _BKNode:
-    """
-    A node in a Burkhard-Keller tree (BK-Tree) for distance search.
-    Stores one value and a dict of children keyed by distance.
-    """
-
-    __slots__ = ("value", "children")
-
-    def __init__(self, value: int):
-        self.value = value
-        self.children: dict[int, _BKNode] = {}
-
-    def add(self, h: int, dist_fn: Callable[[int, int], int]) -> None:
-        d = dist_fn(h, self.value)
-        child = self.children.get(d)
-        if child is not None:
-            child.add(h, dist_fn)
-        else:
-            self.children[d] = _BKNode(h)
-
-    def query(
-        self,
-        target: int,
-        threshold: int,
-        dist_fn: Callable[[int, int], int],
-    ) -> list[tuple[int, int]]:
-        """
-        Recursively collect (value, dist) pairs within threshold.
-        """
-        d0 = dist_fn(target, self.value)
-        matches: list[tuple[int, int]] = []
-        if d0 <= threshold:
-            matches.append((self.value, d0))
-        # Only children whose edge-dist \in [d0-threshold, d0+threshold]
-        lower, upper = d0 - threshold, d0 + threshold
-        for edge, child in self.children.items():
-            if lower <= edge <= upper:
-                matches.extend(child.query(target, threshold, dist_fn))
-        return matches
-
-
-class ImageHashStore:
-    """
-    Store and manage image hashes grouped by label, with a BK-Tree index.
-
-    :param path: file path for persistence (".json" or ".npy")
-    :param auto_save: if True, every modification automatically calls save()
-    :param hash_func: function to compute hash from PIL.Image
-    :param ham_dist: function to compute Hamming distance between two hashes
-    """
-
-    def __init__(
-        self,
-        path: str | Path = HASH_STORE_FILE,
-        auto_save: bool = False,
-        hash_func: Callable[[Image.Image], int] = phash,
-        ham_dist: Callable[[int, int], int] = fast_hamming_distance,
-        threshold: int = HASH_DISTANCE_THRESHOLD,
-    ) -> None:
-        self._path = Path(path)
-        self._auto = auto_save
-        self._hf = hash_func
-        self._hd = ham_dist
-        self._th = threshold
-
-        # label -> set of hashes
-        self._hash: dict[str, set[int]] = {}
-        # hash -> list of labels (for reverse lookup)
-        self._hash_to_labels: dict[int, list[str]] = {}
-        # root of BK-Tree (or None if empty)
-        self._bk_root: _BKNode | None = None
-
-        self.load()
-
-    def load(self) -> None:
-        """Load store from disk and rebuild BK-Tree index."""
-        if not self._path.exists():
-            self._hash.clear()
-            logger.debug(
-                "[ImageHashStore] No file found at %s, starting empty.", self._path
-            )
-            return
-
-        txt = self._path.read_text(encoding="utf-8")
-        obj = json.loads(txt) or {}
-        self._hash = {lbl: set(obj.get(lbl, [])) for lbl in obj}
-
-        # rebuild reverse map and BK-Tree
-        self._hash_to_labels.clear()
-        for lbl, hs in self._hash.items():
-            for h in hs:
-                self._hash_to_labels.setdefault(h, []).append(lbl)
-        logger.debug(
-            "[ImageHashStore] Loaded hash store from %s with %d hashes",
-            self._path,
-            sum(len(v) for v in self._hash.values()),
-        )
-
-        self._build_index()
-
-    def _build_index(self) -> None:
-        """Construct a BK-Tree over all stored hashes."""
-        self._bk_root = None
-        for h in self._hash_to_labels:
-            if self._bk_root is None:
-                self._bk_root = _BKNode(h)
-            else:
-                self._bk_root.add(h, self._hd)
-        logger.debug(
-            "[ImageHashStore] BK-tree index built with %d unique hashes",
-            len(self._hash_to_labels),
-        )
-
-    def save(self) -> None:
-        """Persist current store to disk."""
-        self._path.parent.mkdir(parents=True, exist_ok=True)
-        data = {lbl: list(s) for lbl, s in self._hash.items()}
-        txt = json.dumps(data, ensure_ascii=False, indent=2)
-        self._path.write_text(txt, encoding="utf-8")
-        logger.debug("[ImageHashStore] Saved hash store to %s", self._path)
-
-    def _maybe_save(self) -> None:
-        if self._auto:
-            self.save()
-
-    def add_image(self, img_path: str | Path, label: str) -> int:
-        """
-        Compute hash for the given image and add it under `label`.
-        Updates BK-Tree index incrementally.
-        """
-        img = Image.open(img_path).convert("L")
-        h = self._hf(img)
-        self._hash.setdefault(label, set()).add(h)
-        self._hash_to_labels.setdefault(h, []).append(label)
-        # insert into BK-Tree
-        if self._bk_root is None:
-            self._bk_root = _BKNode(h)
-        else:
-            self._bk_root.add(h, self._hd)
-        logger.debug("[ImageHashStore] Added hash %d under label '%s'", h, label)
-        self._maybe_save()
-        return h
-
-    def add_from_map(self, map_path: str | Path) -> None:
-        """
-        Load a JSON file of the form { "image_path": "label", ... }
-        and add each entry.
-        """
-        map_path = Path(map_path)
-        text = map_path.read_text(encoding="utf-8")
-        mapping = json.loads(text)
-        for rel_img_path, lbl in mapping.items():
-            img_path = (map_path.parent / rel_img_path).resolve()
-            try:
-                self.add_image(img_path, lbl)
-            except Exception as e:
-                logger.warning(
-                    "[ImageHashStore] Failed to add image '%s': %s", img_path, str(e)
-                )
-                continue
-
-    def labels(self) -> list[str]:
-        """Return a sorted list of all labels in the store."""
-        return sorted(self._hash.keys())
-
-    def hashes(self, label: str) -> set[int]:
-        """Return the set of hashes for a given `label` (empty set if none)."""
-        return set(self._hash.get(label, ()))
-
-    def remove_label(self, label: str) -> None:
-        """Remove all hashes associated with `label`."""
-        if label in self._hash:
-            del self._hash[label]
-            logger.debug("[ImageHashStore] Removed label '%s'", label)
-            self._maybe_save()
-
-    def remove_hash(self, label: str, this: int | str | Path) -> bool:
-        """
-        Remove a specific hash under `label`.
-        `this` can be:
-         - an integer hash
-         - a Path (image file) -> will compute its hash then remove
-        Returns True if something was removed.
-        """
-        if label not in self._hash:
-            return False
-
-        h = None
-        if isinstance(this, (str | Path)):
-            try:
-                img = Image.open(this).convert("L")
-                h = self._hf(img)
-            except Exception as e:
-                logger.warning(
-                    "[ImageHashStore] Could not open image '%s': %s", this, str(e)
-                )
-                return False
-        else:
-            h = int(this)
-
-        if h in self._hash[label]:
-            self._hash[label].remove(h)
-            logger.debug("[ImageHashStore] Removed hash %d from label '%s'", h, label)
-            self._maybe_save()
-            return True
-        return False
-
-    def query(
-        self,
-        target: int | str | Path | Image.Image,
-        k: int = 1,
-        threshold: int | None = None,
-    ) -> list[tuple[str, float]]:
-        """
-        Find up to `k` distinct labels whose stored hashes are most similar
-        to `target` within `threshold`. Returns a list of (label, score),
-        sorted by descending score. Each label appears at most once.
-
-        :param target: Image path / int hash / PIL.Image
-        :param k: number of labels to return (default=1)
-        :param threshold: Hamming distance cutoff (default=self._th)
-        """
-        if threshold is None:
-            threshold = self._th
-
-        # compute target hash
-        if isinstance(target, Image.Image):
-            img = target.convert("L")
-            thash = self._hf(img)
-        elif isinstance(target, (str | Path)):
-            img = Image.open(target).convert("L")
-            thash = self._hf(img)
-        else:
-            thash = int(target)
-
-        if self._bk_root is None:
-            return []
-
-        # find all (hash,dist) within threshold
-        matches = self._bk_root.query(thash, threshold, self._hd)
-
-        # collapse to one best dist per label
-        best_per_label: dict[str, float] = {}
-        h2l = self._hash_to_labels
-        for h, dist in matches:
-            for lbl in h2l.get(h, ()):
-                score = 1.0 - dist / threshold
-                prev = best_per_label.get(lbl)
-                if prev is None or score > prev:
-                    best_per_label[lbl] = score
-
-        top_k = heapq.nsmallest(k, best_per_label.items(), key=lambda x: x[1])
-        return top_k
-
-
-img_hash_store = ImageHashStore()

novel_downloader/utils/hash_utils.py

@@ -1,103 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.utils.hash_utils
----------------------------------
-
-Utilities for image perceptual hashing and comparison.
-
-Implements a perceptual hash (pHash) based on DCT, following the method
-described in:
-https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html
-
-Provides:
-- pHash computation via DCT and median thresholding
-- Integer hash representation
-- Fast Hamming distance between hashes
-"""
-
-import numpy as np
-from numpy.typing import NDArray
-from PIL import Image
-from scipy.fft import dct as dct_1d
-
-ANTIALIAS = Image.Resampling.LANCZOS
-HASH_SIZE = 10  # default is 8
-HASH_DISTANCE_THRESHOLD = 5
-
-
-def hash_to_int(hash_array: NDArray[np.bool_]) -> int:
-    """
-    Convert a boolean hash array to an integer.
-
-    :param hash_array: A binary array (dtype=bool) from a hash function.
-    :type hash_array: np.ndarray
-    :return: Integer representation of the binary hash.
-    :rtype: int
-    """
-    result = 0
-    for bit in hash_array:
-        result = (result << 1) | int(bit)
-    return result
-
-
-def fast_hamming_distance(hash_1: int, hash_2: int) -> int:
-    """
-    Compute the Hamming distance between two integer-based image hashes.
-
-    Uses bitwise XOR and bit count for fast comparison.
-
-    :param hash_1: First image hash (as integer).
-    :type hash_1: int
-    :param hash_2: Second image hash (as integer).
-    :type hash_2: int
-    :return: Number of differing bits between the two hashes.
-    :rtype: int
-    """
-    x = hash_1 ^ hash_2
-    count = 0
-    while x:
-        x &= x - 1
-        count += 1
-    return count
-
-
-def _threshold_and_pack(dct_low: NDArray[np.float64]) -> int:
-    """
-    Convert a low-frequency DCT matrix into a binary hash.
-
-    Compares each element to the median, builds a boolean mask,
-    then packs it into an integer.
-    """
-    med = np.median(dct_low)
-    diff = dct_low > med
-    return hash_to_int(diff.flatten())
-
-
-def phash(
-    image: Image.Image, hash_size: int = HASH_SIZE, highfreq_factor: int = 4
-) -> int:
-    """
-    Compute the perceptual hash (pHash) of an image.
-
-    This method applies a Discrete Cosine Transform (DCT) to extract
-    low-frequency features, then compares them to the median to create
-    a binary fingerprint of the image.
-
-    :param image: The input image.
-    :type image: PIL.Image.Image
-    :param hash_size: Size of the resulting hash (NxN).
-    :type hash_size: int
-    :param highfreq_factor: Multiplier for the image resize to preserve detail.
-    :type highfreq_factor: int
-    :return: Integer representation of the perceptual hash.
-    :rtype: int
-    """
-    if hash_size < 2:
-        raise ValueError("Hash size must be greater than or equal to 2")
-
-    img_size = hash_size * highfreq_factor
-    image = image.convert("L").resize((img_size, img_size), ANTIALIAS)
-    pixels = np.asarray(image)
-    dct = dct_1d(dct_1d(pixels, axis=0, norm="ortho"), axis=1, norm="ortho")
-    dctlowfreq = dct[:hash_size, :hash_size]
-    return _threshold_and_pack(dctlowfreq)

novel_downloader/utils/text_utils/chapter_formatting.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.utils.text_utils.chapter_formatting
-----------------------------------------------------
-
-Format chapter content with title, paragraph blocks, and optional author notes.
-"""
-
-import re
-
-_IMG_TAG_RE = re.compile(r"<img[^>]*>")
-
-
-def format_chapter(title: str, paragraphs: str, author_say: str | None = None) -> str:
-    """
-    Build a formatted chapter string with title, paragraphs, and optional author note.
-
-    :param title:       The chapter title.
-    :param paragraphs:  Raw multi-line string; lines are treated as paragraphs.
-    :param author_say:  Optional author comment to append at the end.
-    :return:            A single string where title, paragraphs, and author note
-                        are separated by blank lines.
-    """
-    parts: list[str] = [title.strip()]
-
-    # add each nonempty paragraph line
-    paragraphs = _IMG_TAG_RE.sub("", paragraphs)
-    for ln in paragraphs.splitlines():
-        line = ln.strip()
-        if line:
-            parts.append(line)
-
-    # add author_say lines if present
-    if author_say:
-        author_lines = [ln.strip() for ln in author_say.splitlines() if ln.strip()]
-        if author_lines:
-            parts.append("---")
-            parts.append("作者说:")
-            parts.extend(author_lines)
-
-    return "\n\n".join(parts)
-
-
-__all__ = [
-    "format_chapter",
-]

novel_downloader/utils/text_utils/font_mapping.py

@@ -1,28 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.utils.text_utils.font_mapping
-----------------------------------------------
-
-Utility for decoding obfuscated text by applying character-level font mapping.
-
-This is commonly used to reverse font-based obfuscation in scraped content,
-where characters are visually disguised via custom font glyphs but can be
-recovered using a known mapping.
-"""
-
-
-def apply_font_mapping(text: str, font_map: dict[str, str]) -> str:
-    """
-    Replace each character in `text` using `font_map`,
-    leaving unmapped characters unchanged.
-
-    :param text:    The input string, possibly containing obfuscated font chars.
-    :param font_map: A dict mapping obfuscated chars to real chars.
-    :return:        The de-obfuscated text.
-    """
-    return "".join(font_map.get(ch, ch) for ch in text)
-
-
-__all__ = [
-    "apply_font_mapping",
-]

novel_downloader/utils/text_utils/text_cleaning.py

@@ -1,107 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.utils.text_utils.text_cleaning
------------------------------------------------
-
-Tools for detecting and removing promotional or ad-like content from text.
-"""
-
-import math
-import re
-
-from novel_downloader.utils.file_utils.io import load_blacklisted_words
-
-# --- Constants & Precompiled Patterns ---
-
-_BLACKLISTED_WORDS = load_blacklisted_words()
-
-_BRACKET_PATTERN = re.compile(r"[\(（](.*?)[\)）]")
-_K_PROMO_PATTERN = re.compile(r"\b\d{1,4}k\b", re.IGNORECASE)
-
-
-def clean_chapter_title(title: str) -> str:
-    """
-    Remove bracketed promotional content from a chapter title.
-
-    If any blacklisted word appears inside parentheses (Chinese or English),
-    the entire bracketed section is stripped.
-
-    :param title: Original title, possibly containing ad text in brackets.
-    :return:      Title with offending bracketed sections removed.
-    """
-    cleaned = title
-    for content in _BRACKET_PATTERN.findall(title):
-        if any(bw in content for bw in _BLACKLISTED_WORDS):
-            cleaned = re.sub(rf"[\(（]{re.escape(content)}[\)）]", "", cleaned)
-    return cleaned.strip()
-
-
-def is_promotional_line(line: str) -> bool:
-    """
-    Check if a line of text likely contains promotional or ad-like content.
-
-    :param line: A single line of text.
-    :return:     True if it contains promo keywords or a '###k' vote count pattern.
-    """
-    low = line.lower()
-    if any(kw in low for kw in _BLACKLISTED_WORDS):
-        return True
-    if _K_PROMO_PATTERN.search(low):
-        return True
-    return False
-
-
-def content_prefix(
-    text: str,
-    n: int,
-    ignore_chars: set[str] | None = None,
-) -> str:
-    """
-    Return the prefix of `text` containing the first `n` non-ignored characters.
-
-    :param text: The full input string.
-    :param n: Number of content characters to include.
-    :param ignore_chars: Characters to ignore when counting content.
-    :return: Truncated string preserving original whitespace and line breaks.
-    """
-    ignore = ignore_chars or set()
-    cnt = 0
-
-    for i, ch in enumerate(text):
-        if ch not in ignore:
-            cnt += 1
-            if cnt >= n:
-                return text[: i + 1]
-
-    return text
-
-
-def truncate_half_lines(text: str) -> str:
-    """
-    Keep the first half of the lines (rounded up), preserving line breaks.
-
-    :param text: Full input text
-    :return: Truncated text with first half of lines
-    """
-    lines = text.splitlines()
-    non_empty_lines = [line for line in lines if line.strip()]
-    keep_count = math.ceil(len(non_empty_lines) / 2)
-
-    result_lines = []
-    count = 0
-    for line in lines:
-        result_lines.append(line)
-        if line.strip():
-            count += 1
-        if count >= keep_count:
-            break
-
-    return "\n".join(result_lines)
-
-
-__all__ = [
-    "clean_chapter_title",
-    "is_promotional_line",
-    "content_prefix",
-    "truncate_half_lines",
-]