novel-downloader 1.2.2__py3-none-any.whl → 1.3.0__py3-none-any.whl

novel_downloader/utils/chapter_storage.py

@@ -0,0 +1,340 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.utils.chapter_storage
+--------------------------------------
+
+Storage module for managing novel chapters in
+either JSON file form or an SQLite database.
+"""
+
+import contextlib
+import json
+import sqlite3
+import types
+from pathlib import Path
+from typing import Any, Literal, Self, TypedDict, cast
+
+from .file_utils import save_as_json
+
+_CREATE_TABLE_SQL = """
+CREATE TABLE IF NOT EXISTS "{table}" (
+    id TEXT PRIMARY KEY,
+    title TEXT NOT NULL,
+    content TEXT NOT NULL,
+    extra TEXT NOT NULL
+)
+"""
+
+
+class ChapterDict(TypedDict, total=True):
+    """
+    TypedDict for a novel chapter.
+
+    Fields:
+        id      -- Unique chapter identifier
+        title   -- Chapter title
+        content -- Chapter text
+        extra   -- Arbitrary metadata (e.g. author remarks, timestamps)
+    """
+
+    id: str
+    title: str
+    content: str
+    extra: dict[str, Any]
+
+
+BackendType = Literal["json", "sqlite"]
+SaveMode = Literal["overwrite", "skip"]
+
+
+class ChapterStorage:
+    """
+    Manage storage of chapters in JSON files or an SQLite database.
+
+    :param raw_base: Base directory or file path for storage.
+    :param namespace: Novel identifier (subfolder name or DB/table basename).
+    :param backend_type: "json" (default) or "sqlite".
+    """
+
+    def __init__(
+        self,
+        raw_base: str | Path,
+        namespace: str,
+        backend_type: BackendType = "json",
+        *,
+        batch_size: int = 1,
+    ) -> None:
+        self.raw_base = Path(raw_base)
+        self.namespace = namespace
+        self.backend = backend_type
+        self._batch_size = batch_size
+        self._pending = 0
+        self._conn: sqlite3.Connection | None = None
+
+        if self.backend == "json":
+            self._init_json()
+        else:
+            self._init_sql()
+
+    def _init_json(self) -> None:
+        """Prepare directory for JSON files."""
+        self._json_dir = self.raw_base / self.namespace
+        self._json_dir.mkdir(parents=True, exist_ok=True)
+
+    def _init_sql(self) -> None:
+        """Prepare SQLite connection and ensure table exists."""
+        self._db_path = self.raw_base / f"{self.namespace}.sqlite"
+        self._conn = sqlite3.connect(self._db_path)
+        stmt = _CREATE_TABLE_SQL.format(table=self.namespace)
+        self._conn.execute(stmt)
+        self._conn.commit()
+
+    def _json_path(self, chap_id: str) -> Path:
+        """Return Path for JSON file of given chapter ID."""
+        return self._json_dir / f"{chap_id}.json"
+
+    def _exists_json(self, chap_id: str) -> bool:
+        return self._json_path(chap_id).is_file()
+
+    def _exists_sql(self, chap_id: str) -> bool:
+        if self._conn is None:
+            raise RuntimeError("ChapterStorage is closed")
+        cur = self._conn.execute(
+            f'SELECT 1 FROM "{self.namespace}" WHERE id = ? LIMIT 1', (chap_id,)
+        )
+        return cur.fetchone() is not None
+
+    def exists(self, chap_id: str) -> bool:
+        """
+        Check if a chapter exists.
+
+        :param chap_id: Chapter identifier.
+        :return: True if found, else False.
+        """
+        if self.backend == "json":
+            return self._exists_json(chap_id)
+        else:
+            return self._exists_sql(chap_id)
+
+    def _load_json(self, chap_id: str) -> ChapterDict:
+        raw = self._json_path(chap_id).read_text(encoding="utf-8")
+        return cast(ChapterDict, json.loads(raw))
+
+    def _load_sql(self, chap_id: str) -> ChapterDict:
+        if self._conn is None:
+            raise RuntimeError("ChapterStorage is closed")
+        cur = self._conn.execute(
+            f'SELECT id, title, content, extra FROM "{self.namespace}" WHERE id = ?',
+            (chap_id,),
+        )
+        row = cur.fetchone()
+        return {
+            "id": row[0],
+            "title": row[1],
+            "content": row[2],
+            "extra": json.loads(row[3]),
+        }
+
+    def get(self, chap_id: str) -> ChapterDict | dict[str, Any]:
+        """
+        Retrieve chapter by ID.
+
+        :param chap_id: Chapter identifier.
+        :return: ChapterDict if exists, else empty dict.
+        """
+        if not self.exists(chap_id):
+            return {}
+        return (
+            self._load_json(chap_id)
+            if self.backend == "json"
+            else self._load_sql(chap_id)
+        )
+
+    def _save_json(self, data: ChapterDict, on_exist: SaveMode) -> None:
+        path = self._json_path(data["id"])
+        save_as_json(data, path, on_exist=on_exist)
+
+    def _save_sql(self, data: ChapterDict, on_exist: SaveMode) -> None:
+        if self._conn is None:
+            raise RuntimeError("ChapterStorage is closed")
+        sql = (
+            f'INSERT OR REPLACE INTO "{self.namespace}" '
+            "(id, title, content, extra) VALUES (?, ?, ?, ?)"
+            if on_exist == "overwrite"
+            else f'INSERT OR IGNORE INTO "{self.namespace}" '
+            "(id, title, content, extra) VALUES (?, ?, ?, ?)"
+        )
+        self._conn.execute(
+            sql,
+            (
+                data["id"],
+                data["title"],
+                data["content"],
+                json.dumps(data["extra"], ensure_ascii=False),
+            ),
+        )
+        if self._batch_size == 1:
+            self._conn.commit()
+        else:
+            self._pending += 1
+            if self._pending >= self._batch_size:
+                self._conn.commit()
+                self._pending = 0
+
+    def _save_many_sql(
+        self,
+        datas: list[ChapterDict],
+        on_exist: SaveMode = "overwrite",
+    ) -> None:
+        """
+        Bulk-insert into SQLite using executemany + one commit.
+
+        :param datas: List of ChapterDict to store.
+        :param on_exist: "overwrite" to REPLACE, "skip" to IGNORE on conflicts.
+        """
+        if on_exist not in ("overwrite", "skip"):
+            raise ValueError(f"invalid on_exist mode: {on_exist!r}")
+        if self._conn is None:
+            raise RuntimeError("ChapterStorage is closed")
+
+        sql = (
+            f'INSERT OR REPLACE INTO "{self.namespace}" '
+            "(id, title, content, extra) VALUES (?, ?, ?, ?)"
+            if on_exist == "overwrite"
+            else f'INSERT OR IGNORE INTO "{self.namespace}" '
+            "(id, title, content, extra) VALUES (?, ?, ?, ?)"
+        )
+
+        params = [
+            (
+                data["id"],
+                data["title"],
+                data["content"],
+                json.dumps(data["extra"], ensure_ascii=False),
+            )
+            for data in datas
+        ]
+
+        with self._conn:
+            self._conn.executemany(sql, params)
+
+    def save(
+        self,
+        data: ChapterDict,
+        on_exist: SaveMode = "overwrite",
+    ) -> None:
+        """
+        Save a chapter record.
+
+        :param data: ChapterDict to store.
+        :param on_exist: What to do if chap_id already exists
+        """
+        if on_exist not in ("overwrite", "skip"):
+            raise ValueError(f"invalid on_exist mode: {on_exist!r}")
+
+        if self.backend == "json":
+            self._save_json(data, on_exist)
+        else:
+            self._save_sql(data, on_exist)
+
+    def save_many(
+        self,
+        datas: list[ChapterDict],
+        on_exist: SaveMode = "overwrite",
+    ) -> None:
+        """
+        Save multiple chapter records in one shot.
+
+        :param datas: List of ChapterDict to store.
+        :param on_exist: What to do if chap_id already exists.
+        """
+        if on_exist not in ("overwrite", "skip"):
+            raise ValueError(f"invalid on_exist mode: {on_exist!r}")
+
+        if self.backend == "json":
+            for data in datas:
+                self._save_json(data, on_exist)
+        else:
+            self._save_many_sql(datas, on_exist)
+
+    def list_ids(self) -> list[str]:
+        """
+        List all stored chapter IDs.
+        """
+        if self.backend == "json":
+            return [p.stem for p in self._json_dir.glob("*.json") if p.is_file()]
+
+        if self._conn is None:
+            raise RuntimeError("ChapterStorage is closed")
+        cur = self._conn.execute(f'SELECT id FROM "{self.namespace}"')
+        return [row[0] for row in cur.fetchall()]
+
+    def delete(self, chap_id: str) -> bool:
+        """
+        Delete a chapter by ID.
+
+        :param chap_id: Chapter identifier.
+        :return: True if deleted, False if not found.
+        """
+        if not self.exists(chap_id):
+            return False
+        if self.backend == "json":
+            self._json_path(chap_id).unlink()
+            return True
+
+        if self._conn is None:
+            raise RuntimeError("ChapterStorage is closed")
+        cur = self._conn.execute(
+            f'DELETE FROM "{self.namespace}" WHERE id = ?', (chap_id,)
+        )
+        self._conn.commit()
+        return cur.rowcount > 0
+
+    def count(self) -> int:
+        """
+        Count total chapters stored.
+        """
+        if self.backend == "json":
+            return len(self.list_ids())
+
+        if self._conn is None:
+            raise RuntimeError("ChapterStorage is closed")
+        cur = self._conn.execute(f'SELECT COUNT(1) FROM "{self.namespace}"')
+        return int(cur.fetchone()[0])
+
+    def flush(self) -> None:
+        """
+        Write out any leftover rows (< batch_size) at the end.
+        """
+        if self._conn is not None and self._pending > 0:
+            self._conn.commit()
+            self._pending = 0
+
+    def close(self) -> None:
+        """
+        Gracefully close any open resources.
+        """
+        if self.backend != "sqlite" or self._conn is None:
+            return
+
+        with contextlib.suppress(Exception):
+            self.flush()
+
+        with contextlib.suppress(Exception):
+            self._conn.close()
+
+        self._conn = None
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        tb: types.TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def __del__(self) -> None:
+        self.close()

novel_downloader/utils/constants.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.utils.constants
 --------------------------------
@@ -20,6 +19,10 @@ APP_NAME = "NovelDownloader"  # Display name
 APP_DIR_NAME = "novel_downloader"  # Directory name for platformdirs
 LOGGER_NAME = PACKAGE_NAME  # Root logger name
 
+SUPPORTED_SITES = {
+    "qidian",
+    "biquge",
+}
 
 # -----------------------------------------------------------------------------
 # Base directories
@@ -33,7 +36,6 @@ LOCALES_DIR: Path = PACKAGE_ROOT / "locales"
 # Subdirectories under BASE_CONFIG_DIR
 LOGGER_DIR = WORK_DIR / "logs"
 JS_SCRIPT_DIR = BASE_CONFIG_DIR / "scripts"
-STATE_DIR = BASE_CONFIG_DIR / "state"
 DATA_DIR = BASE_CONFIG_DIR / "data"
 CONFIG_DIR = BASE_CONFIG_DIR / "config"
 MODEL_CACHE_DIR = BASE_CONFIG_DIR / "models"
@@ -41,7 +43,7 @@ MODEL_CACHE_DIR = BASE_CONFIG_DIR / "models"
 # -----------------------------------------------------------------------------
 # Default file paths
 # -----------------------------------------------------------------------------
-STATE_FILE = STATE_DIR / "state.json"
+STATE_FILE = DATA_DIR / "state.json"
 HASH_STORE_FILE = DATA_DIR / "image_hashes.json"
 SETTING_FILE = CONFIG_DIR / "settings.json"
 SITE_RULES_FILE = CONFIG_DIR / "site_rules.json"
@@ -76,7 +78,7 @@ DEFAULT_USER_HEADERS = {
 # -----------------------------------------------------------------------------
 # Embedded resources (via importlib.resources)
 # -----------------------------------------------------------------------------
-BASE_CONFIG_PATH = files("novel_downloader.resources.config").joinpath("settings.yaml")
+BASE_CONFIG_PATH = files("novel_downloader.resources.config").joinpath("settings.toml")
 BASE_RULE_PATH = files("novel_downloader.resources.config").joinpath("rules.toml")
 
 DEFAULT_SETTINGS_PATHS = [

novel_downloader/utils/crypto_utils.py

@@ -12,7 +12,7 @@ import hashlib
 import json
 import random
 import time
-from typing import Any, Dict, List
+from typing import Any
 
 
 def rc4_crypt(
@@ -50,7 +50,7 @@ def rc4_crypt(
 
         # Pseudo-Random Generation Algorithm (PRGA)
         i = j = 0
-        out: List[int] = []
+        out: list[int] = []
         for char in data_bytes:
             i = (i + 1) % 256
             j = (j + S[i]) % 256
@@ -110,7 +110,7 @@ def patch_qd_payload_token(
 
     # Step 1 - decrypt --------------------------------------------------
     decrypted_json: str = rc4_crypt(key, enc_token, mode="decrypt")
-    payload: Dict[str, Any] = json.loads(decrypted_json)
+    payload: dict[str, Any] = json.loads(decrypted_json)
 
     # Step 2 - rebuild timing fields -----------------------------------
     loadts = int(time.time() * 1000)  # ms since epoch

novel_downloader/utils/file_utils/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.utils.file_utils
 ---------------------------------

novel_downloader/utils/file_utils/io.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.utils.file_utils.io
 ------------------------------------
@@ -17,7 +16,7 @@ import logging
 import tempfile
 from importlib.resources import files
 from pathlib import Path
-from typing import Any, Dict, List, Literal, Optional, Set, Union
+from typing import Any, Literal
 
 from .sanitize import sanitize_filename
 
@@ -41,9 +40,9 @@ def _get_non_conflicting_path(path: Path) -> Path:
 
 
 def _write_file(
-    content: Union[str, bytes, Dict[Any, Any], List[Any], Any],
-    filepath: Union[str, Path],
-    mode: Optional[str] = None,
+    content: str | bytes | dict[Any, Any] | list[Any] | Any,
+    filepath: str | Path,
+    mode: str | None = None,
     *,
     on_exist: Literal["overwrite", "skip", "rename"] = "overwrite",
     dump_json: bool = False,
@@ -78,7 +77,7 @@ def _write_file(
             logger.debug("[file] '%s' exists, will overwrite", path)
 
     # Prepare content and write mode
-    content_to_write: Union[str, bytes]
+    content_to_write: str | bytes
     if dump_json:
         # Serialize original object to JSON string
         json_str = json.dumps(content, ensure_ascii=False, indent=2)
@@ -87,7 +86,7 @@ def _write_file(
         content_to_write = json_str
         write_mode = "w"
     else:
-        if isinstance(content, (str, bytes)):
+        if isinstance(content, (str | bytes)):
             content_to_write = content
         else:
             raise TypeError("Non-JSON content must be str or bytes.")
@@ -113,7 +112,7 @@ def _write_file(
 
 def save_as_txt(
     content: str,
-    filepath: Union[str, Path],
+    filepath: str | Path,
     *,
     encoding: str = "utf-8",
     on_exist: Literal["overwrite", "skip", "rename"] = "overwrite",
@@ -139,7 +138,7 @@ def save_as_txt(
 
 def save_as_json(
     content: Any,
-    filepath: Union[str, Path],
+    filepath: str | Path,
     *,
     encoding: str = "utf-8",
     on_exist: Literal["overwrite", "skip", "rename"] = "overwrite",
@@ -163,9 +162,7 @@ def save_as_json(
     )
 
 
-def read_text_file(
-    filepath: Union[str, Path], encoding: str = "utf-8"
-) -> Optional[str]:
+def read_text_file(filepath: str | Path, encoding: str = "utf-8") -> str | None:
     """
     Read a UTF-8 text file.
 
@@ -181,9 +178,7 @@ def read_text_file(
         return None
 
 
-def read_json_file(
-    filepath: Union[str, Path], encoding: str = "utf-8"
-) -> Optional[Any]:
+def read_json_file(filepath: str | Path, encoding: str = "utf-8") -> Any | None:
     """
     Read a JSON file and parse it into Python objects.
 
@@ -199,7 +194,7 @@ def read_json_file(
         return None
 
 
-def read_binary_file(filepath: Union[str, Path]) -> Optional[bytes]:
+def read_binary_file(filepath: str | Path) -> bytes | None:
     """
     Read a binary file and return its content as bytes.
 
@@ -231,7 +226,7 @@ def load_text_resource(
     return resource_path.read_text(encoding="utf-8")
 
 
-def load_blacklisted_words() -> Set[str]:
+def load_blacklisted_words() -> set[str]:
     """
     Convenience loader for the blacklist.txt in the text resources.
 

novel_downloader/utils/file_utils/normalize.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.utils.file_utils.normalize
 -------------------------------------------
@@ -12,12 +11,11 @@ Currently includes line-ending normalization for .txt files.
 
 import logging
 from pathlib import Path
-from typing import Union
 
 logger = logging.getLogger(__name__)
 
 
-def normalize_txt_line_endings(folder_path: Union[str, Path]) -> None:
+def normalize_txt_line_endings(folder_path: str | Path) -> None:
     """
     Convert all .txt files in the given folder (recursively)
     to use Unix-style LF (\\n) line endings.

novel_downloader/utils/file_utils/sanitize.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.utils.file_utils.sanitize
 ------------------------------------------
@@ -15,7 +14,6 @@ lengths, and avoids reserved names on Windows systems.
 import logging
 import os
 import re
-from typing import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +31,7 @@ _SANITIZE_PATTERN_WIN = re.compile(r'[<>:"/\\|?*\x00-\x1F]')
 _SANITIZE_PATTERN_POSIX = re.compile(r"[/\x00]")
 
 
-def sanitize_filename(filename: str, max_length: Optional[int] = 255) -> str:
+def sanitize_filename(filename: str, max_length: int | None = 255) -> str:
     """
     Sanitize the given filename by replacing characters
     that are invalid in file paths with '_'.
@@ -47,12 +45,7 @@ def sanitize_filename(filename: str, max_length: Optional[int] = 255) -> str:
     :param max_length: Optional maximum length of the output filename. Defaults to 255.
     :return: The sanitized filename as a string.
     """
-    if os.name == "nt":
-        # Windows: invalid characters in filenames are: <>:"/\\|?*
-        pattern = _SANITIZE_PATTERN_WIN
-    else:
-        # POSIX systems: the forward slash is not allowed
-        pattern = _SANITIZE_PATTERN_POSIX
+    pattern = _SANITIZE_PATTERN_WIN if os.name == "nt" else _SANITIZE_PATTERN_POSIX
 
     name = pattern.sub("_", filename).strip(" .")
 

novel_downloader/utils/fontocr/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.utils.fontocr
 ------------------------------