ranobelib-python-sdk 0.12.0__tar.gz → 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/PKG-INFO +1 -1
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/pyproject.toml +1 -1
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/pyproject.toml.orig +1 -1
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/__init__.py +9 -0
- ranobelib_python_sdk-0.13.0/src/ranobelib/cache.py +179 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/_illustrations.py +26 -13
- ranobelib_python_sdk-0.13.0/src/ranobelib/images.py +73 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/models.py +46 -0
- ranobelib_python_sdk-0.13.0/src/ranobelib/paragraphs.py +85 -0
- ranobelib_python_sdk-0.12.0/src/ranobelib/cache.py +0 -64
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/README.md +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/_http.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/catalog.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/client.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/console.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exceptions.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/__init__.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/_shared.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/epub.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/fb2.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/pdf.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/txt.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/numbering.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/py.typed +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/sdk.py +0 -0
- {ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/sizing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ranobelib-python-sdk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Summary: Async Python SDK for ranobelib.me: title metadata, chapter/volume downloads, and export to epub/fb2/txt/pdf
|
|
5
5
|
Keywords: ranobelib,ranobe,novel,webnovel,epub,fb2,sdk
|
|
6
6
|
Author: LLIJIYAPNIK
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Async Python SDK for ranobelib.me."""
|
|
2
2
|
|
|
3
|
+
from ranobelib.cache import CacheStats, cache_stats, clear_cache
|
|
3
4
|
from ranobelib.catalog import Catalog
|
|
4
5
|
from ranobelib.exceptions import (
|
|
5
6
|
AccessBlockedError,
|
|
@@ -14,6 +15,7 @@ from ranobelib.exceptions import (
|
|
|
14
15
|
TitleNotFoundError,
|
|
15
16
|
VolumeNotFoundError,
|
|
16
17
|
)
|
|
18
|
+
from ranobelib.images import chapter_image_urls, rewrite_image_urls
|
|
17
19
|
from ranobelib.models import (
|
|
18
20
|
CatalogPage,
|
|
19
21
|
Chapter,
|
|
@@ -25,6 +27,7 @@ from ranobelib.models import (
|
|
|
25
27
|
Title,
|
|
26
28
|
Volume,
|
|
27
29
|
)
|
|
30
|
+
from ranobelib.paragraphs import chapter_paragraphs
|
|
28
31
|
from ranobelib.sdk import RanobeLib
|
|
29
32
|
from ranobelib.sizing import chapter_size, volume_size
|
|
30
33
|
|
|
@@ -32,6 +35,7 @@ __all__ = [
|
|
|
32
35
|
"AccessBlockedError",
|
|
33
36
|
"AmbiguousChapter",
|
|
34
37
|
"AuthRequiredError",
|
|
38
|
+
"CacheStats",
|
|
35
39
|
"Catalog",
|
|
36
40
|
"CatalogPage",
|
|
37
41
|
"Chapter",
|
|
@@ -51,6 +55,11 @@ __all__ = [
|
|
|
51
55
|
"TitleNotFoundError",
|
|
52
56
|
"Volume",
|
|
53
57
|
"VolumeNotFoundError",
|
|
58
|
+
"cache_stats",
|
|
59
|
+
"chapter_image_urls",
|
|
60
|
+
"chapter_paragraphs",
|
|
54
61
|
"chapter_size",
|
|
62
|
+
"clear_cache",
|
|
63
|
+
"rewrite_image_urls",
|
|
55
64
|
"volume_size",
|
|
56
65
|
]
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Disk cache for raw JSON API responses, keyed by request identity.
|
|
2
|
+
|
|
3
|
+
Also the public cache-maintenance helpers, ``cache_stats()`` and ``clear_cache()`` — the
|
|
4
|
+
on-disk layout is an SDK internal, so code outside the SDK should go through these rather
|
|
5
|
+
than measure or delete the cache's files itself.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import time
|
|
14
|
+
from collections.abc import Callable, Iterator
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
DEFAULT_CACHE_DIR = Path(".ranobelib_cache")
|
|
20
|
+
"""Default cache directory, relative to the current working directory."""
|
|
21
|
+
|
|
22
|
+
_ENTRY_NAME = re.compile(r"[0-9a-f]{64}\.json")
|
|
23
|
+
"""Name of a file ``DiskCache`` writes (``_path_for``) — anything else in the directory
|
|
24
|
+
isn't the cache's and is never counted or removed."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class CacheStats:
|
|
29
|
+
"""How many cache entries there are (or were removed), and their total size on disk."""
|
|
30
|
+
|
|
31
|
+
entries: int
|
|
32
|
+
bytes: int
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class DiskCache:
|
|
36
|
+
"""Caches JSON-serializable values to disk, keyed by an arbitrary string.
|
|
37
|
+
|
|
38
|
+
One file per key, named by the key's hash (so keys built from slugs, chapter numbers,
|
|
39
|
+
etc. are always safe filenames), holding the cached value plus the time it was written
|
|
40
|
+
so ``ttl`` can be checked on read without touching any other entry. Corrupted or
|
|
41
|
+
unreadable entries are treated as a cache miss rather than raised as an error, since a
|
|
42
|
+
stale/broken cache file shouldn't stop the SDK from just re-fetching the data.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
cache_dir: str | Path,
|
|
48
|
+
*,
|
|
49
|
+
ttl: float | None = None,
|
|
50
|
+
clock: Callable[[], float] = time.time,
|
|
51
|
+
) -> None:
|
|
52
|
+
"""Initialize the cache.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
cache_dir: Directory to store cache files in. Created lazily on first write.
|
|
56
|
+
ttl: Seconds after which a cache entry is treated as a miss. ``None`` (the
|
|
57
|
+
default) means entries never expire.
|
|
58
|
+
clock: Time source used to stamp and check entries. Overridable for tests.
|
|
59
|
+
"""
|
|
60
|
+
self._dir = Path(cache_dir)
|
|
61
|
+
self._ttl = ttl
|
|
62
|
+
self._clock = clock
|
|
63
|
+
|
|
64
|
+
def get(self, key: str) -> Any | None:
|
|
65
|
+
"""Return the cached value for ``key``, or ``None`` on a miss or expiry."""
|
|
66
|
+
try:
|
|
67
|
+
payload = json.loads(self._path_for(key).read_text(encoding="utf-8"))
|
|
68
|
+
except (OSError, json.JSONDecodeError, KeyError):
|
|
69
|
+
return None
|
|
70
|
+
if self._ttl is not None and self._clock() - payload["cached_at"] > self._ttl:
|
|
71
|
+
return None
|
|
72
|
+
return payload["data"]
|
|
73
|
+
|
|
74
|
+
def set(self, key: str, data: Any) -> None:
|
|
75
|
+
"""Write ``data`` to disk under ``key``, creating the cache directory if needed."""
|
|
76
|
+
path = self._path_for(key)
|
|
77
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
78
|
+
payload = {"cached_at": self._clock(), "data": data}
|
|
79
|
+
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
80
|
+
|
|
81
|
+
def stats(self) -> CacheStats:
|
|
82
|
+
"""Count the cache's entries and their total size. See ``cache_stats()``."""
|
|
83
|
+
entries = size = 0
|
|
84
|
+
for path in self._entry_paths():
|
|
85
|
+
try:
|
|
86
|
+
size += path.stat().st_size
|
|
87
|
+
except OSError: # Removed since it was listed.
|
|
88
|
+
continue
|
|
89
|
+
entries += 1
|
|
90
|
+
return CacheStats(entries=entries, bytes=size)
|
|
91
|
+
|
|
92
|
+
def clear(self, *, older_than: float | None = None) -> CacheStats:
|
|
93
|
+
"""Remove the cache's entries, or only those older than ``older_than`` seconds.
|
|
94
|
+
|
|
95
|
+
See ``clear_cache()``.
|
|
96
|
+
"""
|
|
97
|
+
if older_than is not None and older_than < 0:
|
|
98
|
+
raise ValueError(f"older_than must be >= 0, got {older_than}")
|
|
99
|
+
now = self._clock()
|
|
100
|
+
entries = size = 0
|
|
101
|
+
for path in self._entry_paths():
|
|
102
|
+
if older_than is not None and not self._is_older_than(path, now - older_than):
|
|
103
|
+
continue
|
|
104
|
+
try:
|
|
105
|
+
entry_size = path.stat().st_size
|
|
106
|
+
path.unlink()
|
|
107
|
+
except OSError:
|
|
108
|
+
# Already removed by someone else, or (on Windows) open in another process
|
|
109
|
+
# right now — either way, not ours to count as removed.
|
|
110
|
+
continue
|
|
111
|
+
entries += 1
|
|
112
|
+
size += entry_size
|
|
113
|
+
return CacheStats(entries=entries, bytes=size)
|
|
114
|
+
|
|
115
|
+
def _is_older_than(self, path: Path, cutoff: float) -> bool:
|
|
116
|
+
try:
|
|
117
|
+
cached_at = float(json.loads(path.read_text(encoding="utf-8"))["cached_at"])
|
|
118
|
+
except FileNotFoundError:
|
|
119
|
+
return False
|
|
120
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
121
|
+
# Unreadable/corrupted: get() can never serve it, so it's as good as expired.
|
|
122
|
+
return True
|
|
123
|
+
return cached_at < cutoff
|
|
124
|
+
|
|
125
|
+
def _entry_paths(self) -> Iterator[Path]:
|
|
126
|
+
try:
|
|
127
|
+
children = list(self._dir.iterdir())
|
|
128
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
129
|
+
return
|
|
130
|
+
for path in children:
|
|
131
|
+
if _ENTRY_NAME.fullmatch(path.name) and path.is_file() and not path.is_symlink():
|
|
132
|
+
yield path
|
|
133
|
+
|
|
134
|
+
def _path_for(self, key: str) -> Path:
|
|
135
|
+
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
|
|
136
|
+
return self._dir / f"{digest}.json"
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def cache_stats(cache_dir: str | Path = DEFAULT_CACHE_DIR) -> CacheStats:
|
|
140
|
+
"""Count the entries in an SDK disk cache and their total size on disk.
|
|
141
|
+
|
|
142
|
+
Only files the cache itself writes are counted, not anything else that happens to be in
|
|
143
|
+
the directory.
|
|
144
|
+
|
|
145
|
+
Args:
|
|
146
|
+
cache_dir: The cache directory — the same ``cache_dir`` passed to ``RanobeLib`` /
|
|
147
|
+
``Catalog`` (default: theirs, ``.ranobelib_cache`` in the current directory).
|
|
148
|
+
|
|
149
|
+
Returns:
|
|
150
|
+
The number of entries and their total size in bytes. A missing directory is an empty
|
|
151
|
+
cache, ``CacheStats(entries=0, bytes=0)``, not an error.
|
|
152
|
+
"""
|
|
153
|
+
return DiskCache(cache_dir).stats()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def clear_cache(
|
|
157
|
+
cache_dir: str | Path = DEFAULT_CACHE_DIR, *, older_than: float | None = None
|
|
158
|
+
) -> CacheStats:
|
|
159
|
+
"""Remove entries from an SDK disk cache.
|
|
160
|
+
|
|
161
|
+
Only files the cache itself writes are removed — anything else in the directory, and the
|
|
162
|
+
directory itself, is left alone. Safe to call while clients are using the same cache: an
|
|
163
|
+
entry being read as it's removed is just a cache miss, and one written after it's
|
|
164
|
+
removed is simply recreated.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
cache_dir: The cache directory — see ``cache_stats()``.
|
|
168
|
+
older_than: If given, only remove entries written more than this many seconds ago
|
|
169
|
+
(the same age check ``cache_ttl`` uses), plus any corrupted entry, which the
|
|
170
|
+
cache can't serve anyway. ``None`` (the default) removes every entry.
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
The number of entries removed and the bytes they took. A missing directory removes
|
|
174
|
+
nothing, ``CacheStats(entries=0, bytes=0)``, rather than raising.
|
|
175
|
+
|
|
176
|
+
Raises:
|
|
177
|
+
ValueError: If ``older_than`` is negative.
|
|
178
|
+
"""
|
|
179
|
+
return DiskCache(cache_dir).clear(older_than=older_than)
|
|
@@ -11,6 +11,7 @@ from __future__ import annotations
|
|
|
11
11
|
|
|
12
12
|
import asyncio
|
|
13
13
|
import html
|
|
14
|
+
from collections.abc import Callable
|
|
14
15
|
from html.parser import HTMLParser
|
|
15
16
|
from pathlib import Path
|
|
16
17
|
from urllib.parse import urlsplit
|
|
@@ -62,7 +63,7 @@ def extract_image_urls(html_fragment: str) -> list[str]:
|
|
|
62
63
|
|
|
63
64
|
|
|
64
65
|
class _ImageSrcRewriter(HTMLParser):
|
|
65
|
-
"""Re-emits chapter-content HTML with ``<img src="...">``
|
|
66
|
+
"""Re-emits chapter-content HTML with each ``<img src="...">`` passed through a function.
|
|
66
67
|
|
|
67
68
|
Rebuilds the markup tag-by-tag (rather than a targeted regex substitution) for the same
|
|
68
69
|
reason the txt/fb2 exporters use ``HTMLParser`` over regex: HTML-string-format chapters
|
|
@@ -71,15 +72,13 @@ class _ImageSrcRewriter(HTMLParser):
|
|
|
71
72
|
for epub, WeasyPrint's own HTML parser for pdf) are lenient enough not to need strict
|
|
72
73
|
XHTML here.
|
|
73
74
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
a dangling absolute-URL reference isn't a self-contained embed, just a broken-looking
|
|
77
|
-
image in most consumers, so "not embedded" reads as "not present" instead.
|
|
75
|
+
``replace`` gets each ``<img>``'s ``src`` (``""`` if it has none) and returns the new
|
|
76
|
+
value, or ``None`` to drop that ``<img>`` entirely.
|
|
78
77
|
"""
|
|
79
78
|
|
|
80
|
-
def __init__(self,
|
|
79
|
+
def __init__(self, replace: Callable[[str], str | None]) -> None:
|
|
81
80
|
super().__init__(convert_charrefs=True)
|
|
82
|
-
self.
|
|
81
|
+
self._replace = replace
|
|
83
82
|
self._output: list[str] = []
|
|
84
83
|
|
|
85
84
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
@@ -90,10 +89,10 @@ class _ImageSrcRewriter(HTMLParser):
|
|
|
90
89
|
|
|
91
90
|
def _emit(self, tag: str, attrs: list[tuple[str, str | None]], *, self_closing: bool) -> None:
|
|
92
91
|
if tag == "img":
|
|
93
|
-
|
|
94
|
-
if
|
|
92
|
+
new_src = self._replace(dict(attrs).get("src") or "")
|
|
93
|
+
if new_src is None:
|
|
95
94
|
return
|
|
96
|
-
attrs = [(name,
|
|
95
|
+
attrs = [(name, new_src if name == "src" else value) for name, value in attrs]
|
|
97
96
|
|
|
98
97
|
parts = [f"<{tag}"]
|
|
99
98
|
for name, value in attrs:
|
|
@@ -115,13 +114,27 @@ class _ImageSrcRewriter(HTMLParser):
|
|
|
115
114
|
return "".join(self._output)
|
|
116
115
|
|
|
117
116
|
|
|
118
|
-
def
|
|
119
|
-
"""
|
|
120
|
-
|
|
117
|
+
def replace_image_srcs(html_fragment: str, replace: Callable[[str], str | None]) -> str:
|
|
118
|
+
"""Pass every ``<img src="...">`` in ``html_fragment`` through ``replace``.
|
|
119
|
+
|
|
120
|
+
``replace`` returns the new ``src``, or ``None`` to drop that ``<img>``.
|
|
121
|
+
"""
|
|
122
|
+
rewriter = _ImageSrcRewriter(replace)
|
|
121
123
|
rewriter.feed(html_fragment)
|
|
122
124
|
return rewriter.get_html()
|
|
123
125
|
|
|
124
126
|
|
|
127
|
+
def rewrite_image_srcs(html_fragment: str, url_to_local: dict[str, str]) -> str:
|
|
128
|
+
"""Rewrite ``<img src="...">`` URLs in ``html_fragment`` per ``url_to_local``.
|
|
129
|
+
|
|
130
|
+
An ``<img>`` whose URL isn't in ``url_to_local`` (download failed or was never
|
|
131
|
+
attempted) is dropped entirely rather than kept pointing at the original external URL —
|
|
132
|
+
a dangling absolute-URL reference isn't a self-contained embed, just a broken-looking
|
|
133
|
+
image in most consumers, so "not embedded" reads as "not present" instead.
|
|
134
|
+
"""
|
|
135
|
+
return replace_image_srcs(html_fragment, url_to_local.get)
|
|
136
|
+
|
|
137
|
+
|
|
125
138
|
def pick_cover_url(cover: Cover) -> str | None:
|
|
126
139
|
"""The best available cover URL, preferring the largest: default, then md, then thumbnail."""
|
|
127
140
|
return cover.default or cover.md or cover.thumbnail
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Listing and rewriting the images a fetched chapter references.
|
|
2
|
+
|
|
3
|
+
Chapter illustrations are ``<img>`` tags with absolute URLs on ranobelib.me /
|
|
4
|
+
``*.cdnlibs.org`` (see docs/api-notes.md), inside ``Chapter.content`` and, occasionally,
|
|
5
|
+
``Chapter.footnotes``. These helpers expose the same ``<img>`` parsing the epub/pdf
|
|
6
|
+
exporters and ``sizing.chapter_size()`` use internally, so a caller that serves chapters
|
|
7
|
+
itself (e.g. through its own same-origin image proxy, for offline caching) doesn't need a
|
|
8
|
+
second HTML parser of its own.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
|
|
15
|
+
from ranobelib.exporters._illustrations import extract_image_urls, replace_image_srcs
|
|
16
|
+
from ranobelib.models import Chapter, Footnote
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _require_content(chapter: Chapter) -> str:
|
|
20
|
+
if chapter.content is None:
|
|
21
|
+
raise ValueError(
|
|
22
|
+
f"Chapter {chapter.volume}/{chapter.number} has no content — "
|
|
23
|
+
"fetch it first, e.g. via get_chapter()."
|
|
24
|
+
)
|
|
25
|
+
return chapter.content
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def chapter_image_urls(chapter: Chapter) -> list[str]:
|
|
29
|
+
"""List the image URLs a fetched chapter references.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
chapter: A chapter with content already fetched (e.g. via ``get_chapter()``).
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
Every ``<img src>`` in ``chapter.content`` and then ``chapter.footnotes``, in
|
|
36
|
+
document order, each URL listed once (its first occurrence).
|
|
37
|
+
|
|
38
|
+
Raises:
|
|
39
|
+
ValueError: If ``chapter.content`` is ``None`` (not yet fetched — see ``Chapter``'s
|
|
40
|
+
docstring for when that happens).
|
|
41
|
+
"""
|
|
42
|
+
fragments = [_require_content(chapter), *(footnote.content for footnote in chapter.footnotes)]
|
|
43
|
+
urls = (url for fragment in fragments for url in extract_image_urls(fragment))
|
|
44
|
+
return list(dict.fromkeys(urls))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def rewrite_image_urls(chapter: Chapter, replace: Callable[[str], str]) -> Chapter:
|
|
48
|
+
"""Return a copy of a fetched chapter with every ``<img src>`` passed through ``replace``.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
chapter: A chapter with content already fetched (e.g. via ``get_chapter()``).
|
|
52
|
+
replace: Called with each image URL (once per ``<img>``, so possibly more than once
|
|
53
|
+
for the same URL); its result becomes the new ``src``, HTML-escaped as an
|
|
54
|
+
attribute value. An ``<img>`` without a ``src`` is left as it is.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
A new ``Chapter`` whose ``content`` and ``footnotes`` have their ``<img src>``
|
|
58
|
+
rewritten; every other field is unchanged, and ``chapter`` itself is not modified.
|
|
59
|
+
|
|
60
|
+
Raises:
|
|
61
|
+
ValueError: If ``chapter.content`` is ``None`` — see ``chapter_image_urls()``.
|
|
62
|
+
"""
|
|
63
|
+
content = _require_content(chapter)
|
|
64
|
+
|
|
65
|
+
def rewrite(fragment: str) -> str:
|
|
66
|
+
return replace_image_srcs(fragment, lambda src: replace(src) if src else src)
|
|
67
|
+
|
|
68
|
+
return chapter.model_copy(
|
|
69
|
+
update={
|
|
70
|
+
"content": rewrite(content),
|
|
71
|
+
"footnotes": [Footnote(content=rewrite(f.content)) for f in chapter.footnotes],
|
|
72
|
+
}
|
|
73
|
+
)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import html
|
|
6
|
+
import re
|
|
6
7
|
from datetime import datetime
|
|
7
8
|
from html.parser import HTMLParser
|
|
8
9
|
from typing import Any
|
|
@@ -438,6 +439,28 @@ class Team(BaseModel):
|
|
|
438
439
|
name: str
|
|
439
440
|
|
|
440
441
|
|
|
442
|
+
_FORMAT_WORD = (
|
|
443
|
+
r"(?:(?:light|lite|web|лайт|веб|вэб)\s*-?\s*)?(?:novel(?:la|l|a)?|новелл?а)"
|
|
444
|
+
r"|web\s*-?\s*serial|ranobe|ранобэ|ln|wn|лн|quest|квест"
|
|
445
|
+
)
|
|
446
|
+
_FORMAT_SUFFIX = re.compile(
|
|
447
|
+
rf"[\s,:/]*(?:\(\s*(?:{_FORMAT_WORD})\s*\)|\[\s*(?:{_FORMAT_WORD})\s*\])\s*$",
|
|
448
|
+
re.IGNORECASE,
|
|
449
|
+
)
|
|
450
|
+
"""A publication-format marker at the end of a title name — ``(Новелла)``, ``(Novel)``,
|
|
451
|
+
``(LN)``, ``[квест]``, ... — plus the separator (``,``/``:``/``/``) sometimes left before it.
|
|
452
|
+
The word list comes from every name in the live catalog (see docs/api-notes.md, "Title
|
|
453
|
+
names without the format suffix"); other bracketed tails (a year, ``(18+)``, ``(Фанфик)``,
|
|
454
|
+
``[LitRPG]``, a franchise) tell titles apart, so they're not on it."""
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _strip_format_suffix(name: str) -> str:
|
|
458
|
+
"""``name`` without its trailing format marker(s) — they can stack: ``(LN) (Novel)``."""
|
|
459
|
+
while (match := _FORMAT_SUFFIX.search(name)) is not None:
|
|
460
|
+
name = name[: match.start()]
|
|
461
|
+
return name.strip()
|
|
462
|
+
|
|
463
|
+
|
|
441
464
|
class Title(BaseModel):
|
|
442
465
|
"""Metadata for a single title (novel)."""
|
|
443
466
|
|
|
@@ -463,12 +486,35 @@ class Title(BaseModel):
|
|
|
463
486
|
artists: list[Person] = Field(default_factory=list)
|
|
464
487
|
teams: list[Team] = Field(default_factory=list)
|
|
465
488
|
chapter_count: int | None = None
|
|
489
|
+
"""Number of uploaded chapters. Only ``RanobeLib.get_info()`` fills it in: the catalog
|
|
490
|
+
listing endpoint doesn't return it (see docs/api-notes.md), so it's ``None`` on
|
|
491
|
+
``Catalog.list_titles()`` items."""
|
|
466
492
|
last_chapter_at: datetime | None = Field(default=None, alias="last_item_at")
|
|
467
493
|
"""When the most recent chapter was published — the value the catalog's
|
|
468
494
|
``sort="last_chapter_at"`` orders by. Only ``RanobeLib.get_info()`` fills it in: the
|
|
469
495
|
catalog listing endpoint doesn't return it (see docs/api-notes.md), so it's ``None`` on
|
|
470
496
|
``Catalog.list_titles()`` items, and on a title with no chapters yet."""
|
|
471
497
|
|
|
498
|
+
@property
|
|
499
|
+
def base_names(self) -> list[str]:
|
|
500
|
+
"""The title's names without the format suffix ranobelib.me appends to them.
|
|
501
|
+
|
|
502
|
+
``rus_name``, ``eng_name``, ``name`` and ``other_names``, in that order, each with
|
|
503
|
+
trailing publication-format markers removed — ``"Старшая школа D×D (Новелла)"`` →
|
|
504
|
+
``"Старшая школа D×D"``, likewise ``(Novel)``, ``(LN)``, ``(WN)``, ``(Light Novel)``,
|
|
505
|
+
``(Ранобэ)``, ``[квест]``, ... (stacked ones too, ``(LN) (Novel)``). Other bracketed
|
|
506
|
+
tails — a year, ``(18+)``, ``(Фанфик)``, ``[LitRPG]``, a franchise name — distinguish
|
|
507
|
+
one title from another and are kept. Whitespace is trimmed; empty names and
|
|
508
|
+
duplicates (keeping the first) are dropped. Nothing else is normalized (case,
|
|
509
|
+
punctuation, ``×``): that's up to whatever compares the names.
|
|
510
|
+
|
|
511
|
+
The original fields are left as they are. On ``Catalog.list_titles()`` items
|
|
512
|
+
``other_names`` is empty (the listing doesn't return it), so it contributes nothing.
|
|
513
|
+
"""
|
|
514
|
+
names = [self.rus_name, self.eng_name, self.name, *self.other_names]
|
|
515
|
+
stripped = (_strip_format_suffix(name) for name in names if name)
|
|
516
|
+
return list(dict.fromkeys(name for name in stripped if name))
|
|
517
|
+
|
|
472
518
|
@model_validator(mode="before")
|
|
473
519
|
@classmethod
|
|
474
520
|
def _flatten_chapter_count(cls, data: Any) -> Any:
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""A fetched chapter's top-level blocks as plain text, indexed like the DOM.
|
|
2
|
+
|
|
3
|
+
``Chapter.content`` is a flat, sanitized sequence of block-level elements (``<p>``,
|
|
4
|
+
``<hr />``, ``<img>`` — see ``Chapter``), so a client rendering it as HTML sees one child
|
|
5
|
+
element per block. ``chapter_paragraphs()`` gives each of those blocks' text at the same
|
|
6
|
+
index, so e.g. a comment keyed by a paragraph's position in the reader's DOM
|
|
7
|
+
(``content.children[i]``) can be shown next to that paragraph's text without a second HTML
|
|
8
|
+
parser on the caller's side.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from html.parser import HTMLParser
|
|
14
|
+
|
|
15
|
+
from ranobelib.models import Chapter
|
|
16
|
+
|
|
17
|
+
_VOID_TAGS = frozenset({"br", "hr", "img"})
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _TopLevelTextCollector(HTMLParser):
|
|
21
|
+
"""Collects the text of each top-level element of an HTML fragment, in order.
|
|
22
|
+
|
|
23
|
+
Every start tag at depth 0 begins a new entry, void ones (``br``/``hr``/``img``)
|
|
24
|
+
included, since each is an element child in the DOM too. Text outside any element at the
|
|
25
|
+
top level isn't an entry (it's a text node, not an element child) and is ignored. A
|
|
26
|
+
``<br />`` inside an entry separates words, like a space.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self) -> None:
|
|
30
|
+
super().__init__(convert_charrefs=True)
|
|
31
|
+
self._depth = 0
|
|
32
|
+
self.entries: list[list[str]] = []
|
|
33
|
+
|
|
34
|
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
35
|
+
if self._depth == 0:
|
|
36
|
+
self.entries.append([])
|
|
37
|
+
if tag in _VOID_TAGS:
|
|
38
|
+
if self._depth > 0:
|
|
39
|
+
self.entries[-1].append(" ")
|
|
40
|
+
else:
|
|
41
|
+
self._depth += 1
|
|
42
|
+
|
|
43
|
+
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
44
|
+
self.handle_starttag(tag, attrs)
|
|
45
|
+
if tag not in _VOID_TAGS:
|
|
46
|
+
self.handle_endtag(tag)
|
|
47
|
+
|
|
48
|
+
def handle_endtag(self, tag: str) -> None:
|
|
49
|
+
if tag not in _VOID_TAGS and self._depth > 0:
|
|
50
|
+
self._depth -= 1
|
|
51
|
+
|
|
52
|
+
def handle_data(self, data: str) -> None:
|
|
53
|
+
if self._depth > 0:
|
|
54
|
+
self.entries[-1].append(data)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def chapter_paragraphs(chapter: Chapter) -> list[str]:
|
|
58
|
+
"""A fetched chapter's top-level blocks as plain text, in document order.
|
|
59
|
+
|
|
60
|
+
Entry ``i`` is the text of the ``i``-th top-level element of ``chapter.content`` — the
|
|
61
|
+
same position a browser gives it among the rendered content's element children — so a
|
|
62
|
+
paragraph index counted in the DOM can be used to look it up directly.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
chapter: A chapter with content already fetched (e.g. via ``get_chapter()``).
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
One string per top-level element: its text with entities decoded and whitespace
|
|
69
|
+
collapsed to single spaces (``""`` for a block without text, such as ``<hr />`` or
|
|
70
|
+
an image-only paragraph). ``chapter.footnotes`` aren't included — they're not part
|
|
71
|
+
of ``content`` (see ``Chapter.footnotes``).
|
|
72
|
+
|
|
73
|
+
Raises:
|
|
74
|
+
ValueError: If ``chapter.content`` is ``None`` (not yet fetched — see ``Chapter``'s
|
|
75
|
+
docstring for when that happens).
|
|
76
|
+
"""
|
|
77
|
+
if chapter.content is None:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
f"Chapter {chapter.volume}/{chapter.number} has no content — "
|
|
80
|
+
"fetch it first, e.g. via get_chapter()."
|
|
81
|
+
)
|
|
82
|
+
collector = _TopLevelTextCollector()
|
|
83
|
+
collector.feed(chapter.content)
|
|
84
|
+
collector.close()
|
|
85
|
+
return [" ".join("".join(parts).split()) for parts in collector.entries]
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
"""Disk cache for raw JSON API responses, keyed by request identity."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import hashlib
|
|
6
|
-
import json
|
|
7
|
-
import time
|
|
8
|
-
from collections.abc import Callable
|
|
9
|
-
from pathlib import Path
|
|
10
|
-
from typing import Any
|
|
11
|
-
|
|
12
|
-
DEFAULT_CACHE_DIR = Path(".ranobelib_cache")
|
|
13
|
-
"""Default cache directory, relative to the current working directory."""
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class DiskCache:
|
|
17
|
-
"""Caches JSON-serializable values to disk, keyed by an arbitrary string.
|
|
18
|
-
|
|
19
|
-
One file per key, named by the key's hash (so keys built from slugs, chapter numbers,
|
|
20
|
-
etc. are always safe filenames), holding the cached value plus the time it was written
|
|
21
|
-
so ``ttl`` can be checked on read without touching any other entry. Corrupted or
|
|
22
|
-
unreadable entries are treated as a cache miss rather than raised as an error, since a
|
|
23
|
-
stale/broken cache file shouldn't stop the SDK from just re-fetching the data.
|
|
24
|
-
"""
|
|
25
|
-
|
|
26
|
-
def __init__(
|
|
27
|
-
self,
|
|
28
|
-
cache_dir: str | Path,
|
|
29
|
-
*,
|
|
30
|
-
ttl: float | None = None,
|
|
31
|
-
clock: Callable[[], float] = time.time,
|
|
32
|
-
) -> None:
|
|
33
|
-
"""Initialize the cache.
|
|
34
|
-
|
|
35
|
-
Args:
|
|
36
|
-
cache_dir: Directory to store cache files in. Created lazily on first write.
|
|
37
|
-
ttl: Seconds after which a cache entry is treated as a miss. ``None`` (the
|
|
38
|
-
default) means entries never expire.
|
|
39
|
-
clock: Time source used to stamp and check entries. Overridable for tests.
|
|
40
|
-
"""
|
|
41
|
-
self._dir = Path(cache_dir)
|
|
42
|
-
self._ttl = ttl
|
|
43
|
-
self._clock = clock
|
|
44
|
-
|
|
45
|
-
def get(self, key: str) -> Any | None:
|
|
46
|
-
"""Return the cached value for ``key``, or ``None`` on a miss or expiry."""
|
|
47
|
-
try:
|
|
48
|
-
payload = json.loads(self._path_for(key).read_text(encoding="utf-8"))
|
|
49
|
-
except (OSError, json.JSONDecodeError, KeyError):
|
|
50
|
-
return None
|
|
51
|
-
if self._ttl is not None and self._clock() - payload["cached_at"] > self._ttl:
|
|
52
|
-
return None
|
|
53
|
-
return payload["data"]
|
|
54
|
-
|
|
55
|
-
def set(self, key: str, data: Any) -> None:
|
|
56
|
-
"""Write ``data`` to disk under ``key``, creating the cache directory if needed."""
|
|
57
|
-
path = self._path_for(key)
|
|
58
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
59
|
-
payload = {"cached_at": self._clock(), "data": data}
|
|
60
|
-
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
61
|
-
|
|
62
|
-
def _path_for(self, key: str) -> Path:
|
|
63
|
-
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
|
|
64
|
-
return self._dir / f"{digest}.json"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/__init__.py
RENAMED
|
File without changes
|
{ranobelib_python_sdk-0.12.0 → ranobelib_python_sdk-0.13.0}/src/ranobelib/exporters/_shared.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|