zimlore 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zimlore/__init__.py +5 -0
- zimlore/__main__.py +8 -0
- zimlore/_libzim.py +37 -0
- zimlore/catalog.py +592 -0
- zimlore/cli.py +618 -0
- zimlore/errors.py +45 -0
- zimlore/groups.py +30 -0
- zimlore/library.py +648 -0
- zimlore/manage.py +632 -0
- zimlore/markdown.py +1247 -0
- zimlore/mcp.py +183 -0
- zimlore/purl.py +130 -0
- zimlore/py.typed +0 -0
- zimlore/refs.py +90 -0
- zimlore/service.py +1126 -0
- zimlore/snapshot.py +195 -0
- zimlore/sync.py +269 -0
- zimlore-0.1.0.dist-info/METADATA +206 -0
- zimlore-0.1.0.dist-info/RECORD +23 -0
- zimlore-0.1.0.dist-info/WHEEL +4 -0
- zimlore-0.1.0.dist-info/entry_points.txt +2 -0
- zimlore-0.1.0.dist-info/licenses/LICENSE +21 -0
- zimlore-0.1.0.dist-info/licenses/NOTICE +21 -0
zimlore/__init__.py
ADDED
zimlore/__main__.py
ADDED
zimlore/_libzim.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""The only place that imports the native binding.
|
|
2
|
+
|
|
3
|
+
`libzim` is a compiled extension (it carries libzim and Xapian). It ships wheels for CPython 3.11+ on
|
|
4
|
+
Windows x64, macOS (x86-64, arm64) and Linux (glibc and musl, x86-64 and aarch64), and nothing else, so on any
|
|
5
|
+
other platform `import libzim` fails. Everything that does not read a ZIM (the CLI's `info`, `catalog`, `get`,
|
|
6
|
+
the reference grammar, the converter) must keep working there, which is why the import is lazy and why its
|
|
7
|
+
failure is a `ZimloreError` naming the remedy instead of a traceback from deep inside a tool.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import importlib
|
|
13
|
+
from types import ModuleType
|
|
14
|
+
|
|
15
|
+
from .errors import ZimloreError
|
|
16
|
+
|
|
17
|
+
REMEDY = (
|
|
18
|
+
"pip install libzim (wheels: Windows x64, macOS, Linux glibc/musl; CPython 3.11+). On a platform without a "
|
|
19
|
+
"wheel, build libzim from source (https://github.com/openzim/python-libzim) or serve the same files with "
|
|
20
|
+
"kiwix-serve and read them in a browser"
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def load(name: str) -> ModuleType:
|
|
25
|
+
"""Import `libzim.<name>` (reader, search, suggestion, writer)."""
|
|
26
|
+
try:
|
|
27
|
+
return importlib.import_module(f"libzim.{name}")
|
|
28
|
+
except ImportError as error:
|
|
29
|
+
raise ZimloreError(f"the libzim binding is not available ({error})", REMEDY) from error
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def available() -> bool:
|
|
33
|
+
try:
|
|
34
|
+
load("reader")
|
|
35
|
+
except ZimloreError:
|
|
36
|
+
return False
|
|
37
|
+
return True
|
zimlore/catalog.py
ADDED
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
"""The public Kiwix catalogue (OPDS) and verified downloads. The only module that touches the network.
|
|
2
|
+
|
|
3
|
+
Everything else in zimlore is offline. This module exists because "put ZIM files in a directory" is a poor first
|
|
4
|
+
step for someone who does not yet know which file holds Flask's documentation.
|
|
5
|
+
|
|
6
|
+
Facts measured against the live service on 2026-11-07 and the reason for each rule below:
|
|
7
|
+
|
|
8
|
+
* `https://library.kiwix.org/catalog/v2/...` answers 301 and the feed lives at `opds.library.kiwix.org`.
|
|
9
|
+
* `entries?q=...&lang=eng&count=N` filters; `name=<exact>` returns every dated edition of one name.
|
|
10
|
+
* The acquisition link in the feed ends in `.zim.meta4`. The `.meta4` is a Metalink document carrying the
|
|
11
|
+
file's real byte size and its SHA-256, and the `.zim` URL redirects to an arbitrary mirror. So the hash is
|
|
12
|
+
read from the kiwix.org family of hosts (`.meta4` redirects stay inside it too: `download.kiwix.org` answers 301
|
|
13
|
+
to `lb.download.kiwix.org`) and the bytes may come from anywhere: a download is accepted only if it matches.
|
|
14
|
+
* The feed's own `length` is rounded (617,472 for a file of 617,313 bytes), so it is never used as a check.
|
|
15
|
+
|
|
16
|
+
Every hop of every fetch, redirects included, is checked again: https only (plain http only to a local host),
|
|
17
|
+
never https down to http, never to a loopback, private or link-local address unless the request began at a local
|
|
18
|
+
host. Host names are not resolved, so a public name that points at a private address is not caught here.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import contextlib
|
|
24
|
+
import hashlib
|
|
25
|
+
import http.client
|
|
26
|
+
import ipaddress
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
import shutil
|
|
30
|
+
import socket
|
|
31
|
+
import time
|
|
32
|
+
import unicodedata
|
|
33
|
+
import urllib.error
|
|
34
|
+
import urllib.parse
|
|
35
|
+
import urllib.request
|
|
36
|
+
import xml.etree.ElementTree as ET
|
|
37
|
+
from collections.abc import Callable
|
|
38
|
+
from dataclasses import dataclass
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from typing import Any
|
|
41
|
+
|
|
42
|
+
from . import __version__, library, manage
|
|
43
|
+
from .errors import ZimloreError
|
|
44
|
+
|
|
45
|
+
CATALOG_URL = "https://opds.library.kiwix.org/catalog/v2/entries"
|
|
46
|
+
USER_AGENT = f"zimlore/{__version__}"
|
|
47
|
+
ATOM = {"a": "http://www.w3.org/2005/Atom", "dc": "http://purl.org/dc/terms/"}
|
|
48
|
+
METALINK = "{urn:ietf:params:xml:ns:metalink}"
|
|
49
|
+
LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1"}
|
|
50
|
+
CATALOGUE_DOMAIN = "kiwix.org"
|
|
51
|
+
"""The `.meta4` carrying the trusted size and hash is fetched from this domain or a subdomain (redirects included).
|
|
52
|
+
Only the `.zim` bytes may come from another mirror."""
|
|
53
|
+
STALE_LOCK_SECONDS = 3600
|
|
54
|
+
"""A `.part.lock` is touched on every chunk, so one whose last touch is older than this belongs to a process that
|
|
55
|
+
died. It is reported with the remedy and never broken silently: deleting a live lock would let two downloads write
|
|
56
|
+
one `.part`."""
|
|
57
|
+
DOWNLOAD_DEADLINE_SECONDS = 6 * 3600
|
|
58
|
+
"""A download that has not finished this long after it started is stopped (the `.part` stays and resumes). The
|
|
59
|
+
socket timeout is per read, so without this a mirror sending one byte a minute would hold the lock for ever."""
|
|
60
|
+
CHUNK = 1 << 20
|
|
61
|
+
"""The read size of a download: also the granularity of the deadline and byte-cap checks."""
|
|
62
|
+
PAGE_SIZE = 100
|
|
63
|
+
MAX_PAGES = 50
|
|
64
|
+
|
|
65
|
+
Opener = Callable[[urllib.request.Request], Any]
|
|
66
|
+
|
|
67
|
+
RETRY = "retry later; installed libraries still work"
|
|
68
|
+
_RESERVED_NAMES = {"CON", "PRN", "AUX", "NUL", *(f"COM{n}" for n in range(1, 10)), *(f"LPT{n}" for n in range(1, 10))}
|
|
69
|
+
_FILE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,190}\.zim")
|
|
70
|
+
_ANSI = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b\[[0-?]*[ -/]*[@-~]|\x1b[@-_]")
|
|
71
|
+
_BIDI = set("")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class CatalogEntry:
|
|
76
|
+
name: str
|
|
77
|
+
title: str
|
|
78
|
+
date: str
|
|
79
|
+
language: str
|
|
80
|
+
summary: str
|
|
81
|
+
creator: str
|
|
82
|
+
articles: int
|
|
83
|
+
approx_bytes: int # the feed's rounded length: a size estimate, never a check
|
|
84
|
+
url: str # the .zim URL (the feed's .meta4 link without the suffix)
|
|
85
|
+
tags: tuple[str, ...]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def clean(text: str) -> str:
|
|
89
|
+
"""Catalogue text with escape sequences and control or bidi-override characters removed, so a hostile feed
|
|
90
|
+
cannot recolour, overwrite or reorder what a terminal prints."""
|
|
91
|
+
text = _ANSI.sub("", text)
|
|
92
|
+
return "".join(
|
|
93
|
+
" " if ch in "\t\n\r" else ch
|
|
94
|
+
for ch in text
|
|
95
|
+
if ch in "\t\n\r" or (unicodedata.category(ch) != "Cc" and ch not in _BIDI)
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def is_catalogue_host(host: str | None) -> bool:
|
|
100
|
+
return bool(host) and (host == CATALOGUE_DOMAIN or str(host).endswith("." + CATALOGUE_DOMAIN))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _non_public(host: str | None) -> bool:
|
|
104
|
+
"""A loopback, private, link-local or otherwise non-global address, in any spelling `inet_aton` accepts."""
|
|
105
|
+
name = (host or "").lower().rstrip(".")
|
|
106
|
+
if name == "localhost" or name.endswith(".localhost"):
|
|
107
|
+
return True
|
|
108
|
+
try:
|
|
109
|
+
address = ipaddress.ip_address(name)
|
|
110
|
+
except ValueError:
|
|
111
|
+
try:
|
|
112
|
+
address = ipaddress.ip_address(socket.inet_ntoa(socket.inet_aton(name)))
|
|
113
|
+
except OSError:
|
|
114
|
+
return False
|
|
115
|
+
return not address.is_global
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _check_url(url: str, previous: str = "", *, origin_host: str | None = None) -> None:
|
|
119
|
+
"""Refuse a URL that is not https (http only to a local host), that downgrades from `previous`, or that points
|
|
120
|
+
at a non-public address when the request did not begin at a local host."""
|
|
121
|
+
parts = urllib.parse.urlsplit(url)
|
|
122
|
+
local = parts.hostname in LOCAL_HOSTS
|
|
123
|
+
if parts.scheme != "https" and not (parts.scheme == "http" and local):
|
|
124
|
+
raise ZimloreError(f"refusing to fetch {url}: only https is allowed", "")
|
|
125
|
+
if previous and urllib.parse.urlsplit(previous).scheme == "https" and parts.scheme != "https":
|
|
126
|
+
raise ZimloreError(f"refusing to follow a redirect from {previous} down to {url}: https must stay https", "")
|
|
127
|
+
started_local = (origin_host if origin_host is not None else parts.hostname) in LOCAL_HOSTS
|
|
128
|
+
if _non_public(parts.hostname) and not started_local:
|
|
129
|
+
raise ZimloreError(f"refusing to fetch {url}: it is a loopback, private or link-local address", "")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class _Redirects(urllib.request.HTTPRedirectHandler):
|
|
133
|
+
"""Re-runs the URL rules on every hop. With `pinned`, a request that did not begin at a local host may only
|
|
134
|
+
be redirected within the catalogue domain (the `.meta4` is the root of trust for the whole download)."""
|
|
135
|
+
|
|
136
|
+
def __init__(self, pinned: bool = False) -> None:
|
|
137
|
+
super().__init__()
|
|
138
|
+
self.pinned = pinned
|
|
139
|
+
|
|
140
|
+
def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> Any:
|
|
141
|
+
origin = req.origin_req_host
|
|
142
|
+
_check_url(newurl, req.full_url, origin_host=origin)
|
|
143
|
+
host = urllib.parse.urlsplit(newurl).hostname
|
|
144
|
+
if self.pinned and origin not in LOCAL_HOSTS and not is_catalogue_host(host):
|
|
145
|
+
raise ZimloreError(
|
|
146
|
+
f"refusing to follow a redirect to {newurl}: the .meta4 must come from {CATALOGUE_DOMAIN}",
|
|
147
|
+
"the size and hash would otherwise come from the host that serves the file",
|
|
148
|
+
)
|
|
149
|
+
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _open(request: urllib.request.Request, pinned: bool = False) -> Any:
|
|
153
|
+
return urllib.request.build_opener(_Redirects(pinned)).open(request, timeout=60) # noqa: S310
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def whole_number(value: object, what: str, *, default: int | None = None, minimum: int = 0) -> int:
|
|
157
|
+
"""A catalogue number as an int. Anything but plain ASCII digits (a sign, a decimal point, `²`, text, a value
|
|
158
|
+
of more than 15 digits) is a `ZimloreError`, never a bare `ValueError`. Missing or empty gives `default` when
|
|
159
|
+
set."""
|
|
160
|
+
text = str(value).strip() if value is not None else ""
|
|
161
|
+
if not text and default is not None:
|
|
162
|
+
return default
|
|
163
|
+
if not (text.isascii() and text.isdigit()) or len(text) > 15:
|
|
164
|
+
raise ZimloreError(f"the catalogue returned {what} {text[:40]!r}, which is not a whole number", RETRY)
|
|
165
|
+
number = int(text)
|
|
166
|
+
if number < minimum:
|
|
167
|
+
raise ZimloreError(f"the catalogue returned {what} {number}, below the minimum {minimum}", RETRY)
|
|
168
|
+
return number
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _fetch_error(what: str, error: BaseException) -> ZimloreError:
|
|
172
|
+
return ZimloreError(f"could not fetch {what}: {error}", "check the network, then " + RETRY)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _get(url: str, opener: Opener | None = None, limit: int = 20_000_000, *, pinned: bool = False) -> bytes:
|
|
176
|
+
_check_url(url)
|
|
177
|
+
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) # noqa: S310
|
|
178
|
+
try:
|
|
179
|
+
with (opener or (lambda r: _open(r, pinned)))(request) as response:
|
|
180
|
+
data: bytes = response.read(limit + 1)
|
|
181
|
+
except (OSError, http.client.HTTPException, ValueError) as error:
|
|
182
|
+
raise _fetch_error(url, error) from error
|
|
183
|
+
if len(data) > limit:
|
|
184
|
+
raise ZimloreError(f"{url} is larger than {limit} bytes; refusing to read it into memory")
|
|
185
|
+
return data
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def parse_feed(xml: bytes) -> tuple[list[CatalogEntry], int]:
|
|
189
|
+
"""Entries and the feed's `totalResults`. Entries without a ZIM acquisition link are skipped (the catalogue
|
|
190
|
+
also lists non-ZIM resources in principle) and are not counted as results. Text fields are `clean`ed and
|
|
191
|
+
numbers go through `whole_number`."""
|
|
192
|
+
try:
|
|
193
|
+
root = ET.fromstring(xml) # noqa: S314 - a catalogue feed from kiwix.org; no external entities in expat
|
|
194
|
+
except ET.ParseError as error:
|
|
195
|
+
raise ZimloreError(f"the catalogue did not return valid XML ({error})", "try again later") from error
|
|
196
|
+
total_node = root.find("a:totalResults", ATOM)
|
|
197
|
+
total = whole_number(total_node.text if total_node is not None else None, "totalResults", default=0)
|
|
198
|
+
entries: list[CatalogEntry] = []
|
|
199
|
+
for node in root.findall("a:entry", ATOM):
|
|
200
|
+
|
|
201
|
+
def text(tag: str, node: ET.Element = node) -> str:
|
|
202
|
+
found = node.find(f"a:{tag}", ATOM)
|
|
203
|
+
return clean((found.text or "").strip()) if found is not None else ""
|
|
204
|
+
|
|
205
|
+
link = next((n for n in node.findall("a:link", ATOM) if n.get("type") == "application/x-zim"), None)
|
|
206
|
+
if link is None or not text("name"):
|
|
207
|
+
continue
|
|
208
|
+
href = link.get("href", "")
|
|
209
|
+
author = node.find("a:author/a:name", ATOM)
|
|
210
|
+
entries.append(
|
|
211
|
+
CatalogEntry(
|
|
212
|
+
name=text("name"),
|
|
213
|
+
title=text("title"),
|
|
214
|
+
date=text("updated")[:10],
|
|
215
|
+
language=text("language"),
|
|
216
|
+
summary=text("summary"),
|
|
217
|
+
creator=clean((author.text or "").strip()) if author is not None else "",
|
|
218
|
+
articles=whole_number(text("articleCount"), "articleCount", default=0),
|
|
219
|
+
approx_bytes=whole_number(link.get("length"), "length", default=0),
|
|
220
|
+
url=href[: -len(".meta4")] if href.endswith(".meta4") else href,
|
|
221
|
+
tags=tuple(t for t in text("tags").split(";") if t and not t.startswith("_")),
|
|
222
|
+
)
|
|
223
|
+
)
|
|
224
|
+
return entries, total
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def query(
|
|
228
|
+
text: str = "",
|
|
229
|
+
*,
|
|
230
|
+
name: str = "",
|
|
231
|
+
language: str = "eng",
|
|
232
|
+
count: int = 20,
|
|
233
|
+
start: int = 0,
|
|
234
|
+
tag: str = "",
|
|
235
|
+
max_bytes: int = 0,
|
|
236
|
+
base: str | None = None,
|
|
237
|
+
opener: Opener | None = None,
|
|
238
|
+
) -> tuple[list[CatalogEntry], int]:
|
|
239
|
+
base = base or CATALOG_URL # looked up per call so a test can point the whole module at a local feed
|
|
240
|
+
params = {"count": str(count)}
|
|
241
|
+
if start:
|
|
242
|
+
params["start"] = str(start)
|
|
243
|
+
if text:
|
|
244
|
+
params["q"] = text
|
|
245
|
+
if name:
|
|
246
|
+
params["name"] = name
|
|
247
|
+
if language:
|
|
248
|
+
params["lang"] = language
|
|
249
|
+
if tag:
|
|
250
|
+
params["tag"] = tag # e.g. "devdocs": 243 ZIMs on 2026-11-07
|
|
251
|
+
if max_bytes > 0:
|
|
252
|
+
params["maxsize"] = str(max_bytes) # bytes (checked: 140 DevDocs ZIMs are <= 1,000,000)
|
|
253
|
+
return parse_feed(_get(f"{base}?{urllib.parse.urlencode(params)}", opener))
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def editions_of(name: str, *, opener: Opener | None = None) -> list[CatalogEntry]:
|
|
257
|
+
"""EVERY catalogue edition of one name, paged until the feed's `totalResults` is reached, a page adds nothing
|
|
258
|
+
new, or `MAX_PAGES`. A caller choosing 'the newest' must see all of them:
|
|
259
|
+
the feed's own order is not a promise."""
|
|
260
|
+
found: dict[str, CatalogEntry] = {}
|
|
261
|
+
start = 0
|
|
262
|
+
for _ in range(MAX_PAGES):
|
|
263
|
+
page, total = query(name=name, language="", count=PAGE_SIZE, start=start, opener=opener)
|
|
264
|
+
fresh = [e for e in page if e.url not in found]
|
|
265
|
+
found.update((e.url, e) for e in fresh)
|
|
266
|
+
start += len(page)
|
|
267
|
+
if not fresh or len(found) >= total:
|
|
268
|
+
break
|
|
269
|
+
return list(found.values())
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def latest(entries: list[CatalogEntry]) -> list[CatalogEntry]:
|
|
273
|
+
"""One entry per name: the newest by parsed date, whatever order `entries` arrive in."""
|
|
274
|
+
newest: dict[str, CatalogEntry] = {}
|
|
275
|
+
for entry in sorted(entries, key=lambda e: library.date_key(e.date), reverse=True):
|
|
276
|
+
newest.setdefault(entry.name, entry)
|
|
277
|
+
return list(newest.values())
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
@dataclass(frozen=True)
|
|
281
|
+
class Metalink:
|
|
282
|
+
size: int
|
|
283
|
+
sha256: str
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def parse_metalink(xml: bytes) -> Metalink:
|
|
287
|
+
try:
|
|
288
|
+
root = ET.fromstring(xml) # noqa: S314
|
|
289
|
+
except ET.ParseError as error:
|
|
290
|
+
raise ZimloreError(f"the .meta4 file is not valid XML ({error})") from error
|
|
291
|
+
for file in root.iter(f"{METALINK}file"): # size and hash must describe the SAME file element
|
|
292
|
+
size = file.find(f"{METALINK}size")
|
|
293
|
+
digest = next((h for h in file.iter(f"{METALINK}hash") if h.get("type") == "sha-256"), None)
|
|
294
|
+
text = (digest.text or "").strip().lower() if digest is not None else ""
|
|
295
|
+
if size is not None and re.fullmatch(r"[0-9a-f]{64}", text):
|
|
296
|
+
return Metalink(whole_number(size.text, "the .meta4 size", minimum=1), text)
|
|
297
|
+
raise ZimloreError("the .meta4 file carries no size and SHA-256", "the download cannot be verified; not fetching")
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def target_name(entry: CatalogEntry) -> str:
|
|
301
|
+
"""The file name a download of `entry` gets. It carries the edition date, so a newer edition is a different
|
|
302
|
+
file and an update never touches the older one. Only a plain name is accepted: no separators, no spaces or
|
|
303
|
+
control characters, no Windows device name (`CON.zim` would open the console, not a file)."""
|
|
304
|
+
name = Path(urllib.parse.urlsplit(entry.url).path).name
|
|
305
|
+
if not name.endswith(".zim"):
|
|
306
|
+
raise ZimloreError(f"{entry.url} does not name a .zim file")
|
|
307
|
+
if not _FILE_NAME.fullmatch(name) or name.split(".")[0].upper() in _RESERVED_NAMES:
|
|
308
|
+
raise ZimloreError(
|
|
309
|
+
f"{entry.url} names a file ({name[:60]!r}) that zimlore will not create",
|
|
310
|
+
"ZIM file names are letters, digits and . _ + - and may not be a Windows device name",
|
|
311
|
+
)
|
|
312
|
+
return name
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _touch(path: Path) -> None:
|
|
316
|
+
with contextlib.suppress(OSError): # a lock that cannot be touched only risks being reported stale
|
|
317
|
+
os.utime(path)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _acquire_lock(lock: Path, stale_after: float) -> None:
|
|
321
|
+
"""Create `lock` exclusively. An existing lock is an error either way: a young one means another download is
|
|
322
|
+
writing the same `.part`; an old one is reported as probably stale, with the remedy, but never removed here."""
|
|
323
|
+
try:
|
|
324
|
+
descriptor = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
325
|
+
except FileExistsError:
|
|
326
|
+
try:
|
|
327
|
+
age = time.time() - lock.stat().st_mtime
|
|
328
|
+
except OSError:
|
|
329
|
+
age = 0.0
|
|
330
|
+
if age > stale_after:
|
|
331
|
+
raise ZimloreError(
|
|
332
|
+
f"{lock} is {age / 60:.0f} minutes old: a download that died probably left it behind",
|
|
333
|
+
f"if no zimlore is running, delete {lock} and run the same command again; the .part resumes",
|
|
334
|
+
) from None
|
|
335
|
+
raise ZimloreError(
|
|
336
|
+
f"another zimlore is downloading to {lock.with_suffix('').name} (lock {lock})",
|
|
337
|
+
"wait for it to finish; two downloads must not write one .part",
|
|
338
|
+
) from None
|
|
339
|
+
with os.fdopen(descriptor, "w") as handle:
|
|
340
|
+
handle.write(f"pid {os.getpid()}\n")
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _header(response: Any, name: str) -> str | None:
|
|
344
|
+
headers = getattr(response, "headers", None)
|
|
345
|
+
return headers.get(name) if headers is not None else None
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _range_start(response: Any) -> int | None:
|
|
349
|
+
match = re.fullmatch(r"bytes (\d+)-\d+/(?:\d+|\*)", _header(response, "Content-Range") or "", re.ASCII)
|
|
350
|
+
return int(match.group(1)) if match else None
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _transfer(
|
|
354
|
+
entry: CatalogEntry,
|
|
355
|
+
part: Path,
|
|
356
|
+
lock: Path,
|
|
357
|
+
meta: Metalink,
|
|
358
|
+
opener: Opener | None,
|
|
359
|
+
progress: Callable[[int, int], None] | None,
|
|
360
|
+
deadline: float = DOWNLOAD_DEADLINE_SECONDS,
|
|
361
|
+
) -> tuple[int, str]:
|
|
362
|
+
"""Bring `part` to `meta.size` bytes and return (bytes, SHA-256 of the WHOLE file). An existing `.part` is
|
|
363
|
+
resumed with a Range request after its prefix is re-hashed; a mirror that refuses the range (200 instead of
|
|
364
|
+
206, a wrong Content-Range, 416) makes the download start over from byte zero, never splice. Never more than
|
|
365
|
+
`meta.size` bytes reach the disk, and no more than `deadline` seconds are spent."""
|
|
366
|
+
started = time.monotonic()
|
|
367
|
+
for _ in range(2):
|
|
368
|
+
digest = hashlib.sha256()
|
|
369
|
+
offset = part.stat().st_size if part.exists() else 0
|
|
370
|
+
if offset > meta.size: # longer than the file can be: not a prefix of it
|
|
371
|
+
part.unlink()
|
|
372
|
+
offset = 0
|
|
373
|
+
if offset:
|
|
374
|
+
with part.open("rb") as prefix:
|
|
375
|
+
while chunk := prefix.read(1 << 20):
|
|
376
|
+
digest.update(chunk)
|
|
377
|
+
_touch(lock)
|
|
378
|
+
if progress:
|
|
379
|
+
progress(offset, meta.size)
|
|
380
|
+
if offset == meta.size:
|
|
381
|
+
return offset, digest.hexdigest() # a complete .part: nothing to fetch, the hash check decides
|
|
382
|
+
headers = {"User-Agent": USER_AGENT}
|
|
383
|
+
if offset:
|
|
384
|
+
headers["Range"] = f"bytes={offset}-"
|
|
385
|
+
request = urllib.request.Request(entry.url, headers=headers) # noqa: S310
|
|
386
|
+
try:
|
|
387
|
+
reply = (opener or _open)(request)
|
|
388
|
+
except urllib.error.HTTPError as error:
|
|
389
|
+
if error.code == 416 and offset: # the mirror's file is shorter than our part
|
|
390
|
+
part.unlink(missing_ok=True)
|
|
391
|
+
continue
|
|
392
|
+
raise ZimloreError(f"download of {entry.name} failed ({error})", RETRY) from error
|
|
393
|
+
except (OSError, http.client.HTTPException, ValueError) as error:
|
|
394
|
+
raise ZimloreError(f"download of {entry.name} failed ({error})", RETRY) from error
|
|
395
|
+
with reply as response:
|
|
396
|
+
status = getattr(response, "status", 200)
|
|
397
|
+
if offset and not (status == 206 and _range_start(response) == offset):
|
|
398
|
+
part.unlink(missing_ok=True)
|
|
399
|
+
if status != 200:
|
|
400
|
+
continue
|
|
401
|
+
digest, offset = hashlib.sha256(), 0 # Range refused: this 200 is the whole file, use it from zero
|
|
402
|
+
announced = _header(response, "Content-Length")
|
|
403
|
+
promised = int(announced) if announced and announced.isascii() and announced.isdigit() else None
|
|
404
|
+
if promised is not None and promised > meta.size - offset:
|
|
405
|
+
part.unlink(missing_ok=True)
|
|
406
|
+
raise ZimloreError(
|
|
407
|
+
f"{entry.name}: the server announces {promised} bytes; the .meta4 allows {meta.size - offset}",
|
|
408
|
+
"the mirror is serving a different file; nothing was kept",
|
|
409
|
+
)
|
|
410
|
+
done = offset
|
|
411
|
+
too_much = False
|
|
412
|
+
try:
|
|
413
|
+
with part.open("ab" if offset else "wb") as out:
|
|
414
|
+
while chunk := response.read(min(CHUNK, meta.size - done + 1)):
|
|
415
|
+
if done + len(chunk) > meta.size:
|
|
416
|
+
too_much = True
|
|
417
|
+
break
|
|
418
|
+
out.write(chunk)
|
|
419
|
+
digest.update(chunk)
|
|
420
|
+
done += len(chunk)
|
|
421
|
+
_touch(lock)
|
|
422
|
+
if progress:
|
|
423
|
+
progress(done, meta.size)
|
|
424
|
+
if done < meta.size and time.monotonic() - started >= deadline:
|
|
425
|
+
raise ZimloreError(
|
|
426
|
+
f"download of {entry.name} passed its {deadline:.0f} s deadline at {done} of "
|
|
427
|
+
f"{meta.size} bytes",
|
|
428
|
+
f"run the same command again: it resumes from {part}",
|
|
429
|
+
)
|
|
430
|
+
except (OSError, http.client.HTTPException, ValueError) as error:
|
|
431
|
+
raise ZimloreError(
|
|
432
|
+
f"download of {entry.name} was interrupted at {done} of {meta.size} bytes ({error})",
|
|
433
|
+
f"run the same command again: it resumes from {part}",
|
|
434
|
+
) from error
|
|
435
|
+
if too_much:
|
|
436
|
+
part.unlink(missing_ok=True)
|
|
437
|
+
raise ZimloreError(
|
|
438
|
+
f"{entry.name}: the server sent more than the {meta.size} bytes the .meta4 allows; discarded",
|
|
439
|
+
"the mirror is serving a different file",
|
|
440
|
+
)
|
|
441
|
+
if promised is not None and done - offset < promised:
|
|
442
|
+
raise ZimloreError(
|
|
443
|
+
f"download of {entry.name} was cut off at {done} of {meta.size} bytes",
|
|
444
|
+
f"run the same command again: it resumes from {part}",
|
|
445
|
+
)
|
|
446
|
+
return done, digest.hexdigest()
|
|
447
|
+
raise ZimloreError(f"the mirror for {entry.name} kept refusing to serve the file", RETRY)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
@dataclass(frozen=True)
|
|
451
|
+
class Downloaded:
|
|
452
|
+
path: Path
|
|
453
|
+
status: str # "installed" | "replaced" | "already-installed"
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _ensure_replaceable(final: Path) -> None:
|
|
457
|
+
"""Fail BEFORE any download when `final` cannot be replaced. On Windows libzim (or any reader) holds a ZIM
|
|
458
|
+
without share-delete, so the file cannot be renamed or replaced until the last handle is gone: probe with a
|
|
459
|
+
rename and put it back."""
|
|
460
|
+
library.close_path(final)
|
|
461
|
+
probe = final.with_name(final.name + ".probe")
|
|
462
|
+
try:
|
|
463
|
+
os.rename(final, probe)
|
|
464
|
+
except OSError as error:
|
|
465
|
+
raise ZimloreError(
|
|
466
|
+
f"{final} is in use by another process and cannot be replaced ({error}); nothing was downloaded",
|
|
467
|
+
"stop any running 'zimlore mcp' server or kiwix reader that has it open, then run the same command again",
|
|
468
|
+
) from error
|
|
469
|
+
try:
|
|
470
|
+
os.rename(probe, final)
|
|
471
|
+
except OSError as error: # should be impossible: we just renamed it
|
|
472
|
+
raise ZimloreError(
|
|
473
|
+
f"{final} was moved to {probe} and could not be moved back ({error})", f"rename {probe}"
|
|
474
|
+
) from error
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _same_file(final: Path, meta: Metalink) -> bool:
|
|
478
|
+
record, _ = manage.read_sidecar(final)
|
|
479
|
+
return bool(record and final.is_file() and record.sha256 == meta.sha256 and final.stat().st_size == meta.size)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _check_space(directory: Path, part: Path, meta: Metalink, name: str) -> None:
|
|
483
|
+
need = meta.size - (part.stat().st_size if part.exists() else 0)
|
|
484
|
+
try:
|
|
485
|
+
free = shutil.disk_usage(directory).free
|
|
486
|
+
except OSError:
|
|
487
|
+
return
|
|
488
|
+
if need > free:
|
|
489
|
+
raise ZimloreError(
|
|
490
|
+
f"{name} needs {need} more bytes and {directory} has {free} free; nothing was downloaded",
|
|
491
|
+
"free space: zimlore du shows the use, zimlore prune --yes drops old editions",
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def download(
|
|
496
|
+
entry: CatalogEntry,
|
|
497
|
+
directory: Path,
|
|
498
|
+
*,
|
|
499
|
+
opener: Opener | None = None,
|
|
500
|
+
progress: Callable[[int, int], None] | None = None,
|
|
501
|
+
added_by: str = "get",
|
|
502
|
+
stale_lock_seconds: float = STALE_LOCK_SECONDS,
|
|
503
|
+
replace: bool = False,
|
|
504
|
+
deadline_seconds: float = DOWNLOAD_DEADLINE_SECONDS,
|
|
505
|
+
) -> Path:
|
|
506
|
+
"""`download_result(...).path`: see there."""
|
|
507
|
+
return download_result(
|
|
508
|
+
entry,
|
|
509
|
+
directory,
|
|
510
|
+
opener=opener,
|
|
511
|
+
progress=progress,
|
|
512
|
+
added_by=added_by,
|
|
513
|
+
stale_lock_seconds=stale_lock_seconds,
|
|
514
|
+
replace=replace,
|
|
515
|
+
deadline_seconds=deadline_seconds,
|
|
516
|
+
).path
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def download_result(
|
|
520
|
+
entry: CatalogEntry,
|
|
521
|
+
directory: Path,
|
|
522
|
+
*,
|
|
523
|
+
opener: Opener | None = None,
|
|
524
|
+
progress: Callable[[int, int], None] | None = None,
|
|
525
|
+
added_by: str = "get",
|
|
526
|
+
stale_lock_seconds: float = STALE_LOCK_SECONDS,
|
|
527
|
+
replace: bool = False,
|
|
528
|
+
deadline_seconds: float = DOWNLOAD_DEADLINE_SECONDS,
|
|
529
|
+
) -> Downloaded:
|
|
530
|
+
"""Fetch `entry` into `directory`, verifying size and SHA-256 against the `.meta4` before the file takes its
|
|
531
|
+
final name. A failed or interrupted download leaves a `.part` file, never a `.zim` that looks complete, and
|
|
532
|
+
the next call resumes it. The size cap comes from the `.meta4` before the first byte, a refusal to exceed the
|
|
533
|
+
free disk space too, and a `.part.lock` keeps two downloads from writing one `.part`.
|
|
534
|
+
|
|
535
|
+
An existing file of that name is never replaced unless `replace`: if its sidecar already records this
|
|
536
|
+
`.meta4`'s SHA-256 (and its size matches) the call is a no-op returning `already-installed`. With `replace`
|
|
537
|
+
on Windows a file that is open elsewhere is reported before the download starts."""
|
|
538
|
+
final = directory / target_name(entry)
|
|
539
|
+
_check_url(entry.url)
|
|
540
|
+
host = urllib.parse.urlsplit(entry.url).hostname
|
|
541
|
+
if not (is_catalogue_host(host) or host in LOCAL_HOSTS):
|
|
542
|
+
raise ZimloreError(
|
|
543
|
+
f"{entry.url} is not on {CATALOGUE_DOMAIN}; its .meta4 would come from the host that serves the file",
|
|
544
|
+
"zimlore downloads only catalogue entries; to install a file from elsewhere use zimlore add",
|
|
545
|
+
)
|
|
546
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
547
|
+
part = final.with_name(final.name + ".part")
|
|
548
|
+
lock = part.with_name(part.name + ".lock")
|
|
549
|
+
_acquire_lock(lock, stale_lock_seconds)
|
|
550
|
+
try:
|
|
551
|
+
existing = os.path.lexists(final)
|
|
552
|
+
if existing and replace:
|
|
553
|
+
_ensure_replaceable(final)
|
|
554
|
+
meta = parse_metalink(_get(entry.url + ".meta4", opener, pinned=True))
|
|
555
|
+
if existing:
|
|
556
|
+
if _same_file(final, meta):
|
|
557
|
+
return Downloaded(final, "already-installed")
|
|
558
|
+
if not replace:
|
|
559
|
+
raise ZimloreError(
|
|
560
|
+
f"{final} already exists and is not the file the catalogue describes; not replacing it",
|
|
561
|
+
"zimlore get NAME --replace replaces it (the old file is lost); otherwise move it aside",
|
|
562
|
+
)
|
|
563
|
+
_check_space(directory, part, meta, entry.name)
|
|
564
|
+
done, hexdigest = _transfer(entry, part, lock, meta, opener, progress, deadline_seconds)
|
|
565
|
+
if done != meta.size or hexdigest != meta.sha256:
|
|
566
|
+
part.unlink(missing_ok=True)
|
|
567
|
+
raise ZimloreError(
|
|
568
|
+
f"{entry.name}: downloaded {done} bytes with sha256 {hexdigest[:12]}…, expected "
|
|
569
|
+
f"{meta.size} bytes with {meta.sha256[:12]}…; the file was discarded",
|
|
570
|
+
"retry; if it persists the mirror is serving a different file",
|
|
571
|
+
)
|
|
572
|
+
if existing:
|
|
573
|
+
library.close_path(final)
|
|
574
|
+
try:
|
|
575
|
+
os.replace(part, final)
|
|
576
|
+
except OSError as error:
|
|
577
|
+
raise ZimloreError(
|
|
578
|
+
f"the verified download of {entry.name} could not be installed as {final} ({error})",
|
|
579
|
+
f"it is kept as {part}; free the target and run the same command again (no re-download is needed)",
|
|
580
|
+
) from error
|
|
581
|
+
finally:
|
|
582
|
+
lock.unlink(missing_ok=True)
|
|
583
|
+
record = manage.Provenance(entry.url, entry.name, entry.date, meta.sha256, meta.size, manage.now(), added_by)
|
|
584
|
+
try:
|
|
585
|
+
manage.write_sidecar(final, record)
|
|
586
|
+
except OSError as error:
|
|
587
|
+
manage.sidecar_path(final).unlink(missing_ok=True) # never leave the previous file's record on this one
|
|
588
|
+
raise ZimloreError(
|
|
589
|
+
f"{final} was installed and verified but its provenance record could not be written ({error})",
|
|
590
|
+
"free disk space, then zimlore verify --adopt records what the file is",
|
|
591
|
+
) from error
|
|
592
|
+
return Downloaded(final, "replaced" if existing else "installed")
|