zimlore 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
zimlore/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """zimlore: a Kiwix library (ZIM files) as token-budgeted, citable documentation for AI agents."""
2
+
3
+ from __future__ import annotations
4
+
5
+ __version__ = "0.1.0"
zimlore/__main__.py ADDED
@@ -0,0 +1,8 @@
1
+ """`python -m zimlore` is the same as the `zimlore` command."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .cli import main
6
+
7
+ if __name__ == "__main__":
8
+ raise SystemExit(main())
zimlore/_libzim.py ADDED
@@ -0,0 +1,37 @@
1
+ """The only place that imports the native binding.
2
+
3
+ `libzim` is a compiled extension (it carries libzim and Xapian). It ships wheels for CPython 3.11+ on
4
+ Windows x64, macOS (x86-64, arm64) and Linux (glibc and musl, x86-64 and aarch64), and nothing else, so on any
5
+ other platform `import libzim` fails. Everything that does not read a ZIM (the CLI's `info`, `catalog`, `get`,
6
+ the reference grammar, the converter) must keep working there, which is why the import is lazy and why its
7
+ failure is a `ZimloreError` naming the remedy instead of a traceback from deep inside a tool.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import importlib
13
+ from types import ModuleType
14
+
15
+ from .errors import ZimloreError
16
+
17
+ REMEDY = (
18
+ "pip install libzim (wheels: Windows x64, macOS, Linux glibc/musl; CPython 3.11+). On a platform without a "
19
+ "wheel, build libzim from source (https://github.com/openzim/python-libzim) or serve the same files with "
20
+ "kiwix-serve and read them in a browser"
21
+ )
22
+
23
+
24
+ def load(name: str) -> ModuleType:
25
+ """Import `libzim.<name>` (reader, search, suggestion, writer)."""
26
+ try:
27
+ return importlib.import_module(f"libzim.{name}")
28
+ except ImportError as error:
29
+ raise ZimloreError(f"the libzim binding is not available ({error})", REMEDY) from error
30
+
31
+
32
+ def available() -> bool:
33
+ try:
34
+ load("reader")
35
+ except ZimloreError:
36
+ return False
37
+ return True
zimlore/catalog.py ADDED
@@ -0,0 +1,592 @@
1
+ """The public Kiwix catalogue (OPDS) and verified downloads. The only module that touches the network.
2
+
3
+ Everything else in zimlore is offline. This module exists because "put ZIM files in a directory" is a poor first
4
+ step for someone who does not yet know which file holds Flask's documentation.
5
+
6
+ Facts measured against the live service on 2026-11-07 and the reason for each rule below:
7
+
8
+ * `https://library.kiwix.org/catalog/v2/...` answers 301 and the feed lives at `opds.library.kiwix.org`.
9
+ * `entries?q=...&lang=eng&count=N` filters; `name=<exact>` returns every dated edition of one name.
10
+ * The acquisition link in the feed ends in `.zim.meta4`. The `.meta4` is a Metalink document carrying the
11
+ file's real byte size and its SHA-256, and the `.zim` URL redirects to an arbitrary mirror. So the hash is
12
+ read from the kiwix.org family of hosts (`.meta4` redirects stay inside it too: `download.kiwix.org` answers 301
13
+ to `lb.download.kiwix.org`) and the bytes may come from anywhere: a download is accepted only if it matches.
14
+ * The feed's own `length` is rounded (617,472 for a file of 617,313 bytes), so it is never used as a check.
15
+
16
+ Every hop of every fetch, redirects included, is checked again: https only (plain http only to a local host),
17
+ never https down to http, never to a loopback, private or link-local address unless the request began at a local
18
+ host. Host names are not resolved, so a public name that points at a private address is not caught here.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import contextlib
24
+ import hashlib
25
+ import http.client
26
+ import ipaddress
27
+ import os
28
+ import re
29
+ import shutil
30
+ import socket
31
+ import time
32
+ import unicodedata
33
+ import urllib.error
34
+ import urllib.parse
35
+ import urllib.request
36
+ import xml.etree.ElementTree as ET
37
+ from collections.abc import Callable
38
+ from dataclasses import dataclass
39
+ from pathlib import Path
40
+ from typing import Any
41
+
42
+ from . import __version__, library, manage
43
+ from .errors import ZimloreError
44
+
45
+ CATALOG_URL = "https://opds.library.kiwix.org/catalog/v2/entries"
46
+ USER_AGENT = f"zimlore/{__version__}"
47
+ ATOM = {"a": "http://www.w3.org/2005/Atom", "dc": "http://purl.org/dc/terms/"}
48
+ METALINK = "{urn:ietf:params:xml:ns:metalink}"
49
+ LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1"}
50
+ CATALOGUE_DOMAIN = "kiwix.org"
51
+ """The `.meta4` carrying the trusted size and hash is fetched from this domain or a subdomain (redirects included).
52
+ Only the `.zim` bytes may come from another mirror."""
53
+ STALE_LOCK_SECONDS = 3600
54
+ """A `.part.lock` is touched on every chunk, so one whose last touch is older than this belongs to a process that
55
+ died. It is reported with the remedy and never broken silently: deleting a live lock would let two downloads write
56
+ one `.part`."""
57
+ DOWNLOAD_DEADLINE_SECONDS = 6 * 3600
58
+ """A download that has not finished this long after it started is stopped (the `.part` stays and resumes). The
59
+ socket timeout is per read, so without this a mirror sending one byte a minute would hold the lock for ever."""
60
+ CHUNK = 1 << 20
61
+ """The read size of a download: also the granularity of the deadline and byte-cap checks."""
62
+ PAGE_SIZE = 100
63
+ MAX_PAGES = 50
64
+
65
+ Opener = Callable[[urllib.request.Request], Any]
66
+
67
+ RETRY = "retry later; installed libraries still work"
68
+ _RESERVED_NAMES = {"CON", "PRN", "AUX", "NUL", *(f"COM{n}" for n in range(1, 10)), *(f"LPT{n}" for n in range(1, 10))}
69
+ _FILE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,190}\.zim")
70
+ _ANSI = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b\[[0-?]*[ -/]*[@-~]|\x1b[@-_]")
71
+ _BIDI = set("‪‫‬‭‮⁦⁧⁨⁩")
72
+
73
+
74
+ @dataclass(frozen=True)
75
+ class CatalogEntry:
76
+ name: str
77
+ title: str
78
+ date: str
79
+ language: str
80
+ summary: str
81
+ creator: str
82
+ articles: int
83
+ approx_bytes: int # the feed's rounded length: a size estimate, never a check
84
+ url: str # the .zim URL (the feed's .meta4 link without the suffix)
85
+ tags: tuple[str, ...]
86
+
87
+
88
+ def clean(text: str) -> str:
89
+ """Catalogue text with escape sequences and control or bidi-override characters removed, so a hostile feed
90
+ cannot recolour, overwrite or reorder what a terminal prints."""
91
+ text = _ANSI.sub("", text)
92
+ return "".join(
93
+ " " if ch in "\t\n\r" else ch
94
+ for ch in text
95
+ if ch in "\t\n\r" or (unicodedata.category(ch) != "Cc" and ch not in _BIDI)
96
+ )
97
+
98
+
99
+ def is_catalogue_host(host: str | None) -> bool:
100
+ return bool(host) and (host == CATALOGUE_DOMAIN or str(host).endswith("." + CATALOGUE_DOMAIN))
101
+
102
+
103
+ def _non_public(host: str | None) -> bool:
104
+ """A loopback, private, link-local or otherwise non-global address, in any spelling `inet_aton` accepts."""
105
+ name = (host or "").lower().rstrip(".")
106
+ if name == "localhost" or name.endswith(".localhost"):
107
+ return True
108
+ try:
109
+ address = ipaddress.ip_address(name)
110
+ except ValueError:
111
+ try:
112
+ address = ipaddress.ip_address(socket.inet_ntoa(socket.inet_aton(name)))
113
+ except OSError:
114
+ return False
115
+ return not address.is_global
116
+
117
+
118
+ def _check_url(url: str, previous: str = "", *, origin_host: str | None = None) -> None:
119
+ """Refuse a URL that is not https (http only to a local host), that downgrades from `previous`, or that points
120
+ at a non-public address when the request did not begin at a local host."""
121
+ parts = urllib.parse.urlsplit(url)
122
+ local = parts.hostname in LOCAL_HOSTS
123
+ if parts.scheme != "https" and not (parts.scheme == "http" and local):
124
+ raise ZimloreError(f"refusing to fetch {url}: only https is allowed", "")
125
+ if previous and urllib.parse.urlsplit(previous).scheme == "https" and parts.scheme != "https":
126
+ raise ZimloreError(f"refusing to follow a redirect from {previous} down to {url}: https must stay https", "")
127
+ started_local = (origin_host if origin_host is not None else parts.hostname) in LOCAL_HOSTS
128
+ if _non_public(parts.hostname) and not started_local:
129
+ raise ZimloreError(f"refusing to fetch {url}: it is a loopback, private or link-local address", "")
130
+
131
+
132
+ class _Redirects(urllib.request.HTTPRedirectHandler):
133
+ """Re-runs the URL rules on every hop. With `pinned`, a request that did not begin at a local host may only
134
+ be redirected within the catalogue domain (the `.meta4` is the root of trust for the whole download)."""
135
+
136
+ def __init__(self, pinned: bool = False) -> None:
137
+ super().__init__()
138
+ self.pinned = pinned
139
+
140
+ def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> Any:
141
+ origin = req.origin_req_host
142
+ _check_url(newurl, req.full_url, origin_host=origin)
143
+ host = urllib.parse.urlsplit(newurl).hostname
144
+ if self.pinned and origin not in LOCAL_HOSTS and not is_catalogue_host(host):
145
+ raise ZimloreError(
146
+ f"refusing to follow a redirect to {newurl}: the .meta4 must come from {CATALOGUE_DOMAIN}",
147
+ "the size and hash would otherwise come from the host that serves the file",
148
+ )
149
+ return super().redirect_request(req, fp, code, msg, headers, newurl)
150
+
151
+
152
+ def _open(request: urllib.request.Request, pinned: bool = False) -> Any:
153
+ return urllib.request.build_opener(_Redirects(pinned)).open(request, timeout=60) # noqa: S310
154
+
155
+
156
+ def whole_number(value: object, what: str, *, default: int | None = None, minimum: int = 0) -> int:
157
+ """A catalogue number as an int. Anything but plain ASCII digits (a sign, a decimal point, `²`, text, a value
158
+ of more than 15 digits) is a `ZimloreError`, never a bare `ValueError`. Missing or empty gives `default` when
159
+ set."""
160
+ text = str(value).strip() if value is not None else ""
161
+ if not text and default is not None:
162
+ return default
163
+ if not (text.isascii() and text.isdigit()) or len(text) > 15:
164
+ raise ZimloreError(f"the catalogue returned {what} {text[:40]!r}, which is not a whole number", RETRY)
165
+ number = int(text)
166
+ if number < minimum:
167
+ raise ZimloreError(f"the catalogue returned {what} {number}, below the minimum {minimum}", RETRY)
168
+ return number
169
+
170
+
171
+ def _fetch_error(what: str, error: BaseException) -> ZimloreError:
172
+ return ZimloreError(f"could not fetch {what}: {error}", "check the network, then " + RETRY)
173
+
174
+
175
+ def _get(url: str, opener: Opener | None = None, limit: int = 20_000_000, *, pinned: bool = False) -> bytes:
176
+ _check_url(url)
177
+ request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) # noqa: S310
178
+ try:
179
+ with (opener or (lambda r: _open(r, pinned)))(request) as response:
180
+ data: bytes = response.read(limit + 1)
181
+ except (OSError, http.client.HTTPException, ValueError) as error:
182
+ raise _fetch_error(url, error) from error
183
+ if len(data) > limit:
184
+ raise ZimloreError(f"{url} is larger than {limit} bytes; refusing to read it into memory")
185
+ return data
186
+
187
+
188
+ def parse_feed(xml: bytes) -> tuple[list[CatalogEntry], int]:
189
+ """Entries and the feed's `totalResults`. Entries without a ZIM acquisition link are skipped (the catalogue
190
+ also lists non-ZIM resources in principle) and are not counted as results. Text fields are `clean`ed and
191
+ numbers go through `whole_number`."""
192
+ try:
193
+ root = ET.fromstring(xml) # noqa: S314 - a catalogue feed from kiwix.org; no external entities in expat
194
+ except ET.ParseError as error:
195
+ raise ZimloreError(f"the catalogue did not return valid XML ({error})", "try again later") from error
196
+ total_node = root.find("a:totalResults", ATOM)
197
+ total = whole_number(total_node.text if total_node is not None else None, "totalResults", default=0)
198
+ entries: list[CatalogEntry] = []
199
+ for node in root.findall("a:entry", ATOM):
200
+
201
+ def text(tag: str, node: ET.Element = node) -> str:
202
+ found = node.find(f"a:{tag}", ATOM)
203
+ return clean((found.text or "").strip()) if found is not None else ""
204
+
205
+ link = next((n for n in node.findall("a:link", ATOM) if n.get("type") == "application/x-zim"), None)
206
+ if link is None or not text("name"):
207
+ continue
208
+ href = link.get("href", "")
209
+ author = node.find("a:author/a:name", ATOM)
210
+ entries.append(
211
+ CatalogEntry(
212
+ name=text("name"),
213
+ title=text("title"),
214
+ date=text("updated")[:10],
215
+ language=text("language"),
216
+ summary=text("summary"),
217
+ creator=clean((author.text or "").strip()) if author is not None else "",
218
+ articles=whole_number(text("articleCount"), "articleCount", default=0),
219
+ approx_bytes=whole_number(link.get("length"), "length", default=0),
220
+ url=href[: -len(".meta4")] if href.endswith(".meta4") else href,
221
+ tags=tuple(t for t in text("tags").split(";") if t and not t.startswith("_")),
222
+ )
223
+ )
224
+ return entries, total
225
+
226
+
227
+ def query(
228
+ text: str = "",
229
+ *,
230
+ name: str = "",
231
+ language: str = "eng",
232
+ count: int = 20,
233
+ start: int = 0,
234
+ tag: str = "",
235
+ max_bytes: int = 0,
236
+ base: str | None = None,
237
+ opener: Opener | None = None,
238
+ ) -> tuple[list[CatalogEntry], int]:
239
+ base = base or CATALOG_URL # looked up per call so a test can point the whole module at a local feed
240
+ params = {"count": str(count)}
241
+ if start:
242
+ params["start"] = str(start)
243
+ if text:
244
+ params["q"] = text
245
+ if name:
246
+ params["name"] = name
247
+ if language:
248
+ params["lang"] = language
249
+ if tag:
250
+ params["tag"] = tag # e.g. "devdocs": 243 ZIMs on 2026-11-07
251
+ if max_bytes > 0:
252
+ params["maxsize"] = str(max_bytes) # bytes (checked: 140 DevDocs ZIMs are <= 1,000,000)
253
+ return parse_feed(_get(f"{base}?{urllib.parse.urlencode(params)}", opener))
254
+
255
+
256
+ def editions_of(name: str, *, opener: Opener | None = None) -> list[CatalogEntry]:
257
+ """EVERY catalogue edition of one name, paged until the feed's `totalResults` is reached, a page adds nothing
258
+ new, or `MAX_PAGES`. A caller choosing 'the newest' must see all of them:
259
+ the feed's own order is not a promise."""
260
+ found: dict[str, CatalogEntry] = {}
261
+ start = 0
262
+ for _ in range(MAX_PAGES):
263
+ page, total = query(name=name, language="", count=PAGE_SIZE, start=start, opener=opener)
264
+ fresh = [e for e in page if e.url not in found]
265
+ found.update((e.url, e) for e in fresh)
266
+ start += len(page)
267
+ if not fresh or len(found) >= total:
268
+ break
269
+ return list(found.values())
270
+
271
+
272
+ def latest(entries: list[CatalogEntry]) -> list[CatalogEntry]:
273
+ """One entry per name: the newest by parsed date, whatever order `entries` arrive in."""
274
+ newest: dict[str, CatalogEntry] = {}
275
+ for entry in sorted(entries, key=lambda e: library.date_key(e.date), reverse=True):
276
+ newest.setdefault(entry.name, entry)
277
+ return list(newest.values())
278
+
279
+
280
+ @dataclass(frozen=True)
281
+ class Metalink:
282
+ size: int
283
+ sha256: str
284
+
285
+
286
+ def parse_metalink(xml: bytes) -> Metalink:
287
+ try:
288
+ root = ET.fromstring(xml) # noqa: S314
289
+ except ET.ParseError as error:
290
+ raise ZimloreError(f"the .meta4 file is not valid XML ({error})") from error
291
+ for file in root.iter(f"{METALINK}file"): # size and hash must describe the SAME file element
292
+ size = file.find(f"{METALINK}size")
293
+ digest = next((h for h in file.iter(f"{METALINK}hash") if h.get("type") == "sha-256"), None)
294
+ text = (digest.text or "").strip().lower() if digest is not None else ""
295
+ if size is not None and re.fullmatch(r"[0-9a-f]{64}", text):
296
+ return Metalink(whole_number(size.text, "the .meta4 size", minimum=1), text)
297
+ raise ZimloreError("the .meta4 file carries no size and SHA-256", "the download cannot be verified; not fetching")
298
+
299
+
300
+ def target_name(entry: CatalogEntry) -> str:
301
+ """The file name a download of `entry` gets. It carries the edition date, so a newer edition is a different
302
+ file and an update never touches the older one. Only a plain name is accepted: no separators, no spaces or
303
+ control characters, no Windows device name (`CON.zim` would open the console, not a file)."""
304
+ name = Path(urllib.parse.urlsplit(entry.url).path).name
305
+ if not name.endswith(".zim"):
306
+ raise ZimloreError(f"{entry.url} does not name a .zim file")
307
+ if not _FILE_NAME.fullmatch(name) or name.split(".")[0].upper() in _RESERVED_NAMES:
308
+ raise ZimloreError(
309
+ f"{entry.url} names a file ({name[:60]!r}) that zimlore will not create",
310
+ "ZIM file names are letters, digits and . _ + - and may not be a Windows device name",
311
+ )
312
+ return name
313
+
314
+
315
+ def _touch(path: Path) -> None:
316
+ with contextlib.suppress(OSError): # a lock that cannot be touched only risks being reported stale
317
+ os.utime(path)
318
+
319
+
320
+ def _acquire_lock(lock: Path, stale_after: float) -> None:
321
+ """Create `lock` exclusively. An existing lock is an error either way: a young one means another download is
322
+ writing the same `.part`; an old one is reported as probably stale, with the remedy, but never removed here."""
323
+ try:
324
+ descriptor = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
325
+ except FileExistsError:
326
+ try:
327
+ age = time.time() - lock.stat().st_mtime
328
+ except OSError:
329
+ age = 0.0
330
+ if age > stale_after:
331
+ raise ZimloreError(
332
+ f"{lock} is {age / 60:.0f} minutes old: a download that died probably left it behind",
333
+ f"if no zimlore is running, delete {lock} and run the same command again; the .part resumes",
334
+ ) from None
335
+ raise ZimloreError(
336
+ f"another zimlore is downloading to {lock.with_suffix('').name} (lock {lock})",
337
+ "wait for it to finish; two downloads must not write one .part",
338
+ ) from None
339
+ with os.fdopen(descriptor, "w") as handle:
340
+ handle.write(f"pid {os.getpid()}\n")
341
+
342
+
343
+ def _header(response: Any, name: str) -> str | None:
344
+ headers = getattr(response, "headers", None)
345
+ return headers.get(name) if headers is not None else None
346
+
347
+
348
+ def _range_start(response: Any) -> int | None:
349
+ match = re.fullmatch(r"bytes (\d+)-\d+/(?:\d+|\*)", _header(response, "Content-Range") or "", re.ASCII)
350
+ return int(match.group(1)) if match else None
351
+
352
+
353
+ def _transfer(
354
+ entry: CatalogEntry,
355
+ part: Path,
356
+ lock: Path,
357
+ meta: Metalink,
358
+ opener: Opener | None,
359
+ progress: Callable[[int, int], None] | None,
360
+ deadline: float = DOWNLOAD_DEADLINE_SECONDS,
361
+ ) -> tuple[int, str]:
362
+ """Bring `part` to `meta.size` bytes and return (bytes, SHA-256 of the WHOLE file). An existing `.part` is
363
+ resumed with a Range request after its prefix is re-hashed; a mirror that refuses the range (200 instead of
364
+ 206, a wrong Content-Range, 416) makes the download start over from byte zero, never splice. Never more than
365
+ `meta.size` bytes reach the disk, and no more than `deadline` seconds are spent."""
366
+ started = time.monotonic()
367
+ for _ in range(2):
368
+ digest = hashlib.sha256()
369
+ offset = part.stat().st_size if part.exists() else 0
370
+ if offset > meta.size: # longer than the file can be: not a prefix of it
371
+ part.unlink()
372
+ offset = 0
373
+ if offset:
374
+ with part.open("rb") as prefix:
375
+ while chunk := prefix.read(1 << 20):
376
+ digest.update(chunk)
377
+ _touch(lock)
378
+ if progress:
379
+ progress(offset, meta.size)
380
+ if offset == meta.size:
381
+ return offset, digest.hexdigest() # a complete .part: nothing to fetch, the hash check decides
382
+ headers = {"User-Agent": USER_AGENT}
383
+ if offset:
384
+ headers["Range"] = f"bytes={offset}-"
385
+ request = urllib.request.Request(entry.url, headers=headers) # noqa: S310
386
+ try:
387
+ reply = (opener or _open)(request)
388
+ except urllib.error.HTTPError as error:
389
+ if error.code == 416 and offset: # the mirror's file is shorter than our part
390
+ part.unlink(missing_ok=True)
391
+ continue
392
+ raise ZimloreError(f"download of {entry.name} failed ({error})", RETRY) from error
393
+ except (OSError, http.client.HTTPException, ValueError) as error:
394
+ raise ZimloreError(f"download of {entry.name} failed ({error})", RETRY) from error
395
+ with reply as response:
396
+ status = getattr(response, "status", 200)
397
+ if offset and not (status == 206 and _range_start(response) == offset):
398
+ part.unlink(missing_ok=True)
399
+ if status != 200:
400
+ continue
401
+ digest, offset = hashlib.sha256(), 0 # Range refused: this 200 is the whole file, use it from zero
402
+ announced = _header(response, "Content-Length")
403
+ promised = int(announced) if announced and announced.isascii() and announced.isdigit() else None
404
+ if promised is not None and promised > meta.size - offset:
405
+ part.unlink(missing_ok=True)
406
+ raise ZimloreError(
407
+ f"{entry.name}: the server announces {promised} bytes; the .meta4 allows {meta.size - offset}",
408
+ "the mirror is serving a different file; nothing was kept",
409
+ )
410
+ done = offset
411
+ too_much = False
412
+ try:
413
+ with part.open("ab" if offset else "wb") as out:
414
+ while chunk := response.read(min(CHUNK, meta.size - done + 1)):
415
+ if done + len(chunk) > meta.size:
416
+ too_much = True
417
+ break
418
+ out.write(chunk)
419
+ digest.update(chunk)
420
+ done += len(chunk)
421
+ _touch(lock)
422
+ if progress:
423
+ progress(done, meta.size)
424
+ if done < meta.size and time.monotonic() - started >= deadline:
425
+ raise ZimloreError(
426
+ f"download of {entry.name} passed its {deadline:.0f} s deadline at {done} of "
427
+ f"{meta.size} bytes",
428
+ f"run the same command again: it resumes from {part}",
429
+ )
430
+ except (OSError, http.client.HTTPException, ValueError) as error:
431
+ raise ZimloreError(
432
+ f"download of {entry.name} was interrupted at {done} of {meta.size} bytes ({error})",
433
+ f"run the same command again: it resumes from {part}",
434
+ ) from error
435
+ if too_much:
436
+ part.unlink(missing_ok=True)
437
+ raise ZimloreError(
438
+ f"{entry.name}: the server sent more than the {meta.size} bytes the .meta4 allows; discarded",
439
+ "the mirror is serving a different file",
440
+ )
441
+ if promised is not None and done - offset < promised:
442
+ raise ZimloreError(
443
+ f"download of {entry.name} was cut off at {done} of {meta.size} bytes",
444
+ f"run the same command again: it resumes from {part}",
445
+ )
446
+ return done, digest.hexdigest()
447
+ raise ZimloreError(f"the mirror for {entry.name} kept refusing to serve the file", RETRY)
448
+
449
+
450
+ @dataclass(frozen=True)
451
+ class Downloaded:
452
+ path: Path
453
+ status: str # "installed" | "replaced" | "already-installed"
454
+
455
+
456
+ def _ensure_replaceable(final: Path) -> None:
457
+ """Fail BEFORE any download when `final` cannot be replaced. On Windows libzim (or any reader) holds a ZIM
458
+ without share-delete, so the file cannot be renamed or replaced until the last handle is gone: probe with a
459
+ rename and put it back."""
460
+ library.close_path(final)
461
+ probe = final.with_name(final.name + ".probe")
462
+ try:
463
+ os.rename(final, probe)
464
+ except OSError as error:
465
+ raise ZimloreError(
466
+ f"{final} is in use by another process and cannot be replaced ({error}); nothing was downloaded",
467
+ "stop any running 'zimlore mcp' server or kiwix reader that has it open, then run the same command again",
468
+ ) from error
469
+ try:
470
+ os.rename(probe, final)
471
+ except OSError as error: # should be impossible: we just renamed it
472
+ raise ZimloreError(
473
+ f"{final} was moved to {probe} and could not be moved back ({error})", f"rename {probe}"
474
+ ) from error
475
+
476
+
477
+ def _same_file(final: Path, meta: Metalink) -> bool:
478
+ record, _ = manage.read_sidecar(final)
479
+ return bool(record and final.is_file() and record.sha256 == meta.sha256 and final.stat().st_size == meta.size)
480
+
481
+
482
+ def _check_space(directory: Path, part: Path, meta: Metalink, name: str) -> None:
483
+ need = meta.size - (part.stat().st_size if part.exists() else 0)
484
+ try:
485
+ free = shutil.disk_usage(directory).free
486
+ except OSError:
487
+ return
488
+ if need > free:
489
+ raise ZimloreError(
490
+ f"{name} needs {need} more bytes and {directory} has {free} free; nothing was downloaded",
491
+ "free space: zimlore du shows the use, zimlore prune --yes drops old editions",
492
+ )
493
+
494
+
495
+ def download(
496
+ entry: CatalogEntry,
497
+ directory: Path,
498
+ *,
499
+ opener: Opener | None = None,
500
+ progress: Callable[[int, int], None] | None = None,
501
+ added_by: str = "get",
502
+ stale_lock_seconds: float = STALE_LOCK_SECONDS,
503
+ replace: bool = False,
504
+ deadline_seconds: float = DOWNLOAD_DEADLINE_SECONDS,
505
+ ) -> Path:
506
+ """`download_result(...).path`: see there."""
507
+ return download_result(
508
+ entry,
509
+ directory,
510
+ opener=opener,
511
+ progress=progress,
512
+ added_by=added_by,
513
+ stale_lock_seconds=stale_lock_seconds,
514
+ replace=replace,
515
+ deadline_seconds=deadline_seconds,
516
+ ).path
517
+
518
+
519
+ def download_result(
520
+ entry: CatalogEntry,
521
+ directory: Path,
522
+ *,
523
+ opener: Opener | None = None,
524
+ progress: Callable[[int, int], None] | None = None,
525
+ added_by: str = "get",
526
+ stale_lock_seconds: float = STALE_LOCK_SECONDS,
527
+ replace: bool = False,
528
+ deadline_seconds: float = DOWNLOAD_DEADLINE_SECONDS,
529
+ ) -> Downloaded:
530
+ """Fetch `entry` into `directory`, verifying size and SHA-256 against the `.meta4` before the file takes its
531
+ final name. A failed or interrupted download leaves a `.part` file, never a `.zim` that looks complete, and
532
+ the next call resumes it. The size cap comes from the `.meta4` before the first byte, a refusal to exceed the
533
+ free disk space too, and a `.part.lock` keeps two downloads from writing one `.part`.
534
+
535
+ An existing file of that name is never replaced unless `replace`: if its sidecar already records this
536
+ `.meta4`'s SHA-256 (and its size matches) the call is a no-op returning `already-installed`. With `replace`
537
+ on Windows a file that is open elsewhere is reported before the download starts."""
538
+ final = directory / target_name(entry)
539
+ _check_url(entry.url)
540
+ host = urllib.parse.urlsplit(entry.url).hostname
541
+ if not (is_catalogue_host(host) or host in LOCAL_HOSTS):
542
+ raise ZimloreError(
543
+ f"{entry.url} is not on {CATALOGUE_DOMAIN}; its .meta4 would come from the host that serves the file",
544
+ "zimlore downloads only catalogue entries; to install a file from elsewhere use zimlore add",
545
+ )
546
+ directory.mkdir(parents=True, exist_ok=True)
547
+ part = final.with_name(final.name + ".part")
548
+ lock = part.with_name(part.name + ".lock")
549
+ _acquire_lock(lock, stale_lock_seconds)
550
+ try:
551
+ existing = os.path.lexists(final)
552
+ if existing and replace:
553
+ _ensure_replaceable(final)
554
+ meta = parse_metalink(_get(entry.url + ".meta4", opener, pinned=True))
555
+ if existing:
556
+ if _same_file(final, meta):
557
+ return Downloaded(final, "already-installed")
558
+ if not replace:
559
+ raise ZimloreError(
560
+ f"{final} already exists and is not the file the catalogue describes; not replacing it",
561
+ "zimlore get NAME --replace replaces it (the old file is lost); otherwise move it aside",
562
+ )
563
+ _check_space(directory, part, meta, entry.name)
564
+ done, hexdigest = _transfer(entry, part, lock, meta, opener, progress, deadline_seconds)
565
+ if done != meta.size or hexdigest != meta.sha256:
566
+ part.unlink(missing_ok=True)
567
+ raise ZimloreError(
568
+ f"{entry.name}: downloaded {done} bytes with sha256 {hexdigest[:12]}…, expected "
569
+ f"{meta.size} bytes with {meta.sha256[:12]}…; the file was discarded",
570
+ "retry; if it persists the mirror is serving a different file",
571
+ )
572
+ if existing:
573
+ library.close_path(final)
574
+ try:
575
+ os.replace(part, final)
576
+ except OSError as error:
577
+ raise ZimloreError(
578
+ f"the verified download of {entry.name} could not be installed as {final} ({error})",
579
+ f"it is kept as {part}; free the target and run the same command again (no re-download is needed)",
580
+ ) from error
581
+ finally:
582
+ lock.unlink(missing_ok=True)
583
+ record = manage.Provenance(entry.url, entry.name, entry.date, meta.sha256, meta.size, manage.now(), added_by)
584
+ try:
585
+ manage.write_sidecar(final, record)
586
+ except OSError as error:
587
+ manage.sidecar_path(final).unlink(missing_ok=True) # never leave the previous file's record on this one
588
+ raise ZimloreError(
589
+ f"{final} was installed and verified but its provenance record could not be written ({error})",
590
+ "free disk space, then zimlore verify --adopt records what the file is",
591
+ ) from error
592
+ return Downloaded(final, "replaced" if existing else "installed")