xkcdkit 0.3.0__tar.gz → 0.4.0__tar.gz

{xkcdkit-0.3.0 → xkcdkit-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xkcdkit
-Version: 0.3.0
+Version: 0.4.0
 Summary: xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations.
 Keywords: xkcd,comics,cli,tui,textual
 Author: Eivind Teig
@@ -113,8 +113,8 @@ xkcd random              # a random comic
 xkcd search "woodpecker" # fuzzy search over cached titles, alt text, transcripts and explanations
 xkcd explain 614         # just the explanation, as plain text
 xkcd sync                # refresh the archive index and the latest comic
-xkcd sync --full         # download every comic and explanation (under an hour, see below)
-xkcd sync --full --no-explain  # comics only, a minute or two; explanations come later
+xkcd sync --full         # download every comic and explanation (a couple of minutes)
+xkcd sync --full --no-explain  # comics only, under a minute; explanations come later
 xkcd resync              # re-fetch comics older than 30 days or still missing an explanation
 xkcd tui                 # open the browser
 ```
@@ -272,19 +272,18 @@ async def xkcd(ctx, num: int):
 ### Being polite to explainxkcd
 
 explainxkcd is a volunteer-run wiki. Its robots policy asks for one request
-per second, so a full sync sends at most one explainxkcd request per second
-and a User-Agent that names this project and its repository. xkcd.com has no
-such limit and is fetched by several workers at once, so explanations are the
-pacing item: a complete first `sync --full` takes a little under an hour for
-the whole archive.
-
-Two ways to shorten the wait:
-
-- `xkcd sync --full --no-explain` fetches only the comics, which takes a
-  minute or two. Explanations are then filled in on demand when you open a
-  comic, or in bulk with `xkcd resync`, which is paced the same way.
-- Ctrl-C at any point is safe. Every comic is committed to the database the
-  moment it is fetched, and the next `sync --full` skips everything already
+per second, so bulk syncs send at most one explainxkcd request per second and
+a User-Agent that names this project and its repository. To stay fast anyway,
+each request asks the MediaWiki API for fifty pages at once, the maximum it
+allows, so the whole archive's explanations arrive in about seventy requests.
+xkcd.com has no such limit and is fetched by several workers at once. A
+complete first `sync --full` takes a couple of minutes.
+
+- `xkcd sync --full --no-explain` fetches only the comics, in under a minute.
+  Explanations are then filled in on demand when you open a comic, or in bulk
+  with `xkcd resync`, paced the same way.
+- Ctrl-C at any point is safe. Each batch of fifty is committed as soon as it
+  is fetched, and the next `sync --full` or `resync` skips everything already
   cached, so you can sync in several sittings.
 
 The sync walks every number from 1 to the latest comic, so it does not depend

{xkcdkit-0.3.0 → xkcdkit-0.4.0}/README.md

@@ -81,8 +81,8 @@ xkcd random              # a random comic
 xkcd search "woodpecker" # fuzzy search over cached titles, alt text, transcripts and explanations
 xkcd explain 614         # just the explanation, as plain text
 xkcd sync                # refresh the archive index and the latest comic
-xkcd sync --full         # download every comic and explanation (under an hour, see below)
-xkcd sync --full --no-explain  # comics only, a minute or two; explanations come later
+xkcd sync --full         # download every comic and explanation (a couple of minutes)
+xkcd sync --full --no-explain  # comics only, under a minute; explanations come later
 xkcd resync              # re-fetch comics older than 30 days or still missing an explanation
 xkcd tui                 # open the browser
 ```
@@ -240,19 +240,18 @@ async def xkcd(ctx, num: int):
 ### Being polite to explainxkcd
 
 explainxkcd is a volunteer-run wiki. Its robots policy asks for one request
-per second, so a full sync sends at most one explainxkcd request per second
-and a User-Agent that names this project and its repository. xkcd.com has no
-such limit and is fetched by several workers at once, so explanations are the
-pacing item: a complete first `sync --full` takes a little under an hour for
-the whole archive.
-
-Two ways to shorten the wait:
-
-- `xkcd sync --full --no-explain` fetches only the comics, which takes a
-  minute or two. Explanations are then filled in on demand when you open a
-  comic, or in bulk with `xkcd resync`, which is paced the same way.
-- Ctrl-C at any point is safe. Every comic is committed to the database the
-  moment it is fetched, and the next `sync --full` skips everything already
+per second, so bulk syncs send at most one explainxkcd request per second and
+a User-Agent that names this project and its repository. To stay fast anyway,
+each request asks the MediaWiki API for fifty pages at once, the maximum it
+allows, so the whole archive's explanations arrive in about seventy requests.
+xkcd.com has no such limit and is fetched by several workers at once. A
+complete first `sync --full` takes a couple of minutes.
+
+- `xkcd sync --full --no-explain` fetches only the comics, in under a minute.
+  Explanations are then filled in on demand when you open a comic, or in bulk
+  with `xkcd resync`, paced the same way.
+- Ctrl-C at any point is safe. Each batch of fifty is committed as soon as it
+  is fetched, and the next `sync --full` or `resync` skips everything already
   cached, so you can sync in several sittings.
 
 The sync walks every number from 1 to the latest comic, so it does not depend

{xkcdkit-0.3.0 → xkcdkit-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "xkcdkit"
-version = "0.3.0"
+version = "0.4.0"
 description = "xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations."
 readme = "README.md"
 license = "MIT"

{xkcdkit-0.3.0 → xkcdkit-0.4.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "xkcdkit"
-version = "0.3.0"
+version = "0.4.0"
 description = "xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations."
 readme = "README.md"
 license = "MIT"

{xkcdkit-0.3.0 → xkcdkit-0.4.0}/src/xkcdkit/client.py

@@ -13,7 +13,14 @@ import httpx2
 
 from xkcdkit import ComicNotFound, XKCDError
 from xkcdkit.models import ArchiveEntry, Comic
-from xkcdkit.sources import fetch_archive, fetch_comic, fetch_explanation, user_agent
+from xkcdkit.sources import (
+    EXPLAIN_BATCH,
+    fetch_archive,
+    fetch_comic,
+    fetch_explanation,
+    fetch_explanations,
+    user_agent,
+)
 from xkcdkit.store import Store, utc_now
 
 Progress = Callable[[int, int], None]
@@ -72,6 +79,10 @@ class AsyncXKCD:
         """Attach the explainxkcd page when it exists and stamp the sync time."""
         found = await fetch_explanation(self.http, comic.num)
         self.last_explain = time.monotonic()
+        return self.attach(comic, found)
+
+    def attach(self, comic: Comic, found: tuple[str, int] | None) -> Comic:
+        """Copy a comic with explanation data (or a note of its absence) and a fresh sync time."""
         wikitext, revid = found if found else (None, None)
         return comic.model_copy(
             update={
@@ -155,34 +166,38 @@ class AsyncXKCD:
         return await self.fetch_many(self.store.stale(max_age), progress)
 
     async def fetch_many(self, nums: list[int], progress: Progress | None, explain: bool = True) -> int:
-        """Fetch and store comics with several workers; explainxkcd is paced to one request at a time.
+        """Fetch and store comics in batches of ``EXPLAIN_BATCH``.
 
-        xkcd.com requests overlap freely. Each stored comic is committed at once, so an
-        interrupted run keeps everything finished so far. Numbers xkcd lacks are skipped.
+        Within a batch xkcd.com is fetched concurrently and explainxkcd answers for the
+        whole batch in one paced request. Each batch is committed as a unit, so an
+        interrupted run keeps every finished batch. Numbers xkcd lacks are skipped.
         """
-        pending, done, fetched = list(reversed(nums)), 0, 0
+        fetched = 0
+        for start in range(0, len(nums), EXPLAIN_BATCH):
+            comics = await self.fetch_comics(nums[start : start + EXPLAIN_BATCH])
+            if explain and comics:
+                await self.pace()
+                found = await fetch_explanations(self.http, [comic.num for comic in comics])
+                comics = [self.attach(comic, found[comic.num]) for comic in comics]
+            for comic in comics:
+                self.store.replace(comic)
+            fetched += len(comics)
+            if progress:
+                progress(min(start + EXPLAIN_BATCH, len(nums)), len(nums))
+        return fetched
+
+    async def fetch_comics(self, nums: list[int]) -> list[Comic]:
+        """Fetch several comics from xkcd.com, a few at a time, skipping numbers it lacks."""
+        limit = asyncio.Semaphore(WORKERS)
 
-        async def worker() -> None:
-            nonlocal done, fetched
-            while pending:
-                num = pending.pop()
+        async def one(num: int) -> Comic | None:
+            async with limit:
                 try:
-                    comic = await fetch_comic(self.http, num)
-                    if explain:
-                        await self.pace()
-                        comic = await self.complete(comic)
-                    self.store.replace(comic)
-                    fetched += 1
+                    return await fetch_comic(self.http, num)
                 except ComicNotFound:
-                    pass
-                done += 1
-                if progress:
-                    progress(done, len(nums))
-
-        async with asyncio.TaskGroup() as group:
-            for _ in range(min(WORKERS, len(nums))):
-                group.create_task(worker())
-        return fetched
+                    return None
+
+        return [comic for comic in await asyncio.gather(*(one(num) for num in nums)) if comic]
 
     async def pace(self) -> None:
         """Wait until ``explain_delay`` has passed since the previous explainxkcd request."""

{xkcdkit-0.3.0 → xkcdkit-0.4.0}/src/xkcdkit/sources.py

@@ -15,6 +15,7 @@ from xkcdkit.models import ArchiveEntry, Comic
 
 XKCD = "https://xkcd.com"
 EXPLAIN_API = "https://www.explainxkcd.com/wiki/api.php"
+EXPLAIN_BATCH = 50  # pages per API request allowed for ordinary users
 ARCHIVE_LINK = re.compile(r'<a href="/(\d+)/" title="([^"]+)">(.*?)</a>', re.DOTALL)
 HTML_TAG = re.compile(r"</?[a-z][a-z0-9]*(?:\s[^>]*)?>")
 
@@ -59,15 +60,23 @@ async def fetch_archive(client: httpx2.AsyncClient) -> list[ArchiveEntry]:
     return parse_archive(response.text)
 
 
-async def fetch_explanation(client: httpx2.AsyncClient, num: int) -> tuple[str, int] | None:
-    """Fetch the explainxkcd page for ``num`` as ``(wikitext, revid)``, or None if absent."""
+async def fetch_explanations(
+    client: httpx2.AsyncClient, nums: list[int]
+) -> dict[int, tuple[str, int] | None]:
+    """Fetch up to ``EXPLAIN_BATCH`` explainxkcd pages in one request, keyed by comic number.
+
+    The wiki pages are named ``614: Woodpecker``; the bare number redirects there, and
+    the API resolves that for us. Numbers without a page map to None.
+    """
     response = await request(
         client,
         EXPLAIN_API,
-        action="parse",
-        page=str(num),
+        action="query",
+        titles="|".join(str(num) for num in nums),
         redirects=1,
-        prop="wikitext|revid",
+        prop="revisions",
+        rvprop="ids|content",
+        rvslots="main",
         format="json",
         formatversion=2,
     )
@@ -75,10 +84,23 @@ async def fetch_explanation(client: httpx2.AsyncClient, num: int) -> tuple[str,
         raise XKCDError(f"explainxkcd: HTTP {response.status_code}")
     payload = response.json()
     if "error" in payload:
-        if payload["error"].get("code") == "missingtitle":
-            return None
         raise XKCDError(f"explainxkcd: {payload['error'].get('info', 'unknown error')}")
-    return payload["parse"]["wikitext"], int(payload["parse"]["revid"])
+    target = {r["from"]: r["to"] for r in payload["query"].get("redirects", [])}
+    pages = {page["title"]: page for page in payload["query"]["pages"]}
+    return {num: page_content(pages.get(target.get(str(num), str(num)))) for num in nums}
+
+
+def page_content(page: dict[str, Any] | None) -> tuple[str, int] | None:
+    """The latest revision's wikitext and id, or None for an absent page."""
+    if not page or page.get("missing"):
+        return None
+    revision = page["revisions"][0]
+    return revision["slots"]["main"]["content"], int(revision["revid"])
+
+
+async def fetch_explanation(client: httpx2.AsyncClient, num: int) -> tuple[str, int] | None:
+    """Fetch the explainxkcd page for ``num`` as ``(wikitext, revid)``, or None if absent."""
+    return (await fetch_explanations(client, [num]))[num]
 
 
 async def fetch_image(client: httpx2.AsyncClient, comic: Comic) -> bytes: