xkcdkit 0.2.0__tar.gz → 0.4.0__tar.gz

{xkcdkit-0.2.0 → xkcdkit-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xkcdkit
-Version: 0.2.0
+Version: 0.4.0
 Summary: xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations.
 Keywords: xkcd,comics,cli,tui,textual
 Author: Eivind Teig
@@ -113,7 +113,8 @@ xkcd random              # a random comic
 xkcd search "woodpecker" # fuzzy search over cached titles, alt text, transcripts and explanations
 xkcd explain 614         # just the explanation, as plain text
 xkcd sync                # refresh the archive index and the latest comic
-xkcd sync --full         # download every comic and explanation (about an hour, see below)
+xkcd sync --full         # download every comic and explanation (a couple of minutes)
+xkcd sync --full --no-explain  # comics only, under a minute; explanations come later
 xkcd resync              # re-fetch comics older than 30 days or still missing an explanation
 xkcd tui                 # open the browser
 ```
@@ -271,10 +272,22 @@ async def xkcd(ctx, num: int):
 ### Being polite to explainxkcd
 
 explainxkcd is a volunteer-run wiki. Its robots policy asks for one request
-per second, so a full sync fetches one explanation per second and sends a
-User-Agent that names this project and its repository. A complete first
-`sync --full` therefore takes about an hour. On-demand lookups are not
-delayed.
+per second, so bulk syncs send at most one explainxkcd request per second and
+a User-Agent that names this project and its repository. To stay fast anyway,
+each request asks the MediaWiki API for fifty pages at once, the maximum it
+allows, so the whole archive's explanations arrive in about seventy requests.
+xkcd.com has no such limit and is fetched by several workers at once. A
+complete first `sync --full` takes a couple of minutes.
+
+- `xkcd sync --full --no-explain` fetches only the comics, in under a minute.
+  Explanations are then filled in on demand when you open a comic, or in bulk
+  with `xkcd resync`, paced the same way.
+- Ctrl-C at any point is safe. Each batch of fifty is committed as soon as it
+  is fetched, and the next `sync --full` or `resync` skips everything already
+  cached, so you can sync in several sittings.
+
+The sync walks every number from 1 to the latest comic, so it does not depend
+on the archive page. On-demand lookups are never delayed.
 
 ## Terminal image support
 

{xkcdkit-0.2.0 → xkcdkit-0.4.0}/README.md

@@ -81,7 +81,8 @@ xkcd random              # a random comic
 xkcd search "woodpecker" # fuzzy search over cached titles, alt text, transcripts and explanations
 xkcd explain 614         # just the explanation, as plain text
 xkcd sync                # refresh the archive index and the latest comic
-xkcd sync --full         # download every comic and explanation (about an hour, see below)
+xkcd sync --full         # download every comic and explanation (a couple of minutes)
+xkcd sync --full --no-explain  # comics only, under a minute; explanations come later
 xkcd resync              # re-fetch comics older than 30 days or still missing an explanation
 xkcd tui                 # open the browser
 ```
@@ -239,10 +240,22 @@ async def xkcd(ctx, num: int):
 ### Being polite to explainxkcd
 
 explainxkcd is a volunteer-run wiki. Its robots policy asks for one request
-per second, so a full sync fetches one explanation per second and sends a
-User-Agent that names this project and its repository. A complete first
-`sync --full` therefore takes about an hour. On-demand lookups are not
-delayed.
+per second, so bulk syncs send at most one explainxkcd request per second and
+a User-Agent that names this project and its repository. To stay fast anyway,
+each request asks the MediaWiki API for fifty pages at once, the maximum it
+allows, so the whole archive's explanations arrive in about seventy requests.
+xkcd.com has no such limit and is fetched by several workers at once. A
+complete first `sync --full` takes a couple of minutes.
+
+- `xkcd sync --full --no-explain` fetches only the comics, in under a minute.
+  Explanations are then filled in on demand when you open a comic, or in bulk
+  with `xkcd resync`, paced the same way.
+- Ctrl-C at any point is safe. Each batch of fifty is committed as soon as it
+  is fetched, and the next `sync --full` or `resync` skips everything already
+  cached, so you can sync in several sittings.
+
+The sync walks every number from 1 to the latest comic, so it does not depend
+on the archive page. On-demand lookups are never delayed.
 
 ## Terminal image support
 

{xkcdkit-0.2.0 → xkcdkit-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "xkcdkit"
-version = "0.2.0"
+version = "0.4.0"
 description = "xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations."
 readme = "README.md"
 license = "MIT"

{xkcdkit-0.2.0 → xkcdkit-0.4.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "xkcdkit"
-version = "0.2.0"
+version = "0.4.0"
 description = "xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations."
 readme = "README.md"
 license = "MIT"

{xkcdkit-0.2.0 → xkcdkit-0.4.0}/src/xkcdkit/cli.py

@@ -69,6 +69,12 @@ def run(ctx: typer.Context, action: Callable[[AsyncXKCD], Awaitable[Any]]) -> An
     except XKCDError as exc:
         errors.print(f"error: {exc}", markup=False, highlight=False)
         raise typer.Exit(1) from exc
+    except KeyboardInterrupt:
+        errors.print(
+            "Interrupted. Everything fetched so far is saved; run the command again to continue.",
+            soft_wrap=True,
+        )
+        raise typer.Exit(130) from None
 
 
 @dataclass
@@ -173,11 +179,17 @@ def explain(ctx: typer.Context, num: Num, json: Json = False) -> None:
 def sync(
     ctx: typer.Context,
     full: Annotated[bool, typer.Option("--full", help="Fetch every comic and explanation.")] = False,
+    no_explain: Annotated[
+        bool, typer.Option("--no-explain", help="Skip explainxkcd for now (much faster); resync fills it in.")
+    ] = False,
 ) -> None:
     """Refresh the archive index and latest comic; with --full, download everything."""
     with console.status("Syncing") as status:
         added: int = run(
-            ctx, lambda xk: xk.sync(full, lambda done, total: status.update(f"Syncing {done}/{total}"))
+            ctx,
+            lambda xk: xk.sync(
+                full, lambda done, total: status.update(f"Syncing {done}/{total}"), explain=not no_explain
+            ),
         )
     console.print(f"{added} new comic{'s' if added != 1 else ''} cached.")
 

{xkcdkit-0.2.0 → xkcdkit-0.4.0}/src/xkcdkit/client.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import random
+import time
 from collections.abc import Callable
 from datetime import timedelta
 from pathlib import Path
@@ -12,11 +13,19 @@ import httpx2
 
 from xkcdkit import ComicNotFound, XKCDError
 from xkcdkit.models import ArchiveEntry, Comic
-from xkcdkit.sources import fetch_archive, fetch_comic, fetch_explanation, user_agent
+from xkcdkit.sources import (
+    EXPLAIN_BATCH,
+    fetch_archive,
+    fetch_comic,
+    fetch_explanation,
+    fetch_explanations,
+    user_agent,
+)
 from xkcdkit.store import Store, utc_now
 
 Progress = Callable[[int, int], None]
 RETRY_MISSING_AFTER = timedelta(days=1)
+WORKERS = 8
 
 
 class AsyncXKCD:
@@ -32,7 +41,7 @@ class AsyncXKCD:
         db: Database path. Defaults to the platform user data directory or ``$XKCDKIT_DB``.
         client: An ``httpx2.AsyncClient`` to use instead of the built-in one.
         timeout: Request timeout in seconds for the built-in client.
-        explain_delay: Seconds to wait between explainxkcd requests during bulk syncs.
+        explain_delay: Minimum seconds between explainxkcd requests during bulk syncs.
     """
 
     def __init__(
@@ -48,6 +57,8 @@ class AsyncXKCD:
             timeout=timeout, headers={"User-Agent": user_agent()}, follow_redirects=True
         )
         self.explain_delay = explain_delay
+        self.explain_lock = asyncio.Lock()
+        self.last_explain = 0.0
 
     async def __aenter__(self) -> Self:
         """Enter the async context manager."""
@@ -67,6 +78,11 @@ class AsyncXKCD:
     async def complete(self, comic: Comic) -> Comic:
         """Attach the explainxkcd page when it exists and stamp the sync time."""
         found = await fetch_explanation(self.http, comic.num)
+        self.last_explain = time.monotonic()
+        return self.attach(comic, found)
+
+    def attach(self, comic: Comic, found: tuple[str, int] | None) -> Comic:
+        """Copy a comic with explanation data (or a note of its absence) and a fresh sync time."""
         wikitext, revid = found if found else (None, None)
         return comic.model_copy(
             update={
@@ -126,39 +142,70 @@ class AsyncXKCD:
         self.store.put_archive(entries)
         return entries
 
-    async def sync(self, full: bool = False, progress: Progress | None = None) -> int:
+    async def sync(self, full: bool = False, progress: Progress | None = None, explain: bool = True) -> int:
         """Refresh the archive index and the latest comic; with ``full`` fetch every comic.
 
         A full sync walks every number from 1 to the latest, so it does not depend
-        on the archive page. Returns the number of comics newly added to the cache.
+        on the archive page. With ``explain=False`` it skips explainxkcd, which is
+        much faster; ``resync`` or a later ``get`` fills the explanations in.
+        Returns the number of comics newly added to the cache.
         """
         before = self.store.count()
         await self.refresh_archive()
         latest = await self.latest()
         if full:
             cached = set(self.store.nums())
-            await self.fetch_many([n for n in range(1, latest.num + 1) if n not in cached], progress)
+            nums = [n for n in range(1, latest.num + 1) if n not in cached]
+            await self.fetch_many(nums, progress, explain)
             self.store.set_meta("last_full_sync_at", utc_now().isoformat())
         self.store.set_meta("last_sync_at", utc_now().isoformat())
         return self.store.count() - before
 
     async def resync(self, max_age: timedelta = timedelta(days=30), progress: Progress | None = None) -> int:
-        """Re-fetch comics older than ``max_age`` or missing an explanation. Returns the count."""
+        """Re-fetch comics older than ``max_age`` or without explanation data. Returns the count."""
         return await self.fetch_many(self.store.stale(max_age), progress)
 
-    async def fetch_many(self, nums: list[int], progress: Progress | None) -> int:
-        """Fetch and store each comic in turn, politely paced, skipping numbers xkcd lacks."""
-        done = 0
-        for index, num in enumerate(nums, start=1):
-            try:
-                self.store.replace(await self.complete(await fetch_comic(self.http, num)))
-                done += 1
-            except ComicNotFound:
-                pass
+    async def fetch_many(self, nums: list[int], progress: Progress | None, explain: bool = True) -> int:
+        """Fetch and store comics in batches of ``EXPLAIN_BATCH``.
+
+        Within a batch xkcd.com is fetched concurrently and explainxkcd answers for the
+        whole batch in one paced request. Each batch is committed as a unit, so an
+        interrupted run keeps every finished batch. Numbers xkcd lacks are skipped.
+        """
+        fetched = 0
+        for start in range(0, len(nums), EXPLAIN_BATCH):
+            comics = await self.fetch_comics(nums[start : start + EXPLAIN_BATCH])
+            if explain and comics:
+                await self.pace()
+                found = await fetch_explanations(self.http, [comic.num for comic in comics])
+                comics = [self.attach(comic, found[comic.num]) for comic in comics]
+            for comic in comics:
+                self.store.replace(comic)
+            fetched += len(comics)
             if progress:
-                progress(index, len(nums))
-            await asyncio.sleep(self.explain_delay)
-        return done
+                progress(min(start + EXPLAIN_BATCH, len(nums)), len(nums))
+        return fetched
+
+    async def fetch_comics(self, nums: list[int]) -> list[Comic]:
+        """Fetch several comics from xkcd.com, a few at a time, skipping numbers it lacks."""
+        limit = asyncio.Semaphore(WORKERS)
+
+        async def one(num: int) -> Comic | None:
+            async with limit:
+                try:
+                    return await fetch_comic(self.http, num)
+                except ComicNotFound:
+                    return None
+
+        return [comic for comic in await asyncio.gather(*(one(num) for num in nums)) if comic]
+
+    async def pace(self) -> None:
+        """Wait until ``explain_delay`` has passed since the previous explainxkcd request."""
+        async with self.explain_lock:
+            wait = self.last_explain + self.explain_delay - time.monotonic()
+            if wait > 0:
+                await asyncio.sleep(wait)
+            self.last_explain = time.monotonic()
 
 
 class XKCD:
@@ -224,9 +271,9 @@ class XKCD:
         """See ``AsyncXKCD.explain``."""
         return self.run("explain", num)  # type: ignore[no-any-return]
 
-    def sync(self, full: bool = False, progress: Progress | None = None) -> int:
+    def sync(self, full: bool = False, progress: Progress | None = None, explain: bool = True) -> int:
         """See ``AsyncXKCD.sync``."""
-        return self.run("sync", full, progress)  # type: ignore[no-any-return]
+        return self.run("sync", full, progress, explain)  # type: ignore[no-any-return]
 
     def resync(self, max_age: timedelta = timedelta(days=30), progress: Progress | None = None) -> int:
         """See ``AsyncXKCD.resync``."""

{xkcdkit-0.2.0 → xkcdkit-0.4.0}/src/xkcdkit/sources.py

@@ -15,6 +15,7 @@ from xkcdkit.models import ArchiveEntry, Comic
 
 XKCD = "https://xkcd.com"
 EXPLAIN_API = "https://www.explainxkcd.com/wiki/api.php"
+EXPLAIN_BATCH = 50  # pages per API request allowed for ordinary users
 ARCHIVE_LINK = re.compile(r'<a href="/(\d+)/" title="([^"]+)">(.*?)</a>', re.DOTALL)
 HTML_TAG = re.compile(r"</?[a-z][a-z0-9]*(?:\s[^>]*)?>")
 
@@ -59,15 +60,23 @@ async def fetch_archive(client: httpx2.AsyncClient) -> list[ArchiveEntry]:
     return parse_archive(response.text)
 
 
-async def fetch_explanation(client: httpx2.AsyncClient, num: int) -> tuple[str, int] | None:
-    """Fetch the explainxkcd page for ``num`` as ``(wikitext, revid)``, or None if absent."""
+async def fetch_explanations(
+    client: httpx2.AsyncClient, nums: list[int]
+) -> dict[int, tuple[str, int] | None]:
+    """Fetch up to ``EXPLAIN_BATCH`` explainxkcd pages in one request, keyed by comic number.
+
+    The wiki pages are named ``614: Woodpecker``; the bare number redirects there, and
+    the API resolves that for us. Numbers without a page map to None.
+    """
     response = await request(
         client,
         EXPLAIN_API,
-        action="parse",
-        page=str(num),
+        action="query",
+        titles="|".join(str(num) for num in nums),
         redirects=1,
-        prop="wikitext|revid",
+        prop="revisions",
+        rvprop="ids|content",
+        rvslots="main",
         format="json",
         formatversion=2,
     )
@@ -75,10 +84,23 @@ async def fetch_explanation(client: httpx2.AsyncClient, num: int) -> tuple[str,
         raise XKCDError(f"explainxkcd: HTTP {response.status_code}")
     payload = response.json()
     if "error" in payload:
-        if payload["error"].get("code") == "missingtitle":
-            return None
         raise XKCDError(f"explainxkcd: {payload['error'].get('info', 'unknown error')}")
-    return payload["parse"]["wikitext"], int(payload["parse"]["revid"])
+    target = {r["from"]: r["to"] for r in payload["query"].get("redirects", [])}
+    pages = {page["title"]: page for page in payload["query"]["pages"]}
+    return {num: page_content(pages.get(target.get(str(num), str(num)))) for num in nums}
+
+
+def page_content(page: dict[str, Any] | None) -> tuple[str, int] | None:
+    """The latest revision's wikitext and id, or None for an absent page."""
+    if not page or page.get("missing"):
+        return None
+    revision = page["revisions"][0]
+    return revision["slots"]["main"]["content"], int(revision["revid"])
+
+
+async def fetch_explanation(client: httpx2.AsyncClient, num: int) -> tuple[str, int] | None:
+    """Fetch the explainxkcd page for ``num`` as ``(wikitext, revid)``, or None if absent."""
+    return (await fetch_explanations(client, [num]))[num]
 
 
 async def fetch_image(client: httpx2.AsyncClient, comic: Comic) -> bytes:

{xkcdkit-0.2.0 → xkcdkit-0.4.0}/src/xkcdkit/store.py

@@ -129,10 +129,10 @@ class Store:
         return str(row[0]) if row else None
 
     def stale(self, max_age: timedelta) -> list[int]:
-        """Numbers of comics older than ``max_age`` or still missing an explanation."""
+        """Numbers of comics older than ``max_age`` or without explanation data yet."""
         cutoff = (utc_now() - max_age).isoformat()
         rows = self.db.execute(
-            "SELECT num FROM comics WHERE synced_at < ? OR explanation_missing ORDER BY num",
+            "SELECT num FROM comics WHERE synced_at < ? OR explanation IS NULL ORDER BY num",
             (cutoff,),
         )
         return [row[0] for row in rows]