xkcdkit 0.2.0__tar.gz → 0.3.0__tar.gz

{xkcdkit-0.2.0 → xkcdkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xkcdkit
-Version: 0.2.0
+Version: 0.3.0
 Summary: xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations.
 Keywords: xkcd,comics,cli,tui,textual
 Author: Eivind Teig
@@ -113,7 +113,8 @@ xkcd random              # a random comic
 xkcd search "woodpecker" # fuzzy search over cached titles, alt text, transcripts and explanations
 xkcd explain 614         # just the explanation, as plain text
 xkcd sync                # refresh the archive index and the latest comic
-xkcd sync --full         # download every comic and explanation (about an hour, see below)
+xkcd sync --full         # download every comic and explanation (under an hour, see below)
+xkcd sync --full --no-explain  # comics only, a minute or two; explanations come later
 xkcd resync              # re-fetch comics older than 30 days or still missing an explanation
 xkcd tui                 # open the browser
 ```
@@ -271,10 +272,23 @@ async def xkcd(ctx, num: int):
 ### Being polite to explainxkcd
 
 explainxkcd is a volunteer-run wiki. Its robots policy asks for one request
-per second, so a full sync fetches one explanation per second and sends a
-User-Agent that names this project and its repository. A complete first
-`sync --full` therefore takes about an hour. On-demand lookups are not
-delayed.
+per second, so a full sync sends at most one explainxkcd request per second
+and a User-Agent that names this project and its repository. xkcd.com has no
+such limit and is fetched by several workers at once, so explanations are the
+pacing item: a complete first `sync --full` takes a little under an hour for
+the whole archive.
+
+Two ways to shorten the wait:
+
+- `xkcd sync --full --no-explain` fetches only the comics, which takes a
+  minute or two. Explanations are then filled in on demand when you open a
+  comic, or in bulk with `xkcd resync`, which is paced the same way.
+- Ctrl-C at any point is safe. Every comic is committed to the database the
+  moment it is fetched, and the next `sync --full` skips everything already
+  cached, so you can sync in several sittings.
+
+The sync walks every number from 1 to the latest comic, so it does not depend
+on the archive page. On-demand lookups are never delayed.
 
 ## Terminal image support
 

{xkcdkit-0.2.0 → xkcdkit-0.3.0}/README.md

@@ -81,7 +81,8 @@ xkcd random              # a random comic
 xkcd search "woodpecker" # fuzzy search over cached titles, alt text, transcripts and explanations
 xkcd explain 614         # just the explanation, as plain text
 xkcd sync                # refresh the archive index and the latest comic
-xkcd sync --full         # download every comic and explanation (about an hour, see below)
+xkcd sync --full         # download every comic and explanation (under an hour, see below)
+xkcd sync --full --no-explain  # comics only, a minute or two; explanations come later
 xkcd resync              # re-fetch comics older than 30 days or still missing an explanation
 xkcd tui                 # open the browser
 ```
@@ -239,10 +240,23 @@ async def xkcd(ctx, num: int):
 ### Being polite to explainxkcd
 
 explainxkcd is a volunteer-run wiki. Its robots policy asks for one request
-per second, so a full sync fetches one explanation per second and sends a
-User-Agent that names this project and its repository. A complete first
-`sync --full` therefore takes about an hour. On-demand lookups are not
-delayed.
+per second, so a full sync sends at most one explainxkcd request per second
+and a User-Agent that names this project and its repository. xkcd.com has no
+such limit and is fetched by several workers at once, so explanations are the
+pacing item: a complete first `sync --full` takes a little under an hour for
+the whole archive.
+
+Two ways to shorten the wait:
+
+- `xkcd sync --full --no-explain` fetches only the comics, which takes a
+  minute or two. Explanations are then filled in on demand when you open a
+  comic, or in bulk with `xkcd resync`, which is paced the same way.
+- Ctrl-C at any point is safe. Every comic is committed to the database the
+  moment it is fetched, and the next `sync --full` skips everything already
+  cached, so you can sync in several sittings.
+
+The sync walks every number from 1 to the latest comic, so it does not depend
+on the archive page. On-demand lookups are never delayed.
 
 ## Terminal image support
 

{xkcdkit-0.2.0 → xkcdkit-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "xkcdkit"
-version = "0.2.0"
+version = "0.3.0"
 description = "xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations."
 readme = "README.md"
 license = "MIT"

{xkcdkit-0.2.0 → xkcdkit-0.3.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "xkcdkit"
-version = "0.2.0"
+version = "0.3.0"
 description = "xkcd comics as a Python library, a CLI and a TUI, with a local cache and explainxkcd explanations."
 readme = "README.md"
 license = "MIT"

{xkcdkit-0.2.0 → xkcdkit-0.3.0}/src/xkcdkit/cli.py

@@ -69,6 +69,12 @@ def run(ctx: typer.Context, action: Callable[[AsyncXKCD], Awaitable[Any]]) -> An
     except XKCDError as exc:
         errors.print(f"error: {exc}", markup=False, highlight=False)
         raise typer.Exit(1) from exc
+    except KeyboardInterrupt:
+        errors.print(
+            "Interrupted. Everything fetched so far is saved; run the command again to continue.",
+            soft_wrap=True,
+        )
+        raise typer.Exit(130) from None
 
 
 @dataclass
@@ -173,11 +179,17 @@ def explain(ctx: typer.Context, num: Num, json: Json = False) -> None:
 def sync(
     ctx: typer.Context,
     full: Annotated[bool, typer.Option("--full", help="Fetch every comic and explanation.")] = False,
+    no_explain: Annotated[
+        bool, typer.Option("--no-explain", help="Skip explainxkcd for now (much faster); resync fills it in.")
+    ] = False,
 ) -> None:
     """Refresh the archive index and latest comic; with --full, download everything."""
     with console.status("Syncing") as status:
         added: int = run(
-            ctx, lambda xk: xk.sync(full, lambda done, total: status.update(f"Syncing {done}/{total}"))
+            ctx,
+            lambda xk: xk.sync(
+                full, lambda done, total: status.update(f"Syncing {done}/{total}"), explain=not no_explain
+            ),
         )
     console.print(f"{added} new comic{'s' if added != 1 else ''} cached.")
 

{xkcdkit-0.2.0 → xkcdkit-0.3.0}/src/xkcdkit/client.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import random
+import time
 from collections.abc import Callable
 from datetime import timedelta
 from pathlib import Path
@@ -17,6 +18,7 @@ from xkcdkit.store import Store, utc_now
 
 Progress = Callable[[int, int], None]
 RETRY_MISSING_AFTER = timedelta(days=1)
+WORKERS = 8
 
 
 class AsyncXKCD:
@@ -32,7 +34,7 @@ class AsyncXKCD:
         db: Database path. Defaults to the platform user data directory or ``$XKCDKIT_DB``.
         client: An ``httpx2.AsyncClient`` to use instead of the built-in one.
         timeout: Request timeout in seconds for the built-in client.
-        explain_delay: Seconds to wait between explainxkcd requests during bulk syncs.
+        explain_delay: Minimum seconds between explainxkcd requests during bulk syncs.
     """
 
     def __init__(
@@ -48,6 +50,8 @@ class AsyncXKCD:
             timeout=timeout, headers={"User-Agent": user_agent()}, follow_redirects=True
         )
         self.explain_delay = explain_delay
+        self.explain_lock = asyncio.Lock()
+        self.last_explain = 0.0
 
     async def __aenter__(self) -> Self:
         """Enter the async context manager."""
@@ -67,6 +71,7 @@ class AsyncXKCD:
     async def complete(self, comic: Comic) -> Comic:
         """Attach the explainxkcd page when it exists and stamp the sync time."""
         found = await fetch_explanation(self.http, comic.num)
+        self.last_explain = time.monotonic()
         wikitext, revid = found if found else (None, None)
         return comic.model_copy(
             update={
@@ -126,39 +131,66 @@ class AsyncXKCD:
         self.store.put_archive(entries)
         return entries
 
-    async def sync(self, full: bool = False, progress: Progress | None = None) -> int:
+    async def sync(self, full: bool = False, progress: Progress | None = None, explain: bool = True) -> int:
         """Refresh the archive index and the latest comic; with ``full`` fetch every comic.
 
         A full sync walks every number from 1 to the latest, so it does not depend
-        on the archive page. Returns the number of comics newly added to the cache.
+        on the archive page. With ``explain=False`` it skips explainxkcd, which is
+        much faster; ``resync`` or a later ``get`` fills the explanations in.
+        Returns the number of comics newly added to the cache.
         """
         before = self.store.count()
         await self.refresh_archive()
         latest = await self.latest()
         if full:
             cached = set(self.store.nums())
-            await self.fetch_many([n for n in range(1, latest.num + 1) if n not in cached], progress)
+            nums = [n for n in range(1, latest.num + 1) if n not in cached]
+            await self.fetch_many(nums, progress, explain)
             self.store.set_meta("last_full_sync_at", utc_now().isoformat())
         self.store.set_meta("last_sync_at", utc_now().isoformat())
         return self.store.count() - before
 
     async def resync(self, max_age: timedelta = timedelta(days=30), progress: Progress | None = None) -> int:
-        """Re-fetch comics older than ``max_age`` or missing an explanation. Returns the count."""
+        """Re-fetch comics older than ``max_age`` or without explanation data. Returns the count."""
         return await self.fetch_many(self.store.stale(max_age), progress)
 
-    async def fetch_many(self, nums: list[int], progress: Progress | None) -> int:
-        """Fetch and store each comic in turn, politely paced, skipping numbers xkcd lacks."""
-        done = 0
-        for index, num in enumerate(nums, start=1):
-            try:
-                self.store.replace(await self.complete(await fetch_comic(self.http, num)))
+    async def fetch_many(self, nums: list[int], progress: Progress | None, explain: bool = True) -> int:
+        """Fetch and store comics with several workers; explainxkcd is paced to one request at a time.
+
+        xkcd.com requests overlap freely. Each stored comic is committed at once, so an
+        interrupted run keeps everything finished so far. Numbers xkcd lacks are skipped.
+        """
+        pending, done, fetched = list(reversed(nums)), 0, 0
+
+        async def worker() -> None:
+            nonlocal done, fetched
+            while pending:
+                num = pending.pop()
+                try:
+                    comic = await fetch_comic(self.http, num)
+                    if explain:
+                        await self.pace()
+                        comic = await self.complete(comic)
+                    self.store.replace(comic)
+                    fetched += 1
+                except ComicNotFound:
+                    pass
                 done += 1
-            except ComicNotFound:
-                pass
-            if progress:
-                progress(index, len(nums))
-            await asyncio.sleep(self.explain_delay)
-        return done
+                if progress:
+                    progress(done, len(nums))
+
+        async with asyncio.TaskGroup() as group:
+            for _ in range(min(WORKERS, len(nums))):
+                group.create_task(worker())
+        return fetched
+
+    async def pace(self) -> None:
+        """Wait until ``explain_delay`` has passed since the previous explainxkcd request."""
+        async with self.explain_lock:
+            wait = self.last_explain + self.explain_delay - time.monotonic()
+            if wait > 0:
+                await asyncio.sleep(wait)
+            self.last_explain = time.monotonic()
 
 
 class XKCD:
@@ -224,9 +256,9 @@ class XKCD:
         """See ``AsyncXKCD.explain``."""
         return self.run("explain", num)  # type: ignore[no-any-return]
 
-    def sync(self, full: bool = False, progress: Progress | None = None) -> int:
+    def sync(self, full: bool = False, progress: Progress | None = None, explain: bool = True) -> int:
         """See ``AsyncXKCD.sync``."""
-        return self.run("sync", full, progress)  # type: ignore[no-any-return]
+        return self.run("sync", full, progress, explain)  # type: ignore[no-any-return]
 
     def resync(self, max_age: timedelta = timedelta(days=30), progress: Progress | None = None) -> int:
         """See ``AsyncXKCD.resync``."""

{xkcdkit-0.2.0 → xkcdkit-0.3.0}/src/xkcdkit/store.py

@@ -129,10 +129,10 @@ class Store:
         return str(row[0]) if row else None
 
     def stale(self, max_age: timedelta) -> list[int]:
-        """Numbers of comics older than ``max_age`` or still missing an explanation."""
+        """Numbers of comics older than ``max_age`` or without explanation data yet."""
         cutoff = (utc_now() - max_age).isoformat()
         rows = self.db.execute(
-            "SELECT num FROM comics WHERE synced_at < ? OR explanation_missing ORDER BY num",
+            "SELECT num FROM comics WHERE synced_at < ? OR explanation IS NULL ORDER BY num",
             (cutoff,),
         )
         return [row[0] for row in rows]