bookfetch 0.3.0__py3-none-any.whl

bookfetch/__init__.py

@@ -0,0 +1,3 @@
+"""bookfetch — agent-friendly ebook finder CLI."""
+
+__version__ = "0.3.0"

bookfetch/cli.py

@@ -0,0 +1,169 @@
+"""bookfetch CLI — JSON-first output for agents, --human for people."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from . import __version__
+from .model import Book, Chapter, FetchResult
+from .sources import get_source, search_all, source_names
+from .util import FetchError, sanitize_filename
+from .util.epub import build_epub
+from .util.simplify import to_simplified
+from .util.splitters import split_headings
+
+DESC = "Agent-friendly ebook finder: routes book queries to working sources."
+
+
+def _human_search(obj: dict) -> None:
+    for i, b in enumerate(obj["results"], 1):
+        extra = f" [{b['subtitle']}]" if b.get("subtitle") else ""
+        print(f"{i}. [{b['source']}] {b['title']} (id={b['id']}){extra}")
+    if obj.get("errors"):
+        for src, err in obj["errors"].items():
+            print(f"   ! {src}: {err}", file=sys.stderr)
+
+
+def _human_get(obj: dict) -> None:
+    r = obj["result"]
+    print(f"Saved: {r['out_path']}")
+    extra = f" | {len(r['chapters'])} chapters" if r.get("chapters") else ""
+    print(f"  {r['title']} | {r['lines']} paragraphs | {r['chars']} chars | {r['format']}{extra}")
+
+
+def _ensure_chapters(fr: FetchResult) -> list[Chapter]:
+    """Chapters from the source, or heading-split, or one whole-text chapter."""
+    if fr.chapters:
+        return list(fr.chapters)
+    chs = split_headings(fr.content.splitlines())
+    if chs:
+        return chs
+    return [Chapter(title=fr.title or fr.id, text=fr.content)]
+
+
+def _render_get(src, book: Book, args) -> FetchResult:
+    """Fetch -> optional simplify -> render txt/epub under --out.
+
+    Binary sources (libgen etc.) return FetchResult.raw and are saved
+    byte-for-byte; text-only flags (--simplify/--split/--format) reject them.
+    """
+    fr = src.fetch(book)
+
+    if fr.raw is not None:  # binary passthrough: save the original file
+        if args.simplify or args.split or args.format != "txt":
+            raise ValueError(f"源 {fr.source} 是二进制原文件（.{fr.format}），不支持 --simplify/--split/--format")
+        fname = sanitize_filename(fr.title) or fr.id
+        out_dir = Path(args.out)
+        out_dir.mkdir(parents=True, exist_ok=True)
+        path = out_dir / f"{fname}.{fr.format}"
+        path.write_bytes(fr.raw)
+        fr.out_path = str(path)
+        return fr
+
+    chapters = _ensure_chapters(fr)
+
+    if args.simplify:
+        chapters = [Chapter(title=to_simplified(c.title), text=to_simplified(c.text)) for c in chapters]
+        fr.title = to_simplified(fr.title)
+
+    fname = sanitize_filename(fr.title) or fr.id
+    out_dir = Path(args.out)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    if args.format == "epub":
+        path = build_epub(fr.title, chapters, out_dir / f"{fname}.epub")
+        text_chars = sum(len(c.text) for c in chapters)
+        n_lines = sum(len(c.text.splitlines()) for c in chapters)
+    else:  # txt
+        if args.split and len(chapters) > 1:
+            parts = []
+            for i, c in enumerate(chapters, 1):
+                head = c.title or f"第{i}部分"
+                parts.append(f"=== {head} ===\n{c.text}")
+            text = "\n\n".join(parts) + "\n"
+        elif args.simplify:
+            text = "\n".join(c.text for c in chapters) + "\n"
+        else:
+            text = fr.content  # byte-identical to the fetched text
+        path = out_dir / f"{fname}.txt"
+        path.write_text(text, encoding="utf-8")
+        text_chars = len(text)
+        n_lines = len(text.splitlines())
+
+    fr.out_path = str(path)
+    fr.format = args.format
+    fr.chars = text_chars
+    fr.lines = n_lines
+    return fr
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(prog="bookfetch", description=DESC)
+    p.add_argument("--version", action="version", version=f"bookfetch {__version__}")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    sp = sub.add_parser("search", help="search sources for a book")
+    sp.add_argument("query", help="book title or keywords (Chinese OK)")
+    sp.add_argument("--source", action="append", default=None, help="only search this source (repeatable)")
+    sp.add_argument("--limit", type=int, default=20, help="max results (default 20)")
+    sp.add_argument("--human", action="store_true", help="human-readable output")
+
+    gp = sub.add_parser("get", help="download a book edition (id from search)")
+    gp.add_argument("source", help="source name, e.g. ctext")
+    gp.add_argument("id", help="edition id from search results")
+    gp.add_argument("--title", default="", help="optional title override for the output filename")
+    gp.add_argument("--out", default=".", help="output directory (default: current dir)")
+    gp.add_argument(
+        "--format",
+        choices=["txt", "epub"],
+        default="txt",
+        help="output format (default: txt; epub needs no extra deps)",
+    )
+    gp.add_argument(
+        "--split",
+        action="store_true",
+        help="insert '=== 章节 ===' separators into txt output (epub is always split)",
+    )
+    gp.add_argument(
+        "--simplify",
+        action="store_true",
+        help="convert Traditional Chinese to Simplified (requires the [simp] extra: OpenCC)",
+    )
+    gp.add_argument("--human", action="store_true", help="human-readable output")
+
+    args = p.parse_args(argv)
+    try:
+        if args.cmd == "search":
+            results, errors = search_all(args.query, args.source, args.limit)
+            obj = {
+                "cmd": "search",
+                "query": args.query,
+                "results": [b.to_dict() for b in results],
+                "count": len(results),
+                "errors": errors,
+            }
+        else:  # get
+            src = get_source(args.source)
+            if src is None:
+                raise ValueError(f"unknown source {args.source!r} (known: {', '.join(source_names())})")
+            book = Book(source=args.source, id=args.id, title=args.title)
+            fr = _render_get(src, book, args)
+            obj = {"cmd": "get", "result": fr.to_dict()}
+
+        if getattr(args, "human", False) and args.cmd == "search":
+            _human_search(obj)
+        elif getattr(args, "human", False) and args.cmd == "get":
+            _human_get(obj)
+        else:
+            print(json.dumps(obj, ensure_ascii=False))
+        return 0
+    except (ValueError, FetchError) as e:
+        print(json.dumps({"cmd": getattr(args, "cmd", None), "error": str(e)}, ensure_ascii=False))
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bookfetch/model.py

@@ -0,0 +1,64 @@
+"""Shared result models."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+
+
+@dataclass
+class Book:
+    """One downloadable edition found by a source."""
+
+    source: str
+    id: str
+    title: str
+    url: str = ""
+    subtitle: str = ""
+    format_hint: str = "txt"
+    extra: dict = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class Chapter:
+    """A titled section of a fetched book (source pages, 《》 headings, ...)."""
+
+    title: str
+    text: str
+
+
+@dataclass
+class FetchResult:
+    """Parsed content of a downloaded Book.
+
+    Sources return content + optional chapter structure WITHOUT writing files;
+    the CLI renders the requested format (txt/epub) and sets ``out_path``.
+    ``content`` is the plain merged text: joining each chapter's ``text`` with
+    ``"\\n"`` reproduces it exactly (chapters are ordered slices of lines), so
+    format rendering never loses or reorders anything.
+    """
+
+    source: str
+    id: str
+    title: str
+    out_path: str = ""
+    chars: int = 0
+    lines: int = 0
+    format: str = "txt"
+    content: str = ""
+    chapters: list[Chapter] | None = None
+    raw: bytes | None = None  # binary sources (libgen): file bytes, no text pipeline
+
+    def to_dict(self) -> dict:
+        return {
+            "source": self.source,
+            "id": self.id,
+            "title": self.title,
+            "out_path": self.out_path,
+            "chars": self.chars,
+            "lines": self.lines,
+            "format": self.format,
+            "chapters": [c.title for c in self.chapters] if self.chapters else None,
+        }

bookfetch/sources/__init__.py

@@ -0,0 +1,49 @@
+"""Source registry."""
+
+from __future__ import annotations
+
+from ..model import Book
+from .base import Source
+from .ctext import Ctext
+from .github import GithubBooks
+from .libgen import Libgen
+from .wikisource import Wikisource
+
+_REGISTRY: dict[str, Source] = {}
+
+
+def _register(src: Source) -> Source:
+    _REGISTRY[src.name] = src
+    return src
+
+
+_register(Ctext())
+_register(GithubBooks())
+_register(Wikisource("zh"))
+_register(Wikisource("en"))
+_register(Libgen())
+
+
+def get_source(name: str) -> Source | None:
+    return _REGISTRY.get(name)
+
+
+def source_names() -> list[str]:
+    return list(_REGISTRY)
+
+
+def search_all(query: str, names: list[str] | None = None, limit: int = 20):
+    """Search across sources. Returns (results, errors_by_source)."""
+    names = names or source_names()
+    results: list[Book] = []
+    errors: dict[str, str] = {}
+    for n in names:
+        src = _REGISTRY.get(n)
+        if src is None:
+            errors[n] = f"unknown source (known: {', '.join(source_names())})"
+            continue
+        try:
+            results.extend(src.search(query))
+        except Exception as e:  # source failure must not kill the whole search
+            errors[n] = f"{type(e).__name__}: {e}"
+    return results[:limit], errors

bookfetch/sources/base.py

@@ -0,0 +1,24 @@
+"""Source adapter interface."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from ..model import Book, FetchResult
+
+
+class Source(ABC):
+    """One book source (ctext, github, ...). Implementations must be stateless
+    except for util's built-in rate limiting."""
+
+    name: str = "base"
+
+    @abstractmethod
+    def search(self, query: str) -> list[Book]:
+        """Return editions matching query. Never raises for network issues —
+        callers surface errors via the errors dict instead."""
+
+    @abstractmethod
+    def fetch(self, book: Book) -> FetchResult:
+        """Fetch and parse one edition into content + optional chapter
+        structure. Never writes files — the CLI renders txt/epub."""

bookfetch/sources/ctext.py

@@ -0,0 +1,151 @@
+"""ctext.org source adapter (Chinese classics, punctuated full text)."""
+
+from __future__ import annotations
+
+import html
+import re
+from urllib.parse import parse_qs, quote, urlparse
+
+from ..model import Book, Chapter, FetchResult
+from ..util import FetchError, fetch
+from ..util.splitters import split_headings
+from .base import Source
+
+SEARCH_URL = "https://ctext.org/searchbooks.pl?if=gb&searchu={q}"
+RES_URL = "https://ctext.org/wiki.pl?if=gb&res={rid}"
+CHAPTER_URL = "https://ctext.org/wiki.pl?if=gb&chapter={cid}"
+
+# A paragraph row is: <tr class="result" id="pN">
+#   <td class="ctext" style="width: 60px;" ...>N <a onclick=showDic>...</td>   <- line number
+#   <td class="ctext">正文…</td>                                                 <- the text
+_TD_RE = re.compile(r'<td class="ctext"([^>]*)>(.*?)</td>', re.S)
+_NUM_CELL = re.compile(r"width:\s*60px")
+_ANCHOR_RE = re.compile(r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', re.S)
+_AUTHOR_RE = re.compile(r'<span style="font-weight: bold;">(.*?)</span>', re.S)
+_CHAPTER_RE = re.compile(r'href="[^"]*chapter=(\d+)"[^>]*>(.*?)</a>', re.S)
+
+_TAG_RE = re.compile(r"<[^>]+>")
+
+
+def _strip_tags(s: str) -> str:
+    return re.sub(r"\s+", " ", html.unescape(_TAG_RE.sub("", s))).strip()
+
+
+def _parse_search_page(page: str) -> list[Book]:
+    """Parse a ctext 書名檢索 result page into Book records."""
+    books: list[Book] = []
+    for li in re.split(r"<li", page)[1:]:
+        m = _ANCHOR_RE.search(li)
+        if not m:
+            continue
+        href = html.unescape(m.group(1))
+        title = _strip_tags(m.group(2))
+        if "wiki.pl" not in href or "res=" not in href:
+            continue  # only wiki text editions are downloadable in V1
+        rid = parse_qs(urlparse(href).query).get("res", [""])[0]
+        if not rid:
+            continue
+        author = ""
+        am = _AUTHOR_RE.search(li)
+        if am:
+            author = _strip_tags(am.group(1))
+            # "（宋）徐子平" -> "徐子平" (drop the era wrapper)
+            author = re.sub(r"^[（(][^）)]*[）)]", "", author).strip()
+        note = ""
+        # note text sits after the bold author span (usually after a <br>),
+        # up to the end of the <li> record
+        if am:
+            tail = li[am.end() :]
+            end = tail.find("</li>")
+            if end >= 0:
+                tail = tail[:end]
+            note = _strip_tags(tail)
+        books.append(
+            Book(
+                source="ctext",
+                id=rid,
+                title=title,
+                url=href,
+                subtitle=note,
+                format_hint="txt",
+                extra={"author": author} if author else {},
+            )
+        )
+    return books
+
+
+def _parse_text_cells(page: str) -> list[str]:
+    """Extract paragraph text from a ctext wiki text page (td.ctext cells)."""
+    lines: list[str] = []
+    for attrs, body in _TD_RE.findall(page):
+        if _NUM_CELL.search(attrs or ""):
+            continue  # line-number cell
+        txt = _strip_tags(body)
+        if txt:
+            lines.append(txt)
+    return lines
+
+
+def _chapter_title(anchor: str, idx: int) -> str:
+    """Wiki-page anchor text, or a neutral ordinal for unnamed pages
+    (most wiki pages of one book share the book title as anchor)."""
+    t = _strip_tags(anchor)
+    return t or f"第{idx}部分"
+
+
+class Ctext(Source):
+    name = "ctext"
+
+    def search(self, query: str) -> list[Book]:
+        """Search ctext 書名檢索. Network errors propagate to the CLI errors dict."""
+        page = fetch(SEARCH_URL.format(q=quote(query)))
+        return _parse_search_page(page)
+
+    def fetch(self, book: Book) -> FetchResult:
+        """Fetch a whole book: res page -> ordered wiki pages -> chapters.
+
+        Each wiki page becomes one or more chapters: standalone 《》/卷
+        heading lines split it into titled sections; a page without such
+        structure stays one chapter named by its anchor.
+        """
+        rid = book.id
+        if not rid.isdigit():
+            raise ValueError(f"ctext id must be a res/chapter number, got: {book.id!r}")
+
+        res_page = fetch(RES_URL.format(rid=rid))
+        anchors = list(dict.fromkeys(_CHAPTER_RE.findall(res_page)))
+        if not anchors:
+            raise FetchError(f"ctext res page {rid} lists no text chapters")
+
+        title = book.title or _strip_tags(anchors[0][1])
+        chapters: list[Chapter] = []
+        all_lines: list[str] = []
+        for idx, (cid, anchor) in enumerate(anchors, 1):
+            try:
+                page = fetch(CHAPTER_URL.format(cid=cid))
+            except FetchError as e:
+                raise FetchError(f"chapter {cid} failed: {e}") from e
+            lines = _parse_text_cells(page)
+            if not lines:
+                continue
+            all_lines.extend(lines)
+            page_chs = split_headings(lines)
+            if page_chs:
+                chapters.extend(page_chs)
+            else:
+                chapters.append(Chapter(title=_chapter_title(anchor, idx), text="\n".join(lines)))
+
+        if not all_lines:
+            raise FetchError(f"no text extracted from chapters of res {rid}")
+
+        content = "\n".join(c.text for c in chapters) + "\n"
+        return FetchResult(
+            source=self.name,
+            id=rid,
+            title=title,
+            chars=len(content),
+            lines=len(all_lines),
+            format="txt",
+            content=content,
+            chapters=chapters,
+        )

bookfetch/sources/github.py

@@ -0,0 +1,152 @@
+"""GitHub-hosted public-domain Chinese classic text collections.
+
+Design: each curated repo's blob list is fetched once via the git trees API
+and cached on disk for 7 days (unauthenticated GitHub API quota is 60/hr —
+this keeps usage at ~2 calls per repo per week). Searches match filenames
+locally; downloads go straight to raw.githubusercontent.com (not rate-limited
+the same way).
+
+The curated repo list is deliberately small and reviewed: only public-domain
+text collections.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+from urllib.parse import quote
+
+from ..model import Book, FetchResult
+from ..util import FetchError, fetch
+from ..util.splitters import split_headings
+from .base import Source
+
+_REPOS: list[dict] = [
+    {
+        "repo": "mymmsc/books",
+        "branch": "master",
+        "note": "综合资料库，含《国学/八字 - 渊海子平.txt》等公版古籍文本",
+    },
+]
+
+_TREE_URL = "https://api.github.com/repos/{repo}/git/trees/HEAD?recursive=1"
+_RAW_URL = "https://raw.githubusercontent.com/{repo}/{branch}/{path}"
+_BLOB_URL = "https://github.com/{repo}/blob/{branch}/{path}"
+_CACHE_TTL = 7 * 24 * 3600
+
+
+def _cache_dir() -> Path:
+    base = os.environ.get("BOOKFETCH_CACHE") or Path.home() / ".cache" / "bookfetch"
+    d = Path(base)
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def _paths_from_tree(tree: dict) -> list[str]:
+    return [t["path"] for t in tree.get("tree", []) if t.get("type") == "blob"]
+
+
+def _title_matches(query: str, path: str) -> bool:
+    """Loose filename match: query contained in basename or vice versa."""
+    if not path.lower().endswith(".txt"):
+        return False
+    base = path.rsplit("/", 1)[-1][:-4].strip()
+    q = query.strip()
+    if not q or not base:
+        return False
+    return q.lower() in base.lower() or base.lower() in q.lower()
+
+
+def _search_paths(paths: list[str], cfg: dict, query: str) -> list[Book]:
+    books: list[Book] = []
+    repo = cfg["repo"]
+    branch = cfg["branch"]
+    for path in paths:
+        if not _title_matches(query, path):
+            continue
+        base = path.rsplit("/", 1)[-1][:-4].strip()
+        folder = path.rsplit("/", 1)[0] if "/" in path else ""
+        books.append(
+            Book(
+                source="github",
+                id=f"{repo}:{path}",
+                title=base,
+                url=_BLOB_URL.format(repo=repo, branch=branch, path=quote(path, safe="/")),
+                subtitle=cfg.get("note", ""),
+                format_hint="txt",
+                extra={"repo": repo, "folder": folder},
+            )
+        )
+    return books
+
+
+def _repo_paths(cfg: dict) -> list[str]:
+    """Blob paths for a repo, refreshed from the trees API at most every TTL.
+    Falls back to a stale cache when the API is rate-limited or down."""
+    repo = cfg["repo"]
+    cache = _cache_dir() / f"github_tree_{repo.replace('/', '_')}.json"
+    stale: list[str] | None = None
+    if cache.exists():
+        try:
+            data = json.loads(cache.read_text(encoding="utf-8"))
+            if time.time() - data.get("ts", 0) < _CACHE_TTL:
+                return data["paths"]
+            stale = data["paths"]
+        except Exception:
+            stale = None
+    try:
+        tree = json.loads(fetch(_TREE_URL.format(repo=repo)))
+        paths = _paths_from_tree(tree)
+        cache.write_text(json.dumps({"ts": time.time(), "paths": paths}), encoding="utf-8")
+        return paths
+    except FetchError:
+        if stale is not None:
+            return stale
+        raise
+
+
+class GithubBooks(Source):
+    name = "github"
+
+    def search(self, query: str) -> list[Book]:
+        books: list[Book] = []
+        for cfg in _REPOS:
+            try:
+                paths = _repo_paths(cfg)
+            except FetchError:
+                raise  # surfaced via registry errors dict
+            books.extend(_search_paths(paths, cfg, query))
+        return books
+
+    def fetch(self, book: Book) -> FetchResult:
+        """Fetch one raw txt file; chapter structure comes from headings."""
+        repo, sep, path = book.id.partition(":")
+        if not sep or not repo or not path:
+            raise ValueError("github id must look like 'owner/repo:path/to/book.txt'")
+        cfg = next((c for c in _REPOS if c["repo"] == repo), None)
+        if cfg is None:
+            raise ValueError(f"repo {repo!r} not in curated list: {[c['repo'] for c in _REPOS]}")
+        raw_url = _RAW_URL.format(
+            repo=repo, branch=cfg["branch"], path=quote(path, safe="/")
+        )
+        text = fetch(raw_url)  # decode handles UTF-8 / GB18030 / Big5
+        title = book.title or path.rsplit("/", 1)[-1][:-4] or repo
+        lines = text.splitlines()
+        chapters = split_headings(lines)
+        if chapters:
+            content = "\n".join(c.text for c in chapters)
+        else:
+            chapters = None
+            content = text
+        return FetchResult(
+            source=self.name,
+            id=book.id,
+            title=title,
+            chars=len(content),
+            lines=len(lines),
+            format="txt",
+            content=content,
+            chapters=chapters,
+        )