studydesk 1.2.0__py3-none-any.whl

studydesk/__init__.py

@@ -0,0 +1,3 @@
+"""StudyDesk: a local-first workspace for reading research papers."""
+
+__version__ = "1.2.0"

studydesk/__main__.py

@@ -0,0 +1,3 @@
+from studydesk.cli import main
+
+raise SystemExit(main())

studydesk/cli.py

@@ -0,0 +1,312 @@
+"""`studydesk` command: serve the bundled browser app on localhost and open it."""
+
+from __future__ import annotations
+
+import argparse
+import errno
+import functools
+import http.server
+import io
+import json
+import os
+import socket
+import subprocess
+import sys
+import threading
+import webbrowser
+from pathlib import Path
+from urllib.parse import unquote, urlsplit
+
+from studydesk import __version__, lookup
+from studydesk.store import Conflict, Store, default_dir
+
+DEFAULT_PORT = 5173
+AUTHOR = "Subham Divakar"
+ISSUES = "https://github.com/shubham10divakar/studydesk/issues"
+MAX_LIBRARY = 256 << 20   # 256 MB of JSON
+MAX_PDF = 2 << 30         # 2 GB per PDF
+
+
+def say(text: str, ascii_fallback: str) -> None:
+    """Print text, or a plain-ASCII version on consoles that can't encode it (older Windows code pages)."""
+    try:
+        print(text, flush=True)
+    except UnicodeEncodeError:
+        print(ascii_fallback, flush=True)
+
+# Windows can map .js to text/plain through the registry, which browsers refuse to run.
+TYPES = {
+    ".html": "text/html; charset=utf-8",
+    ".js": "text/javascript; charset=utf-8",
+    ".mjs": "text/javascript; charset=utf-8",
+    ".css": "text/css; charset=utf-8",
+    ".json": "application/json; charset=utf-8",
+    ".svg": "image/svg+xml",
+    ".png": "image/png",
+    ".pdf": "application/pdf",
+    ".woff2": "font/woff2",
+    ".txt": "text/plain; charset=utf-8",
+}
+
+
+def open_folder(path: Path) -> None:
+    """Show the folder in the system file manager (File Explorer, Finder, or the Linux default)."""
+    if sys.platform == "win32":
+        os.startfile(str(path))  # type: ignore[attr-defined]
+    elif sys.platform == "darwin":
+        subprocess.Popen(["open", str(path)])
+    else:
+        subprocess.Popen(["xdg-open", str(path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+def static_dir() -> Path:
+    """The app files: bundled inside the wheel, or prototype/ when run from a checkout."""
+    bundled = Path(__file__).parent / "static"
+    if (bundled / "index.html").is_file():
+        return bundled
+    checkout = Path(__file__).parent.parent / "prototype"
+    if (checkout / "index.html").is_file():
+        return checkout
+    raise SystemExit("studydesk: app files are missing from this install. Reinstall with `pip install --force-reinstall studydesk`.")
+
+
+class Handler(http.server.SimpleHTTPRequestHandler):
+    """Static app files, plus the library API under /api/ (spec: docs/disk-storage.md)."""
+
+    extensions_map = {**http.server.SimpleHTTPRequestHandler.extensions_map, **TYPES}
+
+    def end_headers(self) -> None:
+        # Revalidate every load, so an upgraded install never runs a stale app.js.
+        self.send_header("Cache-Control", "no-cache")
+        super().end_headers()
+
+    # ---------- request checks ----------
+    def _allowed(self) -> bool:
+        """Only this machine's browser, at this server's own address, may talk to us.
+
+        The Host check blocks DNS rebinding (a web page whose domain resolves to 127.0.0.1); the Origin
+        check blocks other sites. CORS preflights are never answered, so no other origin can PUT/DELETE.
+        """
+        srv = self.server
+        host = (self.headers.get("Host") or "").lower()
+        origin = self.headers.get("Origin")
+        if host not in srv.allowed_hosts or (origin is not None and origin.lower() not in srv.allowed_origins):  # type: ignore[attr-defined]
+            self.send_error(403, "Forbidden")
+            return False
+        return True
+
+    def _json(self, code: int, obj) -> None:
+        body = json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json; charset=utf-8")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _length(self, limit: int) -> int | None:
+        try:
+            n = int(self.headers.get("Content-Length", ""))
+        except ValueError:
+            self._json(411, {"error": "Content-Length required"})
+            return None
+        if n < 0 or n > limit:
+            self._json(413, {"error": "too large"})
+            return None
+        return n
+
+    def _api_path(self) -> list[str] | None:
+        path = unquote(urlsplit(self.path).path)
+        if not path.startswith("/api/"):
+            return None
+        return path[len("/api/"):].strip("/").split("/")
+
+    # ---------- methods ----------
+    def do_GET(self) -> None:
+        if not self._allowed():
+            return
+        parts = self._api_path()
+        if parts is None:
+            return super().do_GET()
+        store: Store = self.server.store  # type: ignore[attr-defined]
+        if parts == ["info"]:
+            return self._json(200, {"app": "studydesk", "version": __version__, "dataDir": str(store.root), "rev": store.rev(), "platform": sys.platform})
+        if parts == ["library"]:
+            rev, papers = store.read()
+            return self._json(200, {"rev": rev, "papers": papers})
+        if len(parts) == 2 and parts[0] == "pdf":
+            try:
+                data = store.read_pdf(parts[1])
+            except KeyError:
+                return self._json(400, {"error": "bad id"})
+            if data is None:
+                return self._json(404, {"error": "no such PDF"})
+            self.send_response(200)
+            self.send_header("Content-Type", "application/pdf")
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+            return
+        self._json(404, {"error": "not found"})
+
+    def do_HEAD(self) -> None:
+        if not self._allowed():
+            return
+        if self._api_path() is not None:
+            return self._json(405, {"error": "use GET"})
+        super().do_HEAD()
+
+    def do_PUT(self) -> None:
+        if not self._allowed():
+            return
+        parts = self._api_path()
+        store: Store = self.server.store  # type: ignore[attr-defined]
+        if parts == ["library"]:
+            n = self._length(MAX_LIBRARY)
+            if n is None:
+                return
+            try:
+                body = json.loads(self.rfile.read(n).decode("utf-8"))
+                rev = store.write(str(body["baseRev"]), body["papers"])
+            except Conflict as c:
+                return self._json(409, {"rev": c.rev, "papers": c.papers})
+            except (ValueError, KeyError, TypeError, UnicodeDecodeError):
+                return self._json(400, {"error": "expected {baseRev, papers}"})
+            return self._json(200, {"rev": rev})
+        if parts and len(parts) == 2 and parts[0] == "pdf":
+            n = self._length(MAX_PDF)
+            if n is None:
+                return
+            try:
+                store.write_pdf(parts[1], self.rfile, n)
+            except KeyError:
+                return self._json(400, {"error": "bad id"})
+            except ValueError:
+                return self._json(400, {"error": "upload ended early"})
+            return self._json(200, {"ok": True})
+        self._json(404, {"error": "not found"})
+
+    def do_DELETE(self) -> None:
+        if not self._allowed():
+            return
+        parts = self._api_path()
+        if parts and len(parts) == 2 and parts[0] == "pdf":
+            try:
+                self.server.store.delete_pdf(parts[1])  # type: ignore[attr-defined]
+            except KeyError:
+                return self._json(400, {"error": "bad id"})
+            return self._json(200, {"ok": True})
+        self._json(404, {"error": "not found"})
+
+    def _from_app(self) -> bool:
+        """For actions with effects outside the library (opening a program, going online): require the app's own
+        Origin and a JSON content type, which other sites can't send without a CORS preflight we never answer."""
+        if self.headers.get("Origin") is None or not (self.headers.get("Content-Type") or "").startswith("application/json"):
+            self.send_error(403, "Forbidden")
+            return False
+        return True
+
+    def do_POST(self) -> None:
+        if not self._allowed():
+            return
+        parts = self._api_path()
+        if parts == ["open-folder"]:
+            if not self._from_app():
+                return
+            try:
+                open_folder(self.server.store.root)  # type: ignore[attr-defined]
+            except OSError as e:
+                return self._json(500, {"error": str(e.strerror or e)})
+            return self._json(200, {"ok": True})
+        if parts == ["lookup"]:
+            if not self._from_app():
+                return
+            n = self._length(64 << 10)
+            if n is None:
+                return
+            try:
+                body = json.loads(self.rfile.read(n).decode("utf-8"))
+                query, pid = str(body["query"]), str(body["id"])
+                self.server.store.pdf_path(pid)  # type: ignore[attr-defined]  # validates the id
+            except (ValueError, KeyError, TypeError, UnicodeDecodeError):
+                return self._json(400, {"error": "expected {query, id}"})
+            try:
+                meta, pdf = lookup.look_up(query)
+            except lookup.LookupError_ as e:
+                return self._json(e.status, {"error": str(e)})
+            if pdf is not None:
+                self.server.store.write_pdf(pid, io.BytesIO(pdf), len(pdf))  # type: ignore[attr-defined]
+            meta.pop("pdfUrl", None)
+            return self._json(200, {"paper": meta, "hasFile": pdf is not None})
+        self._json(405, {"error": "not supported"})
+
+    def do_OPTIONS(self) -> None:
+        # Never answer CORS preflights: other websites must not be able to write to the library.
+        self.send_error(403, "Forbidden")
+
+    def log_message(self, format: str, *args) -> None:  # quiet by default
+        if self.server.verbose:  # type: ignore[attr-defined]
+            super().log_message(format, *args)
+
+
+class Server(http.server.ThreadingHTTPServer):
+    # On Windows SO_REUSEADDR lets a second server bind the same port silently, so two copies would
+    # "run" at once. Claim the port exclusively there instead; elsewhere keep fast restarts.
+    allow_reuse_address = sys.platform != "win32"
+    daemon_threads = True
+
+    def server_bind(self) -> None:
+        if sys.platform == "win32" and hasattr(socket, "SO_EXCLUSIVEADDRUSE"):
+            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)
+        super().server_bind()
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="studydesk",
+        description="Start StudyDesk on this computer and open it in your browser.",
+    )
+    parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"port to serve on (default {DEFAULT_PORT})")
+    parser.add_argument("--data-dir", type=Path, default=None,
+                        help="folder for your library (default: $STUDYDESK_DATA, or StudyDesk in your home folder; an existing PDFDesk folder is used as it is)")
+    parser.add_argument("--no-browser", action="store_true", help="don't open a browser window")
+    parser.add_argument("--verbose", action="store_true", help="log every request")
+    parser.add_argument("--version", action="version", version=f"studydesk {__version__}")
+    args = parser.parse_args(argv)
+
+    try:
+        store = Store(args.data_dir or default_dir())
+    except OSError as e:
+        print(f"studydesk: can't use {args.data_dir or default_dir()} as the library folder: {e.strerror or e}", file=sys.stderr)
+        return 1
+    handler = functools.partial(Handler, directory=str(static_dir()))
+    try:
+        server = Server(("127.0.0.1", args.port), handler)
+    except OSError as e:
+        if e.errno in (errno.EADDRINUSE, errno.EACCES) or getattr(e, "winerror", None) in (10048, 10013):
+            print(
+                f"studydesk: port {args.port} is already in use. StudyDesk may already be running at http://localhost:{args.port}\n"
+                "Open that address, or start on another port: studydesk --port <number>\n"
+                f"(This command would use the library folder {store.root}; the server already running may use a different one.)",
+                file=sys.stderr,
+            )
+            return 1
+        raise
+    server.verbose = args.verbose  # type: ignore[attr-defined]
+    server.store = store  # type: ignore[attr-defined]
+    server.allowed_hosts = {f"localhost:{args.port}", f"127.0.0.1:{args.port}"}  # type: ignore[attr-defined]
+    server.allowed_origins = {f"http://{h}" for h in server.allowed_hosts}  # type: ignore[attr-defined]
+
+    url = f"http://localhost:{args.port}/"
+    print(f"StudyDesk {__version__} running at {url}  (Ctrl+C to stop)")
+    print(f"Library folder: {store.root}", flush=True)
+    say(f"Built by {AUTHOR} with ♥", f"Built by {AUTHOR} with <3")
+    print(f"Feedback and questions: {ISSUES}", flush=True)
+    if not args.no_browser:
+        threading.Timer(0.3, webbrowser.open, args=(url,)).start()
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        print("\nStopped.")
+    finally:
+        server.server_close()
+    return 0

studydesk/lookup.py

@@ -0,0 +1,162 @@
+"""Look up a paper by arXiv id or DOI and fetch its details (and, for arXiv, its PDF). Standard library only.
+
+Only three hosts are ever contacted: export.arxiv.org (metadata), arxiv.org (PDFs) and api.crossref.org (DOIs).
+The id is validated before it goes into a URL, so this can't be used to fetch arbitrary addresses.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+
+from studydesk import __version__
+
+USER_AGENT = f"StudyDesk/{__version__} (+https://github.com/shubham10divakar/studydesk)"
+TIMEOUT = 25
+MAX_PDF = 150 << 20  # 150 MB
+
+ARXIV_NEW = re.compile(r"(\d{4}\.\d{4,5})(v\d+)?", re.I)
+ARXIV_OLD = re.compile(r"([a-z][a-z.\-]*/\d{7})(v\d+)?", re.I)
+DOI = re.compile(r"(10\.\d{4,9}/[^\s\"<>]+)")
+
+
+class LookupError_(Exception):
+    """A lookup that failed for a reason worth telling the user. `status` is the HTTP status to answer with."""
+
+    def __init__(self, message: str, status: int = 400):
+        super().__init__(message)
+        self.status = status
+
+
+def parse_query(q: str) -> tuple[str, str]:
+    """('arxiv', '1706.03762v5') or ('doi', '10.1145/3292500.3330701') from what someone pasted."""
+    s = (q or "").strip()
+    s = re.sub(r"^(https?://)?(www\.)?", "", s, flags=re.I)
+    low = s.lower()
+    # arXiv DOIs (10.48550/arXiv.1706.03762) are arXiv papers
+    m = re.match(r"(doi\.org/|dx\.doi\.org/|doi:\s*)?10\.48550/arxiv\.(.+)$", low)
+    if m:
+        s = m.group(2)
+        low = s.lower()
+    if low.startswith(("arxiv.org/abs/", "arxiv.org/pdf/", "export.arxiv.org/abs/")):
+        s = s.split("/", 2)[2]
+        s = re.sub(r"\.pdf$", "", s, flags=re.I)
+    s = re.sub(r"^arxiv:\s*", "", s, flags=re.I)
+    for rx in (ARXIV_NEW, ARXIV_OLD):
+        m = rx.fullmatch(s)
+        if m:
+            return "arxiv", m.group(1) + (m.group(2) or "")
+    s2 = re.sub(r"^(doi\.org/|dx\.doi\.org/|doi:\s*)", "", s, flags=re.I)
+    m = DOI.fullmatch(s2.rstrip(".,;"))
+    if m:
+        return "doi", m.group(1)
+    raise LookupError_("That doesn't look like an arXiv id (like 1706.03762) or a DOI (like 10.1145/3292500.3330701).")
+
+
+def fetch(url: str, accept: str = "*/*", limit: int = 5 << 20) -> tuple[bytes, str]:
+    """GET url and return (body, content type). Replaced in tests."""
+    host = urllib.parse.urlsplit(url).hostname
+    if host not in ("export.arxiv.org", "arxiv.org", "api.crossref.org"):
+        raise LookupError_("refusing to contact " + str(host), 500)
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": accept})
+    try:
+        with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
+            body = r.read(limit + 1)
+            if len(body) > limit:
+                raise LookupError_("The file is too large to download.", 413)
+            return body, r.headers.get("Content-Type", "")
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            raise LookupError_("Not found.", 404) from e
+        raise LookupError_(f"The server answered {e.code}. Try again in a minute.", 502) from e
+    except (urllib.error.URLError, TimeoutError, OSError) as e:
+        raise LookupError_("Couldn't reach the internet to look this up. Check your connection and try again.", 502) from e
+
+
+def clean(s) -> str:
+    """Crossref text can carry HTML: tags like <i>…</i> and entities like &amp;."""
+    return " ".join(html.unescape(re.sub(r"<[^>]+>", "", str(s or ""))).split())
+
+
+def authors_line(names: list[str]) -> str:
+    """Surnames, comma separated, like the rest of the library ("Vaswani, Shazeer, et al.")."""
+    sur = [n.split()[-1] if n.split() else n for n in names if n and n.strip()]
+    return ", ".join(sur[:6] + ["et al."]) if len(sur) > 7 else ", ".join(sur)
+
+
+NS = {"a": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
+
+
+def arxiv(aid: str) -> dict:
+    body, _ = fetch("https://export.arxiv.org/api/query?" + urllib.parse.urlencode({"id_list": aid}), "application/atom+xml")
+    try:
+        root = ET.fromstring(body)
+    except ET.ParseError as e:
+        raise LookupError_("arXiv sent something StudyDesk couldn't read. Try again in a minute.", 502) from e
+    entry = root.find("a:entry", NS)
+    title = entry.findtext("a:title", "", NS) if entry is not None else ""
+    if entry is None or not title.strip() or "/api/errors" in (entry.findtext("a:id", "", NS) or ""):
+        raise LookupError_(f"arXiv has no paper {aid}.", 404)
+    names = [a.findtext("a:name", "", NS).strip() for a in entry.findall("a:author", NS)]
+    published = entry.findtext("a:published", "", NS)
+    journal = (entry.findtext("arxiv:journal_ref", "", NS) or "").strip()
+    base = re.sub(r"v\d+$", "", aid)
+    return {
+        "source": "arxiv",
+        "arxiv": base,
+        "doi": (entry.findtext("arxiv:doi", "", NS) or "").strip(),
+        "title": " ".join(title.split()),
+        "authors": authors_line(names),
+        "year": int(published[:4]) if published[:4].isdigit() else "",
+        "venue": journal or "arXiv",
+        "url": f"https://arxiv.org/abs/{base}",
+        "pdfUrl": f"https://arxiv.org/pdf/{aid}",
+    }
+
+
+def arxiv_pdf(meta: dict) -> bytes:
+    body, ctype = fetch(meta["pdfUrl"], "application/pdf", MAX_PDF)
+    if not body.startswith(b"%PDF"):
+        raise LookupError_("arXiv didn't send a PDF for this paper (it may still be processing). Try again later.", 502)
+    return body
+
+
+def crossref(doi: str) -> dict:
+    body, _ = fetch("https://api.crossref.org/works/" + urllib.parse.quote(doi, safe="/:;()"), "application/json")
+    try:
+        msg = json.loads(body.decode("utf-8"))["message"]
+    except (ValueError, KeyError) as e:
+        raise LookupError_("Crossref sent something StudyDesk couldn't read. Try again in a minute.", 502) from e
+    title = clean((msg.get("title") or [""])[0])
+    sub = clean((msg.get("subtitle") or [""])[0])
+    if sub and sub.lower() not in title.lower():
+        title = f"{title}: {sub}"
+    if not title:
+        raise LookupError_(f"Crossref has no title for {doi}.", 404)
+    names = [clean(" ".join(x for x in (a.get("given"), a.get("family")) if x) or a.get("name", "")) for a in msg.get("author") or []]
+    parts = (msg.get("issued") or msg.get("published") or {}).get("date-parts") or [[None]]
+    year = parts[0][0] if parts and parts[0] else ""
+    return {
+        "source": "doi",
+        "arxiv": "",
+        "doi": msg.get("DOI", doi),
+        "title": title,
+        "authors": authors_line(names),
+        "year": year or "",
+        "venue": clean((msg.get("container-title") or [""])[0] or msg.get("publisher") or ""),
+        "url": "https://doi.org/" + msg.get("DOI", doi),
+    }
+
+
+def look_up(query: str) -> tuple[dict, bytes | None]:
+    """Details for the paper, and its PDF when one can be downloaded (arXiv)."""
+    kind, ident = parse_query(query)
+    if kind == "arxiv":
+        meta = arxiv(ident)
+        return meta, arxiv_pdf(meta)
+    return crossref(ident), None