overleaf-sync 2.0.0a3__tar.gz → 2.0.0a4__tar.gz

{overleaf_sync-2.0.0a3 → overleaf_sync-2.0.0a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: overleaf-sync
-Version: 2.0.0a3
+Version: 2.0.0a4
 Summary: Overleaf Two-Way Sync Tool
 Keywords: overleaf,sync,latex,tex
 Author-email: Moritz Glöckl <moritzgloeckl@users.noreply.github.com>
@@ -115,7 +115,9 @@ Use `ols download` to compile and download your project's PDF. Specify a downloa
 moritz@github:~/test$ ols [-l/--local-only -r/--remote-only --store-path -p/--path -i/--olignore]
 ```
 
-Just calling `ols` will two-way sync your project. When there are changes both locally, and remotely you will be asked which file to keep. Using the `-l` or `-r` option you can specify to either sync local project files to Overleaf only or Overleaf files to local ones only respectively. When using these options you can also sync deleted files. If a file has been deleted it can either be deleted on the target (remote when `-l`, local when `-r`) as well, restored on the source (local when `-l`, remote when `-r`) or ignored.
+Just calling `ols` will two-way sync your project. overleaf-sync remembers what each file looked like after the last sync (in a local `.olsync-state.json` file, which is never uploaded), so it knows on which side a file changed: files changed locally are uploaded, files changed on Overleaf are downloaded. If a file was changed on both sides, or differs on the first sync, you are asked which version to keep (or to skip it). Nothing is overwritten silently.
+
+Using the `-l` or `-r` option you can specify to either sync local project files to Overleaf only or Overleaf files to local ones only respectively. When using these options you can also sync deleted files. If a file has been deleted it can either be deleted on the target (remote when `-l`, local when `-r`) as well, restored on the source (local when `-l`, remote when `-r`) or ignored.
 
 The option `--store-path` specifies the path of the cookie file created by the `login` command. If you did not change its path, you do not need to specify this argument. The `-p/--path` option allows you to specify a different sync folder than the one you're calling `ols` from. The `-i/--olignore` option allows you to specify the path of an `.olignore` file. It uses `fnmatch` internally, so it may have some similarity to `.gitignore` but doesn't work exactly the same. For example, if you wish to exclude a specific folder named `out`, you need to specify it as `out/*`. See [here](https://docs.python.org/3/library/fnmatch.html) for more information.
 

{overleaf_sync-2.0.0a3 → overleaf_sync-2.0.0a4}/README.md

@@ -86,7 +86,9 @@ Use `ols download` to compile and download your project's PDF. Specify a downloa
 moritz@github:~/test$ ols [-l/--local-only -r/--remote-only --store-path -p/--path -i/--olignore]
 ```
 
-Just calling `ols` will two-way sync your project. When there are changes both locally, and remotely you will be asked which file to keep. Using the `-l` or `-r` option you can specify to either sync local project files to Overleaf only or Overleaf files to local ones only respectively. When using these options you can also sync deleted files. If a file has been deleted it can either be deleted on the target (remote when `-l`, local when `-r`) as well, restored on the source (local when `-l`, remote when `-r`) or ignored.
+Just calling `ols` will two-way sync your project. overleaf-sync remembers what each file looked like after the last sync (in a local `.olsync-state.json` file, which is never uploaded), so it knows on which side a file changed: files changed locally are uploaded, files changed on Overleaf are downloaded. If a file was changed on both sides, or differs on the first sync, you are asked which version to keep (or to skip it). Nothing is overwritten silently.
+
+Using the `-l` or `-r` option you can specify to either sync local project files to Overleaf only or Overleaf files to local ones only respectively. When using these options you can also sync deleted files. If a file has been deleted it can either be deleted on the target (remote when `-l`, local when `-r`) as well, restored on the source (local when `-l`, remote when `-r`) or ignored.
 
 The option `--store-path` specifies the path of the cookie file created by the `login` command. If you did not change its path, you do not need to specify this argument. The `-p/--path` option allows you to specify a different sync folder than the one you're calling `ols` from. The `-i/--olignore` option allows you to specify the path of an `.olignore` file. It uses `fnmatch` internally, so it may have some similarity to `.gitignore` but doesn't work exactly the same. For example, if you wish to exclude a specific folder named `out`, you need to specify it as `out/*`. See [here](https://docs.python.org/3/library/fnmatch.html) for more information.
 

{overleaf_sync-2.0.0a3 → overleaf_sync-2.0.0a4}/olsync/__init__.py

@@ -1,3 +1,3 @@
 """Overleaf Two-Way Sync Tool"""
 
-__version__ = '2.0.0a3'
+__version__ = '2.0.0a4'

{overleaf_sync-2.0.0a3 → overleaf_sync-2.0.0a4}/olsync/olclient.py

@@ -12,6 +12,7 @@ import requests as reqs
 from bs4 import BeautifulSoup
 import json
 import ssl
+import sys
 import uuid
 import websocket
 from socketIO_client import SocketIO
@@ -36,6 +37,35 @@ COMPILE_PATH = "/project/{}/compile?enable_pdf_caching=true"  # Compile the proj
 SOCKET_IO_PATH = "/socket.io/1/"  # Real-time service handshake
 PATH_SEP = "/"  # Use hardcoded path separator for both windows and posix system
 
+# Overleaf answers with HTTP 429 (without Retry-After) when a rate limit is reached, e.g. 30 dashboard loads or 10 zip
+# downloads per minute and project. Its limits use one-minute windows, so wait this many seconds before each retry.
+RATE_LIMIT_WAITS = (5, 10, 20, 30)
+
+
+class RateLimitError(reqs.HTTPError):
+    """Overleaf's rate limit was still reached after waiting and retrying"""
+
+
+def _request(method, url, **kwargs):
+    """requests.request that waits and retries when Overleaf's rate limit is reached"""
+    for wait in RATE_LIMIT_WAITS:
+        r = reqs.request(method, url, **kwargs)
+        if r.status_code != 429:
+            return r
+        print(f"Overleaf's rate limit was reached, retrying in {wait}s ...", file=sys.stderr)
+        time.sleep(wait)
+        # Rewind uploaded files, the rejected request has already read them
+        for value in (kwargs.get("files") or {}).values():
+            file = value[1] if isinstance(value, tuple) else value
+            if hasattr(file, "seek"):
+                file.seek(0)
+    r = reqs.request(method, url, **kwargs)
+    if r.status_code == 429:
+        raise RateLimitError("Overleaf's rate limit was reached (too many requests in a short time). "
+                             "Please wait a minute and try again.", response=r)
+    return r
+
+
 class RealtimeSocket(SocketIO):
     """
     socketIO-client's SocketIO, minus its cleanup quirk: SocketIO.__del__ calls disconnect(), which opens a brand-new
@@ -136,7 +166,7 @@ class OverleafClient(object):
         Get the CSRF token from the project dashboard, e.g. after logging in with a cookie copied from a browser
         Returns: the CSRF token, or None if the session cookie is not (or no longer) valid
         """
-        projects_page = reqs.get(self._url(PROJECT_PATH), cookies=self._cookie, allow_redirects=False)
+        projects_page = _request("GET", self._url(PROJECT_PATH), cookies=self._cookie, allow_redirects=False)
         if not projects_page.ok or projects_page.is_redirect:
             return None
         csrf_tag = BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-csrfToken'})
@@ -149,7 +179,7 @@ class OverleafClient(object):
         Get all of a user's active projects (= not archived and not trashed)
         Returns: List of project objects
         """
-        projects_page = reqs.get(self._url(PROJECT_PATH), cookies=self._cookie)
+        projects_page = _request("GET", self._url(PROJECT_PATH), cookies=self._cookie)
         json_content = OverleafClient._extract_projects(projects_page.content)
         return list(OverleafClient.filter_projects(json_content))
 
@@ -160,7 +190,7 @@ class OverleafClient(object):
         Returns: project object
         """
 
-        projects_page = reqs.get(self._url(PROJECT_PATH), cookies=self._cookie)
+        projects_page = _request("GET", self._url(PROJECT_PATH), cookies=self._cookie)
         json_content = OverleafClient._extract_projects(projects_page.content)
         return next(OverleafClient.filter_projects(json_content, {"name": project_name}), None)
 
@@ -170,7 +200,7 @@ class OverleafClient(object):
         Params: project_id, the id of the project
         Returns: bytes string (zip file)
         """
-        r = reqs.get(self._url(DOWNLOAD_PATH.format(project_id)),
+        r = _request("GET", self._url(DOWNLOAD_PATH.format(project_id)),
                      stream=True, cookies=self._cookie)
         return r.content
 
@@ -193,7 +223,7 @@ class OverleafClient(object):
         headers = {
             "X-Csrf-Token": self._csrf
         }
-        r = reqs.post(self._url(FOLDER_PATH.format(project_id)),
+        r = _request("POST", self._url(FOLDER_PATH.format(project_id)),
                       cookies=self._cookie, headers=headers, json=params)
 
         if r.ok:
@@ -240,7 +270,7 @@ class OverleafClient(object):
         # (Built from whatever cookies are present rather than hardcoded names:
         # a session may not carry every cookie -- this previously raised
         # KeyError here even though overleaf_session2 alone is enough to connect.)
-        handshake = reqs.get(
+        handshake = _request("GET", 
             self._url(SOCKET_IO_PATH),
             params={"t": int(time.time() * 1000)},
             cookies=self._cookie,
@@ -348,7 +378,7 @@ class OverleafClient(object):
         }
 
         # Upload the file to the predefined folder
-        r = reqs.post(self._url(UPLOAD_PATH.format(project_id)), cookies=self._cookie, params=params, files=files)
+        r = _request("POST", self._url(UPLOAD_PATH.format(project_id)), cookies=self._cookie, params=params, files=files)
 
         # status_code is an int; comparing to str(200) was always False here,
         # so this reported failure on every call regardless of the real
@@ -392,7 +422,7 @@ class OverleafClient(object):
             "X-Csrf-Token": self._csrf
         }
 
-        r = reqs.delete(self._url(DELETE_PATH.format(project_id, entity_type, file['_id'])), cookies=self._cookie,
+        r = _request("DELETE", self._url(DELETE_PATH.format(project_id, entity_type, file['_id'])), cookies=self._cookie,
                         headers=headers, json={})
 
         # status_code is an int; comparing to str(204) was always False
@@ -421,7 +451,7 @@ class OverleafClient(object):
             "stopOnFirstError": False
         }
 
-        r = reqs.post(self._url(COMPILE_PATH.format(project_id)), cookies=self._cookie, headers=headers, json=body)
+        r = _request("POST", self._url(COMPILE_PATH.format(project_id)), cookies=self._cookie, headers=headers, json=body)
 
         if not r.ok:
             raise reqs.HTTPError()
@@ -441,7 +471,7 @@ class OverleafClient(object):
         if compile_result.get("clsiServerId"):
             params["clsiserverid"] = compile_result["clsiServerId"]
 
-        download_req = reqs.get(self._url(pdf_file['url']), cookies=self._cookie, headers=headers, params=params)
+        download_req = _request("GET", self._url(pdf_file['url']), cookies=self._cookie, headers=headers, params=params)
 
         if download_req.ok:
             return pdf_file['path'], download_req.content

{overleaf_sync-2.0.0a3 → overleaf_sync-2.0.0a4}/olsync/olsync.py

@@ -23,10 +23,12 @@ from pathlib import Path
 
 try:
     # Import for pip installation / wheel
-    from olsync.olclient import BASE_URL, OverleafClient
+    from olsync.olclient import BASE_URL, OverleafClient, RateLimitError
+    from olsync import syncstate
 except ImportError:
     # Import for development
-    from olclient import BASE_URL, OverleafClient
+    from olclient import BASE_URL, OverleafClient, RateLimitError
+    import syncstate
 
 # Session cookie names: overleaf.com, and self-hosted servers (Community Edition, "sharelatex.sid" before version 5)
 OVERLEAF_COM_SESSION_COOKIES = ["overleaf_session2"]
@@ -98,9 +100,13 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
             "Project could not be downloaded.",
             verbose)
 
-        sync = not (local or remote)
+        if not (local or remote):
+            two_way_sync(overleaf_client, project, project_infos, zip_file, olignore_path, verbose)
+            return
 
-        if remote or sync:
+        sync = False  # -l and -r below sync in one direction only
+
+        if remote:
             remote_files = zip_file.namelist()
             deleted_files = [] if sync else [f for f in olignore_keep_list(olignore_path) if f not in remote_files]
             sync_func(
@@ -117,8 +123,8 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
                 from_name="remote",
                 to_name="local",
                 verbose=verbose)
-        if local or sync:
-            # Computed once (#60), and after the remote -> local pass so newly downloaded files are included
+        if local:
+            # Computed once (#60)
             local_files = olignore_keep_list(olignore_path)
             deleted_files = [] if sync else [f for f in zip_file.namelist() if f not in local_files]
             sync_func(
@@ -326,6 +332,93 @@ def write_file(path, content):
         f.write(content)
 
 
+def two_way_sync(overleaf_client, project, project_infos, zip_file, olignore_path, verbose=False):
+    """
+    Two-way sync that decides per file, based on the state of the last sync (see syncstate.py), instead of the
+    project-wide lastUpdated timestamp. Changes on one side are synced; files changed on both sides, or differing
+    without sync history, are only overwritten after asking.
+    """
+    def read(path):
+        with open(path, 'rb') as f:
+            return f.read()
+
+    server = overleaf_client.base_url
+    remote_content = {name: zip_file.read(name) for name in zip_file.namelist() if not name.endswith("/")}
+    # Local candidates: files allowed by .olignore, plus local copies of remote files (to compare them)
+    local_paths = set(olignore_keep_list(olignore_path)) | {name for name in remote_content if os.path.isfile(name)}
+    local = {path: syncstate.file_hash(read(path)) for path in local_paths}
+    remote = {name: syncstate.file_hash(content) for name, content in remote_content.items()}
+    state = syncstate.load_state(".", server, project["id"])
+    plan = syncstate.plan_two_way(local, remote, state)
+
+    click.echo("\nSyncing files between [local] and [remote]")
+    click.echo('=' * 40)
+
+    downloads, uploads, skipped = list(plan["download"]), list(plan["upload"]), []
+    for path in plan["conflict"]:
+        reason = "was changed locally and on Overleaf since the last sync" if path in state \
+            else "differs between local and Overleaf, and there is no sync history for it yet"
+        choice = click.prompt(
+            '\n-> Conflict: <%s> %s.\nKeep the [l]ocal version, keep the [r]emote version or [s]kip?' % (path, reason),
+            default="s", type=click.Choice(['l', 'r', 's']))
+        {"l": uploads, "r": downloads, "s": skipped}[choice].append(path)
+
+    failed = []
+    for path in downloads:
+        try:
+            write_file(path, remote_content[path])
+        except OSError:
+            if verbose:
+                print(traceback.format_exc())
+            failed.append(path)
+    for path in uploads:
+        try:
+            with open(path, 'rb') as f:
+                ok = overleaf_client.upload_file(project["id"], project_infos, path, os.path.getsize(path), f)
+        except Exception:
+            if verbose:
+                print(traceback.format_exc())
+            ok = False
+        if not ok:
+            failed.append(path)
+
+    for title, paths in [("[DOWNLOAD] Following file(s) downloaded from [remote]", downloads),
+                         ("[UPLOAD] Following file(s) uploaded to [remote]", uploads),
+                         ("[SYNC] Following file(s) are up to date", plan["in_sync"]),
+                         ("[SKIP] Following conflicting file(s) have not been synced", skipped),
+                         ("[ERROR] Following file(s) could not be synced", failed)]:
+        click.echo("\n" + title)
+        for path in paths:
+            click.echo("\t%s" % path)
+
+    # Record what both sides look like now. After uploads, read the project again, so that changes Overleaf makes
+    # to uploaded files (e.g. line endings) are not mistaken for remote edits next time.
+    if set(uploads) - set(failed):
+        try:
+            fresh = zipfile.ZipFile(io.BytesIO(overleaf_client.download_project(project["id"])))
+            remote_content = {name: fresh.read(name) for name in fresh.namelist() if not name.endswith("/")}
+        except Exception:
+            if verbose:
+                print(traceback.format_exc())
+            for path in set(uploads) - set(failed):
+                remote_content[path] = read(path)
+    new_state = {}
+    for path in sorted(set(local_paths) | set(remote_content) | set(downloads)):
+        if path in skipped or path in failed:
+            if path in state:
+                new_state[path] = state[path]
+        elif path in remote_content and os.path.isfile(path):
+            new_state[path] = {"local": syncstate.file_hash(read(path)),
+                               "remote": syncstate.file_hash(remote_content[path])}
+    syncstate.save_state(".", server, project["id"], new_state)
+
+    click.echo("")
+    if failed:
+        raise click.ClickException("%d file(s) could not be synced, see above" % len(failed))
+    click.echo("✅  Synced files between [local] and [remote]")
+    click.echo("")
+
+
 def sync_func(files_from, deleted_files, create_file_at_to, delete_file_at_to, create_file_at_from, from_exists_in_to,
               from_equal_to_to, from_newer_than_to, from_name,
               to_name, verbose=False):
@@ -436,6 +529,10 @@ def execute_action(action, progress_message, success_message, fail_message, verb
     with yaspin(text=progress_message, color="green") as spinner:
         try:
             success = action()
+        except RateLimitError as e:
+            # A clear reason is more helpful than the generic failure message
+            spinner.fail("💥 ")
+            raise click.ClickException(str(e))
         except:
             if verbose_error_logging:
                 print(traceback.format_exc())

overleaf_sync-2.0.0a4/olsync/syncstate.py

@@ -0,0 +1,98 @@
+"""Sync state for two-way sync"""
+##################################################
+# MIT License
+##################################################
+# File: syncstate.py
+# Description: Per-file sync state, so two-way sync can tell which side changed a file
+# Author: Moritz Glöckl
+# License: MIT
+##################################################
+
+import hashlib
+import json
+import os
+import tempfile
+from pathlib import Path
+
+# Stored in the sync folder. Dotfiles are never uploaded (glob skips them), so it stays local.
+STATE_FILE = ".olsync-state.json"
+STATE_VERSION = 1
+
+
+def file_hash(content):
+    """
+    Content hash used to detect changes
+    Params: content, the file content (bytes)
+    Returns: hex digest
+    """
+    return hashlib.sha256(content).hexdigest()
+
+
+def load_state(sync_dir, server, project_id):
+    """
+    Load the per-file state of the last successful sync of this project
+    Params: sync_dir, server, project_id
+    Returns: {path: {"local": hash, "remote": hash}}, empty if there is no (valid) state for this server and project
+    """
+    try:
+        with open(Path(sync_dir) / STATE_FILE) as f:
+            state = json.load(f)
+    except (OSError, ValueError):
+        return {}
+    if not isinstance(state, dict) or state.get("version") != STATE_VERSION \
+            or state.get("server") != server or state.get("project_id") != project_id:
+        return {}
+    return state.get("files", {})
+
+
+def save_state(sync_dir, server, project_id, files):
+    """
+    Save the per-file state atomically, so an interrupted sync never leaves a broken state file
+    Params: sync_dir, server, project_id, files ({path: {"local": hash, "remote": hash}})
+    """
+    state = {"version": STATE_VERSION, "server": server, "project_id": project_id, "files": files}
+    fd, tmp_path = tempfile.mkstemp(dir=sync_dir, prefix=STATE_FILE + ".", suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w") as f:
+            json.dump(state, f, indent=1, sort_keys=True)
+        os.replace(tmp_path, Path(sync_dir) / STATE_FILE)
+    except BaseException:
+        os.unlink(tmp_path)
+        raise
+
+
+def plan_two_way(local, remote, state):
+    """
+    Decide per file what a two-way sync has to do
+    Params:
+    local: {path: hash} of the local files
+    remote: {path: hash} of the files on Overleaf
+    state: {path: {"local": hash, "remote": hash}} from the last successful sync
+    Returns: {"upload": [...], "download": [...], "conflict": [...], "in_sync": [...]} (sorted paths)
+    """
+    plan = {"upload": [], "download": [], "conflict": [], "in_sync": []}
+    for path in sorted(set(local) | set(remote)):
+        local_hash, remote_hash, last = local.get(path), remote.get(path), state.get(path)
+        if remote_hash is None:
+            action = "upload"
+        elif local_hash is None:
+            action = "download"
+        elif local_hash == remote_hash:
+            action = "in_sync"
+        elif last is None:
+            # No sync history for this file: never guess which version is newer
+            action = "conflict"
+        else:
+            local_changed = local_hash != last.get("local")
+            remote_changed = remote_hash != last.get("remote")
+            if local_changed and remote_changed:
+                action = "conflict"
+            elif local_changed:
+                action = "upload"
+            elif remote_changed:
+                action = "download"
+            else:
+                # Differs, but neither side changed since the last sync (e.g. Overleaf normalised the file)
+                action = "in_sync"
+        plan[action].append(path)
+    return plan