overleaf-sync 2.0.0a1__tar.gz → 2.0.0a2__tar.gz

{overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: overleaf-sync
-Version: 2.0.0a1
+Version: 2.0.0a2
 Summary: Overleaf Two-Way Sync Tool
 Keywords: overleaf,sync,latex,tex
 Author-email: Moritz Glöckl <moritzgloeckl@users.noreply.github.com>
@@ -74,6 +74,15 @@ It then stores your *cookie* (**not** your login credentials) in a hidden file c
 
 Keep the `.olauth` file save, as it can be used to log in into your account.
 
+#### Login without the login window
+On a server without a display, or if the login window doesn't work for your account (e.g. single sign-on via ORCID or your institution), log in with a session cookie instead:
+
+1. Log in to Overleaf in your normal browser.
+2. Open the developer tools → Application (Firefox: Storage) → Cookies → `https://www.overleaf.com` and copy the value of `overleaf_session2`.
+3. Run `ols login --cookie` and paste the value at the (hidden) prompt, or provide it via the `OLSYNC_COOKIE` environment variable.
+
+This works without installing the `gui` extra.
+
 ### Listing all projects
 ```
 moritz@github:~/test$ ols list [--store-path -v/--verbose]

{overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a2}/README.md

@@ -45,6 +45,15 @@ It then stores your *cookie* (**not** your login credentials) in a hidden file c
 
 Keep the `.olauth` file save, as it can be used to log in into your account.
 
+#### Login without the login window
+On a server without a display, or if the login window doesn't work for your account (e.g. single sign-on via ORCID or your institution), log in with a session cookie instead:
+
+1. Log in to Overleaf in your normal browser.
+2. Open the developer tools → Application (Firefox: Storage) → Cookies → `https://www.overleaf.com` and copy the value of `overleaf_session2`.
+3. Run `ols login --cookie` and paste the value at the (hidden) prompt, or provide it via the `OLSYNC_COOKIE` environment variable.
+
+This works without installing the `gui` extra.
+
 ### Listing all projects
 ```
 moritz@github:~/test$ ols list [--store-path -v/--verbose]

{overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a2}/olsync/__init__.py

@@ -1,3 +1,3 @@
 """Overleaf Two-Way Sync Tool"""
 
-__version__ = '2.0.0a1'
+__version__ = '2.0.0a2'

{overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a2}/olsync/olclient.py

@@ -31,7 +31,7 @@ PROJECT_URL = "https://www.overleaf.com/project"  # The dashboard URL
 DOWNLOAD_URL = "https://www.overleaf.com/project/{}/download/zip"
 UPLOAD_URL = "https://www.overleaf.com/project/{}/upload"  # The URL to upload files
 FOLDER_URL = "https://www.overleaf.com/project/{}/folder"  # The URL to create folders
-DELETE_URL = "https://www.overleaf.com/project/{}/doc/{}"  # The URL to delete files
+DELETE_URL = "https://www.overleaf.com/project/{}/{}/{}"  # The URL to delete files (entity type: doc or file)
 COMPILE_URL = "https://www.overleaf.com/project/{}/compile?enable_pdf_caching=true"  # The URL to compile the project
 BASE_URL = "https://www.overleaf.com"  # The Overleaf Base URL
 PATH_SEP = "/"  # Use hardcoded path separator for both windows and posix system
@@ -109,6 +109,19 @@ class OverleafClient(object):
             json_content = json_content.get('projects', [])
         return json_content
 
+    def fetch_csrf(self):
+        """
+        Get the CSRF token from the project dashboard, e.g. after logging in with a cookie copied from a browser
+        Returns: the CSRF token, or None if the session cookie is not (or no longer) valid
+        """
+        projects_page = reqs.get(PROJECT_URL, cookies=self._cookie, allow_redirects=False)
+        if not projects_page.ok or projects_page.is_redirect:
+            return None
+        csrf_tag = BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-csrfToken'})
+        if csrf_tag is None:
+            return None
+        return csrf_tag.get('content')
+
     def all_projects(self):
         """
         Get all of a user's active projects (= not archived and not trashed)
@@ -330,23 +343,21 @@ class OverleafClient(object):
         Returns: True on success, False on fail
         """
 
-        file = None
-
-        # The file name contains path separators, check folders
-        if PATH_SEP in file_name:
-            local_folders = file_name.split(PATH_SEP)[:-1]  # Remove last item since this is the file name
-            current_overleaf_folder = project_infos['rootFolder'][0]['folders']  # Set the current remote folder
-
-            for local_folder in local_folders:
-                for remote_folder in current_overleaf_folder:
-                    if local_folder.lower() == remote_folder['name'].lower():
-                        file = next((v for v in remote_folder['docs'] if v['name'] == file_name.split(PATH_SEP)[-1]),
-                                    None)
-                        current_overleaf_folder = remote_folder['folders']
-                        break
-        # File is in root folder
-        else:
-            file = next((v for v in project_infos['rootFolder'][0]['docs'] if v['name'] == file_name), None)
+        # Walk down to the folder that contains the file (folder names match case-insensitively, as in upload_file)
+        *local_folders, name = file_name.split(PATH_SEP)
+        current_folder = project_infos['rootFolder'][0]
+        for local_folder in local_folders:
+            current_folder = next((f for f in current_folder['folders'] if f['name'].lower() == local_folder.lower()),
+                                  None)
+            if current_folder is None:
+                return False
+
+        # Text files are "docs", binary files (images, PDFs, ...) are "fileRefs" with their own endpoint
+        file = next((v for v in current_folder.get('docs', []) if v['name'] == name), None)
+        entity_type = "doc"
+        if file is None:
+            file = next((v for v in current_folder.get('fileRefs', []) if v['name'] == name), None)
+            entity_type = "file"
 
         # File not found!
         if file is None:
@@ -356,7 +367,8 @@ class OverleafClient(object):
             "X-Csrf-Token": self._csrf
         }
 
-        r = reqs.delete(DELETE_URL.format(project_id, file['_id']), cookies=self._cookie, headers=headers, json={})
+        r = reqs.delete(DELETE_URL.format(project_id, entity_type, file['_id']), cookies=self._cookie,
+                        headers=headers, json={})
 
         # status_code is an int; comparing to str(204) was always False
         # regardless of the real result, same bug class as upload_file and
@@ -380,7 +392,7 @@ class OverleafClient(object):
             "check": "silent",
             "draft": False,
             "incrementalCompilesEnabled": True,
-            "rootDoc_id": "",
+            # No rootDoc_id: Overleaf validates it as an optional ObjectId and rejects "" (400)
             "stopOnFirstError": False
         }
 
@@ -396,7 +408,15 @@ class OverleafClient(object):
 
         pdf_file = next(v for v in compile_result['outputFiles'] if v['type'] == 'pdf')
 
-        download_req = reqs.get(BASE_URL + pdf_file['url'], cookies=self._cookie, headers=headers)
+        # The output is only served (otherwise 404) when the compile group and CLSI server are passed along,
+        # like Overleaf's own frontend does. Self-hosted servers may not send them.
+        params = {}
+        if compile_result.get("compileGroup"):
+            params["compileGroup"] = compile_result["compileGroup"]
+        if compile_result.get("clsiServerId"):
+            params["clsiserverid"] = compile_result["clsiServerId"]
+
+        download_req = reqs.get(BASE_URL + pdf_file['url'], cookies=self._cookie, headers=headers, params=params)
 
         if download_req.ok:
             return pdf_file['path'], download_req.content

{overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a2}/olsync/olsync.py

@@ -98,9 +98,11 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
         sync = not (local or remote)
 
         if remote or sync:
+            remote_files = zip_file.namelist()
+            deleted_files = [] if sync else [f for f in olignore_keep_list(olignore_path) if f not in remote_files]
             sync_func(
-                files_from=zip_file.namelist(),
-                deleted_files=[f for f in olignore_keep_list(olignore_path) if f not in zip_file.namelist() and not sync],
+                files_from=remote_files,
+                deleted_files=deleted_files,
                 create_file_at_to=lambda name: write_file(name, zip_file.read(name)),
                 delete_file_at_to=lambda name: delete_file(name),
                 create_file_at_from=lambda name: overleaf_client.upload_file(
@@ -113,9 +115,12 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
                 to_name="local",
                 verbose=verbose)
         if local or sync:
+            # Computed once (#60), and after the remote -> local pass so newly downloaded files are included
+            local_files = olignore_keep_list(olignore_path)
+            deleted_files = [] if sync else [f for f in zip_file.namelist() if f not in local_files]
             sync_func(
-                files_from=olignore_keep_list(olignore_path),
-                deleted_files=[f for f in zip_file.namelist() if f not in olignore_keep_list(olignore_path) and not sync],
+                files_from=local_files,
+                deleted_files=deleted_files,
                 create_file_at_to=lambda name: overleaf_client.upload_file(
                     project["id"], project_infos, name, os.path.getsize(name), open(name, 'rb')),
                 delete_file_at_to=lambda name: overleaf_client.delete_file(project["id"], project_infos, name),
@@ -132,11 +137,24 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
 @main.command()
 @click.option('--path', 'cookie_path', default=".olauth", type=click.Path(exists=False),
               help="Path to store the persisted Overleaf cookie.")
+@click.option('--cookie', 'use_cookie', is_flag=True,
+              help="Log in without the login window, using the `overleaf_session2` cookie from a browser that is "
+                   "logged in to Overleaf (asked for at a hidden prompt, or read from OLSYNC_COOKIE). Useful on "
+                   "servers without a display and for single sign-on accounts.")
 @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
-def login(cookie_path, verbose):
+def login(cookie_path, use_cookie, verbose):
     if os.path.isfile(cookie_path) and not click.confirm(
             'Persisted Overleaf cookie already exist. Do you want to override it?'):
         return
+    if use_cookie:
+        # Never accept the cookie as a command line argument, it would end up in the shell history
+        cookie = os.environ.get("OLSYNC_COOKIE") or click.prompt("Value of the overleaf_session2 cookie",
+                                                                 hide_input=True)
+        execute_action(lambda: cookie_login_handler(cookie.strip(), cookie_path), "Login",
+                       "Login successful. Cookie persisted as `" + click.format_filename(
+                           cookie_path) + "`. You may now sync your project.",
+                       "Login failed: the cookie is not valid (expired, or copied incompletely).", verbose)
+        return
     olbrowserlogin = load_browser_login()
     click.clear()
     execute_action(lambda: login_handler(olbrowserlogin, cookie_path), "Login",
@@ -225,15 +243,22 @@ def load_browser_login():
             # Import for pip installation / wheel
             import olsync.olbrowserlogin as olbrowserlogin
         except ImportError as e:
-            if (e.name or "").startswith("PySide6"):
+            if not (e.name or "").startswith("olsync"):
                 raise
-            # Import for development
+            # Import for development (running olsync.py from inside the olsync folder)
             import olbrowserlogin
     except ImportError as e:
-        if not (e.name or "").startswith("PySide6"):
-            raise
+        if (e.name or "").startswith("PySide6"):
+            raise click.ClickException(
+                "The login window needs PySide6. Install it with: pip install \"overleaf-sync[gui]\"\n"
+                "Alternatively, log in without the window: ols login --cookie")
+        # PySide6 is installed but can't be loaded, e.g. because another Qt installation (such as Anaconda's)
+        # conflicts with it (#59)
         raise click.ClickException(
-            "The login window needs PySide6. Install it with: pip install \"overleaf-sync[gui]\"")
+            f"The login window could not be loaded: {e}\n"
+            "This usually means another Qt installation (e.g. from Anaconda) conflicts with PySide6. Installing "
+            "overleaf-sync into a fresh virtual environment based on a non-conda Python usually helps.\n"
+            "Alternatively, log in without the window: ols login --cookie")
     return olbrowserlogin
 
 
@@ -246,6 +271,16 @@ def login_handler(olbrowserlogin, path):
     return True
 
 
+def cookie_login_handler(cookie, path):
+    cookies = {"overleaf_session2": cookie}
+    csrf = OverleafClient(cookies).fetch_csrf()
+    if csrf is None:
+        return False
+    with open(path, 'wb+') as f:
+        pickle.dump({"cookie": cookies, "csrf": csrf}, f)
+    return True
+
+
 def delete_file(path):
     _dir = os.path.dirname(path)
     if _dir == path:
@@ -410,7 +445,8 @@ def olignore_keep_list(olignore_path):
     else:
         click.echo("\n.olignore: using %s to filter items" % olignore_path)
         with open(olignore_path, 'r') as f:
-            ignore_pattern = f.read().splitlines()
+            # "./main.tex" and "main.tex" mean the same file, matched paths have no "./" prefix (#72)
+            ignore_pattern = [pattern.removeprefix("./") for pattern in f.read().splitlines()]
 
         keep_list = [f for f in files if not any(
             fnmatch.fnmatch(f, ignore) for ignore in ignore_pattern)]