overleaf-sync 2.0.0a1__tar.gz → 2.0.0a3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a3}/PKG-INFO +19 -1
- {overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a3}/README.md +18 -0
- {overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a3}/olsync/__init__.py +1 -1
- {overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a3}/olsync/olbrowserlogin.py +13 -10
- {overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a3}/olsync/olclient.py +98 -53
- {overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a3}/olsync/olsync.py +100 -43
- {overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a3}/LICENSE +0 -0
- {overleaf_sync-2.0.0a1 → overleaf_sync-2.0.0a3}/pyproject.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: overleaf-sync
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.0a3
|
|
4
4
|
Summary: Overleaf Two-Way Sync Tool
|
|
5
5
|
Keywords: overleaf,sync,latex,tex
|
|
6
6
|
Author-email: Moritz Glöckl <moritzgloeckl@users.noreply.github.com>
|
|
@@ -74,6 +74,24 @@ It then stores your *cookie* (**not** your login credentials) in a hidden file c
|
|
|
74
74
|
|
|
75
75
|
Keep the `.olauth` file save, as it can be used to log in into your account.
|
|
76
76
|
|
|
77
|
+
#### Login without the login window
|
|
78
|
+
On a server without a display, or if the login window doesn't work for your account (e.g. single sign-on via ORCID or your institution), log in with a session cookie instead:
|
|
79
|
+
|
|
80
|
+
1. Log in to Overleaf in your normal browser.
|
|
81
|
+
2. Open the developer tools → Application (Firefox: Storage) → Cookies → `https://www.overleaf.com` and copy the value of `overleaf_session2` (on a self-hosted server: the cookie `overleaf.sid` of your server).
|
|
82
|
+
3. Run `ols login --cookie` and paste the value at the (hidden) prompt, or provide it via the `OLSYNC_COOKIE` environment variable.
|
|
83
|
+
|
|
84
|
+
This works without installing the `gui` extra.
|
|
85
|
+
|
|
86
|
+
### Self-hosted Overleaf servers
|
|
87
|
+
overleaf-sync also works with self-hosted Overleaf servers. It is tested against Overleaf Community Edition; Server Pro shares its code and should work too, but is untested. Pass the server's URL when logging in:
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
moritz@github:~/test$ ols login --server https://overleaf.example.com
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
The server is stored together with the login, so all other commands (`ols`, `ols list`, `ols download`) use it automatically. You can also set it with `--server` on any command or with the `OLSYNC_SERVER` environment variable. `ols login --cookie --server ...` works too; self-hosted servers name the session cookie `overleaf.sid` (`sharelatex.sid` before version 5).
|
|
94
|
+
|
|
77
95
|
### Listing all projects
|
|
78
96
|
```
|
|
79
97
|
moritz@github:~/test$ ols list [--store-path -v/--verbose]
|
|
@@ -45,6 +45,24 @@ It then stores your *cookie* (**not** your login credentials) in a hidden file c
|
|
|
45
45
|
|
|
46
46
|
Keep the `.olauth` file save, as it can be used to log in into your account.
|
|
47
47
|
|
|
48
|
+
#### Login without the login window
|
|
49
|
+
On a server without a display, or if the login window doesn't work for your account (e.g. single sign-on via ORCID or your institution), log in with a session cookie instead:
|
|
50
|
+
|
|
51
|
+
1. Log in to Overleaf in your normal browser.
|
|
52
|
+
2. Open the developer tools → Application (Firefox: Storage) → Cookies → `https://www.overleaf.com` and copy the value of `overleaf_session2` (on a self-hosted server: the cookie `overleaf.sid` of your server).
|
|
53
|
+
3. Run `ols login --cookie` and paste the value at the (hidden) prompt, or provide it via the `OLSYNC_COOKIE` environment variable.
|
|
54
|
+
|
|
55
|
+
This works without installing the `gui` extra.
|
|
56
|
+
|
|
57
|
+
### Self-hosted Overleaf servers
|
|
58
|
+
overleaf-sync also works with self-hosted Overleaf servers. It is tested against Overleaf Community Edition; Server Pro shares its code and should work too, but is untested. Pass the server's URL when logging in:
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
moritz@github:~/test$ ols login --server https://overleaf.example.com
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The server is stored together with the login, so all other commands (`ols`, `ols list`, `ols download`) use it automatically. You can also set it with `--server` on any command or with the `OLSYNC_SERVER` environment variable. `ols login --cookie --server ...` works too; self-hosted servers name the session cookie `overleaf.sid` (`sharelatex.sid` before version 5).
|
|
65
|
+
|
|
48
66
|
### Listing all projects
|
|
49
67
|
```
|
|
50
68
|
moritz@github:~/test$ ols list [--store-path -v/--verbose]
|
|
@@ -13,13 +13,14 @@ from PySide6.QtWidgets import *
|
|
|
13
13
|
from PySide6.QtWebEngineWidgets import *
|
|
14
14
|
from PySide6.QtWebEngineCore import QWebEngineProfile, QWebEngineSettings, QWebEnginePage
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
BASE_URL = "https://www.overleaf.com" # The default Overleaf server
|
|
17
|
+
LOGIN_PATH = "/login" # Where the user logs in
|
|
18
|
+
PROJECT_PATH = "/project" # The dashboard, reached after a successful login
|
|
19
19
|
# JS snippet to extract the csrfToken
|
|
20
20
|
JAVASCRIPT_CSRF_EXTRACTOR = "document.getElementsByName('ol-csrfToken')[0].content"
|
|
21
|
-
# Name of the cookies we want to extract
|
|
22
|
-
|
|
21
|
+
# Name of the cookies we want to extract: overleaf.com's session and load balancer cookies, and the session cookie of
|
|
22
|
+
# self-hosted servers (Community Edition; "sharelatex.sid" before version 5)
|
|
23
|
+
COOKIE_NAMES = ["overleaf_session2", "GCLB", "overleaf.sid", "sharelatex.sid"]
|
|
23
24
|
|
|
24
25
|
|
|
25
26
|
class OlBrowserLoginWindow(QMainWindow):
|
|
@@ -28,9 +29,11 @@ class OlBrowserLoginWindow(QMainWindow):
|
|
|
28
29
|
Opens a browser window to securely login the user and returns relevant login data.
|
|
29
30
|
"""
|
|
30
31
|
|
|
31
|
-
def __init__(self, *args, **kwargs):
|
|
32
|
+
def __init__(self, base_url=BASE_URL, *args, **kwargs):
|
|
32
33
|
super(OlBrowserLoginWindow, self).__init__(*args, **kwargs)
|
|
33
34
|
|
|
35
|
+
self._base_url = base_url.rstrip("/")
|
|
36
|
+
|
|
34
37
|
self.webview = QWebEngineView()
|
|
35
38
|
|
|
36
39
|
self._cookies = {}
|
|
@@ -46,7 +49,7 @@ class OlBrowserLoginWindow(QMainWindow):
|
|
|
46
49
|
|
|
47
50
|
webpage = QWebEnginePage(self.profile, self)
|
|
48
51
|
self.webview.setPage(webpage)
|
|
49
|
-
self.webview.load(QUrl.fromUserInput(
|
|
52
|
+
self.webview.load(QUrl.fromUserInput(self._base_url + LOGIN_PATH))
|
|
50
53
|
self.webview.loadFinished.connect(self.handle_load_finished)
|
|
51
54
|
|
|
52
55
|
self.setCentralWidget(self.webview)
|
|
@@ -58,7 +61,7 @@ class OlBrowserLoginWindow(QMainWindow):
|
|
|
58
61
|
self._login_success = True
|
|
59
62
|
QCoreApplication.quit()
|
|
60
63
|
|
|
61
|
-
if self.webview.url().toString() ==
|
|
64
|
+
if self.webview.url().toString().rstrip("/") == self._base_url + PROJECT_PATH:
|
|
62
65
|
self.webview.page().runJavaScript(
|
|
63
66
|
JAVASCRIPT_CSRF_EXTRACTOR, 0, callback
|
|
64
67
|
)
|
|
@@ -81,14 +84,14 @@ class OlBrowserLoginWindow(QMainWindow):
|
|
|
81
84
|
return self._login_success
|
|
82
85
|
|
|
83
86
|
|
|
84
|
-
def login():
|
|
87
|
+
def login(base_url=BASE_URL):
|
|
85
88
|
from PySide6.QtCore import QLoggingCategory
|
|
86
89
|
QLoggingCategory.setFilterRules('''\
|
|
87
90
|
qt.webenginecontext.info=false
|
|
88
91
|
''')
|
|
89
92
|
|
|
90
93
|
app = QApplication([])
|
|
91
|
-
ol_browser_login_window = OlBrowserLoginWindow()
|
|
94
|
+
ol_browser_login_window = OlBrowserLoginWindow(base_url)
|
|
92
95
|
ol_browser_login_window.show()
|
|
93
96
|
app.exec()
|
|
94
97
|
|
|
@@ -24,18 +24,32 @@ import time
|
|
|
24
24
|
if not hasattr(websocket, "SSLError"):
|
|
25
25
|
websocket.SSLError = ssl.SSLError
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
# The
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
27
|
+
BASE_URL = "https://www.overleaf.com" # The default Overleaf server; self-hosted servers are passed as base_url
|
|
28
|
+
# Paths relative to the server's base URL
|
|
29
|
+
LOGIN_PATH = "/login" # Where to get the CSRF Token and where to send the login request to
|
|
30
|
+
PROJECT_PATH = "/project" # The dashboard
|
|
31
|
+
DOWNLOAD_PATH = "/project/{}/download/zip" # Download all the files in zip format
|
|
32
|
+
UPLOAD_PATH = "/project/{}/upload" # Upload files
|
|
33
|
+
FOLDER_PATH = "/project/{}/folder" # Create folders
|
|
34
|
+
DELETE_PATH = "/project/{}/{}/{}" # Delete files (entity type: doc or file)
|
|
35
|
+
COMPILE_PATH = "/project/{}/compile?enable_pdf_caching=true" # Compile the project
|
|
36
|
+
SOCKET_IO_PATH = "/socket.io/1/" # Real-time service handshake
|
|
37
37
|
PATH_SEP = "/" # Use hardcoded path separator for both windows and posix system
|
|
38
38
|
|
|
39
|
+
class RealtimeSocket(SocketIO):
|
|
40
|
+
"""
|
|
41
|
+
socketIO-client's SocketIO, minus its cleanup quirk: SocketIO.__del__ calls disconnect(), which opens a brand-new
|
|
42
|
+
connection when the old one is already closed (and, with wait_for_connection=True, retries that forever).
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __del__(self):
|
|
46
|
+
try:
|
|
47
|
+
if self.connected:
|
|
48
|
+
self.disconnect()
|
|
49
|
+
except Exception:
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
|
|
39
53
|
class OverleafClient(object):
|
|
40
54
|
"""
|
|
41
55
|
Overleaf API Wrapper
|
|
@@ -51,9 +65,17 @@ class OverleafClient(object):
|
|
|
51
65
|
if all(p.get(k) == v for k, v in more_attrs.items()):
|
|
52
66
|
yield p
|
|
53
67
|
|
|
54
|
-
def __init__(self, cookie=None, csrf=None):
|
|
68
|
+
def __init__(self, cookie=None, csrf=None, base_url=BASE_URL):
|
|
55
69
|
self._cookie = cookie # Store the cookie for authenticated requests
|
|
56
70
|
self._csrf = csrf # Store the CSRF token since it is needed for some requests
|
|
71
|
+
self._base_url = base_url.rstrip("/") # overleaf.com or a self-hosted server
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def base_url(self):
|
|
75
|
+
return self._base_url
|
|
76
|
+
|
|
77
|
+
def _url(self, path):
|
|
78
|
+
return self._base_url + path
|
|
57
79
|
|
|
58
80
|
def login(self, username, password):
|
|
59
81
|
"""
|
|
@@ -63,7 +85,7 @@ class OverleafClient(object):
|
|
|
63
85
|
Returns: Dict of cookie and CSRF
|
|
64
86
|
"""
|
|
65
87
|
|
|
66
|
-
get_login = reqs.get(
|
|
88
|
+
get_login = reqs.get(self._url(LOGIN_PATH))
|
|
67
89
|
self._csrf = BeautifulSoup(get_login.content, 'html.parser').find(
|
|
68
90
|
'input', {'name': '_csrf'}).get('value')
|
|
69
91
|
login_json = {
|
|
@@ -71,7 +93,7 @@ class OverleafClient(object):
|
|
|
71
93
|
"email": username,
|
|
72
94
|
"password": password
|
|
73
95
|
}
|
|
74
|
-
post_login = reqs.post(
|
|
96
|
+
post_login = reqs.post(self._url(LOGIN_PATH), json=login_json,
|
|
75
97
|
cookies=get_login.cookies)
|
|
76
98
|
|
|
77
99
|
# On a successful authentication the Overleaf API returns a new authenticated cookie.
|
|
@@ -84,7 +106,7 @@ class OverleafClient(object):
|
|
|
84
106
|
self._cookie['GCLB'] = get_login.cookies['GCLB']
|
|
85
107
|
|
|
86
108
|
# CSRF changes after making the login request, new CSRF token will be on the projects page
|
|
87
|
-
projects_page = reqs.get(
|
|
109
|
+
projects_page = reqs.get(self._url(PROJECT_PATH), cookies=self._cookie)
|
|
88
110
|
self._csrf = BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-csrfToken'}) \
|
|
89
111
|
.get('content')
|
|
90
112
|
|
|
@@ -109,12 +131,25 @@ class OverleafClient(object):
|
|
|
109
131
|
json_content = json_content.get('projects', [])
|
|
110
132
|
return json_content
|
|
111
133
|
|
|
134
|
+
def fetch_csrf(self):
|
|
135
|
+
"""
|
|
136
|
+
Get the CSRF token from the project dashboard, e.g. after logging in with a cookie copied from a browser
|
|
137
|
+
Returns: the CSRF token, or None if the session cookie is not (or no longer) valid
|
|
138
|
+
"""
|
|
139
|
+
projects_page = reqs.get(self._url(PROJECT_PATH), cookies=self._cookie, allow_redirects=False)
|
|
140
|
+
if not projects_page.ok or projects_page.is_redirect:
|
|
141
|
+
return None
|
|
142
|
+
csrf_tag = BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-csrfToken'})
|
|
143
|
+
if csrf_tag is None:
|
|
144
|
+
return None
|
|
145
|
+
return csrf_tag.get('content')
|
|
146
|
+
|
|
112
147
|
def all_projects(self):
|
|
113
148
|
"""
|
|
114
149
|
Get all of a user's active projects (= not archived and not trashed)
|
|
115
150
|
Returns: List of project objects
|
|
116
151
|
"""
|
|
117
|
-
projects_page = reqs.get(
|
|
152
|
+
projects_page = reqs.get(self._url(PROJECT_PATH), cookies=self._cookie)
|
|
118
153
|
json_content = OverleafClient._extract_projects(projects_page.content)
|
|
119
154
|
return list(OverleafClient.filter_projects(json_content))
|
|
120
155
|
|
|
@@ -125,7 +160,7 @@ class OverleafClient(object):
|
|
|
125
160
|
Returns: project object
|
|
126
161
|
"""
|
|
127
162
|
|
|
128
|
-
projects_page = reqs.get(
|
|
163
|
+
projects_page = reqs.get(self._url(PROJECT_PATH), cookies=self._cookie)
|
|
129
164
|
json_content = OverleafClient._extract_projects(projects_page.content)
|
|
130
165
|
return next(OverleafClient.filter_projects(json_content, {"name": project_name}), None)
|
|
131
166
|
|
|
@@ -135,7 +170,7 @@ class OverleafClient(object):
|
|
|
135
170
|
Params: project_id, the id of the project
|
|
136
171
|
Returns: bytes string (zip file)
|
|
137
172
|
"""
|
|
138
|
-
r = reqs.get(
|
|
173
|
+
r = reqs.get(self._url(DOWNLOAD_PATH.format(project_id)),
|
|
139
174
|
stream=True, cookies=self._cookie)
|
|
140
175
|
return r.content
|
|
141
176
|
|
|
@@ -158,7 +193,7 @@ class OverleafClient(object):
|
|
|
158
193
|
headers = {
|
|
159
194
|
"X-Csrf-Token": self._csrf
|
|
160
195
|
}
|
|
161
|
-
r = reqs.post(
|
|
196
|
+
r = reqs.post(self._url(FOLDER_PATH.format(project_id)),
|
|
162
197
|
cookies=self._cookie, headers=headers, json=params)
|
|
163
198
|
|
|
164
199
|
if r.ok:
|
|
@@ -206,7 +241,7 @@ class OverleafClient(object):
|
|
|
206
241
|
# a session may not carry every cookie -- this previously raised
|
|
207
242
|
# KeyError here even though overleaf_session2 alone is enough to connect.)
|
|
208
243
|
handshake = reqs.get(
|
|
209
|
-
|
|
244
|
+
self._url(SOCKET_IO_PATH),
|
|
210
245
|
params={"t": int(time.time() * 1000)},
|
|
211
246
|
cookies=self._cookie,
|
|
212
247
|
)
|
|
@@ -224,10 +259,12 @@ class OverleafClient(object):
|
|
|
224
259
|
# handshake"); confirmed it rejects even when projectId is only on
|
|
225
260
|
# the WebSocket upgrade URL -- it must be on the initial
|
|
226
261
|
# /socket.io/1/ GET specifically.
|
|
227
|
-
socket_io =
|
|
228
|
-
|
|
262
|
+
socket_io = RealtimeSocket(
|
|
263
|
+
self._base_url,
|
|
229
264
|
params={'t': int(time.time()), 'projectId': project_id},
|
|
230
|
-
headers={'Cookie': cookie}
|
|
265
|
+
headers={'Cookie': cookie},
|
|
266
|
+
# Fail instead of retrying forever (and hanging the CLI) when the real-time service is unreachable
|
|
267
|
+
wait_for_connection=False
|
|
231
268
|
)
|
|
232
269
|
|
|
233
270
|
# Wait until we connect to the socket
|
|
@@ -242,15 +279,16 @@ class OverleafClient(object):
|
|
|
242
279
|
# .on() never registers -- it would return immediately without
|
|
243
280
|
# reading anything. Wait in short slices instead, stopping as soon as
|
|
244
281
|
# the event has actually been processed.
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
socket_io.
|
|
282
|
+
try:
|
|
283
|
+
socket_io.on('joinProjectResponse', set_project_infos)
|
|
284
|
+
for _ in range(20):
|
|
285
|
+
if project_infos is not None:
|
|
286
|
+
break
|
|
287
|
+
socket_io.wait(seconds=1)
|
|
288
|
+
finally:
|
|
289
|
+
# Disconnect from the socket if still connected
|
|
290
|
+
if socket_io.connected:
|
|
291
|
+
socket_io.disconnect()
|
|
254
292
|
|
|
255
293
|
return project_infos
|
|
256
294
|
|
|
@@ -310,7 +348,7 @@ class OverleafClient(object):
|
|
|
310
348
|
}
|
|
311
349
|
|
|
312
350
|
# Upload the file to the predefined folder
|
|
313
|
-
r = reqs.post(
|
|
351
|
+
r = reqs.post(self._url(UPLOAD_PATH.format(project_id)), cookies=self._cookie, params=params, files=files)
|
|
314
352
|
|
|
315
353
|
# status_code is an int; comparing to str(200) was always False here,
|
|
316
354
|
# so this reported failure on every call regardless of the real
|
|
@@ -330,23 +368,21 @@ class OverleafClient(object):
|
|
|
330
368
|
Returns: True on success, False on fail
|
|
331
369
|
"""
|
|
332
370
|
|
|
333
|
-
file
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
else:
|
|
349
|
-
file = next((v for v in project_infos['rootFolder'][0]['docs'] if v['name'] == file_name), None)
|
|
371
|
+
# Walk down to the folder that contains the file (folder names match case-insensitively, as in upload_file)
|
|
372
|
+
*local_folders, name = file_name.split(PATH_SEP)
|
|
373
|
+
current_folder = project_infos['rootFolder'][0]
|
|
374
|
+
for local_folder in local_folders:
|
|
375
|
+
current_folder = next((f for f in current_folder['folders'] if f['name'].lower() == local_folder.lower()),
|
|
376
|
+
None)
|
|
377
|
+
if current_folder is None:
|
|
378
|
+
return False
|
|
379
|
+
|
|
380
|
+
# Text files are "docs", binary files (images, PDFs, ...) are "fileRefs" with their own endpoint
|
|
381
|
+
file = next((v for v in current_folder.get('docs', []) if v['name'] == name), None)
|
|
382
|
+
entity_type = "doc"
|
|
383
|
+
if file is None:
|
|
384
|
+
file = next((v for v in current_folder.get('fileRefs', []) if v['name'] == name), None)
|
|
385
|
+
entity_type = "file"
|
|
350
386
|
|
|
351
387
|
# File not found!
|
|
352
388
|
if file is None:
|
|
@@ -356,7 +392,8 @@ class OverleafClient(object):
|
|
|
356
392
|
"X-Csrf-Token": self._csrf
|
|
357
393
|
}
|
|
358
394
|
|
|
359
|
-
r = reqs.delete(
|
|
395
|
+
r = reqs.delete(self._url(DELETE_PATH.format(project_id, entity_type, file['_id'])), cookies=self._cookie,
|
|
396
|
+
headers=headers, json={})
|
|
360
397
|
|
|
361
398
|
# status_code is an int; comparing to str(204) was always False
|
|
362
399
|
# regardless of the real result, same bug class as upload_file and
|
|
@@ -380,11 +417,11 @@ class OverleafClient(object):
|
|
|
380
417
|
"check": "silent",
|
|
381
418
|
"draft": False,
|
|
382
419
|
"incrementalCompilesEnabled": True,
|
|
383
|
-
|
|
420
|
+
# No rootDoc_id: Overleaf validates it as an optional ObjectId and rejects "" (400)
|
|
384
421
|
"stopOnFirstError": False
|
|
385
422
|
}
|
|
386
423
|
|
|
387
|
-
r = reqs.post(
|
|
424
|
+
r = reqs.post(self._url(COMPILE_PATH.format(project_id)), cookies=self._cookie, headers=headers, json=body)
|
|
388
425
|
|
|
389
426
|
if not r.ok:
|
|
390
427
|
raise reqs.HTTPError()
|
|
@@ -396,7 +433,15 @@ class OverleafClient(object):
|
|
|
396
433
|
|
|
397
434
|
pdf_file = next(v for v in compile_result['outputFiles'] if v['type'] == 'pdf')
|
|
398
435
|
|
|
399
|
-
|
|
436
|
+
# The output is only served (otherwise 404) when the compile group and CLSI server are passed along,
|
|
437
|
+
# like Overleaf's own frontend does. Self-hosted servers may not send them.
|
|
438
|
+
params = {}
|
|
439
|
+
if compile_result.get("compileGroup"):
|
|
440
|
+
params["compileGroup"] = compile_result["compileGroup"]
|
|
441
|
+
if compile_result.get("clsiServerId"):
|
|
442
|
+
params["clsiserverid"] = compile_result["clsiServerId"]
|
|
443
|
+
|
|
444
|
+
download_req = reqs.get(self._url(pdf_file['url']), cookies=self._cookie, headers=headers, params=params)
|
|
400
445
|
|
|
401
446
|
if download_req.ok:
|
|
402
447
|
return pdf_file['path'], download_req.content
|
|
@@ -23,10 +23,19 @@ from pathlib import Path
|
|
|
23
23
|
|
|
24
24
|
try:
|
|
25
25
|
# Import for pip installation / wheel
|
|
26
|
-
from olsync.olclient import OverleafClient
|
|
26
|
+
from olsync.olclient import BASE_URL, OverleafClient
|
|
27
27
|
except ImportError:
|
|
28
28
|
# Import for development
|
|
29
|
-
from olclient import OverleafClient
|
|
29
|
+
from olclient import BASE_URL, OverleafClient
|
|
30
|
+
|
|
31
|
+
# Session cookie names: overleaf.com, and self-hosted servers (Community Edition, "sharelatex.sid" before version 5)
|
|
32
|
+
OVERLEAF_COM_SESSION_COOKIES = ["overleaf_session2"]
|
|
33
|
+
SELF_HOSTED_SESSION_COOKIES = ["overleaf.sid", "sharelatex.sid"]
|
|
34
|
+
|
|
35
|
+
server_option = click.option(
|
|
36
|
+
'--server', 'server', default=None, envvar='OLSYNC_SERVER', metavar='URL',
|
|
37
|
+
help="URL of a self-hosted Overleaf server (default: the server used at login, or https://www.overleaf.com). "
|
|
38
|
+
"Can also be set with OLSYNC_SERVER.")
|
|
30
39
|
|
|
31
40
|
|
|
32
41
|
def configure_console():
|
|
@@ -54,20 +63,14 @@ def configure_console():
|
|
|
54
63
|
@click.option('-i', '--olignore', 'olignore_path', default=".olignore", type=click.Path(exists=False),
|
|
55
64
|
help="Path to the .olignore file relative to sync path (ignored if syncing from remote to local). See "
|
|
56
65
|
"fnmatch / unix filename pattern matching for information on how to use it.")
|
|
66
|
+
@server_option
|
|
57
67
|
@click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
|
|
58
68
|
@click.version_option(package_name='overleaf-sync')
|
|
59
69
|
@click.pass_context
|
|
60
|
-
def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path, verbose):
|
|
70
|
+
def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path, server, verbose):
|
|
61
71
|
configure_console()
|
|
62
72
|
if ctx.invoked_subcommand is None:
|
|
63
|
-
|
|
64
|
-
raise click.ClickException(
|
|
65
|
-
"Persisted Overleaf cookie not found. Please login or check store path.")
|
|
66
|
-
|
|
67
|
-
with open(cookie_path, 'rb') as f:
|
|
68
|
-
store = pickle.load(f)
|
|
69
|
-
|
|
70
|
-
overleaf_client = OverleafClient(store["cookie"], store["csrf"])
|
|
73
|
+
overleaf_client = load_client(cookie_path, server)
|
|
71
74
|
|
|
72
75
|
# Change the current directory to the specified sync path
|
|
73
76
|
os.chdir(sync_path)
|
|
@@ -98,9 +101,11 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
|
|
|
98
101
|
sync = not (local or remote)
|
|
99
102
|
|
|
100
103
|
if remote or sync:
|
|
104
|
+
remote_files = zip_file.namelist()
|
|
105
|
+
deleted_files = [] if sync else [f for f in olignore_keep_list(olignore_path) if f not in remote_files]
|
|
101
106
|
sync_func(
|
|
102
|
-
files_from=
|
|
103
|
-
deleted_files=
|
|
107
|
+
files_from=remote_files,
|
|
108
|
+
deleted_files=deleted_files,
|
|
104
109
|
create_file_at_to=lambda name: write_file(name, zip_file.read(name)),
|
|
105
110
|
delete_file_at_to=lambda name: delete_file(name),
|
|
106
111
|
create_file_at_from=lambda name: overleaf_client.upload_file(
|
|
@@ -113,9 +118,12 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
|
|
|
113
118
|
to_name="local",
|
|
114
119
|
verbose=verbose)
|
|
115
120
|
if local or sync:
|
|
121
|
+
# Computed once (#60), and after the remote -> local pass so newly downloaded files are included
|
|
122
|
+
local_files = olignore_keep_list(olignore_path)
|
|
123
|
+
deleted_files = [] if sync else [f for f in zip_file.namelist() if f not in local_files]
|
|
116
124
|
sync_func(
|
|
117
|
-
files_from=
|
|
118
|
-
deleted_files=
|
|
125
|
+
files_from=local_files,
|
|
126
|
+
deleted_files=deleted_files,
|
|
119
127
|
create_file_at_to=lambda name: overleaf_client.upload_file(
|
|
120
128
|
project["id"], project_infos, name, os.path.getsize(name), open(name, 'rb')),
|
|
121
129
|
delete_file_at_to=lambda name: overleaf_client.delete_file(project["id"], project_infos, name),
|
|
@@ -132,14 +140,32 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
|
|
|
132
140
|
@main.command()
|
|
133
141
|
@click.option('--path', 'cookie_path', default=".olauth", type=click.Path(exists=False),
|
|
134
142
|
help="Path to store the persisted Overleaf cookie.")
|
|
143
|
+
@click.option('--cookie', 'use_cookie', is_flag=True,
|
|
144
|
+
help="Log in without the login window, using the session cookie from a browser that is logged in to "
|
|
145
|
+
"Overleaf (`overleaf_session2` on overleaf.com, `overleaf.sid` on self-hosted servers), asked for "
|
|
146
|
+
"at a hidden prompt or read from OLSYNC_COOKIE. Useful on servers without a display and for single "
|
|
147
|
+
"sign-on accounts.")
|
|
148
|
+
@server_option
|
|
135
149
|
@click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
|
|
136
|
-
|
|
150
|
+
@click.pass_context
|
|
151
|
+
def login(ctx, cookie_path, use_cookie, server, verbose):
|
|
152
|
+
server = resolve_login_server(ctx, server)
|
|
137
153
|
if os.path.isfile(cookie_path) and not click.confirm(
|
|
138
154
|
'Persisted Overleaf cookie already exist. Do you want to override it?'):
|
|
139
155
|
return
|
|
156
|
+
if use_cookie:
|
|
157
|
+
cookie_names = OVERLEAF_COM_SESSION_COOKIES if server == BASE_URL else SELF_HOSTED_SESSION_COOKIES
|
|
158
|
+
# Never accept the cookie as a command line argument, it would end up in the shell history
|
|
159
|
+
cookie = os.environ.get("OLSYNC_COOKIE") or click.prompt(f"Value of the {cookie_names[0]} cookie",
|
|
160
|
+
hide_input=True)
|
|
161
|
+
execute_action(lambda: cookie_login_handler(cookie.strip(), cookie_path, server, cookie_names), "Login",
|
|
162
|
+
"Login successful. Cookie persisted as `" + click.format_filename(
|
|
163
|
+
cookie_path) + "`. You may now sync your project.",
|
|
164
|
+
"Login failed: the cookie is not valid (expired, or copied incompletely).", verbose)
|
|
165
|
+
return
|
|
140
166
|
olbrowserlogin = load_browser_login()
|
|
141
167
|
click.clear()
|
|
142
|
-
execute_action(lambda: login_handler(olbrowserlogin, cookie_path), "Login",
|
|
168
|
+
execute_action(lambda: login_handler(olbrowserlogin, cookie_path, server), "Login",
|
|
143
169
|
"Login successful. Cookie persisted as `" + click.format_filename(
|
|
144
170
|
cookie_path) + "`. You may now sync your project.",
|
|
145
171
|
"Login failed. Please try again.", verbose)
|
|
@@ -148,8 +174,10 @@ def login(cookie_path, verbose):
|
|
|
148
174
|
@main.command(name='list')
|
|
149
175
|
@click.option('--store-path', 'cookie_path', default=".olauth", type=click.Path(exists=False),
|
|
150
176
|
help="Relative path to load the persisted Overleaf cookie.")
|
|
177
|
+
@server_option
|
|
151
178
|
@click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
|
|
152
|
-
|
|
179
|
+
@click.pass_context
|
|
180
|
+
def list_projects(ctx, cookie_path, server, verbose):
|
|
153
181
|
def query_projects():
|
|
154
182
|
for index, p in enumerate(sorted(overleaf_client.all_projects(), key=lambda x: x['lastUpdated'], reverse=True)):
|
|
155
183
|
if not index:
|
|
@@ -157,14 +185,7 @@ def list_projects(cookie_path, verbose):
|
|
|
157
185
|
click.echo(f"{dateutil.parser.isoparse(p['lastUpdated']).strftime('%m/%d/%Y, %H:%M:%S')} - {p['name']}")
|
|
158
186
|
return True
|
|
159
187
|
|
|
160
|
-
|
|
161
|
-
raise click.ClickException(
|
|
162
|
-
"Persisted Overleaf cookie not found. Please login or check store path.")
|
|
163
|
-
|
|
164
|
-
with open(cookie_path, 'rb') as f:
|
|
165
|
-
store = pickle.load(f)
|
|
166
|
-
|
|
167
|
-
overleaf_client = OverleafClient(store["cookie"], store["csrf"])
|
|
188
|
+
overleaf_client = load_client(cookie_path, server or ctx.find_root().params.get("server"))
|
|
168
189
|
|
|
169
190
|
click.clear()
|
|
170
191
|
execute_action(query_projects, "Querying all projects",
|
|
@@ -178,8 +199,10 @@ def list_projects(cookie_path, verbose):
|
|
|
178
199
|
@click.option('--download-path', 'download_path', default=".", type=click.Path(exists=True))
|
|
179
200
|
@click.option('--store-path', 'cookie_path', default=".olauth", type=click.Path(exists=False),
|
|
180
201
|
help="Relative path to load the persisted Overleaf cookie.")
|
|
202
|
+
@server_option
|
|
181
203
|
@click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
|
|
182
|
-
|
|
204
|
+
@click.pass_context
|
|
205
|
+
def download_pdf(ctx, project_name, download_path, cookie_path, server, verbose):
|
|
183
206
|
def download_project_pdf():
|
|
184
207
|
nonlocal project_name
|
|
185
208
|
project_name = project_name or os.path.basename(os.getcwd())
|
|
@@ -199,14 +222,7 @@ def download_pdf(project_name, download_path, cookie_path, verbose):
|
|
|
199
222
|
|
|
200
223
|
return True
|
|
201
224
|
|
|
202
|
-
|
|
203
|
-
raise click.ClickException(
|
|
204
|
-
"Persisted Overleaf cookie not found. Please login or check store path.")
|
|
205
|
-
|
|
206
|
-
with open(cookie_path, 'rb') as f:
|
|
207
|
-
store = pickle.load(f)
|
|
208
|
-
|
|
209
|
-
overleaf_client = OverleafClient(store["cookie"], store["csrf"])
|
|
225
|
+
overleaf_client = load_client(cookie_path, server or ctx.find_root().params.get("server"))
|
|
210
226
|
|
|
211
227
|
click.clear()
|
|
212
228
|
|
|
@@ -225,27 +241,67 @@ def load_browser_login():
|
|
|
225
241
|
# Import for pip installation / wheel
|
|
226
242
|
import olsync.olbrowserlogin as olbrowserlogin
|
|
227
243
|
except ImportError as e:
|
|
228
|
-
if (e.name or "").startswith("
|
|
244
|
+
if not (e.name or "").startswith("olsync"):
|
|
229
245
|
raise
|
|
230
|
-
# Import for development
|
|
246
|
+
# Import for development (running olsync.py from inside the olsync folder)
|
|
231
247
|
import olbrowserlogin
|
|
232
248
|
except ImportError as e:
|
|
233
|
-
if
|
|
234
|
-
raise
|
|
249
|
+
if (e.name or "").startswith("PySide6"):
|
|
250
|
+
raise click.ClickException(
|
|
251
|
+
"The login window needs PySide6. Install it with: pip install \"overleaf-sync[gui]\"\n"
|
|
252
|
+
"Alternatively, log in without the window: ols login --cookie")
|
|
253
|
+
# PySide6 is installed but can't be loaded, e.g. because another Qt installation (such as Anaconda's)
|
|
254
|
+
# conflicts with it (#59)
|
|
235
255
|
raise click.ClickException(
|
|
236
|
-
"The login window
|
|
256
|
+
f"The login window could not be loaded: {e}\n"
|
|
257
|
+
"This usually means another Qt installation (e.g. from Anaconda) conflicts with PySide6. Installing "
|
|
258
|
+
"overleaf-sync into a fresh virtual environment based on a non-conda Python usually helps.\n"
|
|
259
|
+
"Alternatively, log in without the window: ols login --cookie")
|
|
237
260
|
return olbrowserlogin
|
|
238
261
|
|
|
239
262
|
|
|
240
|
-
def
|
|
241
|
-
|
|
263
|
+
def resolve_login_server(ctx, server):
|
|
264
|
+
"""--server on the login command, else on the group (both also read OLSYNC_SERVER), else overleaf.com"""
|
|
265
|
+
return (server or ctx.find_root().params.get("server") or BASE_URL).rstrip("/")
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def load_client(cookie_path, server=None):
|
|
269
|
+
"""
|
|
270
|
+
Create an OverleafClient from the persisted login. The server is --server/OLSYNC_SERVER if given, otherwise the
|
|
271
|
+
server stored at login (auth files from before 2.0.0a3 have none and mean overleaf.com).
|
|
272
|
+
"""
|
|
273
|
+
if not os.path.isfile(cookie_path):
|
|
274
|
+
raise click.ClickException(
|
|
275
|
+
"Persisted Overleaf cookie not found. Please login or check store path.")
|
|
276
|
+
|
|
277
|
+
with open(cookie_path, 'rb') as f:
|
|
278
|
+
store = pickle.load(f)
|
|
279
|
+
|
|
280
|
+
return OverleafClient(store["cookie"], store["csrf"], base_url=server or store.get("server") or BASE_URL)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def login_handler(olbrowserlogin, path, server):
|
|
284
|
+
store = olbrowserlogin.login(server)
|
|
242
285
|
if store is None:
|
|
243
286
|
return False
|
|
287
|
+
store["server"] = server
|
|
244
288
|
with open(path, 'wb+') as f:
|
|
245
289
|
pickle.dump(store, f)
|
|
246
290
|
return True
|
|
247
291
|
|
|
248
292
|
|
|
293
|
+
def cookie_login_handler(cookie, path, server, cookie_names):
|
|
294
|
+
# Self-hosted servers name the session cookie differently depending on the version, so try each name
|
|
295
|
+
for cookie_name in cookie_names:
|
|
296
|
+
cookies = {cookie_name: cookie}
|
|
297
|
+
csrf = OverleafClient(cookies, base_url=server).fetch_csrf()
|
|
298
|
+
if csrf is not None:
|
|
299
|
+
with open(path, 'wb+') as f:
|
|
300
|
+
pickle.dump({"cookie": cookies, "csrf": csrf, "server": server}, f)
|
|
301
|
+
return True
|
|
302
|
+
return False
|
|
303
|
+
|
|
304
|
+
|
|
249
305
|
def delete_file(path):
|
|
250
306
|
_dir = os.path.dirname(path)
|
|
251
307
|
if _dir == path:
|
|
@@ -410,7 +466,8 @@ def olignore_keep_list(olignore_path):
|
|
|
410
466
|
else:
|
|
411
467
|
click.echo("\n.olignore: using %s to filter items" % olignore_path)
|
|
412
468
|
with open(olignore_path, 'r') as f:
|
|
413
|
-
|
|
469
|
+
# "./main.tex" and "main.tex" mean the same file, matched paths have no "./" prefix (#72)
|
|
470
|
+
ignore_pattern = [pattern.removeprefix("./") for pattern in f.read().splitlines()]
|
|
414
471
|
|
|
415
472
|
keep_list = [f for f in files if not any(
|
|
416
473
|
fnmatch.fnmatch(f, ignore) for ignore in ignore_pattern)]
|
|
File without changes
|
|
File without changes
|