wpsync 1.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wpsync might be problematic. Click here for more details.
- wpsync/__init__.py +3 -0
- wpsync/__main__.py +3 -0
- wpsync/cli.py +740 -0
- wpsync/config.py +173 -0
- wpsync/documents.py +258 -0
- wpsync/markup/__init__.py +0 -0
- wpsync/markup/dom.py +313 -0
- wpsync/markup/markdown.py +526 -0
- wpsync/media.py +92 -0
- wpsync/posts.py +76 -0
- wpsync/pull.py +324 -0
- wpsync/state.py +177 -0
- wpsync/storage.py +28 -0
- wpsync/sync.py +260 -0
- wpsync/wordpress/__init__.py +0 -0
- wpsync/wordpress/autop.py +114 -0
- wpsync/wordpress/client.py +208 -0
- wpsync/wordpress/content.py +565 -0
- wpsync/wordpress/remote.py +100 -0
- wpsync-1.2.1.dist-info/METADATA +312 -0
- wpsync-1.2.1.dist-info/RECORD +24 -0
- wpsync-1.2.1.dist-info/WHEEL +4 -0
- wpsync-1.2.1.dist-info/entry_points.txt +2 -0
- wpsync-1.2.1.dist-info/licenses/LICENSE +61 -0
wpsync/pull.py
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""Bringing posts from the platform into local files."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from dataclasses import dataclass, replace
|
|
8
|
+
from pathlib import Path, PurePosixPath
|
|
9
|
+
from typing import Protocol
|
|
10
|
+
|
|
11
|
+
from wpsync.documents import (
|
|
12
|
+
POSTS_DIR,
|
|
13
|
+
Document,
|
|
14
|
+
LocalDocument,
|
|
15
|
+
body_checksum,
|
|
16
|
+
fields_checksum,
|
|
17
|
+
parse_document,
|
|
18
|
+
serialize_document,
|
|
19
|
+
)
|
|
20
|
+
from wpsync.markup.markdown import image_sources
|
|
21
|
+
from wpsync.media import Media, MediaLibrary
|
|
22
|
+
from wpsync.posts import Conversion, FetchedPost, UnrepresentableContent
|
|
23
|
+
from wpsync.state import MediaRecord, PostRecord, write_state
|
|
24
|
+
from wpsync.storage import write_atomic, write_text_atomic
|
|
25
|
+
from wpsync.sync import Action, DiffEntry, SelectionError, match_post
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class PullRemote(Protocol):
|
|
29
|
+
def fetch_posts(
|
|
30
|
+
self, ids: list[str] | None = None, slug: str | None = None
|
|
31
|
+
) -> list[FetchedPost]: ...
|
|
32
|
+
|
|
33
|
+
def convert(self, post: FetchedPost, media: Media) -> Conversion: ...
|
|
34
|
+
|
|
35
|
+
def download(self, url: str) -> bytes: ...
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class _LocalCandidate:
|
|
40
|
+
"""A local file without a post ID, as a match candidate."""
|
|
41
|
+
|
|
42
|
+
id: str
|
|
43
|
+
title: str
|
|
44
|
+
slug: str
|
|
45
|
+
document: LocalDocument
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class PullResult:
|
|
50
|
+
new: int = 0
|
|
51
|
+
updated: int = 0
|
|
52
|
+
linked: int = 0
|
|
53
|
+
images: int = 0
|
|
54
|
+
skipped: int = 0
|
|
55
|
+
changes_kept: int = 0
|
|
56
|
+
|
|
57
|
+
def summary(self) -> str:
|
|
58
|
+
pulled = self.new + self.updated + self.linked
|
|
59
|
+
if not pulled and not self.skipped:
|
|
60
|
+
kept = self.changes_kept
|
|
61
|
+
if kept:
|
|
62
|
+
posts = f"{kept} post{'s' * (kept != 1)}"
|
|
63
|
+
return f"Nothing to pull; {posts} with local changes."
|
|
64
|
+
return "Nothing to pull."
|
|
65
|
+
parts = [
|
|
66
|
+
f"{count} {label}"
|
|
67
|
+
for count, label in (
|
|
68
|
+
(self.new, "new"),
|
|
69
|
+
(self.updated, "updated"),
|
|
70
|
+
(self.linked, "linked"),
|
|
71
|
+
)
|
|
72
|
+
if count
|
|
73
|
+
]
|
|
74
|
+
line = f"Pulled {pulled} post{'s' * (pulled != 1)}"
|
|
75
|
+
line += f" ({', '.join(parts)})" if parts else ""
|
|
76
|
+
if self.images:
|
|
77
|
+
line += f", {self.images} image{'s' * (self.images != 1)}"
|
|
78
|
+
if self.skipped:
|
|
79
|
+
line += f"; {self.skipped} skipped"
|
|
80
|
+
if self.changes_kept:
|
|
81
|
+
line += f"; {self.changes_kept} with local changes left alone"
|
|
82
|
+
return line + "."
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class Puller:
|
|
86
|
+
def __init__(
|
|
87
|
+
self,
|
|
88
|
+
root: Path,
|
|
89
|
+
api_root: str,
|
|
90
|
+
remote: PullRemote,
|
|
91
|
+
documents: list[LocalDocument],
|
|
92
|
+
records: dict[str, PostRecord],
|
|
93
|
+
media: list[MediaRecord],
|
|
94
|
+
confirm: Callable[[str], bool],
|
|
95
|
+
report: Callable[[DiffEntry], None],
|
|
96
|
+
) -> None:
|
|
97
|
+
self.root = root
|
|
98
|
+
self.api_root = api_root
|
|
99
|
+
self.remote = remote
|
|
100
|
+
self.records = records
|
|
101
|
+
self.confirm = confirm
|
|
102
|
+
self.report = report
|
|
103
|
+
self.library = MediaLibrary(root, media)
|
|
104
|
+
self.documents = {
|
|
105
|
+
d.document.post_id: d for d in documents if d.document.post_id
|
|
106
|
+
}
|
|
107
|
+
self.unlinked = [
|
|
108
|
+
_LocalCandidate(str(d.path), d.document.post.title, d.document.post.slug, d)
|
|
109
|
+
for d in documents
|
|
110
|
+
if d.document.post_id is None
|
|
111
|
+
]
|
|
112
|
+
self.taken = {PurePosixPath(d.path) for d in documents}
|
|
113
|
+
self.result = PullResult()
|
|
114
|
+
|
|
115
|
+
def pull(self, ids: list[str] | None = None, slug: str | None = None) -> PullResult:
|
|
116
|
+
fetched = self.remote.fetch_posts(ids=ids, slug=slug)
|
|
117
|
+
if not fetched and (ids or slug):
|
|
118
|
+
named = f"slug '{slug}'" if slug else f"id {ids[0]}" # type: ignore[index]
|
|
119
|
+
raise SelectionError(f"WordPress has no post with {named}")
|
|
120
|
+
for post in fetched:
|
|
121
|
+
self._pull_post(post)
|
|
122
|
+
if ids is None and slug is None:
|
|
123
|
+
present = {post.id for post in fetched}
|
|
124
|
+
for post_id, record in sorted(
|
|
125
|
+
self.records.items(), key=lambda r: int(r[0])
|
|
126
|
+
):
|
|
127
|
+
if post_id not in present and post_id in self.documents:
|
|
128
|
+
self._line(
|
|
129
|
+
Action.DELETED,
|
|
130
|
+
"r",
|
|
131
|
+
post_id,
|
|
132
|
+
record.path,
|
|
133
|
+
"deleted on WordPress; local file kept",
|
|
134
|
+
)
|
|
135
|
+
return self.result
|
|
136
|
+
|
|
137
|
+
def _pull_post(self, post: FetchedPost) -> None:
|
|
138
|
+
document = self.documents.get(post.id)
|
|
139
|
+
record = self.records.get(post.id)
|
|
140
|
+
if record is not None and document is None:
|
|
141
|
+
self._line(
|
|
142
|
+
Action.DELETED,
|
|
143
|
+
"l",
|
|
144
|
+
post.id,
|
|
145
|
+
record.path,
|
|
146
|
+
"deleted locally; not restored",
|
|
147
|
+
)
|
|
148
|
+
return
|
|
149
|
+
if record is not None and document is not None:
|
|
150
|
+
self._pull_tracked(post, document, record)
|
|
151
|
+
elif document is not None:
|
|
152
|
+
self._pull_unrecorded(post, document)
|
|
153
|
+
else:
|
|
154
|
+
self._pull_new(post)
|
|
155
|
+
|
|
156
|
+
def _pull_tracked(
|
|
157
|
+
self, post: FetchedPost, document: LocalDocument, record: PostRecord
|
|
158
|
+
) -> None:
|
|
159
|
+
local = document.document.post
|
|
160
|
+
changed_locally = (
|
|
161
|
+
body_checksum(local) != record.body_checksum
|
|
162
|
+
or fields_checksum(local) != record.fields_checksum
|
|
163
|
+
)
|
|
164
|
+
if post.state.fingerprint == record.remote_fingerprint:
|
|
165
|
+
self.result.changes_kept += changed_locally
|
|
166
|
+
return
|
|
167
|
+
path = PurePosixPath(document.path)
|
|
168
|
+
if changed_locally and not self.confirm(
|
|
169
|
+
f"#{post.id} {path} changed locally and on WordPress. "
|
|
170
|
+
"Replace your file with WordPress's version?"
|
|
171
|
+
):
|
|
172
|
+
self._skip(post.id, path, "changed on both sides; your file is unchanged")
|
|
173
|
+
return
|
|
174
|
+
if self._write(post, path, document.document, Action.CHANGED):
|
|
175
|
+
self.result.updated += 1
|
|
176
|
+
|
|
177
|
+
def _pull_unrecorded(self, post: FetchedPost, document: LocalDocument) -> None:
|
|
178
|
+
path = PurePosixPath(document.path)
|
|
179
|
+
conversion = self._convert(post, path)
|
|
180
|
+
if conversion is None:
|
|
181
|
+
return
|
|
182
|
+
remote = replace(post.post, body=conversion.markdown.strip("\n"))
|
|
183
|
+
if remote == document.document.post:
|
|
184
|
+
for download in self.library.take_pending():
|
|
185
|
+
self.library.release(download)
|
|
186
|
+
self._record(post, path, document.document, [])
|
|
187
|
+
self._line(Action.UNCHANGED, "", post.id, path, "linked; already identical")
|
|
188
|
+
self.result.linked += 1
|
|
189
|
+
return
|
|
190
|
+
if not self.confirm(
|
|
191
|
+
f"{path} has id {post.id} but was never synced here. "
|
|
192
|
+
"Replace it with WordPress's version?"
|
|
193
|
+
):
|
|
194
|
+
for download in self.library.take_pending():
|
|
195
|
+
self.library.release(download)
|
|
196
|
+
self._skip(post.id, path, "never synced here; your file is unchanged")
|
|
197
|
+
return
|
|
198
|
+
if self._write(post, path, document.document, Action.CHANGED, conversion):
|
|
199
|
+
self.result.linked += 1
|
|
200
|
+
|
|
201
|
+
def _pull_new(self, post: FetchedPost) -> None:
|
|
202
|
+
match = match_post(post.post, self.unlinked)
|
|
203
|
+
if match.candidates:
|
|
204
|
+
files = ", ".join(c.id for c in match.candidates)
|
|
205
|
+
self._skip(post.id, None, f"matches several local files: {files}")
|
|
206
|
+
return
|
|
207
|
+
if match.post is not None:
|
|
208
|
+
candidate = match.post
|
|
209
|
+
path = PurePosixPath(candidate.id)
|
|
210
|
+
if not self.confirm(
|
|
211
|
+
f"WordPress post #{post.id} {post.post.title!r} matches {path} "
|
|
212
|
+
f"({match.reason}). Link them and replace the file with "
|
|
213
|
+
"WordPress's version?"
|
|
214
|
+
):
|
|
215
|
+
self._skip(
|
|
216
|
+
post.id, path, f"matches this file ({match.reason}); not linked"
|
|
217
|
+
)
|
|
218
|
+
return
|
|
219
|
+
self.unlinked.remove(candidate)
|
|
220
|
+
if self._write(post, path, candidate.document.document, Action.NEW):
|
|
221
|
+
self.result.linked += 1
|
|
222
|
+
return
|
|
223
|
+
path = self._new_path(post)
|
|
224
|
+
self.taken.add(path)
|
|
225
|
+
if self._write(post, path, Document(post.id, post.post), Action.NEW):
|
|
226
|
+
self.result.new += 1
|
|
227
|
+
|
|
228
|
+
def _new_path(self, post: FetchedPost) -> PurePosixPath:
|
|
229
|
+
for name in (post.name, f"{post.name}-{post.id}"):
|
|
230
|
+
path = PurePosixPath(POSTS_DIR) / f"{name}.md"
|
|
231
|
+
if path not in self.taken and not (self.root / path).exists():
|
|
232
|
+
return path
|
|
233
|
+
raise ValueError(f"no free file name for post {post.id}")
|
|
234
|
+
|
|
235
|
+
def _write(
|
|
236
|
+
self,
|
|
237
|
+
post: FetchedPost,
|
|
238
|
+
path: PurePosixPath,
|
|
239
|
+
existing: Document,
|
|
240
|
+
action: Action,
|
|
241
|
+
conversion: Conversion | None = None,
|
|
242
|
+
) -> bool:
|
|
243
|
+
if conversion is None:
|
|
244
|
+
conversion = self._convert(post, path)
|
|
245
|
+
if conversion is None:
|
|
246
|
+
return False
|
|
247
|
+
images = []
|
|
248
|
+
used = image_sources(conversion.markdown)
|
|
249
|
+
for download in self.library.take_pending():
|
|
250
|
+
if str(download.path) not in used:
|
|
251
|
+
self.library.release(download)
|
|
252
|
+
continue
|
|
253
|
+
data = self.remote.download(download.url)
|
|
254
|
+
write_atomic(self.root / download.path, data, mode=0o644)
|
|
255
|
+
image = MediaRecord(
|
|
256
|
+
download.path,
|
|
257
|
+
hashlib.sha256(data).hexdigest(),
|
|
258
|
+
download.media_id,
|
|
259
|
+
download.url,
|
|
260
|
+
download.size,
|
|
261
|
+
)
|
|
262
|
+
self.library.add(image)
|
|
263
|
+
images.append(image)
|
|
264
|
+
written = Document(
|
|
265
|
+
post.id,
|
|
266
|
+
replace(post.post, body=conversion.markdown.strip("\n")),
|
|
267
|
+
existing.extra,
|
|
268
|
+
)
|
|
269
|
+
write_text_atomic(self.root / path, serialize_document(written), mode=0o644)
|
|
270
|
+
self._record(post, path, written, images)
|
|
271
|
+
self.result.images += len(images)
|
|
272
|
+
details = []
|
|
273
|
+
if images:
|
|
274
|
+
details.append(f"{len(images)} image{'s' * (len(images) != 1)}")
|
|
275
|
+
if conversion.kept:
|
|
276
|
+
details.append("kept as HTML: " + ", ".join(dict.fromkeys(conversion.kept)))
|
|
277
|
+
self._line(action, "r", post.id, path, "; ".join(details))
|
|
278
|
+
return True
|
|
279
|
+
|
|
280
|
+
def _convert(self, post: FetchedPost, path: PurePosixPath) -> Conversion | None:
|
|
281
|
+
try:
|
|
282
|
+
return self.remote.convert(post, self.library)
|
|
283
|
+
except UnrepresentableContent as error:
|
|
284
|
+
for download in self.library.take_pending():
|
|
285
|
+
self.library.release(download)
|
|
286
|
+
self._skip(post.id, path, f"not pulled: {error}")
|
|
287
|
+
return None
|
|
288
|
+
|
|
289
|
+
def _record(
|
|
290
|
+
self,
|
|
291
|
+
post: FetchedPost,
|
|
292
|
+
path: PurePosixPath,
|
|
293
|
+
document: Document,
|
|
294
|
+
images: list[MediaRecord],
|
|
295
|
+
) -> None:
|
|
296
|
+
# Checksums describe the file as it reads back, exactly as diff will see it.
|
|
297
|
+
written = parse_document(serialize_document(document)).post
|
|
298
|
+
with write_state(self.root, self.api_root) as state:
|
|
299
|
+
for image in images:
|
|
300
|
+
state.save_media(image)
|
|
301
|
+
state.save(
|
|
302
|
+
PostRecord(
|
|
303
|
+
post.id,
|
|
304
|
+
path,
|
|
305
|
+
body_checksum(written),
|
|
306
|
+
fields_checksum(written),
|
|
307
|
+
post.state.fingerprint,
|
|
308
|
+
post.state.format,
|
|
309
|
+
)
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
def _skip(self, post_id: str, path: PurePosixPath | None, detail: str) -> None:
|
|
313
|
+
self.result.skipped += 1
|
|
314
|
+
self._line(Action.CONFLICT, "", post_id, path or PurePosixPath("-"), detail)
|
|
315
|
+
|
|
316
|
+
def _line(
|
|
317
|
+
self,
|
|
318
|
+
action: Action,
|
|
319
|
+
side: str,
|
|
320
|
+
post_id: str,
|
|
321
|
+
path: PurePosixPath,
|
|
322
|
+
detail: str = "",
|
|
323
|
+
) -> None:
|
|
324
|
+
self.report(DiffEntry(action, side, post_id, path, detail))
|
wpsync/state.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Synchronization state: what each linked post looked like at its last sync."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sqlite3
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path, PurePosixPath
|
|
10
|
+
|
|
11
|
+
from wpsync.posts import ContentFormat
|
|
12
|
+
|
|
13
|
+
STATE_PATH = Path(".wpsync") / "state.db"
|
|
14
|
+
SCHEMA_VERSION = 1
|
|
15
|
+
SCHEMA = """
|
|
16
|
+
CREATE TABLE site (
|
|
17
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
18
|
+
api_root TEXT NOT NULL
|
|
19
|
+
);
|
|
20
|
+
CREATE TABLE posts (
|
|
21
|
+
post_id TEXT PRIMARY KEY,
|
|
22
|
+
path TEXT NOT NULL UNIQUE,
|
|
23
|
+
body_checksum TEXT NOT NULL,
|
|
24
|
+
fields_checksum TEXT NOT NULL,
|
|
25
|
+
remote_fingerprint TEXT NOT NULL,
|
|
26
|
+
remote_format TEXT NOT NULL CHECK (remote_format IN ('blocks', 'html'))
|
|
27
|
+
);
|
|
28
|
+
CREATE TABLE media (
|
|
29
|
+
path TEXT PRIMARY KEY,
|
|
30
|
+
checksum TEXT NOT NULL,
|
|
31
|
+
media_id TEXT NOT NULL,
|
|
32
|
+
url TEXT NOT NULL UNIQUE,
|
|
33
|
+
size TEXT NOT NULL
|
|
34
|
+
);
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class StateError(Exception):
|
|
39
|
+
"""Raised when the state database is missing, foreign, or unreadable."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class PostRecord:
|
|
44
|
+
post_id: str
|
|
45
|
+
path: PurePosixPath
|
|
46
|
+
"""Relative to the profile's local root."""
|
|
47
|
+
body_checksum: str
|
|
48
|
+
fields_checksum: str
|
|
49
|
+
remote_fingerprint: str
|
|
50
|
+
remote_format: ContentFormat
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class MediaRecord:
|
|
55
|
+
path: PurePosixPath
|
|
56
|
+
"""Relative to the profile's local root."""
|
|
57
|
+
checksum: str
|
|
58
|
+
"""Of the file as downloaded or uploaded."""
|
|
59
|
+
media_id: str
|
|
60
|
+
url: str
|
|
61
|
+
size: str
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _check_site(connection: sqlite3.Connection, path: Path, api_root: str) -> None:
|
|
65
|
+
version = connection.execute("PRAGMA user_version").fetchone()[0]
|
|
66
|
+
if version != SCHEMA_VERSION:
|
|
67
|
+
raise StateError(f"{path} is not a version {SCHEMA_VERSION} wpsync database")
|
|
68
|
+
row = connection.execute("SELECT api_root FROM site").fetchone()
|
|
69
|
+
if row is None or row[0] != api_root:
|
|
70
|
+
recorded = row[0] if row else "no blog"
|
|
71
|
+
raise StateError(
|
|
72
|
+
f"{path} belongs to {recorded}, not {api_root}; this folder was "
|
|
73
|
+
"synchronized with a different blog"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@contextmanager
|
|
78
|
+
def _read_only(root: Path, api_root: str) -> Iterator[sqlite3.Connection | None]:
|
|
79
|
+
path = root / STATE_PATH
|
|
80
|
+
if not path.exists():
|
|
81
|
+
yield None
|
|
82
|
+
return
|
|
83
|
+
try:
|
|
84
|
+
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
85
|
+
try:
|
|
86
|
+
_check_site(connection, path, api_root)
|
|
87
|
+
yield connection
|
|
88
|
+
finally:
|
|
89
|
+
connection.close()
|
|
90
|
+
except sqlite3.Error as error:
|
|
91
|
+
raise StateError(f"cannot read {path}: {error}") from error
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def read_records(root: Path, api_root: str) -> dict[str, PostRecord]:
|
|
95
|
+
"""Read every post record without creating or changing the database."""
|
|
96
|
+
with _read_only(root, api_root) as connection:
|
|
97
|
+
if connection is None:
|
|
98
|
+
return {}
|
|
99
|
+
rows = connection.execute(
|
|
100
|
+
"SELECT post_id, path, body_checksum, fields_checksum, "
|
|
101
|
+
"remote_fingerprint, remote_format FROM posts"
|
|
102
|
+
).fetchall()
|
|
103
|
+
return {
|
|
104
|
+
row[0]: PostRecord(
|
|
105
|
+
row[0],
|
|
106
|
+
PurePosixPath(row[1]),
|
|
107
|
+
row[2],
|
|
108
|
+
row[3],
|
|
109
|
+
row[4],
|
|
110
|
+
ContentFormat(row[5]),
|
|
111
|
+
)
|
|
112
|
+
for row in rows
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def read_media(root: Path, api_root: str) -> list[MediaRecord]:
|
|
117
|
+
with _read_only(root, api_root) as connection:
|
|
118
|
+
if connection is None:
|
|
119
|
+
return []
|
|
120
|
+
rows = connection.execute(
|
|
121
|
+
"SELECT path, checksum, media_id, url, size FROM media"
|
|
122
|
+
).fetchall()
|
|
123
|
+
return [MediaRecord(PurePosixPath(row[0]), *row[1:]) for row in rows]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class StateWriter:
|
|
127
|
+
"""Writes records inside one transaction per synchronized post."""
|
|
128
|
+
|
|
129
|
+
def __init__(self, connection: sqlite3.Connection) -> None:
|
|
130
|
+
self.connection = connection
|
|
131
|
+
|
|
132
|
+
def save(self, record: PostRecord) -> None:
|
|
133
|
+
self.connection.execute(
|
|
134
|
+
"INSERT OR REPLACE INTO posts VALUES (?, ?, ?, ?, ?, ?)",
|
|
135
|
+
(
|
|
136
|
+
record.post_id,
|
|
137
|
+
str(record.path),
|
|
138
|
+
record.body_checksum,
|
|
139
|
+
record.fields_checksum,
|
|
140
|
+
record.remote_fingerprint,
|
|
141
|
+
record.remote_format.value,
|
|
142
|
+
),
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
def save_media(self, record: MediaRecord) -> None:
|
|
146
|
+
self.connection.execute(
|
|
147
|
+
"INSERT OR REPLACE INTO media VALUES (?, ?, ?, ?, ?)",
|
|
148
|
+
(
|
|
149
|
+
str(record.path),
|
|
150
|
+
record.checksum,
|
|
151
|
+
record.media_id,
|
|
152
|
+
record.url,
|
|
153
|
+
record.size,
|
|
154
|
+
),
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@contextmanager
|
|
159
|
+
def write_state(root: Path, api_root: str) -> Iterator[StateWriter]:
|
|
160
|
+
"""Open the database for one transaction, creating it for a new folder."""
|
|
161
|
+
path = root / STATE_PATH
|
|
162
|
+
path.parent.mkdir(exist_ok=True)
|
|
163
|
+
try:
|
|
164
|
+
connection = sqlite3.connect(path)
|
|
165
|
+
try:
|
|
166
|
+
if connection.execute("PRAGMA user_version").fetchone()[0] == 0:
|
|
167
|
+
with connection:
|
|
168
|
+
connection.executescript(SCHEMA)
|
|
169
|
+
connection.execute("INSERT INTO site VALUES (1, ?)", (api_root,))
|
|
170
|
+
connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
|
|
171
|
+
_check_site(connection, path, api_root)
|
|
172
|
+
with connection:
|
|
173
|
+
yield StateWriter(connection)
|
|
174
|
+
finally:
|
|
175
|
+
connection.close()
|
|
176
|
+
except sqlite3.Error as error:
|
|
177
|
+
raise StateError(f"cannot update {path}: {error}") from error
|
wpsync/storage.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import tempfile
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def write_atomic(path: Path, data: bytes, *, mode: int) -> None:
|
|
9
|
+
"""Replace a file so readers see either the old or the new content."""
|
|
10
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
11
|
+
descriptor, temporary_name = tempfile.mkstemp(
|
|
12
|
+
dir=path.parent, prefix=f".{path.name}."
|
|
13
|
+
)
|
|
14
|
+
temporary_path = Path(temporary_name)
|
|
15
|
+
try:
|
|
16
|
+
with os.fdopen(descriptor, "wb") as handle:
|
|
17
|
+
os.chmod(temporary_path, mode)
|
|
18
|
+
handle.write(data)
|
|
19
|
+
handle.flush()
|
|
20
|
+
os.fsync(handle.fileno())
|
|
21
|
+
temporary_path.replace(path)
|
|
22
|
+
except BaseException:
|
|
23
|
+
temporary_path.unlink(missing_ok=True)
|
|
24
|
+
raise
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def write_text_atomic(path: Path, text: str, *, mode: int) -> None:
|
|
28
|
+
write_atomic(path, text.encode(), mode=mode)
|