wpsync 1.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of wpsync might be problematic. Click here for more details.

wpsync/pull.py ADDED
@@ -0,0 +1,324 @@
1
+ """Bringing posts from the platform into local files."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ from collections.abc import Callable
7
+ from dataclasses import dataclass, replace
8
+ from pathlib import Path, PurePosixPath
9
+ from typing import Protocol
10
+
11
+ from wpsync.documents import (
12
+ POSTS_DIR,
13
+ Document,
14
+ LocalDocument,
15
+ body_checksum,
16
+ fields_checksum,
17
+ parse_document,
18
+ serialize_document,
19
+ )
20
+ from wpsync.markup.markdown import image_sources
21
+ from wpsync.media import Media, MediaLibrary
22
+ from wpsync.posts import Conversion, FetchedPost, UnrepresentableContent
23
+ from wpsync.state import MediaRecord, PostRecord, write_state
24
+ from wpsync.storage import write_atomic, write_text_atomic
25
+ from wpsync.sync import Action, DiffEntry, SelectionError, match_post
26
+
27
+
28
+ class PullRemote(Protocol):
29
+ def fetch_posts(
30
+ self, ids: list[str] | None = None, slug: str | None = None
31
+ ) -> list[FetchedPost]: ...
32
+
33
+ def convert(self, post: FetchedPost, media: Media) -> Conversion: ...
34
+
35
+ def download(self, url: str) -> bytes: ...
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class _LocalCandidate:
40
+ """A local file without a post ID, as a match candidate."""
41
+
42
+ id: str
43
+ title: str
44
+ slug: str
45
+ document: LocalDocument
46
+
47
+
48
+ @dataclass
49
+ class PullResult:
50
+ new: int = 0
51
+ updated: int = 0
52
+ linked: int = 0
53
+ images: int = 0
54
+ skipped: int = 0
55
+ changes_kept: int = 0
56
+
57
+ def summary(self) -> str:
58
+ pulled = self.new + self.updated + self.linked
59
+ if not pulled and not self.skipped:
60
+ kept = self.changes_kept
61
+ if kept:
62
+ posts = f"{kept} post{'s' * (kept != 1)}"
63
+ return f"Nothing to pull; {posts} with local changes."
64
+ return "Nothing to pull."
65
+ parts = [
66
+ f"{count} {label}"
67
+ for count, label in (
68
+ (self.new, "new"),
69
+ (self.updated, "updated"),
70
+ (self.linked, "linked"),
71
+ )
72
+ if count
73
+ ]
74
+ line = f"Pulled {pulled} post{'s' * (pulled != 1)}"
75
+ line += f" ({', '.join(parts)})" if parts else ""
76
+ if self.images:
77
+ line += f", {self.images} image{'s' * (self.images != 1)}"
78
+ if self.skipped:
79
+ line += f"; {self.skipped} skipped"
80
+ if self.changes_kept:
81
+ line += f"; {self.changes_kept} with local changes left alone"
82
+ return line + "."
83
+
84
+
85
+ class Puller:
86
+ def __init__(
87
+ self,
88
+ root: Path,
89
+ api_root: str,
90
+ remote: PullRemote,
91
+ documents: list[LocalDocument],
92
+ records: dict[str, PostRecord],
93
+ media: list[MediaRecord],
94
+ confirm: Callable[[str], bool],
95
+ report: Callable[[DiffEntry], None],
96
+ ) -> None:
97
+ self.root = root
98
+ self.api_root = api_root
99
+ self.remote = remote
100
+ self.records = records
101
+ self.confirm = confirm
102
+ self.report = report
103
+ self.library = MediaLibrary(root, media)
104
+ self.documents = {
105
+ d.document.post_id: d for d in documents if d.document.post_id
106
+ }
107
+ self.unlinked = [
108
+ _LocalCandidate(str(d.path), d.document.post.title, d.document.post.slug, d)
109
+ for d in documents
110
+ if d.document.post_id is None
111
+ ]
112
+ self.taken = {PurePosixPath(d.path) for d in documents}
113
+ self.result = PullResult()
114
+
115
+ def pull(self, ids: list[str] | None = None, slug: str | None = None) -> PullResult:
116
+ fetched = self.remote.fetch_posts(ids=ids, slug=slug)
117
+ if not fetched and (ids or slug):
118
+ named = f"slug '{slug}'" if slug else f"id {ids[0]}" # type: ignore[index]
119
+ raise SelectionError(f"WordPress has no post with {named}")
120
+ for post in fetched:
121
+ self._pull_post(post)
122
+ if ids is None and slug is None:
123
+ present = {post.id for post in fetched}
124
+ for post_id, record in sorted(
125
+ self.records.items(), key=lambda r: int(r[0])
126
+ ):
127
+ if post_id not in present and post_id in self.documents:
128
+ self._line(
129
+ Action.DELETED,
130
+ "r",
131
+ post_id,
132
+ record.path,
133
+ "deleted on WordPress; local file kept",
134
+ )
135
+ return self.result
136
+
137
+ def _pull_post(self, post: FetchedPost) -> None:
138
+ document = self.documents.get(post.id)
139
+ record = self.records.get(post.id)
140
+ if record is not None and document is None:
141
+ self._line(
142
+ Action.DELETED,
143
+ "l",
144
+ post.id,
145
+ record.path,
146
+ "deleted locally; not restored",
147
+ )
148
+ return
149
+ if record is not None and document is not None:
150
+ self._pull_tracked(post, document, record)
151
+ elif document is not None:
152
+ self._pull_unrecorded(post, document)
153
+ else:
154
+ self._pull_new(post)
155
+
156
+ def _pull_tracked(
157
+ self, post: FetchedPost, document: LocalDocument, record: PostRecord
158
+ ) -> None:
159
+ local = document.document.post
160
+ changed_locally = (
161
+ body_checksum(local) != record.body_checksum
162
+ or fields_checksum(local) != record.fields_checksum
163
+ )
164
+ if post.state.fingerprint == record.remote_fingerprint:
165
+ self.result.changes_kept += changed_locally
166
+ return
167
+ path = PurePosixPath(document.path)
168
+ if changed_locally and not self.confirm(
169
+ f"#{post.id} {path} changed locally and on WordPress. "
170
+ "Replace your file with WordPress's version?"
171
+ ):
172
+ self._skip(post.id, path, "changed on both sides; your file is unchanged")
173
+ return
174
+ if self._write(post, path, document.document, Action.CHANGED):
175
+ self.result.updated += 1
176
+
177
+ def _pull_unrecorded(self, post: FetchedPost, document: LocalDocument) -> None:
178
+ path = PurePosixPath(document.path)
179
+ conversion = self._convert(post, path)
180
+ if conversion is None:
181
+ return
182
+ remote = replace(post.post, body=conversion.markdown.strip("\n"))
183
+ if remote == document.document.post:
184
+ for download in self.library.take_pending():
185
+ self.library.release(download)
186
+ self._record(post, path, document.document, [])
187
+ self._line(Action.UNCHANGED, "", post.id, path, "linked; already identical")
188
+ self.result.linked += 1
189
+ return
190
+ if not self.confirm(
191
+ f"{path} has id {post.id} but was never synced here. "
192
+ "Replace it with WordPress's version?"
193
+ ):
194
+ for download in self.library.take_pending():
195
+ self.library.release(download)
196
+ self._skip(post.id, path, "never synced here; your file is unchanged")
197
+ return
198
+ if self._write(post, path, document.document, Action.CHANGED, conversion):
199
+ self.result.linked += 1
200
+
201
+ def _pull_new(self, post: FetchedPost) -> None:
202
+ match = match_post(post.post, self.unlinked)
203
+ if match.candidates:
204
+ files = ", ".join(c.id for c in match.candidates)
205
+ self._skip(post.id, None, f"matches several local files: {files}")
206
+ return
207
+ if match.post is not None:
208
+ candidate = match.post
209
+ path = PurePosixPath(candidate.id)
210
+ if not self.confirm(
211
+ f"WordPress post #{post.id} {post.post.title!r} matches {path} "
212
+ f"({match.reason}). Link them and replace the file with "
213
+ "WordPress's version?"
214
+ ):
215
+ self._skip(
216
+ post.id, path, f"matches this file ({match.reason}); not linked"
217
+ )
218
+ return
219
+ self.unlinked.remove(candidate)
220
+ if self._write(post, path, candidate.document.document, Action.NEW):
221
+ self.result.linked += 1
222
+ return
223
+ path = self._new_path(post)
224
+ self.taken.add(path)
225
+ if self._write(post, path, Document(post.id, post.post), Action.NEW):
226
+ self.result.new += 1
227
+
228
+ def _new_path(self, post: FetchedPost) -> PurePosixPath:
229
+ for name in (post.name, f"{post.name}-{post.id}"):
230
+ path = PurePosixPath(POSTS_DIR) / f"{name}.md"
231
+ if path not in self.taken and not (self.root / path).exists():
232
+ return path
233
+ raise ValueError(f"no free file name for post {post.id}")
234
+
235
+ def _write(
236
+ self,
237
+ post: FetchedPost,
238
+ path: PurePosixPath,
239
+ existing: Document,
240
+ action: Action,
241
+ conversion: Conversion | None = None,
242
+ ) -> bool:
243
+ if conversion is None:
244
+ conversion = self._convert(post, path)
245
+ if conversion is None:
246
+ return False
247
+ images = []
248
+ used = image_sources(conversion.markdown)
249
+ for download in self.library.take_pending():
250
+ if str(download.path) not in used:
251
+ self.library.release(download)
252
+ continue
253
+ data = self.remote.download(download.url)
254
+ write_atomic(self.root / download.path, data, mode=0o644)
255
+ image = MediaRecord(
256
+ download.path,
257
+ hashlib.sha256(data).hexdigest(),
258
+ download.media_id,
259
+ download.url,
260
+ download.size,
261
+ )
262
+ self.library.add(image)
263
+ images.append(image)
264
+ written = Document(
265
+ post.id,
266
+ replace(post.post, body=conversion.markdown.strip("\n")),
267
+ existing.extra,
268
+ )
269
+ write_text_atomic(self.root / path, serialize_document(written), mode=0o644)
270
+ self._record(post, path, written, images)
271
+ self.result.images += len(images)
272
+ details = []
273
+ if images:
274
+ details.append(f"{len(images)} image{'s' * (len(images) != 1)}")
275
+ if conversion.kept:
276
+ details.append("kept as HTML: " + ", ".join(dict.fromkeys(conversion.kept)))
277
+ self._line(action, "r", post.id, path, "; ".join(details))
278
+ return True
279
+
280
+ def _convert(self, post: FetchedPost, path: PurePosixPath) -> Conversion | None:
281
+ try:
282
+ return self.remote.convert(post, self.library)
283
+ except UnrepresentableContent as error:
284
+ for download in self.library.take_pending():
285
+ self.library.release(download)
286
+ self._skip(post.id, path, f"not pulled: {error}")
287
+ return None
288
+
289
+ def _record(
290
+ self,
291
+ post: FetchedPost,
292
+ path: PurePosixPath,
293
+ document: Document,
294
+ images: list[MediaRecord],
295
+ ) -> None:
296
+ # Checksums describe the file as it reads back, exactly as diff will see it.
297
+ written = parse_document(serialize_document(document)).post
298
+ with write_state(self.root, self.api_root) as state:
299
+ for image in images:
300
+ state.save_media(image)
301
+ state.save(
302
+ PostRecord(
303
+ post.id,
304
+ path,
305
+ body_checksum(written),
306
+ fields_checksum(written),
307
+ post.state.fingerprint,
308
+ post.state.format,
309
+ )
310
+ )
311
+
312
+ def _skip(self, post_id: str, path: PurePosixPath | None, detail: str) -> None:
313
+ self.result.skipped += 1
314
+ self._line(Action.CONFLICT, "", post_id, path or PurePosixPath("-"), detail)
315
+
316
+ def _line(
317
+ self,
318
+ action: Action,
319
+ side: str,
320
+ post_id: str,
321
+ path: PurePosixPath,
322
+ detail: str = "",
323
+ ) -> None:
324
+ self.report(DiffEntry(action, side, post_id, path, detail))
wpsync/state.py ADDED
@@ -0,0 +1,177 @@
1
+ """Synchronization state: what each linked post looked like at its last sync."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import sqlite3
6
+ from collections.abc import Iterator
7
+ from contextlib import contextmanager
8
+ from dataclasses import dataclass
9
+ from pathlib import Path, PurePosixPath
10
+
11
+ from wpsync.posts import ContentFormat
12
+
13
+ STATE_PATH = Path(".wpsync") / "state.db"
14
+ SCHEMA_VERSION = 1
15
+ SCHEMA = """
16
+ CREATE TABLE site (
17
+ id INTEGER PRIMARY KEY CHECK (id = 1),
18
+ api_root TEXT NOT NULL
19
+ );
20
+ CREATE TABLE posts (
21
+ post_id TEXT PRIMARY KEY,
22
+ path TEXT NOT NULL UNIQUE,
23
+ body_checksum TEXT NOT NULL,
24
+ fields_checksum TEXT NOT NULL,
25
+ remote_fingerprint TEXT NOT NULL,
26
+ remote_format TEXT NOT NULL CHECK (remote_format IN ('blocks', 'html'))
27
+ );
28
+ CREATE TABLE media (
29
+ path TEXT PRIMARY KEY,
30
+ checksum TEXT NOT NULL,
31
+ media_id TEXT NOT NULL,
32
+ url TEXT NOT NULL UNIQUE,
33
+ size TEXT NOT NULL
34
+ );
35
+ """
36
+
37
+
38
+ class StateError(Exception):
39
+ """Raised when the state database is missing, foreign, or unreadable."""
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class PostRecord:
44
+ post_id: str
45
+ path: PurePosixPath
46
+ """Relative to the profile's local root."""
47
+ body_checksum: str
48
+ fields_checksum: str
49
+ remote_fingerprint: str
50
+ remote_format: ContentFormat
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class MediaRecord:
55
+ path: PurePosixPath
56
+ """Relative to the profile's local root."""
57
+ checksum: str
58
+ """Of the file as downloaded or uploaded."""
59
+ media_id: str
60
+ url: str
61
+ size: str
62
+
63
+
64
+ def _check_site(connection: sqlite3.Connection, path: Path, api_root: str) -> None:
65
+ version = connection.execute("PRAGMA user_version").fetchone()[0]
66
+ if version != SCHEMA_VERSION:
67
+ raise StateError(f"{path} is not a version {SCHEMA_VERSION} wpsync database")
68
+ row = connection.execute("SELECT api_root FROM site").fetchone()
69
+ if row is None or row[0] != api_root:
70
+ recorded = row[0] if row else "no blog"
71
+ raise StateError(
72
+ f"{path} belongs to {recorded}, not {api_root}; this folder was "
73
+ "synchronized with a different blog"
74
+ )
75
+
76
+
77
+ @contextmanager
78
+ def _read_only(root: Path, api_root: str) -> Iterator[sqlite3.Connection | None]:
79
+ path = root / STATE_PATH
80
+ if not path.exists():
81
+ yield None
82
+ return
83
+ try:
84
+ connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
85
+ try:
86
+ _check_site(connection, path, api_root)
87
+ yield connection
88
+ finally:
89
+ connection.close()
90
+ except sqlite3.Error as error:
91
+ raise StateError(f"cannot read {path}: {error}") from error
92
+
93
+
94
+ def read_records(root: Path, api_root: str) -> dict[str, PostRecord]:
95
+ """Read every post record without creating or changing the database."""
96
+ with _read_only(root, api_root) as connection:
97
+ if connection is None:
98
+ return {}
99
+ rows = connection.execute(
100
+ "SELECT post_id, path, body_checksum, fields_checksum, "
101
+ "remote_fingerprint, remote_format FROM posts"
102
+ ).fetchall()
103
+ return {
104
+ row[0]: PostRecord(
105
+ row[0],
106
+ PurePosixPath(row[1]),
107
+ row[2],
108
+ row[3],
109
+ row[4],
110
+ ContentFormat(row[5]),
111
+ )
112
+ for row in rows
113
+ }
114
+
115
+
116
+ def read_media(root: Path, api_root: str) -> list[MediaRecord]:
117
+ with _read_only(root, api_root) as connection:
118
+ if connection is None:
119
+ return []
120
+ rows = connection.execute(
121
+ "SELECT path, checksum, media_id, url, size FROM media"
122
+ ).fetchall()
123
+ return [MediaRecord(PurePosixPath(row[0]), *row[1:]) for row in rows]
124
+
125
+
126
+ class StateWriter:
127
+ """Writes records inside one transaction per synchronized post."""
128
+
129
+ def __init__(self, connection: sqlite3.Connection) -> None:
130
+ self.connection = connection
131
+
132
+ def save(self, record: PostRecord) -> None:
133
+ self.connection.execute(
134
+ "INSERT OR REPLACE INTO posts VALUES (?, ?, ?, ?, ?, ?)",
135
+ (
136
+ record.post_id,
137
+ str(record.path),
138
+ record.body_checksum,
139
+ record.fields_checksum,
140
+ record.remote_fingerprint,
141
+ record.remote_format.value,
142
+ ),
143
+ )
144
+
145
+ def save_media(self, record: MediaRecord) -> None:
146
+ self.connection.execute(
147
+ "INSERT OR REPLACE INTO media VALUES (?, ?, ?, ?, ?)",
148
+ (
149
+ str(record.path),
150
+ record.checksum,
151
+ record.media_id,
152
+ record.url,
153
+ record.size,
154
+ ),
155
+ )
156
+
157
+
158
+ @contextmanager
159
+ def write_state(root: Path, api_root: str) -> Iterator[StateWriter]:
160
+ """Open the database for one transaction, creating it for a new folder."""
161
+ path = root / STATE_PATH
162
+ path.parent.mkdir(exist_ok=True)
163
+ try:
164
+ connection = sqlite3.connect(path)
165
+ try:
166
+ if connection.execute("PRAGMA user_version").fetchone()[0] == 0:
167
+ with connection:
168
+ connection.executescript(SCHEMA)
169
+ connection.execute("INSERT INTO site VALUES (1, ?)", (api_root,))
170
+ connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
171
+ _check_site(connection, path, api_root)
172
+ with connection:
173
+ yield StateWriter(connection)
174
+ finally:
175
+ connection.close()
176
+ except sqlite3.Error as error:
177
+ raise StateError(f"cannot update {path}: {error}") from error
wpsync/storage.py ADDED
@@ -0,0 +1,28 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import tempfile
5
+ from pathlib import Path
6
+
7
+
8
+ def write_atomic(path: Path, data: bytes, *, mode: int) -> None:
9
+ """Replace a file so readers see either the old or the new content."""
10
+ path.parent.mkdir(parents=True, exist_ok=True)
11
+ descriptor, temporary_name = tempfile.mkstemp(
12
+ dir=path.parent, prefix=f".{path.name}."
13
+ )
14
+ temporary_path = Path(temporary_name)
15
+ try:
16
+ with os.fdopen(descriptor, "wb") as handle:
17
+ os.chmod(temporary_path, mode)
18
+ handle.write(data)
19
+ handle.flush()
20
+ os.fsync(handle.fileno())
21
+ temporary_path.replace(path)
22
+ except BaseException:
23
+ temporary_path.unlink(missing_ok=True)
24
+ raise
25
+
26
+
27
+ def write_text_atomic(path: Path, text: str, *, mode: int) -> None:
28
+ write_atomic(path, text.encode(), mode=mode)