wpsync 1.17.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
wpsync/posts.py ADDED
@@ -0,0 +1,78 @@
1
+ """Platform-neutral post types."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from datetime import datetime
7
+ from enum import StrEnum
8
+
9
+
10
+ class ContentFormat(StrEnum):
11
+ BLOCKS = "blocks"
12
+ HTML = "html"
13
+
14
+
15
+ class PostStatus(StrEnum):
16
+ """Whether push publishes a post. A private post is never pushed."""
17
+
18
+ PUBLIC = "public"
19
+ PRIVATE = "private"
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class Post:
24
+ """The synchronized fields of a post. Terms are sorted and unique."""
25
+
26
+ title: str = ""
27
+ slug: str = ""
28
+ date: datetime | None = None
29
+ status: PostStatus = PostStatus.PRIVATE
30
+ categories: tuple[str, ...] = ()
31
+ tags: tuple[str, ...] = ()
32
+ body: str = ""
33
+
34
+
35
+ @dataclass(frozen=True)
36
+ class PostSummary:
37
+ id: str
38
+ title: str
39
+ slug: str
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class RemoteState:
44
+ """What a post looks like on the platform, reduced to what sync compares."""
45
+
46
+ fingerprint: str
47
+ format: ContentFormat
48
+ published: bool
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class FetchedPost:
53
+ """A post as the platform stores it, before its content is converted."""
54
+
55
+ id: str
56
+ post: Post
57
+ """Every field except the body, which conversion produces."""
58
+ content: str
59
+ state: RemoteState
60
+ name: str
61
+ """The file name stem the post prefers, usually its slug."""
62
+ url: str
63
+ """Where the post is on the blog."""
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class Conversion:
68
+ """A post body converted to Markdown."""
69
+
70
+ markdown: str
71
+ kept: tuple[str, ...]
72
+ """Descriptions of elements kept as the platform's markup."""
73
+ embedded_pages: tuple[str, ...] = ()
74
+ """Pages the post embeds from the platform's own uploaded files."""
75
+
76
+
77
+ class UnrepresentableContent(ValueError):
78
+ """Content whose display cannot be reproduced, even keeping it as source."""
wpsync/pull.py ADDED
@@ -0,0 +1,400 @@
1
+ """Bringing posts from the platform into local files."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ from collections.abc import Callable
7
+ from dataclasses import dataclass, field, replace
8
+ from pathlib import Path, PurePosixPath
9
+ from typing import Protocol
10
+
11
+ from wpsync.documents import (
12
+ CONTENT_DIR,
13
+ Document,
14
+ LocalDocument,
15
+ body_checksum,
16
+ display_path,
17
+ fields_checksum,
18
+ parse_document,
19
+ serialize_document,
20
+ )
21
+ from wpsync.markup.markdown import image_sources
22
+ from wpsync.media import Media, MediaLibrary, MissingImage, PostMedia, image_path
23
+ from wpsync.posts import Conversion, FetchedPost, Post, UnrepresentableContent
24
+ from wpsync.state import MediaRecord, PostRecord, write_state
25
+ from wpsync.storage import write_atomic, write_text_atomic
26
+ from wpsync.sync import Action, Outcome, SelectionError, match_post
27
+
28
+
29
+ class PullRemote(Protocol):
30
+ def fetch_posts(
31
+ self, ids: list[str] | None = None, slug: str | None = None
32
+ ) -> list[FetchedPost]: ...
33
+
34
+ def convert(self, post: FetchedPost, media: Media) -> Conversion: ...
35
+
36
+ def download(self, url: str) -> bytes: ...
37
+
38
+ def available(self, url: str) -> bool: ...
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class _LocalCandidate:
43
+ """A local file without a post ID, as a match candidate."""
44
+
45
+ id: str
46
+ title: str
47
+ slug: str
48
+ document: LocalDocument
49
+
50
+
51
+ @dataclass
52
+ class PullResult:
53
+ changes: int = 0
54
+ """Post files and images written."""
55
+ missing_files: int = 0
56
+ skipped: int = 0
57
+ declined: int = 0
58
+ """Skipped posts that answering yes would have pulled."""
59
+ deleted_locally: list[str] = field(default_factory=list)
60
+ deleted_remotely: list[str] = field(default_factory=list)
61
+
62
+ def summary(self) -> str:
63
+ tail = ""
64
+ if self.missing_files:
65
+ missing = self.missing_files
66
+ tail += f"; {missing} file{'s' * (missing != 1)} missing from the blog"
67
+ if self.skipped:
68
+ tail += f"; {self.skipped} skipped"
69
+ if self.changes:
70
+ changes = f"{self.changes} change{'s' * (self.changes != 1)}"
71
+ lines = [f"Pull complete: {changes}{tail}."]
72
+ else:
73
+ lines = [f" Nothing to pull{tail}."]
74
+ for heading, paths in (
75
+ ("Deleted locally, not restored:", self.deleted_locally),
76
+ ("Deleted on WordPress, local files kept:", self.deleted_remotely),
77
+ ):
78
+ if paths:
79
+ lines.append(heading)
80
+ lines.extend(f" {display_path(path)}" for path in paths)
81
+ return "\n".join(lines)
82
+
83
+
84
+ class Puller:
85
+ def __init__(
86
+ self,
87
+ root: Path,
88
+ api_root: str,
89
+ remote: PullRemote,
90
+ documents: list[LocalDocument],
91
+ records: dict[str, PostRecord],
92
+ media: list[MediaRecord],
93
+ confirm: Callable[[str], bool],
94
+ report: Callable[[Outcome], None],
95
+ started: Callable[[int], None],
96
+ advanced: Callable[[], None],
97
+ ) -> None:
98
+ self.root = root
99
+ self.api_root = api_root
100
+ self.remote = remote
101
+ self.records = records
102
+ self.confirm = confirm
103
+ self.report = report
104
+ self.started = started
105
+ self.advanced = advanced
106
+ self.library = MediaLibrary(root, media)
107
+ self.documents = {
108
+ d.document.post_id: d for d in documents if d.document.post_id
109
+ }
110
+ self.unlinked = [
111
+ _LocalCandidate(str(d.path), d.document.post.title, d.document.post.slug, d)
112
+ for d in documents
113
+ if d.document.post_id is None
114
+ ]
115
+ self.taken = {PurePosixPath(d.path) for d in documents}
116
+ self.result = PullResult()
117
+
118
+ def pull(self, ids: list[str] | None = None, slug: str | None = None) -> PullResult:
119
+ fetched = self.remote.fetch_posts(ids=ids, slug=slug)
120
+ if not fetched and (ids or slug):
121
+ named = f"slug '{slug}'" if slug else f"id {ids[0]}" # type: ignore[index]
122
+ raise SelectionError(f"WordPress has no post with {named}")
123
+ self.started(len(fetched))
124
+ for post in fetched:
125
+ self._pull_post(post)
126
+ self.advanced()
127
+ if ids is None and slug is None:
128
+ present = {post.id for post in fetched}
129
+ for post_id, record in sorted(
130
+ self.records.items(), key=lambda r: int(r[0])
131
+ ):
132
+ if post_id not in present and post_id in self.documents:
133
+ path = str(self.documents[post_id].path)
134
+ self.result.deleted_remotely.append(path)
135
+ return self.result
136
+
137
+ def _pull_post(self, post: FetchedPost) -> None:
138
+ document = self.documents.get(post.id)
139
+ record = self.records.get(post.id)
140
+ if record is not None and document is None:
141
+ self.result.deleted_locally.append(str(record.path))
142
+ return
143
+ if record is not None and document is not None:
144
+ self._pull_tracked(post, document, record)
145
+ elif document is not None:
146
+ self._pull_unrecorded(post, document)
147
+ else:
148
+ self._pull_new(post)
149
+
150
+ def _pull_tracked(
151
+ self, post: FetchedPost, document: LocalDocument, record: PostRecord
152
+ ) -> None:
153
+ local = document.document.post
154
+ changed_locally = (
155
+ body_checksum(local) != record.body_checksum
156
+ or fields_checksum(local) != record.fields_checksum
157
+ )
158
+ if post.state.fingerprint == record.remote_fingerprint:
159
+ self._refresh(post, record)
160
+ return
161
+ path = PurePosixPath(document.path)
162
+ if changed_locally and not self.confirm(
163
+ f"#{post.id} {path} changed locally and on WordPress. "
164
+ "Replace your file with WordPress's version?"
165
+ ):
166
+ self._decline(path)
167
+ return
168
+ self._write(post, path, document.document, Action.CHANGED)
169
+
170
+ def _refresh(self, post: FetchedPost, record: PostRecord) -> None:
171
+ """Bring an unchanged post's address and missing files up to date."""
172
+ missing = tuple(url for url in record.missing_files if not self._is_back(url))
173
+ self.result.missing_files += len(missing)
174
+ current = replace(record, url=post.url, missing_files=missing)
175
+ if current != record:
176
+ with write_state(self.root, self.api_root) as state:
177
+ state.save(current)
178
+
179
+ def _is_back(self, url: str) -> bool:
180
+ """Whether a missing file is on the server again, saving it if an image."""
181
+ if url not in self.library.by_url:
182
+ return self.remote.available(url)
183
+ image = self.library.at_url(url)
184
+ if image.checksum:
185
+ return True
186
+ try:
187
+ data = self.remote.download(url)
188
+ except MissingImage:
189
+ return False
190
+ # A file put there by hand since is the user's: it stays, and differs
191
+ # from the checksum as any edited image does.
192
+ if not (self.root / image.path).exists():
193
+ write_atomic(self.root / image.path, data, mode=0o644)
194
+ restored = replace(image, checksum=hashlib.sha256(data).hexdigest())
195
+ self.library.add(restored)
196
+ with write_state(self.root, self.api_root) as state:
197
+ state.save_media(restored)
198
+ self._report(Action.NEW, image.path)
199
+ return True
200
+
201
+ def _pull_unrecorded(self, post: FetchedPost, document: LocalDocument) -> None:
202
+ path = PurePosixPath(document.path)
203
+ question = (
204
+ f"{path} has id {post.id} but was never synced here. "
205
+ "Replace it with WordPress's version?"
206
+ )
207
+ self._link(post, path, document.document, Action.CHANGED, question)
208
+
209
+ def _pull_new(self, post: FetchedPost) -> None:
210
+ match = match_post(post.post, self.unlinked)
211
+ if match.candidates:
212
+ files = ", ".join(c.id for c in match.candidates)
213
+ self._fail(f"#{post.id}", f"same slug as several local files: {files}")
214
+ return
215
+ if match.post is not None:
216
+ candidate = match.post
217
+ question = (
218
+ f"WordPress post #{post.id} {post.post.title!r} has the same slug "
219
+ f"as {candidate.id}. Link them and replace the file with "
220
+ "WordPress's version?"
221
+ )
222
+ if self._link(
223
+ post,
224
+ PurePosixPath(candidate.id),
225
+ candidate.document.document,
226
+ Action.NEW,
227
+ question,
228
+ ):
229
+ self.unlinked.remove(candidate)
230
+ return
231
+ path = self._new_path(post)
232
+ self.taken.add(path)
233
+ self._write(post, path, Document(post.id, post.post), Action.NEW)
234
+
235
+ def _link(
236
+ self,
237
+ post: FetchedPost,
238
+ path: PurePosixPath,
239
+ document: Document,
240
+ action: Action,
241
+ question: str,
242
+ ) -> bool:
243
+ """Link a file to a post, asking only if WordPress's version changes it."""
244
+ conversion = self._convert(post, path)
245
+ if conversion is None:
246
+ return False
247
+ remote = replace(post.post, body=conversion.markdown.strip("\n"))
248
+ if document.post_id == post.id and remote == document.post:
249
+ # The file is untouched, but the images it shows may not be there yet.
250
+ images, missing = self._save_media(path, conversion)
251
+ self._record(post, path, document, images, missing)
252
+ return True
253
+ if not _only_fills_in(document.post, remote) and not self.confirm(question):
254
+ for download in self.library.take_pending():
255
+ self.library.release(download)
256
+ self._decline(path)
257
+ return False
258
+ return self._write(post, path, document, action, conversion)
259
+
260
+ def _new_path(self, post: FetchedPost) -> PurePosixPath:
261
+ for name in (post.name, f"{post.name}-{post.id}"):
262
+ path = PurePosixPath(CONTENT_DIR) / f"{name}.md"
263
+ if path not in self.taken and not (self.root / path).exists():
264
+ return path
265
+ raise ValueError(f"no free file name for post {post.id}")
266
+
267
+ def _write(
268
+ self,
269
+ post: FetchedPost,
270
+ path: PurePosixPath,
271
+ existing: Document,
272
+ action: Action,
273
+ conversion: Conversion | None = None,
274
+ ) -> bool:
275
+ if conversion is None:
276
+ conversion = self._convert(post, path)
277
+ if conversion is None:
278
+ return False
279
+ images, missing = self._save_media(path, conversion)
280
+ written = Document(
281
+ post.id,
282
+ replace(post.post, body=conversion.markdown.strip("\n")),
283
+ existing.extra,
284
+ )
285
+ write_text_atomic(self.root / path, serialize_document(written), mode=0o644)
286
+ self._record(post, path, written, images, missing)
287
+ self._report(action, path)
288
+ return True
289
+
290
+ def _save_media(
291
+ self, path: PurePosixPath, conversion: Conversion
292
+ ) -> tuple[list[MediaRecord], list[str]]:
293
+ """Save the images a converted post shows, and report missing files."""
294
+ used = {
295
+ image
296
+ for source in image_sources(conversion.markdown)
297
+ if (image := image_path(path, source)) is not None
298
+ }
299
+ # Download first, so a failure leaves nothing of this post behind.
300
+ files = []
301
+ for download in self.library.take_pending():
302
+ if download.path not in used:
303
+ self.library.release(download)
304
+ continue
305
+ existing = self.root / download.path
306
+ if existing.is_file():
307
+ # A file at the image's own upload path is this image, saved
308
+ # by an earlier pull whose record is gone.
309
+ files.append((download, existing.read_bytes(), False))
310
+ continue
311
+ try:
312
+ files.append((download, self.remote.download(download.url), True))
313
+ except MissingImage:
314
+ files.append((download, None, True))
315
+ pages_gone = [
316
+ url for url in conversion.embedded_pages if not self.remote.available(url)
317
+ ]
318
+ images = []
319
+ for download, data, fresh in files:
320
+ checksum = ""
321
+ if data is not None:
322
+ if fresh:
323
+ write_atomic(self.root / download.path, data, mode=0o644)
324
+ self._report(Action.NEW, download.path)
325
+ checksum = hashlib.sha256(data).hexdigest()
326
+ image = MediaRecord(
327
+ download.path, checksum, download.media_id, download.url, download.size
328
+ )
329
+ self.library.add(image)
330
+ images.append(image)
331
+ shown = [self.library.by_path.get(image) for image in sorted(used)]
332
+ missing = [
333
+ record.url for record in shown if record and not record.checksum
334
+ ] + pages_gone
335
+ self.result.missing_files += len(missing)
336
+ return images, missing
337
+
338
+ def _convert(self, post: FetchedPost, path: PurePosixPath) -> Conversion | None:
339
+ try:
340
+ return self.remote.convert(post, PostMedia(self.library, path))
341
+ except UnrepresentableContent as error:
342
+ for download in self.library.take_pending():
343
+ self.library.release(download)
344
+ self._fail(str(path), str(error))
345
+ return None
346
+
347
+ def _record(
348
+ self,
349
+ post: FetchedPost,
350
+ path: PurePosixPath,
351
+ document: Document,
352
+ images: list[MediaRecord],
353
+ missing: list[str],
354
+ ) -> None:
355
+ # Checksums describe the file as it reads back, exactly as diff will see it.
356
+ written = parse_document(serialize_document(document)).post
357
+ with write_state(self.root, self.api_root) as state:
358
+ for image in images:
359
+ state.save_media(image)
360
+ state.save(
361
+ PostRecord(
362
+ post.id,
363
+ path,
364
+ body_checksum(written),
365
+ fields_checksum(written),
366
+ post.state.fingerprint,
367
+ post.state.format,
368
+ post.url,
369
+ tuple(missing),
370
+ )
371
+ )
372
+
373
+ def _report(self, action: Action, path: PurePosixPath) -> None:
374
+ self.result.changes += 1
375
+ self.report(Outcome(action, str(path)))
376
+
377
+ def _decline(self, path: PurePosixPath) -> None:
378
+ self.result.skipped += 1
379
+ self.result.declined += 1
380
+ self.report(Outcome(Action.CONFLICT, str(path)))
381
+
382
+ def _fail(self, label: str, reason: str) -> None:
383
+ self.result.skipped += 1
384
+ self.report(Outcome(Action.CONFLICT, label, reason))
385
+
386
+
387
+ def _only_fills_in(local: Post, remote: Post) -> bool:
388
+ """Whether a post differs from a file only in fields the file leaves empty.
389
+
390
+ The title and content never count as empty: an untitled or blank file
391
+ sharing a slug with a post is more likely meant as a new post.
392
+ """
393
+ filled = replace(
394
+ local,
395
+ slug=local.slug or remote.slug,
396
+ date=local.date or remote.date,
397
+ categories=local.categories or remote.categories,
398
+ tags=local.tags or remote.tags,
399
+ )
400
+ return filled == remote