wpsync 1.17.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
wpsync/push.py ADDED
@@ -0,0 +1,288 @@
1
+ """Sending local posts to the platform."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ from collections.abc import Callable
7
+ from dataclasses import dataclass, field, replace
8
+ from functools import cached_property
9
+ from pathlib import Path, PurePosixPath
10
+ from typing import Protocol
11
+
12
+ from wpsync.documents import (
13
+ Document,
14
+ LocalDocument,
15
+ body_checksum,
16
+ display_path,
17
+ fields_checksum,
18
+ parse_document,
19
+ serialize_document,
20
+ write_document,
21
+ )
22
+ from wpsync.markup.markdown import image_sources
23
+ from wpsync.media import Media, MediaImage, MediaLibrary, PostMedia, image_path
24
+ from wpsync.posts import ContentFormat, Conversion, FetchedPost, Post
25
+ from wpsync.state import MediaRecord, PostRecord, write_state
26
+ from wpsync.sync import Action, DeleteStep, DiffEntry, Outcome, PushStep, Remote
27
+ from wpsync.trash import DeleteRemote, delete_post
28
+
29
+
30
+ class PushRemote(Remote, DeleteRemote, Protocol):
31
+ def fetch_posts(self, ids: list[str]) -> list[FetchedPost]: ...
32
+
33
+ def convert(self, post: FetchedPost, media: Media) -> Conversion: ...
34
+
35
+ def render(self, markdown: str, media: Media) -> str: ...
36
+
37
+ def upload(self, name: str, data: bytes) -> MediaImage: ...
38
+
39
+ def missing_terms(self, post: Post) -> list[str]:
40
+ """Descriptions of the post's categories and tags the platform lacks."""
41
+ ...
42
+
43
+ def save(
44
+ self,
45
+ post_id: str | None,
46
+ post: Post,
47
+ content: str | None,
48
+ fields: bool,
49
+ unpublish: bool,
50
+ ) -> FetchedPost: ...
51
+
52
+ def keeps_display(self, sent: str, stored: str) -> bool:
53
+ """Whether the platform stored content that displays as it was sent."""
54
+ ...
55
+
56
+
57
+ class _CannotSend(Exception):
58
+ """A post that cannot be sent as it stands, which does not stop the others."""
59
+
60
+
61
+ @dataclass
62
+ class PushResult:
63
+ changes: int = 0
64
+ """Posts and images created, updated, or deleted on the blog."""
65
+ skipped: int = 0
66
+ declined: int = 0
67
+ """Skipped posts that answering yes would have sent."""
68
+ notes: list[str] = field(default_factory=list)
69
+ """What WordPress did differently from what was sent."""
70
+
71
+ def summary(self) -> str:
72
+ skipped = f"; {self.skipped} skipped" if self.skipped else ""
73
+ if not self.changes:
74
+ return f" Nothing to push{skipped}."
75
+ changes = f"{self.changes} change{'s' * (self.changes != 1)}"
76
+ return f"Push complete: {changes}{skipped}."
77
+
78
+
79
+ class Pusher:
80
+ def __init__(
81
+ self,
82
+ root: Path,
83
+ api_root: str,
84
+ connect: Callable[[], PushRemote],
85
+ records: dict[str, PostRecord],
86
+ media: list[MediaRecord],
87
+ confirm: Callable[[str], bool],
88
+ report: Callable[[Outcome], None],
89
+ ) -> None:
90
+ self.root = root
91
+ self.api_root = api_root
92
+ self.connect = connect
93
+ self.records = records
94
+ self.library = MediaLibrary(root, media)
95
+ self.confirm = confirm
96
+ self.report = report
97
+ self.result = PushResult()
98
+
99
+ @cached_property
100
+ def remote(self) -> PushRemote:
101
+ return self.connect()
102
+
103
+ def push(self, entries: list[DiffEntry]) -> PushResult:
104
+ for entry in entries:
105
+ if isinstance(entry.step, DeleteStep):
106
+ self._delete(entry, entry.step)
107
+ elif entry.step is not None:
108
+ self._send(entry, entry.step)
109
+ elif entry.action is Action.CONFLICT:
110
+ self._skip(entry, entry.reason)
111
+ return self.result
112
+
113
+ def _send(self, entry: DiffEntry, step: PushStep) -> None:
114
+ document = step.document
115
+ post = document.document.post
116
+ path = PurePosixPath(document.path)
117
+ if step.question and not self.confirm(step.question):
118
+ self._decline(entry)
119
+ return
120
+ if step.target and step.content and self._is_block_post(step.target):
121
+ question = (
122
+ f"{entry.path} updates a block post, which becomes plain HTML. "
123
+ "Continue?"
124
+ )
125
+ if not self.confirm(question):
126
+ self._decline(entry)
127
+ return
128
+ missing = self.remote.missing_terms(post) if step.fields else []
129
+ if missing and not self.confirm(
130
+ f"{entry.path} needs {', '.join(missing)}. Create on WordPress?"
131
+ ):
132
+ self._decline(entry)
133
+ return
134
+ try:
135
+ if step.content:
136
+ self._upload_images(path, post.body)
137
+ except _CannotSend as error:
138
+ self._skip(entry, str(error))
139
+ return
140
+ media = PostMedia(self.library, path)
141
+ content = self.remote.render(post.body, media) if step.content else None
142
+ saved = self.remote.save(
143
+ step.target, post, content, step.fields, step.unpublish
144
+ )
145
+ self._record(step, document, saved, content)
146
+ self._report_sent(entry, step, saved, content)
147
+
148
+ def _delete(self, entry: DiffEntry, step: DeleteStep) -> None:
149
+ if step.question and not self.confirm(step.question):
150
+ self._decline(entry)
151
+ return
152
+ self.records.pop(step.target, None)
153
+ fetched = [] if step.gone else self.remote.fetch_posts([step.target])
154
+ if not fetched:
155
+ with write_state(self.root, self.api_root) as state:
156
+ state.forget(step.target)
157
+ self._report(Action.DELETED, entry.path)
158
+ return
159
+ post = fetched[0]
160
+ # The trash keeps the blog's version, since the local file is gone.
161
+ conversion = self.remote.convert(post, PostMedia(self.library, entry.path))
162
+ for download in self.library.take_pending():
163
+ self.library.release(download)
164
+ copy = Document(
165
+ post.id, replace(post.post, body=conversion.markdown.strip("\n"))
166
+ )
167
+ deleted = delete_post(
168
+ self.root,
169
+ self.api_root,
170
+ self.remote,
171
+ post.id,
172
+ entry.path,
173
+ copy,
174
+ on_disk=False,
175
+ )
176
+ self._report(Action.DELETED, entry.path)
177
+ for image in deleted.media:
178
+ self._report(Action.DELETED, image)
179
+
180
+ def _is_block_post(self, post_id: str) -> bool:
181
+ state = self.remote.post_states([post_id]).get(post_id)
182
+ return state is not None and state.format is ContentFormat.BLOCKS
183
+
184
+ def _upload_images(self, post: PurePosixPath, body: str) -> None:
185
+ """Upload the local images a body shows that the media table lacks."""
186
+ for source in sorted(image_sources(body)):
187
+ path = image_path(post, source)
188
+ if path is None:
189
+ continue
190
+ if path.is_absolute() or ".." in path.parts:
191
+ raise _CannotSend(f"image {source} is outside the profile folder")
192
+ known = self.library.by_path.get(path)
193
+ file = self.root / path
194
+ if not file.is_file():
195
+ # A media row without a file is an image missing from the blog
196
+ # too: its library markup is all there is to send.
197
+ if known is not None:
198
+ continue
199
+ raise _CannotSend(f"image not found: {source}")
200
+ data = file.read_bytes()
201
+ checksum = hashlib.sha256(data).hexdigest()
202
+ if known is not None and known.checksum in ("", checksum):
203
+ continue
204
+ image = self.remote.upload(path.name, data)
205
+ record = MediaRecord(path, checksum, image.id, image.url, image.size)
206
+ self.library.add(record)
207
+ # Saved at once, so a failure below cannot upload it twice.
208
+ with write_state(self.root, self.api_root) as state:
209
+ state.save_media(record)
210
+ self._report(Action.NEW if known is None else Action.CHANGED, path)
211
+
212
+ def _record(
213
+ self,
214
+ step: PushStep,
215
+ document: LocalDocument,
216
+ saved: FetchedPost,
217
+ content: str | None,
218
+ ) -> None:
219
+ current = document.document
220
+ post = current.post
221
+ if step.fields:
222
+ # What the platform decided becomes the file's, as a pull would write.
223
+ post = replace(
224
+ post,
225
+ slug=saved.post.slug or post.slug,
226
+ date=post.date or saved.post.date,
227
+ categories=saved.post.categories,
228
+ tags=saved.post.tags,
229
+ )
230
+ written = Document(saved.id, post, current.extra)
231
+ if written != current:
232
+ write_document(self.root / document.path, written)
233
+ reread = parse_document(serialize_document(written)).post
234
+ previous = self.records.get(saved.id)
235
+ # A body push did not send must still count as unsent next time.
236
+ body = body_checksum(reread)
237
+ if previous is not None and content is None:
238
+ body = previous.body_checksum
239
+ missing = previous.missing_files if previous else ()
240
+ if content is not None:
241
+ missing = tuple(url for url in missing if url in content)
242
+ record = PostRecord(
243
+ saved.id,
244
+ PurePosixPath(document.path),
245
+ body,
246
+ fields_checksum(reread),
247
+ saved.state.fingerprint,
248
+ saved.state.format,
249
+ saved.url,
250
+ missing,
251
+ )
252
+ self.records[saved.id] = record
253
+ with write_state(self.root, self.api_root) as state:
254
+ state.save(record)
255
+
256
+ def _report_sent(
257
+ self,
258
+ entry: DiffEntry,
259
+ step: PushStep,
260
+ saved: FetchedPost,
261
+ content: str | None,
262
+ ) -> None:
263
+ wanted = step.document.document.post.slug
264
+ if step.fields and wanted and saved.post.slug != wanted:
265
+ self.result.notes.append(
266
+ f"{display_path(entry.path)}: WordPress set the slug to "
267
+ f"{saved.post.slug}"
268
+ )
269
+ if content is not None and not self.remote.keeps_display(
270
+ content, saved.content
271
+ ):
272
+ self.result.notes.append(
273
+ f"{display_path(entry.path)}: WordPress changed the content; "
274
+ "check the post"
275
+ )
276
+ self._report(Action.NEW if step.target is None else Action.CHANGED, entry.path)
277
+
278
+ def _report(self, action: Action, path: PurePosixPath) -> None:
279
+ self.result.changes += 1
280
+ self.report(Outcome(action, str(path)))
281
+
282
+ def _decline(self, entry: DiffEntry) -> None:
283
+ self.result.declined += 1
284
+ self._skip(entry, "")
285
+
286
+ def _skip(self, entry: DiffEntry, failure: str) -> None:
287
+ self.result.skipped += 1
288
+ self.report(Outcome(Action.CONFLICT, str(entry.path), failure))
wpsync/state.py ADDED
@@ -0,0 +1,219 @@
1
+ """Synchronization state: what each linked post looked like at its last sync."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import sqlite3
7
+ from collections.abc import Iterator
8
+ from contextlib import contextmanager
9
+ from dataclasses import dataclass, replace
10
+ from pathlib import Path, PurePosixPath
11
+
12
+ from wpsync.posts import ContentFormat
13
+
14
+ STATE_PATH = Path(".wpsync") / "state.db"
15
+ SCHEMA_VERSION = 5
16
+ SCHEMA = """
17
+ CREATE TABLE site (
18
+ id INTEGER PRIMARY KEY CHECK (id = 1),
19
+ api_root TEXT NOT NULL
20
+ );
21
+ CREATE TABLE posts (
22
+ post_id TEXT PRIMARY KEY,
23
+ path TEXT NOT NULL,
24
+ body_checksum TEXT NOT NULL,
25
+ fields_checksum TEXT NOT NULL,
26
+ remote_fingerprint TEXT NOT NULL,
27
+ remote_format TEXT NOT NULL CHECK (remote_format IN ('blocks', 'html')),
28
+ url TEXT NOT NULL,
29
+ missing_files TEXT NOT NULL
30
+ );
31
+ CREATE TABLE media (
32
+ path TEXT PRIMARY KEY,
33
+ checksum TEXT NOT NULL,
34
+ media_id TEXT NOT NULL,
35
+ url TEXT NOT NULL UNIQUE,
36
+ size TEXT NOT NULL
37
+ );
38
+ """
39
+
40
+
41
+ class StateError(Exception):
42
+ """Raised when the state database is missing, foreign, or unreadable."""
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class PostRecord:
47
+ post_id: str
48
+ path: PurePosixPath
49
+ """Relative to the profile's local root."""
50
+ body_checksum: str
51
+ fields_checksum: str
52
+ remote_fingerprint: str
53
+ remote_format: ContentFormat
54
+ url: str
55
+ """Where the post is on the blog."""
56
+ missing_files: tuple[str, ...]
57
+ """Images and embedded pages whose files were gone at the last pull."""
58
+
59
+
60
+ @dataclass(frozen=True)
61
+ class MediaRecord:
62
+ path: PurePosixPath
63
+ """Relative to the profile's local root."""
64
+ checksum: str
65
+ """Of the file as downloaded or uploaded; empty while the file is missing."""
66
+ media_id: str
67
+ url: str
68
+ size: str
69
+
70
+
71
+ def _check_site(connection: sqlite3.Connection, path: Path, api_root: str) -> None:
72
+ version = connection.execute("PRAGMA user_version").fetchone()[0]
73
+ if version != SCHEMA_VERSION:
74
+ raise StateError(
75
+ f"{path} is from another version of wpsync; move it aside and run "
76
+ "wpsync pull to rebuild it"
77
+ )
78
+ row = connection.execute("SELECT api_root FROM site").fetchone()
79
+ if row is None or row[0] != api_root:
80
+ recorded = row[0] if row else "no blog"
81
+ raise StateError(
82
+ f"{path} belongs to {recorded}, not {api_root}; this folder was "
83
+ "synchronized with a different blog"
84
+ )
85
+
86
+
87
+ @contextmanager
88
+ def _read_only(root: Path, api_root: str) -> Iterator[sqlite3.Connection | None]:
89
+ path = root / STATE_PATH
90
+ if not path.exists():
91
+ yield None
92
+ return
93
+ try:
94
+ connection = sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True)
95
+ try:
96
+ _check_site(connection, path, api_root)
97
+ yield connection
98
+ finally:
99
+ connection.close()
100
+ except sqlite3.Error as error:
101
+ raise StateError(f"cannot read {path}: {error}") from error
102
+
103
+
104
+ def read_records(root: Path, api_root: str) -> dict[str, PostRecord]:
105
+ """Read every post record without creating or changing the database."""
106
+ with _read_only(root, api_root) as connection:
107
+ if connection is None:
108
+ return {}
109
+ rows = connection.execute(
110
+ "SELECT post_id, path, body_checksum, fields_checksum, "
111
+ "remote_fingerprint, remote_format, url, missing_files FROM posts"
112
+ ).fetchall()
113
+ return {
114
+ row[0]: PostRecord(
115
+ row[0],
116
+ PurePosixPath(row[1]),
117
+ row[2],
118
+ row[3],
119
+ row[4],
120
+ ContentFormat(row[5]),
121
+ row[6],
122
+ tuple(json.loads(row[7])),
123
+ )
124
+ for row in rows
125
+ }
126
+
127
+
128
+ def read_media(root: Path, api_root: str) -> list[MediaRecord]:
129
+ with _read_only(root, api_root) as connection:
130
+ if connection is None:
131
+ return []
132
+ rows = connection.execute(
133
+ "SELECT path, checksum, media_id, url, size FROM media"
134
+ ).fetchall()
135
+ return [MediaRecord(PurePosixPath(row[0]), *row[1:]) for row in rows]
136
+
137
+
138
+ class StateWriter:
139
+ """Writes records inside one transaction per synchronized post."""
140
+
141
+ def __init__(self, connection: sqlite3.Connection) -> None:
142
+ self.connection = connection
143
+
144
+ def save(self, record: PostRecord) -> None:
145
+ self.connection.execute(
146
+ "INSERT OR REPLACE INTO posts VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
147
+ (
148
+ record.post_id,
149
+ str(record.path),
150
+ record.body_checksum,
151
+ record.fields_checksum,
152
+ record.remote_fingerprint,
153
+ record.remote_format.value,
154
+ record.url,
155
+ json.dumps(record.missing_files),
156
+ ),
157
+ )
158
+
159
+ def forget(self, post_id: str) -> None:
160
+ self.connection.execute("DELETE FROM posts WHERE post_id = ?", (post_id,))
161
+
162
+ def forget_media(self, path: PurePosixPath) -> None:
163
+ self.connection.execute("DELETE FROM media WHERE path = ?", (str(path),))
164
+
165
+ def save_media(self, record: MediaRecord) -> None:
166
+ self.connection.execute(
167
+ "INSERT OR REPLACE INTO media VALUES (?, ?, ?, ?, ?)",
168
+ (
169
+ str(record.path),
170
+ record.checksum,
171
+ record.media_id,
172
+ record.url,
173
+ record.size,
174
+ ),
175
+ )
176
+
177
+
178
+ def follow_moves(
179
+ root: Path,
180
+ api_root: str,
181
+ records: dict[str, PostRecord],
182
+ paths: dict[str, PurePosixPath],
183
+ ) -> dict[str, PostRecord]:
184
+ """The records with each post's path updated to where its file is now.
185
+
186
+ A post is its ID; its path is only where its file was last seen.
187
+ """
188
+ moved = {
189
+ post_id: replace(record, path=paths[post_id])
190
+ for post_id, record in records.items()
191
+ if post_id in paths and paths[post_id] != record.path
192
+ }
193
+ if moved:
194
+ with write_state(root, api_root) as state:
195
+ for record in moved.values():
196
+ state.save(record)
197
+ return {**records, **moved}
198
+
199
+
200
+ @contextmanager
201
+ def write_state(root: Path, api_root: str) -> Iterator[StateWriter]:
202
+ """Open the database for one transaction, creating it for a new folder."""
203
+ path = root / STATE_PATH
204
+ path.parent.mkdir(exist_ok=True)
205
+ try:
206
+ connection = sqlite3.connect(path)
207
+ try:
208
+ if connection.execute("PRAGMA user_version").fetchone()[0] == 0:
209
+ with connection:
210
+ connection.executescript(SCHEMA)
211
+ connection.execute("INSERT INTO site VALUES (1, ?)", (api_root,))
212
+ connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
213
+ _check_site(connection, path, api_root)
214
+ with connection:
215
+ yield StateWriter(connection)
216
+ finally:
217
+ connection.close()
218
+ except sqlite3.Error as error:
219
+ raise StateError(f"cannot update {path}: {error}") from error
wpsync/storage.py ADDED
@@ -0,0 +1,28 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import tempfile
5
+ from pathlib import Path
6
+
7
+
8
+ def write_atomic(path: Path, data: bytes, *, mode: int) -> None:
9
+ """Replace a file so readers see either the old or the new content."""
10
+ path.parent.mkdir(parents=True, exist_ok=True)
11
+ descriptor, temporary_name = tempfile.mkstemp(
12
+ dir=path.parent, prefix=f".{path.name}."
13
+ )
14
+ temporary_path = Path(temporary_name)
15
+ try:
16
+ with os.fdopen(descriptor, "wb") as handle:
17
+ os.chmod(temporary_path, mode)
18
+ handle.write(data)
19
+ handle.flush()
20
+ os.fsync(handle.fileno())
21
+ temporary_path.replace(path)
22
+ except BaseException:
23
+ temporary_path.unlink(missing_ok=True)
24
+ raise
25
+
26
+
27
+ def write_text_atomic(path: Path, text: str, *, mode: int) -> None:
28
+ write_atomic(path, text.encode(), mode=mode)