margin 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- margin/__init__.py +0 -0
- margin/batch.py +41 -0
- margin/claude.py +89 -0
- margin/cli.py +294 -0
- margin/config.py +188 -0
- margin/fold.py +750 -0
- margin/images.py +66 -0
- margin/inbox.py +335 -0
- margin/outline.py +282 -0
- margin/patches.py +243 -0
- margin/pinch_check.py +363 -0
- margin/skills/margin/SKILL.md +95 -0
- margin/store.py +186 -0
- margin/sync.py +666 -0
- margin-0.2.1.dist-info/METADATA +113 -0
- margin-0.2.1.dist-info/RECORD +20 -0
- margin-0.2.1.dist-info/WHEEL +5 -0
- margin-0.2.1.dist-info/entry_points.txt +2 -0
- margin-0.2.1.dist-info/licenses/LICENSE +21 -0
- margin-0.2.1.dist-info/top_level.txt +1 -0
margin/sync.py
ADDED
|
@@ -0,0 +1,666 @@
|
|
|
1
|
+
"""Publish a repo's markdown into an Outline collection, keeping comments alive.
|
|
2
|
+
|
|
3
|
+
Each repo is one collection. Folders become parent documents, and a folder's README.md is that
|
|
4
|
+
folder document's text. A file's first `# Heading` becomes the document title.
|
|
5
|
+
|
|
6
|
+
The rule this module exists to keep: a document that may carry comments is only ever PATCHED,
|
|
7
|
+
never replaced, because a replace removes every comment anchor. To patch, margin needs the new
|
|
8
|
+
file in Outline's own markdown style (Outline rewrites `-` lists as `*`, escapes, and so on), so it
|
|
9
|
+
writes the file into a private draft first and reads back what Outline made of it.
|
|
10
|
+
|
|
11
|
+
Direction (phase 1): repo to Outline. Both sides' text at the last sync is kept, so:
|
|
12
|
+
only the repo changed -> patch Outline
|
|
13
|
+
only Outline changed -> leave both alone and report it (phase 2 brings it back to the repo)
|
|
14
|
+
both changed -> conflict: leave both alone and report it
|
|
15
|
+
Nothing is ever settled by "the newest wins".
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import difflib
|
|
20
|
+
import fnmatch
|
|
21
|
+
import hashlib
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from pathlib import Path, PurePosixPath
|
|
24
|
+
|
|
25
|
+
from . import images
|
|
26
|
+
from .config import ALWAYS_EXCLUDE, Repo
|
|
27
|
+
from .outline import Outline, OutlineError
|
|
28
|
+
from .patches import CannotPatch, adopt, apply, plan, spelled_alike
|
|
29
|
+
from .store import Store, repo_lock
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Result:
|
|
34
|
+
path: str
|
|
35
|
+
action: str # created | patched | unchanged | outline-edited | conflict | ...
|
|
36
|
+
detail: str = ""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
TITLE_MAX = 100 # Outline refuses a longer title with a 400
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def loose(text: str) -> str:
|
|
43
|
+
"""Text with only its spacing removed: blank lines between blocks and spaces at the end of a
|
|
44
|
+
line. Outline keeps an empty paragraph a patch tries to remove (a third newline), and one
|
|
45
|
+
between two list items splits the list in two, so the words match while the newlines do not."""
|
|
46
|
+
import re
|
|
47
|
+
return re.sub(r"\n+", "\n", re.sub(r"[ \t]+\n", "\n", text)).strip()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _epoch(stamp: str | None) -> float:
|
|
51
|
+
"""Seconds since 1970 for an Outline timestamp ("2026-10-09T12:12:30.547Z"); 0 when absent."""
|
|
52
|
+
import datetime as dt
|
|
53
|
+
if not stamp:
|
|
54
|
+
return 0.0
|
|
55
|
+
return dt.datetime.fromisoformat(stamp.replace("Z", "+00:00")).timestamp()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def clamp_title(title: str, limit: int = TITLE_MAX) -> str:
|
|
59
|
+
"""A title Outline accepts: cut at a word boundary, with an ellipsis when it was cut."""
|
|
60
|
+
title = " ".join(title.split())
|
|
61
|
+
if len(title) <= limit:
|
|
62
|
+
return title
|
|
63
|
+
cut = title[: limit - 1].rsplit(" ", 1)[0] or title[: limit - 1]
|
|
64
|
+
return cut.rstrip(" ,.;:-") + "…"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def split_title(text: str, fallback: str) -> tuple[str, str]:
|
|
68
|
+
lines = text.lstrip("").split("\n")
|
|
69
|
+
i = 0
|
|
70
|
+
while i < len(lines) and not lines[i].strip():
|
|
71
|
+
i += 1
|
|
72
|
+
if i < len(lines) and lines[i].startswith("# "):
|
|
73
|
+
body = "\n".join(lines[i + 1:]).strip("\n")
|
|
74
|
+
return lines[i][2:].strip() or fallback, body
|
|
75
|
+
return fallback, text.strip("\n")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _match(rel: str, patterns: list[str]) -> bool:
|
|
79
|
+
p = PurePosixPath(rel)
|
|
80
|
+
return any(p.match(pat) or fnmatch.fnmatch(rel, pat) or
|
|
81
|
+
(pat.startswith("**/") and fnmatch.fnmatch(rel, pat[3:])) for pat in patterns)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def files(repo: Repo) -> list[str]:
|
|
85
|
+
"""Repo-relative paths of every markdown file margin publishes. Working tree, not git, so
|
|
86
|
+
untracked and git-ignored docs are included on purpose.
|
|
87
|
+
|
|
88
|
+
Excluded folders are skipped during the walk, not filtered afterwards, so a root that holds
|
|
89
|
+
tens of thousands of other files (a home config folder full of transcripts) costs nothing.
|
|
90
|
+
|
|
91
|
+
A linked folder is walked like a real one (a skills folder whose skills are links to where
|
|
92
|
+
each package keeps it): unwalked, its documents read as gone from the repo. A folder already
|
|
93
|
+
walked under another name is not walked again, so a link to an ancestor cannot loop.
|
|
94
|
+
|
|
95
|
+
A repository inside another repository is skipped: it is someone else's code cloned into a
|
|
96
|
+
project (a library's documentation, a vendored model zoo), not the project's own writing. A
|
|
97
|
+
folder of projects that is not a repository keeps each project, even though each has its own
|
|
98
|
+
`.git`."""
|
|
99
|
+
import os
|
|
100
|
+
excludes = ALWAYS_EXCLUDE + repo.exclude
|
|
101
|
+
md_only = all(p.lower().endswith(".md") for p in repo.include)
|
|
102
|
+
out = []
|
|
103
|
+
seen: set[str] = set()
|
|
104
|
+
in_repo = {os.path.realpath(repo.path): (Path(repo.path) / ".git").exists()}
|
|
105
|
+
for dirpath, dirnames, filenames in os.walk(repo.path, followlinks=True):
|
|
106
|
+
real = os.path.realpath(dirpath)
|
|
107
|
+
if real in seen:
|
|
108
|
+
dirnames[:] = []
|
|
109
|
+
continue
|
|
110
|
+
seen.add(real)
|
|
111
|
+
inside = in_repo.get(real, False)
|
|
112
|
+
base = Path(dirpath).relative_to(repo.path).as_posix()
|
|
113
|
+
base = "" if base == "." else base + "/"
|
|
114
|
+
# a folder is skipped when anything inside it would be excluded, or when it is a
|
|
115
|
+
# repository of its own inside a repository
|
|
116
|
+
keep = []
|
|
117
|
+
for d in sorted(dirnames):
|
|
118
|
+
if _match(f"{base}{d}/x", excludes):
|
|
119
|
+
continue
|
|
120
|
+
own = os.path.exists(os.path.join(dirpath, d, ".git"))
|
|
121
|
+
if own and inside:
|
|
122
|
+
continue
|
|
123
|
+
in_repo[os.path.realpath(os.path.join(dirpath, d))] = inside or own
|
|
124
|
+
keep.append(d)
|
|
125
|
+
dirnames[:] = keep
|
|
126
|
+
for name in sorted(filenames):
|
|
127
|
+
if md_only and not name.lower().endswith(".md"):
|
|
128
|
+
continue # cheap: most of a big folder is not markdown at all
|
|
129
|
+
rel = base + name
|
|
130
|
+
if _match(rel, repo.include) and not _match(rel, excludes):
|
|
131
|
+
out.append(rel)
|
|
132
|
+
return sorted(out)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def tree(paths: list[str]) -> list[tuple[str, str | None]]:
|
|
136
|
+
"""(key, source file) for every document, parents first. A folder's key is its path and its
|
|
137
|
+
source is its README.md, when it has one. A README at the repo root is an ordinary page."""
|
|
138
|
+
keys: dict[str, str | None] = {}
|
|
139
|
+
for rel in paths:
|
|
140
|
+
parts = rel.split("/")
|
|
141
|
+
for depth in range(1, len(parts)):
|
|
142
|
+
keys.setdefault("/".join(parts[:depth]), None)
|
|
143
|
+
for rel in paths:
|
|
144
|
+
parent, name = rel.rpartition("/")[0], rel.rpartition("/")[2]
|
|
145
|
+
if parent and name.lower() == "readme.md":
|
|
146
|
+
keys[parent] = rel
|
|
147
|
+
else:
|
|
148
|
+
keys[rel] = rel
|
|
149
|
+
return sorted(keys.items(), key=lambda kv: (kv[0].count("/"), kv[0]))
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def outline_hunks(base: str, current: str) -> list[tuple[list[str], list[str]]]:
|
|
154
|
+
"""An edit in Outline as hunks, each (lines it removed, lines it added), blank lines left out."""
|
|
155
|
+
a, b = base.splitlines(), current.splitlines()
|
|
156
|
+
out = []
|
|
157
|
+
for op, i1, i2, j1, j2 in difflib.SequenceMatcher(None, a, b, autojunk=False).get_opcodes():
|
|
158
|
+
if op != "equal":
|
|
159
|
+
out.append(([x for x in a[i1:i2] if x.strip()], [x for x in b[j1:j2] if x.strip()]))
|
|
160
|
+
return out
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def outline_edit(base: str, current: str) -> tuple[list[str], list[str]]:
|
|
164
|
+
"""The non-blank lines an edit in Outline added (or changed to) and the ones it removed, from
|
|
165
|
+
the text at the last sync to the text now. A removed line that is still somewhere in the
|
|
166
|
+
document is not counted as removed."""
|
|
167
|
+
a, b = base.splitlines(), current.splitlines()
|
|
168
|
+
added, removed = [], []
|
|
169
|
+
for op, i1, i2, j1, j2 in difflib.SequenceMatcher(None, a, b, autojunk=False).get_opcodes():
|
|
170
|
+
if op in ("replace", "insert"):
|
|
171
|
+
added += [line for line in b[j1:j2] if line.strip()]
|
|
172
|
+
if op in ("replace", "delete"):
|
|
173
|
+
removed += [line for line in a[i1:i2] if line.strip()]
|
|
174
|
+
still = {line.strip() for line in b}
|
|
175
|
+
return added, [line for line in removed if line.strip() not in still]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
#: what a document's detail says when its whole text was replaced instead of patched
|
|
179
|
+
REPLACED = "replaced: no person had edited it and it had no comments"
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class Syncer:
|
|
183
|
+
def __init__(self, outline: Outline, store: Store, repo: Repo, full: bool = False):
|
|
184
|
+
self.o, self.s, self.repo = outline, store, repo
|
|
185
|
+
self.full = full # also fetch documents whose file did not change
|
|
186
|
+
self._collection: str | None = None
|
|
187
|
+
|
|
188
|
+
# ---- the collection and the scratch draft
|
|
189
|
+
@property
|
|
190
|
+
def collection(self) -> str:
|
|
191
|
+
if self._collection is None:
|
|
192
|
+
key = f"collection:{self.repo.name}"
|
|
193
|
+
cid = self.s.get(key)
|
|
194
|
+
if cid:
|
|
195
|
+
try:
|
|
196
|
+
self.o.data("collections.info", {"id": cid})
|
|
197
|
+
except OutlineError as e:
|
|
198
|
+
if e.status != 404:
|
|
199
|
+
raise
|
|
200
|
+
cid = None
|
|
201
|
+
if not cid:
|
|
202
|
+
cid = self.o.ensure_collection(
|
|
203
|
+
self.repo.name, f"Markdown from {self.repo.name}, published by margin.")["id"]
|
|
204
|
+
self.s.put(key, cid)
|
|
205
|
+
self._collection = cid
|
|
206
|
+
return self._collection
|
|
207
|
+
|
|
208
|
+
def canonical(self, body: str) -> str:
|
|
209
|
+
"""`body` as Outline itself writes it back, via a private draft only margin uses."""
|
|
210
|
+
if not body.strip():
|
|
211
|
+
return ""
|
|
212
|
+
h = hashlib.sha256(body.encode()).hexdigest()
|
|
213
|
+
cached = self.s.get(f"canon:{h}")
|
|
214
|
+
if cached is not None:
|
|
215
|
+
return cached
|
|
216
|
+
key = "scratch:" + self.repo.name
|
|
217
|
+
sid = self.s.get(key)
|
|
218
|
+
doc = None
|
|
219
|
+
if sid:
|
|
220
|
+
try:
|
|
221
|
+
doc = self.o.replace(sid, body)
|
|
222
|
+
except OutlineError as e:
|
|
223
|
+
if e.status != 404:
|
|
224
|
+
raise
|
|
225
|
+
if doc is None:
|
|
226
|
+
doc = self.o.data("documents.create", {"collectionId": self.collection,
|
|
227
|
+
"title": "margin scratch (private draft)",
|
|
228
|
+
"text": body, "publish": False})
|
|
229
|
+
self.s.put(key, doc["id"])
|
|
230
|
+
text = doc["text"]
|
|
231
|
+
self.s.put(f"canon:{h}", text)
|
|
232
|
+
return text
|
|
233
|
+
|
|
234
|
+
def resolve(self, body: str, base: Path, doc_id: str) -> str:
|
|
235
|
+
"""`body` with each local picture it shows uploaded as an attachment of `doc_id` and
|
|
236
|
+
pointed at. A picture already uploaded for this document with the same content is reused."""
|
|
237
|
+
pics = images.local(body, base)
|
|
238
|
+
if not pics:
|
|
239
|
+
return body
|
|
240
|
+
urls = {}
|
|
241
|
+
for ref, path in pics:
|
|
242
|
+
data = path.read_bytes()
|
|
243
|
+
key = f"image:{doc_id}:{hashlib.sha256(data).hexdigest()}"
|
|
244
|
+
url = self.s.get(key)
|
|
245
|
+
if url is None:
|
|
246
|
+
url = self.o.upload(path.name, images.TYPES[path.suffix.lower()], data, doc_id)
|
|
247
|
+
self.s.put(key, url)
|
|
248
|
+
urls[ref] = url
|
|
249
|
+
return images.point(body, urls)
|
|
250
|
+
|
|
251
|
+
# ---- one pass
|
|
252
|
+
def run(self) -> list[Result]:
|
|
253
|
+
with repo_lock(self.s, self.repo.name):
|
|
254
|
+
return self._run()
|
|
255
|
+
|
|
256
|
+
def publish(self, rels: list[str]) -> list[tuple[str, dict]]:
|
|
257
|
+
"""Publish just these repo-relative files (and the folders above them) now, and return
|
|
258
|
+
(path, document) for each, so a caller can hand out links without waiting for a pass."""
|
|
259
|
+
return [(rel, doc) for rel, doc, _status, _detail in self.publish_status(rels)]
|
|
260
|
+
|
|
261
|
+
def publish_status(self, rels: list[str], root: Path | None = None
|
|
262
|
+
) -> list[tuple[str, dict, str, str]]:
|
|
263
|
+
"""`publish`, with each file's sync status after it ("ok", "conflict", "cannot-patch",
|
|
264
|
+
"drift", "outline-edited", ...) and its detail. Only "ok" means the document now says
|
|
265
|
+
what the file says; anything else means the link points at an older or different text."""
|
|
266
|
+
out = []
|
|
267
|
+
with repo_lock(self.s, self.repo.name):
|
|
268
|
+
if root is not None and Path(root).resolve() != self.repo.path.resolve():
|
|
269
|
+
for rel in rels: # linked from a worktree: pin them
|
|
270
|
+
parent, name = rel.rpartition("/")[0], rel.rpartition("/")[2]
|
|
271
|
+
self.pin(parent if parent and name.lower() == "readme.md" else rel,
|
|
272
|
+
Path(root) / rel)
|
|
273
|
+
wanted = set()
|
|
274
|
+
for rel in rels:
|
|
275
|
+
parts = rel.split("/")
|
|
276
|
+
wanted.update("/".join(parts[:d]) for d in range(1, len(parts)))
|
|
277
|
+
parent, name = rel.rpartition("/")[0], rel.rpartition("/")[2]
|
|
278
|
+
wanted.add(parent if parent and name.lower() == "readme.md" else rel)
|
|
279
|
+
done = set()
|
|
280
|
+
for key, src in tree(files(self.repo)):
|
|
281
|
+
if key in wanted:
|
|
282
|
+
self.one(key, src)
|
|
283
|
+
done.add(key)
|
|
284
|
+
for key in sorted((wanted - done) & set(self.pins())): # only in a worktree so far
|
|
285
|
+
if self.source(key, None) is not None:
|
|
286
|
+
self.one(key, None)
|
|
287
|
+
for rel in rels:
|
|
288
|
+
parent, name = rel.rpartition("/")[0], rel.rpartition("/")[2]
|
|
289
|
+
key = parent if parent and name.lower() == "readme.md" else rel
|
|
290
|
+
row = self.s.doc(self.repo.name, key)
|
|
291
|
+
if row:
|
|
292
|
+
out.append((rel, self.o.document(row["doc_id"]), row["status"] or "ok",
|
|
293
|
+
row["detail"] or ""))
|
|
294
|
+
return out
|
|
295
|
+
|
|
296
|
+
def accept_file(self, rel: str, root: Path | None = None) -> Result:
|
|
297
|
+
"""Let the file's side win a conflict that no person made: the document's text since the
|
|
298
|
+
last sync was written only by margin's own user, and nobody has commented on it. Outline's
|
|
299
|
+
current text becomes the base, so the next pass sees only the file's change and patches it
|
|
300
|
+
in. Anything a person did refuses it, because their edit or their comment's anchor would be
|
|
301
|
+
lost; then ask them."""
|
|
302
|
+
parent, name = rel.rpartition("/")[0], rel.rpartition("/")[2]
|
|
303
|
+
key = parent if parent and name.lower() == "readme.md" else rel
|
|
304
|
+
with repo_lock(self.s, self.repo.name):
|
|
305
|
+
row = self.s.doc(self.repo.name, key)
|
|
306
|
+
if row is None:
|
|
307
|
+
return Result(key, "refused", "it was never published; margin link publishes it")
|
|
308
|
+
if root is not None and Path(root).resolve() != self.repo.path.resolve():
|
|
309
|
+
self.pin(key, Path(root) / rel) # the worktree's version is the file
|
|
310
|
+
if row["status"] in ("ok", None):
|
|
311
|
+
return Result(key, "refused", "there is no conflict: the document matches the file")
|
|
312
|
+
me = self.o.me()["id"]
|
|
313
|
+
notes = list(self.o.comments(document_id=row["doc_id"]))
|
|
314
|
+
if notes:
|
|
315
|
+
return Result(key, "refused", f"{len(notes)} comment(s) on the document; answer them "
|
|
316
|
+
"first, because the file's text would move their anchors")
|
|
317
|
+
since = (row["synced_at"] or 0) - 60
|
|
318
|
+
people = set()
|
|
319
|
+
for r in self.o.paged("revisions.list", {"documentId": row["doc_id"]}):
|
|
320
|
+
who = r.get("createdBy") or {}
|
|
321
|
+
if who.get("id") != me and _epoch(r.get("createdAt")) > since:
|
|
322
|
+
people.add(who.get("name") or who.get("id") or "someone")
|
|
323
|
+
doc = self.o.document(row["doc_id"])
|
|
324
|
+
last = doc.get("updatedBy") or {}
|
|
325
|
+
if last.get("id") not in (None, me):
|
|
326
|
+
people.add(last.get("name") or "someone")
|
|
327
|
+
if people:
|
|
328
|
+
return Result(key, "refused", f"edited in Outline by {', '.join(sorted(people))} "
|
|
329
|
+
"since the last sync; ask them which text is right")
|
|
330
|
+
# Outline's text becomes the base, and the file's baseline is cleared so the next step
|
|
331
|
+
# patches the file in even when only Outline had moved (outline-edited)
|
|
332
|
+
self.s.save_doc(self.repo.name, key, row["doc_id"], row["collection_id"],
|
|
333
|
+
"", doc["text"], "ok", None)
|
|
334
|
+
src = dict(tree(files(self.repo))).get(key)
|
|
335
|
+
r = self.one(key, src)
|
|
336
|
+
if r.action not in ("cannot-patch", "drift"):
|
|
337
|
+
return r
|
|
338
|
+
# Nobody but margin wrote it and nothing is anchored in it, so the whole text may be
|
|
339
|
+
# replaced, the same reason an empty document is replaced (a replace drops comment
|
|
340
|
+
# anchors, and here there are none).
|
|
341
|
+
file = self.repo.path / self.source(key, src)
|
|
342
|
+
text = file.read_text(errors="replace")
|
|
343
|
+
title, body = split_title(text, PurePosixPath(key).name)
|
|
344
|
+
ident = text + images.digest(body, file.parent)
|
|
345
|
+
after = self.o.replace(row["doc_id"], self.resolve(body, file.parent, row["doc_id"]))["text"]
|
|
346
|
+
self.s.save_doc(self.repo.name, key, row["doc_id"], row["collection_id"],
|
|
347
|
+
ident, after, "ok",
|
|
348
|
+
"the file replaced the document (no person had edited it, no comments)")
|
|
349
|
+
return Result(key, "patched", "replaced: no person had edited it and it had no comments")
|
|
350
|
+
|
|
351
|
+
# ---- files linked from a worktree
|
|
352
|
+
# ⚠️ EVERY CHAT WORKS IN ITS OWN GIT WORKTREE (2026-10-12), and margin read only the main
|
|
353
|
+
# folder, so a new document was published by copying it there by hand. A file linked from a
|
|
354
|
+
# worktree is PINNED: until the main folder has the same text (or the worktree file is gone),
|
|
355
|
+
# the sync pass publishes the worktree's version instead of the main folder's older one, and
|
|
356
|
+
# does not mark a file that exists only in the worktree as missing.
|
|
357
|
+
def _pin_key(self, key: str) -> str:
|
|
358
|
+
return f"pin:{self.repo.name}:{key}"
|
|
359
|
+
|
|
360
|
+
def pin(self, key: str, file: Path) -> None:
|
|
361
|
+
self.s.put(self._pin_key(key), str(Path(file).resolve()))
|
|
362
|
+
|
|
363
|
+
def pins(self) -> dict[str, Path]:
|
|
364
|
+
prefix = self._pin_key("")
|
|
365
|
+
rows = self.s.db.execute("SELECT k, v FROM kv WHERE k LIKE ?", (prefix + "%",)).fetchall()
|
|
366
|
+
return {r["k"][len(prefix):]: Path(r["v"]) for r in rows if r["v"]}
|
|
367
|
+
|
|
368
|
+
def unpin(self, key: str) -> None:
|
|
369
|
+
self.s.db.execute("DELETE FROM kv WHERE k=?", (self._pin_key(key),))
|
|
370
|
+
|
|
371
|
+
def accept_merged(self, rel: str, root: Path | None = None, undo: tuple[str, ...] = ()) -> Result:
|
|
372
|
+
"""Settle a conflict a person made, once their Outline edit has been copied into the file.
|
|
373
|
+
Every line they added or changed since the last sync must be in the file as Outline writes
|
|
374
|
+
it, and no line they removed may be in it; otherwise it refuses and names the line, so
|
|
375
|
+
their edit is never lost. Then Outline's text becomes the base and the file is patched in,
|
|
376
|
+
which leaves every comment anchored on text the file did not change.
|
|
377
|
+
|
|
378
|
+
`undo` names parts of their edit the person agreed to undo (2026-10-13: a heading marker and
|
|
379
|
+
a deleted phrase that were slips while reviewing on a phone). A line of their edit that
|
|
380
|
+
contains one of them is not required in the file, so the file's version wins there."""
|
|
381
|
+
parent, name = rel.rpartition("/")[0], rel.rpartition("/")[2]
|
|
382
|
+
key = parent if parent and name.lower() == "readme.md" else rel
|
|
383
|
+
with repo_lock(self.s, self.repo.name):
|
|
384
|
+
row = self.s.doc(self.repo.name, key)
|
|
385
|
+
if row is None:
|
|
386
|
+
return Result(key, "refused", "it was never published; margin link publishes it")
|
|
387
|
+
if root is not None and Path(root).resolve() != self.repo.path.resolve():
|
|
388
|
+
self.pin(key, Path(root) / rel) # the worktree's version is the file
|
|
389
|
+
src = self.source(key, dict(tree(files(self.repo))).get(key))
|
|
390
|
+
if src is None:
|
|
391
|
+
return Result(key, "refused", "the file is gone")
|
|
392
|
+
current = self.o.document(row["doc_id"])["text"]
|
|
393
|
+
if current == row["base_outline"]:
|
|
394
|
+
return Result(key, "refused", "nobody edited the document in Outline since the "
|
|
395
|
+
"last sync; margin link publishes the file")
|
|
396
|
+
file = self.repo.path / src
|
|
397
|
+
title, body = split_title(file.read_text(errors="replace"), PurePosixPath(key).name)
|
|
398
|
+
target = self.canonical(self.resolve(body, file.parent, row["doc_id"]))
|
|
399
|
+
agreed = [u for u in undo if u.strip()]
|
|
400
|
+
if agreed:
|
|
401
|
+
# a hunk the person agreed to undo is left out whole: its new lines and the lines
|
|
402
|
+
# they replaced, so the file's version of that place wins
|
|
403
|
+
kept = [(r, a) for r, a in outline_hunks(row["base_outline"], current)
|
|
404
|
+
if not any(u in ln for u in agreed for ln in r + a)]
|
|
405
|
+
still = {ln.strip() for ln in current.splitlines()}
|
|
406
|
+
added = [ln for _, a in kept for ln in a]
|
|
407
|
+
removed = [ln for r, _ in kept for ln in r if ln.strip() not in still]
|
|
408
|
+
else:
|
|
409
|
+
added, removed = outline_edit(row["base_outline"], current)
|
|
410
|
+
have = {line.strip() for line in target.splitlines()}
|
|
411
|
+
missing = [line for line in added if line.strip() not in have]
|
|
412
|
+
kept = [line for line in removed if line.strip() in have]
|
|
413
|
+
if missing or kept:
|
|
414
|
+
said = [f"lacks {m.strip()[:80]!r}" for m in missing[:3]]
|
|
415
|
+
said += [f"still has {k.strip()[:80]!r}, which was removed in Outline" for k in kept[:3]]
|
|
416
|
+
return Result(key, "refused", "the file does not carry the Outline edit: "
|
|
417
|
+
+ ", ".join(said))
|
|
418
|
+
# Outline's text is the base now, and the file's baseline is cleared so the next step
|
|
419
|
+
# patches the file in even when the file had not moved since the last sync
|
|
420
|
+
self.s.save_doc(self.repo.name, key, row["doc_id"], row["collection_id"],
|
|
421
|
+
"", current, "ok", None)
|
|
422
|
+
return self.one(key, src)
|
|
423
|
+
|
|
424
|
+
def source(self, key: str, src: str | None) -> str | None:
|
|
425
|
+
"""Where `key`'s text comes from: its pinned worktree file while that differs from the main
|
|
426
|
+
folder's, else the main folder's file. A pin lifts itself once the main folder caught up or
|
|
427
|
+
the worktree file is gone."""
|
|
428
|
+
pinned = self.pins().get(key)
|
|
429
|
+
if pinned is None:
|
|
430
|
+
return src
|
|
431
|
+
if not pinned.exists():
|
|
432
|
+
self.unpin(key)
|
|
433
|
+
return src
|
|
434
|
+
main = self.repo.path / src if src else None
|
|
435
|
+
if main is not None and main.exists() and main.read_bytes() == pinned.read_bytes():
|
|
436
|
+
self.unpin(key)
|
|
437
|
+
return src
|
|
438
|
+
return str(pinned)
|
|
439
|
+
|
|
440
|
+
def _run(self) -> list[Result]:
|
|
441
|
+
paths = files(self.repo)
|
|
442
|
+
results = []
|
|
443
|
+
seen = set()
|
|
444
|
+
for key, src in tree(paths):
|
|
445
|
+
seen.add(key)
|
|
446
|
+
try:
|
|
447
|
+
results.append(self.one(key, src))
|
|
448
|
+
except OutlineError as e:
|
|
449
|
+
results.append(Result(key, "error", str(e)))
|
|
450
|
+
for key in sorted(set(self.pins()) - seen): # files so far only in a worktree
|
|
451
|
+
if self.source(key, None) is None:
|
|
452
|
+
continue # the pin lifted and the main folder has no file: missing, below
|
|
453
|
+
seen.add(key)
|
|
454
|
+
try:
|
|
455
|
+
results.append(self.one(key, None))
|
|
456
|
+
except OutlineError as e:
|
|
457
|
+
results.append(Result(key, "error", str(e)))
|
|
458
|
+
for row in self.s.docs(self.repo.name):
|
|
459
|
+
if row["path"] not in seen and row["status"] != "missing-file":
|
|
460
|
+
self.s.mark(self.repo.name, row["path"], "missing-file",
|
|
461
|
+
"the file is gone from the repo; the document is kept")
|
|
462
|
+
results.append(Result(row["path"], "missing-file"))
|
|
463
|
+
return results
|
|
464
|
+
|
|
465
|
+
def converge(self, doc_id: str, current: str, target: str, tries: int = 4) -> None:
|
|
466
|
+
"""Patch until Outline's text is the target. Every patch's result is checked against what
|
|
467
|
+
it should have produced; on a difference, re-plan from what Outline really has, with
|
|
468
|
+
patches that cross one more block boundary."""
|
|
469
|
+
if target.startswith(current) and target[len(current):].startswith("\n\n"):
|
|
470
|
+
# only new blocks at the end: append them, which leaves every existing block alone
|
|
471
|
+
current = self.o.append(doc_id, target[len(current):])["text"]
|
|
472
|
+
if spelled_alike(current, target):
|
|
473
|
+
return
|
|
474
|
+
# ⚠️ ONE HUNK OUTLINE WILL NOT TAKE MUST NOT STOP THE OTHERS (2026-10-12). Outline keeps an
|
|
475
|
+
# empty paragraph a patch tries to remove (it reads as a third newline between blocks), and
|
|
476
|
+
# the hunks go bottom-up, so that one hunk used to end the pass and every change above it
|
|
477
|
+
# (a report's intro, its title) never landed. A hunk whose result differs from what it
|
|
478
|
+
# should have produced is set aside, and the rest are re-planned from what Outline has;
|
|
479
|
+
# only when nothing else is left do the hunks widen across more block boundaries.
|
|
480
|
+
failed: set = set()
|
|
481
|
+
widen = 0
|
|
482
|
+
budget = 2 * len(plan(current, target, 0)) + 4 * tries
|
|
483
|
+
# ⭐ OUTLINE SPELLS SOME TEXT TWO WAYS (2026-10-16). Saved whole, two literal asterisks are `\**`
|
|
484
|
+
# and a paragraph's lines end in plain newlines; saved through a patch, the same text comes
|
|
485
|
+
# back as `\*\*` and with hard breaks (a backslash at each line's end). The target is the
|
|
486
|
+
# whole-body spelling, so those hunks never matched and fifteen documents stayed at
|
|
487
|
+
# cannot-patch, retried on every pass. Lines spelled alike keep Outline's own spelling
|
|
488
|
+
# (patches.adopt), and a result spelled alike counts as the one expected.
|
|
489
|
+
while budget > 0 and not spelled_alike(current, target):
|
|
490
|
+
hunks = [h for h in plan(current, adopt(current, target), widen) if h not in failed]
|
|
491
|
+
if not hunks:
|
|
492
|
+
if widen + 1 >= tries:
|
|
493
|
+
break
|
|
494
|
+
widen += 1
|
|
495
|
+
continue
|
|
496
|
+
for find, repl in hunks:
|
|
497
|
+
budget -= 1
|
|
498
|
+
if find not in current:
|
|
499
|
+
# ⛔ planned to follow a hunk that was set aside or spelled differently: plan
|
|
500
|
+
# again from what Outline has. It used to end the pass with "find text not present"
|
|
501
|
+
break
|
|
502
|
+
expected = apply(current, [(find, repl)])
|
|
503
|
+
current = self.o.patch(doc_id, find, repl)["text"]
|
|
504
|
+
if current != expected:
|
|
505
|
+
if not spelled_alike(current, expected):
|
|
506
|
+
failed.add((find, repl))
|
|
507
|
+
break
|
|
508
|
+
if not spelled_alike(current, target) and loose(current) != loose(target):
|
|
509
|
+
raise CannotPatch("Outline's text still differs from the file after widening")
|
|
510
|
+
|
|
511
|
+
def only_margin_edited(self, doc: dict) -> bool:
|
|
512
|
+
"""Whether nobody but margin's own account has edited this document: its last editor is
|
|
513
|
+
margin and every saved revision was made by margin. Then a difference from the baseline
|
|
514
|
+
is margin's own (an earlier pass stopped part way), not a person's edit. Any other editor,
|
|
515
|
+
or no answer, keeps the person's edit safe."""
|
|
516
|
+
last = (doc.get("updatedBy") or {}).get("id")
|
|
517
|
+
if not last:
|
|
518
|
+
return False
|
|
519
|
+
try:
|
|
520
|
+
me = (self.o.me() or {}).get("id")
|
|
521
|
+
if last != me:
|
|
522
|
+
return False
|
|
523
|
+
revs = self.o.revisions(doc["id"])
|
|
524
|
+
except OutlineError:
|
|
525
|
+
return False
|
|
526
|
+
return bool(revs) and all((r.get("createdBy") or {}).get("id") == me for r in revs)
|
|
527
|
+
|
|
528
|
+
def replaceable(self, doc_id: str, doc: dict) -> bool:
|
|
529
|
+
"""Whether the whole text of this document may be replaced without losing anything a person
|
|
530
|
+
made: nobody but margin has edited it (only_margin_edited) and no comment is anchored in
|
|
531
|
+
it. Any comment, or any answer Outline withholds, keeps it patch-only."""
|
|
532
|
+
if not self.only_margin_edited(doc):
|
|
533
|
+
return False
|
|
534
|
+
try:
|
|
535
|
+
return not list(self.o.comments(document_id=doc_id))
|
|
536
|
+
except OutlineError:
|
|
537
|
+
return False
|
|
538
|
+
|
|
539
|
+
def _parent_id(self, key: str) -> str | None:
|
|
540
|
+
parent = key.rpartition("/")[0]
|
|
541
|
+
if not parent:
|
|
542
|
+
return None
|
|
543
|
+
row = self.s.doc(self.repo.name, parent)
|
|
544
|
+
return row["doc_id"] if row else None
|
|
545
|
+
|
|
546
|
+
def one(self, key: str, src: str | None) -> Result:
|
|
547
|
+
src = self.source(key, src)
|
|
548
|
+
name = PurePosixPath(key).name
|
|
549
|
+
fallback = name[:-3] if name.lower().endswith(".md") else name
|
|
550
|
+
text = (self.repo.path / src).read_text(errors="replace") if src else ""
|
|
551
|
+
title, body = split_title(text, fallback)
|
|
552
|
+
title = clamp_title(title)
|
|
553
|
+
base = (self.repo.path / src).parent if src else self.repo.path
|
|
554
|
+
# what margin compares: the file, plus each picture's content (a changed picture is a change)
|
|
555
|
+
ident = text + images.digest(body, base)
|
|
556
|
+
row = self.s.doc(self.repo.name, key)
|
|
557
|
+
|
|
558
|
+
if row is None:
|
|
559
|
+
# a picture is attached to its document, so a document with pictures is made first
|
|
560
|
+
# and filled once it has an id (it is new: no comment can be anchored in it yet)
|
|
561
|
+
pics = images.local(body, base)
|
|
562
|
+
doc = self.o.create_document(self.collection, title, "" if pics else body,
|
|
563
|
+
self._parent_id(key))
|
|
564
|
+
if pics:
|
|
565
|
+
doc = self.o.replace(doc["id"], self.resolve(body, base, doc["id"]))
|
|
566
|
+
self.s.save_doc(self.repo.name, key, doc["id"], self.collection, ident, doc["text"])
|
|
567
|
+
return Result(key, "created")
|
|
568
|
+
|
|
569
|
+
repo_changed = ident != row["base_repo"]
|
|
570
|
+
failed_key = f"failed:{row['doc_id']}"
|
|
571
|
+
if (row["status"] == "cannot-patch" and repo_changed and not self.full
|
|
572
|
+
and self.s.get(failed_key) == ident):
|
|
573
|
+
# ⛔ NOT AGAIN FOR THE SAME FILE (2026-10-14). Every patch is an update to the document
|
|
574
|
+
# (a revision, a webhook delivery), and a version Outline would not take was sent again
|
|
575
|
+
# on every pass: 81,566 updates in a day on 17 documents. It is tried again when the file
|
|
576
|
+
# changes, or with --full; until then it stays reported as cannot-patch.
|
|
577
|
+
# ⭐ Unless nothing would be lost by replacing it whole (replaceable, 2026-10-26): then it
|
|
578
|
+
# is replaced, once per version of the file, so a chat's `margin link` publishes it
|
|
579
|
+
tried_key = f"replace-tried:{row['doc_id']}"
|
|
580
|
+
if self.s.get(tried_key) != ident:
|
|
581
|
+
self.s.put(tried_key, ident)
|
|
582
|
+
doc = self.o.document(row["doc_id"])
|
|
583
|
+
if self.replaceable(row["doc_id"], doc):
|
|
584
|
+
body = self.resolve(body, base, row["doc_id"])
|
|
585
|
+
self.o.replace(row["doc_id"], body)
|
|
586
|
+
return self._finish(key, row, ident, title, doc["title"], self.canonical(body), REPLACED)
|
|
587
|
+
return Result(key, "cannot-patch", row["detail"] if "detail" in row.keys() else None)
|
|
588
|
+
dirty_key = f"dirty:{row['doc_id']}"
|
|
589
|
+
if not (repo_changed or self.full or self.s.get(dirty_key) or row["status"] != "ok"):
|
|
590
|
+
return Result(key, "unchanged") # nothing says either side moved: no API call
|
|
591
|
+
self.s.db.execute("DELETE FROM kv WHERE k=?", (dirty_key,))
|
|
592
|
+
try:
|
|
593
|
+
doc = self.o.document(row["doc_id"])
|
|
594
|
+
except OutlineError as e:
|
|
595
|
+
if e.status == 404:
|
|
596
|
+
self.s.mark(self.repo.name, key, "deleted-in-outline",
|
|
597
|
+
"the document was deleted in Outline; margin will not recreate it")
|
|
598
|
+
return Result(key, "deleted-in-outline")
|
|
599
|
+
raise
|
|
600
|
+
current = doc["text"]
|
|
601
|
+
outline_changed = current != row["base_outline"] and not self.only_margin_edited(doc)
|
|
602
|
+
|
|
603
|
+
if outline_changed and repo_changed:
|
|
604
|
+
self.s.mark(self.repo.name, key, "conflict",
|
|
605
|
+
"both the file and the Outline document changed since the last sync")
|
|
606
|
+
return Result(key, "conflict")
|
|
607
|
+
if outline_changed:
|
|
608
|
+
self.s.mark(self.repo.name, key, "outline-edited",
|
|
609
|
+
"the Outline document was edited; the repo file is unchanged")
|
|
610
|
+
return Result(key, "outline-edited")
|
|
611
|
+
if not repo_changed:
|
|
612
|
+
if row["status"] != "ok":
|
|
613
|
+
self.s.mark(self.repo.name, key, "ok")
|
|
614
|
+
return Result(key, "unchanged")
|
|
615
|
+
|
|
616
|
+
body = self.resolve(body, base, row["doc_id"])
|
|
617
|
+
target = self.canonical(body)
|
|
618
|
+
replaced = None
|
|
619
|
+
try:
|
|
620
|
+
if not current.strip():
|
|
621
|
+
self.o.replace(row["doc_id"], body) # an empty document cannot hold a comment
|
|
622
|
+
else:
|
|
623
|
+
self.converge(row["doc_id"], current, target)
|
|
624
|
+
except CannotPatch as e:
|
|
625
|
+
if self.replaceable(row["doc_id"], doc):
|
|
626
|
+
# ⭐ A DOCUMENT NOBODY EDITED AND NOBODY COMMENTED ON IS REPLACED WHOLE (2026-10-26).
|
|
627
|
+
# A chat's plan was published by the serve pass, the chat went on rewriting it, and
|
|
628
|
+
# the patches from the first rendering to the new text failed on its tables ("still
|
|
629
|
+
# differs after widening"), so the chat's own `margin link` met a document it could
|
|
630
|
+
# not change and had to run accept-file by hand. A replace drops comment anchors and
|
|
631
|
+
# a person's edits, so it is only for a document with no comments that no person
|
|
632
|
+
# has edited, the same rule accept_file applies.
|
|
633
|
+
self.o.replace(row["doc_id"], body)
|
|
634
|
+
replaced = REPLACED
|
|
635
|
+
else:
|
|
636
|
+
# ⛔ the patches that did apply changed Outline's text: keep that text as the baseline,
|
|
637
|
+
# or the next pass reads margin's own edits as someone else's and calls it a conflict
|
|
638
|
+
# (2026-10-13, a long plan with tables). The file's baseline stays, so it is retried.
|
|
639
|
+
now = self.o.document(row["doc_id"])["text"]
|
|
640
|
+
self.s.save_doc(self.repo.name, key, row["doc_id"], row["collection_id"],
|
|
641
|
+
row["base_repo"], now, "cannot-patch", str(e))
|
|
642
|
+
self.s.put(failed_key, ident) # this version is not sent again (above)
|
|
643
|
+
return Result(key, "cannot-patch", str(e))
|
|
644
|
+
return self._finish(key, row, ident, title, doc["title"], target, replaced)
|
|
645
|
+
|
|
646
|
+
def _finish(self, key: str, row, ident: str, title: str, doc_title: str, target: str,
|
|
647
|
+
replaced: str | None) -> Result:
|
|
648
|
+
"""After the document was patched or replaced: its title, then Outline's text against the
|
|
649
|
+
file's, and the baselines moved only when they match."""
|
|
650
|
+
if title != doc_title:
|
|
651
|
+
self.o.set_title(row["doc_id"], title)
|
|
652
|
+
after = self.o.document(row["doc_id"])["text"]
|
|
653
|
+
if after == target:
|
|
654
|
+
status, detail = "ok", replaced
|
|
655
|
+
elif loose(after) == loose(target):
|
|
656
|
+
status, detail = "ok", "only blank lines differ from the file"
|
|
657
|
+
elif spelled_alike(after, target):
|
|
658
|
+
status, detail = "ok", "only Outline's spelling of escapes and line breaks differs from the file"
|
|
659
|
+
else:
|
|
660
|
+
status, detail = ("drift", "after patching, Outline's text differs from the file as "
|
|
661
|
+
"Outline renders it")
|
|
662
|
+
# The file's baseline only moves when Outline really matches it, so a failed pass is
|
|
663
|
+
# retried next time instead of being remembered as done.
|
|
664
|
+
self.s.save_doc(self.repo.name, key, row["doc_id"], row["collection_id"],
|
|
665
|
+
ident if status == "ok" else row["base_repo"], after, status, detail)
|
|
666
|
+
return Result(key, "patched" if status == "ok" else "drift", detail or "")
|