pubdocrules 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. pubdoc/__init__.py +1 -0
  2. pubdoc/__main__.py +5 -0
  3. pubdoc/cli.py +800 -0
  4. pubdoc/compare.py +121 -0
  5. pubdoc/corpus.py +245 -0
  6. pubdoc/fonts/DejaVuSans-Bold.ttf +0 -0
  7. pubdoc/fonts/DejaVuSans.ttf +0 -0
  8. pubdoc/fonts/LICENCE-Ubuntu.txt +96 -0
  9. pubdoc/fonts/LICENSE-DejaVu.txt +78 -0
  10. pubdoc/fonts/LICENSE-Liberation.txt +130 -0
  11. pubdoc/fonts/LiberationSans-Bold.ttf +0 -0
  12. pubdoc/fonts/LiberationSans-BoldItalic.ttf +0 -0
  13. pubdoc/fonts/LiberationSans-Italic.ttf +0 -0
  14. pubdoc/fonts/LiberationSans-Regular.ttf +0 -0
  15. pubdoc/fonts/README.md +21 -0
  16. pubdoc/fonts/Ubuntu-R.ttf +0 -0
  17. pubdoc/projects.py +501 -0
  18. pubdoc/registry.py +54 -0
  19. pubdoc/schema/doc.schema.json +14 -0
  20. pubdoc/schema/project.schema.json +142 -0
  21. pubdoc/schema/pubdoc.schema.json +16 -0
  22. pubdoc/settings.py +66 -0
  23. pubdoc/storage.py +126 -0
  24. pubdoc/templates/annotations.mjs +116 -0
  25. pubdoc/templates/charged-ieee/template.typ +41 -0
  26. pubdoc/templates/charged-ieee/template.yml +28 -0
  27. pubdoc/templates/cv-sidebar/cv-sidebar.typ +113 -0
  28. pubdoc/templates/cv-sidebar/template.typ +11 -0
  29. pubdoc/templates/cv-sidebar/template.yml +23 -0
  30. pubdoc/templates/lapreprint-typst/LICENSE +21 -0
  31. pubdoc/templates/lapreprint-typst/frontmatter.typ +244 -0
  32. pubdoc/templates/lapreprint-typst/lapreprint.typ +314 -0
  33. pubdoc/templates/lapreprint-typst/template.typ +112 -0
  34. pubdoc/templates/lapreprint-typst/template.yml +82 -0
  35. pubdoc/templates/linkedin-text.mjs +127 -0
  36. pubdoc/templates/pub-plugin.mjs +117 -0
  37. pubdoc/versions.py +383 -0
  38. pubdocrules-0.5.0.dist-info/METADATA +355 -0
  39. pubdocrules-0.5.0.dist-info/RECORD +43 -0
  40. pubdocrules-0.5.0.dist-info/WHEEL +5 -0
  41. pubdocrules-0.5.0.dist-info/entry_points.txt +2 -0
  42. pubdocrules-0.5.0.dist-info/licenses/LICENSE +202 -0
  43. pubdocrules-0.5.0.dist-info/top_level.txt +1 -0
pubdoc/versions.py ADDED
@@ -0,0 +1,383 @@
1
+ """A doc's versions: each one a signed publet, its renders in storage.
2
+
3
+ Approving a doc's variant builds every layout of it, uploads each render
4
+ to the project's storage tenant (named there by its SHA-256), writes the
5
+ version publet's source into the corpus (corpus/publets/<slug>/publet.json)
6
+ and has `pub` sign and publish it, then records the version in
7
+ <doc>/versions/versions.yml, the index people read. Each doc and variant is
8
+ one lineage, slug doc.<project>.<doc>.<variant>: a new version supersedes
9
+ the one before. Its claims:
10
+
11
+ #identity which doc, variant and version; who approved it, when; the
12
+ pubdoc that built it and the commit it was built from
13
+ #sources every file it was built from, by path and SHA-256
14
+ #renders every render: layout, format, SHA-256, size, address
15
+
16
+ Renders are outputs, not publets, so a publet holds their checksums and
17
+ addresses, never the files. Signed objects are committed in the corpus's
18
+ objects/; the key is in its untracked .publet/, on the approver's machine.
19
+
20
+ verify checks every versions.yml against the corpus's signed objects:
21
+ `pub` verifies the corpus as the parent of a scratch corpus, which checks
22
+ every generation record's signature and every object's hash, and each
23
+ version's claims must state what versions.yml says.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import datetime
29
+ import hashlib
30
+ import json
31
+ import os
32
+ import re
33
+ import shutil
34
+ import subprocess
35
+ import tempfile
36
+ from dataclasses import dataclass
37
+ from pathlib import Path
38
+
39
+ import yaml
40
+
41
+ from . import projects, settings, storage
42
+ from .projects import VERSIONS, Doc, Render
43
+
44
+ INDEX = "versions.yml"
45
+ LOCK = "corpus.lock"
46
+ PUB_SUPPORTED = ("0.4.",)
47
+ PUB_INSTALL = ("cargo install --locked --git https://github.com/Sustenaut/publet "
48
+ "--tag v0.4.0 publet-cli")
49
+ IDENTITY = ["doc", "variant", "version", "approved", "approved_by", "pubdoc", "commit", "rebuilt_from"]
50
+ SOURCES = ["path", "sha256"]
51
+ RENDERS = ["layout", "format", "sha256", "bytes", "url"]
52
+
53
+
54
+ class VersionError(Exception):
55
+ pass
56
+
57
+
58
+ # pub ---------------------------------------------------------------------
59
+
60
+ def pub_program() -> str:
61
+ return os.environ.get("PUB", "pub")
62
+
63
+
64
+ def require_pub() -> str:
65
+ program = shutil.which(pub_program())
66
+ if not program:
67
+ raise VersionError(f"no `{pub_program()}` on PATH: pubdoc signs and verifies versions with "
68
+ f"publet's pub 0.4; install it: {PUB_INSTALL}")
69
+ first = subprocess.run([program, "--version"], capture_output=True, text=True).stdout.split("\n")[0]
70
+ if not any(first.startswith(f"pub {v}") for v in PUB_SUPPORTED):
71
+ raise VersionError(f"pubdoc drives pub 0.4, and `{program} --version` says {first!r}; "
72
+ f"install it: {PUB_INSTALL}")
73
+ return program
74
+
75
+
76
+ def run_pub(where: Path, *args: str) -> str:
77
+ done = subprocess.run([require_pub(), *args], cwd=where, capture_output=True, text=True)
78
+ if done.returncode != 0:
79
+ raise VersionError(f"pub {' '.join(args)} failed in {projects.shown(where)}:\n"
80
+ f"{done.stdout}{done.stderr}".rstrip())
81
+ return done.stdout
82
+
83
+
84
+ def lock(corpus: Path) -> dict:
85
+ path = corpus / LOCK
86
+ return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
87
+
88
+
89
+ def locked(corpus: Path, slug: str) -> str | None:
90
+ for row in lock(corpus).get("publets", []):
91
+ if row.get("slug") == slug:
92
+ return row.get("cid")
93
+ return None
94
+
95
+
96
+ def read_object(corpus: Path, cid: str) -> dict:
97
+ return json.loads(run_pub(corpus, "read", "--dir=objects", "--json", cid))["object"]
98
+
99
+
100
+ # Naming ------------------------------------------------------------------
101
+
102
+ def slug(doc: Doc, variant: str) -> str:
103
+ return f"doc.{doc.project.name}.{doc.name}.{variant}"
104
+
105
+
106
+ def tag(name: str, date: str, taken: set[str]) -> str:
107
+ """A Section 9.1 citation tag for a new lineage: DOC-<six letters>-MM-YYYY,
108
+ the letters from the doc's name and variant (acme, research: ACMERE),
109
+ else, when those are taken, from a hash of the slug."""
110
+ _, project, doc, variant = name.split(".", 3)
111
+ alpha = lambda s: re.sub(r"[^A-Za-z]", "", s).upper() # noqa: E731
112
+ letters = (alpha(doc)[:4] + alpha(variant)[:2] + alpha(project))
113
+ year, month = date[:4], date[5:7]
114
+ candidates = [letters[:6]] if len(letters) >= 6 else []
115
+ digest = hashlib.sha256(name.encode()).digest()
116
+ candidates.append("".join(chr(65 + b % 26) for b in digest[:6]))
117
+ for word in candidates:
118
+ found = f"DOC-{word}-{month}-{year}"
119
+ if found not in taken:
120
+ return found
121
+ raise VersionError(f"no free citation tag for {name}")
122
+
123
+
124
+ def taken_tags(corpus: Path) -> set[str]:
125
+ return {row["tag"] for row in lock(corpus).get("publets", []) if row.get("tag")}
126
+
127
+
128
+ # The index ---------------------------------------------------------------
129
+
130
+ def index_path(doc: Doc) -> Path:
131
+ return doc.path / VERSIONS / INDEX
132
+
133
+
134
+ def read_index(doc: Doc) -> dict:
135
+ path = index_path(doc)
136
+ if not path.is_file():
137
+ return {"doc": doc.label, "versions": []}
138
+ data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
139
+ data.setdefault("versions", [])
140
+ return data
141
+
142
+
143
+ def write_index(doc: Doc, data: dict) -> Path:
144
+ path = index_path(doc)
145
+ path.parent.mkdir(exist_ok=True)
146
+ path.write_text("# Written by `pubdoc approve` from the signed version publets: never edit.\n"
147
+ + yaml.safe_dump(data, sort_keys=False, allow_unicode=True), encoding="utf-8")
148
+ return path
149
+
150
+
151
+ def latest(index: dict, variant: str) -> dict | None:
152
+ return next((v for v in index["versions"] if v.get("variant") == variant), None)
153
+
154
+
155
+ # Approving ---------------------------------------------------------------
156
+
157
+ @dataclass(frozen=True)
158
+ class Approval:
159
+ approver: str
160
+ date: str # YYYY-MM-DD
161
+ commit: str
162
+ rebuilt_from: str # "" unless rebuilt from an earlier commit's sources
163
+ pubdoc: str
164
+
165
+ @property
166
+ def timestamp(self) -> int:
167
+ """The renders' creation time: midnight UTC of the approval date, so a
168
+ rebuild of the same sources with the same tools is the same bytes."""
169
+ day = datetime.date.fromisoformat(self.date)
170
+ return int(datetime.datetime(day.year, day.month, day.day, tzinfo=datetime.timezone.utc).timestamp())
171
+
172
+
173
+ def source_rows(doc: Doc, files: dict[Path, bytes]) -> list[list[str]]:
174
+ root = projects.repo_root(doc.project.path).resolve()
175
+ rows = {}
176
+ for path in [doc.project.file, doc.path / projects.DOC_FILE]:
177
+ if path.is_file():
178
+ rows[os.path.relpath(path.resolve(), root)] = storage.sha256(path.read_bytes())
179
+ for rel, data in files.items():
180
+ rows[os.path.relpath((doc.project.path / rel).resolve(), root)] = storage.sha256(data)
181
+ return [[path, rows[path]] for path in sorted(rows)]
182
+
183
+
184
+ def columns(names: list[str]) -> list[dict]:
185
+ return [{"name": n} for n in names]
186
+
187
+
188
+ def publet_source(doc: Doc, variant: str, number: int, approval: Approval, the_tag: str,
189
+ sources: list[list[str]], renders: list[list]) -> dict:
190
+ who = f"{doc.label} ({variant})"
191
+ scope = f"version {number} of the doc {doc.label}, variant {variant}"
192
+ evidence = [{"ref": doc.project.name, "revision": approval.commit}]
193
+ rebuilt = (f", rebuilt from the sources of commit {approval.rebuilt_from}"
194
+ if approval.rebuilt_from else "")
195
+ return {
196
+ "slug": slug(doc, variant), "tag": the_tag, "title": who,
197
+ "created": f"{approval.date}T00:00:00Z",
198
+ "claims": {
199
+ "identity": {
200
+ "class": "archival", "scope": scope,
201
+ "content": (f"Version {number} of {who} was approved by {approval.approver} on "
202
+ f"{approval.date}, built by pubdoc {approval.pubdoc} from commit "
203
+ f"{approval.commit}{rebuilt}."),
204
+ "data": {"columns": columns(IDENTITY),
205
+ "rows": [[doc.label, variant, str(number), approval.date, approval.approver,
206
+ approval.pubdoc, approval.commit, approval.rebuilt_from]]},
207
+ "sources": evidence,
208
+ },
209
+ "sources": {
210
+ "class": "archival", "scope": scope, "depends": ["#identity"],
211
+ "content": f"Version {number} of {who} was built from these files.",
212
+ "data": {"columns": columns(SOURCES), "rows": sources},
213
+ "sources": evidence,
214
+ },
215
+ "renders": {
216
+ "class": "archival", "scope": scope, "depends": ["#identity"],
217
+ "content": f"Version {number} of {who} is rendered as these files, kept in storage.",
218
+ "data": {"columns": columns(RENDERS), "rows": renders},
219
+ "sources": evidence,
220
+ },
221
+ },
222
+ "sections": [{"heading": h, "items": [{"ref": f"#{c}"}]}
223
+ for h, c in (("Identity", "identity"), ("Sources", "sources"), ("Renders", "renders"))],
224
+ }
225
+
226
+
227
+ def publish(doc: Doc, variant: str, approval: Approval, renders: list[Render],
228
+ files: dict[Path, bytes]) -> dict:
229
+ """Upload `renders` (built), sign the version publet, and record it in
230
+ versions.yml. Returns the new index entry."""
231
+ conf = settings.of(doc.project)
232
+ if not conf.tenant:
233
+ raise VersionError(f"{projects.shown(doc.project.path)} names no storage tenant: set "
234
+ "storage: {tenant: ...} in its project.yml, or in "
235
+ f"{settings.ROOT_FILE} at the repository's root")
236
+ corpus = conf.corpus
237
+ if not (corpus / "corpus.json").is_file():
238
+ raise VersionError(f"no corpus at {projects.shown(corpus)}: make it with `pub corpus init` there, "
239
+ "and its signing key with `pub sign --generate-key --principal=human`")
240
+ require_pub()
241
+ index = read_index(doc)
242
+ before = latest(index, variant)
243
+ number = int(before["version"]) + 1 if before else 1
244
+ rows, addresses = [], {}
245
+ for r in renders:
246
+ data = r.output.read_bytes()
247
+ name = f"{doc.name}-{variant}-{r.layout}-v{number}.{r.extension}"
248
+ url = storage.upload(conf.storage_url, conf.tenant, name, data,
249
+ f"{doc.label} {variant} {r.layout}, version {number}")
250
+ rows.append([r.layout, r.spec["format"], storage.sha256(data), len(data), url])
251
+ addresses[r.layout] = {"url": url, "sha256": storage.sha256(data)}
252
+ the_slug = slug(doc, variant)
253
+ source_file = corpus / "publets" / the_slug / "publet.json"
254
+ if source_file.is_file():
255
+ the_tag = json.loads(source_file.read_text(encoding="utf-8"))["tag"]
256
+ else:
257
+ the_tag = tag(the_slug, approval.date, taken_tags(corpus))
258
+ source = publet_source(doc, variant, number, approval, the_tag, source_rows(doc, files), rows)
259
+ source_file.parent.mkdir(parents=True, exist_ok=True)
260
+ source_file.write_text(json.dumps(source, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
261
+ run_pub(corpus, "build")
262
+ run_pub(corpus, "publish", the_slug)
263
+ identity = locked(corpus, f"{the_slug}#identity")
264
+ key = lock(corpus).get("corpus", {}).get("key")
265
+ if conf.key and key != conf.key:
266
+ raise VersionError(f"the corpus at {projects.shown(corpus)} signed with {key}, "
267
+ f"and the project trusts {conf.key}")
268
+ entry = {
269
+ "version": number, "variant": variant, "approved": approval.date,
270
+ "approved_by": approval.approver,
271
+ "publet": identity,
272
+ "objects": {c: locked(corpus, f"{the_slug}#{c}") for c in ("identity", "sources", "renders")},
273
+ "supersedes": before["publet"] if before else None,
274
+ "pubdoc": approval.pubdoc, "commit": approval.commit,
275
+ }
276
+ if approval.rebuilt_from:
277
+ entry["rebuilt_from"] = approval.rebuilt_from
278
+ entry["renders"] = addresses
279
+ write_index(doc, {"doc": doc.label, "corpus": os.path.relpath(corpus, index_path(doc).parent),
280
+ "key": key, "versions": [entry] + index["versions"]})
281
+ return entry
282
+
283
+
284
+ # Verifying ---------------------------------------------------------------
285
+
286
+ def tenant_base(conf: settings.Settings) -> str:
287
+ return f"{conf.storage_url}/{conf.tenant}/"
288
+
289
+
290
+ def source_problems(doc: Doc, where: str, commit: str, rows: list[list]) -> list[str]:
291
+ """Each source row against the file at `commit`, so a version approved
292
+ with uncommitted changes shows. A commit this clone lacks (a shallow
293
+ CI checkout) is a note, not a problem."""
294
+ root = projects.repo_root(doc.project.path)
295
+ known = subprocess.run(["git", "-C", str(root), "cat-file", "-e", f"{commit}^{{commit}}"],
296
+ capture_output=True)
297
+ if known.returncode != 0:
298
+ return [f"note: {where}: commit {commit} is not in this clone; its sources are not checked"]
299
+ found = []
300
+ for path, digest in rows:
301
+ shown = subprocess.run(["git", "-C", str(root), "show", f"{commit}:{path}"], capture_output=True)
302
+ if shown.returncode != 0:
303
+ found.append(f"{where}: {path} is not in commit {commit}")
304
+ elif storage.sha256(shown.stdout) != digest:
305
+ found.append(f"{where}: {path} at commit {commit} is not the file it was built from")
306
+ return found
307
+
308
+
309
+ def verify_corpus(corpus: Path) -> None:
310
+ """Every generation of `corpus` signed by its key, every object its hash:
311
+ pub checks a parent so whenever a corpus names one, so a scratch corpus
312
+ naming this one as its parent has pub check it, without a key store."""
313
+ with tempfile.TemporaryDirectory(prefix="pubdoc-verify-") as scratch:
314
+ (Path(scratch) / "corpus.json").write_text(json.dumps({
315
+ "name": "verify", "label": "verify", "bound": 67108864,
316
+ "parents": [{"alias": "checked", "path": str(corpus)}]}), encoding="utf-8")
317
+ run_pub(Path(scratch), "corpus", "status")
318
+
319
+
320
+ def claim_rows(corpus: Path, cid: str, names: list[str]) -> list[list]:
321
+ body = read_object(corpus, cid).get("body", {})
322
+ stated = [c.get("name") for c in body.get("data", {}).get("columns", [])]
323
+ if stated != names:
324
+ raise VersionError(f"{cid} has columns {stated}, not {names}")
325
+ return body["data"]["rows"]
326
+
327
+
328
+ def verify_doc(doc: Doc, remote: bool) -> list[str]:
329
+ """Why this doc's versions.yml does not hold, if it does not."""
330
+ problems = []
331
+ path = index_path(doc)
332
+ index = read_index(doc)
333
+ conf = settings.of(doc.project)
334
+ # The corpus is the one the project's settings name, the one verify_corpus
335
+ # checked: a versions.yml naming any other is not taken at its word.
336
+ corpus = conf.corpus
337
+ named = (path.parent / index.get("corpus", "")).resolve() if index.get("corpus") else None
338
+ if named != corpus:
339
+ problems.append(f"{projects.shown(path)}: names the corpus {index.get('corpus')!r}, "
340
+ f"and the project's is {projects.shown(corpus)}")
341
+ signer = lock(corpus).get("corpus", {}).get("key")
342
+ if index.get("key") != signer:
343
+ problems.append(f"{projects.shown(path)}: key {index.get('key')} is not the corpus's, {signer}")
344
+ if conf.key and signer != conf.key:
345
+ problems.append(f"{projects.shown(corpus)}: signed by {signer}, and the project trusts {conf.key}")
346
+ numbers: dict[str, list[int]] = {}
347
+ for entry in index["versions"]:
348
+ where = f"{projects.shown(path)}: version {entry.get('version')} ({entry.get('variant')})"
349
+ numbers.setdefault(entry.get("variant"), []).append(int(entry.get("version", 0)))
350
+ objects = entry.get("objects", {})
351
+ try:
352
+ identity = claim_rows(corpus, objects.get("identity", ""), IDENTITY)[0]
353
+ rendered = claim_rows(corpus, objects.get("renders", ""), RENDERS)
354
+ sourced = claim_rows(corpus, objects.get("sources", ""), SOURCES)
355
+ except (VersionError, KeyError, IndexError, json.JSONDecodeError) as e:
356
+ problems.append(f"{where}: its publet cannot be read: {e}")
357
+ continue
358
+ stated = dict(zip(IDENTITY, identity, strict=False))
359
+ for field, value in (("doc", doc.label), ("variant", entry.get("variant")),
360
+ ("version", str(entry.get("version"))), ("approved", entry.get("approved")),
361
+ ("approved_by", entry.get("approved_by")), ("commit", entry.get("commit"))):
362
+ if stated.get(field) != value:
363
+ problems.append(f"{where}: says {field} {value!r}, its publet {stated.get(field)!r}")
364
+ problems.extend(source_problems(doc, where, stated.get("commit", ""), sourced))
365
+ renders = {row[0]: {"url": row[4], "sha256": row[2]} for row in rendered}
366
+ if renders != entry.get("renders"):
367
+ problems.append(f"{where}: its renders are not the ones its publet records")
368
+ if remote:
369
+ for layout, render in renders.items():
370
+ try:
371
+ storage.download(render["url"], render["sha256"], tenant_base(conf))
372
+ except storage.StorageError as e:
373
+ problems.append(f"{where}: {layout}: {e}")
374
+ for variant in numbers:
375
+ head = locked(corpus, f"{slug(doc, variant)}#identity")
376
+ newest = latest(index, variant)
377
+ if newest and head != newest["objects"].get("identity"):
378
+ problems.append(f"{projects.shown(path)}: {variant}'s newest version is not the corpus's, "
379
+ f"{head}: a version is missing from versions.yml")
380
+ for variant, seen in numbers.items():
381
+ if seen != sorted(seen, reverse=True) or sorted(seen) != list(range(1, len(seen) + 1)):
382
+ problems.append(f"{projects.shown(path)}: {variant}'s versions are {seen}, not newest first from 1")
383
+ return problems