pubhost 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pubhost/site.py ADDED
@@ -0,0 +1,348 @@
1
+ """`pubhost site check DIR`: what a public site's built pages need so that
2
+ robots and other sites read them correctly.
3
+
4
+ Each check belongs to a rule with a stable ID, so a repository can declare
5
+ an exception for it in `pubrules.json` as for the Rust crate standard
6
+ (`pubrules conform`). The rules grow by family: WEB-PAGE for what every page
7
+ names about itself, WEB-OG for the card a shared link shows (LinkedIn,
8
+ Facebook, Slack, WhatsApp...), WEB-ROBOTS for what robots.txt lets in. A
9
+ later family (accessibility, say) is a new entry in RULES and a function in
10
+ CHECKS.
11
+
12
+ It reads the built directory only, never the network: an og:image under the
13
+ site's own address (--url) is looked up in DIR; one elsewhere is only
14
+ checked to be an https address. Pages that are not for sharing are left
15
+ out: 404.html, and redirects (`<meta http-equiv="refresh">`). A page that
16
+ asks not to be indexed (`<meta name="robots" content="noindex">`, or
17
+ "none") is not for sharing or search either, so only the rules that hold
18
+ for any page apply to it: its language and title, not its description,
19
+ card or image, nor what robots.txt lets in.
20
+
21
+ It uses only the standard library, so deploy-static runs it from pubhost's
22
+ source without installing pubhost.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import datetime
28
+ import json
29
+ import os
30
+ import struct
31
+ import sys
32
+ from html.parser import HTMLParser
33
+ from pathlib import Path
34
+ from urllib.parse import urljoin, urlsplit
35
+ from urllib.robotparser import RobotFileParser
36
+
37
+ RULES = {
38
+ "WEB-PAGE-01": "every page names its language: <html lang>",
39
+ "WEB-PAGE-02": "every page has a <title> and a <meta name=\"description\">",
40
+ "WEB-OG-01": "every page has an Open Graph card: og:title, og:description, og:type, og:url, og:image",
41
+ "WEB-OG-02": "og:image is an https address of a PNG or JPEG of at least 1200×627 and under 5 MB",
42
+ "WEB-OG-03": "every page asks for a large card: <meta name=\"twitter:card\" content=\"summary_large_image\">",
43
+ "WEB-ROBOTS-01": "robots.txt lets the link-preview bots read every page and its og:image",
44
+ }
45
+
46
+ # The crawlers that draw link previews. Each obeys robots.txt, so a
47
+ # Disallow meant for search engines also hides the card.
48
+ PREVIEW_BOTS = ("LinkedInBot", "facebookexternalhit", "Twitterbot", "Slackbot")
49
+
50
+ # LinkedIn's minimum for a large card, and its largest image.
51
+ MIN_WIDTH, MIN_HEIGHT = 1200, 627
52
+ MAX_BYTES = 5 * 1024 * 1024
53
+
54
+ # The address pages are checked against robots.txt at without --url: only
55
+ # the path matters to it.
56
+ PLACEHOLDER = "https://site.invalid/"
57
+
58
+
59
+ class Page(HTMLParser):
60
+ """The parts of one HTML page the rules read."""
61
+
62
+ def __init__(self, text: str) -> None:
63
+ super().__init__(convert_charrefs=True)
64
+ self.lang = ""
65
+ self.title = ""
66
+ self.meta: dict[str, str] = {}
67
+ self.redirect = False
68
+ self.noindex = False
69
+ self._in_title = False
70
+ self.feed(text)
71
+
72
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
73
+ a = {k: (v or "") for k, v in attrs}
74
+ if tag == "html":
75
+ self.lang = a.get("lang", "").strip()
76
+ elif tag == "title":
77
+ self._in_title = True
78
+ elif tag == "meta":
79
+ if a.get("http-equiv", "").lower() == "refresh":
80
+ self.redirect = True
81
+ if a.get("name", "").lower() == "robots":
82
+ words = a.get("content", "").lower().replace(",", " ").split()
83
+ self.noindex = self.noindex or "noindex" in words or "none" in words
84
+ key = a.get("property") or a.get("name")
85
+ if key:
86
+ self.meta.setdefault(key.lower(), a.get("content", "").strip())
87
+
88
+ def handle_endtag(self, tag: str) -> None:
89
+ if tag == "title":
90
+ self._in_title = False
91
+
92
+ def handle_data(self, data: str) -> None:
93
+ if self._in_title:
94
+ self.title += data
95
+
96
+
97
+ def image_size(data: bytes) -> tuple[str, int, int] | None:
98
+ """(format, width, height) of a PNG or JPEG, read from its header, or
99
+ None for anything else."""
100
+ if data[:8] == b"\x89PNG\r\n\x1a\n" and data[12:16] == b"IHDR":
101
+ width, height = struct.unpack(">II", data[16:24])
102
+ return "PNG", width, height
103
+ if data[:2] != b"\xff\xd8":
104
+ return None
105
+ i = 2
106
+ while i + 4 <= len(data):
107
+ if data[i] != 0xFF:
108
+ return None
109
+ marker = data[i + 1]
110
+ if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
111
+ i += 2
112
+ continue
113
+ (length,) = struct.unpack(">H", data[i + 2:i + 4])
114
+ # Start of frame: every SOFn but DHT (C4), JPG (C8) and DAC (CC).
115
+ if 0xC0 <= marker <= 0xCF and marker not in (0xC4, 0xC8, 0xCC):
116
+ height, width = struct.unpack(">HH", data[i + 5:i + 9])
117
+ return "JPEG", width, height
118
+ i += 2 + length
119
+ return None
120
+
121
+
122
+ def pages(root: Path) -> list[Path]:
123
+ """Every HTML page under `root` that is meant to be shared."""
124
+ return sorted(p for p in root.rglob("*.html") if p.name != "404.html")
125
+
126
+
127
+ def address(root: Path, path: Path, base: str) -> str:
128
+ """The address a page in `root` is served at under `base`."""
129
+ rel = path.relative_to(root).as_posix()
130
+ if rel == "index.html" or rel.endswith("/index.html"):
131
+ rel = rel[: -len("index.html")]
132
+ return urljoin(base, rel)
133
+
134
+
135
+ def local(root: Path, url: str, base: str | None) -> Path | None:
136
+ """The file in `root` an address under `base` is, or None when the
137
+ address is elsewhere."""
138
+ if not base or not url.startswith(base):
139
+ return None
140
+ rel = urlsplit(url).path[len(urlsplit(base).path):]
141
+ target = (root / rel).resolve()
142
+ if not target.is_relative_to(root.resolve()):
143
+ return None
144
+ return target / "index.html" if target.is_dir() else target
145
+
146
+
147
+ def check_page(rel: str, page: Page) -> dict[str, list[str]]:
148
+ """The WEB-PAGE and WEB-OG-01/03 problems of one page; for a page that
149
+ asks not to be indexed, only those of WEB-PAGE-01 and its title."""
150
+ out: dict[str, list[str]] = {}
151
+
152
+ def problem(rule: str, text: str) -> None:
153
+ out.setdefault(rule, []).append(f"{rel}: {text}")
154
+
155
+ if not page.lang:
156
+ problem("WEB-PAGE-01", 'no <html lang>; set it to the page\'s language, like <html lang="en">, '
157
+ "so screen readers and translators read it right")
158
+ if not page.title.strip():
159
+ problem("WEB-PAGE-02", "no <title>; give the page a title, which search results and tabs show")
160
+ if page.noindex:
161
+ return out
162
+ if not page.meta.get("description"):
163
+ problem("WEB-PAGE-02", 'no <meta name="description">; add one or two sentences on what the page is')
164
+ missing = [k for k in ("og:title", "og:description", "og:type", "og:url", "og:image")
165
+ if not page.meta.get(k)]
166
+ if missing:
167
+ problem("WEB-OG-01", f"no {', '.join(missing)}; add "
168
+ + " ".join(f'<meta property="{k}" content="...">' for k in missing)
169
+ + " to its <head>, or a shared link shows no card")
170
+ if page.meta.get("og:url") and not page.meta["og:url"].startswith("https://"):
171
+ problem("WEB-OG-01", f"og:url is {page.meta['og:url']}; give the page's full https:// address")
172
+ if page.meta.get("twitter:card") != "summary_large_image":
173
+ problem("WEB-OG-03", '<meta name="twitter:card" content="summary_large_image"> is missing; '
174
+ "without it some sites show the image as a small thumbnail")
175
+ return out
176
+
177
+
178
+ def check_image(root: Path, rel: str, page: Page, base: str | None,
179
+ seen: dict[Path, str | None]) -> list[str]:
180
+ """The WEB-OG-02 problems of one page's og:image."""
181
+ url = page.meta.get("og:image")
182
+ if not url:
183
+ return []
184
+ if not url.startswith("https://"):
185
+ return [f"{rel}: og:image is {url}; give its full https:// address, which a crawler can fetch"]
186
+ path = local(root, url, base)
187
+ if path is None:
188
+ return []
189
+ if path not in seen:
190
+ seen[path] = image_problem(path, url)
191
+ problems = [f"{rel}: {seen[path]}"] if seen[path] else []
192
+ size = image_size(path.read_bytes()) if path.is_file() else None
193
+ for key, index in (("og:image:width", 1), ("og:image:height", 2)):
194
+ declared = page.meta.get(key)
195
+ if size and declared and declared != str(size[index]):
196
+ problems.append(f"{rel}: {key} says {declared}, but {url} is {size[index]}; "
197
+ "make them agree, or a card is laid out wrong")
198
+ return problems
199
+
200
+
201
+ def image_problem(path: Path, url: str) -> str | None:
202
+ if not path.is_file():
203
+ return f"og:image {url} is not in the site; build it, or point og:image at a file that is"
204
+ data = path.read_bytes()
205
+ size = image_size(data)
206
+ if size is None:
207
+ return f"og:image {url} is not a PNG or JPEG, which LinkedIn needs; save it as one"
208
+ _, width, height = size
209
+ if width < MIN_WIDTH or height < MIN_HEIGHT:
210
+ return (f"og:image {url} is {width}×{height}; make it at least {MIN_WIDTH}×{MIN_HEIGHT} "
211
+ "(1200×630 is usual), or LinkedIn shows no image")
212
+ if len(data) > MAX_BYTES:
213
+ return f"og:image {url} is {len(data) // 1024} KB; keep it under 5 MB, or LinkedIn drops it"
214
+ return None
215
+
216
+
217
+ def check_robots(root: Path, urls: list[str]) -> list[str]:
218
+ """The WEB-ROBOTS-01 problems: an address in `urls` that robots.txt
219
+ keeps a preview bot from."""
220
+ robots_txt = root / "robots.txt"
221
+ if not robots_txt.is_file():
222
+ return []
223
+ robots = RobotFileParser()
224
+ robots.parse(robots_txt.read_text(encoding="utf-8", errors="replace").splitlines())
225
+ problems = []
226
+ for bot in PREVIEW_BOTS:
227
+ blocked = [u for u in urls if not robots.can_fetch(bot, u)]
228
+ if blocked:
229
+ shown = ", ".join(blocked[:3]) + (f" and {len(blocked) - 3} more" if len(blocked) > 3 else "")
230
+ problems.append(f"robots.txt keeps {bot} from {shown}, so links to it show no card; "
231
+ f"add a group for the preview bots ({', '.join(PREVIEW_BOTS)}) with Allow: /, "
232
+ "and keep search engines out with <meta name=\"robots\" content=\"noindex\"> instead")
233
+ return problems
234
+
235
+
236
+ def checks(root: Path, base: str | None) -> tuple[int, dict[str, list[str]]]:
237
+ """(pages checked, problems by rule)."""
238
+ out: dict[str, list[str]] = {}
239
+ seen: dict[Path, str | None] = {}
240
+ count, urls = 0, []
241
+ for path in pages(root):
242
+ page = Page(path.read_text(encoding="utf-8", errors="replace"))
243
+ if page.redirect:
244
+ continue
245
+ count += 1
246
+ rel = path.relative_to(root).as_posix()
247
+ for rule, texts in check_page(rel, page).items():
248
+ out.setdefault(rule, []).extend(texts)
249
+ if page.noindex:
250
+ continue
251
+ if image := check_image(root, rel, page, base, seen):
252
+ out.setdefault("WEB-OG-02", []).extend(image)
253
+ urls.append(address(root, path, base or PLACEHOLDER))
254
+ if base and page.meta.get("og:image", "").startswith(base):
255
+ urls.append(page.meta["og:image"])
256
+ if robots := check_robots(root, list(dict.fromkeys(urls))):
257
+ out["WEB-ROBOTS-01"] = robots
258
+ return count, out
259
+
260
+
261
+ # A copy of pubrules' conform.today and conform.exceptions (pubrules 0.34.0),
262
+ # so this module needs only the standard library: deploy-static runs it from
263
+ # its own source with nothing installed.
264
+ def today() -> datetime.date:
265
+ """Today, or PUBHOST_TODAY (YYYY-MM-DD), which tests set."""
266
+ fixed = os.environ.get("PUBHOST_TODAY")
267
+ return datetime.date.fromisoformat(fixed) if fixed else datetime.date.today()
268
+
269
+
270
+ def exceptions(config: dict, known: set[str],
271
+ family=lambda rule: str(rule).startswith("WEB-")) -> tuple[set[str], list[str], list[str]]:
272
+ """The rule IDs waived, a description of each open exception, and the
273
+ problems with the declarations. Only the exceptions whose rule is in
274
+ `family` are read: the site rules' (WEB-); the rest of pubrules.json's
275
+ exceptions are the Rust crate standard's, which `pubrules conform` reads."""
276
+ waived, open_, problems = set(), [], []
277
+ for i, e in enumerate(config.get("exceptions", [])):
278
+ where = f"pubrules.json exceptions[{i}]"
279
+ if isinstance(e, dict) and "rule" in e and not family(e["rule"]):
280
+ continue
281
+ if not isinstance(e, dict) or set(e) != {"rule", "reason", "until"}:
282
+ problems.append(f"{where}: give exactly a rule, a reason and an until date")
283
+ continue
284
+ rule, reason, until = e["rule"], str(e["reason"]).strip(), e["until"]
285
+ if rule not in known:
286
+ problems.append(f"{where}: {rule} is not a rule of the standard")
287
+ continue
288
+ if not reason:
289
+ problems.append(f"{where}: {rule} needs a reason")
290
+ continue
291
+ try:
292
+ date = datetime.date.fromisoformat(str(until))
293
+ except ValueError:
294
+ problems.append(f"{where}: {rule}'s until must be a date, YYYY-MM-DD")
295
+ continue
296
+ if date < today():
297
+ problems.append(f"{rule}: the exception expired on {date}; meet the rule, or "
298
+ f"review the exception and set a new date ({reason})")
299
+ continue
300
+ waived.add(rule)
301
+ open_.append(f"{rule} until {date}: {reason}")
302
+ return waived, open_, problems
303
+
304
+
305
+ def run(root: Path, directory: Path, base: str | None) -> int:
306
+ if base is not None:
307
+ if not base.startswith("https://"):
308
+ sys.exit(f"pubhost: --url ({base}) must be the site's https:// address")
309
+ base = base if base.endswith("/") else base + "/"
310
+ if not directory.is_dir():
311
+ sys.exit(f"pubhost: {directory} is not a directory; build the site first")
312
+ config_file = root / "pubrules.json"
313
+ config = json.loads(config_file.read_text()) if config_file.is_file() else {}
314
+ waived, open_, problems = exceptions(config, set(RULES))
315
+ count, found = checks(directory, base)
316
+ if count == 0:
317
+ problems.append(f"{directory} has no HTML pages to check; is it the built site?")
318
+ for rule, texts in sorted(found.items()):
319
+ if rule not in waived:
320
+ problems += [f"{rule} {t}" for t in texts]
321
+ for e in open_:
322
+ print(f"open exception: {e}")
323
+ if problems:
324
+ for p in problems:
325
+ print(p, file=sys.stderr)
326
+ print(f"pubhost site check: {len(problems)} problem(s) in {count} page(s)", file=sys.stderr)
327
+ return 1
328
+ print(f"{count} page(s) meet the {len(RULES) - len(waived)} site rules"
329
+ + ("" if base else "; give --url to check the og:image files too"))
330
+ return 0
331
+
332
+
333
+ def main(argv: list[str] | None = None) -> int:
334
+ """`python -m pubhost.site check DIR [--url URL]`, as deploy-static runs
335
+ it without installing pubhost."""
336
+ import argparse
337
+
338
+ parser = argparse.ArgumentParser(prog="pubhost site")
339
+ commands = parser.add_subparsers(dest="command", required=True)
340
+ p = commands.add_parser("check")
341
+ p.add_argument("dir", type=Path)
342
+ p.add_argument("--url")
343
+ args = parser.parse_args(argv)
344
+ return run(Path.cwd(), args.dir, args.url)
345
+
346
+
347
+ if __name__ == "__main__":
348
+ sys.exit(main())