pubhost 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubhost/__init__.py +24 -0
- pubhost/cli.py +65 -0
- pubhost/dns.py +318 -0
- pubhost/site.py +348 -0
- pubhost-0.2.0.dist-info/METADATA +574 -0
- pubhost-0.2.0.dist-info/RECORD +10 -0
- pubhost-0.2.0.dist-info/WHEEL +5 -0
- pubhost-0.2.0.dist-info/entry_points.txt +2 -0
- pubhost-0.2.0.dist-info/licenses/LICENSE +202 -0
- pubhost-0.2.0.dist-info/top_level.txt +1 -0
pubhost/site.py
ADDED
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
"""`pubhost site check DIR`: what a public site's built pages need so that
|
|
2
|
+
robots and other sites read them correctly.
|
|
3
|
+
|
|
4
|
+
Each check belongs to a rule with a stable ID, so a repository can declare
|
|
5
|
+
an exception for it in `pubrules.json` as for the Rust crate standard
|
|
6
|
+
(`pubrules conform`). The rules grow by family: WEB-PAGE for what every page
|
|
7
|
+
names about itself, WEB-OG for the card a shared link shows (LinkedIn,
|
|
8
|
+
Facebook, Slack, WhatsApp...), WEB-ROBOTS for what robots.txt lets in. A
|
|
9
|
+
later family (accessibility, say) is a new entry in RULES and a function in
|
|
10
|
+
CHECKS.
|
|
11
|
+
|
|
12
|
+
It reads the built directory only, never the network: an og:image under the
|
|
13
|
+
site's own address (--url) is looked up in DIR; one elsewhere is only
|
|
14
|
+
checked to be an https address. Pages that are not for sharing are left
|
|
15
|
+
out: 404.html, and redirects (`<meta http-equiv="refresh">`). A page that
|
|
16
|
+
asks not to be indexed (`<meta name="robots" content="noindex">`, or
|
|
17
|
+
"none") is not for sharing or search either, so only the rules that hold
|
|
18
|
+
for any page apply to it: its language and title, not its description,
|
|
19
|
+
card or image, nor what robots.txt lets in.
|
|
20
|
+
|
|
21
|
+
It uses only the standard library, so deploy-static runs it from pubhost's
|
|
22
|
+
source without installing pubhost.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import datetime
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import struct
|
|
31
|
+
import sys
|
|
32
|
+
from html.parser import HTMLParser
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from urllib.parse import urljoin, urlsplit
|
|
35
|
+
from urllib.robotparser import RobotFileParser
|
|
36
|
+
|
|
37
|
+
RULES = {
|
|
38
|
+
"WEB-PAGE-01": "every page names its language: <html lang>",
|
|
39
|
+
"WEB-PAGE-02": "every page has a <title> and a <meta name=\"description\">",
|
|
40
|
+
"WEB-OG-01": "every page has an Open Graph card: og:title, og:description, og:type, og:url, og:image",
|
|
41
|
+
"WEB-OG-02": "og:image is an https address of a PNG or JPEG of at least 1200×627 and under 5 MB",
|
|
42
|
+
"WEB-OG-03": "every page asks for a large card: <meta name=\"twitter:card\" content=\"summary_large_image\">",
|
|
43
|
+
"WEB-ROBOTS-01": "robots.txt lets the link-preview bots read every page and its og:image",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
# The crawlers that draw link previews. Each obeys robots.txt, so a
|
|
47
|
+
# Disallow meant for search engines also hides the card.
|
|
48
|
+
PREVIEW_BOTS = ("LinkedInBot", "facebookexternalhit", "Twitterbot", "Slackbot")
|
|
49
|
+
|
|
50
|
+
# LinkedIn's minimum for a large card, and its largest image.
|
|
51
|
+
MIN_WIDTH, MIN_HEIGHT = 1200, 627
|
|
52
|
+
MAX_BYTES = 5 * 1024 * 1024
|
|
53
|
+
|
|
54
|
+
# The address pages are checked against robots.txt at without --url: only
|
|
55
|
+
# the path matters to it.
|
|
56
|
+
PLACEHOLDER = "https://site.invalid/"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Page(HTMLParser):
|
|
60
|
+
"""The parts of one HTML page the rules read."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, text: str) -> None:
|
|
63
|
+
super().__init__(convert_charrefs=True)
|
|
64
|
+
self.lang = ""
|
|
65
|
+
self.title = ""
|
|
66
|
+
self.meta: dict[str, str] = {}
|
|
67
|
+
self.redirect = False
|
|
68
|
+
self.noindex = False
|
|
69
|
+
self._in_title = False
|
|
70
|
+
self.feed(text)
|
|
71
|
+
|
|
72
|
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
73
|
+
a = {k: (v or "") for k, v in attrs}
|
|
74
|
+
if tag == "html":
|
|
75
|
+
self.lang = a.get("lang", "").strip()
|
|
76
|
+
elif tag == "title":
|
|
77
|
+
self._in_title = True
|
|
78
|
+
elif tag == "meta":
|
|
79
|
+
if a.get("http-equiv", "").lower() == "refresh":
|
|
80
|
+
self.redirect = True
|
|
81
|
+
if a.get("name", "").lower() == "robots":
|
|
82
|
+
words = a.get("content", "").lower().replace(",", " ").split()
|
|
83
|
+
self.noindex = self.noindex or "noindex" in words or "none" in words
|
|
84
|
+
key = a.get("property") or a.get("name")
|
|
85
|
+
if key:
|
|
86
|
+
self.meta.setdefault(key.lower(), a.get("content", "").strip())
|
|
87
|
+
|
|
88
|
+
def handle_endtag(self, tag: str) -> None:
|
|
89
|
+
if tag == "title":
|
|
90
|
+
self._in_title = False
|
|
91
|
+
|
|
92
|
+
def handle_data(self, data: str) -> None:
|
|
93
|
+
if self._in_title:
|
|
94
|
+
self.title += data
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def image_size(data: bytes) -> tuple[str, int, int] | None:
|
|
98
|
+
"""(format, width, height) of a PNG or JPEG, read from its header, or
|
|
99
|
+
None for anything else."""
|
|
100
|
+
if data[:8] == b"\x89PNG\r\n\x1a\n" and data[12:16] == b"IHDR":
|
|
101
|
+
width, height = struct.unpack(">II", data[16:24])
|
|
102
|
+
return "PNG", width, height
|
|
103
|
+
if data[:2] != b"\xff\xd8":
|
|
104
|
+
return None
|
|
105
|
+
i = 2
|
|
106
|
+
while i + 4 <= len(data):
|
|
107
|
+
if data[i] != 0xFF:
|
|
108
|
+
return None
|
|
109
|
+
marker = data[i + 1]
|
|
110
|
+
if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
|
|
111
|
+
i += 2
|
|
112
|
+
continue
|
|
113
|
+
(length,) = struct.unpack(">H", data[i + 2:i + 4])
|
|
114
|
+
# Start of frame: every SOFn but DHT (C4), JPG (C8) and DAC (CC).
|
|
115
|
+
if 0xC0 <= marker <= 0xCF and marker not in (0xC4, 0xC8, 0xCC):
|
|
116
|
+
height, width = struct.unpack(">HH", data[i + 5:i + 9])
|
|
117
|
+
return "JPEG", width, height
|
|
118
|
+
i += 2 + length
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def pages(root: Path) -> list[Path]:
|
|
123
|
+
"""Every HTML page under `root` that is meant to be shared."""
|
|
124
|
+
return sorted(p for p in root.rglob("*.html") if p.name != "404.html")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def address(root: Path, path: Path, base: str) -> str:
|
|
128
|
+
"""The address a page in `root` is served at under `base`."""
|
|
129
|
+
rel = path.relative_to(root).as_posix()
|
|
130
|
+
if rel == "index.html" or rel.endswith("/index.html"):
|
|
131
|
+
rel = rel[: -len("index.html")]
|
|
132
|
+
return urljoin(base, rel)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def local(root: Path, url: str, base: str | None) -> Path | None:
|
|
136
|
+
"""The file in `root` an address under `base` is, or None when the
|
|
137
|
+
address is elsewhere."""
|
|
138
|
+
if not base or not url.startswith(base):
|
|
139
|
+
return None
|
|
140
|
+
rel = urlsplit(url).path[len(urlsplit(base).path):]
|
|
141
|
+
target = (root / rel).resolve()
|
|
142
|
+
if not target.is_relative_to(root.resolve()):
|
|
143
|
+
return None
|
|
144
|
+
return target / "index.html" if target.is_dir() else target
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def check_page(rel: str, page: Page) -> dict[str, list[str]]:
|
|
148
|
+
"""The WEB-PAGE and WEB-OG-01/03 problems of one page; for a page that
|
|
149
|
+
asks not to be indexed, only those of WEB-PAGE-01 and its title."""
|
|
150
|
+
out: dict[str, list[str]] = {}
|
|
151
|
+
|
|
152
|
+
def problem(rule: str, text: str) -> None:
|
|
153
|
+
out.setdefault(rule, []).append(f"{rel}: {text}")
|
|
154
|
+
|
|
155
|
+
if not page.lang:
|
|
156
|
+
problem("WEB-PAGE-01", 'no <html lang>; set it to the page\'s language, like <html lang="en">, '
|
|
157
|
+
"so screen readers and translators read it right")
|
|
158
|
+
if not page.title.strip():
|
|
159
|
+
problem("WEB-PAGE-02", "no <title>; give the page a title, which search results and tabs show")
|
|
160
|
+
if page.noindex:
|
|
161
|
+
return out
|
|
162
|
+
if not page.meta.get("description"):
|
|
163
|
+
problem("WEB-PAGE-02", 'no <meta name="description">; add one or two sentences on what the page is')
|
|
164
|
+
missing = [k for k in ("og:title", "og:description", "og:type", "og:url", "og:image")
|
|
165
|
+
if not page.meta.get(k)]
|
|
166
|
+
if missing:
|
|
167
|
+
problem("WEB-OG-01", f"no {', '.join(missing)}; add "
|
|
168
|
+
+ " ".join(f'<meta property="{k}" content="...">' for k in missing)
|
|
169
|
+
+ " to its <head>, or a shared link shows no card")
|
|
170
|
+
if page.meta.get("og:url") and not page.meta["og:url"].startswith("https://"):
|
|
171
|
+
problem("WEB-OG-01", f"og:url is {page.meta['og:url']}; give the page's full https:// address")
|
|
172
|
+
if page.meta.get("twitter:card") != "summary_large_image":
|
|
173
|
+
problem("WEB-OG-03", '<meta name="twitter:card" content="summary_large_image"> is missing; '
|
|
174
|
+
"without it some sites show the image as a small thumbnail")
|
|
175
|
+
return out
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def check_image(root: Path, rel: str, page: Page, base: str | None,
|
|
179
|
+
seen: dict[Path, str | None]) -> list[str]:
|
|
180
|
+
"""The WEB-OG-02 problems of one page's og:image."""
|
|
181
|
+
url = page.meta.get("og:image")
|
|
182
|
+
if not url:
|
|
183
|
+
return []
|
|
184
|
+
if not url.startswith("https://"):
|
|
185
|
+
return [f"{rel}: og:image is {url}; give its full https:// address, which a crawler can fetch"]
|
|
186
|
+
path = local(root, url, base)
|
|
187
|
+
if path is None:
|
|
188
|
+
return []
|
|
189
|
+
if path not in seen:
|
|
190
|
+
seen[path] = image_problem(path, url)
|
|
191
|
+
problems = [f"{rel}: {seen[path]}"] if seen[path] else []
|
|
192
|
+
size = image_size(path.read_bytes()) if path.is_file() else None
|
|
193
|
+
for key, index in (("og:image:width", 1), ("og:image:height", 2)):
|
|
194
|
+
declared = page.meta.get(key)
|
|
195
|
+
if size and declared and declared != str(size[index]):
|
|
196
|
+
problems.append(f"{rel}: {key} says {declared}, but {url} is {size[index]}; "
|
|
197
|
+
"make them agree, or a card is laid out wrong")
|
|
198
|
+
return problems
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def image_problem(path: Path, url: str) -> str | None:
|
|
202
|
+
if not path.is_file():
|
|
203
|
+
return f"og:image {url} is not in the site; build it, or point og:image at a file that is"
|
|
204
|
+
data = path.read_bytes()
|
|
205
|
+
size = image_size(data)
|
|
206
|
+
if size is None:
|
|
207
|
+
return f"og:image {url} is not a PNG or JPEG, which LinkedIn needs; save it as one"
|
|
208
|
+
_, width, height = size
|
|
209
|
+
if width < MIN_WIDTH or height < MIN_HEIGHT:
|
|
210
|
+
return (f"og:image {url} is {width}×{height}; make it at least {MIN_WIDTH}×{MIN_HEIGHT} "
|
|
211
|
+
"(1200×630 is usual), or LinkedIn shows no image")
|
|
212
|
+
if len(data) > MAX_BYTES:
|
|
213
|
+
return f"og:image {url} is {len(data) // 1024} KB; keep it under 5 MB, or LinkedIn drops it"
|
|
214
|
+
return None
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def check_robots(root: Path, urls: list[str]) -> list[str]:
|
|
218
|
+
"""The WEB-ROBOTS-01 problems: an address in `urls` that robots.txt
|
|
219
|
+
keeps a preview bot from."""
|
|
220
|
+
robots_txt = root / "robots.txt"
|
|
221
|
+
if not robots_txt.is_file():
|
|
222
|
+
return []
|
|
223
|
+
robots = RobotFileParser()
|
|
224
|
+
robots.parse(robots_txt.read_text(encoding="utf-8", errors="replace").splitlines())
|
|
225
|
+
problems = []
|
|
226
|
+
for bot in PREVIEW_BOTS:
|
|
227
|
+
blocked = [u for u in urls if not robots.can_fetch(bot, u)]
|
|
228
|
+
if blocked:
|
|
229
|
+
shown = ", ".join(blocked[:3]) + (f" and {len(blocked) - 3} more" if len(blocked) > 3 else "")
|
|
230
|
+
problems.append(f"robots.txt keeps {bot} from {shown}, so links to it show no card; "
|
|
231
|
+
f"add a group for the preview bots ({', '.join(PREVIEW_BOTS)}) with Allow: /, "
|
|
232
|
+
"and keep search engines out with <meta name=\"robots\" content=\"noindex\"> instead")
|
|
233
|
+
return problems
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def checks(root: Path, base: str | None) -> tuple[int, dict[str, list[str]]]:
|
|
237
|
+
"""(pages checked, problems by rule)."""
|
|
238
|
+
out: dict[str, list[str]] = {}
|
|
239
|
+
seen: dict[Path, str | None] = {}
|
|
240
|
+
count, urls = 0, []
|
|
241
|
+
for path in pages(root):
|
|
242
|
+
page = Page(path.read_text(encoding="utf-8", errors="replace"))
|
|
243
|
+
if page.redirect:
|
|
244
|
+
continue
|
|
245
|
+
count += 1
|
|
246
|
+
rel = path.relative_to(root).as_posix()
|
|
247
|
+
for rule, texts in check_page(rel, page).items():
|
|
248
|
+
out.setdefault(rule, []).extend(texts)
|
|
249
|
+
if page.noindex:
|
|
250
|
+
continue
|
|
251
|
+
if image := check_image(root, rel, page, base, seen):
|
|
252
|
+
out.setdefault("WEB-OG-02", []).extend(image)
|
|
253
|
+
urls.append(address(root, path, base or PLACEHOLDER))
|
|
254
|
+
if base and page.meta.get("og:image", "").startswith(base):
|
|
255
|
+
urls.append(page.meta["og:image"])
|
|
256
|
+
if robots := check_robots(root, list(dict.fromkeys(urls))):
|
|
257
|
+
out["WEB-ROBOTS-01"] = robots
|
|
258
|
+
return count, out
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
# A copy of pubrules' conform.today and conform.exceptions (pubrules 0.34.0),
|
|
262
|
+
# so this module needs only the standard library: deploy-static runs it from
|
|
263
|
+
# its own source with nothing installed.
|
|
264
|
+
def today() -> datetime.date:
|
|
265
|
+
"""Today, or PUBHOST_TODAY (YYYY-MM-DD), which tests set."""
|
|
266
|
+
fixed = os.environ.get("PUBHOST_TODAY")
|
|
267
|
+
return datetime.date.fromisoformat(fixed) if fixed else datetime.date.today()
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def exceptions(config: dict, known: set[str],
|
|
271
|
+
family=lambda rule: str(rule).startswith("WEB-")) -> tuple[set[str], list[str], list[str]]:
|
|
272
|
+
"""The rule IDs waived, a description of each open exception, and the
|
|
273
|
+
problems with the declarations. Only the exceptions whose rule is in
|
|
274
|
+
`family` are read: the site rules' (WEB-); the rest of pubrules.json's
|
|
275
|
+
exceptions are the Rust crate standard's, which `pubrules conform` reads."""
|
|
276
|
+
waived, open_, problems = set(), [], []
|
|
277
|
+
for i, e in enumerate(config.get("exceptions", [])):
|
|
278
|
+
where = f"pubrules.json exceptions[{i}]"
|
|
279
|
+
if isinstance(e, dict) and "rule" in e and not family(e["rule"]):
|
|
280
|
+
continue
|
|
281
|
+
if not isinstance(e, dict) or set(e) != {"rule", "reason", "until"}:
|
|
282
|
+
problems.append(f"{where}: give exactly a rule, a reason and an until date")
|
|
283
|
+
continue
|
|
284
|
+
rule, reason, until = e["rule"], str(e["reason"]).strip(), e["until"]
|
|
285
|
+
if rule not in known:
|
|
286
|
+
problems.append(f"{where}: {rule} is not a rule of the standard")
|
|
287
|
+
continue
|
|
288
|
+
if not reason:
|
|
289
|
+
problems.append(f"{where}: {rule} needs a reason")
|
|
290
|
+
continue
|
|
291
|
+
try:
|
|
292
|
+
date = datetime.date.fromisoformat(str(until))
|
|
293
|
+
except ValueError:
|
|
294
|
+
problems.append(f"{where}: {rule}'s until must be a date, YYYY-MM-DD")
|
|
295
|
+
continue
|
|
296
|
+
if date < today():
|
|
297
|
+
problems.append(f"{rule}: the exception expired on {date}; meet the rule, or "
|
|
298
|
+
f"review the exception and set a new date ({reason})")
|
|
299
|
+
continue
|
|
300
|
+
waived.add(rule)
|
|
301
|
+
open_.append(f"{rule} until {date}: {reason}")
|
|
302
|
+
return waived, open_, problems
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def run(root: Path, directory: Path, base: str | None) -> int:
|
|
306
|
+
if base is not None:
|
|
307
|
+
if not base.startswith("https://"):
|
|
308
|
+
sys.exit(f"pubhost: --url ({base}) must be the site's https:// address")
|
|
309
|
+
base = base if base.endswith("/") else base + "/"
|
|
310
|
+
if not directory.is_dir():
|
|
311
|
+
sys.exit(f"pubhost: {directory} is not a directory; build the site first")
|
|
312
|
+
config_file = root / "pubrules.json"
|
|
313
|
+
config = json.loads(config_file.read_text()) if config_file.is_file() else {}
|
|
314
|
+
waived, open_, problems = exceptions(config, set(RULES))
|
|
315
|
+
count, found = checks(directory, base)
|
|
316
|
+
if count == 0:
|
|
317
|
+
problems.append(f"{directory} has no HTML pages to check; is it the built site?")
|
|
318
|
+
for rule, texts in sorted(found.items()):
|
|
319
|
+
if rule not in waived:
|
|
320
|
+
problems += [f"{rule} {t}" for t in texts]
|
|
321
|
+
for e in open_:
|
|
322
|
+
print(f"open exception: {e}")
|
|
323
|
+
if problems:
|
|
324
|
+
for p in problems:
|
|
325
|
+
print(p, file=sys.stderr)
|
|
326
|
+
print(f"pubhost site check: {len(problems)} problem(s) in {count} page(s)", file=sys.stderr)
|
|
327
|
+
return 1
|
|
328
|
+
print(f"{count} page(s) meet the {len(RULES) - len(waived)} site rules"
|
|
329
|
+
+ ("" if base else "; give --url to check the og:image files too"))
|
|
330
|
+
return 0
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def main(argv: list[str] | None = None) -> int:
|
|
334
|
+
"""`python -m pubhost.site check DIR [--url URL]`, as deploy-static runs
|
|
335
|
+
it without installing pubhost."""
|
|
336
|
+
import argparse
|
|
337
|
+
|
|
338
|
+
parser = argparse.ArgumentParser(prog="pubhost site")
|
|
339
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
340
|
+
p = commands.add_parser("check")
|
|
341
|
+
p.add_argument("dir", type=Path)
|
|
342
|
+
p.add_argument("--url")
|
|
343
|
+
args = parser.parse_args(argv)
|
|
344
|
+
return run(Path.cwd(), args.dir, args.url)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
if __name__ == "__main__":
|
|
348
|
+
sys.exit(main())
|