pubrules 0.48.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. pubrules/__init__.py +14 -0
  2. pubrules/audit.py +116 -0
  3. pubrules/audit_draft.py +295 -0
  4. pubrules/audit_prepare.py +1128 -0
  5. pubrules/cache.py +319 -0
  6. pubrules/cbor.py +45 -0
  7. pubrules/ci.py +910 -0
  8. pubrules/cli.py +1069 -0
  9. pubrules/conform.py +295 -0
  10. pubrules/doc_coverage.py +235 -0
  11. pubrules/freshness.py +75 -0
  12. pubrules/gherkin.py +348 -0
  13. pubrules/hooks.py +59 -0
  14. pubrules/hosting.py +372 -0
  15. pubrules/ignores.py +86 -0
  16. pubrules/new.py +242 -0
  17. pubrules/nightly.py +1002 -0
  18. pubrules/paths.py +32 -0
  19. pubrules/protect.py +596 -0
  20. pubrules/providers.py +129 -0
  21. pubrules/refs.py +213 -0
  22. pubrules/release-commit +1 -0
  23. pubrules/releases.py +107 -0
  24. pubrules/repos.py +67 -0
  25. pubrules/ruleset.json +43 -0
  26. pubrules/secrets_config.py +165 -0
  27. pubrules/security.py +134 -0
  28. pubrules/service_needs.py +467 -0
  29. pubrules/standard.json +635 -0
  30. pubrules/standard.py +266 -0
  31. pubrules/supply_chain.py +188 -0
  32. pubrules/tag-ruleset.json +14 -0
  33. pubrules/templates/common/AGENTS.md +32 -0
  34. pubrules/templates/common/pubrules +26 -0
  35. pubrules/templates/common/tests/features/repository.feature +11 -0
  36. pubrules/templates/common/tests/install-lock +65 -0
  37. pubrules/templates/common/tests/pytest.ini +4 -0
  38. pubrules/templates/common/tests/requirements-pubrules.txt +17 -0
  39. pubrules/templates/common/tests/requirements.txt +3 -0
  40. pubrules/templates/common/tests/steps/test_repository.py +14 -0
  41. pubrules/templates/deploy/pubdeploy +28 -0
  42. pubrules/templates/new-rust/CHANGELOG.md +3 -0
  43. pubrules/templates/new-rust/Cargo.toml +46 -0
  44. pubrules/templates/new-rust/LICENSE +202 -0
  45. pubrules/templates/new-rust/Makefile +18 -0
  46. pubrules/templates/new-rust/README.md +34 -0
  47. pubrules/templates/new-rust/SECURITY.md +26 -0
  48. pubrules/templates/new-rust/corpus/corpus.json +11 -0
  49. pubrules/templates/new-rust/deny.toml +32 -0
  50. pubrules/templates/new-rust/fuzz/Cargo.toml +23 -0
  51. pubrules/templates/new-rust/fuzz/fuzz_targets/greet.rs +8 -0
  52. pubrules/templates/new-rust/fuzz/gitignore +4 -0
  53. pubrules/templates/new-rust/fuzz/regressions/README.md +4 -0
  54. pubrules/templates/new-rust/gitignore +10 -0
  55. pubrules/templates/new-rust/src/lib.rs +91 -0
  56. pubrules/templates/new-rust/src/main.rs +19 -0
  57. pubrules/templates/new-rust/supply-chain/audits.toml +4 -0
  58. pubrules/templates/new-rust/supply-chain/config.toml +23 -0
  59. pubrules/templates/new-rust/supply-chain/imports.lock +2 -0
  60. pubrules/templates/new-rust/tests/conftest.py +9 -0
  61. pubrules/templates/new-rust/tests/features/cli.feature +14 -0
  62. pubrules/templates/new-rust/tests/steps/test_cli.py +15 -0
  63. pubrules/templates/new-rust/tests/vectors.rs +30 -0
  64. pubrules/templates/new-rust/tools/vectors/generate.py +24 -0
  65. pubrules/templates/python/Makefile +5 -0
  66. pubrules/templates/python/pubrules-python.mk +47 -0
  67. pubrules/templates/rust/Makefile +9 -0
  68. pubrules/templates/rust/pubrules.mk +138 -0
  69. pubrules/templates/rust/rust-toolchain.toml +8 -0
  70. pubrules/templates/service/pubsec +27 -0
  71. pubrules/templates/service/with-secrets +20 -0
  72. pubrules/templates/service/with-secrets.json +8 -0
  73. pubrules/variables.py +45 -0
  74. pubrules-0.48.0.dist-info/METADATA +1241 -0
  75. pubrules-0.48.0.dist-info/RECORD +79 -0
  76. pubrules-0.48.0.dist-info/WHEEL +5 -0
  77. pubrules-0.48.0.dist-info/entry_points.txt +2 -0
  78. pubrules-0.48.0.dist-info/licenses/LICENSE +202 -0
  79. pubrules-0.48.0.dist-info/top_level.txt +1 -0
pubrules/cache.py ADDED
@@ -0,0 +1,319 @@
1
+ """Keep what a build fetched or made between runs, by rules the repository states.
2
+
3
+ pubrules.json's `cache` block is one map of named entries, each with the
4
+ same keys:
5
+
6
+ "cache": {
7
+ "data": {"path": "data", "refresh": "week"},
8
+ "packs": {"path": ".packcache", "unit": "data/areas/{unit}/",
9
+ "inputs": ["data/boundaries.geojsonl", "mapbuild/*.py"],
10
+ "tools": ["tippecanoe --version"]},
11
+ "overview": {"path": ".packcache/overview",
12
+ "inputs": ["data/national/", "data/boundaries.geojsonl", "mapbuild/*.py"],
13
+ "tools": ["tippecanoe --version"]}
14
+ }
15
+
16
+ - `path`: the file or directory kept between runs.
17
+ - `refresh`: `run`, `day`, `week` (ISO week), `month` or `never`: how long
18
+ what is kept stands before it is fetched anew.
19
+ - `version`: a free string; changing it fetches anew.
20
+ - `inputs`: files, directories or globs what is kept depends on.
21
+ - `tools`: commands whose output counts, such as `tippecanoe --version`.
22
+ - `unit`: one pattern with `{unit}`, filled from `pubrules cache run
23
+ --unit`, so each unit gets a fingerprint of its own.
24
+
25
+ An entry needs `refresh` or `inputs` (or both).
26
+
27
+ **Fetched entries** (`refresh` other than `run`, and no `inputs` or
28
+ `unit`) are kept under a key that changes with the period and the version:
29
+ the first run of a period fetches them, and later runs restore them. A
30
+ fresh run restores none.
31
+
32
+ **Made entries** (with `inputs`, `unit` or `refresh: run`) are restored
33
+ from the latest run and saved under a key of each run. The build makes
34
+ them through
35
+
36
+ pubrules cache run NAME [--unit VALUE] --output FILE ... -- COMMAND
37
+
38
+ which fingerprints them (the entry's name, the unit, the `unit` pattern
39
+ filled in, every `inputs` entry, every `tools` command's output, and the
40
+ version), copies the outputs from `path` when a build with the same
41
+ fingerprint made them, and otherwise runs COMMAND, which must write every
42
+ `--output`, and keeps what it wrote. Saving the cache drops whatever no run
43
+ used since the last save, so `path` holds one build's worth.
44
+
45
+ `pubrules cache plan` gives the key for each entry, and `actions/cache`
46
+ restores and saves them. `plan --refresh` restores no entry at all, fetched
47
+ or made, so the run fetches and makes everything anew (`actions/pr-label`
48
+ tells a workflow when the pull request it runs for asks for that). Only Python's standard library is used, so the
49
+ action runs this module from its own copy of pubrules without installing
50
+ anything.
51
+ """
52
+
53
+ from __future__ import annotations
54
+
55
+ import argparse
56
+ import datetime as dt
57
+ import glob
58
+ import hashlib
59
+ import json
60
+ import os
61
+ import shutil
62
+ import subprocess
63
+ import sys
64
+ import tempfile
65
+ from pathlib import Path
66
+
67
+ CONFIG = "pubrules.json"
68
+ KEYS = ("path", "refresh", "version", "inputs", "tools", "unit")
69
+ REFRESH = ("run", "day", "week", "month", "never")
70
+ USED = ".pubrules-used"
71
+ # actions/cache gives this many slots: the most entries one cache block takes.
72
+ SLOTS = 8
73
+
74
+
75
+ class CacheError(Exception):
76
+ """The cache block or a request does not fit the rules."""
77
+
78
+
79
+ def check(block: object) -> dict:
80
+ """The cache block, refused with a message naming what is wrong."""
81
+ if not isinstance(block, dict):
82
+ raise CacheError(f"{CONFIG}'s cache is a map of named entries")
83
+ if len(block) > SLOTS:
84
+ raise CacheError(f"{CONFIG}'s cache has {len(block)} entries; it takes at most {SLOTS}")
85
+ for name, entry in block.items():
86
+ if not isinstance(entry, dict):
87
+ raise CacheError(f"cache entry {name} is a map of {', '.join(KEYS)}")
88
+ unknown = sorted(set(entry) - set(KEYS))
89
+ if unknown:
90
+ raise CacheError(f"cache entry {name} has no setting {', '.join(unknown)}; "
91
+ f"it takes {', '.join(KEYS)}")
92
+ if not isinstance(entry.get("path"), str) or not entry["path"]:
93
+ raise CacheError(f"cache entry {name} needs a path: the file or directory kept")
94
+ if "refresh" not in entry and "inputs" not in entry:
95
+ raise CacheError(f"cache entry {name} needs refresh or inputs (or both)")
96
+ if entry.get("refresh", "run") not in REFRESH:
97
+ raise CacheError(f"cache entry {name}: refresh is one of {', '.join(REFRESH)}")
98
+ if "version" in entry and not isinstance(entry["version"], str):
99
+ raise CacheError(f"cache entry {name}: version is a string")
100
+ for key in ("inputs", "tools"):
101
+ items = entry.get(key, [])
102
+ if not isinstance(items, list) or not all(isinstance(e, str) for e in items):
103
+ raise CacheError(f"cache entry {name}: {key} is a list of strings")
104
+ if "unit" in entry and (not isinstance(entry["unit"], str) or "{unit}" not in entry["unit"]):
105
+ raise CacheError(f"cache entry {name}: unit is one pattern with {{unit}} in it")
106
+ return block
107
+
108
+
109
+ def rules(root: Path) -> dict:
110
+ """pubrules.json's cache block, checked."""
111
+ path = root / CONFIG
112
+ if not path.is_file():
113
+ raise CacheError(f"no {CONFIG} here")
114
+ block = json.loads(path.read_text(encoding="utf-8")).get("cache")
115
+ if block is None:
116
+ raise CacheError(f"{CONFIG} has no cache block")
117
+ return check(block)
118
+
119
+
120
+ def fetched(entry: dict) -> bool:
121
+ """Whether an entry is kept for a period rather than made by the build."""
122
+ return entry.get("refresh", "run") != "run" and "unit" not in entry and "inputs" not in entry
123
+
124
+
125
+ def period(refresh: str, today: dt.date) -> str:
126
+ """The part of a fetched entry's key that changes when it should be fetched anew."""
127
+ if refresh == "day":
128
+ return today.isoformat()
129
+ if refresh == "week":
130
+ year, week, _ = today.isocalendar()
131
+ return f"{year}-W{week:02d}"
132
+ if refresh == "month":
133
+ return today.strftime("%Y-%m")
134
+ return "kept"
135
+
136
+
137
+ def _digest(text: str) -> str:
138
+ return hashlib.sha256(text.encode()).hexdigest()[:16]
139
+
140
+
141
+ def plan(root: Path, today: dt.date, run_id: str) -> list[dict]:
142
+ """For each entry, in name order: its path and its actions/cache key. A
143
+ fetched entry's key changes with its period and version and is restored
144
+ exactly; a made entry's key is new each run and is restored from the
145
+ latest by its prefix."""
146
+ out = []
147
+ for name, entry in sorted(rules(root).items()):
148
+ prefix = f"pubrules-cache-{name}-{_digest(json.dumps(entry, sort_keys=True))}-"
149
+ if fetched(entry):
150
+ key = prefix + period(entry["refresh"], today) + "-" + _digest(entry.get("version", ""))
151
+ out.append({"name": name, "path": entry["path"], "key": key, "prefix": "", "fetched": True})
152
+ else:
153
+ out.append({"name": name, "path": entry["path"], "key": prefix + run_id, "prefix": prefix,
154
+ "fetched": False})
155
+ return out
156
+
157
+
158
+ def _entry(root: Path, entry: str, h) -> None:
159
+ """Feed one file, directory or glob into the fingerprint `h`."""
160
+ # Each file is named relative to its entry, which may lie outside the
161
+ # repository (an absolute path, or a unit value naming one).
162
+ if any(c in entry for c in "*?["):
163
+ base = root
164
+ files = sorted(Path(p) for p in glob.glob(str(root / entry), recursive=True))
165
+ else:
166
+ path = root / entry
167
+ if path.is_dir():
168
+ base = path
169
+ files = sorted(p for p in path.rglob("*") if p.is_file())
170
+ elif path.is_file():
171
+ base = path.parent
172
+ files = [path]
173
+ else:
174
+ h.update(f"missing {entry}\0".encode())
175
+ return
176
+ h.update(f"entry {entry}\0".encode())
177
+ for f in files:
178
+ if f.is_file():
179
+ name = Path(os.path.relpath(f, base)).as_posix()
180
+ h.update(name.encode() + b"\0" + f.read_bytes() + b"\0")
181
+
182
+
183
+ def fingerprint(root: Path, name: str, entry: dict, unit: str | None) -> str:
184
+ """What one unit of a made entry depends on, as a hex digest."""
185
+ h = hashlib.sha256(f"entry {name}\0version {entry.get('version', '')}\0".encode())
186
+ if "unit" in entry:
187
+ if unit is None:
188
+ raise CacheError(f"cache entry {name} is made per unit: give --unit")
189
+ h.update(f"unit {unit}\0".encode())
190
+ _entry(root, entry["unit"].replace("{unit}", unit), h)
191
+ elif unit is not None:
192
+ raise CacheError(f"cache entry {name} has no unit pattern, so it takes no --unit")
193
+ for item in entry.get("inputs", []):
194
+ _entry(root, item, h)
195
+ for command in entry.get("tools", []):
196
+ out = subprocess.run(command, shell=True, cwd=root, capture_output=True, text=True)
197
+ h.update(f"tool {command}\0{out.returncode}\0{out.stdout}\0{out.stderr}\0".encode())
198
+ return h.hexdigest()[:32]
199
+
200
+
201
+ def run(root: Path, name: str, unit: str | None, outputs: list[Path], command: list[str]) -> str:
202
+ """Copy `outputs` from the cache when a build with this fingerprint made
203
+ them, else run `command` and keep them. Answers "reused" or "built"."""
204
+ block = rules(root)
205
+ if name not in block:
206
+ raise CacheError(f"{CONFIG}'s cache has no entry {name}")
207
+ entry = block[name]
208
+ if fetched(entry):
209
+ raise CacheError(f"cache entry {name} is fetched for a period, not made: it has no inputs or unit")
210
+ if not outputs:
211
+ raise CacheError("give each file the command writes as --output")
212
+ store = root / entry["path"]
213
+ key = fingerprint(root, name, entry, unit)
214
+ kept = store / key
215
+ files = [kept / f"{i}-{o.name}" for i, o in enumerate(outputs)]
216
+ store.mkdir(parents=True, exist_ok=True)
217
+ with (store / USED).open("a", encoding="utf-8") as used:
218
+ used.write(key + "\n")
219
+ if all(f.is_file() for f in files):
220
+ for f, o in zip(files, outputs):
221
+ o.parent.mkdir(parents=True, exist_ok=True)
222
+ shutil.copyfile(f, o)
223
+ return "reused"
224
+ if not command:
225
+ raise CacheError(f"no {name} kept for this unit, and no command to make it after --")
226
+ # The command's output goes to stderr, so stdout carries only the verdict.
227
+ done = subprocess.run(command, cwd=root, stdout=sys.stderr)
228
+ if done.returncode:
229
+ raise CacheError(f"the command for {name} failed (exit {done.returncode}); nothing kept")
230
+ missing = [str(o) for o in outputs if not o.is_file()]
231
+ if missing:
232
+ raise CacheError(f"the command for {name} did not write {', '.join(missing)}; nothing kept")
233
+ fresh = Path(tempfile.mkdtemp(dir=store, prefix=".new-"))
234
+ for i, o in enumerate(outputs):
235
+ shutil.copyfile(o, fresh / f"{i}-{o.name}")
236
+ if kept.exists():
237
+ shutil.rmtree(kept)
238
+ fresh.rename(kept)
239
+ return "built"
240
+
241
+
242
+ def prune(root: Path) -> list[str]:
243
+ """Drop what no run used since the last prune, in each made entry a run
244
+ used. An entry no run used this time is left as it is."""
245
+ report = []
246
+ for name, entry in sorted(rules(root).items()):
247
+ store = root / entry["path"]
248
+ used_file = store / USED
249
+ if fetched(entry) or not used_file.is_file():
250
+ continue
251
+ used = set(used_file.read_text(encoding="utf-8").split())
252
+ removed = 0
253
+ for child in store.iterdir():
254
+ # A made entry's path may hold another's (.packcache holds
255
+ # .packcache/overview): only fingerprints are this entry's.
256
+ if child.is_dir() and len(child.name) == 32 and child.name not in used:
257
+ shutil.rmtree(child)
258
+ removed += 1
259
+ used_file.unlink()
260
+ report.append(f"{name}: kept {len(used)}, removed {removed}")
261
+ return report
262
+
263
+
264
+ def main(argv: list[str] | None = None, root: Path | None = None) -> int:
265
+ parser = argparse.ArgumentParser(prog="pubrules cache", description=__doc__.split("\n")[0])
266
+ commands = parser.add_subparsers(dest="step", required=True)
267
+ p = commands.add_parser("plan", help="each entry's path and actions/cache key")
268
+ p.add_argument("--github-output", action="store_true", help="also append them to $GITHUB_OUTPUT")
269
+ p.add_argument("--fresh", action="store_true", help="restore no fetched entry")
270
+ p.add_argument("--refresh", action="store_true", help="restore no entry, fetched or made")
271
+ p = commands.add_parser("run", help="reuse a unit's outputs, or make and keep them: "
272
+ "run NAME [--unit VALUE] --output FILE... -- COMMAND")
273
+ p.add_argument("name")
274
+ p.add_argument("--unit")
275
+ p.add_argument("--output", action="append", default=[], type=Path, metavar="FILE")
276
+ commands.add_parser("prune", help="drop what no run used since the last prune "
277
+ "(actions/cache's save step does this)")
278
+ argv = sys.argv[1:] if argv is None else list(argv)
279
+ # Everything after -- is the command, given to it as it is.
280
+ command = argv[argv.index("--") + 1:] if "--" in argv else []
281
+ args = parser.parse_args(argv[:argv.index("--")] if "--" in argv else argv)
282
+ root = root or Path.cwd()
283
+ try:
284
+ if args.step == "plan":
285
+ today = dt.date.fromisoformat(os.environ["PUBRULES_TODAY"]) if os.environ.get("PUBRULES_TODAY") \
286
+ else dt.datetime.now(dt.UTC).date()
287
+ run_id = (f"{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}"
288
+ if os.environ.get("GITHUB_RUN_ID") else dt.datetime.now(dt.UTC).strftime("%Y%m%d%H%M%S"))
289
+ slots = plan(root, today, run_id)
290
+ for s in slots:
291
+ print(f"{s['name']}: path={s['path']} key={s['key']}"
292
+ + (f" restore-prefix={s['prefix']}" if s["prefix"] else "")
293
+ + (" (refresh: not restored)" if args.refresh
294
+ else " (fresh: not restored)" if args.fresh and s["fetched"] else ""))
295
+ if args.github_output:
296
+ with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as gh:
297
+ for i in range(1, SLOTS + 1):
298
+ s = slots[i - 1] if i <= len(slots) else None
299
+ restore = bool(s) and not args.refresh and not (args.fresh and s["fetched"])
300
+ gh.write(f"path-{i}={s['path'] if s else ''}\n")
301
+ gh.write(f"key-{i}={s['key'] if s else ''}\n")
302
+ gh.write(f"prefix-{i}={s['prefix'] if s else ''}\n")
303
+ gh.write(f"restore-{i}={'true' if restore else 'false'}\n")
304
+ return 0
305
+ if args.step == "run":
306
+ verdict = run(root, args.name, args.unit,
307
+ [o if o.is_absolute() else root / o for o in args.output], command)
308
+ print(f"{verdict} {args.name}" + (f" {args.unit}" if args.unit is not None else ""))
309
+ return 0
310
+ for line in prune(root):
311
+ print(line)
312
+ return 0
313
+ except CacheError as e:
314
+ print(f"pubrules cache: {e}", file=sys.stderr)
315
+ return 2
316
+
317
+
318
+ if __name__ == "__main__":
319
+ sys.exit(main())
pubrules/cbor.py ADDED
@@ -0,0 +1,45 @@
1
+ """Publet's canonical CBOR, enough to write back an object `pub read --json`
2
+ shows: integers, text, byte strings (`{"$bytes": hex}`), arrays and maps
3
+ with keys in byte order. Every object written this way is checked against
4
+ its identifier, so a mistake here cannot write a wrong object, only fail."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import base64
9
+ import hashlib
10
+
11
+
12
+ def head(major: int, n: int) -> bytes:
13
+ if n < 24:
14
+ return bytes([major << 5 | n])
15
+ for size, code in ((1, 24), (2, 25), (4, 26), (8, 27)):
16
+ if n < 1 << (8 * size):
17
+ return bytes([major << 5 | code]) + n.to_bytes(size, "big")
18
+ raise ValueError(f"{n} does not fit in CBOR's lengths")
19
+
20
+
21
+ def encode(value) -> bytes:
22
+ if isinstance(value, bool):
23
+ return b"\xf5" if value else b"\xf4"
24
+ if value is None:
25
+ return b"\xf6"
26
+ if isinstance(value, int):
27
+ return head(0, value) if value >= 0 else head(1, -1 - value)
28
+ if isinstance(value, str):
29
+ b = value.encode()
30
+ return head(3, len(b)) + b
31
+ if isinstance(value, list):
32
+ return head(4, len(value)) + b"".join(encode(x) for x in value)
33
+ if isinstance(value, dict):
34
+ if set(value) == {"$bytes"}:
35
+ b = bytes.fromhex(value["$bytes"])
36
+ return head(2, len(b)) + b
37
+ items = sorted(value.items(), key=lambda kv: kv[0].encode())
38
+ return head(5, len(items)) + b"".join(encode(k) + encode(v) for k, v in items)
39
+ raise TypeError(f"no canonical CBOR for {type(value).__name__}")
40
+
41
+
42
+ def cid(data: bytes) -> str:
43
+ """Section 4.2's identifier of `data`."""
44
+ digest = hashlib.sha256(data).digest()
45
+ return "pub:sha2-256:" + base64.b32encode(digest).decode().lower().rstrip("=")