useful-views 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,53 @@
1
+ """useful-views: a fixed project structure in which every derived table
2
+ records the recipe that made it."""
3
+
4
+ from importlib.metadata import PackageNotFoundError, version as _version
5
+
6
+ from .status import Finding, check
7
+ from .errors import (
8
+ OutputError,
9
+ ProjectError,
10
+ ProjectNotFound,
11
+ RecipeRefused,
12
+ StepFailed,
13
+ UsefulViewsError,
14
+ UsefulViewsWarning,
15
+ )
16
+ from .cookbooks import CookbookError, CookbookResult, cookbook
17
+ from .examples import ExampleError, example
18
+ from .new import new_recipe
19
+ from .project import init, root
20
+ from .rulebook import RULES, Rule
21
+ from .runner import RunResult, run
22
+ from .syncing import SyncResult, sync
23
+
24
+ try:
25
+ __version__ = _version("useful-views")
26
+ except PackageNotFoundError: # running from a source tree that is not installed
27
+ __version__ = "0.0.0+unknown"
28
+
29
+ __all__ = [
30
+ "CookbookError",
31
+ "CookbookResult",
32
+ "ExampleError",
33
+ "Finding",
34
+ "OutputError",
35
+ "ProjectError",
36
+ "ProjectNotFound",
37
+ "RULES",
38
+ "RecipeRefused",
39
+ "Rule",
40
+ "RunResult",
41
+ "StepFailed",
42
+ "SyncResult",
43
+ "UsefulViewsError",
44
+ "UsefulViewsWarning",
45
+ "check",
46
+ "cookbook",
47
+ "example",
48
+ "init",
49
+ "new_recipe",
50
+ "root",
51
+ "run",
52
+ "sync",
53
+ ]
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
useful_views/cli.py ADDED
@@ -0,0 +1,393 @@
1
+ """The useful-views command: a thin wrapper over the Python functions."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import contextlib
7
+ import io
8
+ import json
9
+ import os
10
+ import signal
11
+ import sys
12
+ import threading
13
+ import warnings
14
+
15
+ from . import __version__, check, cookbook, example, init, new_recipe, run, sync
16
+ from .errors import UsefulViewsError, UsefulViewsWarning
17
+ from .rulebook import RULES, describe_rules
18
+ from .status import syncable
19
+
20
+ EXIT_OK = 0
21
+ EXIT_PROBLEMS = 1
22
+ EXIT_USAGE = 2
23
+ EXIT_FAILED = 3
24
+
25
+ EXIT_INTERRUPTED = 130 # 128 + SIGINT; a stop by another signal exits 128 + its number
26
+
27
+ EXIT_CODES = """exit codes:
28
+ 0 success
29
+ 1 check found problems, or cookbook --check found the cookbook out of date
30
+ 2 usage error (unknown command or option, or a bad -C path)
31
+ 3 refused or failed (the message says what to fix, and in which file)
32
+ 130 interrupted (Ctrl-C)
33
+ 143 terminated (SIGTERM, from a CI cancel or a tool's time limit; 129 SIGHUP; 149 Ctrl-Break)"""
34
+
35
+
36
+ class UsageError(Exception):
37
+ """The command line itself is wrong."""
38
+
39
+
40
+ class _Parser(argparse.ArgumentParser):
41
+ def __init__(self, *args, **kwargs):
42
+ kwargs.setdefault("allow_abbrev", False) # --js is an error, not --json
43
+ super().__init__(*args, **kwargs)
44
+
45
+ def error(self, message): # raise instead of exiting, so --json can still answer
46
+ raise UsageError(f"{self.prog}: {message}")
47
+
48
+
49
+ def _error(kind: str, message: str) -> dict:
50
+ """An error as JSON, with the same keys as UsefulViewsError.to_dict()."""
51
+ return {"type": kind, "message": message, "recipe": None, "sync_result": None}
52
+
53
+
54
+ class Terminated(KeyboardInterrupt):
55
+ """The command was stopped by a signal other than Ctrl-C. Raised from the
56
+ signal handler, so the step is stopped and cleaned up as for Ctrl-C."""
57
+
58
+ def __init__(self, signum: int):
59
+ super().__init__()
60
+ self.signum = signum
61
+
62
+
63
+ @contextlib.contextmanager
64
+ def _stop_signals():
65
+ """Treat SIGTERM and SIGHUP (POSIX) and SIGBREAK (Windows) like Ctrl-C
66
+ while the command runs, so a cancelled CI job or a tool's timeout does
67
+ not leave a step running on its own. Only the command line does this;
68
+ the Python API leaves signal handling to its caller."""
69
+ if threading.current_thread() is not threading.main_thread():
70
+ yield
71
+ return
72
+
73
+ def stop(signum, frame):
74
+ raise Terminated(signum)
75
+
76
+ previous = {}
77
+ for name in ("SIGTERM", "SIGHUP", "SIGBREAK"):
78
+ sig = getattr(signal, name, None)
79
+ # Only replace the default: a signal ignored on purpose (nohup ignores
80
+ # SIGHUP, and steps inherit that) or handled by a caller stays so.
81
+ if sig is None or signal.getsignal(sig) is not signal.SIG_DFL:
82
+ continue
83
+ try:
84
+ previous[sig] = signal.signal(sig, stop)
85
+ except (OSError, ValueError):
86
+ pass
87
+ try:
88
+ yield
89
+ finally:
90
+ for sig, handler in previous.items():
91
+ try:
92
+ signal.signal(sig, handler)
93
+ except (OSError, ValueError):
94
+ pass
95
+
96
+
97
+ def _parser() -> argparse.ArgumentParser:
98
+ formatter = argparse.RawDescriptionHelpFormatter
99
+ parser = _Parser(
100
+ prog="useful-views",
101
+ description=(
102
+ "Turn raw data into documented derived tables (useful views). Each recipe in recipes/ "
103
+ "turns declared inputs into declared views by running numbered Python steps, and every "
104
+ "run is recorded in datapackage.json."
105
+ ),
106
+ epilog=EXIT_CODES + "\n\nAlso runs as: python -m useful_views",
107
+ formatter_class=formatter,
108
+ )
109
+ parser.add_argument("--version", action="version", version=f"useful-views {__version__}")
110
+ project_help = "run as if started in PATH (like git -C), for example a project folder"
111
+ json_help = "print the result as JSON on standard output; other output goes to standard error"
112
+ parser.add_argument("-C", "--project", metavar="PATH", help=project_help)
113
+ parser.add_argument("--json", action="store_true", help=json_help)
114
+ # The same options after the command; SUPPRESS keeps a value given before it.
115
+ common = _Parser(add_help=False)
116
+ common.add_argument("-C", "--project", metavar="PATH", default=argparse.SUPPRESS, help=project_help)
117
+ common.add_argument("--json", action="store_true", default=argparse.SUPPRESS, help=json_help)
118
+ commands = parser.add_subparsers(dest="command", required=True, metavar="command", parser_class=_Parser)
119
+
120
+ def add(name: str, help: str, description: str) -> argparse.ArgumentParser:
121
+ return commands.add_parser(name, help=help, description=description, epilog=EXIT_CODES,
122
+ parents=[common], formatter_class=formatter)
123
+
124
+ p = add("init", "create the project structure (safe to run twice)",
125
+ "Create the project structure in PATH: raw/, recipes/, work/, useful-views/, a starter "
126
+ "README.md, AGENTS.md (for AI coding assistants), .gitignore and datapackage.json. Never "
127
+ "overwrites a file.")
128
+ p.add_argument("path", nargs="?", default=".", help="project folder (default: the current folder)")
129
+ p.add_argument("--force", action="store_true",
130
+ help="create the project even in your home folder or at the top of a drive")
131
+
132
+ p = add("example", "download a ready-to-run example project",
133
+ "Download an example project (raw data and recipes) from the useful-views-data repository, "
134
+ "check every file's SHA-256, and set it up like init. Then run: useful-views sync")
135
+ p.add_argument("name", help="the example to download, such as pwt")
136
+ p.add_argument("path", nargs="?", default=None, help="folder to create (default: <name>-example)")
137
+
138
+ p = add("new-recipe", "create recipes/<name>/ from a template",
139
+ "Create recipes/NAME/ with a template recipe.toml and a stub step, 01_first_step.py. Both say "
140
+ "TODO and are refused until you fill them in.")
141
+ p.add_argument("name", help="the recipe's name: lowercase letters, digits and underscores, like pwt_panel")
142
+
143
+ p = add("run", "run one recipe and record what it produced",
144
+ "Run one recipe. It is checked against the rules first (useful-views rules), then its views are "
145
+ "deleted and work/RECIPE/ emptied, each step runs in numeric order as its own process with this "
146
+ "Python and the project root as working directory, every declared view must be written, and the "
147
+ "run is recorded in datapackage.json and useful-views/README.md. Other recipes are not run.")
148
+ p.add_argument("recipe", help="the recipe's name, as in pwt_panel or recipes/pwt_panel/")
149
+
150
+ add("check", "report the state of the project",
151
+ "Report what is missing or out of place, every rule a recipe breaks (refused), and recipes that "
152
+ "sync would run. Changes nothing. Does not notice changes inside a step's code.")
153
+
154
+ p = add("sync", "make the views match the recipe folders, like uv sync",
155
+ "Make the views and datapackage.json match the recipe folders, like uv sync. Checks every recipe "
156
+ "first and changes nothing if any would be refused. Then removes the records and view files of "
157
+ "deleted recipes and outputs, and runs, upstream first, every recipe that is new, changed or "
158
+ "missing its views, plus everything downstream of those.")
159
+ p.add_argument("--all", action="store_true",
160
+ help="run every recipe again, upstream first (sync does not notice edits to a step's code)")
161
+
162
+ p = add("cookbook", "write cookbook.json, the ledger of the recipes used",
163
+ "Write cookbook.json at the project root: every recorded recipe, in run order, with its "
164
+ "description, inputs, outputs and steps, and the raw inputs the recipes read. Generated from "
165
+ "datapackage.json, so the project must be in sync to write it. Commit it, like a lock file.")
166
+ p.add_argument("--check", action="store_true",
167
+ help="do not write; exit 1 if cookbook.json is missing or out of date, or a recipe changed "
168
+ "without a sync. Needs no data, so it works in CI")
169
+
170
+ add("rules", "list the rules a recipe must follow",
171
+ "List the numbered rules. A refusal names the rule it broke.")
172
+ return parser
173
+
174
+
175
+ def main(argv: list[str] | None = None) -> int:
176
+ for stream in (sys.stdout, sys.stderr):
177
+ try:
178
+ stream.reconfigure(errors="backslashreplace")
179
+ except (AttributeError, ValueError):
180
+ pass
181
+ argv = sys.argv[1:] if argv is None else list(argv)
182
+ wants_json = any(a == "--json" or a.startswith("--json=") for a in argv)
183
+ try:
184
+ if wants_json:
185
+ # --help and --version print and stop inside argparse; answer them
186
+ # in JSON too, so standard output always carries one document.
187
+ printed = io.StringIO()
188
+ try:
189
+ with contextlib.redirect_stdout(printed):
190
+ args = _parser().parse_args(argv)
191
+ except SystemExit as exc:
192
+ text = printed.getvalue()
193
+ print(text, end="", file=sys.stderr)
194
+ if text.startswith("usage:"):
195
+ data = {"ok": True, "help": text, "warnings": []}
196
+ else:
197
+ data = {"ok": True, "version": __version__, "warnings": []}
198
+ print(json.dumps(data, indent=2))
199
+ return exc.code if isinstance(exc.code, int) else EXIT_OK
200
+ else:
201
+ args = _parser().parse_args(argv)
202
+ except UsageError as exc:
203
+ return _fail(wants_json, str(exc), _error("UsageError", str(exc)), code=EXIT_USAGE)
204
+ try:
205
+ start = os.getcwd()
206
+ except OSError: # the folder this was started in has been deleted
207
+ start = None
208
+ if not args.project:
209
+ message = "the current folder no longer exists; cd into the project, or use -C <path>"
210
+ return _fail(args.json, message, _error("UsageError", message), code=EXIT_USAGE)
211
+ try:
212
+ if args.project:
213
+ try:
214
+ os.chdir(os.path.expanduser(args.project))
215
+ except OSError as exc:
216
+ message = f"cannot use -C {args.project}: {exc.strerror or exc}"
217
+ return _fail(args.json, message, _error("UsageError", message), code=EXIT_USAGE)
218
+ with _stop_signals():
219
+ return _main(args)
220
+ finally:
221
+ try:
222
+ if start is not None:
223
+ os.chdir(start)
224
+ except OSError:
225
+ pass # the folder it started in was deleted meanwhile
226
+
227
+
228
+ def _main(args: argparse.Namespace) -> int:
229
+ shown: list[str] = [] # every warning, for the JSON output
230
+ with warnings.catch_warnings():
231
+ original = warnings.showwarning
232
+
233
+ def show(message, category, filename, lineno, file=None, line=None):
234
+ if issubclass(category, UsefulViewsWarning):
235
+ print(f"warning: {message}", file=sys.stderr)
236
+ if str(message) not in shown:
237
+ shown.append(str(message))
238
+ else:
239
+ original(message, category, filename, lineno, file, line)
240
+
241
+ # Warnings are part of the command's output (and of its JSON), so show
242
+ # them whatever filters the environment sets.
243
+ warnings.simplefilter("always", UsefulViewsWarning)
244
+ warnings.showwarning = show
245
+ out = sys.stdout
246
+ try:
247
+ if args.json:
248
+ # Step output and progress go to standard error; standard
249
+ # output carries only the JSON.
250
+ with contextlib.redirect_stdout(sys.stderr):
251
+ code, data = _dispatch(args, shown)
252
+ data["warnings"] = list(dict.fromkeys([*data.get("warnings", []), *shown]))
253
+ print(json.dumps(data, indent=2), file=out)
254
+ return code
255
+ code, _ = _dispatch(args, shown)
256
+ return code
257
+ except UsefulViewsError as exc:
258
+ return _fail(args.json, str(exc), exc.to_dict(), out, shown=shown)
259
+ except KeyboardInterrupt as exc:
260
+ if isinstance(exc, Terminated):
261
+ name = signal.Signals(exc.signum).name
262
+ message, code = f"terminated ({name})", 128 + exc.signum
263
+ else:
264
+ message, code = "interrupted", EXIT_INTERRUPTED
265
+ note = str(exc) # what sync had done, when it was sync that stopped
266
+ if note:
267
+ message = f"{message}\n\n{note}"
268
+ error = _error("Interrupted", message)
269
+ result = getattr(exc, "sync_result", None)
270
+ error["sync_result"] = result.to_dict() if result is not None else None
271
+ return _fail(args.json, message, error, out, code=code, shown=shown)
272
+ except Exception as exc: # an unexpected failure: still readable, still exit 3
273
+ message = f"{type(exc).__name__}: {exc}"
274
+ return _fail(args.json, message, _error(type(exc).__name__, str(exc)), out, shown=shown)
275
+
276
+
277
+ def _fail(as_json: bool, message: str, error: dict, out=None, *, code: int = EXIT_FAILED,
278
+ shown: list[str] | None = None) -> int:
279
+ print(f"error: {message}", file=sys.stderr)
280
+ if as_json:
281
+ # ASCII-escaped, so the JSON stays valid on any console encoding.
282
+ document = {"ok": False, "error": error, "warnings": list(shown or [])}
283
+ print(json.dumps(document, indent=2), file=out or sys.stdout)
284
+ return code
285
+
286
+
287
+ def _dispatch(args: argparse.Namespace, shown: list[str]) -> tuple[int, dict]:
288
+ if args.command == "init":
289
+ folder = init(args.path, force=args.force)
290
+ print(f"useful-views project ready at {folder}")
291
+ return EXIT_OK, {"ok": True, "project": str(folder)}
292
+ if args.command == "example":
293
+ folder = example(args.name, args.path)
294
+ print(f'Example {args.name!r} ready at {folder}. Make its views with: useful-views -C "{folder}" sync')
295
+ return EXIT_OK, {"ok": True, "project": str(folder)}
296
+ if args.command == "new-recipe":
297
+ folder = new_recipe(args.name)
298
+ print(f"Created {folder}. Edit its recipe.toml and 01_first_step.py, then run: useful-views run {args.name}")
299
+ return EXIT_OK, {"ok": True, "recipe": args.name, "folder": str(folder)}
300
+ if args.command == "run":
301
+ result = run(args.recipe)
302
+ print(f"Ran recipes/{result.recipe}. Wrote {', '.join(result.outputs)}.")
303
+ print("Updated datapackage.json and useful-views/README.md.")
304
+ return EXIT_OK, {"ok": True, **result.to_dict()}
305
+ if args.command == "check":
306
+ findings = check()
307
+ data = {"ok": not findings, "findings": [f.to_dict() for f in findings]}
308
+ if not findings:
309
+ if shown:
310
+ print(f"No problems found ({len(shown)} warning{'s' if len(shown) > 1 else ''} above).")
311
+ else:
312
+ print("No problems found.")
313
+ return EXIT_OK, data
314
+ for finding in findings:
315
+ first, *rest = finding.message.split("\n")
316
+ print(f"{finding.kind}: {first}")
317
+ for line in rest:
318
+ print(f" {line}" if line else "")
319
+ for_sync = [f for f in findings if syncable(f)]
320
+ others = [f for f in findings if not syncable(f) and f.kind not in ("stale-cookbook", "stray-file")]
321
+ if for_sync and not others:
322
+ then = ", then useful-views cookbook" if any(f.kind == "stale-cookbook" for f in findings) else ""
323
+ print(f"\nTo make or update everything that is missing or changed, upstream first, run: "
324
+ f"useful-views sync{then}")
325
+ elif for_sync:
326
+ print("\nFix the other findings first; then useful-views sync makes or updates the rest, upstream first.")
327
+ return EXIT_PROBLEMS, data
328
+ if args.command == "sync":
329
+ result = sync(all=args.all)
330
+ if result.removed:
331
+ print("Removed the records of deleted recipes: " + ", ".join(f"recipes/{n}" for n in result.removed) + ".")
332
+ if result.dropped:
333
+ makers = _makers()
334
+ moved = [f"{v} (now made by recipes/{makers[v]})" for v in result.dropped if v in makers]
335
+ gone = [v for v in result.dropped if v not in makers]
336
+ if gone:
337
+ print("Removed views no recipe declares any more: " + ", ".join(gone) + ".")
338
+ if moved:
339
+ print("Moved to another recipe: " + ", ".join(moved) + ".")
340
+ if result.deleted:
341
+ print("Deleted: " + ", ".join(result.deleted) + ".")
342
+ if result.ran:
343
+ print("Ran, in order: " + ", ".join(f"recipes/{n}" for n in result.ran) + ".")
344
+ if not (result.removed or result.dropped or result.ran):
345
+ print("Everything is in sync: nothing to remove or run.")
346
+ else:
347
+ print("Updated datapackage.json and useful-views/README.md.")
348
+ return EXIT_OK, {"ok": True, **result.to_dict()}
349
+ if args.command == "cookbook":
350
+ result = cookbook(check=args.check)
351
+ data = {"ok": result.up_to_date, **result.to_dict()}
352
+ if args.check:
353
+ if result.up_to_date:
354
+ print("cookbook.json is up to date.")
355
+ return EXIT_OK, data
356
+ if result.findings:
357
+ print("The recipe folders no longer match their records in datapackage.json, so cookbook.json "
358
+ "is out of date:")
359
+ for finding in result.findings:
360
+ print(f" {finding.kind}: {finding.message.splitlines()[0]}")
361
+ if all(syncable(f) for f in result.findings):
362
+ print("Run useful-views sync, then useful-views cookbook, and commit the results.")
363
+ else:
364
+ print("Fix what is listed (useful-views check shows each in full), then run useful-views sync "
365
+ "and useful-views cookbook, and commit the results.")
366
+ else:
367
+ print("cookbook.json is missing or out of date. Update it with: useful-views cookbook")
368
+ return EXIT_PROBLEMS, data
369
+ if result.written:
370
+ print(f"Wrote cookbook.json: {len(result.recipes)} recipe(s), in run order.")
371
+ else:
372
+ print("cookbook.json is already up to date.")
373
+ return EXIT_OK, data
374
+ if args.command == "rules":
375
+ print(describe_rules())
376
+ return EXIT_OK, {"ok": True, "rules": [r._asdict() for r in RULES.values()]}
377
+ raise AssertionError(args.command)
378
+
379
+
380
+ def _makers() -> dict[str, str]:
381
+ """Each recorded view and the recipe that made it, after a sync."""
382
+ from .project import find_root
383
+ from .record import load_datapackage
384
+
385
+ try:
386
+ data = load_datapackage(find_root())
387
+ except UsefulViewsError:
388
+ return {}
389
+ return {out: r["name"] for r in data["recipes"] for out in r["outputs"]}
390
+
391
+
392
+ if __name__ == "__main__":
393
+ sys.exit(main())
@@ -0,0 +1,229 @@
1
+ """cookbook(): write cookbook.json, the ledger of the recipes a project uses.
2
+
3
+ Like `uv lock` writing `uv.lock`, the cookbook is produced by a command and
4
+ never edited by hand. Version 1 records each recipe and its steps, in run
5
+ order, and the raw inputs they read. Later versions are planned to add the
6
+ SHA-256 of every file and the environment the steps ran in.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ from dataclasses import dataclass, field
13
+ from pathlib import Path
14
+
15
+ from .errors import ProjectError, UsefulViewsError
16
+ from .paths import RAW
17
+ from .project import find_root
18
+ from .lock import project_lock
19
+ from .record import load_datapackage, write_atomic
20
+
21
+ COOKBOOK = "cookbook.json"
22
+ FORMAT_VERSION = 1
23
+
24
+ # check() findings that do not stop a cookbook from being written.
25
+ NOT_BLOCKING = {"stray-file", "stale-cookbook", "stale-catalog"}
26
+
27
+ # With check=True, only findings that mean datapackage.json no longer matches
28
+ # the recipe folders count; they need no data on disk, so a CI job with only
29
+ # the git checkout (no raw/, no views) gets a real answer.
30
+ RECORD_KINDS = {"unreadable-recipe", "refused", "never-run", "missing-recipe-folder"}
31
+ RECORD_REASONS = {"changed", "outputs-changed", "outputs-dropped"}
32
+
33
+
34
+ def _out_of_record(finding) -> bool:
35
+ return finding.kind in RECORD_KINDS or (finding.kind == "out-of-date" and finding.reason in RECORD_REASONS)
36
+
37
+
38
+ class CookbookError(UsefulViewsError):
39
+ """The cookbook cannot be written because the project is not in sync.
40
+
41
+ Attributes:
42
+ findings: The `check` findings that must be resolved first.
43
+ """
44
+
45
+ def __init__(self, message: str, *, findings: list | None = None):
46
+ super().__init__(message)
47
+ self.findings = list(findings or [])
48
+
49
+ def to_dict(self) -> dict:
50
+ data = super().to_dict()
51
+ data["findings"] = [f.to_dict() for f in self.findings]
52
+ return data
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class CookbookResult:
57
+ """What `cookbook` did, or found with `check=True`.
58
+
59
+ Attributes:
60
+ path: The cookbook file, `cookbook.json` at the project root.
61
+ written: True if the file was created or changed.
62
+ up_to_date: True if the file on disk matches the project's recipes
63
+ (after writing, always True; with `check=True`, the answer).
64
+ recipes: The recipes in the cookbook, in run order.
65
+ findings: With `check=True`, the `check` findings that mean the
66
+ recorded recipes no longer match the recipe folders (so the
67
+ cookbook cannot be up to date); otherwise empty.
68
+ """
69
+
70
+ path: Path
71
+ written: bool
72
+ up_to_date: bool
73
+ recipes: tuple[str, ...] = field(default=())
74
+ findings: tuple = field(default=())
75
+
76
+ def to_dict(self) -> dict:
77
+ return {
78
+ "path": str(self.path),
79
+ "written": self.written,
80
+ "up_to_date": self.up_to_date,
81
+ "recipes": list(self.recipes),
82
+ "findings": [f.to_dict() for f in self.findings],
83
+ }
84
+
85
+
86
+ def _run_order(records: dict[str, dict]) -> list[str]:
87
+ """Recipe names, upstream first; recipes that do not depend on each other
88
+ in name order."""
89
+ makers = {out: name for name, r in records.items() for out in r["outputs"]}
90
+ upstream = {
91
+ name: sorted({makers[i] for i in r["inputs"] if i in makers} - {name}) for name, r in records.items()
92
+ }
93
+ order: list[str] = []
94
+ done: set[str] = set()
95
+
96
+ def visit(name: str, seen: tuple[str, ...] = ()) -> None:
97
+ if name in done or name in seen:
98
+ return
99
+ for up in upstream[name]:
100
+ visit(up, (*seen, name))
101
+ done.add(name)
102
+ order.append(name)
103
+
104
+ for name in sorted(records):
105
+ visit(name)
106
+ return order
107
+
108
+
109
+ def build(data: dict) -> dict:
110
+ """The cookbook for a project's recorded recipes (datapackage.json)."""
111
+ records = {r["name"]: r for r in data["recipes"]}
112
+ order = _run_order(records)
113
+ raw = sorted({p for r in records.values() for p in r["inputs"] if p.startswith(f"{RAW}/")})
114
+ return {
115
+ "version": FORMAT_VERSION,
116
+ "generator": "useful-views",
117
+ "project": data.get("name"),
118
+ "raw_inputs": raw,
119
+ "recipes": [records[name] for name in order],
120
+ }
121
+
122
+
123
+ def render(book: dict) -> str:
124
+ return json.dumps(book, indent=2, ensure_ascii=False) + "\n"
125
+
126
+
127
+ def expected_text(project: Path) -> str:
128
+ """What cookbook.json should contain for the project's current records."""
129
+ return render(build(load_datapackage(project)))
130
+
131
+
132
+ def read_cookbook(project: Path) -> str | None:
133
+ """The text of cookbook.json with LF line endings, or None if there is none.
134
+
135
+ A file that is not UTF-8 text reads as "", which never matches, so it
136
+ counts as out of date and `cookbook` rewrites it. A file that cannot be
137
+ read at all raises ProjectError.
138
+ """
139
+ path = project / COOKBOOK
140
+ if path.is_dir(): # tested first: Windows reports a folder as "permission denied"
141
+ raise ProjectError(f"{COOKBOOK} is a folder. Delete it, then run useful-views cookbook.")
142
+ try:
143
+ data = path.read_bytes()
144
+ except FileNotFoundError:
145
+ return None
146
+ except OSError as exc:
147
+ raise ProjectError(
148
+ f"Could not read {COOKBOOK}: {exc.strerror or exc}. Check its permissions, or delete it and run "
149
+ "useful-views cookbook."
150
+ ) from exc
151
+ try:
152
+ return data.decode("utf-8").replace("\r\n", "\n")
153
+ except UnicodeDecodeError:
154
+ return ""
155
+
156
+
157
+ def stale_warning(project: Path) -> str | None:
158
+ """A warning to give after records changed, if there is a cookbook and it
159
+ no longer matches them."""
160
+ try:
161
+ current = read_cookbook(project)
162
+ if current is None or current == expected_text(project):
163
+ return None
164
+ except UsefulViewsError:
165
+ return None # check reports a cookbook that cannot be read
166
+ return f"{COOKBOOK} no longer matches the recorded recipes. Update it with: useful-views cookbook"
167
+
168
+
169
+ def cookbook(*, root: str | Path | None = None, check: bool = False) -> CookbookResult:
170
+ """Write `cookbook.json`, the ledger of the recipes the project uses.
171
+
172
+ The cookbook lists every recorded recipe, in run order (upstream first),
173
+ with its description, inputs, outputs and steps, and the raw inputs the
174
+ recipes read. It is generated from the record in `datapackage.json`, so
175
+ to write it the project must be in sync: if `check` finds anything other
176
+ than stray files or a stale catalog or cookbook, nothing is written. Writing an unchanged cookbook
177
+ leaves the file untouched. Commit it to git, like a lock file.
178
+
179
+ `check=True` needs no data: it works on a fresh clone or in CI, where
180
+ raw/ and the views are not present. It compares `cookbook.json` with the
181
+ committed `datapackage.json`, and also reports recipes whose folder no
182
+ longer matches their record (edited, added or removed without a sync).
183
+
184
+ Args:
185
+ root: The project folder. By default it is found by walking up from
186
+ the current folder to `datapackage.json`.
187
+ check: Do not write; report whether `cookbook.json` matches the
188
+ project's recipes. Use it in CI.
189
+
190
+ Returns:
191
+ A `CookbookResult`. With `check=True`, its `up_to_date` is the
192
+ answer, and its `findings` list recipes whose record is out of date.
193
+
194
+ Raises:
195
+ CookbookError: When writing, the project is not in sync; its
196
+ `findings` say why.
197
+ ProjectError: cookbook.json exists but cannot be read.
198
+ ProjectNotFound: No project was found.
199
+ """
200
+ from .status import check as check_project, syncable
201
+
202
+ project = find_root(root)
203
+ with project_lock(project, enabled=not check):
204
+ findings = check_project(root=project)
205
+ data = load_datapackage(project)
206
+ book = build(data)
207
+ text = render(book)
208
+ path = project / COOKBOOK
209
+ current = read_cookbook(project)
210
+ recipes = tuple(r["name"] for r in book["recipes"])
211
+ if check:
212
+ stale = tuple(f for f in findings if _out_of_record(f))
213
+ return CookbookResult(path=path, written=False, up_to_date=current == text and not stale,
214
+ recipes=recipes, findings=stale)
215
+ blocking = [f for f in findings if f.kind not in NOT_BLOCKING]
216
+ if blocking:
217
+ listing = "\n".join(f" - {f.kind}: {f.message.splitlines()[0]}" for f in blocking)
218
+ if all(syncable(f) for f in blocking):
219
+ advice = "Run useful-views sync, then try again."
220
+ else:
221
+ advice = ("Fix what is listed (useful-views check shows each in full), run useful-views sync, "
222
+ "then try again.")
223
+ raise CookbookError(
224
+ "The cookbook describes a project in sync, but this one is not:\n" + listing + "\n" + advice,
225
+ findings=blocking,
226
+ )
227
+ if current != text:
228
+ write_atomic(path, text)
229
+ return CookbookResult(path=path, written=current != text, up_to_date=True, recipes=recipes)