useful-views 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- useful_views/__init__.py +53 -0
- useful_views/__main__.py +5 -0
- useful_views/cli.py +393 -0
- useful_views/cookbooks.py +229 -0
- useful_views/errors.py +151 -0
- useful_views/examples.py +221 -0
- useful_views/lock.py +147 -0
- useful_views/new.py +55 -0
- useful_views/paths.py +216 -0
- useful_views/project.py +216 -0
- useful_views/py.typed +0 -0
- useful_views/recipe.py +396 -0
- useful_views/record.py +427 -0
- useful_views/rulebook.py +94 -0
- useful_views/rules.py +412 -0
- useful_views/runner.py +570 -0
- useful_views/status.py +288 -0
- useful_views/syncing.py +519 -0
- useful_views/templates/01_first_step.py +12 -0
- useful_views/templates/AGENTS.md +21 -0
- useful_views/templates/project_README.md +118 -0
- useful_views/templates/recipe.toml +16 -0
- useful_views-0.1.0.dist-info/METADATA +568 -0
- useful_views-0.1.0.dist-info/RECORD +27 -0
- useful_views-0.1.0.dist-info/WHEEL +4 -0
- useful_views-0.1.0.dist-info/entry_points.txt +2 -0
- useful_views-0.1.0.dist-info/licenses/LICENSE +21 -0
useful_views/__init__.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""useful-views: a fixed project structure in which every derived table
|
|
2
|
+
records the recipe that made it."""
|
|
3
|
+
|
|
4
|
+
from importlib.metadata import PackageNotFoundError, version as _version
|
|
5
|
+
|
|
6
|
+
from .status import Finding, check
|
|
7
|
+
from .errors import (
|
|
8
|
+
OutputError,
|
|
9
|
+
ProjectError,
|
|
10
|
+
ProjectNotFound,
|
|
11
|
+
RecipeRefused,
|
|
12
|
+
StepFailed,
|
|
13
|
+
UsefulViewsError,
|
|
14
|
+
UsefulViewsWarning,
|
|
15
|
+
)
|
|
16
|
+
from .cookbooks import CookbookError, CookbookResult, cookbook
|
|
17
|
+
from .examples import ExampleError, example
|
|
18
|
+
from .new import new_recipe
|
|
19
|
+
from .project import init, root
|
|
20
|
+
from .rulebook import RULES, Rule
|
|
21
|
+
from .runner import RunResult, run
|
|
22
|
+
from .syncing import SyncResult, sync
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
__version__ = _version("useful-views")
|
|
26
|
+
except PackageNotFoundError: # running from a source tree that is not installed
|
|
27
|
+
__version__ = "0.0.0+unknown"
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"CookbookError",
|
|
31
|
+
"CookbookResult",
|
|
32
|
+
"ExampleError",
|
|
33
|
+
"Finding",
|
|
34
|
+
"OutputError",
|
|
35
|
+
"ProjectError",
|
|
36
|
+
"ProjectNotFound",
|
|
37
|
+
"RULES",
|
|
38
|
+
"RecipeRefused",
|
|
39
|
+
"Rule",
|
|
40
|
+
"RunResult",
|
|
41
|
+
"StepFailed",
|
|
42
|
+
"SyncResult",
|
|
43
|
+
"UsefulViewsError",
|
|
44
|
+
"UsefulViewsWarning",
|
|
45
|
+
"check",
|
|
46
|
+
"cookbook",
|
|
47
|
+
"example",
|
|
48
|
+
"init",
|
|
49
|
+
"new_recipe",
|
|
50
|
+
"root",
|
|
51
|
+
"run",
|
|
52
|
+
"sync",
|
|
53
|
+
]
|
useful_views/__main__.py
ADDED
useful_views/cli.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
"""The useful-views command: a thin wrapper over the Python functions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import contextlib
|
|
7
|
+
import io
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import signal
|
|
11
|
+
import sys
|
|
12
|
+
import threading
|
|
13
|
+
import warnings
|
|
14
|
+
|
|
15
|
+
from . import __version__, check, cookbook, example, init, new_recipe, run, sync
|
|
16
|
+
from .errors import UsefulViewsError, UsefulViewsWarning
|
|
17
|
+
from .rulebook import RULES, describe_rules
|
|
18
|
+
from .status import syncable
|
|
19
|
+
|
|
20
|
+
EXIT_OK = 0
|
|
21
|
+
EXIT_PROBLEMS = 1
|
|
22
|
+
EXIT_USAGE = 2
|
|
23
|
+
EXIT_FAILED = 3
|
|
24
|
+
|
|
25
|
+
EXIT_INTERRUPTED = 130 # 128 + SIGINT; a stop by another signal exits 128 + its number
|
|
26
|
+
|
|
27
|
+
EXIT_CODES = """exit codes:
|
|
28
|
+
0 success
|
|
29
|
+
1 check found problems, or cookbook --check found the cookbook out of date
|
|
30
|
+
2 usage error (unknown command or option, or a bad -C path)
|
|
31
|
+
3 refused or failed (the message says what to fix, and in which file)
|
|
32
|
+
130 interrupted (Ctrl-C)
|
|
33
|
+
143 terminated (SIGTERM, from a CI cancel or a tool's time limit; 129 SIGHUP; 149 Ctrl-Break)"""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class UsageError(Exception):
|
|
37
|
+
"""The command line itself is wrong."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class _Parser(argparse.ArgumentParser):
|
|
41
|
+
def __init__(self, *args, **kwargs):
|
|
42
|
+
kwargs.setdefault("allow_abbrev", False) # --js is an error, not --json
|
|
43
|
+
super().__init__(*args, **kwargs)
|
|
44
|
+
|
|
45
|
+
def error(self, message): # raise instead of exiting, so --json can still answer
|
|
46
|
+
raise UsageError(f"{self.prog}: {message}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _error(kind: str, message: str) -> dict:
|
|
50
|
+
"""An error as JSON, with the same keys as UsefulViewsError.to_dict()."""
|
|
51
|
+
return {"type": kind, "message": message, "recipe": None, "sync_result": None}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Terminated(KeyboardInterrupt):
|
|
55
|
+
"""The command was stopped by a signal other than Ctrl-C. Raised from the
|
|
56
|
+
signal handler, so the step is stopped and cleaned up as for Ctrl-C."""
|
|
57
|
+
|
|
58
|
+
def __init__(self, signum: int):
|
|
59
|
+
super().__init__()
|
|
60
|
+
self.signum = signum
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@contextlib.contextmanager
|
|
64
|
+
def _stop_signals():
|
|
65
|
+
"""Treat SIGTERM and SIGHUP (POSIX) and SIGBREAK (Windows) like Ctrl-C
|
|
66
|
+
while the command runs, so a cancelled CI job or a tool's timeout does
|
|
67
|
+
not leave a step running on its own. Only the command line does this;
|
|
68
|
+
the Python API leaves signal handling to its caller."""
|
|
69
|
+
if threading.current_thread() is not threading.main_thread():
|
|
70
|
+
yield
|
|
71
|
+
return
|
|
72
|
+
|
|
73
|
+
def stop(signum, frame):
|
|
74
|
+
raise Terminated(signum)
|
|
75
|
+
|
|
76
|
+
previous = {}
|
|
77
|
+
for name in ("SIGTERM", "SIGHUP", "SIGBREAK"):
|
|
78
|
+
sig = getattr(signal, name, None)
|
|
79
|
+
# Only replace the default: a signal ignored on purpose (nohup ignores
|
|
80
|
+
# SIGHUP, and steps inherit that) or handled by a caller stays so.
|
|
81
|
+
if sig is None or signal.getsignal(sig) is not signal.SIG_DFL:
|
|
82
|
+
continue
|
|
83
|
+
try:
|
|
84
|
+
previous[sig] = signal.signal(sig, stop)
|
|
85
|
+
except (OSError, ValueError):
|
|
86
|
+
pass
|
|
87
|
+
try:
|
|
88
|
+
yield
|
|
89
|
+
finally:
|
|
90
|
+
for sig, handler in previous.items():
|
|
91
|
+
try:
|
|
92
|
+
signal.signal(sig, handler)
|
|
93
|
+
except (OSError, ValueError):
|
|
94
|
+
pass
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _parser() -> argparse.ArgumentParser:
|
|
98
|
+
formatter = argparse.RawDescriptionHelpFormatter
|
|
99
|
+
parser = _Parser(
|
|
100
|
+
prog="useful-views",
|
|
101
|
+
description=(
|
|
102
|
+
"Turn raw data into documented derived tables (useful views). Each recipe in recipes/ "
|
|
103
|
+
"turns declared inputs into declared views by running numbered Python steps, and every "
|
|
104
|
+
"run is recorded in datapackage.json."
|
|
105
|
+
),
|
|
106
|
+
epilog=EXIT_CODES + "\n\nAlso runs as: python -m useful_views",
|
|
107
|
+
formatter_class=formatter,
|
|
108
|
+
)
|
|
109
|
+
parser.add_argument("--version", action="version", version=f"useful-views {__version__}")
|
|
110
|
+
project_help = "run as if started in PATH (like git -C), for example a project folder"
|
|
111
|
+
json_help = "print the result as JSON on standard output; other output goes to standard error"
|
|
112
|
+
parser.add_argument("-C", "--project", metavar="PATH", help=project_help)
|
|
113
|
+
parser.add_argument("--json", action="store_true", help=json_help)
|
|
114
|
+
# The same options after the command; SUPPRESS keeps a value given before it.
|
|
115
|
+
common = _Parser(add_help=False)
|
|
116
|
+
common.add_argument("-C", "--project", metavar="PATH", default=argparse.SUPPRESS, help=project_help)
|
|
117
|
+
common.add_argument("--json", action="store_true", default=argparse.SUPPRESS, help=json_help)
|
|
118
|
+
commands = parser.add_subparsers(dest="command", required=True, metavar="command", parser_class=_Parser)
|
|
119
|
+
|
|
120
|
+
def add(name: str, help: str, description: str) -> argparse.ArgumentParser:
|
|
121
|
+
return commands.add_parser(name, help=help, description=description, epilog=EXIT_CODES,
|
|
122
|
+
parents=[common], formatter_class=formatter)
|
|
123
|
+
|
|
124
|
+
p = add("init", "create the project structure (safe to run twice)",
|
|
125
|
+
"Create the project structure in PATH: raw/, recipes/, work/, useful-views/, a starter "
|
|
126
|
+
"README.md, AGENTS.md (for AI coding assistants), .gitignore and datapackage.json. Never "
|
|
127
|
+
"overwrites a file.")
|
|
128
|
+
p.add_argument("path", nargs="?", default=".", help="project folder (default: the current folder)")
|
|
129
|
+
p.add_argument("--force", action="store_true",
|
|
130
|
+
help="create the project even in your home folder or at the top of a drive")
|
|
131
|
+
|
|
132
|
+
p = add("example", "download a ready-to-run example project",
|
|
133
|
+
"Download an example project (raw data and recipes) from the useful-views-data repository, "
|
|
134
|
+
"check every file's SHA-256, and set it up like init. Then run: useful-views sync")
|
|
135
|
+
p.add_argument("name", help="the example to download, such as pwt")
|
|
136
|
+
p.add_argument("path", nargs="?", default=None, help="folder to create (default: <name>-example)")
|
|
137
|
+
|
|
138
|
+
p = add("new-recipe", "create recipes/<name>/ from a template",
|
|
139
|
+
"Create recipes/NAME/ with a template recipe.toml and a stub step, 01_first_step.py. Both say "
|
|
140
|
+
"TODO and are refused until you fill them in.")
|
|
141
|
+
p.add_argument("name", help="the recipe's name: lowercase letters, digits and underscores, like pwt_panel")
|
|
142
|
+
|
|
143
|
+
p = add("run", "run one recipe and record what it produced",
|
|
144
|
+
"Run one recipe. It is checked against the rules first (useful-views rules), then its views are "
|
|
145
|
+
"deleted and work/RECIPE/ emptied, each step runs in numeric order as its own process with this "
|
|
146
|
+
"Python and the project root as working directory, every declared view must be written, and the "
|
|
147
|
+
"run is recorded in datapackage.json and useful-views/README.md. Other recipes are not run.")
|
|
148
|
+
p.add_argument("recipe", help="the recipe's name, as in pwt_panel or recipes/pwt_panel/")
|
|
149
|
+
|
|
150
|
+
add("check", "report the state of the project",
|
|
151
|
+
"Report what is missing or out of place, every rule a recipe breaks (refused), and recipes that "
|
|
152
|
+
"sync would run. Changes nothing. Does not notice changes inside a step's code.")
|
|
153
|
+
|
|
154
|
+
p = add("sync", "make the views match the recipe folders, like uv sync",
|
|
155
|
+
"Make the views and datapackage.json match the recipe folders, like uv sync. Checks every recipe "
|
|
156
|
+
"first and changes nothing if any would be refused. Then removes the records and view files of "
|
|
157
|
+
"deleted recipes and outputs, and runs, upstream first, every recipe that is new, changed or "
|
|
158
|
+
"missing its views, plus everything downstream of those.")
|
|
159
|
+
p.add_argument("--all", action="store_true",
|
|
160
|
+
help="run every recipe again, upstream first (sync does not notice edits to a step's code)")
|
|
161
|
+
|
|
162
|
+
p = add("cookbook", "write cookbook.json, the ledger of the recipes used",
|
|
163
|
+
"Write cookbook.json at the project root: every recorded recipe, in run order, with its "
|
|
164
|
+
"description, inputs, outputs and steps, and the raw inputs the recipes read. Generated from "
|
|
165
|
+
"datapackage.json, so the project must be in sync to write it. Commit it, like a lock file.")
|
|
166
|
+
p.add_argument("--check", action="store_true",
|
|
167
|
+
help="do not write; exit 1 if cookbook.json is missing or out of date, or a recipe changed "
|
|
168
|
+
"without a sync. Needs no data, so it works in CI")
|
|
169
|
+
|
|
170
|
+
add("rules", "list the rules a recipe must follow",
|
|
171
|
+
"List the numbered rules. A refusal names the rule it broke.")
|
|
172
|
+
return parser
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def main(argv: list[str] | None = None) -> int:
|
|
176
|
+
for stream in (sys.stdout, sys.stderr):
|
|
177
|
+
try:
|
|
178
|
+
stream.reconfigure(errors="backslashreplace")
|
|
179
|
+
except (AttributeError, ValueError):
|
|
180
|
+
pass
|
|
181
|
+
argv = sys.argv[1:] if argv is None else list(argv)
|
|
182
|
+
wants_json = any(a == "--json" or a.startswith("--json=") for a in argv)
|
|
183
|
+
try:
|
|
184
|
+
if wants_json:
|
|
185
|
+
# --help and --version print and stop inside argparse; answer them
|
|
186
|
+
# in JSON too, so standard output always carries one document.
|
|
187
|
+
printed = io.StringIO()
|
|
188
|
+
try:
|
|
189
|
+
with contextlib.redirect_stdout(printed):
|
|
190
|
+
args = _parser().parse_args(argv)
|
|
191
|
+
except SystemExit as exc:
|
|
192
|
+
text = printed.getvalue()
|
|
193
|
+
print(text, end="", file=sys.stderr)
|
|
194
|
+
if text.startswith("usage:"):
|
|
195
|
+
data = {"ok": True, "help": text, "warnings": []}
|
|
196
|
+
else:
|
|
197
|
+
data = {"ok": True, "version": __version__, "warnings": []}
|
|
198
|
+
print(json.dumps(data, indent=2))
|
|
199
|
+
return exc.code if isinstance(exc.code, int) else EXIT_OK
|
|
200
|
+
else:
|
|
201
|
+
args = _parser().parse_args(argv)
|
|
202
|
+
except UsageError as exc:
|
|
203
|
+
return _fail(wants_json, str(exc), _error("UsageError", str(exc)), code=EXIT_USAGE)
|
|
204
|
+
try:
|
|
205
|
+
start = os.getcwd()
|
|
206
|
+
except OSError: # the folder this was started in has been deleted
|
|
207
|
+
start = None
|
|
208
|
+
if not args.project:
|
|
209
|
+
message = "the current folder no longer exists; cd into the project, or use -C <path>"
|
|
210
|
+
return _fail(args.json, message, _error("UsageError", message), code=EXIT_USAGE)
|
|
211
|
+
try:
|
|
212
|
+
if args.project:
|
|
213
|
+
try:
|
|
214
|
+
os.chdir(os.path.expanduser(args.project))
|
|
215
|
+
except OSError as exc:
|
|
216
|
+
message = f"cannot use -C {args.project}: {exc.strerror or exc}"
|
|
217
|
+
return _fail(args.json, message, _error("UsageError", message), code=EXIT_USAGE)
|
|
218
|
+
with _stop_signals():
|
|
219
|
+
return _main(args)
|
|
220
|
+
finally:
|
|
221
|
+
try:
|
|
222
|
+
if start is not None:
|
|
223
|
+
os.chdir(start)
|
|
224
|
+
except OSError:
|
|
225
|
+
pass # the folder it started in was deleted meanwhile
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _main(args: argparse.Namespace) -> int:
|
|
229
|
+
shown: list[str] = [] # every warning, for the JSON output
|
|
230
|
+
with warnings.catch_warnings():
|
|
231
|
+
original = warnings.showwarning
|
|
232
|
+
|
|
233
|
+
def show(message, category, filename, lineno, file=None, line=None):
|
|
234
|
+
if issubclass(category, UsefulViewsWarning):
|
|
235
|
+
print(f"warning: {message}", file=sys.stderr)
|
|
236
|
+
if str(message) not in shown:
|
|
237
|
+
shown.append(str(message))
|
|
238
|
+
else:
|
|
239
|
+
original(message, category, filename, lineno, file, line)
|
|
240
|
+
|
|
241
|
+
# Warnings are part of the command's output (and of its JSON), so show
|
|
242
|
+
# them whatever filters the environment sets.
|
|
243
|
+
warnings.simplefilter("always", UsefulViewsWarning)
|
|
244
|
+
warnings.showwarning = show
|
|
245
|
+
out = sys.stdout
|
|
246
|
+
try:
|
|
247
|
+
if args.json:
|
|
248
|
+
# Step output and progress go to standard error; standard
|
|
249
|
+
# output carries only the JSON.
|
|
250
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
251
|
+
code, data = _dispatch(args, shown)
|
|
252
|
+
data["warnings"] = list(dict.fromkeys([*data.get("warnings", []), *shown]))
|
|
253
|
+
print(json.dumps(data, indent=2), file=out)
|
|
254
|
+
return code
|
|
255
|
+
code, _ = _dispatch(args, shown)
|
|
256
|
+
return code
|
|
257
|
+
except UsefulViewsError as exc:
|
|
258
|
+
return _fail(args.json, str(exc), exc.to_dict(), out, shown=shown)
|
|
259
|
+
except KeyboardInterrupt as exc:
|
|
260
|
+
if isinstance(exc, Terminated):
|
|
261
|
+
name = signal.Signals(exc.signum).name
|
|
262
|
+
message, code = f"terminated ({name})", 128 + exc.signum
|
|
263
|
+
else:
|
|
264
|
+
message, code = "interrupted", EXIT_INTERRUPTED
|
|
265
|
+
note = str(exc) # what sync had done, when it was sync that stopped
|
|
266
|
+
if note:
|
|
267
|
+
message = f"{message}\n\n{note}"
|
|
268
|
+
error = _error("Interrupted", message)
|
|
269
|
+
result = getattr(exc, "sync_result", None)
|
|
270
|
+
error["sync_result"] = result.to_dict() if result is not None else None
|
|
271
|
+
return _fail(args.json, message, error, out, code=code, shown=shown)
|
|
272
|
+
except Exception as exc: # an unexpected failure: still readable, still exit 3
|
|
273
|
+
message = f"{type(exc).__name__}: {exc}"
|
|
274
|
+
return _fail(args.json, message, _error(type(exc).__name__, str(exc)), out, shown=shown)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _fail(as_json: bool, message: str, error: dict, out=None, *, code: int = EXIT_FAILED,
|
|
278
|
+
shown: list[str] | None = None) -> int:
|
|
279
|
+
print(f"error: {message}", file=sys.stderr)
|
|
280
|
+
if as_json:
|
|
281
|
+
# ASCII-escaped, so the JSON stays valid on any console encoding.
|
|
282
|
+
document = {"ok": False, "error": error, "warnings": list(shown or [])}
|
|
283
|
+
print(json.dumps(document, indent=2), file=out or sys.stdout)
|
|
284
|
+
return code
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _dispatch(args: argparse.Namespace, shown: list[str]) -> tuple[int, dict]:
|
|
288
|
+
if args.command == "init":
|
|
289
|
+
folder = init(args.path, force=args.force)
|
|
290
|
+
print(f"useful-views project ready at {folder}")
|
|
291
|
+
return EXIT_OK, {"ok": True, "project": str(folder)}
|
|
292
|
+
if args.command == "example":
|
|
293
|
+
folder = example(args.name, args.path)
|
|
294
|
+
print(f'Example {args.name!r} ready at {folder}. Make its views with: useful-views -C "{folder}" sync')
|
|
295
|
+
return EXIT_OK, {"ok": True, "project": str(folder)}
|
|
296
|
+
if args.command == "new-recipe":
|
|
297
|
+
folder = new_recipe(args.name)
|
|
298
|
+
print(f"Created {folder}. Edit its recipe.toml and 01_first_step.py, then run: useful-views run {args.name}")
|
|
299
|
+
return EXIT_OK, {"ok": True, "recipe": args.name, "folder": str(folder)}
|
|
300
|
+
if args.command == "run":
|
|
301
|
+
result = run(args.recipe)
|
|
302
|
+
print(f"Ran recipes/{result.recipe}. Wrote {', '.join(result.outputs)}.")
|
|
303
|
+
print("Updated datapackage.json and useful-views/README.md.")
|
|
304
|
+
return EXIT_OK, {"ok": True, **result.to_dict()}
|
|
305
|
+
if args.command == "check":
|
|
306
|
+
findings = check()
|
|
307
|
+
data = {"ok": not findings, "findings": [f.to_dict() for f in findings]}
|
|
308
|
+
if not findings:
|
|
309
|
+
if shown:
|
|
310
|
+
print(f"No problems found ({len(shown)} warning{'s' if len(shown) > 1 else ''} above).")
|
|
311
|
+
else:
|
|
312
|
+
print("No problems found.")
|
|
313
|
+
return EXIT_OK, data
|
|
314
|
+
for finding in findings:
|
|
315
|
+
first, *rest = finding.message.split("\n")
|
|
316
|
+
print(f"{finding.kind}: {first}")
|
|
317
|
+
for line in rest:
|
|
318
|
+
print(f" {line}" if line else "")
|
|
319
|
+
for_sync = [f for f in findings if syncable(f)]
|
|
320
|
+
others = [f for f in findings if not syncable(f) and f.kind not in ("stale-cookbook", "stray-file")]
|
|
321
|
+
if for_sync and not others:
|
|
322
|
+
then = ", then useful-views cookbook" if any(f.kind == "stale-cookbook" for f in findings) else ""
|
|
323
|
+
print(f"\nTo make or update everything that is missing or changed, upstream first, run: "
|
|
324
|
+
f"useful-views sync{then}")
|
|
325
|
+
elif for_sync:
|
|
326
|
+
print("\nFix the other findings first; then useful-views sync makes or updates the rest, upstream first.")
|
|
327
|
+
return EXIT_PROBLEMS, data
|
|
328
|
+
if args.command == "sync":
|
|
329
|
+
result = sync(all=args.all)
|
|
330
|
+
if result.removed:
|
|
331
|
+
print("Removed the records of deleted recipes: " + ", ".join(f"recipes/{n}" for n in result.removed) + ".")
|
|
332
|
+
if result.dropped:
|
|
333
|
+
makers = _makers()
|
|
334
|
+
moved = [f"{v} (now made by recipes/{makers[v]})" for v in result.dropped if v in makers]
|
|
335
|
+
gone = [v for v in result.dropped if v not in makers]
|
|
336
|
+
if gone:
|
|
337
|
+
print("Removed views no recipe declares any more: " + ", ".join(gone) + ".")
|
|
338
|
+
if moved:
|
|
339
|
+
print("Moved to another recipe: " + ", ".join(moved) + ".")
|
|
340
|
+
if result.deleted:
|
|
341
|
+
print("Deleted: " + ", ".join(result.deleted) + ".")
|
|
342
|
+
if result.ran:
|
|
343
|
+
print("Ran, in order: " + ", ".join(f"recipes/{n}" for n in result.ran) + ".")
|
|
344
|
+
if not (result.removed or result.dropped or result.ran):
|
|
345
|
+
print("Everything is in sync: nothing to remove or run.")
|
|
346
|
+
else:
|
|
347
|
+
print("Updated datapackage.json and useful-views/README.md.")
|
|
348
|
+
return EXIT_OK, {"ok": True, **result.to_dict()}
|
|
349
|
+
if args.command == "cookbook":
|
|
350
|
+
result = cookbook(check=args.check)
|
|
351
|
+
data = {"ok": result.up_to_date, **result.to_dict()}
|
|
352
|
+
if args.check:
|
|
353
|
+
if result.up_to_date:
|
|
354
|
+
print("cookbook.json is up to date.")
|
|
355
|
+
return EXIT_OK, data
|
|
356
|
+
if result.findings:
|
|
357
|
+
print("The recipe folders no longer match their records in datapackage.json, so cookbook.json "
|
|
358
|
+
"is out of date:")
|
|
359
|
+
for finding in result.findings:
|
|
360
|
+
print(f" {finding.kind}: {finding.message.splitlines()[0]}")
|
|
361
|
+
if all(syncable(f) for f in result.findings):
|
|
362
|
+
print("Run useful-views sync, then useful-views cookbook, and commit the results.")
|
|
363
|
+
else:
|
|
364
|
+
print("Fix what is listed (useful-views check shows each in full), then run useful-views sync "
|
|
365
|
+
"and useful-views cookbook, and commit the results.")
|
|
366
|
+
else:
|
|
367
|
+
print("cookbook.json is missing or out of date. Update it with: useful-views cookbook")
|
|
368
|
+
return EXIT_PROBLEMS, data
|
|
369
|
+
if result.written:
|
|
370
|
+
print(f"Wrote cookbook.json: {len(result.recipes)} recipe(s), in run order.")
|
|
371
|
+
else:
|
|
372
|
+
print("cookbook.json is already up to date.")
|
|
373
|
+
return EXIT_OK, data
|
|
374
|
+
if args.command == "rules":
|
|
375
|
+
print(describe_rules())
|
|
376
|
+
return EXIT_OK, {"ok": True, "rules": [r._asdict() for r in RULES.values()]}
|
|
377
|
+
raise AssertionError(args.command)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _makers() -> dict[str, str]:
|
|
381
|
+
"""Each recorded view and the recipe that made it, after a sync."""
|
|
382
|
+
from .project import find_root
|
|
383
|
+
from .record import load_datapackage
|
|
384
|
+
|
|
385
|
+
try:
|
|
386
|
+
data = load_datapackage(find_root())
|
|
387
|
+
except UsefulViewsError:
|
|
388
|
+
return {}
|
|
389
|
+
return {out: r["name"] for r in data["recipes"] for out in r["outputs"]}
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
if __name__ == "__main__":
|
|
393
|
+
sys.exit(main())
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""cookbook(): write cookbook.json, the ledger of the recipes a project uses.
|
|
2
|
+
|
|
3
|
+
Like `uv lock` writing `uv.lock`, the cookbook is produced by a command and
|
|
4
|
+
never edited by hand. Version 1 records each recipe and its steps, in run
|
|
5
|
+
order, and the raw inputs they read. Later versions are planned to add the
|
|
6
|
+
SHA-256 of every file and the environment the steps ran in.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from .errors import ProjectError, UsefulViewsError
|
|
16
|
+
from .paths import RAW
|
|
17
|
+
from .project import find_root
|
|
18
|
+
from .lock import project_lock
|
|
19
|
+
from .record import load_datapackage, write_atomic
|
|
20
|
+
|
|
21
|
+
COOKBOOK = "cookbook.json"
|
|
22
|
+
FORMAT_VERSION = 1
|
|
23
|
+
|
|
24
|
+
# check() findings that do not stop a cookbook from being written.
|
|
25
|
+
NOT_BLOCKING = {"stray-file", "stale-cookbook", "stale-catalog"}
|
|
26
|
+
|
|
27
|
+
# With check=True, only findings that mean datapackage.json no longer matches
|
|
28
|
+
# the recipe folders count; they need no data on disk, so a CI job with only
|
|
29
|
+
# the git checkout (no raw/, no views) gets a real answer.
|
|
30
|
+
RECORD_KINDS = {"unreadable-recipe", "refused", "never-run", "missing-recipe-folder"}
|
|
31
|
+
RECORD_REASONS = {"changed", "outputs-changed", "outputs-dropped"}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _out_of_record(finding) -> bool:
|
|
35
|
+
return finding.kind in RECORD_KINDS or (finding.kind == "out-of-date" and finding.reason in RECORD_REASONS)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class CookbookError(UsefulViewsError):
|
|
39
|
+
"""The cookbook cannot be written because the project is not in sync.
|
|
40
|
+
|
|
41
|
+
Attributes:
|
|
42
|
+
findings: The `check` findings that must be resolved first.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(self, message: str, *, findings: list | None = None):
|
|
46
|
+
super().__init__(message)
|
|
47
|
+
self.findings = list(findings or [])
|
|
48
|
+
|
|
49
|
+
def to_dict(self) -> dict:
|
|
50
|
+
data = super().to_dict()
|
|
51
|
+
data["findings"] = [f.to_dict() for f in self.findings]
|
|
52
|
+
return data
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class CookbookResult:
|
|
57
|
+
"""What `cookbook` did, or found with `check=True`.
|
|
58
|
+
|
|
59
|
+
Attributes:
|
|
60
|
+
path: The cookbook file, `cookbook.json` at the project root.
|
|
61
|
+
written: True if the file was created or changed.
|
|
62
|
+
up_to_date: True if the file on disk matches the project's recipes
|
|
63
|
+
(after writing, always True; with `check=True`, the answer).
|
|
64
|
+
recipes: The recipes in the cookbook, in run order.
|
|
65
|
+
findings: With `check=True`, the `check` findings that mean the
|
|
66
|
+
recorded recipes no longer match the recipe folders (so the
|
|
67
|
+
cookbook cannot be up to date); otherwise empty.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
path: Path
|
|
71
|
+
written: bool
|
|
72
|
+
up_to_date: bool
|
|
73
|
+
recipes: tuple[str, ...] = field(default=())
|
|
74
|
+
findings: tuple = field(default=())
|
|
75
|
+
|
|
76
|
+
def to_dict(self) -> dict:
|
|
77
|
+
return {
|
|
78
|
+
"path": str(self.path),
|
|
79
|
+
"written": self.written,
|
|
80
|
+
"up_to_date": self.up_to_date,
|
|
81
|
+
"recipes": list(self.recipes),
|
|
82
|
+
"findings": [f.to_dict() for f in self.findings],
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _run_order(records: dict[str, dict]) -> list[str]:
|
|
87
|
+
"""Recipe names, upstream first; recipes that do not depend on each other
|
|
88
|
+
in name order."""
|
|
89
|
+
makers = {out: name for name, r in records.items() for out in r["outputs"]}
|
|
90
|
+
upstream = {
|
|
91
|
+
name: sorted({makers[i] for i in r["inputs"] if i in makers} - {name}) for name, r in records.items()
|
|
92
|
+
}
|
|
93
|
+
order: list[str] = []
|
|
94
|
+
done: set[str] = set()
|
|
95
|
+
|
|
96
|
+
def visit(name: str, seen: tuple[str, ...] = ()) -> None:
|
|
97
|
+
if name in done or name in seen:
|
|
98
|
+
return
|
|
99
|
+
for up in upstream[name]:
|
|
100
|
+
visit(up, (*seen, name))
|
|
101
|
+
done.add(name)
|
|
102
|
+
order.append(name)
|
|
103
|
+
|
|
104
|
+
for name in sorted(records):
|
|
105
|
+
visit(name)
|
|
106
|
+
return order
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def build(data: dict) -> dict:
|
|
110
|
+
"""The cookbook for a project's recorded recipes (datapackage.json)."""
|
|
111
|
+
records = {r["name"]: r for r in data["recipes"]}
|
|
112
|
+
order = _run_order(records)
|
|
113
|
+
raw = sorted({p for r in records.values() for p in r["inputs"] if p.startswith(f"{RAW}/")})
|
|
114
|
+
return {
|
|
115
|
+
"version": FORMAT_VERSION,
|
|
116
|
+
"generator": "useful-views",
|
|
117
|
+
"project": data.get("name"),
|
|
118
|
+
"raw_inputs": raw,
|
|
119
|
+
"recipes": [records[name] for name in order],
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def render(book: dict) -> str:
|
|
124
|
+
return json.dumps(book, indent=2, ensure_ascii=False) + "\n"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def expected_text(project: Path) -> str:
|
|
128
|
+
"""What cookbook.json should contain for the project's current records."""
|
|
129
|
+
return render(build(load_datapackage(project)))
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def read_cookbook(project: Path) -> str | None:
|
|
133
|
+
"""The text of cookbook.json with LF line endings, or None if there is none.
|
|
134
|
+
|
|
135
|
+
A file that is not UTF-8 text reads as "", which never matches, so it
|
|
136
|
+
counts as out of date and `cookbook` rewrites it. A file that cannot be
|
|
137
|
+
read at all raises ProjectError.
|
|
138
|
+
"""
|
|
139
|
+
path = project / COOKBOOK
|
|
140
|
+
if path.is_dir(): # tested first: Windows reports a folder as "permission denied"
|
|
141
|
+
raise ProjectError(f"{COOKBOOK} is a folder. Delete it, then run useful-views cookbook.")
|
|
142
|
+
try:
|
|
143
|
+
data = path.read_bytes()
|
|
144
|
+
except FileNotFoundError:
|
|
145
|
+
return None
|
|
146
|
+
except OSError as exc:
|
|
147
|
+
raise ProjectError(
|
|
148
|
+
f"Could not read {COOKBOOK}: {exc.strerror or exc}. Check its permissions, or delete it and run "
|
|
149
|
+
"useful-views cookbook."
|
|
150
|
+
) from exc
|
|
151
|
+
try:
|
|
152
|
+
return data.decode("utf-8").replace("\r\n", "\n")
|
|
153
|
+
except UnicodeDecodeError:
|
|
154
|
+
return ""
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def stale_warning(project: Path) -> str | None:
|
|
158
|
+
"""A warning to give after records changed, if there is a cookbook and it
|
|
159
|
+
no longer matches them."""
|
|
160
|
+
try:
|
|
161
|
+
current = read_cookbook(project)
|
|
162
|
+
if current is None or current == expected_text(project):
|
|
163
|
+
return None
|
|
164
|
+
except UsefulViewsError:
|
|
165
|
+
return None # check reports a cookbook that cannot be read
|
|
166
|
+
return f"{COOKBOOK} no longer matches the recorded recipes. Update it with: useful-views cookbook"
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def cookbook(*, root: str | Path | None = None, check: bool = False) -> CookbookResult:
|
|
170
|
+
"""Write `cookbook.json`, the ledger of the recipes the project uses.
|
|
171
|
+
|
|
172
|
+
The cookbook lists every recorded recipe, in run order (upstream first),
|
|
173
|
+
with its description, inputs, outputs and steps, and the raw inputs the
|
|
174
|
+
recipes read. It is generated from the record in `datapackage.json`, so
|
|
175
|
+
to write it the project must be in sync: if `check` finds anything other
|
|
176
|
+
than stray files or a stale catalog or cookbook, nothing is written. Writing an unchanged cookbook
|
|
177
|
+
leaves the file untouched. Commit it to git, like a lock file.
|
|
178
|
+
|
|
179
|
+
`check=True` needs no data: it works on a fresh clone or in CI, where
|
|
180
|
+
raw/ and the views are not present. It compares `cookbook.json` with the
|
|
181
|
+
committed `datapackage.json`, and also reports recipes whose folder no
|
|
182
|
+
longer matches their record (edited, added or removed without a sync).
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
root: The project folder. By default it is found by walking up from
|
|
186
|
+
the current folder to `datapackage.json`.
|
|
187
|
+
check: Do not write; report whether `cookbook.json` matches the
|
|
188
|
+
project's recipes. Use it in CI.
|
|
189
|
+
|
|
190
|
+
Returns:
|
|
191
|
+
A `CookbookResult`. With `check=True`, its `up_to_date` is the
|
|
192
|
+
answer, and its `findings` list recipes whose record is out of date.
|
|
193
|
+
|
|
194
|
+
Raises:
|
|
195
|
+
CookbookError: When writing, the project is not in sync; its
|
|
196
|
+
`findings` say why.
|
|
197
|
+
ProjectError: cookbook.json exists but cannot be read.
|
|
198
|
+
ProjectNotFound: No project was found.
|
|
199
|
+
"""
|
|
200
|
+
from .status import check as check_project, syncable
|
|
201
|
+
|
|
202
|
+
project = find_root(root)
|
|
203
|
+
with project_lock(project, enabled=not check):
|
|
204
|
+
findings = check_project(root=project)
|
|
205
|
+
data = load_datapackage(project)
|
|
206
|
+
book = build(data)
|
|
207
|
+
text = render(book)
|
|
208
|
+
path = project / COOKBOOK
|
|
209
|
+
current = read_cookbook(project)
|
|
210
|
+
recipes = tuple(r["name"] for r in book["recipes"])
|
|
211
|
+
if check:
|
|
212
|
+
stale = tuple(f for f in findings if _out_of_record(f))
|
|
213
|
+
return CookbookResult(path=path, written=False, up_to_date=current == text and not stale,
|
|
214
|
+
recipes=recipes, findings=stale)
|
|
215
|
+
blocking = [f for f in findings if f.kind not in NOT_BLOCKING]
|
|
216
|
+
if blocking:
|
|
217
|
+
listing = "\n".join(f" - {f.kind}: {f.message.splitlines()[0]}" for f in blocking)
|
|
218
|
+
if all(syncable(f) for f in blocking):
|
|
219
|
+
advice = "Run useful-views sync, then try again."
|
|
220
|
+
else:
|
|
221
|
+
advice = ("Fix what is listed (useful-views check shows each in full), run useful-views sync, "
|
|
222
|
+
"then try again.")
|
|
223
|
+
raise CookbookError(
|
|
224
|
+
"The cookbook describes a project in sync, but this one is not:\n" + listing + "\n" + advice,
|
|
225
|
+
findings=blocking,
|
|
226
|
+
)
|
|
227
|
+
if current != text:
|
|
228
|
+
write_atomic(path, text)
|
|
229
|
+
return CookbookResult(path=path, written=current != text, up_to_date=True, recipes=recipes)
|