framework-py 0.1.0.dev3__py3-none-any.whl → 0.1.0.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +5 -0
- framework/__main__.py +64 -5
- framework/_agents_md.py +210 -0
- framework/_assets/adopt/rules.json +161 -0
- framework/_assets/features/jupyter.json +39 -0
- framework/_assets/features/quarto.json +18 -0
- framework/_assets/features/quarto_manuscript.json +35 -0
- framework/_assets/features/quarto_slides.json +29 -0
- framework/_assets/features/renv.json +20 -0
- framework/_assets/features/targets.json +34 -0
- framework/_assets/features/uv.json +17 -0
- framework/_assets/schema/layer.v1.json +45 -7
- framework/_assets/schema/layers.v1.json +49 -7
- framework/_assets/schema/spec.v1.json +38 -7
- framework/_assets/sets/packages/tidyverse-analysis.json +42 -0
- framework/_assets/sets/skills/framework.json +25 -0
- framework/_assets/sets/skills/ggplot2.json +26 -0
- framework/_assets/workflows/jupyter.json +14 -0
- framework/_assets/workflows/quarto.json +14 -0
- framework/_assets/workflows/scripts.json +18 -0
- framework/_defaults_cache.py +47 -0
- framework/_paths.py +19 -9
- framework/adopt.py +365 -0
- framework/ai.py +13 -20
- framework/cloud.py +128 -13
- framework/create.py +395 -18
- framework/features.py +195 -0
- framework/hooks.py +69 -7
- framework/notebooks.py +9 -1
- framework/outputs.py +6 -1
- framework/quarto.py +77 -25
- framework/scaffold.py +50 -3
- framework/spec/check.py +31 -7
- framework/spec/merge.py +1 -0
- framework/spec/resolve.py +80 -6
- framework/templates/defaults.yml +0 -1
- {framework_py-0.1.0.dev3.dist-info → framework_py-0.1.0.dev4.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev3.dist-info → framework_py-0.1.0.dev4.dist-info}/RECORD +40 -22
- {framework_py-0.1.0.dev3.dist-info → framework_py-0.1.0.dev4.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev3.dist-info → framework_py-0.1.0.dev4.dist-info}/licenses/LICENSE +0 -0
framework/features.py
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Framework's tools and its bundled skill and package sets.
|
|
2
|
+
|
|
3
|
+
The definitions are canonical in the R package (``inst/features/`` and
|
|
4
|
+
``inst/sets/``) and copied here by ``scripts/sync_assets.py``. Each holds a
|
|
5
|
+
layer in the spec layer format, so an offline ``new()`` and framework.pub build
|
|
6
|
+
the same projects. A tool's ``layer`` applies first, then its layer for the
|
|
7
|
+
project's language. Workflows (``inst/workflows/``) are defined the same way.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from collections.abc import Iterable, Mapping
|
|
14
|
+
from functools import cache
|
|
15
|
+
from importlib import resources
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from framework._log import inform
|
|
21
|
+
from framework.errors import FrameworkError
|
|
22
|
+
|
|
23
|
+
LANGUAGE = "python"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@cache
|
|
27
|
+
def _definitions(folder: str) -> dict[str, dict[str, Any]]:
|
|
28
|
+
root = resources.files("framework").joinpath(f"_assets/{folder}")
|
|
29
|
+
found: dict[str, dict[str, Any]] = {}
|
|
30
|
+
for entry in sorted(root.iterdir(), key=lambda e: e.name):
|
|
31
|
+
if entry.name.endswith(".json"):
|
|
32
|
+
definition = json.loads(entry.read_text(encoding="utf-8"))
|
|
33
|
+
found[definition["id"]] = definition
|
|
34
|
+
return found
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _tools() -> dict[str, dict[str, Any]]:
|
|
38
|
+
return _definitions("features")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _sets(kind: str) -> dict[str, dict[str, Any]]:
|
|
42
|
+
return _definitions(f"sets/{kind}")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _for_python(definition: Mapping[str, Any]) -> bool:
|
|
46
|
+
return LANGUAGE in (definition.get("language") or [])
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _workflows() -> dict[str, dict[str, Any]]:
|
|
50
|
+
return _definitions("workflows")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def workflow_layers(workflow: str) -> list[dict[str, Any]]:
|
|
54
|
+
"""Layer envelopes for a workflow: its required tools, then its own layer, as in R.
|
|
55
|
+
|
|
56
|
+
Raises:
|
|
57
|
+
FrameworkError: For an unknown workflow or one that isn't for Python.
|
|
58
|
+
"""
|
|
59
|
+
definitions = _workflows()
|
|
60
|
+
definition = definitions.get(workflow)
|
|
61
|
+
if definition is None:
|
|
62
|
+
available = ", ".join(i for i, d in definitions.items() if _for_python(d))
|
|
63
|
+
raise FrameworkError(f"Unknown workflow: {workflow}. Available: {available}")
|
|
64
|
+
if not _for_python(definition):
|
|
65
|
+
raise FrameworkError(f"The {definition['name']} workflow isn't available for Python.")
|
|
66
|
+
entries = definition_layers(list(definition.get("requires") or []))
|
|
67
|
+
entries.append(
|
|
68
|
+
{"kind": "facet", "key": f"framework/workflow-{workflow}", "layer": definition["layer"]}
|
|
69
|
+
)
|
|
70
|
+
extra = (definition.get("language_layers") or {}).get(LANGUAGE)
|
|
71
|
+
if extra:
|
|
72
|
+
key = f"framework/workflow-{workflow}-{LANGUAGE}"
|
|
73
|
+
entries.append({"kind": "facet", "key": key, "layer": extra})
|
|
74
|
+
return entries
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def tools_available() -> pd.DataFrame:
|
|
78
|
+
"""Framework's tools for Python projects.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
One row per tool with ``id``, ``name``, ``description``, and ``requires`` (a list of
|
|
82
|
+
tool ids).
|
|
83
|
+
"""
|
|
84
|
+
rows = [
|
|
85
|
+
{
|
|
86
|
+
"id": d["id"],
|
|
87
|
+
"name": d["name"],
|
|
88
|
+
"description": d["description"],
|
|
89
|
+
"requires": list(d.get("requires") or []),
|
|
90
|
+
}
|
|
91
|
+
for d in _tools().values()
|
|
92
|
+
if _for_python(d)
|
|
93
|
+
]
|
|
94
|
+
return pd.DataFrame(rows, columns=["id", "name", "description", "requires"])
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def resolve_tools(tools: Iterable[str], off: Iterable[str] = ()) -> list[str]:
|
|
98
|
+
"""The tools to turn on: those asked for plus what they require, in order.
|
|
99
|
+
|
|
100
|
+
A required tool is added with a message, as R and framework.pub's launch
|
|
101
|
+
page do.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
FrameworkError: For an unknown tool, a tool that isn't for Python, a conflict, or a
|
|
105
|
+
required tool that was turned off.
|
|
106
|
+
"""
|
|
107
|
+
definitions = _tools()
|
|
108
|
+
asked = list(dict.fromkeys(tools))
|
|
109
|
+
unknown = [t for t in asked if t not in definitions]
|
|
110
|
+
if unknown:
|
|
111
|
+
available = ", ".join(t for t, d in definitions.items() if _for_python(d))
|
|
112
|
+
raise FrameworkError(f"Unknown tool(s): {', '.join(unknown)}. Available: {available}")
|
|
113
|
+
turned_off = set(off)
|
|
114
|
+
wanted: list[str] = []
|
|
115
|
+
|
|
116
|
+
def add(tool: str, needed_by: str | None) -> None:
|
|
117
|
+
if tool in wanted:
|
|
118
|
+
return
|
|
119
|
+
if needed_by is not None:
|
|
120
|
+
if tool in turned_off:
|
|
121
|
+
raise FrameworkError(f"The {needed_by} tool needs {tool}, which is turned off.")
|
|
122
|
+
if tool not in asked:
|
|
123
|
+
inform(f"Also turning on {tool}, which {needed_by} needs")
|
|
124
|
+
for required in definitions[tool].get("requires") or []:
|
|
125
|
+
add(required, tool)
|
|
126
|
+
wanted.append(tool)
|
|
127
|
+
|
|
128
|
+
for tool in asked:
|
|
129
|
+
add(tool, None)
|
|
130
|
+
for tool in wanted:
|
|
131
|
+
definition = definitions[tool]
|
|
132
|
+
if not _for_python(definition):
|
|
133
|
+
name = definition["name"]
|
|
134
|
+
raise FrameworkError(f"The {name} tool isn't available for Python projects.")
|
|
135
|
+
clash = [t for t in definition.get("conflicts") or [] if t in wanted]
|
|
136
|
+
if clash:
|
|
137
|
+
raise FrameworkError(f"The {tool} tool can't be used with {', '.join(clash)}.")
|
|
138
|
+
return wanted
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _pick(ids: Iterable[str], kind: str, label: str) -> list[dict[str, Any]]:
|
|
142
|
+
sets = _sets(kind)
|
|
143
|
+
entries = []
|
|
144
|
+
for handle in ids:
|
|
145
|
+
# Framework's handles work offline too
|
|
146
|
+
set_id = handle.removeprefix("framework/")
|
|
147
|
+
definition = sets.get(set_id)
|
|
148
|
+
if definition is None:
|
|
149
|
+
available = ", ".join(i for i, d in sets.items() if _for_python(d))
|
|
150
|
+
raise FrameworkError(f"Unknown {label}: {set_id}. Available: {available}")
|
|
151
|
+
if not _for_python(definition):
|
|
152
|
+
raise FrameworkError(f"The {definition['name']} {label} isn't available for Python.")
|
|
153
|
+
prefix = "packages-" if kind == "packages" else "skills-"
|
|
154
|
+
entries.append(
|
|
155
|
+
{"kind": "facet", "key": f"framework/{prefix}{set_id}", "layer": definition["layer"]}
|
|
156
|
+
)
|
|
157
|
+
return entries
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def definition_layers(
|
|
161
|
+
tools: Mapping[str, bool] | Iterable[str] = (),
|
|
162
|
+
package_sets: Iterable[str] = (),
|
|
163
|
+
skill_sets: Iterable[str] = (),
|
|
164
|
+
) -> list[dict[str, Any]]:
|
|
165
|
+
"""Layer envelopes for tools and sets, in framework.pub's order.
|
|
166
|
+
|
|
167
|
+
Tools come first, then package sets, then skill sets. ``tools`` is a list of tool ids or
|
|
168
|
+
a map of tool id to true or false. Tools set to false are turned off by one last layer,
|
|
169
|
+
``framework/tools-off``, which overrides the project's defaults.
|
|
170
|
+
"""
|
|
171
|
+
choices = dict(tools) if isinstance(tools, Mapping) else dict.fromkeys(tools, True)
|
|
172
|
+
for tool, on in choices.items():
|
|
173
|
+
if not isinstance(on, bool):
|
|
174
|
+
raise FrameworkError(f"tools.{tool} must be true or false")
|
|
175
|
+
off = [t for t, on in choices.items() if on is False]
|
|
176
|
+
unknown = [t for t in off if t not in _tools()]
|
|
177
|
+
if unknown:
|
|
178
|
+
raise FrameworkError(f"Unknown tool(s): {', '.join(unknown)}")
|
|
179
|
+
entries: list[dict[str, Any]] = []
|
|
180
|
+
for tool in resolve_tools([t for t, on in choices.items() if on], off):
|
|
181
|
+
definition = _tools()[tool]
|
|
182
|
+
key = f"framework/tool-{tool}"
|
|
183
|
+
entries.append({"kind": "facet", "key": key, "layer": definition["layer"]})
|
|
184
|
+
extra = (definition.get("language_layers") or {}).get(LANGUAGE)
|
|
185
|
+
if extra:
|
|
186
|
+
entries.append(
|
|
187
|
+
{"kind": "facet", "key": f"framework/tool-{tool}-{LANGUAGE}", "layer": extra}
|
|
188
|
+
)
|
|
189
|
+
entries += _pick(package_sets, "packages", "package set")
|
|
190
|
+
entries += _pick(skill_sets, "skills", "skill set")
|
|
191
|
+
if off:
|
|
192
|
+
# After the tools and sets, as in R, so the project's defaults are overridden
|
|
193
|
+
layer = {"tools": dict.fromkeys(off, False)}
|
|
194
|
+
entries.append({"kind": "facet", "key": "framework/tools-off", "layer": layer})
|
|
195
|
+
return entries
|
framework/hooks.py
CHANGED
|
@@ -9,6 +9,8 @@ settings:
|
|
|
9
9
|
- ``check_sensitive_dirs`` (formerly ``warn_unignored_sensitive``): warn about
|
|
10
10
|
folders named like private, confidential, sensitive, cache, or scratch that
|
|
11
11
|
git does not ignore, and ask before committing when a terminal is attached.
|
|
12
|
+
- ``check_notebook_outputs``: block the commit when a staged ``.ipynb`` has
|
|
13
|
+
cell outputs or execution counts, which can hold printed data.
|
|
12
14
|
- ``ai_sync`` is gone: projects keep AI instructions in AGENTS.md and skills,
|
|
13
15
|
so there is nothing to regenerate. An ``ai_sync`` setting in an older
|
|
14
16
|
project is ignored.
|
|
@@ -17,6 +19,7 @@ settings:
|
|
|
17
19
|
from __future__ import annotations
|
|
18
20
|
|
|
19
21
|
import fnmatch
|
|
22
|
+
import json
|
|
20
23
|
import os
|
|
21
24
|
import shutil
|
|
22
25
|
import stat
|
|
@@ -33,11 +36,12 @@ from framework.errors import FrameworkError
|
|
|
33
36
|
from framework.project import find_settings_file
|
|
34
37
|
from framework.settings import _has_environment_sections, settings_read
|
|
35
38
|
|
|
36
|
-
HOOKS = ("data_security", "check_sensitive_dirs")
|
|
39
|
+
HOOKS = ("data_security", "check_sensitive_dirs", "check_notebook_outputs")
|
|
37
40
|
_ALIASES = {"warn_unignored_sensitive": "check_sensitive_dirs"}
|
|
38
41
|
_DESCRIPTIONS = {
|
|
39
42
|
"data_security": "Check for private data and secrets in git",
|
|
40
43
|
"check_sensitive_dirs": "Warn about unignored sensitive directories",
|
|
44
|
+
"check_notebook_outputs": "Block Jupyter notebooks committed with cell outputs",
|
|
41
45
|
}
|
|
42
46
|
# Headers of the shared hook and of R 1.x's generated hook.
|
|
43
47
|
_MARKERS = ("Framework pre-commit hook", "Framework Git Hooks")
|
|
@@ -224,8 +228,9 @@ def git_hooks_enable(
|
|
|
224
228
|
"""Turn on one hook check in the settings and (re)install the hook.
|
|
225
229
|
|
|
226
230
|
Args:
|
|
227
|
-
hook_name: The check. One of "data_security"
|
|
228
|
-
name "warn_unignored_sensitive" is accepted
|
|
231
|
+
hook_name: The check. One of "data_security", "check_sensitive_dirs", or
|
|
232
|
+
"check_notebook_outputs". The older name "warn_unignored_sensitive" is accepted
|
|
233
|
+
for "check_sensitive_dirs".
|
|
229
234
|
settings_file: The project's settings file. Default: the settings.yml or config.yml
|
|
230
235
|
found from the current directory.
|
|
231
236
|
verbose: Print what was done.
|
|
@@ -254,10 +259,10 @@ def git_hooks_disable(
|
|
|
254
259
|
to false.
|
|
255
260
|
|
|
256
261
|
Args:
|
|
257
|
-
hook_name: The check. One of "data_security", "check_sensitive_dirs",
|
|
258
|
-
"ai_sync". The older name "warn_unignored_sensitive"
|
|
259
|
-
"check_sensitive_dirs", and turning that check off also turns off
|
|
260
|
-
setting.
|
|
262
|
+
hook_name: The check. One of "data_security", "check_sensitive_dirs",
|
|
263
|
+
"check_notebook_outputs", or "ai_sync". The older name "warn_unignored_sensitive"
|
|
264
|
+
is accepted for "check_sensitive_dirs", and turning that check off also turns off
|
|
265
|
+
the older setting.
|
|
261
266
|
settings_file: The project's settings file. Default: the settings.yml or config.yml
|
|
262
267
|
found from the current directory.
|
|
263
268
|
verbose: Print what was done.
|
|
@@ -380,6 +385,61 @@ def _check_sensitive_dirs(root: Path) -> bool:
|
|
|
380
385
|
return answer
|
|
381
386
|
|
|
382
387
|
|
|
388
|
+
def _has_outputs(text: str) -> bool:
|
|
389
|
+
try:
|
|
390
|
+
notebook = json.loads(text)
|
|
391
|
+
except ValueError:
|
|
392
|
+
return False
|
|
393
|
+
cells = notebook.get("cells") if isinstance(notebook, dict) else None
|
|
394
|
+
return any(
|
|
395
|
+
isinstance(cell, dict)
|
|
396
|
+
and cell.get("cell_type") == "code"
|
|
397
|
+
and (cell.get("outputs") or cell.get("execution_count") is not None)
|
|
398
|
+
for cell in cells or []
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _notebooks_with_outputs(root: Path) -> list[str]:
|
|
403
|
+
"""Staged notebooks whose staged version has outputs, as paths from the repository root."""
|
|
404
|
+
staged = _git(root, "diff", "--cached", "--name-only", "--diff-filter=ACMR", "-z")
|
|
405
|
+
found = []
|
|
406
|
+
for path in staged.stdout.split("\0"):
|
|
407
|
+
if path.lower().endswith(".ipynb"):
|
|
408
|
+
# The staged version, which may differ from the file on disk
|
|
409
|
+
shown = subprocess.run(
|
|
410
|
+
["git", "-C", str(root), "show", f":{path}"],
|
|
411
|
+
capture_output=True,
|
|
412
|
+
encoding="utf-8",
|
|
413
|
+
errors="replace",
|
|
414
|
+
check=False,
|
|
415
|
+
)
|
|
416
|
+
if shown.returncode == 0 and _has_outputs(shown.stdout):
|
|
417
|
+
found.append(path)
|
|
418
|
+
return found
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _quote(path: str) -> str:
|
|
422
|
+
"""Single-quote a path for a POSIX shell, as R's shQuote() does."""
|
|
423
|
+
return "'" + path.replace("'", "'\\''") + "'"
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _check_notebook_outputs(root: Path) -> bool:
|
|
427
|
+
notebooks = _notebooks_with_outputs(root)
|
|
428
|
+
if not notebooks:
|
|
429
|
+
return True
|
|
430
|
+
# The same wording as R's hook
|
|
431
|
+
lines = ["✗ These notebooks have saved outputs, which can contain data:"]
|
|
432
|
+
lines += [f" - {path}" for path in notebooks]
|
|
433
|
+
lines += [
|
|
434
|
+
"",
|
|
435
|
+
"Clear the outputs, then stage the notebooks again. In Jupyter, use",
|
|
436
|
+
"Edit > Clear All Outputs, or run:",
|
|
437
|
+
]
|
|
438
|
+
lines += [f" jupyter nbconvert --clear-output --inplace {_quote(path)}" for path in notebooks]
|
|
439
|
+
print("\n".join(lines), file=sys.stderr)
|
|
440
|
+
return False
|
|
441
|
+
|
|
442
|
+
|
|
383
443
|
def hook_run(name: str, settings_file: str | Path | None = None) -> int:
|
|
384
444
|
"""Run the checks for one git hook. Returns the exit status for git.
|
|
385
445
|
|
|
@@ -398,6 +458,8 @@ def hook_run(name: str, settings_file: str | Path | None = None) -> int:
|
|
|
398
458
|
|
|
399
459
|
if enabled["check_sensitive_dirs"] and not _check_sensitive_dirs(root):
|
|
400
460
|
return 1
|
|
461
|
+
if enabled["check_notebook_outputs"] and not _check_notebook_outputs(root):
|
|
462
|
+
return 1
|
|
401
463
|
if enabled["data_security"]:
|
|
402
464
|
from framework.security import _report, git_security_audit
|
|
403
465
|
|
framework/notebooks.py
CHANGED
|
@@ -186,6 +186,13 @@ def _default_type(config: dict[str, Any]) -> NotebookType:
|
|
|
186
186
|
)
|
|
187
187
|
if value in ("jupyter", "ipynb"):
|
|
188
188
|
return "jupyter"
|
|
189
|
+
if value == "script":
|
|
190
|
+
# Scripts-only projects
|
|
191
|
+
return "script"
|
|
192
|
+
# A project that turned the Quarto tool off gets Jupyter notebooks, as in R
|
|
193
|
+
tools = config.get("tools")
|
|
194
|
+
if isinstance(tools, dict) and tools.get("quarto") is False:
|
|
195
|
+
return "jupyter"
|
|
189
196
|
return "quarto"
|
|
190
197
|
|
|
191
198
|
|
|
@@ -208,7 +215,8 @@ def make_notebook(
|
|
|
208
215
|
``data-cleaning.qmd``. An extension (``.qmd``, ``.ipynb``, or
|
|
209
216
|
``.py``) sets the type.
|
|
210
217
|
type: ``"quarto"`` (the default), ``"jupyter"``, or ``"script"``.
|
|
211
|
-
Without it, ``scaffold: notebook_format`` in the settings decides
|
|
218
|
+
Without it, ``scaffold: notebook_format`` in the settings decides, so a
|
|
219
|
+
Scripts-only project (``script``) gets a ``.py`` script.
|
|
212
220
|
dir: The folder. Defaults to ``directories: notebooks`` in the
|
|
213
221
|
settings, relative to the project root.
|
|
214
222
|
stub: The stub to start from, such as ``"default"`` or ``"minimal"``.
|
framework/outputs.py
CHANGED
|
@@ -281,7 +281,12 @@ def save_notebook(
|
|
|
281
281
|
result = subprocess.run(command, capture_output=True, text=True, check=False)
|
|
282
282
|
if result.returncode != 0:
|
|
283
283
|
raise FrameworkError(f"Quarto render failed\n{result.stdout}{result.stderr}")
|
|
284
|
-
|
|
284
|
+
# Inside a Quarto project, the output keeps the file's folder under
|
|
285
|
+
# --output-dir (notebooks/x.qmd -> <temp>/notebooks/x.html)
|
|
286
|
+
extension = _NOTEBOOK_FORMATS[format]
|
|
287
|
+
outputs = sorted(Path(temp).rglob(f"{source.stem}{extension}")) or sorted(
|
|
288
|
+
Path(temp).rglob(f"*{extension}")
|
|
289
|
+
)
|
|
285
290
|
if not outputs:
|
|
286
291
|
raise FrameworkError(f"No {format} output found after rendering")
|
|
287
292
|
shutil.copyfile(outputs[0], destination)
|
framework/quarto.py
CHANGED
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import os
|
|
6
6
|
import re
|
|
7
7
|
import shutil
|
|
8
|
+
from collections import Counter
|
|
8
9
|
from dataclasses import dataclass, field
|
|
9
10
|
from datetime import date, datetime
|
|
10
11
|
from pathlib import Path
|
|
@@ -85,6 +86,16 @@ def render_format(dir_key: str, project_type: str | None = None) -> str:
|
|
|
85
86
|
return "html"
|
|
86
87
|
|
|
87
88
|
|
|
89
|
+
def _shared_output_dir(pairs: list[tuple[str, str, str, str | None]]) -> str | None:
|
|
90
|
+
"""The parent most render targets share, as R picks it. None means in place."""
|
|
91
|
+
counts = Counter(parent for *_, parent in pairs if parent is not None)
|
|
92
|
+
if not counts:
|
|
93
|
+
return None
|
|
94
|
+
# Ties go to the parent that sorts first, as in R
|
|
95
|
+
best = min(counts, key=lambda parent: (-counts[parent], parent))
|
|
96
|
+
return None if best == "." else best
|
|
97
|
+
|
|
98
|
+
|
|
88
99
|
def quarto_generate_all(
|
|
89
100
|
project_path: str | Path,
|
|
90
101
|
project_type: str,
|
|
@@ -92,18 +103,24 @@ def quarto_generate_all(
|
|
|
92
103
|
quarto_settings: dict[str, Any] | None = None,
|
|
93
104
|
directories: dict[str, str] | None = None,
|
|
94
105
|
) -> list[Path]:
|
|
95
|
-
"""Write the
|
|
106
|
+
"""Write the project's Quarto configuration, as the R package does. Returns the files written.
|
|
96
107
|
|
|
97
|
-
|
|
108
|
+
One root ``_quarto.yml`` holds the shared settings. Its ``output-dir`` is
|
|
109
|
+
the folder most render targets share, and Quarto keeps each file's folder
|
|
110
|
+
under it, so ``notebooks/x.ipynb`` renders to ``outputs/notebooks/x.html``.
|
|
111
|
+
A folder whose format differs from the root's, such as a course's
|
|
112
|
+
``slides/``, gets a ``_metadata.yml`` with only its format, which inherits
|
|
113
|
+
the rest from the root. A render target outside the shared folder keeps
|
|
114
|
+
its own ``_quarto.yml`` with a relative ``output-dir``. Existing files are
|
|
115
|
+
overwritten.
|
|
98
116
|
|
|
99
117
|
Args:
|
|
100
118
|
project_path: The project's root folder.
|
|
101
119
|
project_type: The project type. "presentation" renders the root as revealjs. Any other
|
|
102
120
|
type renders it as HTML.
|
|
103
121
|
render_dirs: Maps a directory key to the folder its output goes to, relative to the
|
|
104
|
-
project root.
|
|
105
|
-
|
|
106
|
-
render as revealjs. Default: no render folders.
|
|
122
|
+
project root. Keys with an empty target are skipped. Keys containing "slide" or
|
|
123
|
+
"slides" render as revealjs. Default: no render folders.
|
|
107
124
|
quarto_settings: Format options under ``html`` and ``revealjs`` keys, such as
|
|
108
125
|
``{"html": {"toc": False}}``. They override Framework's defaults, and only options
|
|
109
126
|
that have a default are written.
|
|
@@ -120,29 +137,46 @@ def quarto_generate_all(
|
|
|
120
137
|
root = Path(project_path).expanduser()
|
|
121
138
|
if not root.is_dir():
|
|
122
139
|
raise FrameworkError(f"Project path does not exist: {root}")
|
|
140
|
+
pairs: list[tuple[str, str, str, str | None]] = []
|
|
141
|
+
for key, target in (render_dirs or {}).items():
|
|
142
|
+
source = (directories or {}).get(key, key)
|
|
143
|
+
if not target or not source:
|
|
144
|
+
continue
|
|
145
|
+
if target == source:
|
|
146
|
+
parent: str | None = "."
|
|
147
|
+
elif target.endswith(f"/{source}"):
|
|
148
|
+
parent = target[: -len(source) - 1]
|
|
149
|
+
else:
|
|
150
|
+
parent = None
|
|
151
|
+
pairs.append((key, source, target, parent))
|
|
152
|
+
output_dir = _shared_output_dir(pairs)
|
|
153
|
+
|
|
123
154
|
written = []
|
|
124
155
|
root_format = "revealjs" if project_type == "presentation" else "html"
|
|
125
156
|
root_file = root / "_quarto.yml"
|
|
126
157
|
root_file.write_text(
|
|
127
|
-
quarto_yml(quarto_settings, root_format, project_root=True
|
|
158
|
+
quarto_yml(quarto_settings, root_format, project_root=True, output_dir=output_dir),
|
|
159
|
+
encoding="utf-8",
|
|
128
160
|
)
|
|
129
161
|
written.append(root_file)
|
|
130
162
|
|
|
131
|
-
for key, target in
|
|
132
|
-
if not target:
|
|
133
|
-
continue
|
|
134
|
-
source = (directories or {}).get(key, key)
|
|
135
|
-
if not source:
|
|
136
|
-
continue
|
|
163
|
+
for key, source, target, parent in pairs:
|
|
137
164
|
source_path = root / source
|
|
138
165
|
source_path.mkdir(parents=True, exist_ok=True)
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
166
|
+
format = render_format(key, project_type)
|
|
167
|
+
if parent != (output_dir or "."):
|
|
168
|
+
# Outside the shared output folder: its own Quarto project, since
|
|
169
|
+
# output-dir is a project setting
|
|
170
|
+
relative = Path(os.path.relpath(root / target, source_path)).as_posix()
|
|
171
|
+
path = source_path / "_quarto.yml"
|
|
172
|
+
path.write_text(
|
|
173
|
+
quarto_yml(quarto_settings, format, output_dir=relative), encoding="utf-8"
|
|
174
|
+
)
|
|
175
|
+
written.append(path)
|
|
176
|
+
elif format != root_format:
|
|
177
|
+
path = source_path / "_metadata.yml"
|
|
178
|
+
path.write_text(quarto_yml(quarto_settings, format), encoding="utf-8")
|
|
179
|
+
written.append(path)
|
|
146
180
|
return written
|
|
147
181
|
|
|
148
182
|
|
|
@@ -154,22 +188,25 @@ class RegenerateResult:
|
|
|
154
188
|
|
|
155
189
|
|
|
156
190
|
def quarto_regenerate(project_path: str | Path, backup: bool = True) -> RegenerateResult:
|
|
157
|
-
"""Rewrite
|
|
191
|
+
"""Rewrite the project's Quarto configuration from its settings, backing up the old files.
|
|
158
192
|
|
|
159
193
|
The files are generated by ``quarto_generate_all()`` from ``project_type``,
|
|
160
|
-
``render_dirs``, ``quarto``, and ``directories`` in the settings.
|
|
194
|
+
``render_dirs``, ``quarto``, and ``directories`` in the settings. The
|
|
195
|
+
``_quarto.yml`` and ``_metadata.yml`` files in the render folders are
|
|
196
|
+
removed first, so an older project's per-folder files don't linger.
|
|
161
197
|
|
|
162
198
|
Args:
|
|
163
199
|
project_path: The project's root folder, which must hold settings.yml or config.yml.
|
|
164
|
-
backup: Copy every existing ``_quarto.yml`` into a timestamped
|
|
165
|
-
``.quarto_backups/`` first.
|
|
200
|
+
backup: Copy every existing ``_quarto.yml`` and ``_metadata.yml`` into a timestamped
|
|
201
|
+
folder under ``.quarto_backups/`` first.
|
|
166
202
|
|
|
167
203
|
Returns:
|
|
168
204
|
The files written, the backed-up files' paths relative to the project root, and the
|
|
169
205
|
backup folder (None without ``backup``).
|
|
170
206
|
|
|
171
207
|
Raises:
|
|
172
|
-
FrameworkError: When the project has no settings file
|
|
208
|
+
FrameworkError: When the project has no settings file, or its settings turn the Quarto
|
|
209
|
+
tool off.
|
|
173
210
|
"""
|
|
174
211
|
root = Path(project_path).expanduser()
|
|
175
212
|
settings_file = next(
|
|
@@ -178,12 +215,20 @@ def quarto_regenerate(project_path: str | Path, backup: bool = True) -> Regenera
|
|
|
178
215
|
if settings_file is None:
|
|
179
216
|
raise FrameworkError("Project settings.yml/config.yml not found")
|
|
180
217
|
config = settings_read(settings_file)
|
|
218
|
+
# A project without a tools section predates it and keeps Quarto
|
|
219
|
+
tools = config.get("tools")
|
|
220
|
+
if isinstance(tools, dict) and tools.get("quarto") is not True:
|
|
221
|
+
raise FrameworkError(
|
|
222
|
+
"This project doesn't use Quarto (tools: quarto is off in settings.yml). Set "
|
|
223
|
+
"tools: {quarto: true} to turn it on, then run quarto_regenerate() again."
|
|
224
|
+
)
|
|
181
225
|
|
|
182
226
|
backed_up: list[str] = []
|
|
183
227
|
location = None
|
|
184
228
|
if backup:
|
|
185
229
|
location = root / ".quarto_backups" / f"{datetime.now():%Y%m%d_%H%M%S}"
|
|
186
|
-
|
|
230
|
+
found = [*root.rglob("_quarto.yml"), *root.rglob("_metadata.yml")]
|
|
231
|
+
for existing in sorted(found):
|
|
187
232
|
if ".quarto_backups" in existing.parts:
|
|
188
233
|
continue
|
|
189
234
|
relative = existing.relative_to(root)
|
|
@@ -192,6 +237,13 @@ def quarto_regenerate(project_path: str | Path, backup: bool = True) -> Regenera
|
|
|
192
237
|
shutil.copy2(existing, target)
|
|
193
238
|
backed_up.append(relative.as_posix())
|
|
194
239
|
|
|
240
|
+
directories = config.get("directories") or {}
|
|
241
|
+
for key in config.get("render_dirs") or {}:
|
|
242
|
+
source = directories.get(key, key) if isinstance(directories, dict) else key
|
|
243
|
+
if source:
|
|
244
|
+
for name in ("_quarto.yml", "_metadata.yml"):
|
|
245
|
+
(root / str(source) / name).unlink(missing_ok=True)
|
|
246
|
+
|
|
195
247
|
quarto = config.get("quarto") if isinstance(config.get("quarto"), dict) else None
|
|
196
248
|
written = quarto_generate_all(
|
|
197
249
|
root,
|
framework/scaffold.py
CHANGED
|
@@ -9,7 +9,9 @@ import runpy
|
|
|
9
9
|
import shutil
|
|
10
10
|
import subprocess
|
|
11
11
|
import sys
|
|
12
|
+
import warnings
|
|
12
13
|
from datetime import UTC, datetime
|
|
14
|
+
from importlib.machinery import FrozenImporter, PathFinder
|
|
13
15
|
from pathlib import Path
|
|
14
16
|
from typing import Any
|
|
15
17
|
|
|
@@ -18,7 +20,7 @@ import pandas as pd
|
|
|
18
20
|
from framework import db, dotenv
|
|
19
21
|
from framework._log import inform, show_messages
|
|
20
22
|
from framework._paths import in_root
|
|
21
|
-
from framework.errors import ProjectNotFoundError, SettingsError
|
|
23
|
+
from framework.errors import FrameworkWarning, ProjectNotFoundError, SettingsError
|
|
22
24
|
from framework.project import find_settings_file
|
|
23
25
|
from framework.settings import settings_read
|
|
24
26
|
|
|
@@ -109,10 +111,54 @@ def _set_seed(config: dict[str, Any]) -> int | None:
|
|
|
109
111
|
return value
|
|
110
112
|
|
|
111
113
|
|
|
114
|
+
def _top_level_modules(folder: Path) -> list[tuple[str, str]]:
|
|
115
|
+
"""The modules a folder provides, as (module name, entry name) pairs."""
|
|
116
|
+
found = []
|
|
117
|
+
for entry in sorted(folder.iterdir(), key=lambda e: e.name):
|
|
118
|
+
if entry.is_file() and entry.suffix == ".py":
|
|
119
|
+
name = entry.stem
|
|
120
|
+
elif (entry / "__init__.py").is_file():
|
|
121
|
+
# Folders without __init__.py are namespace packages, which never
|
|
122
|
+
# win over a module found anywhere on the path.
|
|
123
|
+
name = entry.name
|
|
124
|
+
else:
|
|
125
|
+
continue
|
|
126
|
+
if name.isidentifier() and name != "__init__":
|
|
127
|
+
found.append((name, entry.name))
|
|
128
|
+
return found
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _warn_shadowed(path: Path, label: str) -> None:
|
|
132
|
+
"""Warn when a module in a function folder would hide one Python already has.
|
|
133
|
+
|
|
134
|
+
Called before the folder goes on the import path. It checks names only and
|
|
135
|
+
imports nothing.
|
|
136
|
+
"""
|
|
137
|
+
others = [p for p in sys.path if p != str(path)]
|
|
138
|
+
for name, entry in _top_level_modules(path):
|
|
139
|
+
shown = f"{label.rstrip('/')}/{entry}"
|
|
140
|
+
if name in sys.builtin_module_names or FrozenImporter.find_spec(name) is not None:
|
|
141
|
+
# Python finds these before it looks at the path
|
|
142
|
+
continue
|
|
143
|
+
if name in sys.stdlib_module_names:
|
|
144
|
+
message = (
|
|
145
|
+
f"{shown} hides Python's {name} module. "
|
|
146
|
+
f"Rename it so imports of {name} get the standard one."
|
|
147
|
+
)
|
|
148
|
+
elif PathFinder.find_spec(name, others) is not None:
|
|
149
|
+
message = (
|
|
150
|
+
f"{shown} hides the installed {name} module. "
|
|
151
|
+
f"Rename it so imports of {name} get the installed one."
|
|
152
|
+
)
|
|
153
|
+
else:
|
|
154
|
+
continue
|
|
155
|
+
warnings.warn(message, FrameworkWarning, stacklevel=4)
|
|
156
|
+
|
|
157
|
+
|
|
112
158
|
def _add_function_folders(config: dict[str, Any], root: Path) -> list[Path]:
|
|
113
159
|
"""Put the project's function folders on the import path."""
|
|
114
|
-
# New projects write source_all_functions under scaffold:.
|
|
115
|
-
#
|
|
160
|
+
# New projects write source_all_functions under scaffold:. Older ones
|
|
161
|
+
# have it under options:. R and Python read both, as for the seed.
|
|
116
162
|
options = _section(config, "options")
|
|
117
163
|
if _first(config, "source_all_functions") is False:
|
|
118
164
|
return []
|
|
@@ -123,6 +169,7 @@ def _add_function_folders(config: dict[str, Any], root: Path) -> list[Path]:
|
|
|
123
169
|
for folder in folders if isinstance(folders, list) else [folders]:
|
|
124
170
|
path = in_root(str(folder), root).resolve()
|
|
125
171
|
if path.is_dir() and str(path) not in sys.path:
|
|
172
|
+
_warn_shadowed(path, str(folder))
|
|
126
173
|
sys.path.insert(0, str(path))
|
|
127
174
|
added.append(path)
|
|
128
175
|
inform(f"Added {folder} to the import path")
|