framework-py 0.1.0.dev12__py3-none-any.whl → 0.1.0.dev13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +5 -0
- framework/__main__.py +58 -3
- framework/_agents_md.py +3 -1
- framework/_assets/adopt/rules.json +229 -2
- framework/_backups.py +48 -0
- framework/_gitignore.py +128 -0
- framework/_paths.py +3 -2
- framework/_private.py +62 -4
- framework/_yaml.py +24 -0
- framework/adopt.py +395 -96
- framework/ai.py +10 -7
- framework/bootstrap.py +6 -1
- framework/cloud.py +289 -0
- framework/project.py +54 -4
- framework/quarto.py +274 -82
- framework/scaffold.py +5 -0
- framework/security.py +185 -51
- framework/skill_sync.py +70 -24
- framework/templates/gitignore/course +2 -0
- framework/templates/gitignore/presentation +2 -0
- framework/templates/gitignore/project +2 -0
- framework/templates/gitignore/sensitive +2 -0
- framework/templates/skill-history.json +29 -0
- framework/upgrade.py +987 -0
- {framework_py-0.1.0.dev12.dist-info → framework_py-0.1.0.dev13.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev12.dist-info → framework_py-0.1.0.dev13.dist-info}/RECORD +28 -24
- {framework_py-0.1.0.dev12.dist-info → framework_py-0.1.0.dev13.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev12.dist-info → framework_py-0.1.0.dev13.dist-info}/licenses/LICENSE +0 -0
framework/__init__.py
CHANGED
|
@@ -26,6 +26,7 @@ from framework.cloud import (
|
|
|
26
26
|
CloudError,
|
|
27
27
|
cloud_login,
|
|
28
28
|
cloud_logout,
|
|
29
|
+
cloud_register,
|
|
29
30
|
cloud_settings,
|
|
30
31
|
cloud_status,
|
|
31
32
|
cloud_sync,
|
|
@@ -139,6 +140,7 @@ from framework.storage import (
|
|
|
139
140
|
publish_notebook,
|
|
140
141
|
storage_test,
|
|
141
142
|
)
|
|
143
|
+
from framework.upgrade import UpgradeResult, upgrade
|
|
142
144
|
|
|
143
145
|
try:
|
|
144
146
|
__version__ = version("framework-py")
|
|
@@ -165,6 +167,7 @@ __all__ = [
|
|
|
165
167
|
"SpecError",
|
|
166
168
|
"TableDisplay",
|
|
167
169
|
"UnsupportedFormatError",
|
|
170
|
+
"UpgradeResult",
|
|
168
171
|
"__version__",
|
|
169
172
|
"adopt",
|
|
170
173
|
"ai_skills_update",
|
|
@@ -178,6 +181,7 @@ __all__ = [
|
|
|
178
181
|
"capture_output",
|
|
179
182
|
"cloud_login",
|
|
180
183
|
"cloud_logout",
|
|
184
|
+
"cloud_register",
|
|
181
185
|
"cloud_settings",
|
|
182
186
|
"cloud_status",
|
|
183
187
|
"cloud_sync",
|
|
@@ -270,5 +274,6 @@ __all__ = [
|
|
|
270
274
|
"stubs_path",
|
|
271
275
|
"stubs_publish",
|
|
272
276
|
"tools_available",
|
|
277
|
+
"upgrade",
|
|
273
278
|
"write_framework_template",
|
|
274
279
|
]
|
framework/__main__.py
CHANGED
|
@@ -4,7 +4,9 @@
|
|
|
4
4
|
python -m framework create --name NAME --path DIR [--type TYPE] [--workflow ID]
|
|
5
5
|
[--tool ID[=false]]... [--skill-set ID]... [--package-set ID]... [--interactive]
|
|
6
6
|
[--offline]
|
|
7
|
-
python -m framework adopt [--path DIR] [--yes]
|
|
7
|
+
python -m framework adopt [--path DIR] [--yes] [--type TYPE] [--sensitive]
|
|
8
|
+
[--cloud [--slug SLUG]] [--force]
|
|
9
|
+
python -m framework upgrade [--path DIR] [--apply] [--no-backup]
|
|
8
10
|
python -m framework bootstrap [--path DIR] [--interactive] [--no-packages]
|
|
9
11
|
python -m framework hook pre-commit
|
|
10
12
|
|
|
@@ -75,6 +77,42 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
75
77
|
adopt = commands.add_parser("adopt", help="make an existing folder a Framework project")
|
|
76
78
|
adopt.add_argument("--path", default=".", help="the project folder (default: .)")
|
|
77
79
|
adopt.add_argument("--yes", action="store_true", help="accept the proposed folders")
|
|
80
|
+
adopt.add_argument(
|
|
81
|
+
"--type",
|
|
82
|
+
choices=["project", "project_sensitive", "course", "presentation", "bare"],
|
|
83
|
+
help="the project type (default: inferred from the folders)",
|
|
84
|
+
)
|
|
85
|
+
adopt.add_argument(
|
|
86
|
+
"--sensitive",
|
|
87
|
+
action="store_true",
|
|
88
|
+
help="the project holds sensitive data: turn on Framework's guards for private folders",
|
|
89
|
+
)
|
|
90
|
+
adopt.add_argument(
|
|
91
|
+
"--cloud",
|
|
92
|
+
action="store_true",
|
|
93
|
+
help="after adopting, create the project on Framework Cloud and link it",
|
|
94
|
+
)
|
|
95
|
+
adopt.add_argument(
|
|
96
|
+
"--slug", help="with --cloud: the project's slug (default: made from the folder's name)"
|
|
97
|
+
)
|
|
98
|
+
adopt.add_argument(
|
|
99
|
+
"--force",
|
|
100
|
+
action="store_true",
|
|
101
|
+
help="adopt the folder as Python even when its files say another language",
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
upgrade = commands.add_parser(
|
|
105
|
+
"upgrade", help="bring a project made by an older Framework to the current layout"
|
|
106
|
+
)
|
|
107
|
+
upgrade.add_argument("--path", default=".", help="the project folder (default: .)")
|
|
108
|
+
upgrade.add_argument(
|
|
109
|
+
"--apply", action="store_true", help="apply the plan (default: print it only)"
|
|
110
|
+
)
|
|
111
|
+
upgrade.add_argument(
|
|
112
|
+
"--no-backup",
|
|
113
|
+
action="store_true",
|
|
114
|
+
help="don't copy the files it rewrites to .framework/backups/ first",
|
|
115
|
+
)
|
|
78
116
|
|
|
79
117
|
bootstrap = commands.add_parser("bootstrap", help="prepare a cloned project")
|
|
80
118
|
bootstrap.add_argument("--path", default=".", help="the project folder (default: .)")
|
|
@@ -111,7 +149,20 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
111
149
|
if args.command == "adopt":
|
|
112
150
|
from framework.adopt import adopt
|
|
113
151
|
|
|
114
|
-
path = adopt(
|
|
152
|
+
path = adopt(
|
|
153
|
+
args.path,
|
|
154
|
+
yes=args.yes,
|
|
155
|
+
type=args.type,
|
|
156
|
+
sensitive=True if args.sensitive else None,
|
|
157
|
+
cloud=True if args.cloud else None,
|
|
158
|
+
slug=args.slug,
|
|
159
|
+
force=args.force,
|
|
160
|
+
)
|
|
161
|
+
elif args.command == "upgrade":
|
|
162
|
+
from framework.upgrade import upgrade
|
|
163
|
+
|
|
164
|
+
upgrade(args.path, dry_run=not args.apply, backup=not args.no_backup)
|
|
165
|
+
return 0
|
|
115
166
|
elif args.command == "bootstrap":
|
|
116
167
|
from framework.bootstrap import bootstrap
|
|
117
168
|
|
|
@@ -147,7 +198,11 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
147
198
|
)
|
|
148
199
|
except FrameworkError as error:
|
|
149
200
|
# The functions' advice names Python arguments; the command line has flags
|
|
150
|
-
message =
|
|
201
|
+
message = (
|
|
202
|
+
str(error)
|
|
203
|
+
.replace("Pass yes=True.", "Pass --yes.")
|
|
204
|
+
.replace("pass force=True", "pass --force")
|
|
205
|
+
)
|
|
151
206
|
print(f"Error: {message}", file=sys.stderr)
|
|
152
207
|
return 1
|
|
153
208
|
print(path)
|
framework/_agents_md.py
CHANGED
|
@@ -197,7 +197,9 @@ def _data(spec: dict[str, Any]) -> str:
|
|
|
197
197
|
|
|
198
198
|
def _type_section(spec: dict[str, Any]) -> str:
|
|
199
199
|
# Sensitive data is a setting, whatever structure the project uses
|
|
200
|
-
|
|
200
|
+
from framework._private import spec_sensitive
|
|
201
|
+
|
|
202
|
+
if spec_sensitive(spec):
|
|
201
203
|
return (
|
|
202
204
|
"## Privacy Requirements\n\n"
|
|
203
205
|
"**This is a privacy-sensitive project.** Before ANY data operation, output, or\n"
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"description": "How adopt() and structure_import() map an existing project's folders to Framework directory keys. Each rule's patterns are relative folder paths, matched exactly but ignoring case; a rule without languages applies to every language. Shared with framework-py and fw; see inst/conformance/README.md (adopt/).",
|
|
2
|
+
"description": "How adopt() and structure_import() map an existing project's folders to Framework directory keys. Each rule's patterns are relative folder paths, matched exactly but ignoring case; a rule without languages applies to every language, and a rule without types applies to every project type. A skip entry with * matches any characters within one folder name. render_dirs rules mark folders that hold rendered notebooks. types says how the project type is inferred. Shared with framework-py and fw; see inst/conformance/README.md (adopt/).",
|
|
3
3
|
"max_depth": 3,
|
|
4
4
|
"skip": [
|
|
5
5
|
".git",
|
|
@@ -18,9 +18,220 @@
|
|
|
18
18
|
".cache",
|
|
19
19
|
".pytest_cache",
|
|
20
20
|
".mypy_cache",
|
|
21
|
-
".framework"
|
|
21
|
+
".framework",
|
|
22
|
+
"settings",
|
|
23
|
+
"_site",
|
|
24
|
+
"_book",
|
|
25
|
+
"*_files",
|
|
26
|
+
"*_cache"
|
|
22
27
|
],
|
|
28
|
+
"types": {
|
|
29
|
+
"course": {
|
|
30
|
+
"description": "At least min of these top-level folders, ignoring case",
|
|
31
|
+
"signals": [
|
|
32
|
+
"slides",
|
|
33
|
+
"lectures",
|
|
34
|
+
"assignments",
|
|
35
|
+
"homework",
|
|
36
|
+
"modules",
|
|
37
|
+
"readings",
|
|
38
|
+
"syllabus",
|
|
39
|
+
"course_docs"
|
|
40
|
+
],
|
|
41
|
+
"min": 2
|
|
42
|
+
},
|
|
43
|
+
"project_sensitive": {
|
|
44
|
+
"description": "A folder mapped to a key matching this pattern",
|
|
45
|
+
"key_pattern": "^(inputs|outputs)_(private|public)_"
|
|
46
|
+
},
|
|
47
|
+
"project": {
|
|
48
|
+
"description": "Any folder mapped to a Framework key; otherwise bare"
|
|
49
|
+
}
|
|
50
|
+
},
|
|
23
51
|
"rules": [
|
|
52
|
+
{
|
|
53
|
+
"key": "data",
|
|
54
|
+
"patterns": [
|
|
55
|
+
"data"
|
|
56
|
+
],
|
|
57
|
+
"types": [
|
|
58
|
+
"course"
|
|
59
|
+
]
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"key": "slides",
|
|
63
|
+
"patterns": [
|
|
64
|
+
"slides",
|
|
65
|
+
"lectures"
|
|
66
|
+
],
|
|
67
|
+
"types": [
|
|
68
|
+
"course"
|
|
69
|
+
]
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"key": "assignments",
|
|
73
|
+
"patterns": [
|
|
74
|
+
"assignments",
|
|
75
|
+
"homework",
|
|
76
|
+
"problem_sets",
|
|
77
|
+
"problem-sets"
|
|
78
|
+
],
|
|
79
|
+
"types": [
|
|
80
|
+
"course"
|
|
81
|
+
]
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"key": "course_docs",
|
|
85
|
+
"patterns": [
|
|
86
|
+
"course_docs",
|
|
87
|
+
"syllabus"
|
|
88
|
+
],
|
|
89
|
+
"types": [
|
|
90
|
+
"course"
|
|
91
|
+
]
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"key": "readings",
|
|
95
|
+
"patterns": [
|
|
96
|
+
"readings"
|
|
97
|
+
],
|
|
98
|
+
"types": [
|
|
99
|
+
"course"
|
|
100
|
+
]
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
"key": "notebooks",
|
|
104
|
+
"patterns": [
|
|
105
|
+
"modules"
|
|
106
|
+
],
|
|
107
|
+
"types": [
|
|
108
|
+
"course"
|
|
109
|
+
]
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"key": "inputs_private_raw",
|
|
113
|
+
"patterns": [
|
|
114
|
+
"inputs/private/raw",
|
|
115
|
+
"inputs/raw/private",
|
|
116
|
+
"data/private/raw",
|
|
117
|
+
"data/raw/private"
|
|
118
|
+
]
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"key": "inputs_public_raw",
|
|
122
|
+
"patterns": [
|
|
123
|
+
"inputs/public/raw",
|
|
124
|
+
"inputs/raw/public",
|
|
125
|
+
"data/public/raw",
|
|
126
|
+
"data/raw/public"
|
|
127
|
+
]
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"key": "inputs_private_intermediate",
|
|
131
|
+
"patterns": [
|
|
132
|
+
"inputs/private/intermediate",
|
|
133
|
+
"inputs/intermediate/private",
|
|
134
|
+
"data/private/intermediate",
|
|
135
|
+
"data/intermediate/private"
|
|
136
|
+
]
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
"key": "inputs_public_intermediate",
|
|
140
|
+
"patterns": [
|
|
141
|
+
"inputs/public/intermediate",
|
|
142
|
+
"inputs/intermediate/public",
|
|
143
|
+
"data/public/intermediate",
|
|
144
|
+
"data/intermediate/public"
|
|
145
|
+
]
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
"key": "inputs_private_final",
|
|
149
|
+
"patterns": [
|
|
150
|
+
"inputs/private/final",
|
|
151
|
+
"inputs/final/private",
|
|
152
|
+
"data/private/final",
|
|
153
|
+
"data/final/private"
|
|
154
|
+
]
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
"key": "inputs_public_final",
|
|
158
|
+
"patterns": [
|
|
159
|
+
"inputs/public/final",
|
|
160
|
+
"inputs/final/public",
|
|
161
|
+
"data/public/final",
|
|
162
|
+
"data/final/public"
|
|
163
|
+
]
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
"key": "outputs_private_notebooks",
|
|
167
|
+
"patterns": [
|
|
168
|
+
"outputs/private/notebooks",
|
|
169
|
+
"output/private/notebooks"
|
|
170
|
+
]
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
"key": "outputs_public_notebooks",
|
|
174
|
+
"patterns": [
|
|
175
|
+
"outputs/public/notebooks",
|
|
176
|
+
"output/public/notebooks"
|
|
177
|
+
]
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
"key": "outputs_private_tables",
|
|
181
|
+
"patterns": [
|
|
182
|
+
"outputs/private/tables",
|
|
183
|
+
"output/private/tables"
|
|
184
|
+
]
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
"key": "outputs_public_tables",
|
|
188
|
+
"patterns": [
|
|
189
|
+
"outputs/public/tables",
|
|
190
|
+
"output/public/tables"
|
|
191
|
+
]
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
"key": "outputs_private_figures",
|
|
195
|
+
"patterns": [
|
|
196
|
+
"outputs/private/figures",
|
|
197
|
+
"output/private/figures"
|
|
198
|
+
]
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
"key": "outputs_public_figures",
|
|
202
|
+
"patterns": [
|
|
203
|
+
"outputs/public/figures",
|
|
204
|
+
"output/public/figures"
|
|
205
|
+
]
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
"key": "outputs_private_models",
|
|
209
|
+
"patterns": [
|
|
210
|
+
"outputs/private/models",
|
|
211
|
+
"output/private/models"
|
|
212
|
+
]
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
"key": "outputs_public_models",
|
|
216
|
+
"patterns": [
|
|
217
|
+
"outputs/public/models",
|
|
218
|
+
"output/public/models"
|
|
219
|
+
]
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
"key": "outputs_private_reports",
|
|
223
|
+
"patterns": [
|
|
224
|
+
"outputs/private/reports",
|
|
225
|
+
"output/private/reports"
|
|
226
|
+
]
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
"key": "outputs_public_reports",
|
|
230
|
+
"patterns": [
|
|
231
|
+
"outputs/public/reports",
|
|
232
|
+
"output/public/reports"
|
|
233
|
+
]
|
|
234
|
+
},
|
|
24
235
|
{
|
|
25
236
|
"key": "inputs_raw",
|
|
26
237
|
"patterns": [
|
|
@@ -157,5 +368,21 @@
|
|
|
157
368
|
"sandbox"
|
|
158
369
|
]
|
|
159
370
|
}
|
|
371
|
+
],
|
|
372
|
+
"render_dirs": [
|
|
373
|
+
{
|
|
374
|
+
"key": "notebooks",
|
|
375
|
+
"patterns": [
|
|
376
|
+
"outputs/notebooks",
|
|
377
|
+
"output/notebooks"
|
|
378
|
+
]
|
|
379
|
+
},
|
|
380
|
+
{
|
|
381
|
+
"key": "docs",
|
|
382
|
+
"patterns": [
|
|
383
|
+
"outputs/docs",
|
|
384
|
+
"output/docs"
|
|
385
|
+
]
|
|
386
|
+
}
|
|
160
387
|
]
|
|
161
388
|
}
|
framework/_backups.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Backups of files Framework rewrites, for ``upgrade()`` and ``quarto_regenerate()``.
|
|
2
|
+
|
|
3
|
+
They go to ``.framework/backups/<timestamp>/<path from the project root>``.
|
|
4
|
+
``.framework/.gitignore`` ignores ``backups/``, so they never reach git whatever
|
|
5
|
+
the project's ``.gitignore`` says. The layout is R's (``R/backups.R``).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import shutil
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def backup_dir(root: Path) -> str:
|
|
16
|
+
"""A new backup folder's path from the root: ``.framework/backups/<timestamp>``, with
|
|
17
|
+
``-2``, ``-3``, ... when that folder exists."""
|
|
18
|
+
stamp = f"{datetime.now():%Y%m%d-%H%M%S}"
|
|
19
|
+
relative = f".framework/backups/{stamp}"
|
|
20
|
+
n = 2
|
|
21
|
+
while (root / relative).exists():
|
|
22
|
+
relative = f".framework/backups/{stamp}-{n}"
|
|
23
|
+
n += 1
|
|
24
|
+
return relative
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def ignore_in_framework_dir(root: Path, entries: list[str]) -> None:
|
|
28
|
+
"""Add entries to ``.framework/.gitignore`` (created if missing), once each."""
|
|
29
|
+
path = root / ".framework" / ".gitignore"
|
|
30
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
lines = path.read_text(encoding="utf-8").splitlines() if path.is_file() else []
|
|
32
|
+
missing = [entry for entry in entries if entry not in {line.strip() for line in lines}]
|
|
33
|
+
if missing:
|
|
34
|
+
path.write_text("\n".join([*lines, *missing]) + "\n", encoding="utf-8")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def backup_files(root: Path, paths: list[str], relative: str) -> list[str]:
|
|
38
|
+
"""Copy the existing files among ``paths`` (from the root) into the backup folder
|
|
39
|
+
``relative`` (from ``backup_dir()``), keeping their paths. Returns the paths copied."""
|
|
40
|
+
existing = [p for p in dict.fromkeys(paths) if (root / p).is_file()]
|
|
41
|
+
if not existing:
|
|
42
|
+
return []
|
|
43
|
+
ignore_in_framework_dir(root, ["backups/"])
|
|
44
|
+
for path in existing:
|
|
45
|
+
target = root / relative / path
|
|
46
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
shutil.copy2(root / path, target)
|
|
48
|
+
return existing
|
framework/_gitignore.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Merging rules into an existing ``.gitignore``, for ``adopt()`` and ``upgrade()``.
|
|
2
|
+
|
|
3
|
+
The rules, shared with R (``R/gitignore_merge.R``) and fw:
|
|
4
|
+
|
|
5
|
+
- A rule is a line that is neither blank nor a comment, trimmed. Lines are
|
|
6
|
+
compared exactly after trimming, so ``/.quarto/`` and ``.quarto/`` are
|
|
7
|
+
different rules (Quarto looks for its own exact lines).
|
|
8
|
+
- Lines are only ever appended, in one block under a comment that names the
|
|
9
|
+
tool, never removed, reordered, or rewritten.
|
|
10
|
+
- A rule that would ignore a file git already tracks is left out and
|
|
11
|
+
reported, so the merge never hides files the project keeps in git (a
|
|
12
|
+
tracked ``.Rprofile``, rendered ``*_files/`` on GitHub Pages). Negations
|
|
13
|
+
(``!...``) are never left out.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import shutil
|
|
19
|
+
import subprocess
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from framework._private import is_private_folder_name
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def gitignore_rules(text: str) -> list[str]:
|
|
27
|
+
"""The rules in ``.gitignore`` text: lines that are neither blank nor comments,
|
|
28
|
+
trimmed, in order, once each."""
|
|
29
|
+
lines = (line.strip() for line in text.splitlines())
|
|
30
|
+
return list(dict.fromkeys(line for line in lines if line and not line.startswith("#")))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _read(root: Path) -> str:
|
|
34
|
+
path = root / ".gitignore"
|
|
35
|
+
return path.read_text(encoding="utf-8") if path.is_file() else ""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _git(root: Path, *args: str) -> subprocess.CompletedProcess[str] | None:
|
|
39
|
+
if shutil.which("git") is None:
|
|
40
|
+
return None
|
|
41
|
+
return subprocess.run(
|
|
42
|
+
["git", "-C", str(root), *args],
|
|
43
|
+
capture_output=True,
|
|
44
|
+
text=True,
|
|
45
|
+
check=False,
|
|
46
|
+
encoding="utf-8",
|
|
47
|
+
errors="replace",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def in_work_tree(root: Path) -> bool:
|
|
52
|
+
"""Whether ``root`` is inside a git work tree (and git is installed)."""
|
|
53
|
+
result = _git(root, "rev-parse", "--is-inside-work-tree")
|
|
54
|
+
return result is not None and result.returncode == 0 and result.stdout.strip() == "true"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def tracked_matching(root: Path, rule: str) -> list[str]:
|
|
58
|
+
"""Files under ``root`` that git tracks and ``rule`` would ignore."""
|
|
59
|
+
result = _git(root, "ls-files", "--cached", "--ignored", f"--exclude={rule}")
|
|
60
|
+
if result is None or result.returncode != 0:
|
|
61
|
+
return []
|
|
62
|
+
return [line for line in result.stdout.splitlines() if line]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class MissingRules:
|
|
67
|
+
"""The rules to append, in order, and those left out because git tracks files they
|
|
68
|
+
match (rule -> one such file)."""
|
|
69
|
+
|
|
70
|
+
add: list[str] = field(default_factory=list)
|
|
71
|
+
skipped: dict[str, str] = field(default_factory=dict)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def missing_rules(root: Path, wanted: list[str] | str, text: str | None = None) -> MissingRules:
|
|
75
|
+
"""The rules in ``wanted`` that the project's ``.gitignore`` (or ``text``) lacks."""
|
|
76
|
+
have = set(gitignore_rules(_read(root) if text is None else text))
|
|
77
|
+
wanted_text = wanted if isinstance(wanted, str) else "\n".join(wanted)
|
|
78
|
+
add = [rule for rule in gitignore_rules(wanted_text) if rule not in have]
|
|
79
|
+
skipped: dict[str, str] = {}
|
|
80
|
+
if add and in_work_tree(root):
|
|
81
|
+
for rule in add:
|
|
82
|
+
if rule.startswith("!"):
|
|
83
|
+
continue
|
|
84
|
+
tracked = tracked_matching(root, rule)
|
|
85
|
+
if tracked:
|
|
86
|
+
skipped[rule] = tracked[0]
|
|
87
|
+
add = [rule for rule in add if rule not in skipped]
|
|
88
|
+
return MissingRules(add, skipped)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def merged_text(current: str, add: list[str], header: str) -> str:
|
|
92
|
+
"""``.gitignore`` text with ``add`` appended in one block under ``header``."""
|
|
93
|
+
if not add:
|
|
94
|
+
return current
|
|
95
|
+
lines = current.splitlines()
|
|
96
|
+
if lines and lines[-1].strip():
|
|
97
|
+
lines.append("")
|
|
98
|
+
return "\n".join([*lines, header, *add]) + "\n"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def append_rules(root: Path, add: list[str], header: str) -> list[str]:
|
|
102
|
+
"""Append ``add`` to the project's ``.gitignore`` under ``header``. Returns the rules
|
|
103
|
+
added."""
|
|
104
|
+
if add:
|
|
105
|
+
(root / ".gitignore").write_text(merged_text(_read(root), add, header), encoding="utf-8")
|
|
106
|
+
return add
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def unignored_folders(root: Path, folders: list[str], text: str | None = None) -> list[str]:
|
|
110
|
+
"""The folders (paths from the root) that ``.gitignore`` doesn't ignore yet, and that
|
|
111
|
+
the templates' ``**/private/**`` and ``**/*_private/**`` rules don't cover."""
|
|
112
|
+
from framework.security import _matches_gitignore
|
|
113
|
+
|
|
114
|
+
cleaned = [f.removeprefix("./").rstrip("/") for f in folders if f]
|
|
115
|
+
cleaned = list(dict.fromkeys(f for f in cleaned if f))
|
|
116
|
+
lines = (_read(root) if text is None else text).splitlines()
|
|
117
|
+
return [
|
|
118
|
+
folder
|
|
119
|
+
for folder in cleaned
|
|
120
|
+
if not any(is_private_folder_name(part) for part in folder.split("/"))
|
|
121
|
+
and not _matches_gitignore(f"{folder}/.framework-probe", lines)
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def rules_summary(rules: list[str], n: int = 6) -> str:
|
|
126
|
+
"""A short list of rules for messages: the first few, then "and N more"."""
|
|
127
|
+
shown = ", ".join(rules[:n])
|
|
128
|
+
return f"{shown}, and {len(rules) - n} more" if len(rules) > n else shown
|
framework/_paths.py
CHANGED
|
@@ -53,9 +53,10 @@ def output_directory(
|
|
|
53
53
|
"""
|
|
54
54
|
directories = config.get("directories")
|
|
55
55
|
directories = directories if isinstance(directories, dict) else {}
|
|
56
|
-
|
|
56
|
+
from framework._private import settings_sensitive
|
|
57
|
+
|
|
57
58
|
split = (
|
|
58
|
-
(
|
|
59
|
+
settings_sensitive(config)
|
|
59
60
|
or f"outputs_private_{kind}" in directories
|
|
60
61
|
or f"outputs_public_{kind}" in directories
|
|
61
62
|
)
|
framework/_private.py
CHANGED
|
@@ -192,8 +192,66 @@ def confirm_private_upload(
|
|
|
192
192
|
raise FrameworkError("Cancelled. Nothing was uploaded.")
|
|
193
193
|
|
|
194
194
|
|
|
195
|
+
def settings_sensitive(settings: Any) -> bool:
|
|
196
|
+
"""Whether settings describe a project that holds sensitive data.
|
|
197
|
+
|
|
198
|
+
``project_type: project_sensitive`` always counts: projects of that type made before
|
|
199
|
+
Framework 2 have no ``security:`` block. Otherwise ``security: sensitive:`` decides
|
|
200
|
+
when it is set, and without it the privacy-sensitive layout (an ``inputs_private_*``
|
|
201
|
+
or ``outputs_private_*`` folder) counts. The rule is R's ``.fw_settings_sensitive()``.
|
|
202
|
+
"""
|
|
203
|
+
if not isinstance(settings, dict):
|
|
204
|
+
return False
|
|
205
|
+
if settings.get("project_type") == "project_sensitive":
|
|
206
|
+
return True
|
|
207
|
+
security = settings.get("security")
|
|
208
|
+
flag = security.get("sensitive") if isinstance(security, dict) else None
|
|
209
|
+
if flag is not None:
|
|
210
|
+
return flag is True
|
|
211
|
+
directories = settings.get("directories")
|
|
212
|
+
keys = directories if isinstance(directories, dict) else {}
|
|
213
|
+
return any(re.match(r"^(inputs|outputs)_private_", str(key)) for key in keys)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def spec_sensitive(spec: Any) -> bool:
|
|
217
|
+
"""``settings_sensitive()`` for a spec, whose type is ``type``."""
|
|
218
|
+
if not isinstance(spec, dict):
|
|
219
|
+
return False
|
|
220
|
+
return settings_sensitive(
|
|
221
|
+
{
|
|
222
|
+
"project_type": spec.get("type"),
|
|
223
|
+
"security": spec.get("security"),
|
|
224
|
+
"directories": spec.get("directories"),
|
|
225
|
+
}
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def sensitive_mismatch(settings: Any) -> str | None:
|
|
230
|
+
"""Why the settings and the project type disagree about sensitive data, or None.
|
|
231
|
+
|
|
232
|
+
A ``project_sensitive`` project with ``security: sensitive: false``, or with no
|
|
233
|
+
``security:`` block at all, which ``upgrade()`` adds.
|
|
234
|
+
"""
|
|
235
|
+
if not isinstance(settings, dict) or settings.get("project_type") != "project_sensitive":
|
|
236
|
+
return None
|
|
237
|
+
security = settings.get("security")
|
|
238
|
+
flag = security.get("sensitive") if isinstance(security, dict) else None
|
|
239
|
+
if flag is True:
|
|
240
|
+
return None
|
|
241
|
+
if flag is None:
|
|
242
|
+
return (
|
|
243
|
+
"project_type is project_sensitive but settings have no security: block. "
|
|
244
|
+
"Framework treats the project as sensitive; run framework.upgrade() to add "
|
|
245
|
+
"security: sensitive: true."
|
|
246
|
+
)
|
|
247
|
+
return (
|
|
248
|
+
"project_type is project_sensitive but security: sensitive: is false. "
|
|
249
|
+
"Framework treats the project as sensitive because of its type."
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
|
|
195
253
|
def project_is_sensitive() -> bool:
|
|
196
|
-
"""Whether the current project
|
|
254
|
+
"""Whether the current project holds sensitive data (``settings_sensitive()``)."""
|
|
197
255
|
from framework.project import find_settings_file
|
|
198
256
|
from framework.settings import settings_read
|
|
199
257
|
|
|
@@ -201,10 +259,9 @@ def project_is_sensitive() -> bool:
|
|
|
201
259
|
if found is None:
|
|
202
260
|
return False
|
|
203
261
|
try:
|
|
204
|
-
|
|
262
|
+
return settings_sensitive(settings_read(found))
|
|
205
263
|
except FrameworkError:
|
|
206
264
|
return False
|
|
207
|
-
return isinstance(security, dict) and security.get("sensitive") is True
|
|
208
265
|
|
|
209
266
|
|
|
210
267
|
def sensitive_reasons(what: str) -> list[str]:
|
|
@@ -213,7 +270,8 @@ def sensitive_reasons(what: str) -> list[str]:
|
|
|
213
270
|
if not project_is_sensitive():
|
|
214
271
|
return []
|
|
215
272
|
return [
|
|
216
|
-
f"This project holds sensitive data (security: sensitive: true
|
|
273
|
+
f"This project holds sensitive data (security: sensitive: true, or type "
|
|
274
|
+
f"project_sensitive), and {what} can "
|
|
217
275
|
"show it. Check that it holds only results that may be shared."
|
|
218
276
|
]
|
|
219
277
|
|