framework-py 0.1.0.dev13__py3-none-any.whl → 0.1.0.dev14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
framework/__main__.py CHANGED
@@ -5,8 +5,9 @@
5
5
  [--tool ID[=false]]... [--skill-set ID]... [--package-set ID]... [--interactive]
6
6
  [--offline]
7
7
  python -m framework adopt [--path DIR] [--yes] [--type TYPE] [--sensitive]
8
- [--cloud [--slug SLUG]] [--force]
9
- python -m framework upgrade [--path DIR] [--apply] [--no-backup]
8
+ [--private FOLDER ...] [--cloud [--slug SLUG]] [--force]
9
+ python -m framework upgrade [--path DIR] [--apply] [--no-backup] [--type TYPE]
10
+ [--move-settings]
10
11
  python -m framework bootstrap [--path DIR] [--interactive] [--no-packages]
11
12
  python -m framework hook pre-commit
12
13
 
@@ -100,6 +101,12 @@ def _parser() -> argparse.ArgumentParser:
100
101
  action="store_true",
101
102
  help="adopt the folder as Python even when its files say another language",
102
103
  )
104
+ adopt.add_argument(
105
+ "--private",
106
+ action="append",
107
+ metavar="FOLDER",
108
+ help="map a folder to a private key, keeping its path (repeatable)",
109
+ )
103
110
 
104
111
  upgrade = commands.add_parser(
105
112
  "upgrade", help="bring a project made by an older Framework to the current layout"
@@ -113,6 +120,16 @@ def _parser() -> argparse.ArgumentParser:
113
120
  action="store_true",
114
121
  help="don't copy the files it rewrites to .framework/backups/ first",
115
122
  )
123
+ upgrade.add_argument(
124
+ "--type",
125
+ choices=("project", "project_sensitive", "course", "presentation", "bare"),
126
+ help="change the project's type (project_sensitive makes it sensitive)",
127
+ )
128
+ upgrade.add_argument(
129
+ "--move-settings",
130
+ action="store_true",
131
+ help="move settings keys outside default: into settings/<key>.yml",
132
+ )
116
133
 
117
134
  bootstrap = commands.add_parser("bootstrap", help="prepare a cloned project")
118
135
  bootstrap.add_argument("--path", default=".", help="the project folder (default: .)")
@@ -157,11 +174,18 @@ def main(argv: Sequence[str] | None = None) -> int:
157
174
  cloud=True if args.cloud else None,
158
175
  slug=args.slug,
159
176
  force=args.force,
177
+ private=args.private,
160
178
  )
161
179
  elif args.command == "upgrade":
162
180
  from framework.upgrade import upgrade
163
181
 
164
- upgrade(args.path, dry_run=not args.apply, backup=not args.no_backup)
182
+ upgrade(
183
+ args.path,
184
+ dry_run=not args.apply,
185
+ backup=not args.no_backup,
186
+ type=args.type,
187
+ move_settings=args.move_settings,
188
+ )
165
189
  return 0
166
190
  elif args.command == "bootstrap":
167
191
  from framework.bootstrap import bootstrap
framework/_gitignore.py CHANGED
@@ -9,8 +9,13 @@ The rules, shared with R (``R/gitignore_merge.R``) and fw:
9
9
  tool, never removed, reordered, or rewritten.
10
10
  - A rule that would ignore a file git already tracks is left out and
11
11
  reported, so the merge never hides files the project keeps in git (a
12
- tracked ``.Rprofile``, rendered ``*_files/`` on GitHub Pages). Negations
13
- (``!...``) are never left out.
12
+ tracked ``.Rprofile``, rendered ``*_files/`` on GitHub Pages).
13
+ - Negations (``!...``) are left out and reported when the ``.gitignore``
14
+ already has rules: appended after the project's own rules they would win
15
+ over them, and could make files in folders the project ignores committable
16
+ (fore's ``inputs/**`` with ``!inputs/*/``, where the template's
17
+ ``!/inputs/**/README.md`` re-included ``inputs/raw/README.md``). Into a new
18
+ or empty ``.gitignore`` the template goes whole, negations included.
14
19
  """
15
20
 
16
21
  from __future__ import annotations
@@ -64,11 +69,13 @@ def tracked_matching(root: Path, rule: str) -> list[str]:
64
69
 
65
70
  @dataclass
66
71
  class MissingRules:
67
- """The rules to append, in order, and those left out because git tracks files they
68
- match (rule -> one such file)."""
72
+ """The rules to append, in order; those left out because git tracks files they match
73
+ (rule -> one such file); and the negations left out because the ``.gitignore``
74
+ already has rules."""
69
75
 
70
76
  add: list[str] = field(default_factory=list)
71
77
  skipped: dict[str, str] = field(default_factory=dict)
78
+ negations: list[str] = field(default_factory=list)
72
79
 
73
80
 
74
81
  def missing_rules(root: Path, wanted: list[str] | str, text: str | None = None) -> MissingRules:
@@ -76,6 +83,8 @@ def missing_rules(root: Path, wanted: list[str] | str, text: str | None = None)
76
83
  have = set(gitignore_rules(_read(root) if text is None else text))
77
84
  wanted_text = wanted if isinstance(wanted, str) else "\n".join(wanted)
78
85
  add = [rule for rule in gitignore_rules(wanted_text) if rule not in have]
86
+ negations = [rule for rule in add if rule.startswith("!")] if have else []
87
+ add = [rule for rule in add if rule not in negations]
79
88
  skipped: dict[str, str] = {}
80
89
  if add and in_work_tree(root):
81
90
  for rule in add:
@@ -85,7 +94,7 @@ def missing_rules(root: Path, wanted: list[str] | str, text: str | None = None)
85
94
  if tracked:
86
95
  skipped[rule] = tracked[0]
87
96
  add = [rule for rule in add if rule not in skipped]
88
- return MissingRules(add, skipped)
97
+ return MissingRules(add, skipped, negations)
89
98
 
90
99
 
91
100
  def merged_text(current: str, add: list[str], header: str) -> str:
@@ -108,18 +117,38 @@ def append_rules(root: Path, add: list[str], header: str) -> list[str]:
108
117
 
109
118
  def unignored_folders(root: Path, folders: list[str], text: str | None = None) -> list[str]:
110
119
  """The folders (paths from the root) that ``.gitignore`` doesn't ignore yet, and that
111
- the templates' ``**/private/**`` and ``**/*_private/**`` rules don't cover."""
120
+ the templates' ``**/private/**`` and ``**/*_private/**`` rules don't cover. In a git
121
+ work tree git decides, so every rule counts as git reads it (fore's ``inputs/**``
122
+ with ``!inputs/*/`` ignores what is inside ``inputs/raw``); elsewhere, or when
123
+ ``text`` isn't the ``.gitignore`` on disk, R's matcher reads ``text``."""
112
124
  from framework.security import _matches_gitignore
113
125
 
114
126
  cleaned = [f.removeprefix("./").rstrip("/") for f in folders if f]
115
127
  cleaned = list(dict.fromkeys(f for f in cleaned if f))
116
- lines = (_read(root) if text is None else text).splitlines()
117
- return [
128
+ candidates = [
118
129
  folder
119
130
  for folder in cleaned
120
131
  if not any(is_private_folder_name(part) for part in folder.split("/"))
121
- and not _matches_gitignore(f"{folder}/.framework-probe", lines)
122
132
  ]
133
+ if not candidates:
134
+ return []
135
+ on_disk = _read(root)
136
+ if (text is None or text == on_disk) and in_work_tree(root):
137
+ probes = [f"{folder}/.framework-probe" for folder in candidates]
138
+ result = subprocess.run(
139
+ ["git", "-C", str(root), "check-ignore", "--stdin", "-z"],
140
+ input="\0".join(probes),
141
+ capture_output=True,
142
+ text=True,
143
+ check=False,
144
+ encoding="utf-8",
145
+ errors="replace",
146
+ )
147
+ if result.returncode in (0, 1):
148
+ ignored = set(result.stdout.split("\0"))
149
+ return [f for f, probe in zip(candidates, probes, strict=True) if probe not in ignored]
150
+ lines = (on_disk if text is None else text).splitlines()
151
+ return [f for f in candidates if not _matches_gitignore(f"{f}/.framework-probe", lines)]
123
152
 
124
153
 
125
154
  def rules_summary(rules: list[str], n: int = 6) -> str:
@@ -0,0 +1,100 @@
1
+ """Checking a spec against ``spec.v1.json`` before it is written.
2
+
3
+ Framework Cloud validates every spec it receives against the same schema and refuses
4
+ one with keys the schema doesn't allow (HTTP 422). ``create``, ``adopt()``, and
5
+ ``upgrade()`` write specs; each goes through ``ready()``, which drops keys the schema
6
+ doesn't allow and stops on any other problem, so no spec framework-py writes can be
7
+ refused. R: ``R/spec_schema.R``.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ from functools import cache
14
+ from importlib import resources
15
+ from typing import Any
16
+
17
+ from framework._log import inform
18
+ from framework.spec.merge import SpecError
19
+
20
+
21
+ @cache
22
+ def schema() -> dict[str, Any]:
23
+ """The parsed ``spec.v1.json``."""
24
+ text = resources.files("framework").joinpath("_assets/schema/spec.v1.json").read_text("utf-8")
25
+ parsed: dict[str, Any] = json.loads(text)
26
+ return parsed
27
+
28
+
29
+ def _as_json(spec: Any) -> Any:
30
+ # As JSON sees it: written the way .framework/spec.json is, and read back
31
+ return json.loads(json.dumps(spec, default=str))
32
+
33
+
34
+ def _resolve(node: Any) -> Any:
35
+ root = schema()
36
+ while isinstance(node, dict) and "$ref" in node:
37
+ target: Any = root
38
+ for part in str(node["$ref"]).removeprefix("#/").split("/"):
39
+ target = target[part]
40
+ node = target
41
+ return node
42
+
43
+
44
+ def problems(spec: Any) -> list[str]:
45
+ """The spec's problems against spec.v1, as ``"path: what is wrong"``; empty when valid."""
46
+ import jsonschema
47
+
48
+ validator = jsonschema.Draft7Validator(schema())
49
+ out = []
50
+ for error in sorted(validator.iter_errors(_as_json(spec)), key=lambda e: list(e.path)):
51
+ where = ".".join(str(p) for p in error.absolute_path) or "(spec)"
52
+ out.append(f"{where}: {error.message}")
53
+ return out
54
+
55
+
56
+ def prune(spec: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
57
+ """The spec without keys spec.v1 doesn't allow, where the schema says exactly which
58
+ keys an object may hold, and the dot paths dropped. Inside ``anyOf`` and ``oneOf``
59
+ nothing is removed: which form applies is for the check to say."""
60
+ dropped: list[str] = []
61
+
62
+ def walk(value: Any, node: Any, path: str) -> Any:
63
+ node = _resolve(node)
64
+ if not isinstance(node, dict):
65
+ return value
66
+ if isinstance(value, dict):
67
+ props = node.get("properties") or {}
68
+ additional = node.get("additionalProperties")
69
+ out = {}
70
+ for key, item in value.items():
71
+ where = f"{path}.{key}" if path else str(key)
72
+ if key in props:
73
+ out[key] = walk(item, props[key], where)
74
+ elif additional is False:
75
+ dropped.append(where)
76
+ elif isinstance(additional, dict):
77
+ out[key] = walk(item, additional, where)
78
+ else:
79
+ out[key] = item
80
+ return out
81
+ if isinstance(value, list) and isinstance(node.get("items"), dict):
82
+ return [walk(item, node["items"], f"{path}[{i}]") for i, item in enumerate(value)]
83
+ return value
84
+
85
+ return walk(_as_json(spec), schema(), ""), dropped
86
+
87
+
88
+ def ready(spec: dict[str, Any], quiet: bool = False) -> dict[str, Any]:
89
+ """The spec ready to write: keys spec.v1 doesn't allow are dropped (with a message
90
+ naming them), and any other problem raises ``SpecError`` with all of them listed."""
91
+ pruned, dropped = prune(spec)
92
+ if dropped and not quiet:
93
+ inform(
94
+ " Left out of .framework/spec.json (not part of spec.v1, so Framework Cloud "
95
+ f"would refuse them): {', '.join(dropped)}"
96
+ )
97
+ found = problems(pruned)
98
+ if found:
99
+ raise SpecError(".framework/spec.json would not match spec.v1: " + "; ".join(found))
100
+ return pruned
framework/adopt.py CHANGED
@@ -49,6 +49,7 @@ class AdoptPlan:
49
49
  type: str = "bare"
50
50
  reserved: list[str] = field(default_factory=list)
51
51
  sensitive: bool = False
52
+ marked_private: list[str] = field(default_factory=list)
52
53
 
53
54
  @property
54
55
  def directories(self) -> dict[str, str]:
@@ -321,9 +322,81 @@ def _table(plan: AdoptPlan) -> str:
321
322
  width = max(30, *(len(path) for path, _ in _rows(plan))) if _rows(plan) else 30
322
323
  for number, (path, shown) in enumerate(_rows(plan), start=1):
323
324
  lines.append(f" {number:>2}. {path:<{width}} {shown}")
325
+ candidates = restricted_candidates(plan) if plan.sensitive else []
326
+ if candidates:
327
+ quoted = ", ".join(f'"{c}"' for c in candidates)
328
+ lines.append(
329
+ f"May hold restricted data (mark private?): {', '.join(candidates)}. Accept to be "
330
+ f"asked about each, or pass private=[{quoted}] (python -m framework adopt "
331
+ "--private <folder>, or fw adopt --private <folder>) to map them to private keys, "
332
+ "which Framework's checks and agent guards cover."
333
+ )
324
334
  return "\n".join(lines)
325
335
 
326
336
 
337
+ _RESTRICTED_NAME = re.compile(
338
+ r"data|export|extract|report|result|output|raw|table|figure|submission|download|cohort"
339
+ )
340
+
341
+
342
+ def _private_path(path: str) -> bool:
343
+ from framework._private import is_private_folder_name
344
+
345
+ return any(is_private_folder_name(part) for part in path.split("/"))
346
+
347
+
348
+ def restricted_candidates(plan: AdoptPlan) -> list[str]:
349
+ """The mapped folders that may hold restricted data but aren't private, in mapping
350
+ order: ``inputs_*``/``outputs_*`` keys and ``data``, ``cache``, and ``scratch``; then
351
+ folders of their own named like data, exports, extracts, reports, results, outputs,
352
+ raw, tables, figures, submissions, downloads, or cohorts. Folders private by key or by
353
+ a private-named segment don't count. R and fw: the same rule."""
354
+ from_keys = [
355
+ path
356
+ for key, path in plan.directories.items()
357
+ if (re.match(r"^(inputs|outputs)_", key) or key in ("data", "cache", "scratch"))
358
+ and not re.search("private|public", key)
359
+ ]
360
+ from_extras = [
361
+ entry["path"]
362
+ for entry in plan.extra_directories.values()
363
+ if _RESTRICTED_NAME.search(entry["path"].rsplit("/", 1)[-1].lower())
364
+ ]
365
+ return [p for p in dict.fromkeys([*from_keys, *from_extras]) if not _private_path(p)]
366
+
367
+
368
+ def mark_private(plan: AdoptPlan, folder: str) -> None:
369
+ """Map a folder to a private key, keeping its path: ``inputs_raw`` becomes
370
+ ``inputs_private_raw``, ``outputs_reports`` becomes ``outputs_private_reports``, and
371
+ any other folder becomes ``outputs_private_<name>`` when it is named like outputs
372
+ (reports, results, exports, tables, figures) and ``inputs_private_<name>`` otherwise.
373
+ Its path is gitignored like the other private folders."""
374
+ folder = folder.removeprefix("./").rstrip("/")
375
+ role = plan.roles.get(folder)
376
+ if role is None:
377
+ mapped = [*plan.directories.values(), *(e["path"] for e in plan.extra_directories.values())]
378
+ raise FrameworkError(
379
+ f"'{folder}' isn't a folder adopt() maps here, so it can't be marked private. "
380
+ f"Folders: {', '.join(mapped)}"
381
+ )
382
+ if role != EXTRA and "private" in role and "public" not in role:
383
+ return
384
+ base = folder.rsplit("/", 1)[-1]
385
+ slug = _extra_key(base, set())
386
+ if role != EXTRA and re.match(r"^(inputs|outputs)_", role):
387
+ key = re.sub(r"^(inputs|outputs)_", r"\1_private_", role)
388
+ elif re.search("report|result|output|export|table|figure", base.lower()):
389
+ key = f"outputs_private_{slug}"
390
+ else:
391
+ key = f"inputs_private_{slug}"
392
+ taken = set(plan.directories) | set(plan.extra_directories)
393
+ if key in taken:
394
+ key = _extra_key(key, taken)
395
+ _set_role(plan, folder, key)
396
+ if folder not in plan.marked_private:
397
+ plan.marked_private.append(folder)
398
+
399
+
327
400
  def _review(plan: AdoptPlan, ask: Callable[[str], str], cancelled: str) -> AdoptPlan:
328
401
  """Let the person accept the plan, change a folder's role, leave a folder out, or cancel."""
329
402
  while True:
@@ -382,6 +455,7 @@ def adopt(
382
455
  cloud: bool | None = None,
383
456
  slug: str | None = None,
384
457
  force: bool = False,
458
+ private: list[str] | tuple[str, ...] | None = None,
385
459
  ) -> Path:
386
460
  """Make an existing folder a Framework project, in place.
387
461
 
@@ -406,8 +480,9 @@ def adopt(
406
480
  when any folder matches a Framework directory, and ``bare`` otherwise.
407
481
 
408
482
  The spec records the folder as it is, as R's ``adopt()`` writes it: the
409
- mapped folders only, no tools, no default packages, AI files off, and git
410
- on when the folder is a git repository.
483
+ mapped folders only, Quarto on when a ``_quarto.yml`` or ``.qmd`` file is at
484
+ most three folders deep, no other tools, no default packages, AI files off,
485
+ and git on when the folder is a git repository.
411
486
 
412
487
  framework-py adopts Python projects. When the folder's files say another
413
488
  language (an ``.Rproj``, ``renv.lock``, or ``.R`` files for R), it stops
@@ -432,6 +507,14 @@ def adopt(
432
507
  the folder's name.
433
508
  force: Adopt the folder as a Python project even when its files say
434
509
  another language.
510
+ private: Folders (paths from the project folder) to map to private keys,
511
+ keeping their paths: ``inputs_raw`` becomes ``inputs_private_raw``,
512
+ ``outputs_reports`` becomes ``outputs_private_reports``, and another
513
+ folder becomes ``outputs_private_<name>`` or ``inputs_private_<name>``.
514
+ Their paths are added to ``.gitignore``. In a sensitive adopt, folders
515
+ that may hold restricted data (data, exports, reports, results, ...) and
516
+ aren't private are listed in the plan; an interactive review asks about
517
+ each.
435
518
 
436
519
  Returns:
437
520
  The project folder.
@@ -479,6 +562,8 @@ def adopt(
479
562
  raise FrameworkError("adopt() needs a terminal to review the folders. Pass yes=True.")
480
563
 
481
564
  plan = adopt_plan(scan_folders(root), type=type)
565
+ for folder in private or ():
566
+ mark_private(plan, folder)
482
567
  if plan.type == "project_sensitive":
483
568
  sensitive = True
484
569
  plan.sensitive = sensitive is True
@@ -503,8 +588,13 @@ def adopt(
503
588
  "Does this project hold sensitive data? Framework will treat its outputs as "
504
589
  "private by default.",
505
590
  )
591
+ if plan.sensitive:
592
+ for folder in restricted_candidates(plan):
593
+ answer = input(f"'{folder}' may hold restricted data. Mark it private? [Y/n] ")
594
+ if answer.strip().lower() not in ("n", "no"):
595
+ mark_private(plan, folder)
506
596
 
507
- spec = adopt_spec(root.name, plan, use_git=(root / ".git").exists())
597
+ spec = adopt_spec(root.name, plan, use_git=(root / ".git").exists(), quarto=detect_quarto(root))
508
598
  from framework.create import _record, _write_settings
509
599
 
510
600
  _write_settings(root, spec)
@@ -531,11 +621,17 @@ def _adopt_gitignore(root: Path, spec: Mapping[str, Any], plan: AdoptPlan) -> st
531
621
  from framework._gitignore import append_rules, missing_rules, rules_summary, unignored_folders
532
622
 
533
623
  wanted = [str(spec["git"]["gitignore_content"])]
534
- wanted += [f"/{folder}/" for folder in unignored_folders(root, plan.private)]
624
+ folders = list(dict.fromkeys([*plan.private, *plan.marked_private]))
625
+ wanted += [f"/{folder}/" for folder in unignored_folders(root, folders)]
535
626
  missing = missing_rules(root, wanted)
536
627
  append_rules(root, missing.add, "# Added by framework.adopt()")
537
628
  if missing.add:
538
629
  inform(f" Added to .gitignore ({len(missing.add)} rules): {rules_summary(missing.add)}")
630
+ if missing.negations:
631
+ inform(
632
+ f" Left out of .gitignore: {rules_summary(missing.negations)}. Added after the "
633
+ "folder's own rules, they could make files in ignored folders committable."
634
+ )
539
635
  if missing.skipped:
540
636
  shown = ", ".join(f"{rule} ({file})" for rule, file in missing.skipped.items())
541
637
  inform(f" Left out of .gitignore because git tracks files they match: {shown}")
@@ -543,12 +639,30 @@ def _adopt_gitignore(root: Path, spec: Mapping[str, Any], plan: AdoptPlan) -> st
543
639
  return path.read_text(encoding="utf-8") if path.is_file() else ""
544
640
 
545
641
 
546
- def adopt_spec(name: str, plan: AdoptPlan, use_git: bool = False) -> dict[str, Any]:
642
+ def detect_quarto(root: Path, rules: Mapping[str, Any] | None = None) -> bool:
643
+ """Whether the folder uses Quarto: a ``_quarto.yml`` (or ``_quarto.yaml``) or a
644
+ ``.qmd`` file in it or in a folder ``adopt()`` scans (at most the rules' ``max_depth``
645
+ deep, skipping the folders it skips), so a Quarto project in ``notebooks/`` counts.
646
+ R and fw: the same rule."""
647
+ for folder in ["", *scan_folders(root, rules)]:
648
+ try:
649
+ names = [p.name.lower() for p in (root / folder).iterdir() if not p.is_dir()]
650
+ except OSError:
651
+ continue
652
+ if any(n in ("_quarto.yml", "_quarto.yaml") or n.endswith(".qmd") for n in names):
653
+ return True
654
+ return False
655
+
656
+
657
+ def adopt_spec(
658
+ name: str, plan: AdoptPlan, use_git: bool = False, quarto: bool = False
659
+ ) -> dict[str, Any]:
547
660
  """The resolved spec for an adopted folder, as R's ``adopt()`` records it.
548
661
 
549
662
  A project of the mapped type with exactly the mapped folders: the type's own
550
- folders and render folders are not added. No tools, no default packages, no
551
- skills, and no AI files. Git is on when the folder is a git repository.
663
+ folders and render folders are not added. Quarto is on when the folder uses it
664
+ (``detect_quarto()``); no other tools, no default packages, no skills, and no AI
665
+ files. Git is on when the folder is a git repository.
552
666
  """
553
667
  from framework.spec.resolve import spec_resolve
554
668
 
@@ -558,7 +672,7 @@ def adopt_spec(name: str, plan: AdoptPlan, use_git: bool = False) -> dict[str, A
558
672
  "language": "python",
559
673
  "core_files": False,
560
674
  "settings_layout": "flat",
561
- "tools": {"quarto": False, "uv": False},
675
+ "tools": {"quarto": quarto, "uv": False},
562
676
  "git": {"use_git": use_git},
563
677
  "ai": {"enabled": False},
564
678
  "security": {"sensitive": sensitive},
framework/ai.py CHANGED
@@ -36,17 +36,56 @@ def assistant_targets(spec: dict[str, Any]) -> tuple[list[str], list[str]]:
36
36
 
37
37
 
38
38
  def restricted_paths(spec: dict[str, Any]) -> list[str]:
39
- """Folders agents must not read: each configured path up to its ``private`` segment."""
40
- paths = list((spec.get("directories") or {}).values())
41
- paths += [entry.get("path") for entry in (spec.get("extra_directories") or {}).values()]
42
- restricted = set()
39
+ """Folders agents must not read in a sensitive project, each ending in ``/``.
40
+
41
+ Every configured folder (``directories``, ``extra_directories``, ``render_dirs``)
42
+ with a segment named ``private`` or ending in ``_private``, collapsed to that
43
+ segment, and every ``directories`` entry whose key marks it private
44
+ (``inputs_private_raw: inputs/raw``), as configured. ``.``, unsafe paths (outside
45
+ the project), and folders inside another restricted folder are left out. ``.env``
46
+ (secrets) is added by the callers. R: ``.spec_restricted_paths()``.
47
+ """
48
+ from framework._private import is_private_folder_name
49
+ from framework.spec.check import is_safe_path
50
+
51
+ def clean(values: Any) -> list[str]:
52
+ out = []
53
+ for value in values:
54
+ if isinstance(value, str) and value:
55
+ path = value.removeprefix("./").rstrip("/")
56
+ if path and is_safe_path(path):
57
+ out.append(path)
58
+ return out
59
+
60
+ def mapping(value: Any) -> dict[str, Any]:
61
+ return value if isinstance(value, dict) else {}
62
+
63
+ directories = mapping(spec.get("directories"))
64
+ extra = spec.get("extra_directories")
65
+ entries = (
66
+ extra.values() if isinstance(extra, dict) else extra if isinstance(extra, list) else []
67
+ )
68
+ paths = clean(
69
+ [
70
+ *directories.values(),
71
+ *(e.get("path") for e in entries if isinstance(e, dict)),
72
+ *mapping(spec.get("render_dirs")).values(),
73
+ ]
74
+ )
75
+ restricted: list[str] = []
43
76
  for path in paths:
44
- if not isinstance(path, str):
45
- continue
46
77
  parts = path.split("/")
47
- if "private" in parts:
48
- restricted.add("/".join(parts[: parts.index("private") + 1]) + "/")
49
- return sorted(restricted)
78
+ hits = [i for i, part in enumerate(parts) if is_private_folder_name(part)]
79
+ if hits:
80
+ restricted.append("/".join(parts[: hits[0] + 1]))
81
+ restricted += clean(
82
+ value
83
+ for key, value in directories.items()
84
+ if "private" in str(key).lower() and "public" not in str(key).lower()
85
+ )
86
+ restricted = [p for p in dict.fromkeys(restricted) if p != "."]
87
+ kept = [p for p in restricted if not any(p.startswith(f"{o}/") for o in restricted if o != p)]
88
+ return [f"{p}/" for p in sorted(kept)]
50
89
 
51
90
 
52
91
  def restricted_notice(spec: dict[str, Any]) -> str | None:
framework/cloud.py CHANGED
@@ -1506,6 +1506,22 @@ def cloud_register(
1506
1506
  raise FrameworkError(f".framework/spec.json is not valid JSON: {error}") from error
1507
1507
  if not isinstance(spec, dict):
1508
1508
  raise FrameworkError(".framework/spec.json is not a JSON object.")
1509
+ # Framework Cloud refuses a spec that doesn't match spec.v1; say so here, with the
1510
+ # fix, before anything is sent
1511
+ from framework._spec_schema import problems
1512
+
1513
+ found = problems(spec)
1514
+ if found:
1515
+ fix = (
1516
+ f'framework.upgrade("{path}", dry_run=False) (python -m framework upgrade --apply)'
1517
+ if spec.get("language") in (None, "python")
1518
+ else f"the {spec.get('language')} package's upgrade() (or fw project upgrade)"
1519
+ )
1520
+ raise FrameworkError(
1521
+ ".framework/spec.json doesn't match spec.v1, so Framework Cloud would refuse it: "
1522
+ + "; ".join(found[:5])
1523
+ + f". Run {fix} to rewrite it, then cloud_register() again."
1524
+ )
1509
1525
  if _env_tracked(root):
1510
1526
  raise FrameworkError(
1511
1527
  f"git tracks {root / '.env'}, so the project token would be committed. Run "
framework/create.py CHANGED
@@ -383,10 +383,16 @@ def _presentation(spec: Mapping[str, Any]) -> str:
383
383
 
384
384
 
385
385
  def _record(root: Path, spec: Mapping[str, Any]) -> None:
386
+ """Record how the project was made. Keys spec.v1 doesn't allow are left out, and a
387
+ spec that doesn't match it otherwise is never written."""
386
388
  record = dict(spec)
387
389
  source = dict(record.get("source") or {"origin": "python"})
388
390
  source["framework_version"] = _version()
389
391
  record["source"] = source
392
+ # Only what spec.v1 allows, so Framework Cloud accepts the spec as written
393
+ from framework._spec_schema import ready
394
+
395
+ record = ready(record)
390
396
  _write(root, ".framework/spec.json", json.dumps(record, indent=2, ensure_ascii=False) + "\n")
391
397
 
392
398
 
framework/data.py CHANGED
@@ -497,6 +497,25 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
497
497
  return target, type, path
498
498
 
499
499
 
500
+ def _in_settings_folder(folder: Path) -> bool:
501
+ """Whether a folder is one the settings name, or inside one."""
502
+ from framework.security import _project_folders
503
+
504
+ try:
505
+ root = project_root()
506
+ config = settings_read()
507
+ except FrameworkError:
508
+ return False
509
+ full = folder.resolve() if folder.is_absolute() else (root / folder).resolve()
510
+ for relative in _project_folders(config, root):
511
+ if relative in ("", "."):
512
+ continue
513
+ base = (root / relative).resolve()
514
+ if full == base or base in full.parents:
515
+ return True
516
+ return False
517
+
518
+
500
519
  def data_save(
501
520
  data: pd.DataFrame,
502
521
  path: str,
@@ -524,7 +543,11 @@ def data_save(
524
543
  locked: Whether later reads must find the file unchanged. Default: True, as in R.
525
544
  Saving again over locked data with different contents replaces it, as R does,
526
545
  with a ``DataChangedWarning``. The same contents save silently.
527
- force: Create the folder if it does not exist.
546
+ force: Create any missing folder. Without it, a missing folder is created
547
+ only when it is a folder named in the settings (``directories``,
548
+ ``extra_directories``, ``render_dirs``) or inside one, such as
549
+ ``inputs/private/intermediate`` in a fresh checkout; any other missing
550
+ folder is an error.
528
551
 
529
552
  Returns:
530
553
  ``data``, so the call can end a pipeline.
@@ -541,10 +564,15 @@ def data_save(
541
564
  raise UnsupportedFormatError(f"data_save() writes csv or parquet, not {file_type}")
542
565
 
543
566
  folder = target.parent
567
+ # A missing folder is created when it is a settings folder or inside one (git
568
+ # doesn't keep empty or ignored folders, so a fresh checkout of a sensitive project
569
+ # has no inputs/private/...), or with force=True. Anywhere else a missing folder is
570
+ # more likely a typo.
544
571
  if not folder.is_dir():
545
- if not force:
572
+ if not force and not _in_settings_folder(folder):
546
573
  raise FrameworkError(
547
- f"Directory '{folder}' does not exist. Use force=True to create it."
574
+ f"Directory '{folder}' does not exist and isn't a settings folder. Use "
575
+ "force=True to create it."
548
576
  )
549
577
  folder.mkdir(parents=True, exist_ok=True)
550
578
  inform(f"Created directory: {folder}")