framework-py 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +5 -0
- framework/__main__.py +64 -5
- framework/_agents_md.py +210 -0
- framework/_assets/adopt/rules.json +161 -0
- framework/_assets/features/jupyter.json +39 -0
- framework/_assets/features/quarto.json +18 -0
- framework/_assets/features/quarto_manuscript.json +35 -0
- framework/_assets/features/quarto_slides.json +29 -0
- framework/_assets/features/renv.json +20 -0
- framework/_assets/features/targets.json +34 -0
- framework/_assets/features/uv.json +17 -0
- framework/_assets/schema/layer.v1.json +45 -7
- framework/_assets/schema/layers.v1.json +49 -7
- framework/_assets/schema/spec.v1.json +38 -7
- framework/_assets/sets/packages/tidyverse-analysis.json +42 -0
- framework/_assets/sets/skills/framework.json +25 -0
- framework/_assets/sets/skills/ggplot2.json +26 -0
- framework/_assets/workflows/jupyter.json +14 -0
- framework/_assets/workflows/quarto.json +14 -0
- framework/_assets/workflows/scripts.json +18 -0
- framework/_defaults_cache.py +47 -0
- framework/_pages.py +228 -0
- framework/_paths.py +19 -9
- framework/_private.py +69 -0
- framework/adopt.py +365 -0
- framework/ai.py +13 -20
- framework/cloud.py +139 -14
- framework/create.py +395 -18
- framework/features.py +195 -0
- framework/hooks.py +69 -7
- framework/notebooks.py +9 -1
- framework/outputs.py +6 -1
- framework/quarto.py +77 -25
- framework/scaffold.py +50 -3
- framework/spec/check.py +31 -7
- framework/spec/merge.py +1 -0
- framework/spec/resolve.py +80 -6
- framework/storage.py +96 -50
- framework/templates/defaults.yml +0 -1
- framework/templates/skills/framework-outputs/SKILL.md +5 -5
- framework/templates/skills/framework-sensitive-data/SKILL.md +3 -2
- {framework_py-0.1.0.dev2.dist-info → framework_py-0.1.0.dev4.dist-info}/METADATA +1 -1
- framework_py-0.1.0.dev4.dist-info/RECORD +84 -0
- framework_py-0.1.0.dev2.dist-info/RECORD +0 -64
- {framework_py-0.1.0.dev2.dist-info → framework_py-0.1.0.dev4.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev2.dist-info → framework_py-0.1.0.dev4.dist-info}/licenses/LICENSE +0 -0
framework/spec/check.py
CHANGED
|
@@ -29,7 +29,7 @@ REQUIRED = (
|
|
|
29
29
|
"files",
|
|
30
30
|
"skills",
|
|
31
31
|
)
|
|
32
|
-
OPTIONAL = ("quarto", "source")
|
|
32
|
+
OPTIONAL = ("quarto", "source", "tools", "core_files", "settings_layout")
|
|
33
33
|
|
|
34
34
|
# Assistant -> (skills folder, stub file). Adding an assistant is one row.
|
|
35
35
|
ASSISTANTS: dict[str, tuple[str, str | None]] = {
|
|
@@ -38,7 +38,7 @@ ASSISTANTS: dict[str, tuple[str, str | None]] = {
|
|
|
38
38
|
"copilot": (".agents/skills", ".github/copilot-instructions.md"),
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
_LABEL = re.compile(r"^[a-z][a-z0-9_]*$")
|
|
42
42
|
_SOURCES = ("pypi", "github")
|
|
43
43
|
_PACKAGE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
|
44
44
|
_SKILL = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$")
|
|
@@ -72,7 +72,9 @@ def secret_env_values(raw: str | None = None, variables: Any = None) -> list[str
|
|
|
72
72
|
|
|
73
73
|
|
|
74
74
|
_HOOK = re.compile(r"^[^:]+\.py::[A-Za-z_][A-Za-z0-9_.]*$")
|
|
75
|
-
|
|
75
|
+
# Workflow -> the notebook format it allows. R Markdown is R only.
|
|
76
|
+
WORKFLOW_FORMATS = {"quarto": "quarto", "jupyter": "jupyter", "scripts": "script"}
|
|
77
|
+
_NOTEBOOK_FORMATS = tuple(WORKFLOW_FORMATS.values())
|
|
76
78
|
|
|
77
79
|
|
|
78
80
|
def is_safe_path(path: Any) -> bool:
|
|
@@ -141,8 +143,18 @@ def spec_check(spec: dict[str, Any]) -> list[str]:
|
|
|
141
143
|
|
|
142
144
|
if spec.get("spec_version") != 1:
|
|
143
145
|
add(f"spec_version must be 1, got {spec.get('spec_version')}")
|
|
144
|
-
|
|
145
|
-
|
|
146
|
+
# type is a label; behavior follows settings, tools, and core_files
|
|
147
|
+
if not isinstance(spec.get("type"), str) or not _LABEL.match(spec["type"]):
|
|
148
|
+
add(f"type must be a lowercase id such as 'project', got '{spec.get('type')}'")
|
|
149
|
+
if spec.get("core_files") is not None and not isinstance(spec["core_files"], bool):
|
|
150
|
+
add("core_files must be true or false")
|
|
151
|
+
if spec.get("settings_layout") not in (None, "split", "flat"):
|
|
152
|
+
add("settings_layout must be 'split' or 'flat'")
|
|
153
|
+
for tool, on in _dict(spec.get("tools")).items():
|
|
154
|
+
if not _LABEL.match(str(tool)):
|
|
155
|
+
add(f"tools: invalid tool id '{tool}'")
|
|
156
|
+
if not isinstance(on, bool):
|
|
157
|
+
add(f"tools.{tool} must be true or false")
|
|
146
158
|
if spec.get("language") != "python":
|
|
147
159
|
add(f"language must be 'python' for the Python package, got '{spec.get('language')}'")
|
|
148
160
|
|
|
@@ -188,8 +200,20 @@ def spec_check(spec: dict[str, Any]) -> list[str]:
|
|
|
188
200
|
notebook_format = _get(spec, ("scaffold", "notebook_format"))
|
|
189
201
|
if notebook_format is not None and notebook_format not in _NOTEBOOK_FORMATS:
|
|
190
202
|
add(
|
|
191
|
-
"scaffold.notebook_format must be quarto or
|
|
192
|
-
f"not '{notebook_format}'"
|
|
203
|
+
"scaffold.notebook_format must be quarto, jupyter, or script for a Python "
|
|
204
|
+
f"project, not '{notebook_format}'"
|
|
205
|
+
)
|
|
206
|
+
workflow = _get(spec, ("scaffold", "workflow"))
|
|
207
|
+
if workflow is not None and workflow not in WORKFLOW_FORMATS:
|
|
208
|
+
add(f"scaffold.workflow must be quarto, jupyter, or scripts, not '{workflow}'")
|
|
209
|
+
elif (
|
|
210
|
+
workflow is not None
|
|
211
|
+
and notebook_format is not None
|
|
212
|
+
and notebook_format != WORKFLOW_FORMATS[workflow]
|
|
213
|
+
):
|
|
214
|
+
add(
|
|
215
|
+
f"scaffold.notebook_format '{notebook_format}' doesn't fit the {workflow} "
|
|
216
|
+
f"workflow, which uses '{WORKFLOW_FORMATS[workflow]}'"
|
|
193
217
|
)
|
|
194
218
|
|
|
195
219
|
for name, entry in _dict(spec.get("skills")).items():
|
framework/spec/merge.py
CHANGED
framework/spec/resolve.py
CHANGED
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import re
|
|
5
6
|
from collections.abc import Mapping, Sequence
|
|
6
7
|
from dataclasses import dataclass, field
|
|
7
8
|
from importlib import resources
|
|
8
9
|
from typing import Any
|
|
9
10
|
|
|
10
11
|
from framework import _yaml
|
|
11
|
-
from framework.spec.check import spec_check
|
|
12
|
+
from framework.spec.check import WORKFLOW_FORMATS, spec_check
|
|
12
13
|
from framework.spec.merge import (
|
|
13
14
|
MAP_PATHS,
|
|
14
15
|
PACKAGE_DEFAULTS,
|
|
@@ -20,7 +21,10 @@ from framework.spec.merge import (
|
|
|
20
21
|
normalize_layers,
|
|
21
22
|
)
|
|
22
23
|
|
|
24
|
+
# Framework's own structures. Any other lowercase label is a blueprint whose
|
|
25
|
+
# layers bring its folders.
|
|
23
26
|
TYPES = ("project", "project_sensitive", "course", "presentation", "bare")
|
|
27
|
+
_LABEL = re.compile(r"^[a-z][a-z0-9_]*$")
|
|
24
28
|
LANGUAGES = ("r", "python", "stata")
|
|
25
29
|
LANGUAGE = "python"
|
|
26
30
|
|
|
@@ -29,6 +33,9 @@ _ORDER = (
|
|
|
29
33
|
"name",
|
|
30
34
|
"type",
|
|
31
35
|
"language",
|
|
36
|
+
"core_files",
|
|
37
|
+
"settings_layout",
|
|
38
|
+
"tools",
|
|
32
39
|
"author",
|
|
33
40
|
"directories",
|
|
34
41
|
"extra_directories",
|
|
@@ -65,20 +72,61 @@ def _defaults() -> dict[str, Any]:
|
|
|
65
72
|
return data
|
|
66
73
|
|
|
67
74
|
|
|
75
|
+
def default_settings_layout(type: str) -> str:
|
|
76
|
+
"""Split settings files for project and project_sensitive, one file for the rest."""
|
|
77
|
+
return "split" if type in ("project", "project_sensitive") else "flat"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def spec_core_files(spec: Mapping[str, Any]) -> bool:
|
|
81
|
+
"""Whether the project gets .env, scaffold.py, framework.db, and the AI files.
|
|
82
|
+
|
|
83
|
+
A spec without ``core_files`` predates it: every type but bare has them.
|
|
84
|
+
"""
|
|
85
|
+
value = spec.get("core_files")
|
|
86
|
+
return spec.get("type") != "bare" if value is None else value is True
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def spec_settings_layout(spec: Mapping[str, Any]) -> str:
|
|
90
|
+
"""``split`` or ``flat``, with today's default for specs without ``settings_layout``."""
|
|
91
|
+
return str(spec.get("settings_layout") or default_settings_layout(str(spec.get("type"))))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def spec_tool_on(spec: Mapping[str, Any], tool: str) -> bool:
|
|
95
|
+
"""Whether the spec turns a tool on."""
|
|
96
|
+
tools = spec.get("tools")
|
|
97
|
+
return isinstance(tools, Mapping) and tools.get(tool) is True
|
|
98
|
+
|
|
99
|
+
|
|
68
100
|
def package_defaults(type: str) -> dict[str, Any]:
|
|
69
|
-
"""Python's package defaults for a project type: the bottom layer of every spec.
|
|
101
|
+
"""Python's package defaults for a project type: the bottom layer of every spec.
|
|
102
|
+
|
|
103
|
+
Framework's own structures have defaults. Any other label is a blueprint
|
|
104
|
+
whose structure layer brings its folders, so it starts like bare but with
|
|
105
|
+
core files, as in R.
|
|
106
|
+
"""
|
|
70
107
|
defaults = _defaults()
|
|
108
|
+
known = type in TYPES
|
|
109
|
+
label = type
|
|
110
|
+
if not known:
|
|
111
|
+
type = "bare"
|
|
71
112
|
type_config = defaults["types"][type]
|
|
113
|
+
core_files = label != "bare"
|
|
72
114
|
skills: dict[str, Any] = {}
|
|
73
115
|
if type != "bare":
|
|
74
116
|
names = list(defaults["skills"])
|
|
75
117
|
if type == "project_sensitive":
|
|
76
118
|
names += defaults["sensitive_skills"]
|
|
77
119
|
skills = {name: {"package": True} for name in names}
|
|
120
|
+
quarto_on = known and core_files
|
|
78
121
|
quarto = dict(defaults["quarto"])
|
|
79
122
|
if type_config.get("render_dir") is not None:
|
|
80
123
|
quarto["render_dir"] = type_config["render_dir"]
|
|
81
124
|
return {
|
|
125
|
+
"core_files": core_files,
|
|
126
|
+
"settings_layout": default_settings_layout(label),
|
|
127
|
+
# Framework's structures other than bare include Quarto; uv manages every
|
|
128
|
+
# Python project's dependencies
|
|
129
|
+
"tools": {"quarto": quarto_on, "uv": True},
|
|
82
130
|
"author": dict(defaults["author"]),
|
|
83
131
|
"directories": dict(type_config["directories"]),
|
|
84
132
|
"extra_directories": {},
|
|
@@ -86,7 +134,8 @@ def package_defaults(type: str) -> dict[str, Any]:
|
|
|
86
134
|
"packages": defaults["packages"],
|
|
87
135
|
"git": defaults["git"],
|
|
88
136
|
"ai": defaults["ai"],
|
|
89
|
-
|
|
137
|
+
# The workflow decides the notebook format, as in R's built-ins
|
|
138
|
+
"scaffold": {**defaults["scaffold"], **({"workflow": "quarto"} if quarto_on else {})},
|
|
90
139
|
"quarto": quarto,
|
|
91
140
|
"connections": defaults["connections"],
|
|
92
141
|
"security": {"sensitive": type == "project_sensitive"},
|
|
@@ -187,6 +236,27 @@ def _mark_maps(spec: dict[str, Any]) -> None:
|
|
|
187
236
|
node[path[-1]] = {}
|
|
188
237
|
|
|
189
238
|
|
|
239
|
+
def _derive_notebook_format(scaffold: dict[str, Any], spec: Mapping[str, Any]) -> None:
|
|
240
|
+
"""Fill in the notebook format from the workflow, or from the tools without one.
|
|
241
|
+
|
|
242
|
+
The format follows the workflow, so a package default never contradicts it.
|
|
243
|
+
"""
|
|
244
|
+
if scaffold.get("notebook_format"):
|
|
245
|
+
return
|
|
246
|
+
workflow = scaffold.get("workflow")
|
|
247
|
+
if workflow in WORKFLOW_FORMATS:
|
|
248
|
+
derived = WORKFLOW_FORMATS[workflow]
|
|
249
|
+
else:
|
|
250
|
+
# Without the Quarto tool, notebooks are Jupyter notebooks
|
|
251
|
+
derived = "quarto" if spec_tool_on(spec, "quarto") else "jupyter"
|
|
252
|
+
# Keep the key where the package defaults used to put it
|
|
253
|
+
items = list(scaffold.items())
|
|
254
|
+
at = next((i for i, (k, _) in enumerate(items) if k == "ide"), len(items))
|
|
255
|
+
items.insert(at, ("notebook_format", derived))
|
|
256
|
+
scaffold.clear()
|
|
257
|
+
scaffold.update(items)
|
|
258
|
+
|
|
259
|
+
|
|
190
260
|
def _finalize(spec: dict[str, Any], blocks: Sequence[GitignoreBlock]) -> dict[str, Any]:
|
|
191
261
|
spec["spec_version"] = 1
|
|
192
262
|
security = spec.setdefault("security", {})
|
|
@@ -199,6 +269,8 @@ def _finalize(spec: dict[str, Any], blocks: Sequence[GitignoreBlock]) -> dict[st
|
|
|
199
269
|
):
|
|
200
270
|
skills["framework-sensitive-data"] = {"package": True}
|
|
201
271
|
|
|
272
|
+
_derive_notebook_format(spec.setdefault("scaffold", {}), spec)
|
|
273
|
+
|
|
202
274
|
ai = spec.setdefault("ai", {})
|
|
203
275
|
ai["assistants"] = {k: True for k, v in (ai.get("assistants") or {}).items() if v is True}
|
|
204
276
|
ai.setdefault("agents_md", "stub")
|
|
@@ -221,7 +293,9 @@ def spec_resolve(
|
|
|
221
293
|
"""Merge layers over Python's package defaults into one complete project spec.
|
|
222
294
|
|
|
223
295
|
The project type comes from the last layer that sets ``type`` (default
|
|
224
|
-
``"project"``)
|
|
296
|
+
``"project"``). It is a label: Framework's five structures have package
|
|
297
|
+
defaults, and any other lowercase id starts like bare with core files. The
|
|
298
|
+
language from the last layer that sets
|
|
225
299
|
``language`` (default ``"r"``, as in R). This package builds Python
|
|
226
300
|
projects only, so the language must resolve to ``"python"``.
|
|
227
301
|
|
|
@@ -239,8 +313,8 @@ def spec_resolve(
|
|
|
239
313
|
for entry in entries:
|
|
240
314
|
type = entry.layer.get("type") or type
|
|
241
315
|
language = entry.layer.get("language") or language
|
|
242
|
-
if type not
|
|
243
|
-
raise SpecError(f"
|
|
316
|
+
if not isinstance(type, str) or not _LABEL.match(type):
|
|
317
|
+
raise SpecError(f"Project type must be a lowercase id such as 'project', got '{type}'")
|
|
244
318
|
if language != LANGUAGE:
|
|
245
319
|
raise SpecError(language_message(language))
|
|
246
320
|
|
framework/storage.py
CHANGED
|
@@ -12,8 +12,9 @@ Credentials come from the environment (``S3_ACCESS_KEY`` and
|
|
|
12
12
|
environment variables take precedence over values in the settings.
|
|
13
13
|
|
|
14
14
|
Publishing uploads files with a public-read ACL and returns their URLs.
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
A source with "private" anywhere in its full path is listed with a warning
|
|
16
|
+
and needs confirmation: an interactive session asks, and a session with no
|
|
17
|
+
terminal raises an error unless ``include_private=True`` is passed.
|
|
17
18
|
"""
|
|
18
19
|
|
|
19
20
|
from __future__ import annotations
|
|
@@ -33,6 +34,7 @@ from typing import Any
|
|
|
33
34
|
import pandas as pd
|
|
34
35
|
|
|
35
36
|
from framework._log import inform
|
|
37
|
+
from framework._private import confirm_private_upload, include_private_arg
|
|
36
38
|
from framework.errors import FrameworkError
|
|
37
39
|
from framework.settings import settings_read
|
|
38
40
|
|
|
@@ -208,29 +210,6 @@ def resolve_bucket(connection: str | None = None) -> Bucket:
|
|
|
208
210
|
# Uploading --------------------------------------------------------------------------
|
|
209
211
|
|
|
210
212
|
|
|
211
|
-
def _refuse_private(path: Path, allow_private: bool) -> None:
|
|
212
|
-
"""Refuse a source inside a folder named ``private`` within the project.
|
|
213
|
-
|
|
214
|
-
Folders above the project root do not count (macOS keeps temporary files
|
|
215
|
-
under ``/private``), and nothing is refused outside a project.
|
|
216
|
-
"""
|
|
217
|
-
from framework.project import find_settings_file
|
|
218
|
-
|
|
219
|
-
settings_file = find_settings_file()
|
|
220
|
-
if allow_private or settings_file is None:
|
|
221
|
-
return
|
|
222
|
-
root = settings_file.parent.resolve()
|
|
223
|
-
resolved = path.resolve()
|
|
224
|
-
if not resolved.is_relative_to(root):
|
|
225
|
-
return
|
|
226
|
-
relative = resolved.relative_to(root)
|
|
227
|
-
if "private" in relative.parts:
|
|
228
|
-
raise FrameworkError(
|
|
229
|
-
f"Refusing to publish '{relative.as_posix()}': it is inside a private folder, which "
|
|
230
|
-
"holds restricted data. Pass allow_private=True if you really mean to publish it."
|
|
231
|
-
)
|
|
232
|
-
|
|
233
|
-
|
|
234
213
|
def _content_type(path: Path) -> str:
|
|
235
214
|
return (
|
|
236
215
|
_CONTENT_TYPES.get(path.suffix.lower())
|
|
@@ -295,6 +274,37 @@ def _target(connection: str | None) -> str:
|
|
|
295
274
|
)
|
|
296
275
|
|
|
297
276
|
|
|
277
|
+
def _publish_github(
|
|
278
|
+
to: str,
|
|
279
|
+
source: Path,
|
|
280
|
+
dest: str | None,
|
|
281
|
+
connection: str | None,
|
|
282
|
+
slug: str | bool | None,
|
|
283
|
+
self_contained: bool,
|
|
284
|
+
) -> str:
|
|
285
|
+
"""``to="github"``, after rejecting arguments that mean something only elsewhere."""
|
|
286
|
+
from framework._pages import github_publish
|
|
287
|
+
|
|
288
|
+
if to != "github":
|
|
289
|
+
raise FrameworkError(f'to must be "github" or None (got "{to}").')
|
|
290
|
+
given = [
|
|
291
|
+
name
|
|
292
|
+
for name, used in (
|
|
293
|
+
("dest", dest is not None),
|
|
294
|
+
("connection", connection is not None),
|
|
295
|
+
("slug=True/False", isinstance(slug, bool)),
|
|
296
|
+
)
|
|
297
|
+
if used
|
|
298
|
+
]
|
|
299
|
+
if given:
|
|
300
|
+
raise FrameworkError(
|
|
301
|
+
f'Not used with to="github": {", ".join(given)}. Name the published folder with '
|
|
302
|
+
'slug="...".'
|
|
303
|
+
)
|
|
304
|
+
assert not isinstance(slug, bool)
|
|
305
|
+
return github_publish(source, slug=slug, self_contained=self_contained)
|
|
306
|
+
|
|
307
|
+
|
|
298
308
|
def _slug_ignored(slug: str | bool | None) -> None:
|
|
299
309
|
if slug is not None:
|
|
300
310
|
import warnings
|
|
@@ -315,7 +325,10 @@ def publish(
|
|
|
315
325
|
overwrite: bool = True,
|
|
316
326
|
self_contained: bool = True,
|
|
317
327
|
slug: str | bool | None = None,
|
|
318
|
-
|
|
328
|
+
include_private: bool = False,
|
|
329
|
+
*,
|
|
330
|
+
to: str | None = None,
|
|
331
|
+
allow_private: bool | None = None,
|
|
319
332
|
) -> str | list[str]:
|
|
320
333
|
"""Upload a file or folder to the project's storage bucket.
|
|
321
334
|
|
|
@@ -326,8 +339,13 @@ def publish(
|
|
|
326
339
|
overwrite: Accepted for parity with R; uploads always replace.
|
|
327
340
|
self_contained: For Framework Cloud publishing.
|
|
328
341
|
slug: For Framework Cloud publishing.
|
|
329
|
-
|
|
330
|
-
|
|
342
|
+
include_private: Upload files with "private" in their path without asking. Only a
|
|
343
|
+
person who has reviewed the files should set this.
|
|
344
|
+
to: "github" publishes to the project's GitHub Pages site: the page is copied to
|
|
345
|
+
``published/<slug>/`` at the top of the repository, only that folder is
|
|
346
|
+
committed, and the commit is pushed. Pages sites are public, even from a private
|
|
347
|
+
repository. Default: a storage bucket or Framework Cloud.
|
|
348
|
+
allow_private: Deprecated name for ``include_private``.
|
|
331
349
|
|
|
332
350
|
Returns:
|
|
333
351
|
The public URL, or a list of URLs for a folder.
|
|
@@ -335,11 +353,16 @@ def publish(
|
|
|
335
353
|
path = Path(source)
|
|
336
354
|
if not path.exists():
|
|
337
355
|
raise FrameworkError(f"Source not found: {path}")
|
|
338
|
-
|
|
356
|
+
include = include_private_arg(include_private, allow_private)
|
|
357
|
+
confirm_private_upload(_files(path, True, None) if path.is_dir() else [path], include)
|
|
358
|
+
if to is not None:
|
|
359
|
+
return _publish_github(to, path, dest, connection, slug, self_contained)
|
|
339
360
|
if _target(connection) == "cloud":
|
|
340
361
|
from framework.cloud import cloud_publish
|
|
341
362
|
|
|
342
|
-
return cloud_publish(
|
|
363
|
+
return cloud_publish(
|
|
364
|
+
path, dest=dest, self_contained=self_contained, slug=slug, include_private=True
|
|
365
|
+
)
|
|
343
366
|
_slug_ignored(slug)
|
|
344
367
|
bucket = resolve_bucket(connection)
|
|
345
368
|
dest = dest or path.name
|
|
@@ -359,7 +382,10 @@ def publish_notebook(
|
|
|
359
382
|
self_contained: bool = True,
|
|
360
383
|
format: str = "html",
|
|
361
384
|
slug: str | bool | None = None,
|
|
362
|
-
|
|
385
|
+
include_private: bool = False,
|
|
386
|
+
*,
|
|
387
|
+
to: str | None = None,
|
|
388
|
+
allow_private: bool | None = None,
|
|
363
389
|
) -> str:
|
|
364
390
|
"""Render a Quarto notebook with the ``quarto`` command and upload the result.
|
|
365
391
|
|
|
@@ -385,20 +411,30 @@ def publish_notebook(
|
|
|
385
411
|
slug: For Framework Cloud: a string names the document and turns on its readable URL,
|
|
386
412
|
True uses the derived name, and False turns the readable URL off. A storage bucket
|
|
387
413
|
ignores it with a warning.
|
|
388
|
-
|
|
389
|
-
|
|
414
|
+
include_private: Upload files with "private" in their path without asking. Only a
|
|
415
|
+
person who has reviewed the files should set this.
|
|
416
|
+
to: "github" publishes to the project's GitHub Pages site: the page is copied to
|
|
417
|
+
``published/<slug>/`` at the top of the repository, only that folder is
|
|
418
|
+
committed, and the commit is pushed. Pages sites are public, even from a private
|
|
419
|
+
repository. Default: a storage bucket or Framework Cloud.
|
|
420
|
+
allow_private: Deprecated name for ``include_private``.
|
|
390
421
|
|
|
391
422
|
Returns:
|
|
392
423
|
The public URL.
|
|
393
424
|
|
|
394
425
|
Raises:
|
|
395
|
-
FrameworkError: When the file is missing or not a notebook,
|
|
396
|
-
|
|
426
|
+
FrameworkError: When the file is missing or not a notebook, has "private" in its path
|
|
427
|
+
and is not confirmed, Quarto is not installed, rendering fails, or there is nowhere
|
|
428
|
+
to publish.
|
|
397
429
|
"""
|
|
398
430
|
source = Path(file)
|
|
399
431
|
if not source.is_file() or source.suffix.lower() not in (".qmd", ".ipynb"):
|
|
400
432
|
raise FrameworkError(f"publish_notebook() needs an existing .qmd or .ipynb file: {source}")
|
|
401
|
-
|
|
433
|
+
confirm_private_upload([source], include_private_arg(include_private, allow_private))
|
|
434
|
+
if to is not None:
|
|
435
|
+
if format != "html":
|
|
436
|
+
raise FrameworkError(f'GitHub Pages publishes HTML only (got format="{format}").')
|
|
437
|
+
return _publish_github(to, source, dest, connection, slug, self_contained)
|
|
402
438
|
quarto = shutil.which("quarto")
|
|
403
439
|
if quarto is None:
|
|
404
440
|
raise FrameworkError(
|
|
@@ -409,7 +445,9 @@ def publish_notebook(
|
|
|
409
445
|
raise FrameworkError(f'Framework Cloud publishes HTML only (got format="{format}").')
|
|
410
446
|
from framework.cloud import cloud_publish
|
|
411
447
|
|
|
412
|
-
return cloud_publish(
|
|
448
|
+
return cloud_publish(
|
|
449
|
+
source, dest=dest, self_contained=self_contained, slug=slug, include_private=True
|
|
450
|
+
)
|
|
413
451
|
_slug_ignored(slug)
|
|
414
452
|
bucket = resolve_bucket(connection)
|
|
415
453
|
self_contained = self_contained or not bucket.static_hosting
|
|
@@ -449,7 +487,9 @@ def publish_data(
|
|
|
449
487
|
format: str = "csv",
|
|
450
488
|
connection: str | None = None,
|
|
451
489
|
compress: bool = False,
|
|
452
|
-
|
|
490
|
+
include_private: bool = False,
|
|
491
|
+
*,
|
|
492
|
+
allow_private: bool | None = None,
|
|
453
493
|
) -> str:
|
|
454
494
|
"""Upload a DataFrame (as CSV, Parquet, or JSON) or a data file.
|
|
455
495
|
|
|
@@ -463,18 +503,20 @@ def publish_data(
|
|
|
463
503
|
connection: A storage bucket name from the settings. Default: the project's default
|
|
464
504
|
bucket, then one marked ``default: true``, then the first.
|
|
465
505
|
compress: Gzip the file before uploading.
|
|
466
|
-
|
|
467
|
-
|
|
506
|
+
include_private: Upload files with "private" in their path without asking. Only a
|
|
507
|
+
person who has reviewed the files should set this.
|
|
508
|
+
allow_private: Deprecated name for ``include_private``.
|
|
468
509
|
|
|
469
510
|
Returns:
|
|
470
511
|
The public URL.
|
|
471
512
|
|
|
472
513
|
Raises:
|
|
473
|
-
FrameworkError: When ``format`` is not allowed, the file is missing or
|
|
474
|
-
|
|
514
|
+
FrameworkError: When ``format`` is not allowed, the file is missing or has "private"
|
|
515
|
+
in its path and is not confirmed, no bucket is configured, or the upload fails.
|
|
475
516
|
"""
|
|
476
517
|
if format not in ("csv", "parquet", "json"):
|
|
477
518
|
raise FrameworkError("format must be csv, parquet, or json (rds is R-only)")
|
|
519
|
+
include = include_private_arg(include_private, allow_private)
|
|
478
520
|
bucket = resolve_bucket(connection)
|
|
479
521
|
with tempfile.TemporaryDirectory(prefix="publish_data_") as temp:
|
|
480
522
|
if isinstance(data, pd.DataFrame):
|
|
@@ -489,7 +531,7 @@ def publish_data(
|
|
|
489
531
|
source = Path(data)
|
|
490
532
|
if not source.is_file():
|
|
491
533
|
raise FrameworkError(f"File not found: {source}")
|
|
492
|
-
|
|
534
|
+
confirm_private_upload([source], include)
|
|
493
535
|
if compress:
|
|
494
536
|
compressed = Path(temp) / f"{source.name}.gz"
|
|
495
537
|
with source.open("rb") as raw, gzip.open(compressed, "wb") as packed:
|
|
@@ -507,7 +549,9 @@ def publish_dir(
|
|
|
507
549
|
connection: str | None = None,
|
|
508
550
|
pattern: str | None = None,
|
|
509
551
|
recursive: bool = True,
|
|
510
|
-
|
|
552
|
+
include_private: bool = False,
|
|
553
|
+
*,
|
|
554
|
+
allow_private: bool | None = None,
|
|
511
555
|
) -> list[str]:
|
|
512
556
|
"""Upload the files in a folder. ``pattern`` is a regular expression on the paths.
|
|
513
557
|
|
|
@@ -520,25 +564,27 @@ def publish_dir(
|
|
|
520
564
|
pattern: Only upload files whose path matches this regular expression (searched
|
|
521
565
|
anywhere in the path). Default: every file.
|
|
522
566
|
recursive: Include files in subfolders.
|
|
523
|
-
|
|
524
|
-
|
|
567
|
+
include_private: Upload files with "private" in their path without asking. Only a
|
|
568
|
+
person who has reviewed the files should set this.
|
|
569
|
+
allow_private: Deprecated name for ``include_private``.
|
|
525
570
|
|
|
526
571
|
Returns:
|
|
527
572
|
The public URL of each file uploaded. The list is empty when no file matched.
|
|
528
573
|
|
|
529
574
|
Raises:
|
|
530
|
-
FrameworkError: When the folder is missing
|
|
531
|
-
configured, or an upload fails.
|
|
575
|
+
FrameworkError: When the folder is missing, a file has "private" in its path and is
|
|
576
|
+
not confirmed, no bucket is configured, or an upload fails.
|
|
532
577
|
"""
|
|
533
578
|
folder = Path(dir)
|
|
534
579
|
if not folder.is_dir():
|
|
535
580
|
raise FrameworkError(f"Directory not found: {folder}")
|
|
536
|
-
|
|
537
|
-
bucket = resolve_bucket(connection)
|
|
581
|
+
include = include_private_arg(include_private, allow_private)
|
|
538
582
|
files = _files(folder, recursive, pattern)
|
|
539
583
|
if not files:
|
|
540
584
|
inform("No files found to upload")
|
|
541
585
|
return []
|
|
586
|
+
confirm_private_upload(files, include)
|
|
587
|
+
bucket = resolve_bucket(connection)
|
|
542
588
|
dest = dest or folder.resolve().name
|
|
543
589
|
inform(f"Uploading {len(files)} files...")
|
|
544
590
|
urls = _upload_folder(folder, dest, bucket, files)
|
framework/templates/defaults.yml
CHANGED
|
@@ -61,8 +61,8 @@ fw.publish("outputs/report.html")
|
|
|
61
61
|
fw.publish_notebook("notebooks/analysis.qmd")
|
|
62
62
|
```
|
|
63
63
|
|
|
64
|
-
The publish functions
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
reviewed the file can decide that it is safe to publish. See
|
|
68
|
-
`framework-sensitive-data` skill if this project has one.
|
|
64
|
+
The publish functions warn about any file with "private" in its path and ask
|
|
65
|
+
the user to confirm. In a script they stop with an error. Never pass
|
|
66
|
+
`include_private=True` or answer the prompt on the user's behalf: only the
|
|
67
|
+
person who has reviewed the file can decide that it is safe to publish. See
|
|
68
|
+
the `framework-sensitive-data` skill if this project has one.
|
|
@@ -15,8 +15,9 @@ whether something is safe, treat it as private.
|
|
|
15
15
|
2. Raw data with identifiers goes in a `private/` folder.
|
|
16
16
|
3. Only de-identified, aggregated data goes in a `public/` folder.
|
|
17
17
|
4. Review every output before saving it to a public folder.
|
|
18
|
-
5. Never publish anything from a private folder
|
|
19
|
-
`
|
|
18
|
+
5. Never publish anything from a private folder. Never pass
|
|
19
|
+
`include_private=True` to a publish function or answer its prompt on the
|
|
20
|
+
user's behalf.
|
|
20
21
|
6. Do not read or print restricted files. Values you print are sent to the
|
|
21
22
|
model provider. Work from the data dictionary and simulated data instead.
|
|
22
23
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: framework-py
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev4
|
|
4
4
|
Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
|
|
5
5
|
Project-URL: Homepage, https://framework.pub
|
|
6
6
|
Project-URL: Source, https://github.com/frameworktools/framework-py
|