framework-py 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/_pages.py +228 -0
- framework/_private.py +69 -0
- framework/cloud.py +11 -1
- framework/storage.py +96 -50
- framework/templates/skills/framework-outputs/SKILL.md +5 -5
- framework/templates/skills/framework-sensitive-data/SKILL.md +3 -2
- {framework_py-0.1.0.dev2.dist-info → framework_py-0.1.0.dev3.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev2.dist-info → framework_py-0.1.0.dev3.dist-info}/RECORD +10 -8
- {framework_py-0.1.0.dev2.dist-info → framework_py-0.1.0.dev3.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev2.dist-info → framework_py-0.1.0.dev3.dist-info}/licenses/LICENSE +0 -0
framework/_pages.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""Publishing to GitHub Pages: ``publish(..., to="github")``.
|
|
2
|
+
|
|
3
|
+
Framework Cloud's "Publish with GitHub Pages" commits a workflow that deploys
|
|
4
|
+
the repository's ``published/`` folder from the default branch. This side
|
|
5
|
+
copies a document into ``published/<slug>/``, commits only that folder, and
|
|
6
|
+
pushes. Documents never pass through the cloud. R and fw behave the same way.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import tempfile
|
|
16
|
+
import warnings
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from framework._log import inform
|
|
21
|
+
from framework.errors import FrameworkError, FrameworkWarning
|
|
22
|
+
|
|
23
|
+
PUBLIC_NOTICE = "GitHub Pages sites are public, even when the repository is private."
|
|
24
|
+
_SLUG = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
|
|
25
|
+
_REMOTE = re.compile(r"github\.com[:/]+([^/]+)/([^/]+?)(?:\.git)?/?$")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class _Remote:
|
|
30
|
+
name: str
|
|
31
|
+
owner: str
|
|
32
|
+
repo: str
|
|
33
|
+
has_upstream: bool
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _git(top: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
|
37
|
+
return subprocess.run(
|
|
38
|
+
["git", "-C", str(top), *args], capture_output=True, text=True, check=False
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def github_remote(url: str) -> tuple[str, str] | None:
|
|
43
|
+
"""The owner and repository of a GitHub remote URL (https, ssh, or scp-style)."""
|
|
44
|
+
match = _REMOTE.search(url.strip())
|
|
45
|
+
return (match[1], match[2]) if match else None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def pages_base(owner: str, repo: str) -> str:
|
|
49
|
+
"""The URL GitHub gives a repository's Pages site."""
|
|
50
|
+
host = f"{owner.lower()}.github.io"
|
|
51
|
+
if repo.lower() == host:
|
|
52
|
+
return f"https://{host}/"
|
|
53
|
+
return f"https://{host}/{repo}/"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def pages_slug(source: Path) -> str:
|
|
57
|
+
"""The file name lowercased, with each run of other characters as "-".
|
|
58
|
+
|
|
59
|
+
GitHub Pages URLs are case-sensitive. "Final Report (v2).html" becomes
|
|
60
|
+
"final-report-v2".
|
|
61
|
+
"""
|
|
62
|
+
name = source.name if source.is_dir() else source.stem
|
|
63
|
+
return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _remote(top: Path) -> _Remote | None:
|
|
67
|
+
"""The branch's upstream remote, then origin, then any GitHub remote."""
|
|
68
|
+
upstream = _git(top, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}")
|
|
69
|
+
upstream_name = upstream.stdout.strip().split("/")[0] if upstream.returncode == 0 else None
|
|
70
|
+
listed = _git(top, "remote")
|
|
71
|
+
names = listed.stdout.split() if listed.returncode == 0 else []
|
|
72
|
+
candidates = [n for n in (upstream_name, "origin", *names) if n is not None]
|
|
73
|
+
for name in dict.fromkeys(candidates):
|
|
74
|
+
if name not in names:
|
|
75
|
+
continue
|
|
76
|
+
url = _git(top, "remote", "get-url", name)
|
|
77
|
+
found = github_remote(url.stdout) if url.returncode == 0 else None
|
|
78
|
+
if found is not None:
|
|
79
|
+
return _Remote(name, found[0], found[1], name == upstream_name)
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _stage(source: Path, stage: Path, self_contained: bool) -> None:
|
|
84
|
+
"""Copy ``source`` into ``stage`` as a folder whose front page is index.html."""
|
|
85
|
+
suffix = source.suffix.lower()
|
|
86
|
+
if source.is_dir():
|
|
87
|
+
if not (source / "index.html").is_file():
|
|
88
|
+
raise FrameworkError(
|
|
89
|
+
f"A folder published to GitHub Pages needs an index.html at its top: {source}"
|
|
90
|
+
)
|
|
91
|
+
shutil.copytree(source, stage, dirs_exist_ok=True)
|
|
92
|
+
return
|
|
93
|
+
if suffix in (".html", ".htm"):
|
|
94
|
+
shutil.copy2(source, stage / source.name)
|
|
95
|
+
assets = source.with_name(f"{source.stem}_files")
|
|
96
|
+
if assets.is_dir():
|
|
97
|
+
# The supporting folder keeps its name, so relative links still resolve
|
|
98
|
+
shutil.copytree(assets, stage / assets.name)
|
|
99
|
+
elif suffix in (".qmd", ".ipynb"):
|
|
100
|
+
quarto = shutil.which("quarto")
|
|
101
|
+
if quarto is None:
|
|
102
|
+
raise FrameworkError("Rendering requires the quarto command on your PATH.")
|
|
103
|
+
command = [quarto, "render", str(source.resolve()), "--output-dir", str(stage)]
|
|
104
|
+
command += ["--to", "html", *(["--embed-resources"] if self_contained else [])]
|
|
105
|
+
inform(f"Rendering {source.name}...")
|
|
106
|
+
result = subprocess.run(command, capture_output=True, text=True, check=False)
|
|
107
|
+
if result.returncode != 0:
|
|
108
|
+
raise FrameworkError(f"Quarto render failed:\n{result.stdout}{result.stderr}")
|
|
109
|
+
else:
|
|
110
|
+
raise FrameworkError(
|
|
111
|
+
"GitHub Pages publishing takes rendered HTML, a render folder, or a .qmd or "
|
|
112
|
+
f".ipynb to render. Got: {suffix or source.name}"
|
|
113
|
+
)
|
|
114
|
+
if not (stage / "index.html").is_file():
|
|
115
|
+
main = stage / f"{source.stem}.html"
|
|
116
|
+
if not main.is_file():
|
|
117
|
+
pages = sorted(stage.glob("*.htm*"))
|
|
118
|
+
if len(pages) != 1:
|
|
119
|
+
raise FrameworkError(f"No single HTML page found to publish from {source}")
|
|
120
|
+
main = pages[0]
|
|
121
|
+
main.rename(stage / "index.html")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _cloud_pages_url(root: Path) -> tuple[bool, str | None, str | None]:
|
|
125
|
+
"""Whether the cloud answered, its Pages URL (None when Pages is off), and the server.
|
|
126
|
+
|
|
127
|
+
One call to ``/api/v1/me`` with the project token. Not known when there is
|
|
128
|
+
no project token or the call fails.
|
|
129
|
+
"""
|
|
130
|
+
from framework.cloud import CloudError, api, cloud_server, project_token
|
|
131
|
+
|
|
132
|
+
token = project_token(root)
|
|
133
|
+
if token is None:
|
|
134
|
+
return False, None, None
|
|
135
|
+
try:
|
|
136
|
+
server = cloud_server(scope="project", project_dir=root).url
|
|
137
|
+
me = api("/api/v1/me", token, server)
|
|
138
|
+
except CloudError:
|
|
139
|
+
return False, None, None
|
|
140
|
+
if not isinstance(me, dict) or me.get("kind") != "project":
|
|
141
|
+
return False, None, None
|
|
142
|
+
pages = me.get("pages_url")
|
|
143
|
+
return True, pages if isinstance(pages, str) and pages else None, server
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _confirm() -> None:
|
|
147
|
+
# Shown on every run, even when Framework's messages are off
|
|
148
|
+
print(PUBLIC_NOTICE, file=sys.stderr)
|
|
149
|
+
if sys.stdin.isatty():
|
|
150
|
+
answer = input("Publish it? [y/N] ").strip().lower()
|
|
151
|
+
if answer not in ("y", "yes"):
|
|
152
|
+
raise FrameworkError("Cancelled: nothing was published.")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def github_publish(source: Path, slug: str | None = None, self_contained: bool = True) -> str:
|
|
156
|
+
"""Copy ``source`` to ``published/<slug>/``, commit only that folder, and push."""
|
|
157
|
+
from framework.project import find_settings_file
|
|
158
|
+
|
|
159
|
+
if slug is None:
|
|
160
|
+
slug = pages_slug(source)
|
|
161
|
+
if not _SLUG.match(slug):
|
|
162
|
+
raise FrameworkError(
|
|
163
|
+
"slug must start with a lowercase letter or digit and use only lowercase letters, "
|
|
164
|
+
"digits, '.', '_', and '-' (GitHub Pages URLs are case-sensitive)."
|
|
165
|
+
)
|
|
166
|
+
settings_file = find_settings_file()
|
|
167
|
+
root = settings_file.parent if settings_file is not None else Path.cwd()
|
|
168
|
+
shown = _git(root, "rev-parse", "--show-toplevel")
|
|
169
|
+
if shown.returncode != 0:
|
|
170
|
+
raise FrameworkError(
|
|
171
|
+
"This project isn't a git repository. Create the project on GitHub first."
|
|
172
|
+
)
|
|
173
|
+
top = Path(shown.stdout.strip())
|
|
174
|
+
remote = _remote(top)
|
|
175
|
+
if remote is None:
|
|
176
|
+
raise FrameworkError(
|
|
177
|
+
"This project has no GitHub remote. Create the project on GitHub first, then "
|
|
178
|
+
"publish again."
|
|
179
|
+
)
|
|
180
|
+
_confirm()
|
|
181
|
+
|
|
182
|
+
# published/ sits at the top of the repository, where the workflow deploys from
|
|
183
|
+
rel = f"published/{slug}"
|
|
184
|
+
with tempfile.TemporaryDirectory(prefix="fw_pages_") as temp:
|
|
185
|
+
_stage(source, Path(temp), self_contained)
|
|
186
|
+
target = top / rel
|
|
187
|
+
if target.exists():
|
|
188
|
+
shutil.rmtree(target)
|
|
189
|
+
shutil.copytree(temp, target)
|
|
190
|
+
|
|
191
|
+
# Commit only this folder; anything else the user staged stays staged
|
|
192
|
+
_git(top, "add", "-A", "--", rel)
|
|
193
|
+
if _git(top, "diff", "--cached", "--quiet", "--", rel).returncode == 0:
|
|
194
|
+
inform(f"No change in {rel}; nothing committed or pushed.")
|
|
195
|
+
else:
|
|
196
|
+
commit = _git(top, "commit", "-m", f"Publish {slug}", "--", rel)
|
|
197
|
+
if commit.returncode != 0:
|
|
198
|
+
raise FrameworkError(f"git commit failed:\n{commit.stdout}{commit.stderr}")
|
|
199
|
+
branch = _git(top, "rev-parse", "--abbrev-ref", "HEAD").stdout.strip()
|
|
200
|
+
default = _git(top, "symbolic-ref", "--short", f"refs/remotes/{remote.name}/HEAD")
|
|
201
|
+
if default.returncode == 0:
|
|
202
|
+
default_branch = default.stdout.strip().split("/", 1)[-1]
|
|
203
|
+
if branch != default_branch:
|
|
204
|
+
warnings.warn(
|
|
205
|
+
f"You're on '{branch}'. GitHub Pages deploys only from the default branch "
|
|
206
|
+
f"('{default_branch}'), so this won't appear until it's merged there.",
|
|
207
|
+
FrameworkWarning,
|
|
208
|
+
stacklevel=3,
|
|
209
|
+
)
|
|
210
|
+
push = (
|
|
211
|
+
_git(top, "push")
|
|
212
|
+
if remote.has_upstream
|
|
213
|
+
else _git(top, "push", "-u", remote.name, "HEAD")
|
|
214
|
+
)
|
|
215
|
+
if push.returncode != 0:
|
|
216
|
+
raise FrameworkError(f"Committed, but git push failed:\n{push.stdout}{push.stderr}")
|
|
217
|
+
|
|
218
|
+
known, pages_url, server = _cloud_pages_url(root)
|
|
219
|
+
if known and pages_url is None:
|
|
220
|
+
warnings.warn(
|
|
221
|
+
f"GitHub Pages is off for this project. Turn it on under Projects at {server}.",
|
|
222
|
+
FrameworkWarning,
|
|
223
|
+
stacklevel=3,
|
|
224
|
+
)
|
|
225
|
+
base = pages_url or pages_base(remote.owner, remote.repo)
|
|
226
|
+
url = f"{base.rstrip('/')}/{slug}/"
|
|
227
|
+
inform(f"Published to {url}")
|
|
228
|
+
return url
|
framework/_private.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""The check before an upload: a path with "private" in it needs confirmation.
|
|
2
|
+
|
|
3
|
+
Uploads are always the user's explicit choice, so the check never refuses
|
|
4
|
+
outright. Any source whose full path contains "private", in any letter case,
|
|
5
|
+
is listed with a warning. An interactive session asks before uploading. A
|
|
6
|
+
session with no terminal (scripts, CI, notebooks run by agents) raises an
|
|
7
|
+
error unless ``include_private=True`` is passed.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import sys
|
|
13
|
+
import warnings
|
|
14
|
+
from collections.abc import Iterable
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from framework.errors import FrameworkError
|
|
18
|
+
|
|
19
|
+
# macOS resolves /tmp, /var, and /etc to /private/..., which the user did not name
|
|
20
|
+
_SYSTEM_PREFIX = "/private/"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def is_private_path(path: Path) -> bool:
|
|
24
|
+
"""Whether the full absolute path contains "private", ignoring the system prefix."""
|
|
25
|
+
full = path.expanduser().resolve().as_posix()
|
|
26
|
+
if full.startswith(_SYSTEM_PREFIX):
|
|
27
|
+
full = full[len(_SYSTEM_PREFIX) - 1 :]
|
|
28
|
+
return "private" in full.lower()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def include_private_arg(include_private: bool, allow_private: bool | None) -> bool:
|
|
32
|
+
"""``include_private``, honoring the deprecated ``allow_private`` name."""
|
|
33
|
+
if allow_private is None:
|
|
34
|
+
return include_private
|
|
35
|
+
warnings.warn(
|
|
36
|
+
"allow_private is deprecated; use include_private instead.",
|
|
37
|
+
DeprecationWarning,
|
|
38
|
+
stacklevel=3,
|
|
39
|
+
)
|
|
40
|
+
return include_private or allow_private
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def confirm_private_upload(paths: Iterable[Path], include_private: bool) -> None:
|
|
44
|
+
"""Warn about private paths among ``paths`` and ask before uploading them.
|
|
45
|
+
|
|
46
|
+
Raises:
|
|
47
|
+
FrameworkError: When there is no terminal to ask on and ``include_private`` is
|
|
48
|
+
False, or when the user declines.
|
|
49
|
+
"""
|
|
50
|
+
if include_private:
|
|
51
|
+
return
|
|
52
|
+
flagged = [p for p in paths if is_private_path(p)]
|
|
53
|
+
if not flagged:
|
|
54
|
+
return
|
|
55
|
+
shown = "".join(f"\n {p}" for p in flagged[:5])
|
|
56
|
+
if len(flagged) > 5:
|
|
57
|
+
shown += f"\n ... and {len(flagged) - 5} more"
|
|
58
|
+
print(
|
|
59
|
+
f"Warning: {len(flagged)} file(s) to upload have 'private' in their path:{shown}",
|
|
60
|
+
file=sys.stderr,
|
|
61
|
+
)
|
|
62
|
+
if not sys.stdin.isatty():
|
|
63
|
+
raise FrameworkError(
|
|
64
|
+
"Not uploading files with 'private' in their path without confirmation. "
|
|
65
|
+
"Review them, then pass include_private=True to upload them."
|
|
66
|
+
)
|
|
67
|
+
answer = input("Upload them anyway? [y/N] ").strip().lower()
|
|
68
|
+
if answer not in ("y", "yes"):
|
|
69
|
+
raise FrameworkError("Cancelled. Nothing was uploaded.")
|
framework/cloud.py
CHANGED
|
@@ -41,6 +41,7 @@ import pandas as pd
|
|
|
41
41
|
|
|
42
42
|
from framework import _secretservice, _wincred, _yaml, db, dotenv
|
|
43
43
|
from framework._log import inform, show_messages
|
|
44
|
+
from framework._private import confirm_private_upload
|
|
44
45
|
from framework.config_home import config_dir, ensure_config_dir, r_config_dir
|
|
45
46
|
from framework.errors import FrameworkError, FrameworkWarning
|
|
46
47
|
from framework.integrity import file_hash
|
|
@@ -803,14 +804,23 @@ def cloud_publish(
|
|
|
803
804
|
self_contained: bool = True,
|
|
804
805
|
slug: str | bool | None = None,
|
|
805
806
|
title: str | None = None,
|
|
807
|
+
include_private: bool = False,
|
|
806
808
|
) -> str:
|
|
807
809
|
"""Publish rendered HTML (or a .qmd or .ipynb to render) to the project's cloud server.
|
|
808
810
|
|
|
809
811
|
Accepts a self-contained ``.html``, an ``.html`` with a sibling
|
|
810
812
|
``<name>_files/`` folder, a render output folder, or a ``.qmd`` or ``.ipynb``. Each
|
|
811
813
|
publish adds a version. ``slug`` names the document and turns on its
|
|
812
|
-
readable URL (``True`` uses the derived name, ``False`` turns it off).
|
|
814
|
+
readable URL (``True`` uses the derived name, ``False`` turns it off). A source with
|
|
815
|
+
"private" in its path needs confirmation, or ``include_private=True``.
|
|
813
816
|
"""
|
|
817
|
+
sources = [source]
|
|
818
|
+
if source.is_dir():
|
|
819
|
+
sources = sorted(p for p in source.rglob("*") if p.is_file())
|
|
820
|
+
elif source.with_name(f"{source.stem}_files").is_dir():
|
|
821
|
+
assets = source.with_name(f"{source.stem}_files")
|
|
822
|
+
sources += sorted(p for p in assets.rglob("*") if p.is_file())
|
|
823
|
+
confirm_private_upload(sources, include_private)
|
|
814
824
|
token = _require_project_token()
|
|
815
825
|
with tempfile.TemporaryDirectory(prefix="fw_publish_") as temp:
|
|
816
826
|
stage = Path(temp)
|
framework/storage.py
CHANGED
|
@@ -12,8 +12,9 @@ Credentials come from the environment (``S3_ACCESS_KEY`` and
|
|
|
12
12
|
environment variables take precedence over values in the settings.
|
|
13
13
|
|
|
14
14
|
Publishing uploads files with a public-read ACL and returns their URLs.
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
A source with "private" anywhere in its full path is listed with a warning
|
|
16
|
+
and needs confirmation: an interactive session asks, and a session with no
|
|
17
|
+
terminal raises an error unless ``include_private=True`` is passed.
|
|
17
18
|
"""
|
|
18
19
|
|
|
19
20
|
from __future__ import annotations
|
|
@@ -33,6 +34,7 @@ from typing import Any
|
|
|
33
34
|
import pandas as pd
|
|
34
35
|
|
|
35
36
|
from framework._log import inform
|
|
37
|
+
from framework._private import confirm_private_upload, include_private_arg
|
|
36
38
|
from framework.errors import FrameworkError
|
|
37
39
|
from framework.settings import settings_read
|
|
38
40
|
|
|
@@ -208,29 +210,6 @@ def resolve_bucket(connection: str | None = None) -> Bucket:
|
|
|
208
210
|
# Uploading --------------------------------------------------------------------------
|
|
209
211
|
|
|
210
212
|
|
|
211
|
-
def _refuse_private(path: Path, allow_private: bool) -> None:
|
|
212
|
-
"""Refuse a source inside a folder named ``private`` within the project.
|
|
213
|
-
|
|
214
|
-
Folders above the project root do not count (macOS keeps temporary files
|
|
215
|
-
under ``/private``), and nothing is refused outside a project.
|
|
216
|
-
"""
|
|
217
|
-
from framework.project import find_settings_file
|
|
218
|
-
|
|
219
|
-
settings_file = find_settings_file()
|
|
220
|
-
if allow_private or settings_file is None:
|
|
221
|
-
return
|
|
222
|
-
root = settings_file.parent.resolve()
|
|
223
|
-
resolved = path.resolve()
|
|
224
|
-
if not resolved.is_relative_to(root):
|
|
225
|
-
return
|
|
226
|
-
relative = resolved.relative_to(root)
|
|
227
|
-
if "private" in relative.parts:
|
|
228
|
-
raise FrameworkError(
|
|
229
|
-
f"Refusing to publish '{relative.as_posix()}': it is inside a private folder, which "
|
|
230
|
-
"holds restricted data. Pass allow_private=True if you really mean to publish it."
|
|
231
|
-
)
|
|
232
|
-
|
|
233
|
-
|
|
234
213
|
def _content_type(path: Path) -> str:
|
|
235
214
|
return (
|
|
236
215
|
_CONTENT_TYPES.get(path.suffix.lower())
|
|
@@ -295,6 +274,37 @@ def _target(connection: str | None) -> str:
|
|
|
295
274
|
)
|
|
296
275
|
|
|
297
276
|
|
|
277
|
+
def _publish_github(
|
|
278
|
+
to: str,
|
|
279
|
+
source: Path,
|
|
280
|
+
dest: str | None,
|
|
281
|
+
connection: str | None,
|
|
282
|
+
slug: str | bool | None,
|
|
283
|
+
self_contained: bool,
|
|
284
|
+
) -> str:
|
|
285
|
+
"""``to="github"``, after rejecting arguments that mean something only elsewhere."""
|
|
286
|
+
from framework._pages import github_publish
|
|
287
|
+
|
|
288
|
+
if to != "github":
|
|
289
|
+
raise FrameworkError(f'to must be "github" or None (got "{to}").')
|
|
290
|
+
given = [
|
|
291
|
+
name
|
|
292
|
+
for name, used in (
|
|
293
|
+
("dest", dest is not None),
|
|
294
|
+
("connection", connection is not None),
|
|
295
|
+
("slug=True/False", isinstance(slug, bool)),
|
|
296
|
+
)
|
|
297
|
+
if used
|
|
298
|
+
]
|
|
299
|
+
if given:
|
|
300
|
+
raise FrameworkError(
|
|
301
|
+
f'Not used with to="github": {", ".join(given)}. Name the published folder with '
|
|
302
|
+
'slug="...".'
|
|
303
|
+
)
|
|
304
|
+
assert not isinstance(slug, bool)
|
|
305
|
+
return github_publish(source, slug=slug, self_contained=self_contained)
|
|
306
|
+
|
|
307
|
+
|
|
298
308
|
def _slug_ignored(slug: str | bool | None) -> None:
|
|
299
309
|
if slug is not None:
|
|
300
310
|
import warnings
|
|
@@ -315,7 +325,10 @@ def publish(
|
|
|
315
325
|
overwrite: bool = True,
|
|
316
326
|
self_contained: bool = True,
|
|
317
327
|
slug: str | bool | None = None,
|
|
318
|
-
|
|
328
|
+
include_private: bool = False,
|
|
329
|
+
*,
|
|
330
|
+
to: str | None = None,
|
|
331
|
+
allow_private: bool | None = None,
|
|
319
332
|
) -> str | list[str]:
|
|
320
333
|
"""Upload a file or folder to the project's storage bucket.
|
|
321
334
|
|
|
@@ -326,8 +339,13 @@ def publish(
|
|
|
326
339
|
overwrite: Accepted for parity with R; uploads always replace.
|
|
327
340
|
self_contained: For Framework Cloud publishing.
|
|
328
341
|
slug: For Framework Cloud publishing.
|
|
329
|
-
|
|
330
|
-
|
|
342
|
+
include_private: Upload files with "private" in their path without asking. Only a
|
|
343
|
+
person who has reviewed the files should set this.
|
|
344
|
+
to: "github" publishes to the project's GitHub Pages site: the page is copied to
|
|
345
|
+
``published/<slug>/`` at the top of the repository, only that folder is
|
|
346
|
+
committed, and the commit is pushed. Pages sites are public, even from a private
|
|
347
|
+
repository. Default: a storage bucket or Framework Cloud.
|
|
348
|
+
allow_private: Deprecated name for ``include_private``.
|
|
331
349
|
|
|
332
350
|
Returns:
|
|
333
351
|
The public URL, or a list of URLs for a folder.
|
|
@@ -335,11 +353,16 @@ def publish(
|
|
|
335
353
|
path = Path(source)
|
|
336
354
|
if not path.exists():
|
|
337
355
|
raise FrameworkError(f"Source not found: {path}")
|
|
338
|
-
|
|
356
|
+
include = include_private_arg(include_private, allow_private)
|
|
357
|
+
confirm_private_upload(_files(path, True, None) if path.is_dir() else [path], include)
|
|
358
|
+
if to is not None:
|
|
359
|
+
return _publish_github(to, path, dest, connection, slug, self_contained)
|
|
339
360
|
if _target(connection) == "cloud":
|
|
340
361
|
from framework.cloud import cloud_publish
|
|
341
362
|
|
|
342
|
-
return cloud_publish(
|
|
363
|
+
return cloud_publish(
|
|
364
|
+
path, dest=dest, self_contained=self_contained, slug=slug, include_private=True
|
|
365
|
+
)
|
|
343
366
|
_slug_ignored(slug)
|
|
344
367
|
bucket = resolve_bucket(connection)
|
|
345
368
|
dest = dest or path.name
|
|
@@ -359,7 +382,10 @@ def publish_notebook(
|
|
|
359
382
|
self_contained: bool = True,
|
|
360
383
|
format: str = "html",
|
|
361
384
|
slug: str | bool | None = None,
|
|
362
|
-
|
|
385
|
+
include_private: bool = False,
|
|
386
|
+
*,
|
|
387
|
+
to: str | None = None,
|
|
388
|
+
allow_private: bool | None = None,
|
|
363
389
|
) -> str:
|
|
364
390
|
"""Render a Quarto notebook with the ``quarto`` command and upload the result.
|
|
365
391
|
|
|
@@ -385,20 +411,30 @@ def publish_notebook(
|
|
|
385
411
|
slug: For Framework Cloud: a string names the document and turns on its readable URL,
|
|
386
412
|
True uses the derived name, and False turns the readable URL off. A storage bucket
|
|
387
413
|
ignores it with a warning.
|
|
388
|
-
|
|
389
|
-
|
|
414
|
+
include_private: Upload files with "private" in their path without asking. Only a
|
|
415
|
+
person who has reviewed the files should set this.
|
|
416
|
+
to: "github" publishes to the project's GitHub Pages site: the page is copied to
|
|
417
|
+
``published/<slug>/`` at the top of the repository, only that folder is
|
|
418
|
+
committed, and the commit is pushed. Pages sites are public, even from a private
|
|
419
|
+
repository. Default: a storage bucket or Framework Cloud.
|
|
420
|
+
allow_private: Deprecated name for ``include_private``.
|
|
390
421
|
|
|
391
422
|
Returns:
|
|
392
423
|
The public URL.
|
|
393
424
|
|
|
394
425
|
Raises:
|
|
395
|
-
FrameworkError: When the file is missing or not a notebook,
|
|
396
|
-
|
|
426
|
+
FrameworkError: When the file is missing or not a notebook, has "private" in its path
|
|
427
|
+
and is not confirmed, Quarto is not installed, rendering fails, or there is nowhere
|
|
428
|
+
to publish.
|
|
397
429
|
"""
|
|
398
430
|
source = Path(file)
|
|
399
431
|
if not source.is_file() or source.suffix.lower() not in (".qmd", ".ipynb"):
|
|
400
432
|
raise FrameworkError(f"publish_notebook() needs an existing .qmd or .ipynb file: {source}")
|
|
401
|
-
|
|
433
|
+
confirm_private_upload([source], include_private_arg(include_private, allow_private))
|
|
434
|
+
if to is not None:
|
|
435
|
+
if format != "html":
|
|
436
|
+
raise FrameworkError(f'GitHub Pages publishes HTML only (got format="{format}").')
|
|
437
|
+
return _publish_github(to, source, dest, connection, slug, self_contained)
|
|
402
438
|
quarto = shutil.which("quarto")
|
|
403
439
|
if quarto is None:
|
|
404
440
|
raise FrameworkError(
|
|
@@ -409,7 +445,9 @@ def publish_notebook(
|
|
|
409
445
|
raise FrameworkError(f'Framework Cloud publishes HTML only (got format="{format}").')
|
|
410
446
|
from framework.cloud import cloud_publish
|
|
411
447
|
|
|
412
|
-
return cloud_publish(
|
|
448
|
+
return cloud_publish(
|
|
449
|
+
source, dest=dest, self_contained=self_contained, slug=slug, include_private=True
|
|
450
|
+
)
|
|
413
451
|
_slug_ignored(slug)
|
|
414
452
|
bucket = resolve_bucket(connection)
|
|
415
453
|
self_contained = self_contained or not bucket.static_hosting
|
|
@@ -449,7 +487,9 @@ def publish_data(
|
|
|
449
487
|
format: str = "csv",
|
|
450
488
|
connection: str | None = None,
|
|
451
489
|
compress: bool = False,
|
|
452
|
-
|
|
490
|
+
include_private: bool = False,
|
|
491
|
+
*,
|
|
492
|
+
allow_private: bool | None = None,
|
|
453
493
|
) -> str:
|
|
454
494
|
"""Upload a DataFrame (as CSV, Parquet, or JSON) or a data file.
|
|
455
495
|
|
|
@@ -463,18 +503,20 @@ def publish_data(
|
|
|
463
503
|
connection: A storage bucket name from the settings. Default: the project's default
|
|
464
504
|
bucket, then one marked ``default: true``, then the first.
|
|
465
505
|
compress: Gzip the file before uploading.
|
|
466
|
-
|
|
467
|
-
|
|
506
|
+
include_private: Upload files with "private" in their path without asking. Only a
|
|
507
|
+
person who has reviewed the files should set this.
|
|
508
|
+
allow_private: Deprecated name for ``include_private``.
|
|
468
509
|
|
|
469
510
|
Returns:
|
|
470
511
|
The public URL.
|
|
471
512
|
|
|
472
513
|
Raises:
|
|
473
|
-
FrameworkError: When ``format`` is not allowed, the file is missing or
|
|
474
|
-
|
|
514
|
+
FrameworkError: When ``format`` is not allowed, the file is missing or has "private"
|
|
515
|
+
in its path and is not confirmed, no bucket is configured, or the upload fails.
|
|
475
516
|
"""
|
|
476
517
|
if format not in ("csv", "parquet", "json"):
|
|
477
518
|
raise FrameworkError("format must be csv, parquet, or json (rds is R-only)")
|
|
519
|
+
include = include_private_arg(include_private, allow_private)
|
|
478
520
|
bucket = resolve_bucket(connection)
|
|
479
521
|
with tempfile.TemporaryDirectory(prefix="publish_data_") as temp:
|
|
480
522
|
if isinstance(data, pd.DataFrame):
|
|
@@ -489,7 +531,7 @@ def publish_data(
|
|
|
489
531
|
source = Path(data)
|
|
490
532
|
if not source.is_file():
|
|
491
533
|
raise FrameworkError(f"File not found: {source}")
|
|
492
|
-
|
|
534
|
+
confirm_private_upload([source], include)
|
|
493
535
|
if compress:
|
|
494
536
|
compressed = Path(temp) / f"{source.name}.gz"
|
|
495
537
|
with source.open("rb") as raw, gzip.open(compressed, "wb") as packed:
|
|
@@ -507,7 +549,9 @@ def publish_dir(
|
|
|
507
549
|
connection: str | None = None,
|
|
508
550
|
pattern: str | None = None,
|
|
509
551
|
recursive: bool = True,
|
|
510
|
-
|
|
552
|
+
include_private: bool = False,
|
|
553
|
+
*,
|
|
554
|
+
allow_private: bool | None = None,
|
|
511
555
|
) -> list[str]:
|
|
512
556
|
"""Upload the files in a folder. ``pattern`` is a regular expression on the paths.
|
|
513
557
|
|
|
@@ -520,25 +564,27 @@ def publish_dir(
|
|
|
520
564
|
pattern: Only upload files whose path matches this regular expression (searched
|
|
521
565
|
anywhere in the path). Default: every file.
|
|
522
566
|
recursive: Include files in subfolders.
|
|
523
|
-
|
|
524
|
-
|
|
567
|
+
include_private: Upload files with "private" in their path without asking. Only a
|
|
568
|
+
person who has reviewed the files should set this.
|
|
569
|
+
allow_private: Deprecated name for ``include_private``.
|
|
525
570
|
|
|
526
571
|
Returns:
|
|
527
572
|
The public URL of each file uploaded. The list is empty when no file matched.
|
|
528
573
|
|
|
529
574
|
Raises:
|
|
530
|
-
FrameworkError: When the folder is missing
|
|
531
|
-
configured, or an upload fails.
|
|
575
|
+
FrameworkError: When the folder is missing, a file has "private" in its path and is
|
|
576
|
+
not confirmed, no bucket is configured, or an upload fails.
|
|
532
577
|
"""
|
|
533
578
|
folder = Path(dir)
|
|
534
579
|
if not folder.is_dir():
|
|
535
580
|
raise FrameworkError(f"Directory not found: {folder}")
|
|
536
|
-
|
|
537
|
-
bucket = resolve_bucket(connection)
|
|
581
|
+
include = include_private_arg(include_private, allow_private)
|
|
538
582
|
files = _files(folder, recursive, pattern)
|
|
539
583
|
if not files:
|
|
540
584
|
inform("No files found to upload")
|
|
541
585
|
return []
|
|
586
|
+
confirm_private_upload(files, include)
|
|
587
|
+
bucket = resolve_bucket(connection)
|
|
542
588
|
dest = dest or folder.resolve().name
|
|
543
589
|
inform(f"Uploading {len(files)} files...")
|
|
544
590
|
urls = _upload_folder(folder, dest, bucket, files)
|
|
@@ -61,8 +61,8 @@ fw.publish("outputs/report.html")
|
|
|
61
61
|
fw.publish_notebook("notebooks/analysis.qmd")
|
|
62
62
|
```
|
|
63
63
|
|
|
64
|
-
The publish functions
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
reviewed the file can decide that it is safe to publish. See
|
|
68
|
-
`framework-sensitive-data` skill if this project has one.
|
|
64
|
+
The publish functions warn about any file with "private" in its path and ask
|
|
65
|
+
the user to confirm. In a script they stop with an error. Never pass
|
|
66
|
+
`include_private=True` or answer the prompt on the user's behalf: only the
|
|
67
|
+
person who has reviewed the file can decide that it is safe to publish. See
|
|
68
|
+
the `framework-sensitive-data` skill if this project has one.
|
|
@@ -15,8 +15,9 @@ whether something is safe, treat it as private.
|
|
|
15
15
|
2. Raw data with identifiers goes in a `private/` folder.
|
|
16
16
|
3. Only de-identified, aggregated data goes in a `public/` folder.
|
|
17
17
|
4. Review every output before saving it to a public folder.
|
|
18
|
-
5. Never publish anything from a private folder
|
|
19
|
-
`
|
|
18
|
+
5. Never publish anything from a private folder. Never pass
|
|
19
|
+
`include_private=True` to a publish function or answer its prompt on the
|
|
20
|
+
user's behalf.
|
|
20
21
|
6. Do not read or print restricted files. Values you print are sent to the
|
|
21
22
|
model provider. Work from the data dictionary and simulated data instead.
|
|
22
23
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: framework-py
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev3
|
|
4
4
|
Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
|
|
5
5
|
Project-URL: Homepage, https://framework.pub
|
|
6
6
|
Project-URL: Source, https://github.com/frameworktools/framework-py
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
framework/__init__.py,sha256=Mq-Oq0FPzKPL7RbTGbwDaKV3do5DYLsBHkUSREabvoU,5652
|
|
2
2
|
framework/__main__.py,sha256=2YFKuWdA_X9z8nE0STKaTkzrvdc_cEdMxW_XM0h3GBo,3579
|
|
3
3
|
framework/_log.py,sha256=MhVL7izCJUvdg7F9430j8gPFiFyx3VcJ0qyJmrn2uUk,885
|
|
4
|
+
framework/_pages.py,sha256=Bm5b2NWr7H0I4t_kLtwFpaPaoPO2zGfsAuX8Z-95YTY,9250
|
|
4
5
|
framework/_paths.py,sha256=iAnUfmnnLDOUSbf7CGLXdW9SYWxchJxfVvJPbQkxyJk,2200
|
|
6
|
+
framework/_private.py,sha256=pZpvs1CSaUiSPCaBvqF13r2uwbXD8f5UFsCVu-9TzQY,2545
|
|
5
7
|
framework/_secretservice.py,sha256=zv5UvEekQic466qRmVpLG_hfEkisikHfab8e6kyzS2k,2450
|
|
6
8
|
framework/_time.py,sha256=mv-BS3NjOo0hr6bwYRhUBY-3sG6SJmzaboaMD62QcJw,306
|
|
7
9
|
framework/_wincred.py,sha256=oJGS37Dcw_PZVmiAGRu4H_lhcOC5-FqGGYfxVUGPPmU,3891
|
|
@@ -10,7 +12,7 @@ framework/ai.py,sha256=oKJOjHMUByqom8Ngk5dVlJj5ZbWJtFD7Mddvbqsh5qc,8114
|
|
|
10
12
|
framework/bootstrap.py,sha256=giXT20hQ-InqUlbtvOluqVMD4cpk5DcCrD5LqOFTARA,7913
|
|
11
13
|
framework/cache.py,sha256=I7LIPcxiT6j9N7ogJlNb0Nw99SNiMDcmed2xMzBFRBQ,12342
|
|
12
14
|
framework/catalog.py,sha256=7icsQYhdAqi56pN74Lc650L4FhqyB_jOd6d88NhpDV4,8632
|
|
13
|
-
framework/cloud.py,sha256=
|
|
15
|
+
framework/cloud.py,sha256=1_VW0jbvBOSzHrQktI2hko_YQ8lUqjXRanHWmHeC5_c,35475
|
|
14
16
|
framework/config_home.py,sha256=MHvKZ0vV0m2R1uPQmtMlfoim2XMhcLveJyBAyCpTKn8,7218
|
|
15
17
|
framework/connections.py,sha256=Y-pOaoKbd9CCNb31po9xtdutvq00s6jCP5miWhidqoI,17057
|
|
16
18
|
framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
|
|
@@ -32,7 +34,7 @@ framework/scaffold.py,sha256=MvyKbNStU-DJkIfNgXAVHoNmn-oAtYeC5gfYxe07ryk,10072
|
|
|
32
34
|
framework/scratch.py,sha256=idxthUR5BIMX7FIGS7KzIuhOrO3-HKLbBsV8It3EEE4,3890
|
|
33
35
|
framework/security.py,sha256=lPhAU_f3Yiq_yRAOroN_cMqeYvjKUanK9uls1BKDavg,24770
|
|
34
36
|
framework/settings.py,sha256=c0FNmOKZf7_E_HhEWSS0hus2ZC7stOAzDVDHqgZWkzc,18290
|
|
35
|
-
framework/storage.py,sha256=
|
|
37
|
+
framework/storage.py,sha256=7iE1Hx34Vs4oXheCQoeTFhJZPtv_B8YgWr9DlBxPkzA,25424
|
|
36
38
|
framework/_assets/init.sql,sha256=-aIiuIcCmBniGKq-CHdRtUiV9e_CLXoqxxd05snmgOE,1156
|
|
37
39
|
framework/_assets/hooks/pre-commit,sha256=wXkNetsub3sTi2LYewB-b5DRN8yoAd-M8pG9OEa0TPs,2280
|
|
38
40
|
framework/_assets/schema/docs-reference.v1.json,sha256=4RWr0daKXxXUbFeU8F-IBiPpKGGX_9pqTB_o3Lkwn18,5922
|
|
@@ -49,16 +51,16 @@ framework/templates/gitignore/presentation,sha256=CAl2vPcA1Lgv1CXsVmuKf7DtIVs-g6
|
|
|
49
51
|
framework/templates/gitignore/project,sha256=D8jFTFhIyJyTIh1xrPE5wFkvuJlSkqaqeJYafnmRPts,2622
|
|
50
52
|
framework/templates/gitignore/sensitive,sha256=mz0KwMHq1UJhOGJTFzTbzJug6C6p2wbec9ADl2veqhE,3550
|
|
51
53
|
framework/templates/skills/framework-data/SKILL.md,sha256=jdy9b0OgEWEvySgBTEVwEr6haSHYqRgCRRaT9Xd9zso,1802
|
|
52
|
-
framework/templates/skills/framework-outputs/SKILL.md,sha256=
|
|
54
|
+
framework/templates/skills/framework-outputs/SKILL.md,sha256=5ipWm8GBTmc0SL79I9HQ8ezaibpk1zwuHJbdp9Za5Tc,2419
|
|
53
55
|
framework/templates/skills/framework-packages/SKILL.md,sha256=hReYddtMTK9dzqLiGOrSxbE8YA6OFia_E8tqbkSgNxI,1170
|
|
54
|
-
framework/templates/skills/framework-sensitive-data/SKILL.md,sha256=
|
|
56
|
+
framework/templates/skills/framework-sensitive-data/SKILL.md,sha256=1Qid4542CV0sctzUaEXCtNxkl5l8JMuZs6hUwkmGTPc,1567
|
|
55
57
|
framework/templates/skills/framework-workflow/SKILL.md,sha256=-UIc81zJrO-0FfKlt2QK5Zfh0YzqbqNzBPskBpXe_1Q,2139
|
|
56
58
|
framework/templates/stubs/notebook-default.ipynb,sha256=2p3Zj6bQEHlUL1eEVAbPfTTxqGBs84C67tA3MkCzVQs,685
|
|
57
59
|
framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIjKzQeGEy2M2lQCVA1Y,472
|
|
58
60
|
framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
|
|
59
61
|
framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
|
|
60
62
|
framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
|
|
61
|
-
framework_py-0.1.0.
|
|
62
|
-
framework_py-0.1.0.
|
|
63
|
-
framework_py-0.1.0.
|
|
64
|
-
framework_py-0.1.0.
|
|
63
|
+
framework_py-0.1.0.dev3.dist-info/METADATA,sha256=x16APIzNkCGAurfJ06wWvrOT_OaL3YFaET2H9mvoZBM,3730
|
|
64
|
+
framework_py-0.1.0.dev3.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
65
|
+
framework_py-0.1.0.dev3.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
|
|
66
|
+
framework_py-0.1.0.dev3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|