framework-py 0.1.0.dev10__py3-none-any.whl → 0.1.0.dev12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +2 -1
- framework/_depends.py +39 -8
- framework/_private.py +93 -21
- framework/cache.py +5 -4
- framework/catalog.py +12 -9
- framework/cloud.py +1 -1
- framework/connections.py +65 -31
- framework/create.py +10 -5
- framework/outputs.py +5 -3
- framework/project.py +27 -12
- framework/publish_bundle.py +19 -12
- framework/storage.py +5 -5
- framework/templates/skills/framework-data/.framework-skill.json +1 -1
- framework/templates/skills/framework-data/SKILL.md +5 -2
- framework/templates/skills/framework-sensitive-data/.framework-skill.json +1 -1
- framework/templates/skills/framework-sensitive-data/SKILL.md +23 -2
- {framework_py-0.1.0.dev10.dist-info → framework_py-0.1.0.dev12.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev10.dist-info → framework_py-0.1.0.dev12.dist-info}/RECORD +20 -20
- {framework_py-0.1.0.dev10.dist-info → framework_py-0.1.0.dev12.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev10.dist-info → framework_py-0.1.0.dev12.dist-info}/licenses/LICENSE +0 -0
framework/__init__.py
CHANGED
|
@@ -121,7 +121,7 @@ from framework.packages import (
|
|
|
121
121
|
packages_status,
|
|
122
122
|
packages_update,
|
|
123
123
|
)
|
|
124
|
-
from framework.project import path, project_root
|
|
124
|
+
from framework.project import path, project_path, project_root
|
|
125
125
|
from framework.quarto import quarto_generate_all, quarto_regenerate
|
|
126
126
|
from framework.scaffold import project_info, scaffold, status
|
|
127
127
|
from framework.scratch import scratch_capture, scratch_clean
|
|
@@ -234,6 +234,7 @@ __all__ = [
|
|
|
234
234
|
"project_create_from_layers",
|
|
235
235
|
"project_info",
|
|
236
236
|
"project_list",
|
|
237
|
+
"project_path",
|
|
237
238
|
"project_register",
|
|
238
239
|
"project_root",
|
|
239
240
|
"project_unregister",
|
framework/_depends.py
CHANGED
|
@@ -24,7 +24,7 @@ import hashlib
|
|
|
24
24
|
import json
|
|
25
25
|
import os
|
|
26
26
|
import pickle
|
|
27
|
-
from dataclasses import
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
28
|
from pathlib import Path
|
|
29
29
|
from typing import Any
|
|
30
30
|
|
|
@@ -45,6 +45,20 @@ class Dependency:
|
|
|
45
45
|
kind: str
|
|
46
46
|
ref: str
|
|
47
47
|
hash: str
|
|
48
|
+
# A value's name in messages; not stored, since R and Python share the record
|
|
49
|
+
label: str = field(default="", compare=False)
|
|
50
|
+
|
|
51
|
+
def record(self) -> dict[str, str]:
|
|
52
|
+
return {"kind": self.kind, "ref": self.ref, "hash": self.hash}
|
|
53
|
+
|
|
54
|
+
def name(self) -> str:
|
|
55
|
+
"""What messages call this dependency: a dict entry's key, a reference's string,
|
|
56
|
+
or ``dependency <n> (a value)``."""
|
|
57
|
+
if self.label:
|
|
58
|
+
return self.label
|
|
59
|
+
if self.kind == "value":
|
|
60
|
+
return f"dependency {self.ref.removeprefix('value ')} (a value)"
|
|
61
|
+
return self.ref
|
|
48
62
|
|
|
49
63
|
|
|
50
64
|
def _sha256(data: bytes) -> str:
|
|
@@ -122,11 +136,18 @@ def resolve(depends_on: Any) -> list[Dependency]:
|
|
|
122
136
|
return [
|
|
123
137
|
_resolve_reference(item)
|
|
124
138
|
if isinstance(item, str)
|
|
125
|
-
else Dependency("value", f"value {position}", value_hash(item))
|
|
139
|
+
else Dependency("value", f"value {position}", value_hash(item), _value_label(item))
|
|
126
140
|
for position, item in enumerate(items, start=1)
|
|
127
141
|
]
|
|
128
142
|
|
|
129
143
|
|
|
144
|
+
def _value_label(value: Any) -> str:
|
|
145
|
+
"""A dict with one key is named by its key; other values by their position."""
|
|
146
|
+
if isinstance(value, dict) and len(value) == 1:
|
|
147
|
+
return str(next(iter(value)))
|
|
148
|
+
return ""
|
|
149
|
+
|
|
150
|
+
|
|
130
151
|
def key(dependencies: list[Dependency]) -> str:
|
|
131
152
|
lines = sorted(f"{d.kind}\t{d.ref}\t{d.hash}\n" for d in dependencies)
|
|
132
153
|
return _sha256("".join(lines).encode("utf-8"))
|
|
@@ -139,7 +160,10 @@ def recorded(name: str) -> tuple[str, list[Dependency]] | None:
|
|
|
139
160
|
return None
|
|
140
161
|
try:
|
|
141
162
|
data = json.loads(text)
|
|
142
|
-
dependencies = [
|
|
163
|
+
dependencies = [
|
|
164
|
+
Dependency(str(item["kind"]), str(item["ref"]), str(item["hash"]))
|
|
165
|
+
for item in data["dependencies"]
|
|
166
|
+
]
|
|
143
167
|
return str(data["key"]), dependencies
|
|
144
168
|
except (ValueError, KeyError, TypeError):
|
|
145
169
|
return None
|
|
@@ -149,7 +173,7 @@ def record(name: str, dependencies: list[Dependency] | None) -> None:
|
|
|
149
173
|
if dependencies is None:
|
|
150
174
|
forget(name)
|
|
151
175
|
return
|
|
152
|
-
document = {"key": key(dependencies), "dependencies": [
|
|
176
|
+
document = {"key": key(dependencies), "dependencies": [d.record() for d in dependencies]}
|
|
153
177
|
db.set_meta(META_PREFIX + name, json.dumps(document))
|
|
154
178
|
|
|
155
179
|
|
|
@@ -177,11 +201,11 @@ def recheck(dependencies: list[Dependency]) -> tuple[list[Dependency], list[str]
|
|
|
177
201
|
try:
|
|
178
202
|
fresh = _resolve_reference(dependency.ref)
|
|
179
203
|
except FrameworkError:
|
|
180
|
-
changed.append(dependency.
|
|
204
|
+
changed.append(dependency.name())
|
|
181
205
|
continue
|
|
182
206
|
current.append(fresh)
|
|
183
207
|
if fresh.hash != dependency.hash:
|
|
184
|
-
changed.append(dependency.
|
|
208
|
+
changed.append(dependency.name())
|
|
185
209
|
return current, changed
|
|
186
210
|
|
|
187
211
|
|
|
@@ -189,6 +213,13 @@ def changed_between(old: list[Dependency], new: list[Dependency]) -> list[str]:
|
|
|
189
213
|
"""Names for what differs between two dependency lists, for messages."""
|
|
190
214
|
before = {(d.kind, d.ref): d.hash for d in old}
|
|
191
215
|
after = {(d.kind, d.ref): d.hash for d in new}
|
|
192
|
-
names = [
|
|
193
|
-
names += [
|
|
216
|
+
names = [d.name() for d in new if before.get((d.kind, d.ref)) != d.hash]
|
|
217
|
+
names += [d.name() for d in old if (d.kind, d.ref) not in after]
|
|
194
218
|
return list(dict.fromkeys(names)) or ["the dependency list"]
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def listing(names: list[str]) -> str:
|
|
222
|
+
"""``a``, ``a and b``, or ``a, b and c``."""
|
|
223
|
+
if len(names) < 2:
|
|
224
|
+
return "".join(names)
|
|
225
|
+
return f"{', '.join(names[:-1])} and {names[-1]}"
|
framework/_private.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
"""The check before an upload: a
|
|
1
|
+
"""The check before an upload: a file in a private folder needs confirmation.
|
|
2
2
|
|
|
3
3
|
Uploads are always the user's explicit choice, so the check never refuses
|
|
4
|
-
outright. Any source
|
|
5
|
-
|
|
4
|
+
outright. Any source in a private folder (``is_private_path()``) is listed
|
|
5
|
+
with a warning. An interactive session asks before uploading. A
|
|
6
6
|
session with no terminal (scripts, CI, notebooks run by agents) raises an
|
|
7
7
|
error unless ``include_private=True`` is passed.
|
|
8
8
|
|
|
@@ -22,18 +22,63 @@ from collections.abc import Iterable, Sequence
|
|
|
22
22
|
from pathlib import Path
|
|
23
23
|
from typing import Any
|
|
24
24
|
|
|
25
|
+
from framework._paths import in_root
|
|
25
26
|
from framework.errors import FrameworkError, FrameworkWarning
|
|
26
27
|
|
|
27
28
|
# macOS resolves /tmp, /var, and /etc to /private/..., which the user did not name
|
|
28
29
|
_SYSTEM_PREFIX = "/private/"
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
def
|
|
32
|
-
"""Whether
|
|
33
|
-
|
|
32
|
+
def is_private_folder_name(name: str) -> bool:
|
|
33
|
+
"""Whether a folder name marks it private: ``private`` or ``..._private``, in any case."""
|
|
34
|
+
lowered = name.lower()
|
|
35
|
+
return lowered == "private" or lowered.endswith("_private")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def private_directories(root: Path | None = None) -> list[Path]:
|
|
39
|
+
"""The folders the settings' ``directories`` mark private by key
|
|
40
|
+
(``outputs_private_tables``), resolved. Keys that also say public are left out."""
|
|
41
|
+
from framework.project import find_settings_file
|
|
42
|
+
from framework.settings import settings_read
|
|
43
|
+
|
|
44
|
+
found = find_settings_file(root)
|
|
45
|
+
if found is None:
|
|
46
|
+
return []
|
|
47
|
+
root = found.parent
|
|
48
|
+
try:
|
|
49
|
+
directories = settings_read(found).get("directories")
|
|
50
|
+
except FrameworkError:
|
|
51
|
+
return []
|
|
52
|
+
if not isinstance(directories, dict):
|
|
53
|
+
return []
|
|
54
|
+
return [
|
|
55
|
+
in_root(value, root).resolve()
|
|
56
|
+
for key, value in directories.items()
|
|
57
|
+
if isinstance(value, str)
|
|
58
|
+
and value
|
|
59
|
+
and "private" in str(key).lower()
|
|
60
|
+
and "public" not in str(key).lower()
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def is_private_path(path: Path, private_dirs: Sequence[Path] | None = None) -> bool:
|
|
65
|
+
"""Whether ``path`` is in a private folder.
|
|
66
|
+
|
|
67
|
+
A path is private when one of its folders (not the file name) is named ``private`` or
|
|
68
|
+
ends in ``_private``, in any letter case, or when it is under a folder the settings
|
|
69
|
+
mark private (``private_directories()``). macOS's ``/private/`` system prefix does not
|
|
70
|
+
count. A folder's own name counts when ``path`` is a folder.
|
|
71
|
+
"""
|
|
72
|
+
resolved = path.expanduser().resolve()
|
|
73
|
+
full = resolved.as_posix()
|
|
34
74
|
if full.startswith(_SYSTEM_PREFIX):
|
|
35
75
|
full = full[len(_SYSTEM_PREFIX) - 1 :]
|
|
36
|
-
|
|
76
|
+
parts = full.split("/")
|
|
77
|
+
folders = parts if resolved.is_dir() else parts[:-1]
|
|
78
|
+
if any(is_private_folder_name(part) for part in folders):
|
|
79
|
+
return True
|
|
80
|
+
dirs = private_directories() if private_dirs is None else private_dirs
|
|
81
|
+
return any(resolved == folder or folder in resolved.parents for folder in dirs)
|
|
37
82
|
|
|
38
83
|
|
|
39
84
|
def is_private_data(path: Path, root: Path | None) -> bool:
|
|
@@ -120,7 +165,8 @@ def confirm_private_upload(
|
|
|
120
165
|
"""
|
|
121
166
|
if include_private:
|
|
122
167
|
return
|
|
123
|
-
|
|
168
|
+
dirs = private_directories()
|
|
169
|
+
flagged = [p for p in paths if is_private_path(p, dirs)]
|
|
124
170
|
if not flagged and not reasons:
|
|
125
171
|
return
|
|
126
172
|
if flagged:
|
|
@@ -128,18 +174,20 @@ def confirm_private_upload(
|
|
|
128
174
|
if len(flagged) > 5:
|
|
129
175
|
shown += f"\n ... and {len(flagged) - 5} more"
|
|
130
176
|
print(
|
|
131
|
-
f"Warning: {len(flagged)} file(s) to upload
|
|
177
|
+
f"Warning: {len(flagged)} file(s) to upload are in private folders:{shown}",
|
|
132
178
|
file=sys.stderr,
|
|
133
179
|
)
|
|
134
180
|
for reason in reasons:
|
|
135
181
|
print(f"Warning: {reason}", file=sys.stderr)
|
|
136
182
|
if not sys.stdin.isatty():
|
|
137
|
-
what = "files
|
|
183
|
+
what = "files from private folders" if flagged else "this"
|
|
138
184
|
raise FrameworkError(
|
|
139
185
|
f"Not uploading {what} without confirmation. "
|
|
140
186
|
"Review it, then pass include_private=True to upload it."
|
|
141
187
|
)
|
|
142
|
-
|
|
188
|
+
# As R asks: "them" when only paths are flagged
|
|
189
|
+
question = "Upload anyway?" if reasons else "Upload them anyway?"
|
|
190
|
+
answer = input(f"{question} [y/N] ").strip().lower()
|
|
143
191
|
if answer not in ("y", "yes"):
|
|
144
192
|
raise FrameworkError("Cancelled. Nothing was uploaded.")
|
|
145
193
|
|
|
@@ -172,10 +220,8 @@ def sensitive_reasons(what: str) -> list[str]:
|
|
|
172
220
|
|
|
173
221
|
# A quoted string in R or Python code, without f-string or raw-string prefixes
|
|
174
222
|
_STRING = re.compile(r""""((?:[^"\\\n]|\\.)*)"|'((?:[^'\\\n]|\\.)*)'""")
|
|
175
|
-
# A
|
|
176
|
-
|
|
177
|
-
r"(^|[/\\.])[^/\\.\s]*(private|confidential|sensitive)[^/\\.\s]*([/\\.]|$)", re.I
|
|
178
|
-
)
|
|
223
|
+
# A catalog name: dot-separated words
|
|
224
|
+
_CATALOG_NAME = re.compile(r"[A-Za-z0-9_]+(\.[A-Za-z0-9_-]+)+")
|
|
179
225
|
|
|
180
226
|
|
|
181
227
|
def _notebook_code(path: Path) -> str:
|
|
@@ -217,14 +263,40 @@ def _private_catalog_names() -> set[str]:
|
|
|
217
263
|
return private
|
|
218
264
|
|
|
219
265
|
|
|
266
|
+
def _private_path_string(value: str, root: Path | None, dirs: Sequence[Path]) -> bool:
|
|
267
|
+
"""Whether a string is a path with a private folder part, or under a private folder
|
|
268
|
+
from the settings. The last part counts as a folder only when it has no extension, so
|
|
269
|
+
a file named ``01-private-notes.qmd`` doesn't count."""
|
|
270
|
+
if "/" not in value and "\\" not in value:
|
|
271
|
+
return False
|
|
272
|
+
parts = re.split(r"[/\\]", value)
|
|
273
|
+
last = parts[-1]
|
|
274
|
+
folders = parts[:-1] if "." in last else parts
|
|
275
|
+
if any(is_private_folder_name(part) for part in folders):
|
|
276
|
+
return True
|
|
277
|
+
if not dirs or root is None:
|
|
278
|
+
return False
|
|
279
|
+
try:
|
|
280
|
+
candidate = in_root(value, root).resolve()
|
|
281
|
+
except (OSError, ValueError):
|
|
282
|
+
return False
|
|
283
|
+
return any(candidate == folder or folder in candidate.parents for folder in dirs)
|
|
284
|
+
|
|
285
|
+
|
|
220
286
|
def notebook_private_reads(path: Path) -> list[str]:
|
|
221
287
|
"""The strings in a notebook's code that name private data, in order, once each.
|
|
222
288
|
|
|
223
|
-
A string counts when
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
289
|
+
A string counts when it is a path with a private folder part
|
|
290
|
+
(``"inputs/private/raw/ids.csv"``, ``"outputs_private/x.csv"``) or under a folder the
|
|
291
|
+
settings mark private, or when it is a data catalog name whose file is private. Other
|
|
292
|
+
strings, such as settings keys or ``"private|public"``, never count. This is a quick
|
|
293
|
+
text check, not an analysis: data reached any other way is not found.
|
|
227
294
|
"""
|
|
295
|
+
from framework.project import find_settings_file
|
|
296
|
+
|
|
297
|
+
found_settings = find_settings_file()
|
|
298
|
+
root = found_settings.parent if found_settings is not None else None
|
|
299
|
+
dirs = private_directories(root) if root is not None else []
|
|
228
300
|
try:
|
|
229
301
|
code = _notebook_code(path)
|
|
230
302
|
except OSError:
|
|
@@ -235,10 +307,10 @@ def notebook_private_reads(path: Path) -> list[str]:
|
|
|
235
307
|
for value in dict.fromkeys(strings):
|
|
236
308
|
if not value or len(value) > 500 or value in found:
|
|
237
309
|
continue
|
|
238
|
-
if
|
|
310
|
+
if _private_path_string(value, root, dirs):
|
|
239
311
|
found.append(value)
|
|
240
312
|
continue
|
|
241
|
-
if
|
|
313
|
+
if _CATALOG_NAME.fullmatch(value):
|
|
242
314
|
if catalog is None:
|
|
243
315
|
catalog = _private_catalog_names()
|
|
244
316
|
if value in catalog:
|
framework/cache.py
CHANGED
|
@@ -230,7 +230,7 @@ def _lookup(
|
|
|
230
230
|
if dependencies is not None:
|
|
231
231
|
if stored is None or stored[0] != _depends.key(dependencies):
|
|
232
232
|
changed = _depends.changed_between(stored[1] if stored else [], dependencies)
|
|
233
|
-
inform(f"Cache '{name}' is out of date: {
|
|
233
|
+
inform(f"Cache '{name}' is out of date: {_depends.listing(changed)} changed")
|
|
234
234
|
return _MISSING
|
|
235
235
|
elif stored is not None:
|
|
236
236
|
fresh, changed = _depends.recheck(stored[1])
|
|
@@ -238,9 +238,10 @@ def _lookup(
|
|
|
238
238
|
carried.append(fresh)
|
|
239
239
|
if changed:
|
|
240
240
|
warnings.warn(
|
|
241
|
-
f"Cache '{name}' is older than its dependencies:
|
|
242
|
-
"since it was stored, so it is treated
|
|
243
|
-
"cache_remember() to recompute it without
|
|
241
|
+
f"Cache '{name}' is older than its dependencies: "
|
|
242
|
+
f"{_depends.listing(changed)} changed since it was stored, so it is treated "
|
|
243
|
+
"as a miss. Pass depends_on= to cache_remember() to recompute it without "
|
|
244
|
+
"this warning.",
|
|
244
245
|
CacheWarning,
|
|
245
246
|
stacklevel=3,
|
|
246
247
|
)
|
framework/catalog.py
CHANGED
|
@@ -61,9 +61,10 @@ class DataInfo:
|
|
|
61
61
|
name: The catalog name, when the file was found by name.
|
|
62
62
|
extra: Any other fields of the catalog entry.
|
|
63
63
|
columns: With ``data_info(..., columns=True)``: one row per column, with its
|
|
64
|
-
``name``, its ``type`` (logical, integer, double, character,
|
|
65
|
-
datetime, time, duration, or other),
|
|
66
|
-
|
|
64
|
+
``name``, its ``type`` in R's words (logical, integer, double, character,
|
|
65
|
+
factor, date, datetime, time, duration, or other), its pandas ``dtype``
|
|
66
|
+
(``int64``, ``object``, ...), and its count of ``missing`` values. Never any
|
|
67
|
+
values. None otherwise.
|
|
67
68
|
rows: With ``columns=True``, the number of rows. None otherwise.
|
|
68
69
|
"""
|
|
69
70
|
|
|
@@ -229,12 +230,12 @@ def data_info(path: str, columns: bool = False) -> DataInfo:
|
|
|
229
230
|
the project root.
|
|
230
231
|
|
|
231
232
|
With ``columns=True``, the file is read and summarized without showing any values:
|
|
232
|
-
each column's name, type, and missing count, and the number of rows.
|
|
233
|
-
the sensitive-data rules can learn a restricted file's layout this way
|
|
234
|
-
seeing its records::
|
|
233
|
+
each column's name, type, pandas dtype, and missing count, and the number of rows.
|
|
234
|
+
An agent under the sensitive-data rules can learn a restricted file's layout this way
|
|
235
|
+
without seeing its records::
|
|
235
236
|
|
|
236
237
|
info = fw.data_info("inputs.private.raw.patients", columns=True)
|
|
237
|
-
info.columns # name, type, missing
|
|
238
|
+
info.columns # name, type, dtype, missing
|
|
238
239
|
info.rows
|
|
239
240
|
|
|
240
241
|
Args:
|
|
@@ -293,14 +294,16 @@ def _type_name(column: pd.Series) -> str:
|
|
|
293
294
|
|
|
294
295
|
|
|
295
296
|
def column_summary(frame: pd.DataFrame) -> pd.DataFrame:
|
|
296
|
-
"""Each column's name, type,
|
|
297
|
+
"""Each column's name, type (in R's words, as R's summary has it), pandas dtype, and
|
|
298
|
+
missing count. Never any values."""
|
|
297
299
|
return pd.DataFrame(
|
|
298
300
|
{
|
|
299
301
|
"name": [str(name) for name in frame.columns],
|
|
300
302
|
"type": [_type_name(frame.iloc[:, i]) for i in range(frame.shape[1])],
|
|
303
|
+
"dtype": [str(frame.iloc[:, i].dtype) for i in range(frame.shape[1])],
|
|
301
304
|
"missing": [int(frame.iloc[:, i].isna().sum()) for i in range(frame.shape[1])],
|
|
302
305
|
},
|
|
303
|
-
columns=["name", "type", "missing"],
|
|
306
|
+
columns=["name", "type", "dtype", "missing"],
|
|
304
307
|
)
|
|
305
308
|
|
|
306
309
|
|
framework/cloud.py
CHANGED
|
@@ -1054,7 +1054,7 @@ def cloud_publish(
|
|
|
1054
1054
|
visibility alone: a new artifact is unlisted, and an existing one keeps its setting.
|
|
1055
1055
|
``True`` turns the readable URL on with the derived name, and ``False`` turns it off.
|
|
1056
1056
|
Pass ``visibility="public"`` to make an artifact public. An explicit ``visibility``
|
|
1057
|
-
always wins over ``True`` and ``False``. A source
|
|
1057
|
+
always wins over ``True`` and ``False``. A source in a private folder needs
|
|
1058
1058
|
confirmation, or ``include_private=True``.
|
|
1059
1059
|
"""
|
|
1060
1060
|
sources = [source]
|
framework/connections.py
CHANGED
|
@@ -170,15 +170,25 @@ def _writable(path: str) -> str:
|
|
|
170
170
|
return path
|
|
171
171
|
|
|
172
172
|
|
|
173
|
+
def _read_only_file(path: str) -> str:
|
|
174
|
+
"""``path``, after checking that a database to open read-only exists: read-only
|
|
175
|
+
can't create it."""
|
|
176
|
+
if not Path(path).is_file():
|
|
177
|
+
raise FrameworkError(
|
|
178
|
+
f"Read-only database does not exist: {path}. Create it first with a writable "
|
|
179
|
+
"connection (one without read_only): connect to it once, or run its schema with "
|
|
180
|
+
"db_execute_file()."
|
|
181
|
+
)
|
|
182
|
+
return path
|
|
183
|
+
|
|
184
|
+
|
|
173
185
|
def _connect_sqlite(config: Mapping[str, Any]) -> Any:
|
|
174
186
|
path = _file_path(config, "SQLite")
|
|
175
187
|
if not config.get("read_only"):
|
|
176
188
|
return sqlite3.connect(_writable(path))
|
|
177
189
|
# mode=ro opens the file without write access, so every write fails and a file
|
|
178
190
|
# without write permission opens
|
|
179
|
-
|
|
180
|
-
raise FrameworkError(f"Read-only database does not exist: {path}")
|
|
181
|
-
return sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=ro", uri=True)
|
|
191
|
+
return sqlite3.connect(f"{Path(_read_only_file(path)).resolve().as_uri()}?mode=ro", uri=True)
|
|
182
192
|
|
|
183
193
|
|
|
184
194
|
def _connect_postgres(config: Mapping[str, Any]) -> Any:
|
|
@@ -224,7 +234,9 @@ def _connect_duckdb(config: Mapping[str, Any]) -> Any:
|
|
|
224
234
|
path = _file_path(config, "DuckDB")
|
|
225
235
|
# A read-only database needs no write permission
|
|
226
236
|
return duckdb.connect(
|
|
227
|
-
path if read_only else _writable(path),
|
|
237
|
+
_read_only_file(path) if read_only else _writable(path),
|
|
238
|
+
read_only=read_only,
|
|
239
|
+
config=options,
|
|
228
240
|
)
|
|
229
241
|
|
|
230
242
|
|
|
@@ -294,14 +306,24 @@ class Connection:
|
|
|
294
306
|
name: The connection name in the settings.
|
|
295
307
|
driver: sqlite, postgres, mysql, duckdb, or sqlserver.
|
|
296
308
|
raw: The driver's connection object.
|
|
297
|
-
read_only: Whether it was opened
|
|
309
|
+
read_only: Whether it was opened read-only (sqlite and duckdb), by its
|
|
310
|
+
``read_only: true`` setting or a call's ``read_only=True``.
|
|
298
311
|
"""
|
|
299
312
|
|
|
300
|
-
def __init__(
|
|
313
|
+
def __init__(
|
|
314
|
+
self,
|
|
315
|
+
name: str,
|
|
316
|
+
driver: str,
|
|
317
|
+
raw: Any,
|
|
318
|
+
read_only: bool = False,
|
|
319
|
+
read_only_from: str = "settings",
|
|
320
|
+
) -> None:
|
|
301
321
|
self.name = name
|
|
302
322
|
self.driver = driver
|
|
303
323
|
self.raw = raw
|
|
304
324
|
self.read_only = read_only
|
|
325
|
+
# "settings" or "call": where read_only came from, for messages
|
|
326
|
+
self._read_only_from = read_only_from
|
|
305
327
|
self._in_transaction = False
|
|
306
328
|
|
|
307
329
|
def __getattr__(self, attribute: str) -> Any:
|
|
@@ -341,12 +363,14 @@ def db_connect(name: str, read_only: bool | None = None) -> Connection:
|
|
|
341
363
|
|
|
342
364
|
Args:
|
|
343
365
|
name: The connection's name under ``connections`` in the settings.
|
|
344
|
-
read_only: Open a sqlite or duckdb file read-only
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
366
|
+
read_only: Open a sqlite or duckdb file read-only for this call (True). Default:
|
|
367
|
+
the connection's ``read_only`` setting. A call can make a connection read-only
|
|
368
|
+
but not writable: False on a connection with ``read_only: true`` in its settings
|
|
369
|
+
is an error, so the settings decide what may write. A job that builds a
|
|
370
|
+
database other code reads read-only uses its own connection entry without
|
|
371
|
+
``read_only: true``. Server databases (postgres, mysql, sqlserver) control writes
|
|
372
|
+
with the database user's permissions, so True is an error for them and False
|
|
373
|
+
changes nothing.
|
|
350
374
|
|
|
351
375
|
Returns:
|
|
352
376
|
A ``Connection`` that wraps the driver's connection object. Use it in a ``with`` block
|
|
@@ -354,14 +378,16 @@ def db_connect(name: str, read_only: bool | None = None) -> Connection:
|
|
|
354
378
|
|
|
355
379
|
Raises:
|
|
356
380
|
FrameworkError: The connection is not configured, its driver package
|
|
357
|
-
is not installed, connecting failed,
|
|
358
|
-
database
|
|
381
|
+
is not installed, connecting failed, ``read_only=True`` names a server
|
|
382
|
+
database, or ``read_only=False`` names a connection that is read-only in its
|
|
383
|
+
settings.
|
|
359
384
|
"""
|
|
360
385
|
entry = _database_config(name)
|
|
361
386
|
driver_name = str(entry["driver"]).lower()
|
|
362
387
|
if driver_name not in DRIVERS:
|
|
363
388
|
raise FrameworkError(f"Unsupported database driver for '{name}': {entry['driver']}")
|
|
364
389
|
driver = DRIVERS[driver_name][0]
|
|
390
|
+
from_call = False
|
|
365
391
|
if read_only is not None:
|
|
366
392
|
if driver not in _FILE_DRIVERS:
|
|
367
393
|
if read_only:
|
|
@@ -369,7 +395,13 @@ def db_connect(name: str, read_only: bool | None = None) -> Connection:
|
|
|
369
395
|
f"read_only=True applies to sqlite and duckdb files, and '{name}' is "
|
|
370
396
|
f"{driver}. Connect as a database user without write access instead."
|
|
371
397
|
)
|
|
398
|
+
elif entry.get("read_only") and not read_only:
|
|
399
|
+
raise FrameworkError(
|
|
400
|
+
f"Connection '{name}' is read-only in its settings; read_only=False can't "
|
|
401
|
+
"override that."
|
|
402
|
+
)
|
|
372
403
|
else:
|
|
404
|
+
from_call = read_only and not bool(entry.get("read_only"))
|
|
373
405
|
entry = {**entry, "read_only": read_only}
|
|
374
406
|
try:
|
|
375
407
|
raw = _CONNECTORS[driver](entry)
|
|
@@ -378,7 +410,7 @@ def db_connect(name: str, read_only: bool | None = None) -> Connection:
|
|
|
378
410
|
except Exception as error:
|
|
379
411
|
raise FrameworkError(f"Failed to connect to '{name}': {error}") from error
|
|
380
412
|
read_only_open = driver in _FILE_DRIVERS and bool(entry.get("read_only"))
|
|
381
|
-
return Connection(name, driver, raw, read_only_open)
|
|
413
|
+
return Connection(name, driver, raw, read_only_open, "call" if from_call else "settings")
|
|
382
414
|
|
|
383
415
|
|
|
384
416
|
# Queries ------------------------------------------------------------------------------
|
|
@@ -406,8 +438,9 @@ def db_query(
|
|
|
406
438
|
connection_name: The connection's name in the settings. The connection is opened for
|
|
407
439
|
this query and closed afterward.
|
|
408
440
|
params: Values for the query's placeholders, as a sequence or a mapping.
|
|
409
|
-
read_only: Open the connection read-only
|
|
410
|
-
|
|
441
|
+
read_only: Open the connection read-only for this query (True), as
|
|
442
|
+
``db_connect()`` does. Default: the connection's setting, which False can't
|
|
443
|
+
override when it is ``read_only: true``. A read-only DuckDB
|
|
411
444
|
file can be open in several processes at once, so ``read_only=True`` lets a
|
|
412
445
|
notebook query a database another job holds open read-only too.
|
|
413
446
|
|
|
@@ -415,7 +448,8 @@ def db_query(
|
|
|
415
448
|
The result rows, with the query's column names.
|
|
416
449
|
|
|
417
450
|
Raises:
|
|
418
|
-
FrameworkError: When the connection cannot be opened
|
|
451
|
+
FrameworkError: When the connection cannot be opened, ``read_only=False`` names a
|
|
452
|
+
connection that is read-only in its settings, or the query fails.
|
|
419
453
|
"""
|
|
420
454
|
with db_connect(connection_name, read_only=read_only) as connection:
|
|
421
455
|
cursor = connection.cursor()
|
|
@@ -450,11 +484,7 @@ def db_execute(
|
|
|
450
484
|
statement fails.
|
|
451
485
|
"""
|
|
452
486
|
with db_connect(connection_name) as connection:
|
|
453
|
-
|
|
454
|
-
raise FrameworkError(
|
|
455
|
-
f"Connection '{connection_name}' is read-only (read_only: true in its "
|
|
456
|
-
"settings). db_execute() can't change it."
|
|
457
|
-
)
|
|
487
|
+
_writable_connection(connection, "db_execute()")
|
|
458
488
|
cursor = connection.cursor()
|
|
459
489
|
try:
|
|
460
490
|
cursor.execute(query, params) if params is not None else cursor.execute(query)
|
|
@@ -502,11 +532,7 @@ def db_transaction(connection: Connection) -> Iterator[Connection]:
|
|
|
502
532
|
FrameworkError: When the connection is read-only, or the block raises an exception.
|
|
503
533
|
The transaction is rolled back and the original exception is chained as the cause.
|
|
504
534
|
"""
|
|
505
|
-
|
|
506
|
-
raise FrameworkError(
|
|
507
|
-
f"Connection '{connection.name}' is read-only (read_only: true in its settings). "
|
|
508
|
-
"db_transaction() needs a writable connection."
|
|
509
|
-
)
|
|
535
|
+
_writable_connection(connection, "db_transaction()")
|
|
510
536
|
if connection._in_transaction:
|
|
511
537
|
yield connection
|
|
512
538
|
return
|
|
@@ -539,11 +565,19 @@ def _opened(connection: Connection | str) -> Iterator[Connection]:
|
|
|
539
565
|
|
|
540
566
|
|
|
541
567
|
def _writable_connection(connection: Connection, what: str) -> None:
|
|
542
|
-
|
|
568
|
+
"""Refuse a read-only connection, saying where its mode came from."""
|
|
569
|
+
if not connection.read_only:
|
|
570
|
+
return
|
|
571
|
+
if connection._read_only_from == "call":
|
|
543
572
|
raise FrameworkError(
|
|
544
|
-
f"Connection '{connection.name}' is read-only
|
|
545
|
-
f"connection: db_connect('{connection.name}'
|
|
573
|
+
f"Connection '{connection.name}' is read-only (read_only=True in this call). "
|
|
574
|
+
f"{what} needs a writable connection: db_connect('{connection.name}')."
|
|
546
575
|
)
|
|
576
|
+
raise FrameworkError(
|
|
577
|
+
f"Connection '{connection.name}' is read-only (read_only: true in its settings). "
|
|
578
|
+
f"{what} needs a writable connection, from a connection entry without "
|
|
579
|
+
"read_only: true."
|
|
580
|
+
)
|
|
547
581
|
|
|
548
582
|
|
|
549
583
|
def _project_file(path: str | Path) -> Path:
|
framework/create.py
CHANGED
|
@@ -602,11 +602,16 @@ def _defaults_layer(settings: Mapping[str, Any], type: str) -> dict[str, Any]:
|
|
|
602
602
|
layer["ai"]["assistants"] = assistants
|
|
603
603
|
git = defaults.get("git")
|
|
604
604
|
if isinstance(git, dict):
|
|
605
|
-
layer["git"] = {
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
605
|
+
layer["git"] = {} if git.get("initialize") is None else {"use_git": git["initialize"]}
|
|
606
|
+
# As in R, the user's hooks only turn checks on. Global settings hold
|
|
607
|
+
# data_security: false from the shipped defaults for every type, which would
|
|
608
|
+
# otherwise turn off the checks a privacy-sensitive project starts with; the
|
|
609
|
+
# package defaults already have every other check off.
|
|
610
|
+
hooks = git.get("hooks")
|
|
611
|
+
if isinstance(hooks, dict):
|
|
612
|
+
enabled = {name: True for name, value in hooks.items() if value is True}
|
|
613
|
+
if enabled:
|
|
614
|
+
layer["git"]["hooks"] = enabled
|
|
610
615
|
types = settings.get("project_types")
|
|
611
616
|
type_config = types.get(type) if isinstance(types, dict) else None
|
|
612
617
|
if isinstance(type_config, dict) and isinstance(type_config.get("directories"), dict):
|
framework/outputs.py
CHANGED
|
@@ -218,8 +218,9 @@ def _figure_formats(format: str | Sequence[str]) -> list[str]:
|
|
|
218
218
|
)
|
|
219
219
|
if "emf" in formats and not _backend_writes("emf"):
|
|
220
220
|
raise UnsupportedFormatError(
|
|
221
|
-
"emf needs a
|
|
222
|
-
"
|
|
221
|
+
"emf needs a matplotlib backend that writes it, which isn't installed. "
|
|
222
|
+
"Install one that registers emf, or save svg or pdf and convert them "
|
|
223
|
+
"(for example with Inkscape or LibreOffice)."
|
|
223
224
|
)
|
|
224
225
|
return formats
|
|
225
226
|
|
|
@@ -345,7 +346,8 @@ def _data_rows(model: Any) -> int:
|
|
|
345
346
|
if isinstance(frame, pd.DataFrame) and len(frame):
|
|
346
347
|
return len(frame)
|
|
347
348
|
endog = getattr(inner, "endog", None)
|
|
348
|
-
|
|
349
|
+
# As in R, a kept response counts when it has more than one value
|
|
350
|
+
if endog is not None and getattr(endog, "ndim", 0) >= 1 and len(endog) > 1:
|
|
349
351
|
return len(endog)
|
|
350
352
|
attributes = getattr(model, "__dict__", None)
|
|
351
353
|
if isinstance(attributes, dict):
|
framework/project.py
CHANGED
|
@@ -60,15 +60,15 @@ def project_root(start: str | Path | None = None) -> Path:
|
|
|
60
60
|
return found.parent
|
|
61
61
|
|
|
62
62
|
|
|
63
|
-
def
|
|
63
|
+
def project_path(*parts: str | PathLike[str]) -> Path:
|
|
64
64
|
"""An absolute path inside the project, from parts relative to its root.
|
|
65
65
|
|
|
66
|
-
Like R's ``here::here()``: the same call gives the same file from a
|
|
67
|
-
``notebooks/``, a script in the root, or a subfolder. Python never changes the
|
|
68
|
-
directory, so ``Path("outputs/tables/x.csv").exists()`` in a notebook looks under
|
|
69
|
-
``notebooks/``; ``fw.
|
|
66
|
+
Like R's ``project_path()`` and ``here::here()``: the same call gives the same file from a
|
|
67
|
+
notebook in ``notebooks/``, a script in the root, or a subfolder. Python never changes the
|
|
68
|
+
working directory, so ``Path("outputs/tables/x.csv").exists()`` in a notebook looks under
|
|
69
|
+
``notebooks/``; ``fw.project_path("outputs/tables/x.csv").exists()`` looks under the root::
|
|
70
70
|
|
|
71
|
-
if not fw.
|
|
71
|
+
if not fw.project_path("outputs", "tables", "incidence.csv").exists():
|
|
72
72
|
...
|
|
73
73
|
|
|
74
74
|
Args:
|
|
@@ -80,13 +80,28 @@ def path(*parts: str | PathLike[str]) -> Path:
|
|
|
80
80
|
|
|
81
81
|
Raises:
|
|
82
82
|
ProjectNotFoundError: Not inside a Framework project.
|
|
83
|
-
ValueError: A part is
|
|
83
|
+
ValueError: A part is absolute or rooted (``"/etc"``, ``"C:\\data"``, ``"~/x"``).
|
|
84
84
|
"""
|
|
85
|
-
root = project_root()
|
|
86
85
|
for part in parts:
|
|
87
|
-
|
|
88
|
-
|
|
86
|
+
text = str(part)
|
|
87
|
+
# Rooted on either system: "/etc", "C:\\data", "C:data", "\\data", or "~"
|
|
88
|
+
if Path(part).is_absolute() or PureWindowsPath(part).anchor or text.startswith("~"):
|
|
89
89
|
raise ValueError(
|
|
90
|
-
f"
|
|
90
|
+
f"project_path() parts are relative to the project root, but '{text}' is absolute"
|
|
91
91
|
)
|
|
92
|
-
return
|
|
92
|
+
return project_root().joinpath(*parts)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def path(*parts: str | PathLike[str]) -> Path:
|
|
96
|
+
"""An alias of ``project_path()``, the name R uses.
|
|
97
|
+
|
|
98
|
+
``fw.path(...)`` and ``fw.project_path(...)`` are the same call, and both stay supported.
|
|
99
|
+
New code can use ``project_path()`` so it reads the same in R and Python.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
*parts: Path parts relative to the project root, as for ``project_path()``.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
The absolute path. The file need not exist.
|
|
106
|
+
"""
|
|
107
|
+
return project_path(*parts)
|
framework/publish_bundle.py
CHANGED
|
@@ -71,6 +71,8 @@ def artifact_slug(value: str) -> str:
|
|
|
71
71
|
# publish alone at the folder's root, as a GitHub Pages docs/ folder wants
|
|
72
72
|
LAYOUTS = ("versioned", "site-root")
|
|
73
73
|
SITE_RECORD = ".framework-site.json"
|
|
74
|
+
# Sources a site-root publish never writes: only their rendered output goes to the site
|
|
75
|
+
_NOTEBOOK_SOURCES = (".qmd", ".ipynb", ".rmd")
|
|
74
76
|
|
|
75
77
|
|
|
76
78
|
@dataclass(frozen=True)
|
|
@@ -369,8 +371,9 @@ def folder_deliver_root(bundle: Path, site: Path, base_url: str | None = None) -
|
|
|
369
371
|
"""Put the latest publish alone at the folder's root, with no versions or index.
|
|
370
372
|
|
|
371
373
|
The files the previous publish wrote, listed in ``.framework-site.json``, are removed
|
|
372
|
-
first; other files in the folder (``CNAME``, ``.nojekyll``) are left alone.
|
|
373
|
-
``artifact.json`` isn't
|
|
374
|
+
first; other files in the folder (``CNAME``, ``.nojekyll``) are left alone. Only
|
|
375
|
+
rendered output is written: the bundle's ``artifact.json`` isn't, nor a notebook's
|
|
376
|
+
source (``.qmd``, ``.ipynb``, ``.Rmd``), nor a copy of an HTML source identical to
|
|
374
377
|
``index.html``, so a deck is written once.
|
|
375
378
|
"""
|
|
376
379
|
manifest = json.loads((bundle / "artifact.json").read_text(encoding="utf-8"))
|
|
@@ -385,18 +388,20 @@ def folder_deliver_root(bundle: Path, site: Path, base_url: str | None = None) -
|
|
|
385
388
|
previous = previous if isinstance(previous, dict) else {}
|
|
386
389
|
index = (bundle / "index.html").read_bytes() if (bundle / "index.html").is_file() else None
|
|
387
390
|
original = manifest.get("original")
|
|
391
|
+
|
|
392
|
+
def source_copy(path: Path) -> bool:
|
|
393
|
+
"""The bundle's copy of a notebook source, or of an HTML source equal to index.html."""
|
|
394
|
+
if path.parent != bundle or path.name != original or path.name == "index.html":
|
|
395
|
+
return False
|
|
396
|
+
suffix = path.suffix.lower()
|
|
397
|
+
if suffix in _NOTEBOOK_SOURCES:
|
|
398
|
+
return True
|
|
399
|
+
return suffix in (".html", ".htm") and path.read_bytes() == index
|
|
400
|
+
|
|
388
401
|
files = [
|
|
389
402
|
path
|
|
390
403
|
for path in sorted(bundle.rglob("*"))
|
|
391
|
-
if path.is_file()
|
|
392
|
-
and path.name != "artifact.json"
|
|
393
|
-
and not (
|
|
394
|
-
path.parent == bundle
|
|
395
|
-
and path.name == original
|
|
396
|
-
and path.name != "index.html"
|
|
397
|
-
and path.suffix.lower() in (".html", ".htm")
|
|
398
|
-
and path.read_bytes() == index
|
|
399
|
-
)
|
|
404
|
+
if path.is_file() and path.name != "artifact.json" and not source_copy(path)
|
|
400
405
|
]
|
|
401
406
|
root = site.resolve()
|
|
402
407
|
for relative in previous.get("files") or []:
|
|
@@ -416,7 +421,9 @@ def folder_deliver_root(bundle: Path, site: Path, base_url: str | None = None) -
|
|
|
416
421
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
417
422
|
shutil.copy2(path, target)
|
|
418
423
|
written.append(relative.as_posix())
|
|
419
|
-
|
|
424
|
+
# Versions count per slug: a different artifact in the folder starts again at 1
|
|
425
|
+
same_slug = previous.get("slug") == manifest.get("slug")
|
|
426
|
+
version = int(previous.get("version") or 0) + 1 if same_slug else 1
|
|
420
427
|
record = {
|
|
421
428
|
"schema": "framework.site/v1",
|
|
422
429
|
"version": version,
|
framework/storage.py
CHANGED
|
@@ -441,7 +441,7 @@ def _publish_legacy(
|
|
|
441
441
|
overwrite: Accepted for parity with R; uploads always replace.
|
|
442
442
|
self_contained: For Framework Cloud publishing.
|
|
443
443
|
slug: For Framework Cloud publishing.
|
|
444
|
-
include_private: Upload files
|
|
444
|
+
include_private: Upload files in private folders without asking. Only a
|
|
445
445
|
person who has reviewed the files should set this.
|
|
446
446
|
to: "cloud" explicitly publishes to Framework Cloud and "bucket" selects storage.
|
|
447
447
|
"github" publishes to the project's GitHub Pages site: each publish is kept in
|
|
@@ -572,7 +572,7 @@ def publish(
|
|
|
572
572
|
overwrite: Accepted for parity with R; uploads always replace.
|
|
573
573
|
self_contained: For Framework Cloud publishing.
|
|
574
574
|
slug: For Framework Cloud publishing.
|
|
575
|
-
include_private: Upload files
|
|
575
|
+
include_private: Upload files in private folders without asking. Only a
|
|
576
576
|
person who has reviewed the files should set this.
|
|
577
577
|
to: A configured destination name, cloud, github, folder, s3, or bucket.
|
|
578
578
|
Named destinations take priority over publish.default and legacy defaults.
|
|
@@ -770,7 +770,7 @@ def publish_notebook(
|
|
|
770
770
|
slug: For Framework Cloud: a string names the document and leaves its visibility
|
|
771
771
|
alone, True turns the readable URL on with the derived name, and False turns it
|
|
772
772
|
off. A storage bucket ignores it with a warning.
|
|
773
|
-
include_private: Upload files
|
|
773
|
+
include_private: Upload files in private folders without asking. Only a
|
|
774
774
|
person who has reviewed the files should set this.
|
|
775
775
|
to: "cloud" explicitly publishes to Framework Cloud and "bucket" selects storage.
|
|
776
776
|
"github" publishes to the project's GitHub Pages site: each publish is kept in
|
|
@@ -979,7 +979,7 @@ def publish_data(
|
|
|
979
979
|
connection: A storage bucket name from the settings. Default: the project's default
|
|
980
980
|
bucket, then one marked ``default: true``, then the first.
|
|
981
981
|
compress: Gzip the file before uploading.
|
|
982
|
-
include_private: Upload files
|
|
982
|
+
include_private: Upload files in private folders without asking. Only a
|
|
983
983
|
person who has reviewed the files should set this.
|
|
984
984
|
to: A destination name from ``publish.destinations``, or a driver name, as in
|
|
985
985
|
``publish()``. Default: ``publish.default``.
|
|
@@ -1092,7 +1092,7 @@ def publish_dir(
|
|
|
1092
1092
|
(searched anywhere in that path, such as ``"figures/.*\\.png$"``). Default: every
|
|
1093
1093
|
file.
|
|
1094
1094
|
recursive: Include files in subfolders.
|
|
1095
|
-
include_private: Upload files
|
|
1095
|
+
include_private: Upload files in private folders without asking. Only a
|
|
1096
1096
|
person who has reviewed the files should set this.
|
|
1097
1097
|
to: A destination name from ``publish.destinations``, or a driver name, as in
|
|
1098
1098
|
``publish()``. Default: ``publish.default``.
|
|
@@ -30,8 +30,11 @@ or `pyreadstat`.
|
|
|
30
30
|
## Saving
|
|
31
31
|
|
|
32
32
|
```python
|
|
33
|
-
|
|
34
|
-
fw.data_save(final, "inputs/final/analysis_ready.csv"
|
|
33
|
+
# Locked by default: a later read raises an error if the file changed
|
|
34
|
+
fw.data_save(final, "inputs/final/analysis_ready.csv")
|
|
35
|
+
|
|
36
|
+
# Data a pipeline rebuilds on every run: unlocked, so a rebuild doesn't warn
|
|
37
|
+
fw.data_save(clean, "inputs.intermediate.survey_clean", locked=False) # Parquet, in the catalog
|
|
35
38
|
```
|
|
36
39
|
|
|
37
40
|
`data_save()` writes Parquet or CSV, which R can read too. Use it instead of
|
|
@@ -18,10 +18,16 @@ whether something is safe, treat it as private.
|
|
|
18
18
|
2. Raw data with identifiers goes in a `private/` folder.
|
|
19
19
|
3. Only de-identified, aggregated data goes in a `public/` folder.
|
|
20
20
|
4. Review every output before saving it to a public folder.
|
|
21
|
-
5.
|
|
21
|
+
5. Save sensitive data into a private folder:
|
|
22
|
+
`fw.data_save(df, "inputs.private.intermediate.cleaned")`, or a path under
|
|
23
|
+
`inputs/private/` or `outputs/private/`.
|
|
24
|
+
6. Run `fw.git_security_audit()` before commits to scan for data leaks, and
|
|
25
|
+
keep the `data_security` git hook on (`fw.git_hooks_list()`,
|
|
26
|
+
`fw.git_hooks_enable("data_security")`).
|
|
27
|
+
7. Never publish anything from a private folder. Never pass
|
|
22
28
|
`include_private=True` to a publish function or answer its prompt on the
|
|
23
29
|
user's behalf.
|
|
24
|
-
|
|
30
|
+
8. Do not read or print restricted files. Values you print are sent to the
|
|
25
31
|
model provider. Work from the data dictionary and simulated data instead.
|
|
26
32
|
|
|
27
33
|
## Data flow
|
|
@@ -34,6 +40,21 @@ Intermediate -> inputs/private/intermediate/
|
|
|
34
40
|
Public-safe data -> inputs/public/final/
|
|
35
41
|
```
|
|
36
42
|
|
|
43
|
+
## Learning a file's layout
|
|
44
|
+
|
|
45
|
+
To learn what a private or restricted file holds, use
|
|
46
|
+
`fw.data_info(name, columns=True)` instead of reading the file, printing it, or
|
|
47
|
+
calling `.head()`, `.info()`, or `.describe()` on it:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
info = fw.data_info("inputs.private.raw.patients", columns=True)
|
|
51
|
+
info.columns # name, type, and count of missing values for each column
|
|
52
|
+
info.rows # number of rows
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
It never shows values, category levels, ranges, or examples, and it records
|
|
56
|
+
nothing in framework.db.
|
|
57
|
+
|
|
37
58
|
## Writing code
|
|
38
59
|
|
|
39
60
|
- Do not print rows with identifiers in notebook output that will be rendered.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: framework-py
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev12
|
|
4
4
|
Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
|
|
5
5
|
Project-URL: Homepage, https://framework.pub
|
|
6
6
|
Project-URL: Source, https://github.com/frameworktools/framework-py
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
framework/__init__.py,sha256=
|
|
1
|
+
framework/__init__.py,sha256=P87pGZVH2C171Zuxm1JykpqerlskzhYRzoR3zu_cZcQ,6264
|
|
2
2
|
framework/__main__.py,sha256=CnSGHPB54v3Uqed9020bgseg_2AZzb1lnJFl8nlX8CM,6044
|
|
3
3
|
framework/_agents_md.py,sha256=jSFZZd7QUEj9rP4Dlc9w_optMReyaXw5rikilRX4a48,9804
|
|
4
4
|
framework/_artifacts.py,sha256=cOk2XLqctoqLaQSkksmc7cP4W0_AgqSxYNCTJxyZyik,36068
|
|
5
5
|
framework/_csv.py,sha256=oAaLUChyUsHJsh0YuUDt0viC8SAImMgUseNKGk7kLXo,4855
|
|
6
6
|
framework/_defaults_cache.py,sha256=pyqQ7rMcKwo_fTSAvDDNyQXIAjiJZfWccnkRPw340WQ,1711
|
|
7
|
-
framework/_depends.py,sha256=
|
|
7
|
+
framework/_depends.py,sha256=KLU4iog3zdyKcXoUuI0Tg9ArZ-3NdKbMEc8XS-DM7Bo,8579
|
|
8
8
|
framework/_identifiers.py,sha256=1jarqO2hG9E-GAv8rCKfc-f27NKLJFtAligUj7beEak,5951
|
|
9
9
|
framework/_keychain.py,sha256=9QW8dFDhgy0knWCCu9blttIe_nCG0UhdY77wrVa1h6M,2798
|
|
10
10
|
framework/_log.py,sha256=ZgLaWbjiXK7Cybd_mWGURb4OOGaAAYsB6NkR15W3nnk,1160
|
|
11
11
|
framework/_pages.py,sha256=bIUR-eVhrwCF-Jdvs8cL_IpN0FLN2RLAy8AdKctdOF8,12503
|
|
12
12
|
framework/_paths.py,sha256=mjuSD1kz-SwmlaISjas_1XJSo9CGqQ9xe4AkVObzzS8,3479
|
|
13
|
-
framework/_private.py,sha256
|
|
13
|
+
framework/_private.py,sha256=-VTAfKj-gFGa-9jmK83gY2kVBn5MmEIxEcUIIi-SrMI,12320
|
|
14
14
|
framework/_secretservice.py,sha256=zv5UvEekQic466qRmVpLG_hfEkisikHfab8e6kyzS2k,2450
|
|
15
15
|
framework/_sql.py,sha256=RlehpBg2Oq1W7jyZEoXEtlAs3ovIYbj2pz85TVdsSQc,3954
|
|
16
16
|
framework/_tabular.py,sha256=zaUB83lMCCJql24BKv6CYD8d1s-jjViS0H4lTxeBzzQ,15131
|
|
@@ -20,13 +20,13 @@ framework/_yaml.py,sha256=fslcba4AC4Zi-8spaE5GZoQ-PAqaumuxN3RvFeyRzTw,5293
|
|
|
20
20
|
framework/adopt.py,sha256=8Trxg7y8zIN3lrFwKBb_7KTfcXnt4t3f7zox5x0ss4I,14056
|
|
21
21
|
framework/ai.py,sha256=ZI7DmEiDi0hrxOLCvh-3aU9cgGcCznOtovRVl9rrXj8,9145
|
|
22
22
|
framework/bootstrap.py,sha256=kBznHCosXkpVCuMYdIP3Ii8SOiWvtQtEP60oWdka3Qk,9468
|
|
23
|
-
framework/cache.py,sha256=
|
|
24
|
-
framework/catalog.py,sha256=
|
|
25
|
-
framework/cloud.py,sha256=
|
|
23
|
+
framework/cache.py,sha256=jJxWIs6kQ7naMenC96pCPtqDCYiCe9c1KSITufV75q4,17160
|
|
24
|
+
framework/catalog.py,sha256=sl4V1judSCVoa2LYLXeQdDhWdsONTC_5s2_fpyPylto,13202
|
|
25
|
+
framework/cloud.py,sha256=kQzhtqmeW5m3LOrYLkpq3LBQ_p28MMcRiVIcgdTr9nQ,49544
|
|
26
26
|
framework/config_home.py,sha256=iSb0VuENgNOSnsZ_v53xFqjRiGgrB7OzZbHbyDOqKzs,8265
|
|
27
|
-
framework/connections.py,sha256=
|
|
27
|
+
framework/connections.py,sha256=EUkWtuQKW7X-BLJOKXzSbVTSNE6raxjV_GuIELjAOSY,32439
|
|
28
28
|
framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
|
|
29
|
-
framework/create.py,sha256=
|
|
29
|
+
framework/create.py,sha256=sAsLcdxhN3mZdcoO_c17PZNfQZfGuO2SKXvhQeNIjns,45107
|
|
30
30
|
framework/data.py,sha256=qfijeRGMZ7vIspkGZmS_X6-VR0eFWMuoxyJThBrKxOA,30283
|
|
31
31
|
framework/db.py,sha256=Wruez2V352xA3gjk5RcUi_0zncFsDwTs9Gwa2hXBY3Y,8099
|
|
32
32
|
framework/display.py,sha256=0FuKUt67uf5h1WjD4bwzoruc2XAN4y_j3mz_f5FskhE,5334
|
|
@@ -37,10 +37,10 @@ framework/global_config.py,sha256=fYLaxN_6Wc2xrIASn77dWkvGR3uuX-111bvfFPB82A4,11
|
|
|
37
37
|
framework/hooks.py,sha256=BcM9De5i5XINy4Nu68z3TeH6ogFujpgUoY5y1SCYnb8,21073
|
|
38
38
|
framework/integrity.py,sha256=2hXk95tlNGs6HVTFA0AqcPWQg2AiO6Hy0APMIXHpCTU,586
|
|
39
39
|
framework/notebooks.py,sha256=nBPDgZOSPZzTHaYvvXgyC_P2KXo7Zv471h47cvYOirA,21008
|
|
40
|
-
framework/outputs.py,sha256=
|
|
40
|
+
framework/outputs.py,sha256=8CcKKedgR8rrgIgQP96l5x-geEGOoCnGIW3Ywq19oYc,27200
|
|
41
41
|
framework/packages.py,sha256=_ZE_eXDbonl7sNW_mT8MZXbpAhnGYOmBEyAbyQ6qQKo,16100
|
|
42
|
-
framework/project.py,sha256=
|
|
43
|
-
framework/publish_bundle.py,sha256=
|
|
42
|
+
framework/project.py,sha256=EAdc24UxrCHq-sofltFNFqnlZw3KoySMSEUX2W0GeFA,3761
|
|
43
|
+
framework/publish_bundle.py,sha256=0vOrLZXSbdV2Fl05smq-2-04Fsts0na3rB_Y0mCOcuY,28802
|
|
44
44
|
framework/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
45
|
framework/quarto.py,sha256=QpEV7rID4_4j8mbhcrkrprrZL15eOYI_PwL-jvE6dY4,9977
|
|
46
46
|
framework/scaffold.py,sha256=UwJw503gOZIp_Y2fHiDuxeG6BUNsy9Yq2k7ZsvEPqz0,13169
|
|
@@ -48,7 +48,7 @@ framework/scratch.py,sha256=idxthUR5BIMX7FIGS7KzIuhOrO3-HKLbBsV8It3EEE4,3890
|
|
|
48
48
|
framework/security.py,sha256=hMFVo_VLIFLOS9yY7ZbBc0acJmARrm_IrD9khk2tUME,38731
|
|
49
49
|
framework/settings.py,sha256=Sb2YyE53aYLWPj423P-pQVCvRpLsiHydgZh2K0luTQQ,23893
|
|
50
50
|
framework/skill_sync.py,sha256=xiaIPTGIExXcBIMAOeg44WLPMPdixZps_2twxelioow,19412
|
|
51
|
-
framework/storage.py,sha256=
|
|
51
|
+
framework/storage.py,sha256=Jt8_adVwWMQmASzSUb97gLEMWz2GlmDUJmPgoYSgOBU,55485
|
|
52
52
|
framework/_assets/init.sql,sha256=-aIiuIcCmBniGKq-CHdRtUiV9e_CLXoqxxd05snmgOE,1156
|
|
53
53
|
framework/_assets/adopt/rules.json,sha256=au7YTeHzOiYotm-lT8ap3kb1t1WmOAeZeEgoumIb7_4,2868
|
|
54
54
|
framework/_assets/features/jupyter.json,sha256=BlPgR9OIfmlMVTstwpWbYrAA4MNx61QvK9lybrNtGgA,736
|
|
@@ -104,14 +104,14 @@ framework/templates/gitignore/course,sha256=VbNFPxggk8XlXEOVedO9no6B9W6Ixm_ASWe0
|
|
|
104
104
|
framework/templates/gitignore/presentation,sha256=4HOErO4zm2HEX2q_JS_vH-0lFMxCWAv1H6PKqyKBIQg,2903
|
|
105
105
|
framework/templates/gitignore/project,sha256=kHfqjTvWZ51AsApXipBKSCdOIpaYsE0mG8t5-tK1y-A,2782
|
|
106
106
|
framework/templates/gitignore/sensitive,sha256=8mKH9WUvn3TRc6QtvboLFvcbfQOGGebRQNMKZzGzzy0,3710
|
|
107
|
-
framework/templates/skills/framework-data/.framework-skill.json,sha256=
|
|
108
|
-
framework/templates/skills/framework-data/SKILL.md,sha256=
|
|
107
|
+
framework/templates/skills/framework-data/.framework-skill.json,sha256=zvqsxljDqxJZsC7Yp5_ujk-oVBiCud1zhJfAaNI4PZY,139
|
|
108
|
+
framework/templates/skills/framework-data/SKILL.md,sha256=IjqI215VsNfvnYWmjd3V1FFbdSIhfwjQ_1Lqkbw-mdw,2075
|
|
109
109
|
framework/templates/skills/framework-outputs/.framework-skill.json,sha256=RZN5cYD13n9dEt_sJVO3rXJOL1zG3UXeyvlyoFtIzdU,142
|
|
110
110
|
framework/templates/skills/framework-outputs/SKILL.md,sha256=fBeT97JfqUooZebo0xF0fDLPfhaErHVKV5xt1w7-pTc,2561
|
|
111
111
|
framework/templates/skills/framework-packages/.framework-skill.json,sha256=JhqCI0ZM_F53ZsxEFRkjErDaqc8pCi1KiSrugH6KXJw,143
|
|
112
112
|
framework/templates/skills/framework-packages/SKILL.md,sha256=GAtfLA3VjPB9lrQVAeYEOHU9XFyiKv9VS7CyVywRK-M,1278
|
|
113
|
-
framework/templates/skills/framework-sensitive-data/.framework-skill.json,sha256=
|
|
114
|
-
framework/templates/skills/framework-sensitive-data/SKILL.md,sha256=
|
|
113
|
+
framework/templates/skills/framework-sensitive-data/.framework-skill.json,sha256=wq-N0XXaFFju0Fqdb49QPfiShBThJKEH52ie4o1V5ks,149
|
|
114
|
+
framework/templates/skills/framework-sensitive-data/SKILL.md,sha256=6if0juK9GzSu2TP4pFyieNbvKE1ew4Xofhg5EfS693o,2542
|
|
115
115
|
framework/templates/skills/framework-workflow/.framework-skill.json,sha256=t5VdSo3dTcBjM4y53gv25U8E0gaqXLAd1DKdooGpb-g,143
|
|
116
116
|
framework/templates/skills/framework-workflow/SKILL.md,sha256=vEyO-vWSVuHlwWN0N8sdmQNnFzayWV2iIUDs2KUb4as,2279
|
|
117
117
|
framework/templates/stubs/notebook-default.ipynb,sha256=2p3Zj6bQEHlUL1eEVAbPfTTxqGBs84C67tA3MkCzVQs,685
|
|
@@ -119,7 +119,7 @@ framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIj
|
|
|
119
119
|
framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
|
|
120
120
|
framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
|
|
121
121
|
framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
|
|
122
|
-
framework_py-0.1.0.
|
|
123
|
-
framework_py-0.1.0.
|
|
124
|
-
framework_py-0.1.0.
|
|
125
|
-
framework_py-0.1.0.
|
|
122
|
+
framework_py-0.1.0.dev12.dist-info/METADATA,sha256=KfuUaX-fyw2tjz47BpiWe8lRAFzGjZxSUBaDQNIjzYA,4281
|
|
123
|
+
framework_py-0.1.0.dev12.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
124
|
+
framework_py-0.1.0.dev12.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
|
|
125
|
+
framework_py-0.1.0.dev12.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|