framework-py 0.1.0.dev7__py3-none-any.whl → 0.1.0.dev8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +4 -0
- framework/_assets/sets/packages/bayes-r.json +51 -0
- framework/_assets/sets/packages/causal-inference-r.json +34 -0
- framework/_assets/sets/packages/fastapi-python.json +34 -0
- framework/_assets/sets/packages/large-data-r.json +30 -0
- framework/_assets/sets/packages/plumber-r.json +30 -0
- framework/_assets/sets/packages/polars-python.json +26 -0
- framework/_assets/sets/packages/python-bayes.json +26 -0
- framework/_assets/sets/packages/python-database.json +26 -0
- framework/_assets/sets/packages/python-ml.json +22 -0
- framework/_assets/sets/packages/python-spatial.json +22 -0
- framework/_assets/sets/packages/python-viz.json +22 -0
- framework/_assets/sets/packages/regression-r.json +42 -0
- framework/_assets/sets/packages/shiny-python.json +30 -0
- framework/_assets/sets/packages/shiny-r.json +34 -0
- framework/_assets/sets/packages/spatial-r.json +26 -0
- framework/_assets/sets/packages/survey-r.json +22 -0
- framework/_assets/sets/packages/survival-r.json +30 -0
- framework/_assets/sets/packages/tables-r.json +30 -0
- framework/_assets/sets/packages/text-r.json +22 -0
- framework/_assets/sets/packages/tidy-core.json +26 -0
- framework/_assets/sets/packages/tidy-database.json +38 -0
- framework/_assets/sets/packages/tidy-omop.json +58 -0
- framework/_assets/sets/packages/tidymodels.json +22 -0
- framework/_assets/sets/packages/tidyverse-extended.json +70 -0
- framework/bootstrap.py +32 -6
- framework/cache.py +9 -5
- framework/catalog.py +8 -4
- framework/cloud.py +19 -13
- framework/config_home.py +18 -2
- framework/connections.py +45 -7
- framework/create.py +22 -1
- framework/data.py +87 -48
- framework/hooks.py +40 -3
- framework/notebooks.py +43 -21
- framework/packages.py +117 -0
- framework/publish_bundle.py +59 -2
- framework/scaffold.py +10 -2
- framework/security.py +24 -12
- framework/storage.py +142 -11
- framework/templates/defaults.yml +1 -1
- framework/templates/gitignore/course +2 -5
- {framework_py-0.1.0.dev7.dist-info → framework_py-0.1.0.dev8.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev7.dist-info → framework_py-0.1.0.dev8.dist-info}/RECORD +46 -22
- {framework_py-0.1.0.dev7.dist-info → framework_py-0.1.0.dev8.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev7.dist-info → framework_py-0.1.0.dev8.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "tidy-omop",
|
|
3
|
+
"name": "Tidy OMOP",
|
|
4
|
+
"description": "The DARWIN EU packages for OMOP CDM data: CDMConnector and omopgenerics to connect, CodelistGenerator and CohortConstructor for cohorts, PatientProfiles, IncidencePrevalence, CohortCharacteristics, and DrugUtilisation for studies, visOmopResults for results, and omock for test data.",
|
|
5
|
+
"language": [
|
|
6
|
+
"r"
|
|
7
|
+
],
|
|
8
|
+
"layer": {
|
|
9
|
+
"packages": {
|
|
10
|
+
"default_packages": {
|
|
11
|
+
"DBI": {
|
|
12
|
+
"source": "cran",
|
|
13
|
+
"auto_attach": false
|
|
14
|
+
},
|
|
15
|
+
"CDMConnector": {
|
|
16
|
+
"source": "cran",
|
|
17
|
+
"auto_attach": false
|
|
18
|
+
},
|
|
19
|
+
"omopgenerics": {
|
|
20
|
+
"source": "cran",
|
|
21
|
+
"auto_attach": false
|
|
22
|
+
},
|
|
23
|
+
"CodelistGenerator": {
|
|
24
|
+
"source": "cran",
|
|
25
|
+
"auto_attach": false
|
|
26
|
+
},
|
|
27
|
+
"CohortConstructor": {
|
|
28
|
+
"source": "cran",
|
|
29
|
+
"auto_attach": false
|
|
30
|
+
},
|
|
31
|
+
"PatientProfiles": {
|
|
32
|
+
"source": "cran",
|
|
33
|
+
"auto_attach": false
|
|
34
|
+
},
|
|
35
|
+
"IncidencePrevalence": {
|
|
36
|
+
"source": "cran",
|
|
37
|
+
"auto_attach": false
|
|
38
|
+
},
|
|
39
|
+
"CohortCharacteristics": {
|
|
40
|
+
"source": "cran",
|
|
41
|
+
"auto_attach": false
|
|
42
|
+
},
|
|
43
|
+
"DrugUtilisation": {
|
|
44
|
+
"source": "cran",
|
|
45
|
+
"auto_attach": false
|
|
46
|
+
},
|
|
47
|
+
"visOmopResults": {
|
|
48
|
+
"source": "cran",
|
|
49
|
+
"auto_attach": false
|
|
50
|
+
},
|
|
51
|
+
"omock": {
|
|
52
|
+
"source": "cran",
|
|
53
|
+
"auto_attach": false
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "tidymodels",
|
|
3
|
+
"name": "Tidymodels",
|
|
4
|
+
"description": "tidymodels for machine learning with recipes, resampling, and tuning, and vip for variable importance.",
|
|
5
|
+
"language": [
|
|
6
|
+
"r"
|
|
7
|
+
],
|
|
8
|
+
"layer": {
|
|
9
|
+
"packages": {
|
|
10
|
+
"default_packages": {
|
|
11
|
+
"tidymodels": {
|
|
12
|
+
"source": "cran",
|
|
13
|
+
"auto_attach": false
|
|
14
|
+
},
|
|
15
|
+
"vip": {
|
|
16
|
+
"source": "cran",
|
|
17
|
+
"auto_attach": false
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "tidyverse-extended",
|
|
3
|
+
"name": "Tidyverse extended",
|
|
4
|
+
"description": "The tidyverse analysis set plus readers for CSV, Excel, Stata, and SPSS files, janitor, glue, scales, and patchwork.",
|
|
5
|
+
"language": [
|
|
6
|
+
"r"
|
|
7
|
+
],
|
|
8
|
+
"layer": {
|
|
9
|
+
"packages": {
|
|
10
|
+
"default_packages": {
|
|
11
|
+
"dplyr": {
|
|
12
|
+
"source": "cran",
|
|
13
|
+
"auto_attach": true
|
|
14
|
+
},
|
|
15
|
+
"tidyr": {
|
|
16
|
+
"source": "cran",
|
|
17
|
+
"auto_attach": true
|
|
18
|
+
},
|
|
19
|
+
"ggplot2": {
|
|
20
|
+
"source": "cran",
|
|
21
|
+
"auto_attach": true
|
|
22
|
+
},
|
|
23
|
+
"purrr": {
|
|
24
|
+
"source": "cran",
|
|
25
|
+
"auto_attach": true
|
|
26
|
+
},
|
|
27
|
+
"stringr": {
|
|
28
|
+
"source": "cran",
|
|
29
|
+
"auto_attach": true
|
|
30
|
+
},
|
|
31
|
+
"forcats": {
|
|
32
|
+
"source": "cran",
|
|
33
|
+
"auto_attach": true
|
|
34
|
+
},
|
|
35
|
+
"lubridate": {
|
|
36
|
+
"source": "cran",
|
|
37
|
+
"auto_attach": true
|
|
38
|
+
},
|
|
39
|
+
"readr": {
|
|
40
|
+
"source": "cran",
|
|
41
|
+
"auto_attach": false
|
|
42
|
+
},
|
|
43
|
+
"readxl": {
|
|
44
|
+
"source": "cran",
|
|
45
|
+
"auto_attach": false
|
|
46
|
+
},
|
|
47
|
+
"haven": {
|
|
48
|
+
"source": "cran",
|
|
49
|
+
"auto_attach": false
|
|
50
|
+
},
|
|
51
|
+
"janitor": {
|
|
52
|
+
"source": "cran",
|
|
53
|
+
"auto_attach": false
|
|
54
|
+
},
|
|
55
|
+
"glue": {
|
|
56
|
+
"source": "cran",
|
|
57
|
+
"auto_attach": false
|
|
58
|
+
},
|
|
59
|
+
"scales": {
|
|
60
|
+
"source": "cran",
|
|
61
|
+
"auto_attach": false
|
|
62
|
+
},
|
|
63
|
+
"patchwork": {
|
|
64
|
+
"source": "cran",
|
|
65
|
+
"auto_attach": false
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
framework/bootstrap.py
CHANGED
|
@@ -4,7 +4,7 @@ Every step is safe to repeat and none contacts the cloud:
|
|
|
4
4
|
|
|
5
5
|
1. Make sure ``.gitignore`` ignores ``.env``.
|
|
6
6
|
2. Create ``.env`` from ``.env.example`` when it is missing, and list
|
|
7
|
-
variables that are still empty
|
|
7
|
+
variables that are still empty and that the settings read with ``env()``.
|
|
8
8
|
3. Create the project folders, which git does not carry when they are empty
|
|
9
9
|
or ignored.
|
|
10
10
|
4. Create ``framework.db``, or add tables a newer version needs.
|
|
@@ -79,7 +79,30 @@ def _ask(prompt: str) -> str:
|
|
|
79
79
|
return input()
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
# env("VAR"), !expr Sys.getenv("VAR"), and ${VAR} in the settings, as R finds them
|
|
83
|
+
_REFERENCE = re.compile(
|
|
84
|
+
r"""\b(?:env|Sys\.getenv)\(\s*["']([A-Za-z_][A-Za-z0-9_]*)["']"""
|
|
85
|
+
r"|\$\{([A-Za-z_][A-Za-z0-9_]*)\}"
|
|
86
|
+
)
|
|
87
|
+
_COMMENT = re.compile(r"(^|\s)#.*$", re.MULTILINE)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _referenced_variables(settings_file: Path) -> set[str]:
|
|
91
|
+
"""Variables the settings use: in settings.yml and the YAML files under settings/."""
|
|
92
|
+
files = [settings_file, *sorted((settings_file.parent / "settings").glob("*.y*ml"))]
|
|
93
|
+
names: set[str] = set()
|
|
94
|
+
for file in files:
|
|
95
|
+
try:
|
|
96
|
+
text = _COMMENT.sub("", file.read_text(encoding="utf-8"))
|
|
97
|
+
except (OSError, UnicodeError):
|
|
98
|
+
continue
|
|
99
|
+
names.update(a or b for a, b in _REFERENCE.findall(text))
|
|
100
|
+
return names
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _prepare_env(
|
|
104
|
+
root: Path, settings_file: Path, interactive: bool, result: BootstrapResult
|
|
105
|
+
) -> None:
|
|
83
106
|
env = root / ".env"
|
|
84
107
|
example = root / ".env.example"
|
|
85
108
|
if not env.exists() and example.is_file():
|
|
@@ -87,7 +110,9 @@ def _prepare_env(root: Path, interactive: bool, result: BootstrapResult) -> None
|
|
|
87
110
|
result.done.append("Created .env from .env.example")
|
|
88
111
|
if not env.is_file():
|
|
89
112
|
return
|
|
90
|
-
|
|
113
|
+
# Placeholders the settings never read, such as an unused S3 block, are not to-dos
|
|
114
|
+
used = _referenced_variables(settings_file)
|
|
115
|
+
empty = [key for key, value in dotenv.read(env).items() if value == "" and key in used]
|
|
91
116
|
if interactive and empty:
|
|
92
117
|
values: dict[str, str] = {}
|
|
93
118
|
for key in empty:
|
|
@@ -173,8 +198,9 @@ def bootstrap(
|
|
|
173
198
|
|
|
174
199
|
Args:
|
|
175
200
|
path: The project folder.
|
|
176
|
-
interactive: Ask for the values of empty ``.env`` variables
|
|
177
|
-
it, bootstrap never prompts and lists them as to-dos
|
|
201
|
+
interactive: Ask for the values of empty ``.env`` variables that the settings read
|
|
202
|
+
with ``env()``. Without it, bootstrap never prompts and lists them as to-dos
|
|
203
|
+
instead. Empty placeholders the settings never read are left alone.
|
|
178
204
|
packages: Install the packages with uv.
|
|
179
205
|
|
|
180
206
|
Returns:
|
|
@@ -207,7 +233,7 @@ def bootstrap(
|
|
|
207
233
|
|
|
208
234
|
if ensure_secrets_ignored(root):
|
|
209
235
|
result.done.append("Added .env to .gitignore")
|
|
210
|
-
_prepare_env(root, interactive, result)
|
|
236
|
+
_prepare_env(root, settings_file, interactive, result)
|
|
211
237
|
_create_folders(root, config, result)
|
|
212
238
|
existed = db.db_path(root).exists()
|
|
213
239
|
db.ensure_db(root)
|
framework/cache.py
CHANGED
|
@@ -36,15 +36,18 @@ T = TypeVar("T")
|
|
|
36
36
|
_FORMATS = (".parquet", ".pickle")
|
|
37
37
|
_R_FORMAT = ".rds"
|
|
38
38
|
_UNITS = {
|
|
39
|
+
**dict.fromkeys(("second", "seconds", "sec", "secs", "s"), 1 / 3600),
|
|
40
|
+
**dict.fromkeys(("minute", "minutes", "min", "mins"), 1 / 60),
|
|
39
41
|
**dict.fromkeys(("hour", "hours", "hr", "hrs", "h"), 1),
|
|
40
42
|
**dict.fromkeys(("day", "days", "d"), 24),
|
|
41
43
|
**dict.fromkeys(("week", "weeks", "w"), 24 * 7),
|
|
42
44
|
}
|
|
43
|
-
_DURATION = re.compile(r"^([0-9]*\.?[0-9]+)\s*(
|
|
45
|
+
_DURATION = re.compile(r"^([0-9]*\.?[0-9]+)\s*(" + "|".join(_UNITS) + r")$")
|
|
44
46
|
|
|
45
47
|
|
|
46
48
|
def normalize_expire_after(value: float | str | None, default: float | None = None) -> float | None:
|
|
47
|
-
"""Hours until expiry, from a number of hours or a string like ``"
|
|
49
|
+
"""Hours until expiry, from a number of hours or a string like ``"30 minutes"`` or
|
|
50
|
+
``"2 days"``. Seconds, minutes, hours, days, and weeks are accepted, as in R."""
|
|
48
51
|
if value is None or (isinstance(value, str) and not value.strip()):
|
|
49
52
|
return default
|
|
50
53
|
if isinstance(value, str):
|
|
@@ -52,7 +55,7 @@ def normalize_expire_after(value: float | str | None, default: float | None = No
|
|
|
52
55
|
if match is None:
|
|
53
56
|
raise FrameworkError(
|
|
54
57
|
"Could not parse expire_after value. Use numbers (hours) or strings like "
|
|
55
|
-
"'
|
|
58
|
+
"'30 minutes', '2 hours', '1 day', '1 week'."
|
|
56
59
|
)
|
|
57
60
|
return float(match.group(1)) * _UNITS[match.group(2)]
|
|
58
61
|
if value < 0:
|
|
@@ -120,7 +123,7 @@ def cache(
|
|
|
120
123
|
value: The value. DataFrames are stored as Parquet, anything else with pickle.
|
|
121
124
|
file: Store the value in this file instead of the cache folder.
|
|
122
125
|
expire_after: Hours until the value expires, or a string like
|
|
123
|
-
``"2 days"``. Default: ``options.data.cache_default_expire`` in the
|
|
126
|
+
``"30 minutes"`` or ``"2 days"``. Default: ``options.data.cache_default_expire`` in the
|
|
124
127
|
settings, else never.
|
|
125
128
|
"""
|
|
126
129
|
is_frame = isinstance(value, pd.DataFrame)
|
|
@@ -273,7 +276,8 @@ def cache_remember(
|
|
|
273
276
|
name: The cache name.
|
|
274
277
|
compute: Computes the value on a miss.
|
|
275
278
|
file: Store the value in this file instead of the cache folder.
|
|
276
|
-
expire_after: Hours until the value expires, or a string like ``"
|
|
279
|
+
expire_after: Hours until the value expires, or a string like ``"30 minutes"`` or
|
|
280
|
+
``"2 days"``.
|
|
277
281
|
refresh: Recompute even when cached. A function is called to decide.
|
|
278
282
|
"""
|
|
279
283
|
if compute is None:
|
framework/catalog.py
CHANGED
|
@@ -215,9 +215,10 @@ def _not_found(name: str, data: Any) -> DataNotFoundError:
|
|
|
215
215
|
def data_info(path: str) -> DataInfo:
|
|
216
216
|
"""How to read a data file, given a catalog name or a file path.
|
|
217
217
|
|
|
218
|
-
``path`` is a dot-notation catalog name (``"inputs.raw.survey"``), a
|
|
219
|
-
|
|
220
|
-
|
|
218
|
+
``path`` is a dot-notation catalog name (``"inputs.raw.survey"``), a relative
|
|
219
|
+
path (anything with a slash), or an absolute path. A relative path is looked for
|
|
220
|
+
from the project root, then from the current folder. Catalog paths are relative to
|
|
221
|
+
the project root.
|
|
221
222
|
|
|
222
223
|
Args:
|
|
223
224
|
path: A dot-notation catalog name, a relative path containing a slash, or an absolute
|
|
@@ -232,7 +233,10 @@ def data_info(path: str) -> DataInfo:
|
|
|
232
233
|
settings_file = find_settings_file()
|
|
233
234
|
root = settings_file.parent if settings_file else None
|
|
234
235
|
if _is_absolute(path) or "/" in path or "\\" in path:
|
|
235
|
-
|
|
236
|
+
# A relative path starts at the project root when the file is there, as
|
|
237
|
+
# data_save() writes it, else at the current folder
|
|
238
|
+
found = root is not None and (root / Path(path).expanduser()).exists()
|
|
239
|
+
return _build(path, root if found and root else Path.cwd(), None, None, root)
|
|
236
240
|
|
|
237
241
|
base = root or Path.cwd()
|
|
238
242
|
data = settings_read(settings_file).get("data")
|
framework/cloud.py
CHANGED
|
@@ -279,17 +279,24 @@ def cloud_token(url: str) -> str | None:
|
|
|
279
279
|
return token
|
|
280
280
|
|
|
281
281
|
|
|
282
|
+
def login_hint(url: str) -> str:
|
|
283
|
+
"""How to connect to ``url``: browser login first, a pasted token for scripts.
|
|
284
|
+
|
|
285
|
+
``cloud_status()`` and every call that needs a token give the same advice.
|
|
286
|
+
"""
|
|
287
|
+
where = "" if is_default(url) else f'url="{url}"'
|
|
288
|
+
browser = f"fw.cloud_login({where})"
|
|
289
|
+
token = f'fw.cloud_login("fw_...", {where})' if where else 'fw.cloud_login("fw_...")'
|
|
290
|
+
return (
|
|
291
|
+
f"Run {browser} to approve this computer in your browser, or create a token at "
|
|
292
|
+
f"{url}/tokens and run {token}."
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
|
|
282
296
|
def _require_token(url: str) -> str:
|
|
283
297
|
token = cloud_token(url)
|
|
284
298
|
if token is None:
|
|
285
|
-
|
|
286
|
-
'fw.cloud_login("fw_...")'
|
|
287
|
-
if is_default(url)
|
|
288
|
-
else f'fw.cloud_login("fw_...", url="{url}")'
|
|
289
|
-
)
|
|
290
|
-
raise CloudError(
|
|
291
|
-
f"No cloud token found for {host(url)}. Create one at {url}/tokens and run:\n {login}"
|
|
292
|
-
)
|
|
299
|
+
raise CloudError(f"No cloud token found for {host(url)}. {login_hint(url)}")
|
|
293
300
|
return token
|
|
294
301
|
|
|
295
302
|
|
|
@@ -663,10 +670,7 @@ def cloud_status(url: str | None = None) -> bool:
|
|
|
663
670
|
inform(f"This project publishes to {project.url} (from {project.source})")
|
|
664
671
|
token = cloud_token(server.url)
|
|
665
672
|
if token is None:
|
|
666
|
-
|
|
667
|
-
"cloud_login()" if is_default(server.url) else f'cloud_login(token, url="{server.url}")'
|
|
668
|
-
)
|
|
669
|
-
inform(f"Not connected. Run {login} to connect.")
|
|
673
|
+
inform(f"Not connected. {login_hint(server.url)}")
|
|
670
674
|
return False
|
|
671
675
|
try:
|
|
672
676
|
me = api("/api/v1/me", token, server.url)
|
|
@@ -921,7 +925,9 @@ def data_verify(name: str) -> bool:
|
|
|
921
925
|
mismatch.
|
|
922
926
|
|
|
923
927
|
Args:
|
|
924
|
-
name: A catalog name or file path, as accepted by ``data_info()``.
|
|
928
|
+
name: A catalog name or file path, as accepted by ``data_info()``. A relative path
|
|
929
|
+
is looked for from the project root first, so the path ``data_save()`` wrote
|
|
930
|
+
works from a notebook in ``notebooks/``.
|
|
925
931
|
|
|
926
932
|
Returns:
|
|
927
933
|
True when every available record matches the file, otherwise False.
|
framework/config_home.py
CHANGED
|
@@ -204,14 +204,30 @@ def project_unregister(project: int | str | Path) -> bool:
|
|
|
204
204
|
|
|
205
205
|
|
|
206
206
|
def project_list() -> pd.DataFrame:
|
|
207
|
-
"""Registered projects, with columns id, path, and
|
|
207
|
+
"""Registered projects, with columns id, path, created, and exists.
|
|
208
|
+
|
|
209
|
+
A project whose folder no longer exists (such as one made in a temporary folder) has
|
|
210
|
+
``exists`` False, and a message names it. Remove it with ``project_unregister(id)``.
|
|
211
|
+
"""
|
|
212
|
+
from framework._log import inform
|
|
213
|
+
|
|
208
214
|
projects = global_settings().get("projects") or []
|
|
209
215
|
rows = [
|
|
210
216
|
{
|
|
211
217
|
"id": int(p.get("id", 0)),
|
|
212
218
|
"path": str(p.get("path", "")),
|
|
213
219
|
"created": str(p.get("created", "")),
|
|
220
|
+
"exists": Path(str(p.get("path", ""))).expanduser().is_dir(),
|
|
214
221
|
}
|
|
215
222
|
for p in projects
|
|
216
223
|
]
|
|
217
|
-
|
|
224
|
+
missing = [row for row in rows if not row["exists"]]
|
|
225
|
+
if missing:
|
|
226
|
+
one = len(missing) == 1
|
|
227
|
+
names = ", ".join(f"{row['path']} (id {row['id']})" for row in missing)
|
|
228
|
+
inform(
|
|
229
|
+
f"{len(missing)} registered project folder{'' if one else 's'} no longer "
|
|
230
|
+
f"exist{'s' if one else ''}: {names}. Remove {'it' if one else 'them'} with "
|
|
231
|
+
"project_unregister(id)."
|
|
232
|
+
)
|
|
233
|
+
return pd.DataFrame(rows, columns=["id", "path", "created", "exists"])
|
framework/connections.py
CHANGED
|
@@ -18,15 +18,19 @@ Connections live under ``connections`` in the settings (usually
|
|
|
18
18
|
Credentials come from ``.env`` through ``env()``, so they never sit in the
|
|
19
19
|
settings. Drivers: sqlite (built in), postgres (``postgres`` extra), mysql or
|
|
20
20
|
mariadb (``mysql``), duckdb (``duckdb``), and sqlserver (``sqlserver``).
|
|
21
|
+
``read_only: true`` on a sqlite or duckdb connection opens the file read-only:
|
|
22
|
+
writes fail, and ``db_execute()`` and ``db_transaction()`` refuse it.
|
|
21
23
|
"""
|
|
22
24
|
|
|
23
25
|
from __future__ import annotations
|
|
24
26
|
|
|
27
|
+
import os
|
|
25
28
|
import re
|
|
26
29
|
import sqlite3
|
|
27
30
|
from collections.abc import Iterator, Mapping, Sequence
|
|
28
31
|
from contextlib import contextmanager
|
|
29
32
|
from importlib import import_module, metadata
|
|
33
|
+
from pathlib import Path
|
|
30
34
|
from types import TracebackType
|
|
31
35
|
from typing import Any
|
|
32
36
|
|
|
@@ -156,8 +160,25 @@ def _file_path(config: Mapping[str, Any], what: str) -> str:
|
|
|
156
160
|
return str(path)
|
|
157
161
|
|
|
158
162
|
|
|
163
|
+
def _writable(path: str) -> str:
|
|
164
|
+
"""``path``, after checking that an existing file can be written, as R checks."""
|
|
165
|
+
if Path(path).is_file() and not os.access(path, os.W_OK):
|
|
166
|
+
raise FrameworkError(
|
|
167
|
+
f"No write permission for database: {path}. Add read_only: true to the "
|
|
168
|
+
"connection to open it read-only."
|
|
169
|
+
)
|
|
170
|
+
return path
|
|
171
|
+
|
|
172
|
+
|
|
159
173
|
def _connect_sqlite(config: Mapping[str, Any]) -> Any:
|
|
160
|
-
|
|
174
|
+
path = _file_path(config, "SQLite")
|
|
175
|
+
if not config.get("read_only"):
|
|
176
|
+
return sqlite3.connect(_writable(path))
|
|
177
|
+
# mode=ro opens the file without write access, so every write fails and a file
|
|
178
|
+
# without write permission opens
|
|
179
|
+
if not Path(path).is_file():
|
|
180
|
+
raise FrameworkError(f"Read-only database does not exist: {path}")
|
|
181
|
+
return sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=ro", uri=True)
|
|
161
182
|
|
|
162
183
|
|
|
163
184
|
def _connect_postgres(config: Mapping[str, Any]) -> Any:
|
|
@@ -199,8 +220,11 @@ def _connect_duckdb(config: Mapping[str, Any]) -> Any:
|
|
|
199
220
|
options = {
|
|
200
221
|
key: config[key] for key in ("memory_limit", "threads") if config.get(key) is not None
|
|
201
222
|
}
|
|
223
|
+
read_only = bool(config.get("read_only"))
|
|
224
|
+
path = _file_path(config, "DuckDB")
|
|
225
|
+
# A read-only database needs no write permission
|
|
202
226
|
return duckdb.connect(
|
|
203
|
-
|
|
227
|
+
path if read_only else _writable(path), read_only=read_only, config=options
|
|
204
228
|
)
|
|
205
229
|
|
|
206
230
|
|
|
@@ -270,12 +294,14 @@ class Connection:
|
|
|
270
294
|
name: The connection name in the settings.
|
|
271
295
|
driver: sqlite, postgres, mysql, duckdb, or sqlserver.
|
|
272
296
|
raw: The driver's connection object.
|
|
297
|
+
read_only: Whether it was opened with ``read_only: true`` (sqlite and duckdb).
|
|
273
298
|
"""
|
|
274
299
|
|
|
275
|
-
def __init__(self, name: str, driver: str, raw: Any) -> None:
|
|
300
|
+
def __init__(self, name: str, driver: str, raw: Any, read_only: bool = False) -> None:
|
|
276
301
|
self.name = name
|
|
277
302
|
self.driver = driver
|
|
278
303
|
self.raw = raw
|
|
304
|
+
self.read_only = read_only
|
|
279
305
|
self._in_transaction = False
|
|
280
306
|
|
|
281
307
|
def __getattr__(self, attribute: str) -> Any:
|
|
@@ -332,7 +358,8 @@ def db_connect(name: str) -> Connection:
|
|
|
332
358
|
raise
|
|
333
359
|
except Exception as error:
|
|
334
360
|
raise FrameworkError(f"Failed to connect to '{name}': {error}") from error
|
|
335
|
-
|
|
361
|
+
read_only = driver in ("sqlite", "duckdb") and bool(entry.get("read_only"))
|
|
362
|
+
return Connection(name, driver, raw, read_only)
|
|
336
363
|
|
|
337
364
|
|
|
338
365
|
# Queries ------------------------------------------------------------------------------
|
|
@@ -393,9 +420,15 @@ def db_execute(
|
|
|
393
420
|
The driver's count of affected rows, or -1 when the driver does not report one.
|
|
394
421
|
|
|
395
422
|
Raises:
|
|
396
|
-
FrameworkError: When the connection cannot be opened or the
|
|
423
|
+
FrameworkError: When the connection cannot be opened, is read-only, or the
|
|
424
|
+
statement fails.
|
|
397
425
|
"""
|
|
398
426
|
with db_connect(connection_name) as connection:
|
|
427
|
+
if connection.read_only:
|
|
428
|
+
raise FrameworkError(
|
|
429
|
+
f"Connection '{connection_name}' is read-only (read_only: true in its "
|
|
430
|
+
"settings). db_execute() can't change it."
|
|
431
|
+
)
|
|
399
432
|
cursor = connection.cursor()
|
|
400
433
|
try:
|
|
401
434
|
cursor.execute(query, params) if params is not None else cursor.execute(query)
|
|
@@ -440,9 +473,14 @@ def db_transaction(connection: Connection) -> Iterator[Connection]:
|
|
|
440
473
|
A context manager that yields ``connection``.
|
|
441
474
|
|
|
442
475
|
Raises:
|
|
443
|
-
FrameworkError: When the block raises an exception.
|
|
444
|
-
the original exception is chained as the cause.
|
|
476
|
+
FrameworkError: When the connection is read-only, or the block raises an exception.
|
|
477
|
+
The transaction is rolled back and the original exception is chained as the cause.
|
|
445
478
|
"""
|
|
479
|
+
if connection.read_only:
|
|
480
|
+
raise FrameworkError(
|
|
481
|
+
f"Connection '{connection.name}' is read-only (read_only: true in its settings). "
|
|
482
|
+
"db_transaction() needs a writable connection."
|
|
483
|
+
)
|
|
446
484
|
if connection._in_transaction:
|
|
447
485
|
yield connection
|
|
448
486
|
return
|
framework/create.py
CHANGED
|
@@ -283,9 +283,28 @@ def _dependency(name: str, entry: Mapping[str, Any]) -> str:
|
|
|
283
283
|
return f"{name}{version or ''}"
|
|
284
284
|
|
|
285
285
|
|
|
286
|
+
# What Quarto's Jupyter engine imports to run Python chunks (nbformat and nbclient,
|
|
287
|
+
# in Quarto's share/jupyter/jupyter.py), and the kernel it starts.
|
|
288
|
+
_QUARTO_PACKAGES = ("ipykernel", "nbformat", "nbclient")
|
|
289
|
+
# The kernel, and JupyterLab, so `uv run jupyter lab` opens the notebooks.
|
|
290
|
+
_JUPYTER_PACKAGES = ("ipykernel", "jupyterlab")
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _package_key(name: str) -> str:
|
|
294
|
+
return re.sub(r"[-_.]+", "-", name).lower()
|
|
295
|
+
|
|
296
|
+
|
|
286
297
|
def _pyproject(spec: Mapping[str, Any]) -> str:
|
|
287
298
|
packages = (spec.get("packages") or {}).get("default_packages") or {}
|
|
288
299
|
dependencies = ["framework-py", *(_dependency(n, e) for n, e in packages.items())]
|
|
300
|
+
# A project renders and edits with its workflow's tools out of the box. The spec's
|
|
301
|
+
# packages win when they name one of these, with a version for example.
|
|
302
|
+
tools = [
|
|
303
|
+
*(_QUARTO_PACKAGES if spec_tool_on(spec, "quarto") else ()),
|
|
304
|
+
*(_JUPYTER_PACKAGES if spec_tool_on(spec, "jupyter") else ()),
|
|
305
|
+
]
|
|
306
|
+
named = {_package_key(name) for name in packages}
|
|
307
|
+
dependencies += [name for name in dict.fromkeys(tools) if _package_key(name) not in named]
|
|
289
308
|
lines = [
|
|
290
309
|
"[project]",
|
|
291
310
|
f'name = "{kebab_case(spec["name"])}"',
|
|
@@ -368,7 +387,9 @@ def _git_init(root: Path) -> None:
|
|
|
368
387
|
def _is_temporary(root: Path) -> bool:
|
|
369
388
|
temp = Path(tempfile.gettempdir()).resolve()
|
|
370
389
|
resolved = root.resolve()
|
|
371
|
-
return resolved.is_relative_to(temp) or str(resolved).startswith(
|
|
390
|
+
return resolved.is_relative_to(temp) or str(resolved).startswith(
|
|
391
|
+
("/tmp/", "/private/tmp/", "/var/folders/", "/private/var/folders/")
|
|
392
|
+
)
|
|
372
393
|
|
|
373
394
|
|
|
374
395
|
# Public API -----------------------------------------------------------------
|