framework-py 0.1.0.dev9__py3-none-any.whl → 0.1.0.dev11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +12 -1
- framework/_agents_md.py +46 -2
- framework/_csv.py +129 -0
- framework/_depends.py +194 -0
- framework/_identifiers.py +146 -0
- framework/_private.py +204 -15
- framework/_sql.py +117 -0
- framework/_tabular.py +388 -0
- framework/cache.py +92 -7
- framework/catalog.py +87 -8
- framework/connections.py +262 -6
- framework/create.py +41 -9
- framework/data.py +178 -37
- framework/display.py +142 -0
- framework/errors.py +2 -1
- framework/hooks.py +16 -2
- framework/notebooks.py +39 -8
- framework/outputs.py +308 -41
- framework/packages.py +20 -0
- framework/project.py +49 -1
- framework/publish_bundle.py +87 -1
- framework/security.py +326 -30
- framework/settings.py +38 -0
- framework/spec/resolve.py +14 -1
- framework/storage.py +69 -9
- framework/templates/skills/framework-data/.framework-skill.json +1 -1
- framework/templates/skills/framework-data/SKILL.md +5 -2
- framework/templates/skills/framework-sensitive-data/.framework-skill.json +1 -1
- framework/templates/skills/framework-sensitive-data/SKILL.md +23 -2
- {framework_py-0.1.0.dev9.dist-info → framework_py-0.1.0.dev11.dist-info}/METADATA +5 -3
- {framework_py-0.1.0.dev9.dist-info → framework_py-0.1.0.dev11.dist-info}/RECORD +33 -27
- {framework_py-0.1.0.dev9.dist-info → framework_py-0.1.0.dev11.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev9.dist-info → framework_py-0.1.0.dev11.dist-info}/licenses/LICENSE +0 -0
framework/__init__.py
CHANGED
|
@@ -45,8 +45,10 @@ from framework.connections import (
|
|
|
45
45
|
Connection,
|
|
46
46
|
connections_list,
|
|
47
47
|
db_connect,
|
|
48
|
+
db_copy_from,
|
|
48
49
|
db_drivers_status,
|
|
49
50
|
db_execute,
|
|
51
|
+
db_execute_file,
|
|
50
52
|
db_list,
|
|
51
53
|
db_query,
|
|
52
54
|
db_transaction,
|
|
@@ -60,6 +62,7 @@ from framework.create import (
|
|
|
60
62
|
project_create_from_layers,
|
|
61
63
|
)
|
|
62
64
|
from framework.data import data_add, data_read, data_save
|
|
65
|
+
from framework.display import TableDisplay, show_table
|
|
63
66
|
from framework.errors import (
|
|
64
67
|
CacheWarning,
|
|
65
68
|
DataChangedWarning,
|
|
@@ -98,6 +101,7 @@ from framework.notebooks import (
|
|
|
98
101
|
stubs_publish,
|
|
99
102
|
)
|
|
100
103
|
from framework.outputs import (
|
|
104
|
+
load_model,
|
|
101
105
|
result_list,
|
|
102
106
|
save_figure,
|
|
103
107
|
save_model,
|
|
@@ -117,7 +121,7 @@ from framework.packages import (
|
|
|
117
121
|
packages_status,
|
|
118
122
|
packages_update,
|
|
119
123
|
)
|
|
120
|
-
from framework.project import project_root
|
|
124
|
+
from framework.project import path, project_path, project_root
|
|
121
125
|
from framework.quarto import quarto_generate_all, quarto_regenerate
|
|
122
126
|
from framework.scaffold import project_info, scaffold, status
|
|
123
127
|
from framework.scratch import scratch_capture, scratch_clean
|
|
@@ -159,6 +163,7 @@ __all__ = [
|
|
|
159
163
|
"ResolvedSpec",
|
|
160
164
|
"SettingsError",
|
|
161
165
|
"SpecError",
|
|
166
|
+
"TableDisplay",
|
|
162
167
|
"UnsupportedFormatError",
|
|
163
168
|
"__version__",
|
|
164
169
|
"adopt",
|
|
@@ -187,8 +192,10 @@ __all__ = [
|
|
|
187
192
|
"data_save",
|
|
188
193
|
"data_verify",
|
|
189
194
|
"db_connect",
|
|
195
|
+
"db_copy_from",
|
|
190
196
|
"db_drivers_status",
|
|
191
197
|
"db_execute",
|
|
198
|
+
"db_execute_file",
|
|
192
199
|
"db_list",
|
|
193
200
|
"db_query",
|
|
194
201
|
"db_transaction",
|
|
@@ -202,6 +209,7 @@ __all__ = [
|
|
|
202
209
|
"global_settings",
|
|
203
210
|
"ledger_list",
|
|
204
211
|
"ledger_push",
|
|
212
|
+
"load_model",
|
|
205
213
|
"load_settings_catalog",
|
|
206
214
|
"make_notebook",
|
|
207
215
|
"make_presentation",
|
|
@@ -220,11 +228,13 @@ __all__ = [
|
|
|
220
228
|
"packages_snapshot",
|
|
221
229
|
"packages_status",
|
|
222
230
|
"packages_update",
|
|
231
|
+
"path",
|
|
223
232
|
"project_add_directory",
|
|
224
233
|
"project_create",
|
|
225
234
|
"project_create_from_layers",
|
|
226
235
|
"project_info",
|
|
227
236
|
"project_list",
|
|
237
|
+
"project_path",
|
|
228
238
|
"project_register",
|
|
229
239
|
"project_root",
|
|
230
240
|
"project_unregister",
|
|
@@ -249,6 +259,7 @@ __all__ = [
|
|
|
249
259
|
"settings",
|
|
250
260
|
"settings_read",
|
|
251
261
|
"settings_write",
|
|
262
|
+
"show_table",
|
|
252
263
|
"skills_sync",
|
|
253
264
|
"spec_check",
|
|
254
265
|
"spec_resolve",
|
framework/_agents_md.py
CHANGED
|
@@ -7,9 +7,12 @@ user's to edit.
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
import re
|
|
11
|
+
from pathlib import Path
|
|
10
12
|
from typing import Any
|
|
11
13
|
|
|
12
14
|
REGENERATE = "<!-- @framework:regenerate -->"
|
|
15
|
+
PACKAGES_HEADING = "Installed Packages"
|
|
13
16
|
|
|
14
17
|
SKILL_DESCRIPTIONS = {
|
|
15
18
|
"framework-workflow": "scaffold() initialization, critical rules, creating notebooks/scripts",
|
|
@@ -119,13 +122,54 @@ def _environment(spec: dict[str, Any]) -> str:
|
|
|
119
122
|
def _packages(spec: dict[str, Any]) -> str:
|
|
120
123
|
from framework.create import pyproject_dependencies
|
|
121
124
|
|
|
122
|
-
|
|
123
|
-
|
|
125
|
+
return packages_section([name for name, _ in pyproject_dependencies(spec)])
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def packages_section(names: list[str]) -> str:
|
|
129
|
+
"""The Installed Packages section, listing the project's dependencies by name."""
|
|
130
|
+
lines = [f"## {PACKAGES_HEADING} {REGENERATE}", ""]
|
|
124
131
|
lines += [f"**Dependencies** (in pyproject.toml): {', '.join(names)}", ""]
|
|
125
132
|
lines.append('Add packages with `fw.package_add("name")`.')
|
|
126
133
|
return "\n".join(lines)
|
|
127
134
|
|
|
128
135
|
|
|
136
|
+
def context_file(root: Path) -> Path | None:
|
|
137
|
+
"""The project's AI context file: ``ai.canonical_file`` in the settings, else AGENTS.md,
|
|
138
|
+
else CLAUDE.md for older projects, as R's ``ai_regenerate_context()`` picks it."""
|
|
139
|
+
from framework.settings import settings_read
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
ai = settings_read(root / "settings.yml").get("ai")
|
|
143
|
+
name = ai.get("canonical_file") if isinstance(ai, dict) else None
|
|
144
|
+
except Exception:
|
|
145
|
+
name = None
|
|
146
|
+
for candidate in (name, "AGENTS.md", "CLAUDE.md"):
|
|
147
|
+
if isinstance(candidate, str) and candidate and (root / candidate).is_file():
|
|
148
|
+
return root / candidate
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def refresh_packages(root: Path, names: list[str]) -> Path | None:
|
|
153
|
+
"""Rewrite the context file's Installed Packages section with ``names``.
|
|
154
|
+
|
|
155
|
+
Only a file with the marked heading changes; the rest of it is kept. Returns the file
|
|
156
|
+
when it changed.
|
|
157
|
+
"""
|
|
158
|
+
path = context_file(root)
|
|
159
|
+
if path is None:
|
|
160
|
+
return None
|
|
161
|
+
text = path.read_text(encoding="utf-8")
|
|
162
|
+
heading = re.escape(f"## {PACKAGES_HEADING} {REGENERATE}")
|
|
163
|
+
found = re.search(heading + r".*?(?=\n+## |\n*\Z)", text, re.DOTALL)
|
|
164
|
+
if found is None:
|
|
165
|
+
return None
|
|
166
|
+
updated = text[: found.start()] + packages_section(names) + text[found.end() :]
|
|
167
|
+
if updated == text:
|
|
168
|
+
return None
|
|
169
|
+
path.write_text(updated, encoding="utf-8")
|
|
170
|
+
return path
|
|
171
|
+
|
|
172
|
+
|
|
129
173
|
def _data(spec: dict[str, Any]) -> str:
|
|
130
174
|
directories = spec.get("directories") or {}
|
|
131
175
|
rows = [
|
framework/_csv.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Writing CSV files in the form R's readr writes them.
|
|
2
|
+
|
|
3
|
+
R's ``data_save()`` writes CSV with ``readr::write_delim()``. pandas writes whole-number
|
|
4
|
+
doubles as ``1.0``, booleans as ``True``, and missing values as empty fields, so the same
|
|
5
|
+
data gave different files, and different hashes, in the two languages. ``write_csv()``
|
|
6
|
+
writes what readr writes:
|
|
7
|
+
|
|
8
|
+
- Doubles in readr's shortest form: ``1`` and ``0.5``, ``1e15`` and ``1.5e-4`` past the
|
|
9
|
+
limits where readr switches to an exponent, ``Inf``, and ``NA`` for NaN.
|
|
10
|
+
- Logicals as ``TRUE`` and ``FALSE``.
|
|
11
|
+
- Missing values as ``NA``.
|
|
12
|
+
- Date-times in UTC as ``2020-01-02T03:04:05Z``, and dates as ``2020-01-02``. readr drops
|
|
13
|
+
fractional seconds; this writer keeps them (``03:04:05.5Z``), which readr reads back.
|
|
14
|
+
- Fields quoted only when needed, and ``\\n`` line endings on every platform.
|
|
15
|
+
|
|
16
|
+
Other writers that should match R, such as ``save_table()``, can call ``write_csv()``.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import datetime as dt
|
|
22
|
+
import math
|
|
23
|
+
from decimal import Decimal
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
import pandas as pd
|
|
28
|
+
|
|
29
|
+
NA = "NA"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def format_double(value: float) -> str:
|
|
33
|
+
"""One double as readr writes it."""
|
|
34
|
+
if math.isnan(value):
|
|
35
|
+
return NA
|
|
36
|
+
if math.isinf(value):
|
|
37
|
+
return "Inf" if value > 0 else "-Inf"
|
|
38
|
+
if value == 0:
|
|
39
|
+
return "-0" if math.copysign(1.0, value) < 0 else "0"
|
|
40
|
+
# repr() gives the shortest digits that read back as the same double, as readr's
|
|
41
|
+
# writer does. The value is digits * 10**exponent.
|
|
42
|
+
sign, digit_tuple, exponent = Decimal(repr(float(value))).as_tuple()
|
|
43
|
+
assert isinstance(exponent, int)
|
|
44
|
+
digits = "".join(str(d) for d in digit_tuple)
|
|
45
|
+
stripped = digits.rstrip("0")
|
|
46
|
+
exponent += len(digits) - len(stripped)
|
|
47
|
+
digits = stripped
|
|
48
|
+
point = len(digits) + exponent # where the decimal point falls
|
|
49
|
+
prefix = "-" if sign else ""
|
|
50
|
+
if exponent >= 0:
|
|
51
|
+
if exponent == 0 or point <= 15:
|
|
52
|
+
return prefix + digits + "0" * exponent
|
|
53
|
+
return f"{prefix}{digits}e{exponent}"
|
|
54
|
+
if point > 0:
|
|
55
|
+
return f"{prefix}{digits[:point]}.{digits[point:]}"
|
|
56
|
+
if point > -3:
|
|
57
|
+
return f"{prefix}0.{'0' * -point}{digits}"
|
|
58
|
+
mantissa = digits[0] + (f".{digits[1:]}" if len(digits) > 1 else "")
|
|
59
|
+
return f"{prefix}{mantissa}e{point - 1}"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _format_datetime(value: pd.Timestamp | dt.datetime) -> str:
|
|
63
|
+
stamp = pd.Timestamp(value)
|
|
64
|
+
if stamp.tzinfo is not None:
|
|
65
|
+
stamp = stamp.tz_convert("UTC")
|
|
66
|
+
text = stamp.strftime("%Y-%m-%dT%H:%M:%S")
|
|
67
|
+
fraction = stamp.microsecond * 1000 + stamp.nanosecond
|
|
68
|
+
if fraction:
|
|
69
|
+
text += f".{fraction:09d}".rstrip("0")
|
|
70
|
+
return text + "Z"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _format_value(value: Any) -> str:
|
|
74
|
+
if value is None or value is pd.NA or value is pd.NaT:
|
|
75
|
+
return NA
|
|
76
|
+
if isinstance(value, bool):
|
|
77
|
+
return "TRUE" if value else "FALSE"
|
|
78
|
+
if isinstance(value, float):
|
|
79
|
+
return format_double(value)
|
|
80
|
+
if isinstance(value, int):
|
|
81
|
+
return str(value)
|
|
82
|
+
if isinstance(value, pd.Timestamp | dt.datetime):
|
|
83
|
+
return _format_datetime(value)
|
|
84
|
+
if isinstance(value, dt.date):
|
|
85
|
+
return value.isoformat()
|
|
86
|
+
try:
|
|
87
|
+
if pd.isna(value):
|
|
88
|
+
return NA
|
|
89
|
+
except (TypeError, ValueError):
|
|
90
|
+
pass
|
|
91
|
+
if hasattr(value, "item"):
|
|
92
|
+
# numpy scalars: format the Python value
|
|
93
|
+
return _format_value(value.item())
|
|
94
|
+
return str(value)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _format_column(column: pd.Series) -> pd.Series:
|
|
98
|
+
kind = column.dtype
|
|
99
|
+
if pd.api.types.is_bool_dtype(kind):
|
|
100
|
+
values = column.map(lambda v: NA if pd.isna(v) else ("TRUE" if v else "FALSE"))
|
|
101
|
+
elif pd.api.types.is_integer_dtype(kind):
|
|
102
|
+
values = column.map(lambda v: NA if pd.isna(v) else str(int(v)))
|
|
103
|
+
elif pd.api.types.is_float_dtype(kind):
|
|
104
|
+
values = column.map(lambda v: NA if pd.isna(v) else format_double(float(v)))
|
|
105
|
+
elif pd.api.types.is_datetime64_any_dtype(kind):
|
|
106
|
+
values = column.map(lambda v: NA if pd.isna(v) else _format_datetime(v))
|
|
107
|
+
elif isinstance(kind, pd.CategoricalDtype):
|
|
108
|
+
values = column.astype(object).map(_format_value)
|
|
109
|
+
else:
|
|
110
|
+
values = column.map(_format_value)
|
|
111
|
+
return values.astype(object)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def format_like_r(frame: pd.DataFrame) -> pd.DataFrame:
|
|
115
|
+
"""``frame`` with every value as the text readr would write for it."""
|
|
116
|
+
if frame.columns.empty:
|
|
117
|
+
return frame.copy()
|
|
118
|
+
formatted = pd.concat(
|
|
119
|
+
[_format_column(frame.iloc[:, position]) for position in range(frame.shape[1])], axis=1
|
|
120
|
+
)
|
|
121
|
+
formatted.columns = frame.columns
|
|
122
|
+
return formatted
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def write_csv(frame: pd.DataFrame, path: str | Path, sep: str = ",") -> None:
|
|
126
|
+
"""Write ``frame`` without its index, as R's ``readr::write_delim()`` would."""
|
|
127
|
+
format_like_r(frame).to_csv(
|
|
128
|
+
path, sep=sep, index=False, lineterminator="\n", na_rep=NA, encoding="utf-8"
|
|
129
|
+
)
|
framework/_depends.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Cache dependencies: what ``cache_remember(depends_on=...)`` hashes into an entry's key.
|
|
2
|
+
|
|
3
|
+
Each dependency is one of:
|
|
4
|
+
|
|
5
|
+
- ``data``: a catalog name, hashed as its file (SHA-256, as data integrity hashes it).
|
|
6
|
+
- ``file``: a file path, relative to the project root (then the current folder).
|
|
7
|
+
- ``folder``: a folder path, hashed as the sorted list of its files' paths and hashes.
|
|
8
|
+
Hidden files and folders and ``__pycache__`` are skipped.
|
|
9
|
+
- ``value``: any other value, hashed in this language only.
|
|
10
|
+
|
|
11
|
+
A string is always a reference (a catalog name, file, or folder) and must resolve, so a
|
|
12
|
+
misspelled name is an error rather than a silently hashed string. To hash a string value,
|
|
13
|
+
wrap it: ``{"rule": ">= 0"}``.
|
|
14
|
+
|
|
15
|
+
The entry's key is the SHA-256 of its dependencies' ``kind``, ``ref``, and ``hash``, one
|
|
16
|
+
line each, sorted. It is stored in framework.db's ``meta`` table under
|
|
17
|
+
``cache_depends:<name>``, as JSON with the key and the dependency list, so R can read and
|
|
18
|
+
write the same record.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import hashlib
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import pickle
|
|
27
|
+
from dataclasses import asdict, dataclass
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
import pandas as pd
|
|
32
|
+
|
|
33
|
+
from framework import db
|
|
34
|
+
from framework.errors import DataNotFoundError, FrameworkError
|
|
35
|
+
from framework.integrity import file_hash
|
|
36
|
+
from framework.project import find_settings_file
|
|
37
|
+
|
|
38
|
+
META_PREFIX = "cache_depends:"
|
|
39
|
+
_SKIPPED_FOLDERS = frozenset({"__pycache__"})
|
|
40
|
+
_REFERENCES = ("data", "file", "folder")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Dependency:
|
|
45
|
+
kind: str
|
|
46
|
+
ref: str
|
|
47
|
+
hash: str
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _sha256(data: bytes) -> str:
|
|
51
|
+
return hashlib.sha256(data).hexdigest()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def folder_hash(folder: Path) -> str:
|
|
55
|
+
"""The SHA-256 of ``<path>\\t<file hash>\\n`` for every file below ``folder``, sorted by
|
|
56
|
+
path (``/`` separators, relative to the folder)."""
|
|
57
|
+
lines = []
|
|
58
|
+
for current, folders, files in os.walk(folder):
|
|
59
|
+
folders[:] = [f for f in folders if not f.startswith(".") and f not in _SKIPPED_FOLDERS]
|
|
60
|
+
for name in files:
|
|
61
|
+
if name.startswith("."):
|
|
62
|
+
continue
|
|
63
|
+
path = Path(current) / name
|
|
64
|
+
lines.append(f"{path.relative_to(folder).as_posix()}\t{file_hash(path)}\n")
|
|
65
|
+
return _sha256("".join(sorted(lines)).encode("utf-8"))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def value_hash(value: Any) -> str:
|
|
69
|
+
"""A value's hash. Only meaningful within one language."""
|
|
70
|
+
if isinstance(value, pd.DataFrame | pd.Series):
|
|
71
|
+
frame = value.to_frame() if isinstance(value, pd.Series) else value
|
|
72
|
+
digest = hashlib.sha256()
|
|
73
|
+
digest.update(json.dumps([str(c) for c in frame.columns]).encode("utf-8"))
|
|
74
|
+
digest.update(json.dumps([str(t) for t in frame.dtypes]).encode("utf-8"))
|
|
75
|
+
digest.update(pd.util.hash_pandas_object(frame, index=True).to_numpy().tobytes())
|
|
76
|
+
return digest.hexdigest()
|
|
77
|
+
try:
|
|
78
|
+
text = json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=True)
|
|
79
|
+
return _sha256(f"json:{text}".encode())
|
|
80
|
+
except (TypeError, ValueError):
|
|
81
|
+
return _sha256(b"pickle:" + pickle.dumps(value, protocol=4))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _root() -> Path | None:
|
|
85
|
+
found = find_settings_file()
|
|
86
|
+
return found.parent if found is not None else None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _resolve_reference(ref: str) -> Dependency:
|
|
90
|
+
from framework.catalog import data_info
|
|
91
|
+
|
|
92
|
+
root = _root()
|
|
93
|
+
candidate = Path(ref).expanduser()
|
|
94
|
+
bases = [root] if root is not None else []
|
|
95
|
+
if not candidate.is_absolute():
|
|
96
|
+
for base in [*bases, Path.cwd()]:
|
|
97
|
+
if (base / candidate).exists():
|
|
98
|
+
candidate = base / candidate
|
|
99
|
+
break
|
|
100
|
+
if candidate.is_absolute() and candidate.is_dir():
|
|
101
|
+
return Dependency("folder", ref, folder_hash(candidate))
|
|
102
|
+
if candidate.is_absolute() and candidate.is_file():
|
|
103
|
+
return Dependency("file", ref, file_hash(candidate))
|
|
104
|
+
if "/" not in ref and "\\" not in ref:
|
|
105
|
+
try:
|
|
106
|
+
info = data_info(ref)
|
|
107
|
+
except DataNotFoundError:
|
|
108
|
+
info = None
|
|
109
|
+
if info is not None and info.name is not None:
|
|
110
|
+
if not info.path.is_file():
|
|
111
|
+
raise FrameworkError(f"depends_on: the file for '{ref}' is missing: {info.path}")
|
|
112
|
+
return Dependency("data", ref, file_hash(info.path))
|
|
113
|
+
raise FrameworkError(
|
|
114
|
+
f"depends_on: '{ref}' is not a catalog name, file, or folder. Strings name what the "
|
|
115
|
+
'entry depends on; to hash a string value, wrap it, as in {"rule": ">= 0"}.'
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def resolve(depends_on: Any) -> list[Dependency]:
|
|
120
|
+
"""The dependencies of ``depends_on``: one item, or a list or tuple of items."""
|
|
121
|
+
items = list(depends_on) if isinstance(depends_on, list | tuple) else [depends_on]
|
|
122
|
+
return [
|
|
123
|
+
_resolve_reference(item)
|
|
124
|
+
if isinstance(item, str)
|
|
125
|
+
else Dependency("value", f"value {position}", value_hash(item))
|
|
126
|
+
for position, item in enumerate(items, start=1)
|
|
127
|
+
]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def key(dependencies: list[Dependency]) -> str:
|
|
131
|
+
lines = sorted(f"{d.kind}\t{d.ref}\t{d.hash}\n" for d in dependencies)
|
|
132
|
+
return _sha256("".join(lines).encode("utf-8"))
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def recorded(name: str) -> tuple[str, list[Dependency]] | None:
|
|
136
|
+
"""The key and dependencies stored for a cache entry, or None."""
|
|
137
|
+
text = db.get_meta(META_PREFIX + name)
|
|
138
|
+
if not text:
|
|
139
|
+
return None
|
|
140
|
+
try:
|
|
141
|
+
data = json.loads(text)
|
|
142
|
+
dependencies = [Dependency(**item) for item in data["dependencies"]]
|
|
143
|
+
return str(data["key"]), dependencies
|
|
144
|
+
except (ValueError, KeyError, TypeError):
|
|
145
|
+
return None
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def record(name: str, dependencies: list[Dependency] | None) -> None:
|
|
149
|
+
if dependencies is None:
|
|
150
|
+
forget(name)
|
|
151
|
+
return
|
|
152
|
+
document = {"key": key(dependencies), "dependencies": [asdict(d) for d in dependencies]}
|
|
153
|
+
db.set_meta(META_PREFIX + name, json.dumps(document))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def forget(name: str | None = None) -> None:
|
|
157
|
+
"""Remove the stored dependencies of one entry, or of every entry."""
|
|
158
|
+
with db.connect() as connection:
|
|
159
|
+
if name is None:
|
|
160
|
+
connection.execute("DELETE FROM meta WHERE key LIKE ?", (META_PREFIX + "%",))
|
|
161
|
+
else:
|
|
162
|
+
connection.execute("DELETE FROM meta WHERE key = ?", (META_PREFIX + name,))
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def recheck(dependencies: list[Dependency]) -> tuple[list[Dependency], list[str]]:
|
|
166
|
+
"""The stored dependencies hashed again, and the references that changed.
|
|
167
|
+
|
|
168
|
+
Values cannot be hashed again without the caller, so they keep their stored hash. A
|
|
169
|
+
reference that no longer resolves counts as changed.
|
|
170
|
+
"""
|
|
171
|
+
current: list[Dependency] = []
|
|
172
|
+
changed: list[str] = []
|
|
173
|
+
for dependency in dependencies:
|
|
174
|
+
if dependency.kind not in _REFERENCES:
|
|
175
|
+
current.append(dependency)
|
|
176
|
+
continue
|
|
177
|
+
try:
|
|
178
|
+
fresh = _resolve_reference(dependency.ref)
|
|
179
|
+
except FrameworkError:
|
|
180
|
+
changed.append(dependency.ref)
|
|
181
|
+
continue
|
|
182
|
+
current.append(fresh)
|
|
183
|
+
if fresh.hash != dependency.hash:
|
|
184
|
+
changed.append(dependency.ref)
|
|
185
|
+
return current, changed
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def changed_between(old: list[Dependency], new: list[Dependency]) -> list[str]:
|
|
189
|
+
"""Names for what differs between two dependency lists, for messages."""
|
|
190
|
+
before = {(d.kind, d.ref): d.hash for d in old}
|
|
191
|
+
after = {(d.kind, d.ref): d.hash for d in new}
|
|
192
|
+
names = [ref for (kind, ref), digest in after.items() if before.get((kind, ref)) != digest]
|
|
193
|
+
names += [ref for (kind, ref) in before if (kind, ref) not in after]
|
|
194
|
+
return list(dict.fromkeys(names)) or ["the dependency list"]
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Columns that look like direct identifiers, found from column names alone.
|
|
2
|
+
|
|
3
|
+
A cheap check for real data about people where it doesn't belong: ``data_save()``
|
|
4
|
+
warns when it writes such columns outside a private folder, and the security audit
|
|
5
|
+
flags tracked data files whose header has them. Only names are read, never values,
|
|
6
|
+
so a file is opened for its header only.
|
|
7
|
+
|
|
8
|
+
A column name is split into lowercase tokens at non-alphanumeric characters and
|
|
9
|
+
camelCase boundaries (``patientDOB`` -> ``patient``, ``dob``), and ``joined`` is the
|
|
10
|
+
tokens run together. The kinds:
|
|
11
|
+
|
|
12
|
+
- dob: a token ``dob``; or tokens ``birth`` with ``date``, ``dt``, or ``day``; or
|
|
13
|
+
``joined`` containing ``dateofbirth``, ``birthdate``, or ``birthday``.
|
|
14
|
+
- zip: a token ``zip``, ``zipcode``, ``zip5``, ``zip9``, ``postal``, ``postcode``, or
|
|
15
|
+
``postalcode``; or ``joined`` containing ``zipcode`` or ``postalcode``. Three-digit
|
|
16
|
+
``zip3`` is not flagged.
|
|
17
|
+
- ssn: a token ``ssn``, or ``joined`` containing ``socialsecurity``.
|
|
18
|
+
- mrn: a token ``mrn``, or ``joined`` containing ``medicalrecordnumber``,
|
|
19
|
+
``medicalrecordno``, or ``medrec``.
|
|
20
|
+
- name: a token ``name`` with ``first``, ``last``, ``given``, ``family``, ``full``,
|
|
21
|
+
``middle``, ``maiden``, ``patient``, or ``person``; or ``joined`` equal to
|
|
22
|
+
``firstname``, ``lastname``, ``surname``, ``fullname``, ``fname``, or ``lname``.
|
|
23
|
+
Names alone are common in data that isn't about people (a site's name), so they
|
|
24
|
+
count only alongside a dob column.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import csv
|
|
30
|
+
import re
|
|
31
|
+
from collections.abc import Iterable
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
KINDS = ("dob", "zip", "ssn", "mrn", "name")
|
|
35
|
+
HEADER_TYPES = (".csv", ".tsv", ".txt", ".parquet", ".dta", ".sav")
|
|
36
|
+
|
|
37
|
+
_ZIP = {"zip", "zipcode", "zip5", "zip9", "postal", "postcode", "postalcode"}
|
|
38
|
+
_NAME_WITH = {"first", "last", "given", "family", "full", "middle", "maiden", "patient", "person"}
|
|
39
|
+
_NAMES = {"firstname", "lastname", "surname", "fullname", "fname", "lname"}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _tokens(column: str) -> list[str]:
|
|
43
|
+
spaced = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", column)
|
|
44
|
+
spaced = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1 \2", spaced)
|
|
45
|
+
return [token for token in re.split(r"[^a-z0-9]+", spaced.lower()) if token]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _kind(column: str) -> str | None:
|
|
49
|
+
tokens = _tokens(column)
|
|
50
|
+
words = set(tokens)
|
|
51
|
+
joined = "".join(tokens)
|
|
52
|
+
if (
|
|
53
|
+
"dob" in words
|
|
54
|
+
or ("birth" in words and words & {"date", "dt", "day"})
|
|
55
|
+
or any(part in joined for part in ("dateofbirth", "birthdate", "birthday"))
|
|
56
|
+
):
|
|
57
|
+
return "dob"
|
|
58
|
+
if words & _ZIP or "zipcode" in joined or "postalcode" in joined:
|
|
59
|
+
return "zip"
|
|
60
|
+
if "ssn" in words or "socialsecurity" in joined:
|
|
61
|
+
return "ssn"
|
|
62
|
+
if "mrn" in words or any(
|
|
63
|
+
p in joined for p in ("medicalrecordnumber", "medicalrecordno", "medrec")
|
|
64
|
+
):
|
|
65
|
+
return "mrn"
|
|
66
|
+
if ("name" in words and words & _NAME_WITH) or joined in _NAMES:
|
|
67
|
+
return "name"
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def identifier_columns(columns: Iterable[object]) -> list[tuple[str, str]]:
|
|
72
|
+
"""The columns that look like direct identifiers, with their kind, in column order.
|
|
73
|
+
|
|
74
|
+
Name columns count only when a dob column is there too.
|
|
75
|
+
"""
|
|
76
|
+
found = [(str(column), _kind(str(column))) for column in columns]
|
|
77
|
+
has_dob = any(kind == "dob" for _, kind in found)
|
|
78
|
+
return [
|
|
79
|
+
(column, kind) for column, kind in found if kind is not None and (kind != "name" or has_dob)
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def header_columns(path: Path) -> list[str] | None:
|
|
84
|
+
"""A data file's column names, reading only its header. None when the type isn't
|
|
85
|
+
supported, its reader isn't installed, or the file can't be read."""
|
|
86
|
+
suffix = path.suffix.lower()
|
|
87
|
+
try:
|
|
88
|
+
if suffix in (".csv", ".tsv", ".txt"):
|
|
89
|
+
with path.open(encoding="utf-8-sig", errors="replace", newline="") as handle:
|
|
90
|
+
line = handle.readline(65536)
|
|
91
|
+
if not line.strip():
|
|
92
|
+
return None
|
|
93
|
+
delimiter = (
|
|
94
|
+
"\t"
|
|
95
|
+
if suffix == ".tsv" or "\t" in line
|
|
96
|
+
else ";"
|
|
97
|
+
if line.count(";") > line.count(",")
|
|
98
|
+
else ","
|
|
99
|
+
)
|
|
100
|
+
return next(csv.reader([line], delimiter=delimiter))
|
|
101
|
+
if suffix == ".parquet":
|
|
102
|
+
import pyarrow.parquet as pq
|
|
103
|
+
|
|
104
|
+
return list(pq.read_schema(path).names)
|
|
105
|
+
if suffix in (".dta", ".sav"):
|
|
106
|
+
import pyreadstat
|
|
107
|
+
|
|
108
|
+
reader = pyreadstat.read_dta if suffix == ".dta" else pyreadstat.read_sav
|
|
109
|
+
_, meta = reader(str(path), metadataonly=True)
|
|
110
|
+
return list(meta.column_names)
|
|
111
|
+
except ImportError:
|
|
112
|
+
return None
|
|
113
|
+
except Exception: # A file that can't be read is not this check's to report.
|
|
114
|
+
return None
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def warn_identifiers(columns: Iterable[object], target: Path, root: Path) -> None:
|
|
119
|
+
"""Warn when data saved outside a private folder has identifier-like columns."""
|
|
120
|
+
import warnings
|
|
121
|
+
|
|
122
|
+
from framework._paths import has_private_segment, relative_to_root
|
|
123
|
+
from framework._private import project_is_sensitive
|
|
124
|
+
from framework.errors import FrameworkWarning
|
|
125
|
+
|
|
126
|
+
relative = relative_to_root(target, root)
|
|
127
|
+
if has_private_segment(Path(relative).parent):
|
|
128
|
+
return
|
|
129
|
+
found = identifier_columns(columns)
|
|
130
|
+
if not found:
|
|
131
|
+
return
|
|
132
|
+
shown = ", ".join(column for column, _ in found[:5])
|
|
133
|
+
if len(found) > 5:
|
|
134
|
+
shown += f", and {len(found) - 5} more"
|
|
135
|
+
advice = (
|
|
136
|
+
"If this is real data about people, save it in a private folder instead."
|
|
137
|
+
if project_is_sensitive()
|
|
138
|
+
else "If this is real data about people, save it in a private folder, which git "
|
|
139
|
+
"ignores, and consider a privacy-sensitive project (type project_sensitive)."
|
|
140
|
+
)
|
|
141
|
+
warnings.warn(
|
|
142
|
+
f"{relative} has columns that look like direct identifiers ({shown}), and it isn't "
|
|
143
|
+
f"in a private folder. {advice}",
|
|
144
|
+
FrameworkWarning,
|
|
145
|
+
stacklevel=3,
|
|
146
|
+
)
|