framework-py 0.1.0.dev22__py3-none-any.whl → 0.1.0.dev23__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/adopt.py +2 -2
- framework/cache.py +7 -1
- framework/cloud.py +78 -33
- framework/connections.py +1 -1
- framework/create.py +3 -2
- framework/data.py +144 -24
- framework/data_registry.py +2 -2
- framework/db.py +9 -0
- framework/errors.py +3 -2
- framework/notebooks.py +1 -1
- framework/outputs.py +18 -2
- framework/storage.py +1 -1
- framework/sync_all.py +11 -11
- framework/templates/skill-history.json +2 -1
- framework/templates/skills/framework-data/.framework-skill.json +2 -2
- framework/templates/skills/framework-data/SKILL.md +5 -4
- {framework_py-0.1.0.dev22.dist-info → framework_py-0.1.0.dev23.dist-info}/METADATA +15 -33
- {framework_py-0.1.0.dev22.dist-info → framework_py-0.1.0.dev23.dist-info}/RECORD +20 -20
- {framework_py-0.1.0.dev22.dist-info → framework_py-0.1.0.dev23.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev22.dist-info → framework_py-0.1.0.dev23.dist-info}/licenses/LICENSE +0 -0
framework/adopt.py
CHANGED
|
@@ -520,7 +520,7 @@ def adopt(
|
|
|
520
520
|
The project folder.
|
|
521
521
|
|
|
522
522
|
Raises:
|
|
523
|
-
FrameworkError: When the folder
|
|
523
|
+
FrameworkError: When the folder does not exist or is already a Framework project,
|
|
524
524
|
when its files say another language and ``force`` is False, when the
|
|
525
525
|
arguments disagree, or when the review needs a terminal and ``yes`` is False.
|
|
526
526
|
"""
|
|
@@ -743,7 +743,7 @@ def structure_import(
|
|
|
743
743
|
|
|
744
744
|
Raises:
|
|
745
745
|
CloudError: When you are not logged in, or the server rejects the layout or
|
|
746
|
-
|
|
746
|
+
does not support structures.
|
|
747
747
|
FrameworkError: When the review needs a terminal and ``yes`` is False, or when
|
|
748
748
|
you cancel it.
|
|
749
749
|
"""
|
framework/cache.py
CHANGED
|
@@ -26,7 +26,7 @@ import pandas as pd
|
|
|
26
26
|
from framework import _depends, db
|
|
27
27
|
from framework._log import inform
|
|
28
28
|
from framework._paths import in_root, relative_to_root
|
|
29
|
-
from framework.errors import CacheWarning, FrameworkError
|
|
29
|
+
from framework.errors import CacheWarning, FrameworkError, IntegrityError
|
|
30
30
|
from framework.integrity import file_hash
|
|
31
31
|
from framework.project import project_root
|
|
32
32
|
from framework.settings import settings_read
|
|
@@ -149,6 +149,12 @@ def _store(
|
|
|
149
149
|
path = (
|
|
150
150
|
Path(file) if file is not None else cache_dir() / f"{name}{_FORMATS[0 if is_frame else 1]}"
|
|
151
151
|
)
|
|
152
|
+
# A cache file never replaces locked data
|
|
153
|
+
from framework.data import locked_data_at, locked_message
|
|
154
|
+
|
|
155
|
+
held = locked_data_at(path)
|
|
156
|
+
if held is not None:
|
|
157
|
+
raise IntegrityError(locked_message("cache()", held, "Store the cache in another file."))
|
|
152
158
|
if not path.parent.is_dir():
|
|
153
159
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
154
160
|
inform(f"Creating cache directory: {path.parent}")
|
framework/cloud.py
CHANGED
|
@@ -7,13 +7,17 @@ and documents you publish.
|
|
|
7
7
|
|
|
8
8
|
The server is resolved per call, as in R:
|
|
9
9
|
|
|
10
|
-
- User calls (
|
|
11
|
-
|
|
12
|
-
``cloud_login()
|
|
10
|
+
- User calls (settings, sync, ``new(key=...)``): the ``url`` argument, then
|
|
11
|
+
``FW_CLOUD_URL``, then the server saved by the last login (``cloud_login()``,
|
|
12
|
+
``fw login``, or R's ``cloud_login()``).
|
|
13
13
|
- Project calls (publishing, the ledger): the ``url`` argument, then
|
|
14
14
|
``FW_CLOUD_URL`` in the project's ``.env``, then the environment variable,
|
|
15
|
-
then
|
|
16
|
-
|
|
15
|
+
then the saved login. A project token is only valid on the server that
|
|
16
|
+
issued it, so when saved logins name different servers these calls stop
|
|
17
|
+
instead of guessing.
|
|
18
|
+
- With nothing set, calls raise ``NoServerError``. Only ``cloud_login()``
|
|
19
|
+
falls back to https://framework.pub, which its notice names before
|
|
20
|
+
connecting. Using the public cloud is always the user's choice.
|
|
17
21
|
|
|
18
22
|
Plain ``http`` is accepted only for localhost.
|
|
19
23
|
"""
|
|
@@ -69,6 +73,14 @@ class CloudError(FrameworkError):
|
|
|
69
73
|
status: int | None = None
|
|
70
74
|
|
|
71
75
|
|
|
76
|
+
class NoServerError(CloudError):
|
|
77
|
+
"""Nothing names a server: no ``url``, no ``FW_CLOUD_URL``, and no saved login."""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class AmbiguousServerError(CloudError):
|
|
81
|
+
"""A project call found logins to several servers and nothing naming the project's."""
|
|
82
|
+
|
|
83
|
+
|
|
72
84
|
def _version() -> str:
|
|
73
85
|
from framework import __version__
|
|
74
86
|
|
|
@@ -121,14 +133,14 @@ def is_default(url: str) -> bool:
|
|
|
121
133
|
return host(url) == host(DEFAULT_URL)
|
|
122
134
|
|
|
123
135
|
|
|
124
|
-
def
|
|
125
|
-
"""
|
|
136
|
+
def _saved_urls() -> list[str]:
|
|
137
|
+
"""Every server a login saved in fw's and R's cloud.yml, the most recent first.
|
|
126
138
|
|
|
127
|
-
|
|
128
|
-
|
|
139
|
+
URLs are normalized and duplicates dropped. The most recently written file
|
|
140
|
+
comes first, so the last login in any tool counts. On a tie fw's folder
|
|
141
|
+
comes first. fw and R use the same rule.
|
|
129
142
|
"""
|
|
130
|
-
|
|
131
|
-
best_time = -1
|
|
143
|
+
found: list[tuple[int, str]] = []
|
|
132
144
|
for folder in dict.fromkeys((config_dir(), r_config_dir())):
|
|
133
145
|
path = folder / "cloud.yml"
|
|
134
146
|
if not path.is_file():
|
|
@@ -136,10 +148,9 @@ def _saved_url() -> str | None:
|
|
|
136
148
|
data = _yaml.read(path)
|
|
137
149
|
server = data.get("server") if isinstance(data, dict) else None
|
|
138
150
|
if isinstance(server, str) and server:
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
return best
|
|
151
|
+
found.append((path.stat().st_mtime_ns, normalize_url(server)))
|
|
152
|
+
found.sort(key=lambda item: -item[0])
|
|
153
|
+
return list(dict.fromkeys(url for _, url in found))
|
|
143
154
|
|
|
144
155
|
|
|
145
156
|
def _project_dir(project_dir: str | Path | None) -> Path | None:
|
|
@@ -163,16 +174,42 @@ class Server:
|
|
|
163
174
|
def cloud_server(
|
|
164
175
|
url: str | None = None, scope: str = "user", project_dir: str | Path | None = None
|
|
165
176
|
) -> Server:
|
|
166
|
-
"""Which server a call uses, and where that setting came from.
|
|
177
|
+
"""Which server a call uses, and where that setting came from.
|
|
178
|
+
|
|
179
|
+
``scope`` is ``"user"``, ``"project"``, or ``"login"``. Raises
|
|
180
|
+
``NoServerError`` when nothing names a server, except for ``"login"``,
|
|
181
|
+
which falls back to https://framework.pub.
|
|
182
|
+
"""
|
|
167
183
|
candidates: list[tuple[str | None, str]] = [(url, "the url argument")]
|
|
168
184
|
if scope == "project":
|
|
169
185
|
candidates.append((_project_env("FW_CLOUD_URL", project_dir), "the project's .env"))
|
|
170
186
|
candidates.append((os.environ.get("FW_CLOUD_URL"), "the FW_CLOUD_URL environment variable"))
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
187
|
+
for value, source in candidates:
|
|
188
|
+
if value:
|
|
189
|
+
return Server(normalize_url(str(value)), source)
|
|
190
|
+
|
|
191
|
+
saved = _saved_urls()
|
|
192
|
+
if len(saved) == 1 or (saved and scope != "project"):
|
|
193
|
+
return Server(saved[0], "your cloud_login() default")
|
|
194
|
+
if saved:
|
|
195
|
+
hosts = " and ".join(host(u) for u in saved)
|
|
196
|
+
raise AmbiguousServerError(
|
|
197
|
+
"This project does not say which Framework Cloud server it uses: its .env has no "
|
|
198
|
+
f"FW_CLOUD_URL, and you are logged in to {hosts}. Add FW_CLOUD_URL=<server> to the "
|
|
199
|
+
"project's .env, or set FW_CLOUD_URL."
|
|
200
|
+
)
|
|
201
|
+
if scope == "login":
|
|
202
|
+
return Server(DEFAULT_URL, "the cloud_login() default")
|
|
203
|
+
where = (
|
|
204
|
+
"This project's .env has no FW_CLOUD_URL and you have not logged in. "
|
|
205
|
+
"Add FW_CLOUD_URL=<server> to .env, or log in"
|
|
206
|
+
if scope == "project"
|
|
207
|
+
else "Log in"
|
|
208
|
+
)
|
|
209
|
+
raise NoServerError(
|
|
210
|
+
f"No Framework Cloud server is chosen. {where} with fw.cloud_login() (framework.pub) "
|
|
211
|
+
'or fw.cloud_login(url="<server>"), or set FW_CLOUD_URL.'
|
|
212
|
+
)
|
|
176
213
|
|
|
177
214
|
|
|
178
215
|
# Tokens ---------------------------------------------------------------------------
|
|
@@ -546,7 +583,7 @@ def cloud_login(
|
|
|
546
583
|
Args:
|
|
547
584
|
token: A personal token. Omit it to approve this computer in the browser.
|
|
548
585
|
url: The server. Default: ``FW_CLOUD_URL``, then your saved default,
|
|
549
|
-
then https://framework.pub.
|
|
586
|
+
then https://framework.pub, which the notice names before connecting.
|
|
550
587
|
verify: Check the token with the server first.
|
|
551
588
|
consent: ``True`` to confirm a pasted token without a prompt (scripts,
|
|
552
589
|
CI). Approving in the browser is the confirmation for browser login.
|
|
@@ -556,7 +593,7 @@ def cloud_login(
|
|
|
556
593
|
Where the token was stored: ``"keyring"`` or a file path.
|
|
557
594
|
"""
|
|
558
595
|
show_messages()
|
|
559
|
-
server = cloud_server(url).url
|
|
596
|
+
server = cloud_server(url, scope="login").url
|
|
560
597
|
name = host(server)
|
|
561
598
|
if token is None:
|
|
562
599
|
print(consent_notice(server), file=sys.stderr)
|
|
@@ -624,7 +661,7 @@ def _revoke(server: str, token: str) -> None:
|
|
|
624
661
|
def cloud_logout(url: str | None = None) -> list[str]:
|
|
625
662
|
"""Revoke and remove this machine's stored token for a server. Returns what was removed.
|
|
626
663
|
|
|
627
|
-
Each stored token is revoked on the server first. If the server
|
|
664
|
+
Each stored token is revoked on the server first. If the server cannot be reached, the
|
|
628
665
|
local copy is still removed, and you can revoke the token on the server's tokens page.
|
|
629
666
|
The keyring entry, the token file, and the file ``FW_TOKEN_FILE`` names are removed. For
|
|
630
667
|
the default server, the legacy keyring entry and ``~/.secrets/framework/token`` are removed
|
|
@@ -632,8 +669,7 @@ def cloud_logout(url: str | None = None) -> list[str]:
|
|
|
632
669
|
until you unset it.
|
|
633
670
|
|
|
634
671
|
Args:
|
|
635
|
-
url: The server. Default: ``FW_CLOUD_URL``, then your saved default
|
|
636
|
-
https://framework.pub.
|
|
672
|
+
url: The server. Default: ``FW_CLOUD_URL``, then your saved default.
|
|
637
673
|
|
|
638
674
|
Returns:
|
|
639
675
|
A description of each keyring entry and the path of each file removed. The list is
|
|
@@ -677,19 +713,25 @@ def cloud_status(url: str | None = None) -> bool:
|
|
|
677
713
|
publishes to a different server, that server is printed too.
|
|
678
714
|
|
|
679
715
|
Args:
|
|
680
|
-
url: The server. Default: ``FW_CLOUD_URL``, then your saved default
|
|
681
|
-
https://framework.pub.
|
|
716
|
+
url: The server. Default: ``FW_CLOUD_URL``, then your saved default.
|
|
682
717
|
|
|
683
718
|
Returns:
|
|
684
719
|
True when a token is stored and the server accepts it. False when no token is stored,
|
|
685
720
|
the server is unreachable, or the token is invalid.
|
|
686
721
|
"""
|
|
687
722
|
show_messages()
|
|
688
|
-
|
|
723
|
+
try:
|
|
724
|
+
server = cloud_server(url)
|
|
725
|
+
except NoServerError as error:
|
|
726
|
+
inform(str(error))
|
|
727
|
+
return False
|
|
689
728
|
inform(f"Server: {server.url} (from {server.source})")
|
|
690
729
|
if project_token() is not None:
|
|
691
|
-
|
|
692
|
-
|
|
730
|
+
try:
|
|
731
|
+
project: Server | None = cloud_server(scope="project")
|
|
732
|
+
except CloudError:
|
|
733
|
+
project = None
|
|
734
|
+
if project is not None and project.url != server.url:
|
|
693
735
|
inform(f"This project publishes to {project.url} (from {project.source})")
|
|
694
736
|
token = cloud_token(server.url)
|
|
695
737
|
if token is None:
|
|
@@ -1308,7 +1350,10 @@ def register_project(
|
|
|
1308
1350
|
was built from. A server without ``/api/v1/projects`` gets the older
|
|
1309
1351
|
registration.
|
|
1310
1352
|
"""
|
|
1311
|
-
|
|
1353
|
+
try:
|
|
1354
|
+
server = cloud_server().url
|
|
1355
|
+
except NoServerError:
|
|
1356
|
+
return None
|
|
1312
1357
|
token = cloud_token(server)
|
|
1313
1358
|
if token is None:
|
|
1314
1359
|
return None
|
|
@@ -1468,7 +1513,7 @@ def cloud_register(
|
|
|
1468
1513
|
A project that already has a token in ``.env`` is left as it is. When you
|
|
1469
1514
|
already have a project with the slug, an interactive session offers to
|
|
1470
1515
|
link this folder to that project instead, and nothing is created. A slug
|
|
1471
|
-
held by a project in your Trash
|
|
1516
|
+
held by a project in your Trash cannot be reused. A self-hosted server from
|
|
1472
1517
|
before the spec API builds the project from your defaults; the spec is
|
|
1473
1518
|
then sent separately (``PUT /api/v1/project/spec``). This is
|
|
1474
1519
|
``fw project link --create``.
|
framework/connections.py
CHANGED
|
@@ -439,7 +439,7 @@ def db_query(
|
|
|
439
439
|
this query and closed afterward.
|
|
440
440
|
params: Values for the query's placeholders, as a sequence or a mapping.
|
|
441
441
|
read_only: Open the connection read-only for this query (True), as
|
|
442
|
-
``db_connect()`` does. Default: the connection's setting, which False
|
|
442
|
+
``db_connect()`` does. Default: the connection's setting, which False cannot
|
|
443
443
|
override when it is ``read_only: true``. A read-only DuckDB
|
|
444
444
|
file can be open in several processes at once, so ``read_only=True`` lets a
|
|
445
445
|
notebook query a database another job holds open read-only too.
|
framework/create.py
CHANGED
|
@@ -774,11 +774,12 @@ def new_project(
|
|
|
774
774
|
url: With ``key``: the Framework Cloud server that issued the token. No login is
|
|
775
775
|
needed. The server is saved as ``FW_CLOUD_URL`` in the project's ``.env``.
|
|
776
776
|
Default: ``FW_CLOUD_URL``, then the server you last logged in to with
|
|
777
|
-
[`cloud_login()`](ref:cloud_login),
|
|
777
|
+
[`cloud_login()`](ref:cloud_login). With neither, ``new()`` raises
|
|
778
|
+
``NoServerError`` rather than assume https://framework.pub.
|
|
778
779
|
interactive: Whether to prompt. Default: whether stdin is a terminal.
|
|
779
780
|
offline: Build from Framework's built-in defaults without contacting Framework Cloud.
|
|
780
781
|
When you are logged in and pass no choices, ``new()`` resolves your defaults on
|
|
781
|
-
the cloud and caches them. If the cloud
|
|
782
|
+
the cloud and caches them. If the cloud cannot be reached, it builds from that
|
|
782
783
|
cache and says so. With no cache it asks first, or, when not interactive, stops
|
|
783
784
|
unless ``offline`` is True. It never falls back silently.
|
|
784
785
|
|
framework/data.py
CHANGED
|
@@ -13,9 +13,12 @@ too.
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
15
|
import importlib.util
|
|
16
|
+
import os
|
|
16
17
|
import re
|
|
18
|
+
import sys
|
|
17
19
|
import warnings
|
|
18
20
|
from collections.abc import Callable, Mapping, Sequence
|
|
21
|
+
from dataclasses import dataclass
|
|
19
22
|
from pathlib import Path
|
|
20
23
|
from typing import Any
|
|
21
24
|
|
|
@@ -26,7 +29,7 @@ from framework._identifiers import warn_identifiers
|
|
|
26
29
|
from framework._log import inform
|
|
27
30
|
from framework._paths import in_root, relative_to_root
|
|
28
31
|
from framework._private import is_private_data, note_private_read
|
|
29
|
-
from framework.catalog import DataInfo, catalog_locate, data_info
|
|
32
|
+
from framework.catalog import DataInfo, catalog_locate, catalog_names, data_info
|
|
30
33
|
from framework.errors import (
|
|
31
34
|
DataChangedWarning,
|
|
32
35
|
DataNotFoundError,
|
|
@@ -516,16 +519,98 @@ def _in_settings_folder(folder: Path) -> bool:
|
|
|
516
519
|
return False
|
|
517
520
|
|
|
518
521
|
|
|
522
|
+
@dataclass(frozen=True)
|
|
523
|
+
class LockedData:
|
|
524
|
+
"""Locked data a write would change: its name, its file, and where the lock is."""
|
|
525
|
+
|
|
526
|
+
name: str
|
|
527
|
+
file: Path
|
|
528
|
+
source: str # "catalog" or "record"
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _same_file(a: Path, b: Path) -> bool:
|
|
532
|
+
"""Whether two paths name the same file. Case is ignored on macOS and Windows."""
|
|
533
|
+
left, right = os.path.normpath(a.resolve()), os.path.normpath(b.resolve())
|
|
534
|
+
if sys.platform in ("darwin", "win32"):
|
|
535
|
+
return left.lower() == right.lower()
|
|
536
|
+
return left == right
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def locked_data_at(file_path: str | Path, name: str | None = None) -> LockedData | None:
|
|
540
|
+
"""The locked data a write to ``file_path`` would change, or None, as in R.
|
|
541
|
+
|
|
542
|
+
Locked data is a catalog entry with ``locked: true``, or a framework.db record saved
|
|
543
|
+
locked (``data_save()`` to a path has no catalog entry). An entry counts when its file
|
|
544
|
+
exists and is ``file_path``, or when its name is ``name``, since a save by name
|
|
545
|
+
repoints the entry. A catalog entry's flag wins over its record, so ``locked: false``
|
|
546
|
+
in the settings unlocks it.
|
|
547
|
+
"""
|
|
548
|
+
try:
|
|
549
|
+
root = project_root()
|
|
550
|
+
data = settings_read().get("data")
|
|
551
|
+
except FrameworkError:
|
|
552
|
+
return None
|
|
553
|
+
target = Path(file_path)
|
|
554
|
+
|
|
555
|
+
def hit(entry_name: str, entry_path: Any) -> Path | None:
|
|
556
|
+
if not isinstance(entry_path, str) or not entry_path:
|
|
557
|
+
return None
|
|
558
|
+
file = Path(entry_path) if Path(entry_path).is_absolute() else root / entry_path
|
|
559
|
+
if file.is_file() and (entry_name == name or _same_file(file, target)):
|
|
560
|
+
return file
|
|
561
|
+
return None
|
|
562
|
+
|
|
563
|
+
names = catalog_names(data)
|
|
564
|
+
for entry_name in names:
|
|
565
|
+
located = catalog_locate(entry_name, data)
|
|
566
|
+
entry = located[0][located[1]] if located else None
|
|
567
|
+
if isinstance(entry, dict) and entry.get("locked") is True:
|
|
568
|
+
file = hit(entry_name, entry.get("path"))
|
|
569
|
+
if file is not None:
|
|
570
|
+
return LockedData(entry_name, file, "catalog")
|
|
571
|
+
for record_name, record_path in db.locked_data_records(root):
|
|
572
|
+
if record_name in names:
|
|
573
|
+
continue
|
|
574
|
+
file = hit(record_name, record_path)
|
|
575
|
+
if file is not None:
|
|
576
|
+
return LockedData(record_name, file, "record")
|
|
577
|
+
return None
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def locked_message(caller: str, held: LockedData, advice: str) -> str:
|
|
581
|
+
"""The error for a write that would change locked data."""
|
|
582
|
+
where = (
|
|
583
|
+
"locked: true in the data catalog" if held.source == "catalog" else "saved with locked=True"
|
|
584
|
+
)
|
|
585
|
+
try:
|
|
586
|
+
shown = relative_to_root(held.file, project_root())
|
|
587
|
+
except FrameworkError:
|
|
588
|
+
shown = str(held.file)
|
|
589
|
+
return (
|
|
590
|
+
f"{caller} does not overwrite locked data.\n"
|
|
591
|
+
f" dataset: {held.name} ({where})\n"
|
|
592
|
+
f" file: {shown}\n"
|
|
593
|
+
f"{advice}"
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
|
|
519
597
|
def data_save(
|
|
520
598
|
data: pd.DataFrame,
|
|
521
599
|
path: str,
|
|
522
600
|
type: str | None = None,
|
|
523
601
|
delimiter: str = "comma",
|
|
524
|
-
locked: bool =
|
|
602
|
+
locked: bool | None = None,
|
|
525
603
|
force: bool = False,
|
|
604
|
+
unlock: bool = False,
|
|
526
605
|
) -> pd.DataFrame:
|
|
527
606
|
"""Save a DataFrame and record its hash.
|
|
528
607
|
|
|
608
|
+
Data is saved unlocked unless you lock it. Locked data protects inputs:
|
|
609
|
+
``data_read()`` raises when a locked file changed. So ``data_save()`` never changes a
|
|
610
|
+
locked file unless asked. Save under another name, or pass ``unlock=True`` to replace
|
|
611
|
+
it; the new file is then unlocked, and later saves replace it. Pass
|
|
612
|
+
``unlock=True, locked=True`` to replace it and keep it locked.
|
|
613
|
+
|
|
529
614
|
Args:
|
|
530
615
|
data: The data to save.
|
|
531
616
|
path: A catalog name such as ``"inputs.intermediate.cleaned"`` or a
|
|
@@ -540,17 +625,26 @@ def data_save(
|
|
|
540
625
|
type: ``"parquet"`` or ``"csv"``. Default: from the file extension,
|
|
541
626
|
or Parquet for a catalog name.
|
|
542
627
|
delimiter: Delimiter for CSV files: comma, tab, semicolon, or space.
|
|
543
|
-
locked: Whether later reads must find the file unchanged. Default:
|
|
544
|
-
|
|
545
|
-
|
|
628
|
+
locked: Whether later reads must find the file unchanged. Default: unlocked, as
|
|
629
|
+
in R, so a pipeline can save the data again. Pass ``locked=True`` to lock data
|
|
630
|
+
you mean to keep as it is. Left out, a save keeps a lock the data already
|
|
631
|
+
has, unless ``unlock=True``.
|
|
546
632
|
force: Create any missing folder. Without it, a missing folder is created
|
|
547
633
|
only when it is a folder named in the settings (``directories``,
|
|
548
634
|
``extra_directories``, ``render_dirs``) or inside one, such as
|
|
549
635
|
``inputs/private/intermediate`` in a fresh checkout; any other missing
|
|
550
636
|
folder is an error.
|
|
637
|
+
unlock: Replace locked data and save it unlocked, unless ``locked=True``.
|
|
638
|
+
Without it, ``data_save()`` raises ``IntegrityError`` before writing when the
|
|
639
|
+
save would change locked data: a catalog entry with ``locked: true``, or a
|
|
640
|
+
file saved earlier with ``locked=True``. Saving the same contents again needs
|
|
641
|
+
no ``unlock``.
|
|
551
642
|
|
|
552
643
|
Returns:
|
|
553
644
|
``data``, so the call can end a pipeline.
|
|
645
|
+
|
|
646
|
+
Raises:
|
|
647
|
+
IntegrityError: The save would change locked data and ``unlock`` is False.
|
|
554
648
|
"""
|
|
555
649
|
if not isinstance(data, pd.DataFrame):
|
|
556
650
|
raise FrameworkError("data_save() needs a pandas DataFrame")
|
|
@@ -577,12 +671,43 @@ def data_save(
|
|
|
577
671
|
folder.mkdir(parents=True, exist_ok=True)
|
|
578
672
|
inform(f"Created directory: {folder}")
|
|
579
673
|
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
674
|
+
def write(destination: Path) -> None:
|
|
675
|
+
if file_type == "csv":
|
|
676
|
+
# In readr's form (1 not 1.0, TRUE, NA, "\n" line endings), so R reads the same
|
|
677
|
+
# values and the same data hashes the same in both languages
|
|
678
|
+
_csv.write_csv(data, destination, sep=_separator(delimiter))
|
|
679
|
+
else:
|
|
680
|
+
data.to_parquet(destination, index=False)
|
|
681
|
+
|
|
682
|
+
# Locked data stays unchanged unless unlock=True: reads raise when a locked file
|
|
683
|
+
# changed. The new contents go to a temporary file first, so a refused save leaves
|
|
684
|
+
# the locked file as it was. The same contents pass.
|
|
685
|
+
held = locked_data_at(target, name=name)
|
|
686
|
+
if held is not None:
|
|
687
|
+
same = False
|
|
688
|
+
if _same_file(held.file, target):
|
|
689
|
+
temporary = folder / f".fw-save-{os.getpid()}{target.suffix}"
|
|
690
|
+
try:
|
|
691
|
+
write(temporary)
|
|
692
|
+
same = file_hash(temporary) == file_hash(target)
|
|
693
|
+
finally:
|
|
694
|
+
temporary.unlink(missing_ok=True)
|
|
695
|
+
if not same and not unlock:
|
|
696
|
+
raise IntegrityError(
|
|
697
|
+
locked_message(
|
|
698
|
+
"data_save()",
|
|
699
|
+
held,
|
|
700
|
+
"Save the data under another name, or pass unlock=True to replace it.\n"
|
|
701
|
+
"The new file is then unlocked. Add locked=True to keep it locked.",
|
|
702
|
+
)
|
|
703
|
+
)
|
|
704
|
+
if not same:
|
|
705
|
+
inform(f"Replacing locked data '{held.name}' (unlock=True).")
|
|
706
|
+
# Without ``locked``, locked data saved with the same contents stays locked, and
|
|
707
|
+
# unlock=True saves it unlocked. Everything else is saved unlocked.
|
|
708
|
+
if locked is None:
|
|
709
|
+
locked = held is not None and not unlock
|
|
710
|
+
write(target)
|
|
586
711
|
root = project_root()
|
|
587
712
|
relative = relative_to_root(target, root)
|
|
588
713
|
inform(f"Data saved to: {relative}")
|
|
@@ -592,17 +717,6 @@ def data_save(
|
|
|
592
717
|
key = name or relative
|
|
593
718
|
|
|
594
719
|
digest = file_hash(target)
|
|
595
|
-
previous = db.get_data_record(key, root)
|
|
596
|
-
if previous is not None and previous.locked and previous.hash and previous.hash != digest:
|
|
597
|
-
# As in R, the save goes ahead: data_save() is how locked data is rebuilt. It
|
|
598
|
-
# says so, since the lock means reads expect the file unchanged.
|
|
599
|
-
warnings.warn(
|
|
600
|
-
f"data_save() replaced locked data '{key}' with different contents.\n"
|
|
601
|
-
f"Previous hash: {previous.hash[:12]}\nCurrent hash: {digest[:12]}\n"
|
|
602
|
-
"Pass locked=False for data a pipeline rebuilds.",
|
|
603
|
-
DataChangedWarning,
|
|
604
|
-
stacklevel=2,
|
|
605
|
-
)
|
|
606
720
|
db.set_data_record(
|
|
607
721
|
key,
|
|
608
722
|
path=relative,
|
|
@@ -714,7 +828,7 @@ def data_add(
|
|
|
714
828
|
name: str | None = None,
|
|
715
829
|
type: str | None = None,
|
|
716
830
|
delimiter: str = "comma",
|
|
717
|
-
locked: bool =
|
|
831
|
+
locked: bool | None = None,
|
|
718
832
|
update_config: bool = True,
|
|
719
833
|
) -> dict[str, Any]:
|
|
720
834
|
"""Add an existing file to the data catalog and record its hash.
|
|
@@ -727,7 +841,8 @@ def data_add(
|
|
|
727
841
|
``inputs.private.raw.<file>`` in ``inputs_private_raw``), else the file name.
|
|
728
842
|
type: File type. Default: from the extension.
|
|
729
843
|
delimiter: Delimiter for CSV files.
|
|
730
|
-
locked: Whether later reads must find the file unchanged.
|
|
844
|
+
locked: Whether later reads must find the file unchanged. Default: unlocked.
|
|
845
|
+
Left out, a file already locked in the catalog or framework.db stays locked.
|
|
731
846
|
update_config: Write the entry to the settings, not just framework.db.
|
|
732
847
|
|
|
733
848
|
Returns:
|
|
@@ -750,6 +865,11 @@ def data_add(
|
|
|
750
865
|
if name is None:
|
|
751
866
|
name = _derive_name(path, settings_read(), root)
|
|
752
867
|
|
|
868
|
+
# Left out, ``locked`` keeps a lock the file already has, so registering locked
|
|
869
|
+
# data again doesn't unlock it
|
|
870
|
+
if locked is None:
|
|
871
|
+
locked = locked_data_at(path, name=name) is not None
|
|
872
|
+
|
|
753
873
|
relative = relative_to_root(path, root)
|
|
754
874
|
stored_delimiter = delimiter if type == "csv" else None
|
|
755
875
|
db.set_data_record(
|
framework/data_registry.py
CHANGED
|
@@ -461,7 +461,7 @@ def pull_data_registry(
|
|
|
461
461
|
project's framework.db. Applying writes the new entries into the settings' data section
|
|
462
462
|
(inline, or in its split file such as ``settings/data.yml``), keeping the file's comments.
|
|
463
463
|
Local entries are never removed, and an entry whose local definition differs keeps it
|
|
464
|
-
unless ``overwrite=True``. Fields Framework Cloud
|
|
464
|
+
unless ``overwrite=True``. Fields Framework Cloud does not store remain unchanged. The
|
|
465
465
|
revision pulled is kept in ``.framework/data.lock.json`` for ``push_data_registry()``.
|
|
466
466
|
``pull_all()`` pulls it with the packages and skills.
|
|
467
467
|
|
|
@@ -544,7 +544,7 @@ def push_data_registry(
|
|
|
544
544
|
|
|
545
545
|
Args:
|
|
546
546
|
project_path: The project's root directory. Default: the current directory.
|
|
547
|
-
remove: Remove entries from Cloud's registry that this project
|
|
547
|
+
remove: Remove entries from Cloud's registry that this project does not have.
|
|
548
548
|
force: Push even when Cloud's registry changed since the last pull.
|
|
549
549
|
confirm: True pushes without asking; False cancels. None asks in a console.
|
|
550
550
|
dry_run: List the changes and send nothing.
|
framework/db.py
CHANGED
|
@@ -190,6 +190,15 @@ def set_data_record(
|
|
|
190
190
|
)
|
|
191
191
|
|
|
192
192
|
|
|
193
|
+
def locked_data_records(root: str | Path | None = None) -> list[tuple[str, str]]:
|
|
194
|
+
"""Name and path of each ``data`` row marked locked; none without framework.db."""
|
|
195
|
+
if not db_path(root).is_file():
|
|
196
|
+
return []
|
|
197
|
+
with connect(root) as connection:
|
|
198
|
+
rows = connection.execute("SELECT name, path FROM data WHERE locked").fetchall()
|
|
199
|
+
return [(row["name"], row["path"]) for row in rows if row["path"]]
|
|
200
|
+
|
|
201
|
+
|
|
193
202
|
def update_data_hash(name: str, hash: str, root: str | Path | None = None) -> None:
|
|
194
203
|
"""Record a new hash for a data file, as R's ``.update_data_with_hash()`` does."""
|
|
195
204
|
now = timestamp()
|
framework/errors.py
CHANGED
|
@@ -24,7 +24,8 @@ class UnsupportedFormatError(FrameworkError):
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class IntegrityError(FrameworkError):
|
|
27
|
-
"""A locked data file no longer matches the hash recorded in framework.db
|
|
27
|
+
"""A locked data file no longer matches the hash recorded in framework.db, or a save
|
|
28
|
+
would change locked data."""
|
|
28
29
|
|
|
29
30
|
|
|
30
31
|
class FrameworkWarning(UserWarning):
|
|
@@ -33,7 +34,7 @@ class FrameworkWarning(UserWarning):
|
|
|
33
34
|
|
|
34
35
|
class DataChangedWarning(FrameworkWarning):
|
|
35
36
|
"""A data file changed since Framework last recorded its hash: an unlocked file read
|
|
36
|
-
after it changed
|
|
37
|
+
after it changed."""
|
|
37
38
|
|
|
38
39
|
|
|
39
40
|
class CacheWarning(FrameworkWarning):
|
framework/notebooks.py
CHANGED
|
@@ -242,7 +242,7 @@ def make_notebook(
|
|
|
242
242
|
overwrite: Replace an existing file.
|
|
243
243
|
subdir: A subfolder of ``dir``.
|
|
244
244
|
title: The title, used exactly as given. Default: made from ``name`` in sentence
|
|
245
|
-
case, which keeps a name with capitals as typed but
|
|
245
|
+
case, which keeps a name with capitals as typed but cannot infer whether a
|
|
246
246
|
lowercase file name such as "bp-for-ldl" holds acronyms. Pass
|
|
247
247
|
``title="BP for LDL"`` for those.
|
|
248
248
|
|
framework/outputs.py
CHANGED
|
@@ -24,7 +24,12 @@ from framework import _csv, db
|
|
|
24
24
|
from framework._log import inform
|
|
25
25
|
from framework._paths import output_directory, relative_to_root
|
|
26
26
|
from framework._private import private_read_note, warn_public_save
|
|
27
|
-
from framework.errors import
|
|
27
|
+
from framework.errors import (
|
|
28
|
+
FrameworkError,
|
|
29
|
+
FrameworkWarning,
|
|
30
|
+
IntegrityError,
|
|
31
|
+
UnsupportedFormatError,
|
|
32
|
+
)
|
|
28
33
|
from framework.integrity import file_hash
|
|
29
34
|
from framework.settings import settings_read
|
|
30
35
|
|
|
@@ -64,6 +69,15 @@ def _check_overwrite(path: Path, overwrite: bool) -> None:
|
|
|
64
69
|
raise FrameworkError(f"File already exists: {path}. Use overwrite=True to replace.")
|
|
65
70
|
|
|
66
71
|
|
|
72
|
+
def _check_locked(caller: str, path: Path) -> None:
|
|
73
|
+
"""An output never replaces locked data."""
|
|
74
|
+
from framework.data import locked_data_at, locked_message
|
|
75
|
+
|
|
76
|
+
held = locked_data_at(path)
|
|
77
|
+
if held is not None:
|
|
78
|
+
raise IntegrityError(locked_message(caller, held, "Save under another name."))
|
|
79
|
+
|
|
80
|
+
|
|
67
81
|
def _result_name(path: Path) -> str:
|
|
68
82
|
"""The results key: the path from the project root, so same-named files stay apart."""
|
|
69
83
|
from framework.project import find_settings_file
|
|
@@ -130,6 +144,7 @@ def save_table(
|
|
|
130
144
|
raise FrameworkError(f"format must be one of {', '.join(_TABLE_FORMATS)}")
|
|
131
145
|
path = _output_file("tables", public, f"{name}{_TABLE_FORMATS[format]}")
|
|
132
146
|
_check_overwrite(path, overwrite)
|
|
147
|
+
_check_locked("save_table()", path)
|
|
133
148
|
if format == "csv":
|
|
134
149
|
if kwargs:
|
|
135
150
|
# The same readr form, with the caller's pandas options on top
|
|
@@ -300,7 +315,7 @@ def save_figure(
|
|
|
300
315
|
The saved file, or for a list of formats, the saved files in that order.
|
|
301
316
|
|
|
302
317
|
Raises:
|
|
303
|
-
FrameworkError: When a format is unknown, ``plot``
|
|
318
|
+
FrameworkError: When a format is unknown, ``plot`` cannot be saved, or a file
|
|
304
319
|
exists and ``overwrite`` is False. Nothing is written then.
|
|
305
320
|
UnsupportedFormatError: When no backend writes a format (emf).
|
|
306
321
|
"""
|
|
@@ -413,6 +428,7 @@ def save_model(model: Any, name: str, public: bool = False, overwrite: bool = Tr
|
|
|
413
428
|
rows_warned = True
|
|
414
429
|
path = _output_file("models", public, f"{name}.pkl")
|
|
415
430
|
_check_overwrite(path, overwrite)
|
|
431
|
+
_check_locked("save_model()", path)
|
|
416
432
|
# Pickle to a temporary file first, so a failed dump leaves any existing model intact
|
|
417
433
|
descriptor, temp = tempfile.mkstemp(prefix=f".{path.stem}.", suffix=".tmp", dir=path.parent)
|
|
418
434
|
try:
|
framework/storage.py
CHANGED
|
@@ -1225,7 +1225,7 @@ def publish_list(
|
|
|
1225
1225
|
|
|
1226
1226
|
Like ``publish()``, it follows ``to``, then ``publish.default``. For a folder
|
|
1227
1227
|
destination it lists the published artifacts, one row per slug. GitHub and Cloud
|
|
1228
|
-
destinations
|
|
1228
|
+
destinations cannot be listed here: Framework Cloud artifacts are on the server's
|
|
1229
1229
|
Artifacts page, ``<server>/published``.
|
|
1230
1230
|
|
|
1231
1231
|
Args:
|
framework/sync_all.py
CHANGED
|
@@ -469,9 +469,9 @@ def pull_all(
|
|
|
469
469
|
are kept unless ``overwrite=True`` (edited skills are backed up first).
|
|
470
470
|
|
|
471
471
|
The revisions pulled are kept in ``.framework/setup.lock.json`` (packages) and
|
|
472
|
-
``.framework/data.lock.json`` (data registry) for ``push_all()``. When a part
|
|
473
|
-
read or applied, the others still
|
|
474
|
-
and which
|
|
472
|
+
``.framework/data.lock.json`` (data registry) for ``push_all()``. When a part cannot be
|
|
473
|
+
read or applied, the others still proceed, and the error names which parts were applied
|
|
474
|
+
and which were skipped.
|
|
475
475
|
|
|
476
476
|
Args:
|
|
477
477
|
project_path: The project's root directory. Default: the current directory.
|
|
@@ -683,8 +683,8 @@ def _skills_push_plan(root: Path, force: bool) -> dict[str, Any]:
|
|
|
683
683
|
from framework.skill_pull import _push_apply, _push_edited, _push_plan, _warning
|
|
684
684
|
|
|
685
685
|
edited = _push_edited(root)
|
|
686
|
-
|
|
687
|
-
if edited and cloud_token(url) is None:
|
|
686
|
+
# Only edited skills need a server; with none chosen, NoServerError says how to pick one
|
|
687
|
+
if edited and cloud_token(url := cloud_server().url) is None:
|
|
688
688
|
names = ", ".join(Path(paths[0]).name for _, paths in edited)
|
|
689
689
|
raise CloudError(
|
|
690
690
|
f"Not signed in, so edited skills ({names}) can't be pushed. {login_hint(url)}"
|
|
@@ -767,7 +767,7 @@ def push_all(
|
|
|
767
767
|
edited skills, and its data registry. Cloud previews each part first (as a dry run),
|
|
768
768
|
and the changes are listed together, grouped by part (Packages, Skills, Data registry),
|
|
769
769
|
with what each push affects: new setups, zips, and Update on GitHub use the package
|
|
770
|
-
list and data registry, and a skill's new version is used wherever it
|
|
770
|
+
list and data registry, and a skill's new version is used wherever it is attached. Then
|
|
771
771
|
asks "Push these changes?" once. framework-py and the unpinned workflow tool packages
|
|
772
772
|
every project gets are not sent.
|
|
773
773
|
|
|
@@ -779,13 +779,13 @@ def push_all(
|
|
|
779
779
|
packages and the data registry go with the project token. Only data definitions are
|
|
780
780
|
sent, never data files or hashes.
|
|
781
781
|
|
|
782
|
-
When a part
|
|
783
|
-
were pushed and which
|
|
782
|
+
When a part cannot be pushed, the others still proceed, and the error names which parts
|
|
783
|
+
were pushed and which were skipped. A push resolves the project again from its blueprint's
|
|
784
784
|
latest version.
|
|
785
785
|
|
|
786
786
|
Args:
|
|
787
787
|
project_path: The project's root directory. Default: the current directory.
|
|
788
|
-
remove: Remove packages and data entries from Cloud that this project
|
|
788
|
+
remove: Remove packages and data entries from Cloud that this project does not have.
|
|
789
789
|
force: Push even when a part changed on Cloud since the last pull.
|
|
790
790
|
confirm: True pushes without asking; False cancels. None asks in a console.
|
|
791
791
|
dry_run: List the changes and send nothing.
|
|
@@ -794,11 +794,11 @@ def push_all(
|
|
|
794
794
|
A dict: ``packages`` and ``data_registry`` (each with ``revision`` and Cloud's
|
|
795
795
|
``changes``: ``added``, ``changed``, ``removed``, and ``kept``), ``skills`` (each
|
|
796
796
|
skill's ``name``, ``status``, and ``path``), ``applied`` (the parts pushed),
|
|
797
|
-
``failed`` (a reason for each part that
|
|
797
|
+
``failed`` (a reason for each part that failed), ``dry_run``, and ``message``.
|
|
798
798
|
|
|
799
799
|
Raises:
|
|
800
800
|
CloudError: Without a project token, offline, or when the server has no setup
|
|
801
|
-
endpoint (nothing changes then), and when a part
|
|
801
|
+
endpoint (nothing changes then), and when a part could not be pushed.
|
|
802
802
|
FrameworkError: Without a console and without ``confirm=True`` or ``dry_run=True``.
|
|
803
803
|
"""
|
|
804
804
|
from framework._log import inform, show_messages
|
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
"0cdf0d8c885825802fee7688344ed2d607ad40df5dae99c4905b275d861a3f0b",
|
|
9
9
|
"223a88db5e55b0d7ef9d85a68dddd5d4515b7522217f08d0ff52ea91bc3e99dc",
|
|
10
10
|
"75d663831d581fbb5d30d12068b44f521bbeb42022df1cdc3c9acae9267bff28",
|
|
11
|
-
"8ddcbd6f43a011612fc928014c457012bea16921d8a91802451693f5777dceca"
|
|
11
|
+
"8ddcbd6f43a011612fc928014c457012bea16921d8a91802451693f5777dceca",
|
|
12
|
+
"d26fb86e46ab55008ec3d9fb1b69066db2dcd05edf4a15fe0f3ea98f2c7d51b2"
|
|
12
13
|
],
|
|
13
14
|
"framework-databases": [
|
|
14
15
|
"b151906ffa9a8c886de99a9dbefd8c975b45cade5af8f7d1654dac9ba2596f38"
|
|
@@ -36,11 +36,12 @@ Prefer `fw.data_save()` to `DataFrame.to_csv()` and `DataFrame.to_parquet()`
|
|
|
36
36
|
for project data. It writes Parquet or CSV, which R can also read.
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
|
-
#
|
|
40
|
-
fw.data_save(
|
|
39
|
+
# Unlocked by default, so a pipeline can save it again
|
|
40
|
+
fw.data_save(clean, "inputs.intermediate.survey_clean")
|
|
41
41
|
|
|
42
|
-
#
|
|
43
|
-
|
|
42
|
+
# Locked: a later read raises an error if the file changed, and a save with
|
|
43
|
+
# new contents raises unless it passes unlock=True
|
|
44
|
+
fw.data_save(final, "inputs/final/analysis_ready.csv", locked=True)
|
|
44
45
|
```
|
|
45
46
|
|
|
46
47
|
## Folders
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: framework-py
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev23
|
|
4
4
|
Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
|
|
5
5
|
Project-URL: Homepage, https://framework.pub
|
|
6
6
|
Project-URL: Source, https://github.com/frameworktools/framework-py
|
|
@@ -50,19 +50,13 @@ Description-Content-Type: text/markdown
|
|
|
50
50
|
|
|
51
51
|
# framework-py
|
|
52
52
|
|
|
53
|
-
Framework for Python
|
|
54
|
-
data integrity tracking, caching, saved outputs, and guardrails for AI coding
|
|
55
|
-
assistants.
|
|
53
|
+
Framework for Python provides structured data science projects with a data catalog, integrity tracking, caching, saved outputs, and AI coding assistant guardrails.
|
|
56
54
|
|
|
57
|
-
framework-py is the Python implementation of
|
|
58
|
-
[Framework](https://framework.table1.org). It reads and writes the same
|
|
59
|
-
project files as the R package (`settings.yml`, the data catalog, and
|
|
60
|
-
`framework.db`), so either language can work with a project's data. It works
|
|
61
|
-
without Framework Cloud and without the `fw` command-line tool.
|
|
55
|
+
`framework-py` is the Python implementation of [Framework](https://framework.pub) ([documentation](https://py.framework.pub)). It reads and writes the same project files as the R package (`settings.yml`, the data catalog, and `framework.db`), so either language can read and verify a project's data. It runs standalone without Framework Cloud or the `fw` CLI.
|
|
62
56
|
|
|
63
|
-
Status: in development.
|
|
57
|
+
Status: in development. See [docs/plan.md](docs/plan.md) for the implementation plan.
|
|
64
58
|
|
|
65
|
-
##
|
|
59
|
+
## Installation
|
|
66
60
|
|
|
67
61
|
```sh
|
|
68
62
|
uv add framework-py
|
|
@@ -70,22 +64,19 @@ uv add framework-py
|
|
|
70
64
|
pip install framework-py
|
|
71
65
|
```
|
|
72
66
|
|
|
73
|
-
Optional extras
|
|
74
|
-
`postgres`, `mysql`, `duckdb`, `sqlserver`, `s3`, `excel`, `spss`, `sas`
|
|
75
|
-
(version 8 SAS transport files), and `rds` (R's `.rds` data frames).
|
|
67
|
+
Optional extras install database drivers, S3 storage, and additional file formats: `postgres`, `mysql`, `duckdb`, `sqlserver`, `s3`, `excel`, `spss`, `sas` (version 8 SAS transport files), and `rds` (R `.rds` data frames).
|
|
76
68
|
|
|
77
69
|
```sh
|
|
78
70
|
uv add "framework-py[postgres,s3]"
|
|
79
71
|
```
|
|
80
72
|
|
|
81
|
-
|
|
82
|
-
check that a downloaded wheel or sdist was built from this repository:
|
|
73
|
+
GitHub Actions builds releases with signed build provenance. Verify that a wheel or sdist came from this repository:
|
|
83
74
|
|
|
84
75
|
```sh
|
|
85
76
|
gh attestation verify framework_py-<version>-py3-none-any.whl --repo frameworktools/framework-py
|
|
86
77
|
```
|
|
87
78
|
|
|
88
|
-
##
|
|
79
|
+
## Quick start
|
|
89
80
|
|
|
90
81
|
```python
|
|
91
82
|
import framework as fw
|
|
@@ -95,17 +86,15 @@ info = fw.data_info("inputs.raw.survey")
|
|
|
95
86
|
fw.data_list()
|
|
96
87
|
```
|
|
97
88
|
|
|
98
|
-
|
|
89
|
+
Prepare a cloned project:
|
|
99
90
|
|
|
100
91
|
```sh
|
|
101
92
|
uv run python -m framework bootstrap
|
|
102
93
|
```
|
|
103
94
|
|
|
104
|
-
##
|
|
95
|
+
## Framework Cloud sync
|
|
105
96
|
|
|
106
|
-
Pull or push
|
|
107
|
-
packages, skills, and data registry. Each call lists the changes by part and
|
|
108
|
-
asks once.
|
|
97
|
+
Pull or push packages, skills, and the data registry for a linked Framework Cloud project. Each call lists changes by section and asks once before applying them.
|
|
109
98
|
|
|
110
99
|
```python
|
|
111
100
|
fw.pull_all(dry_run=True)
|
|
@@ -113,8 +102,7 @@ fw.pull_all()
|
|
|
113
102
|
fw.push_all()
|
|
114
103
|
```
|
|
115
104
|
|
|
116
|
-
`pull_skills()
|
|
117
|
-
`push_data_registry()` sync one part.
|
|
105
|
+
`pull_skills()`, `push_skills()`, `pull_data_registry()`, and `push_data_registry()` sync individual sections.
|
|
118
106
|
|
|
119
107
|
## Skill sync
|
|
120
108
|
|
|
@@ -125,9 +113,7 @@ fw.skills_sync(".agents/skills/my-skill", dry_run=True)
|
|
|
125
113
|
fw.skills_sync(".agents/skills/my-skill")
|
|
126
114
|
```
|
|
127
115
|
|
|
128
|
-
Local edits create account versions. Edited Framework skills become private
|
|
129
|
-
forks with an exact source version. Concurrent edits stop for reconciliation.
|
|
130
|
-
[Skill sync guide](https://framework.pub/docs/skill-sync).
|
|
116
|
+
Local edits create account versions. Edited Framework skills become private forks pinned to an exact source version. Concurrent edits stop for reconciliation. See the [Skill sync guide](https://framework.pub/docs/skill-sync).
|
|
131
117
|
|
|
132
118
|
## Development
|
|
133
119
|
|
|
@@ -140,10 +126,6 @@ uv run python scripts/check_parity.py
|
|
|
140
126
|
uv run python scripts/sync_assets.py --check
|
|
141
127
|
```
|
|
142
128
|
|
|
143
|
-
|
|
144
|
-
and the R export list) are copied from the R package with
|
|
145
|
-
`scripts/sync_assets.py`. Change them in the R repository, then copy them
|
|
146
|
-
again.
|
|
129
|
+
`scripts/sync_assets.py` copies shared files (JSON Schemas, the `framework.db` schema, conformance fixtures, and the R export list) from the R package. Edit shared assets in the R repository, then run `scripts/sync_assets.py` here.
|
|
147
130
|
|
|
148
|
-
`parity.toml` maps every
|
|
149
|
-
equivalent, or gives the reason there is none.
|
|
131
|
+
`parity.toml` maps every exported R function to its Python equivalent or records why none exists.
|
|
@@ -20,28 +20,28 @@ framework/_tabular.py,sha256=zaUB83lMCCJql24BKv6CYD8d1s-jjViS0H4lTxeBzzQ,15131
|
|
|
20
20
|
framework/_time.py,sha256=mv-BS3NjOo0hr6bwYRhUBY-3sG6SJmzaboaMD62QcJw,306
|
|
21
21
|
framework/_wincred.py,sha256=oJGS37Dcw_PZVmiAGRu4H_lhcOC5-FqGGYfxVUGPPmU,3891
|
|
22
22
|
framework/_yaml.py,sha256=eYqL4BsjyavrnJGxh88YziZtTgtWDzfeVAtA_Ru78wc,6200
|
|
23
|
-
framework/adopt.py,sha256=
|
|
23
|
+
framework/adopt.py,sha256=oOfpfwo8ehTMDthctihGHi11sEg8StQ3R_85Bq-oq8Y,33441
|
|
24
24
|
framework/ai.py,sha256=hqBNXngw7VyQBcSt7kB20Zpa6N_5WS1hNhDi2zWJw0U,9218
|
|
25
25
|
framework/bootstrap.py,sha256=mYlD1BG9KaG9PAIHlw3l6677fJYwQdtih95W9Rqmnho,9687
|
|
26
|
-
framework/cache.py,sha256=
|
|
26
|
+
framework/cache.py,sha256=z8Zaem5Q7dsAghuX9CgK_j0vhX5dDlRw7fl9f-CF3eA,17440
|
|
27
27
|
framework/catalog.py,sha256=sl4V1judSCVoa2LYLXeQdDhWdsONTC_5s2_fpyPylto,13202
|
|
28
|
-
framework/cloud.py,sha256=
|
|
28
|
+
framework/cloud.py,sha256=aiLjTstIxQJvBqm7NGGbq-osfa61A6L5H8auOE3X544,65949
|
|
29
29
|
framework/config_home.py,sha256=iSb0VuENgNOSnsZ_v53xFqjRiGgrB7OzZbHbyDOqKzs,8265
|
|
30
|
-
framework/connections.py,sha256=
|
|
30
|
+
framework/connections.py,sha256=99JqrNAi2zBpNYyxLV6SYj-NAJjh56pYh9jaMSYOHCU,32742
|
|
31
31
|
framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
|
|
32
|
-
framework/create.py,sha256=
|
|
33
|
-
framework/data.py,sha256=
|
|
34
|
-
framework/data_registry.py,sha256=
|
|
35
|
-
framework/db.py,sha256=
|
|
32
|
+
framework/create.py,sha256=3ykMUhMslVsjiE__9Scx5v_6CU2Ye36wWqlp17j_vGw,53030
|
|
33
|
+
framework/data.py,sha256=cAOxwmF_nowXIZPl6vW5-ZUyI_z16H1bRJM60gJgeoo,36522
|
|
34
|
+
framework/data_registry.py,sha256=2hbXsoY9IF5Zj6lkiTE2RO2c5j2Z2vXKm2iTN7FdsQk,22499
|
|
35
|
+
framework/db.py,sha256=UYEUCvk4vGytBkmdNTUxRpYPtoFSDHwTpgpfWFw31aE,8522
|
|
36
36
|
framework/display.py,sha256=0FuKUt67uf5h1WjD4bwzoruc2XAN4y_j3mz_f5FskhE,5334
|
|
37
37
|
framework/dotenv.py,sha256=6YQVR8d0CnHXHAYxvP3E4Am4Ei9mzqN2g63w_ft7qa0,4642
|
|
38
|
-
framework/errors.py,sha256=
|
|
38
|
+
framework/errors.py,sha256=kOXK0D-7xpcal8XsVnGAV9Nga-3xXxUFq-gdMn--El4,1177
|
|
39
39
|
framework/features.py,sha256=NlR-0Xa4_d_nEBX9MicVyWqE_5L1-sA-zF_pJeNWznw,7729
|
|
40
40
|
framework/global_config.py,sha256=fYLaxN_6Wc2xrIASn77dWkvGR3uuX-111bvfFPB82A4,11036
|
|
41
41
|
framework/hooks.py,sha256=7FbEWBeMaI0QG7lW1myc3KjVmHd92MqhNjuFoTv1OFQ,21204
|
|
42
42
|
framework/integrity.py,sha256=2hXk95tlNGs6HVTFA0AqcPWQg2AiO6Hy0APMIXHpCTU,586
|
|
43
|
-
framework/notebooks.py,sha256=
|
|
44
|
-
framework/outputs.py,sha256=
|
|
43
|
+
framework/notebooks.py,sha256=mZoxYrjG1y2MYb34w7oCPwoQtuKV8MwRgStnxR-Q468,21013
|
|
44
|
+
framework/outputs.py,sha256=bwc5_cEWvhBAW6PiDVjAXSM08mj9kBrgYSsNMTQR9nA,27627
|
|
45
45
|
framework/packages.py,sha256=_ZE_eXDbonl7sNW_mT8MZXbpAhnGYOmBEyAbyQ6qQKo,16100
|
|
46
46
|
framework/project.py,sha256=1aMzfqLZVkxdImyG3ZQcI-ayfV7Rc3Lrqv1JCsY0GVI,5719
|
|
47
47
|
framework/publish_bundle.py,sha256=0vOrLZXSbdV2Fl05smq-2-04Fsts0na3rB_Y0mCOcuY,28802
|
|
@@ -53,8 +53,8 @@ framework/security.py,sha256=-x66bSIIE_xOzkTf5Ojt7xIECKuAIpKVJZ5TtIGrwys,51147
|
|
|
53
53
|
framework/settings.py,sha256=Sb2YyE53aYLWPj423P-pQVCvRpLsiHydgZh2K0luTQQ,23893
|
|
54
54
|
framework/skill_pull.py,sha256=E6N1kMLJAww_wt-kSSNWhcqu-KOsago1C8ClqQ-XHlY,21669
|
|
55
55
|
framework/skill_sync.py,sha256=0gqaeVkiNaqGJxmrVrKrYaX8wI3DNFZUsJEgG63xcKc,23060
|
|
56
|
-
framework/storage.py,sha256=
|
|
57
|
-
framework/sync_all.py,sha256=
|
|
56
|
+
framework/storage.py,sha256=UnTL9buYwawFFGll4D0FRDqsVXjj_B2uYMELXntfq_M,55486
|
|
57
|
+
framework/sync_all.py,sha256=Jub0bWECsWokL7DF5tLYlLUHcDzNRmsD-Nu5X8IAPEY,34531
|
|
58
58
|
framework/upgrade.py,sha256=WtghpLR65YFH_JQFmo1JlH9lHdwsXQd-ZzzOTmyOL28,65593
|
|
59
59
|
framework/_assets/init.sql,sha256=-aIiuIcCmBniGKq-CHdRtUiV9e_CLXoqxxd05snmgOE,1156
|
|
60
60
|
framework/_assets/adopt/rules.json,sha256=Ao_8reCfxohbF27hzVBjXJeMUP2E9MzlLMqrLutv41o,7603
|
|
@@ -113,15 +113,15 @@ framework/spec/merge.py,sha256=gvauBNsFJYe_1J3sO9eF7PSt7mii8ONmnPe8qmyOaGg,6303
|
|
|
113
113
|
framework/spec/resolve.py,sha256=gyAewGbMcdTbh4feUp6DEj_D9wPBDzRiDC8PTOpyp6Q,12590
|
|
114
114
|
framework/templates/defaults.yml,sha256=OPNe85BXmZ1G8CN-RZ62Ku-_uLtx5yC7nqltBL224kE,3933
|
|
115
115
|
framework/templates/reader.html,sha256=SBw3I3rRMdAyoBosMVuZ9Yp177P11CoxIAaG0OOuvQg,19772
|
|
116
|
-
framework/templates/skill-history.json,sha256=
|
|
116
|
+
framework/templates/skill-history.json,sha256=thmGpcHEK67UjWnRvWY0I3WA3Xu2oFbZKlMS_vZkeEw,2118
|
|
117
117
|
framework/templates/gitignore/course,sha256=6SL07s0J71lPepEdAd5BFWBF6Q3WM-SbEXy8mS-uCwc,3085
|
|
118
118
|
framework/templates/gitignore/presentation,sha256=T4iJwMjivw4MjZExBM427nSwJmJoCbutIsJW6QOAomU,2987
|
|
119
119
|
framework/templates/gitignore/project,sha256=A8y6g1SgkOb2uGfPRuQqhtwUEU5ev5vrronkgZZmcVI,2866
|
|
120
120
|
framework/templates/gitignore/sensitive,sha256=gkJwc2bYDtR3QFgGflvJlihpsb5rQkX4JQ7Frlxg0Bo,3794
|
|
121
121
|
framework/templates/skills/framework-cache/.framework-skill.json,sha256=aGE7pPEXxDDgfCXbwpgxN_3XEvAJ3i0udvDswboOoNM,140
|
|
122
122
|
framework/templates/skills/framework-cache/SKILL.md,sha256=HMFnPD5VooarzBo9HU-TCodK7AgKes6rz6kBSjgsUqs,768
|
|
123
|
-
framework/templates/skills/framework-data/.framework-skill.json,sha256=
|
|
124
|
-
framework/templates/skills/framework-data/SKILL.md,sha256=
|
|
123
|
+
framework/templates/skills/framework-data/.framework-skill.json,sha256=JlgWP9twOVFrQyw54trjTCJWd-ZnvdKo-CcG8eLPCGo,139
|
|
124
|
+
framework/templates/skills/framework-data/SKILL.md,sha256=0m-4bkarVQCOw9n7G2kGbbLc0F7fShX-Dz6pjyx9UbI,1850
|
|
125
125
|
framework/templates/skills/framework-databases/.framework-skill.json,sha256=_xbitYo6BaH5P5auj_flabnt4WfIHCnhsm5PAmKJmi4,144
|
|
126
126
|
framework/templates/skills/framework-databases/SKILL.md,sha256=sVGQb_qajIht6Zqdvv2Ml1tFyt5a-PfRZU2sm6JZbzg,768
|
|
127
127
|
framework/templates/skills/framework-outputs/.framework-skill.json,sha256=l-guJxNb4E_ZW74LIT-aMWn278jPDdg1Lv6FvB1CpfI,142
|
|
@@ -139,7 +139,7 @@ framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIj
|
|
|
139
139
|
framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
|
|
140
140
|
framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
|
|
141
141
|
framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
|
|
142
|
-
framework_py-0.1.0.
|
|
143
|
-
framework_py-0.1.0.
|
|
144
|
-
framework_py-0.1.0.
|
|
145
|
-
framework_py-0.1.0.
|
|
142
|
+
framework_py-0.1.0.dev23.dist-info/METADATA,sha256=DZohonN6a3kDz2_LwtsH0KsqLk71R1j40j522qyRyhg,4682
|
|
143
|
+
framework_py-0.1.0.dev23.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
144
|
+
framework_py-0.1.0.dev23.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
|
|
145
|
+
framework_py-0.1.0.dev23.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|