framework-py 0.1.0.dev6__py3-none-any.whl → 0.1.0.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +2 -0
- framework/__main__.py +9 -4
- framework/_assets/hooks/pre-commit +10 -0
- framework/_assets/schema/docs-reference.v1.json +1 -1
- framework/_assets/schema/layer.v1.json +35 -0
- framework/_assets/schema/layers.v1.json +35 -0
- framework/_assets/schema/spec.v1.json +31 -0
- framework/_assets/sets/packages/python-data-analysis.json +34 -0
- framework/_keychain.py +79 -0
- framework/_log.py +9 -1
- framework/_pages.py +98 -14
- framework/_paths.py +19 -0
- framework/adopt.py +3 -2
- framework/ai.py +22 -4
- framework/bootstrap.py +17 -3
- framework/cache.py +19 -2
- framework/catalog.py +22 -2
- framework/cloud.py +208 -52
- framework/config_home.py +7 -1
- framework/connections.py +49 -4
- framework/create.py +32 -9
- framework/data.py +18 -5
- framework/db.py +11 -1
- framework/dotenv.py +57 -6
- framework/hooks.py +18 -11
- framework/notebooks.py +13 -5
- framework/outputs.py +51 -15
- framework/packages.py +13 -8
- framework/publish_bundle.py +532 -0
- framework/scaffold.py +11 -1
- framework/security.py +23 -6
- framework/settings.py +91 -22
- framework/skill_sync.py +3 -3
- framework/spec/check.py +31 -1
- framework/spec/resolve.py +1 -0
- framework/storage.py +280 -28
- framework/templates/reader.html +12 -0
- {framework_py-0.1.0.dev6.dist-info → framework_py-0.1.0.dev7.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev6.dist-info → framework_py-0.1.0.dev7.dist-info}/RECORD +41 -37
- {framework_py-0.1.0.dev6.dist-info → framework_py-0.1.0.dev7.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev6.dist-info → framework_py-0.1.0.dev7.dist-info}/licenses/LICENSE +0 -0
framework/cache.py
CHANGED
|
@@ -321,13 +321,30 @@ def cache_forget(name: str, file: str | Path | None = None) -> None:
|
|
|
321
321
|
|
|
322
322
|
|
|
323
323
|
def cache_flush() -> None:
|
|
324
|
-
"""Remove every cached value (from R and Python) and every cache record.
|
|
324
|
+
"""Remove every cached value (from R and Python) and every cache record.
|
|
325
|
+
|
|
326
|
+
Values stored with ``file=`` outside the cache folder are removed too, as in R, when
|
|
327
|
+
the file has a cache extension (.parquet, .pickle, or .rds).
|
|
328
|
+
"""
|
|
329
|
+
suffixes = (*_FORMATS, _R_FORMAT)
|
|
325
330
|
folder = cache_dir()
|
|
326
331
|
if folder.is_dir():
|
|
327
332
|
for path in folder.iterdir():
|
|
328
|
-
if path.is_file() and path.suffix in
|
|
333
|
+
if path.is_file() and path.suffix in suffixes:
|
|
329
334
|
path.unlink()
|
|
330
335
|
with db.connect() as connection:
|
|
336
|
+
custom = [
|
|
337
|
+
row["file_path"]
|
|
338
|
+
for row in connection.execute(
|
|
339
|
+
"SELECT file_path FROM cache WHERE file_path IS NOT NULL AND file_path != ''"
|
|
340
|
+
)
|
|
341
|
+
]
|
|
342
|
+
if custom:
|
|
343
|
+
root = project_root()
|
|
344
|
+
for stored in custom:
|
|
345
|
+
path = in_root(str(stored), root)
|
|
346
|
+
if path.suffix.lower() in suffixes and path.is_file():
|
|
347
|
+
path.unlink()
|
|
331
348
|
connection.execute("DELETE FROM cache")
|
|
332
349
|
|
|
333
350
|
|
framework/catalog.py
CHANGED
|
@@ -47,8 +47,9 @@ class DataInfo:
|
|
|
47
47
|
|
|
48
48
|
Attributes:
|
|
49
49
|
path: Absolute path to the file.
|
|
50
|
-
type: File type: csv, parquet, rds, excel, stata, spss, spss_por, sas,
|
|
51
|
-
or sas_xpt.
|
|
50
|
+
type: File type: csv, tsv, parquet, rds, excel, stata, spss, spss_por, sas,
|
|
51
|
+
or sas_xpt. A ``.tsv`` file found by its extension is csv with a tab
|
|
52
|
+
delimiter; ``type: tsv`` in the catalog is kept as tsv.
|
|
52
53
|
delimiter: For delimited text: comma, tab, semicolon, or space. None
|
|
53
54
|
means it is guessed from the extension.
|
|
54
55
|
locked: Whether the file's hash is checked against framework.db on read.
|
|
@@ -132,6 +133,22 @@ def _build(
|
|
|
132
133
|
)
|
|
133
134
|
|
|
134
135
|
|
|
136
|
+
def _pathless_entry(item: Any) -> bool:
|
|
137
|
+
"""Whether a mapping is an entry's fields without a path, rather than a group.
|
|
138
|
+
|
|
139
|
+
Every key must be an entry field, and no value may look like a file path, so a
|
|
140
|
+
group with an entry named ``type: data/type.csv`` still reads as a group.
|
|
141
|
+
"""
|
|
142
|
+
if not isinstance(item, dict) or not item or not set(item) <= _KNOWN_FIELDS:
|
|
143
|
+
return False
|
|
144
|
+
return not any(
|
|
145
|
+
isinstance(value, (str, dict))
|
|
146
|
+
and (isinstance(value, dict) or "/" in value or "\\" in value or Path(value).suffix)
|
|
147
|
+
for key, value in item.items()
|
|
148
|
+
if key != "description"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
135
152
|
def catalog_names(data: Any, prefix: str = "") -> list[str]:
|
|
136
153
|
"""Every dot-notation name in a catalog section, in file order."""
|
|
137
154
|
names: list[str] = []
|
|
@@ -141,6 +158,9 @@ def catalog_names(data: Any, prefix: str = "") -> list[str]:
|
|
|
141
158
|
full = f"{prefix}.{key}" if prefix else str(key)
|
|
142
159
|
if isinstance(item, str) or (isinstance(item, dict) and "path" in item):
|
|
143
160
|
names.append(full)
|
|
161
|
+
elif _pathless_entry(item):
|
|
162
|
+
# broken: {type: csv} is an entry missing its path, not a group
|
|
163
|
+
names.append(full)
|
|
144
164
|
elif isinstance(item, dict):
|
|
145
165
|
names.extend(catalog_names(item, full))
|
|
146
166
|
return names
|
framework/cloud.py
CHANGED
|
@@ -20,6 +20,7 @@ Plain ``http`` is accepted only for localhost.
|
|
|
20
20
|
|
|
21
21
|
from __future__ import annotations
|
|
22
22
|
|
|
23
|
+
import contextlib
|
|
23
24
|
import hashlib
|
|
24
25
|
import json
|
|
25
26
|
import mimetypes
|
|
@@ -30,7 +31,9 @@ import shutil
|
|
|
30
31
|
import subprocess
|
|
31
32
|
import sys
|
|
32
33
|
import tempfile
|
|
34
|
+
import time
|
|
33
35
|
import warnings
|
|
36
|
+
import webbrowser
|
|
34
37
|
import zipfile
|
|
35
38
|
from collections.abc import Mapping, Sequence
|
|
36
39
|
from dataclasses import dataclass
|
|
@@ -41,8 +44,9 @@ from typing import Any
|
|
|
41
44
|
import httpx
|
|
42
45
|
import pandas as pd
|
|
43
46
|
|
|
44
|
-
from framework import _secretservice, _wincred, _yaml, db, dotenv
|
|
47
|
+
from framework import _keychain, _secretservice, _wincred, _yaml, db, dotenv
|
|
45
48
|
from framework._log import inform, show_messages
|
|
49
|
+
from framework._paths import rendered_page
|
|
46
50
|
from framework._private import confirm_private_upload
|
|
47
51
|
from framework.config_home import config_dir, ensure_config_dir, r_config_dir
|
|
48
52
|
from framework.errors import FrameworkError, FrameworkWarning
|
|
@@ -112,14 +116,24 @@ def is_default(url: str) -> bool:
|
|
|
112
116
|
|
|
113
117
|
|
|
114
118
|
def _saved_url() -> str | None:
|
|
115
|
-
|
|
119
|
+
"""The default server a login saved, from the newer of fw's and R's cloud.yml.
|
|
120
|
+
|
|
121
|
+
The most recently written file wins, so the last login in any tool counts.
|
|
122
|
+
On a tie fw's folder comes first. fw and R use the same rule.
|
|
123
|
+
"""
|
|
124
|
+
best: str | None = None
|
|
125
|
+
best_time = -1
|
|
126
|
+
for folder in dict.fromkeys((config_dir(), r_config_dir())):
|
|
116
127
|
path = folder / "cloud.yml"
|
|
117
|
-
if path.is_file():
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
128
|
+
if not path.is_file():
|
|
129
|
+
continue
|
|
130
|
+
data = _yaml.read(path)
|
|
131
|
+
server = data.get("server") if isinstance(data, dict) else None
|
|
132
|
+
if isinstance(server, str) and server:
|
|
133
|
+
written = path.stat().st_mtime_ns
|
|
134
|
+
if written > best_time:
|
|
135
|
+
best, best_time = server, written
|
|
136
|
+
return best
|
|
123
137
|
|
|
124
138
|
|
|
125
139
|
def _project_dir(project_dir: str | Path | None) -> Path | None:
|
|
@@ -187,10 +201,13 @@ def _write_token_file(path: Path, token: str) -> None:
|
|
|
187
201
|
|
|
188
202
|
|
|
189
203
|
# Windows and Linux entries are read and written in R's format, so R, fw,
|
|
190
|
-
# and Python share one entry per server. macOS
|
|
204
|
+
# and Python share one entry per server. On macOS, /usr/bin/security reads and
|
|
205
|
+
# writes them, as fw does, so the Keychain never asks for access.
|
|
191
206
|
|
|
192
207
|
|
|
193
208
|
def _keyring_get(service: str) -> str | None:
|
|
209
|
+
if sys.platform == "darwin":
|
|
210
|
+
return _keychain.get(service, "token")
|
|
194
211
|
if sys.platform == "win32":
|
|
195
212
|
return _wincred.get(service, "token")
|
|
196
213
|
if sys.platform == "linux":
|
|
@@ -208,6 +225,8 @@ def _keyring_get(service: str) -> str | None:
|
|
|
208
225
|
|
|
209
226
|
|
|
210
227
|
def _keyring_set(service: str, token: str) -> bool:
|
|
228
|
+
if sys.platform == "darwin":
|
|
229
|
+
return _keychain.set(service, "token", token)
|
|
211
230
|
try:
|
|
212
231
|
if sys.platform == "win32":
|
|
213
232
|
_wincred.set(service, "token", token)
|
|
@@ -223,6 +242,8 @@ def _keyring_set(service: str, token: str) -> bool:
|
|
|
223
242
|
|
|
224
243
|
|
|
225
244
|
def _keyring_delete(service: str) -> bool:
|
|
245
|
+
if sys.platform == "darwin":
|
|
246
|
+
return _keychain.delete(service, "token")
|
|
226
247
|
try:
|
|
227
248
|
if sys.platform == "win32":
|
|
228
249
|
return _wincred.delete(service, "token")
|
|
@@ -379,7 +400,7 @@ def resolve_choices(
|
|
|
379
400
|
|
|
380
401
|
Each Framework-Notice header is shown as a message.
|
|
381
402
|
"""
|
|
382
|
-
response = _request("/api/
|
|
403
|
+
response = _request("/api/v1/resolve", token, url, method="POST", body=body)
|
|
383
404
|
for notice in response.headers.get_list("Framework-Notice"):
|
|
384
405
|
inform(notice)
|
|
385
406
|
envelope = response.json()
|
|
@@ -405,30 +426,114 @@ def consent_notice(url: str) -> str:
|
|
|
405
426
|
)
|
|
406
427
|
|
|
407
428
|
|
|
429
|
+
# Tests replace the pause between approval checks.
|
|
430
|
+
_sleep = time.sleep
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _device_login(server: str, browser: bool) -> str:
|
|
434
|
+
"""Approve this machine in the browser and return the new token.
|
|
435
|
+
|
|
436
|
+
The same flow as ``fw login`` and Stata's ``fw_login``: the server issues a
|
|
437
|
+
short code, you approve it on the server's page, and the token comes back
|
|
438
|
+
here. Nothing is typed or pasted.
|
|
439
|
+
"""
|
|
440
|
+
name = host(server)
|
|
441
|
+
try:
|
|
442
|
+
with _client() as client:
|
|
443
|
+
response = client.post(
|
|
444
|
+
f"{server}/api/v1/cli/device", json={"client_name": f"Python on {platform.node()}"}
|
|
445
|
+
)
|
|
446
|
+
except httpx.HTTPError as error:
|
|
447
|
+
raise CloudOfflineError(f"Could not reach {name}: {error}") from error
|
|
448
|
+
if response.status_code in (404, 405):
|
|
449
|
+
raise CloudError(
|
|
450
|
+
f"{name} does not offer browser login. Create a token at {server}/tokens, "
|
|
451
|
+
"then call cloud_login(token)."
|
|
452
|
+
)
|
|
453
|
+
if response.is_error:
|
|
454
|
+
raise _error(response, "Browser login")
|
|
455
|
+
start = response.json()
|
|
456
|
+
code = start["device_code"]
|
|
457
|
+
page = start.get("verification_url_complete") or start["verification_url"]
|
|
458
|
+
interval = int(start.get("interval") or 5)
|
|
459
|
+
expires = int(start.get("expires_in") or 600)
|
|
460
|
+
print(
|
|
461
|
+
f"Approve this computer in your browser.\n Code: {start.get('user_code', '')}\n"
|
|
462
|
+
f" Page: {page}\nWaiting for approval. Press Ctrl+C to stop.",
|
|
463
|
+
file=sys.stderr,
|
|
464
|
+
)
|
|
465
|
+
if browser:
|
|
466
|
+
# A missing browser only means opening the link by hand.
|
|
467
|
+
with contextlib.suppress(Exception):
|
|
468
|
+
webbrowser.open(page)
|
|
469
|
+
waited = 0
|
|
470
|
+
while waited < expires:
|
|
471
|
+
_sleep(interval)
|
|
472
|
+
waited += interval
|
|
473
|
+
try:
|
|
474
|
+
with _client() as client:
|
|
475
|
+
response = client.post(
|
|
476
|
+
f"{server}/api/v1/cli/device/token", json={"device_code": code}
|
|
477
|
+
)
|
|
478
|
+
except httpx.HTTPError as error:
|
|
479
|
+
raise CloudOfflineError(f"Could not reach {name}: {error}") from error
|
|
480
|
+
if response.status_code == 428:
|
|
481
|
+
continue
|
|
482
|
+
if response.status_code == 429:
|
|
483
|
+
interval += 5
|
|
484
|
+
continue
|
|
485
|
+
if response.status_code == 403:
|
|
486
|
+
raise CloudError("The login request was denied in the browser.")
|
|
487
|
+
if response.status_code == 410:
|
|
488
|
+
break
|
|
489
|
+
if response.is_error:
|
|
490
|
+
raise _error(response, "Browser login")
|
|
491
|
+
token = response.json().get("token")
|
|
492
|
+
if not isinstance(token, str) or not token:
|
|
493
|
+
raise CloudError(f"{name} approved the login but returned no token.")
|
|
494
|
+
return token
|
|
495
|
+
raise CloudError("The login code expired. Run cloud_login() again.")
|
|
496
|
+
|
|
497
|
+
|
|
408
498
|
def cloud_login(
|
|
409
|
-
token: str
|
|
499
|
+
token: str | None = None,
|
|
500
|
+
url: str | None = None,
|
|
501
|
+
verify: bool = True,
|
|
502
|
+
consent: bool | None = None,
|
|
503
|
+
browser: bool = True,
|
|
410
504
|
) -> str:
|
|
411
505
|
"""Connect this machine to a Framework Cloud server.
|
|
412
506
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
507
|
+
Without a token, ``cloud_login()`` opens the server's approval page in your
|
|
508
|
+
browser. Sign in, or create a free account, and approve this computer; the
|
|
509
|
+
token comes back here. With a token from the server's token page,
|
|
510
|
+
``cloud_login(token)`` stores that token instead.
|
|
511
|
+
|
|
512
|
+
The token is stored per server in the system keyring (the same entry R,
|
|
513
|
+
Stata, and ``fw`` use), or in a file readable only by you when no keyring
|
|
514
|
+
is available. The server becomes your default.
|
|
416
515
|
|
|
417
516
|
Args:
|
|
418
|
-
token: A personal token
|
|
517
|
+
token: A personal token. Omit it to approve this computer in the browser.
|
|
419
518
|
url: The server. Default: ``FW_CLOUD_URL``, then your saved default,
|
|
420
519
|
then https://framework.pub.
|
|
421
520
|
verify: Check the token with the server first.
|
|
422
|
-
consent: ``True`` to confirm without a prompt (scripts,
|
|
521
|
+
consent: ``True`` to confirm a pasted token without a prompt (scripts,
|
|
522
|
+
CI). Approving in the browser is the confirmation for browser login.
|
|
523
|
+
browser: Open the approval page. ``False`` prints the link only.
|
|
423
524
|
|
|
424
525
|
Returns:
|
|
425
526
|
Where the token was stored: ``"keyring"`` or a file path.
|
|
426
527
|
"""
|
|
427
528
|
show_messages()
|
|
428
|
-
if len(token) < 10:
|
|
429
|
-
raise CloudError("That does not look like a Framework Cloud token.")
|
|
430
529
|
server = cloud_server(url).url
|
|
431
530
|
name = host(server)
|
|
531
|
+
if token is None:
|
|
532
|
+
print(consent_notice(server), file=sys.stderr)
|
|
533
|
+
token = _device_login(server, browser)
|
|
534
|
+
consent = True
|
|
535
|
+
if len(token) < 10:
|
|
536
|
+
raise CloudError("That does not look like a Framework Cloud token.")
|
|
432
537
|
if consent is not True:
|
|
433
538
|
print(consent_notice(server), file=sys.stderr)
|
|
434
539
|
if not sys.stdin.isatty():
|
|
@@ -469,11 +574,32 @@ def cloud_login(
|
|
|
469
574
|
return stored
|
|
470
575
|
|
|
471
576
|
|
|
577
|
+
def _revoke(server: str, token: str) -> None:
|
|
578
|
+
"""Revoke a token on the server. A token the server already rejects is fine."""
|
|
579
|
+
try:
|
|
580
|
+
with _client() as client:
|
|
581
|
+
response = client.delete(
|
|
582
|
+
f"{server}/api/v1/token", headers={"Authorization": f"Bearer {token}"}
|
|
583
|
+
)
|
|
584
|
+
if response.status_code == 401 or not response.is_error:
|
|
585
|
+
return
|
|
586
|
+
except httpx.HTTPError:
|
|
587
|
+
pass
|
|
588
|
+
inform(
|
|
589
|
+
f"The token could not be revoked on {host(server)}. "
|
|
590
|
+
f"Revoke it on the tokens page: {server}/tokens"
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
|
|
472
594
|
def cloud_logout(url: str | None = None) -> list[str]:
|
|
473
|
-
"""
|
|
595
|
+
"""Revoke and remove this machine's stored token for a server. Returns what was removed.
|
|
474
596
|
|
|
475
|
-
|
|
476
|
-
|
|
597
|
+
Each stored token is revoked on the server first. If the server can't be reached, the
|
|
598
|
+
local copy is still removed, and you can revoke the token on the server's tokens page.
|
|
599
|
+
The keyring entry, the token file, and the file ``FW_TOKEN_FILE`` names are removed. For
|
|
600
|
+
the default server, the legacy keyring entry and ``~/.secrets/framework/token`` are removed
|
|
601
|
+
too. A token in ``FW_CLOUD_TOKEN`` is neither revoked nor removed, and stays in effect
|
|
602
|
+
until you unset it.
|
|
477
603
|
|
|
478
604
|
Args:
|
|
479
605
|
url: The server. Default: ``FW_CLOUD_URL``, then your saved default, then
|
|
@@ -487,12 +613,20 @@ def cloud_logout(url: str | None = None) -> list[str]:
|
|
|
487
613
|
server = cloud_server(url).url
|
|
488
614
|
removed = []
|
|
489
615
|
services = [keyring_service(server)] + ([LEGACY_SERVICE] if is_default(server) else [])
|
|
490
|
-
for service in services:
|
|
491
|
-
if _keyring_delete(service):
|
|
492
|
-
removed.append(f"keyring ({service})")
|
|
493
616
|
paths = [config_dir() / token_file_name(server)]
|
|
494
617
|
if is_default(server):
|
|
495
618
|
paths.append(_legacy_token_file())
|
|
619
|
+
if os.environ.get("FW_TOKEN_FILE"):
|
|
620
|
+
# cloud_login() writes here when it is set, and cloud_token() reads it first
|
|
621
|
+
paths.insert(0, Path(os.environ["FW_TOKEN_FILE"]).expanduser())
|
|
622
|
+
# Only tokens this call removes are revoked, never one in FW_CLOUD_TOKEN
|
|
623
|
+
stored = [_keyring_get(service) for service in services]
|
|
624
|
+
stored += [_read_token_file(path) for path in paths]
|
|
625
|
+
for token in dict.fromkeys(t for t in stored if t):
|
|
626
|
+
_revoke(server, token)
|
|
627
|
+
for service in services:
|
|
628
|
+
if _keyring_delete(service):
|
|
629
|
+
removed.append(f"keyring ({service})")
|
|
496
630
|
for path in paths:
|
|
497
631
|
if path.exists():
|
|
498
632
|
path.unlink()
|
|
@@ -501,6 +635,8 @@ def cloud_logout(url: str | None = None) -> list[str]:
|
|
|
501
635
|
inform(f"[ok] Removed: {', '.join(removed)}")
|
|
502
636
|
else:
|
|
503
637
|
inform(f"No stored token found for {host(server)}.")
|
|
638
|
+
if os.environ.get("FW_CLOUD_TOKEN"):
|
|
639
|
+
inform("FW_CLOUD_TOKEN is still set, so this machine stays connected until you unset it.")
|
|
504
640
|
return removed
|
|
505
641
|
|
|
506
642
|
|
|
@@ -603,8 +739,8 @@ def cloud_sync(direction: str = "auto") -> int:
|
|
|
603
739
|
elif local_changed and remote_ahead:
|
|
604
740
|
raise CloudError(
|
|
605
741
|
"Both your local mirror and the cloud changed since the last sync.\n"
|
|
606
|
-
' cloud_sync("pull")
|
|
607
|
-
'
|
|
742
|
+
' cloud_sync("pull") takes the cloud version and backs up your file.\n'
|
|
743
|
+
' Then copy your edits from the backup and run cloud_sync("push").'
|
|
608
744
|
)
|
|
609
745
|
elif local_changed:
|
|
610
746
|
direction = "push"
|
|
@@ -657,7 +793,9 @@ def cloud_sync(direction: str = "auto") -> int:
|
|
|
657
793
|
|
|
658
794
|
|
|
659
795
|
def _ledger_live() -> bool:
|
|
660
|
-
|
|
796
|
+
from framework.settings import project_dotenv
|
|
797
|
+
|
|
798
|
+
mode = os.environ.get("FW_LEDGER", "") or project_dotenv().get("FW_LEDGER", "")
|
|
661
799
|
if not mode:
|
|
662
800
|
try:
|
|
663
801
|
from framework.settings import settings_read
|
|
@@ -686,7 +824,8 @@ def ledger_push_quietly(name: str, digest: str, file_path: Path | None = None) -
|
|
|
686
824
|
"algo": "sha256",
|
|
687
825
|
"hash": digest,
|
|
688
826
|
"size_bytes": file_path.stat().st_size if file_path and file_path.is_file() else None,
|
|
689
|
-
|
|
827
|
+
# UTC with a Z, as ledger_push() sends: the server stores the clock time as UTC
|
|
828
|
+
"recorded_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
690
829
|
"client": {"device": platform.node(), "package_version": _version()},
|
|
691
830
|
}
|
|
692
831
|
try:
|
|
@@ -804,17 +943,25 @@ def data_verify(name: str) -> bool:
|
|
|
804
943
|
inform(f"Verifying {name} ({info.path})")
|
|
805
944
|
inform(f" file sha256: {digest[:16]}...")
|
|
806
945
|
|
|
807
|
-
record
|
|
946
|
+
# A file path finds the record saved under its catalog name, by its stored path
|
|
947
|
+
from framework._paths import relative_to_root
|
|
948
|
+
from framework.project import project_root
|
|
949
|
+
|
|
950
|
+
stored = relative_to_root(info.path, project_root())
|
|
951
|
+
record = db.get_data_record(name, path=stored)
|
|
808
952
|
if record is not None and record.hash:
|
|
809
953
|
if record.hash == digest:
|
|
810
954
|
inform(" [ok] matches local framework.db record")
|
|
811
955
|
else:
|
|
812
956
|
inform(f" [MISMATCH] local framework.db has {record.hash[:16]}...")
|
|
813
957
|
ok = False
|
|
958
|
+
else:
|
|
959
|
+
inform(" (no framework.db record for this file yet)")
|
|
960
|
+
ledger_name = record.name if record is not None else name
|
|
814
961
|
|
|
815
962
|
if project_token() is not None:
|
|
816
963
|
try:
|
|
817
|
-
entries = _ledger_entries(
|
|
964
|
+
entries = _ledger_entries(ledger_name)
|
|
818
965
|
except CloudError as error:
|
|
819
966
|
inform(f" (cloud ledger unreachable: {error})")
|
|
820
967
|
entries = None
|
|
@@ -863,7 +1010,7 @@ def _publish_collection(
|
|
|
863
1010
|
) -> dict[str, Any]:
|
|
864
1011
|
if isinstance(collection, bool) or not isinstance(collection, (str, int)):
|
|
865
1012
|
raise CloudError("collection must be a collection ID or slug.")
|
|
866
|
-
rows = api("/api/
|
|
1013
|
+
rows = api("/api/v1/collections", token, url)["collections"]
|
|
867
1014
|
matches = [
|
|
868
1015
|
row for row in rows if str(row["id"]) == str(collection) or row["slug"] == str(collection)
|
|
869
1016
|
]
|
|
@@ -897,9 +1044,12 @@ def cloud_publish(
|
|
|
897
1044
|
|
|
898
1045
|
Accepts a self-contained ``.html``, an ``.html`` with a sibling
|
|
899
1046
|
``<name>_files/`` folder, a render output folder, or a ``.qmd`` or ``.ipynb``. Each
|
|
900
|
-
publish adds a version. ``slug`` names the document and
|
|
901
|
-
|
|
902
|
-
|
|
1047
|
+
publish adds a version. A string ``slug`` names the document and leaves its
|
|
1048
|
+
visibility alone: a new artifact is unlisted, and an existing one keeps its setting.
|
|
1049
|
+
``True`` turns the readable URL on with the derived name, and ``False`` turns it off.
|
|
1050
|
+
Pass ``visibility="public"`` to make an artifact public. An explicit ``visibility``
|
|
1051
|
+
always wins over ``True`` and ``False``. A source with "private" in its path needs
|
|
1052
|
+
confirmation, or ``include_private=True``.
|
|
903
1053
|
"""
|
|
904
1054
|
sources = [source]
|
|
905
1055
|
if source.is_dir():
|
|
@@ -910,12 +1060,10 @@ def cloud_publish(
|
|
|
910
1060
|
confirm_private_upload(sources, include_private)
|
|
911
1061
|
token = _require_project_token()
|
|
912
1062
|
url = cloud_server(scope="project").url
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
else (visibility == "public" if visibility is not None else None)
|
|
918
|
-
)
|
|
1063
|
+
# An explicit visibility wins. ``readable`` is the older switch between public
|
|
1064
|
+
# and unlisted, sent only for slug=True or False. A string slug never sets it.
|
|
1065
|
+
readable = slug if isinstance(slug, bool) and visibility is None else None
|
|
1066
|
+
public = visibility == "public" if visibility is not None else readable
|
|
919
1067
|
selected = (
|
|
920
1068
|
None
|
|
921
1069
|
if collection is None
|
|
@@ -954,13 +1102,14 @@ def cloud_publish(
|
|
|
954
1102
|
command.append("--embed-resources")
|
|
955
1103
|
if subprocess.run(command, capture_output=True, check=False).returncode != 0:
|
|
956
1104
|
raise CloudError("Quarto render failed.")
|
|
957
|
-
|
|
958
|
-
|
|
1105
|
+
# A project's _quarto.yml keeps the notebook's folder under the output folder
|
|
1106
|
+
page = rendered_page(out, source.stem)
|
|
1107
|
+
if page is None:
|
|
959
1108
|
raise CloudError("Quarto render produced no HTML output.")
|
|
960
1109
|
if self_contained:
|
|
961
|
-
html =
|
|
1110
|
+
html = page
|
|
962
1111
|
else:
|
|
963
|
-
bundle, entrypoint =
|
|
1112
|
+
bundle, entrypoint = page.parent, page.name
|
|
964
1113
|
elif suffix in (".html", ".htm"):
|
|
965
1114
|
assets = source.with_name(f"{source.stem}_files")
|
|
966
1115
|
if assets.is_dir():
|
|
@@ -974,7 +1123,6 @@ def cloud_publish(
|
|
|
974
1123
|
else:
|
|
975
1124
|
html = source
|
|
976
1125
|
|
|
977
|
-
readable = None if slug is None else slug is not False
|
|
978
1126
|
name = slug if isinstance(slug, str) else (dest or source.stem)
|
|
979
1127
|
data: dict[str, str] = {"slug": re.sub(r"\.html?$", "", name)}
|
|
980
1128
|
if readable is not None:
|
|
@@ -1021,14 +1169,22 @@ def cloud_publish(
|
|
|
1021
1169
|
inform(f"Published version {result['version']}: {result.get('url')}")
|
|
1022
1170
|
else:
|
|
1023
1171
|
inform(f"Published: {result.get('url')}")
|
|
1024
|
-
|
|
1025
|
-
|
|
1172
|
+
# The server's notice assumes an unlisted link. Say it plainly for a public
|
|
1173
|
+
# artifact whose owner has no handle yet.
|
|
1174
|
+
notice = str(result.get("notice") or "")
|
|
1175
|
+
public = result.get("visibility") == "public" or (
|
|
1176
|
+
result.get("visibility") is None and "handle" in notice
|
|
1177
|
+
)
|
|
1178
|
+
if public and not result.get("readable_url"):
|
|
1179
|
+
inform("Public. Set a handle in Account > Account details to get a readable URL.")
|
|
1180
|
+
elif notice and "handle" not in notice:
|
|
1181
|
+
inform(notice)
|
|
1026
1182
|
if selected is not None:
|
|
1027
1183
|
try:
|
|
1028
1184
|
if not result.get("id"):
|
|
1029
1185
|
raise CloudError("The server did not return an artifact ID.")
|
|
1030
1186
|
api(
|
|
1031
|
-
f"/api/
|
|
1187
|
+
f"/api/v1/collections/{selected['id']}/items",
|
|
1032
1188
|
token,
|
|
1033
1189
|
url,
|
|
1034
1190
|
method="POST",
|
|
@@ -1083,7 +1239,7 @@ def registration_body(
|
|
|
1083
1239
|
package_sets: Sequence[str] = (),
|
|
1084
1240
|
skill_sets: Sequence[str] = (),
|
|
1085
1241
|
) -> dict[str, Any]:
|
|
1086
|
-
"""The choices a project built offline was made from, for ``/api/
|
|
1242
|
+
"""The choices a project built offline was made from, for ``/api/v1/projects``.
|
|
1087
1243
|
|
|
1088
1244
|
Each of Framework's five types is a structure of the same name.
|
|
1089
1245
|
"""
|
|
@@ -1120,7 +1276,7 @@ def register_project(
|
|
|
1120
1276
|
"""Register a new local project on the cloud when logged in. Never fails creation.
|
|
1121
1277
|
|
|
1122
1278
|
The cloud records the structure, tools, sets, and settings the project
|
|
1123
|
-
was built from. A server without ``/api/
|
|
1279
|
+
was built from. A server without ``/api/v1/projects`` gets the older
|
|
1124
1280
|
registration.
|
|
1125
1281
|
"""
|
|
1126
1282
|
server = cloud_server().url
|
|
@@ -1131,7 +1287,7 @@ def register_project(
|
|
|
1131
1287
|
if spec is not None:
|
|
1132
1288
|
body = registration_body(name, type, spec, package_sets, skill_sets)
|
|
1133
1289
|
try:
|
|
1134
|
-
result = api("/api/
|
|
1290
|
+
result = api("/api/v1/projects", token, server, method="POST", body=body)
|
|
1135
1291
|
slug, project_token = result.get("slug"), result.get("token")
|
|
1136
1292
|
except _MissingEndpointError:
|
|
1137
1293
|
spec = None
|
|
@@ -1152,7 +1308,7 @@ def fetch_layers(key: str, url: str | None = None) -> tuple[dict[str, Any], str]
|
|
|
1152
1308
|
"""A project's layers file from the cloud, by its project token."""
|
|
1153
1309
|
server = cloud_server(url).url
|
|
1154
1310
|
inform(f"Fetching this project's definition from {server} (what is shared: {server}/privacy)")
|
|
1155
|
-
envelope = api("/api/
|
|
1311
|
+
envelope = api("/api/v1/projects/pull", key, server)
|
|
1156
1312
|
if not isinstance(envelope, dict):
|
|
1157
1313
|
raise CloudError("The server returned an unexpected project definition.")
|
|
1158
1314
|
return envelope, server
|
framework/config_home.py
CHANGED
|
@@ -19,6 +19,7 @@ from typing import Any
|
|
|
19
19
|
import pandas as pd
|
|
20
20
|
|
|
21
21
|
from framework import _yaml
|
|
22
|
+
from framework.errors import SettingsError
|
|
22
23
|
|
|
23
24
|
# Files config_sync() copies. Token files hold a secret and are written 0600.
|
|
24
25
|
_SYNC_FILES = ("settings.yml", "cloud.yml")
|
|
@@ -77,6 +78,8 @@ class SyncResult:
|
|
|
77
78
|
|
|
78
79
|
def __str__(self) -> str:
|
|
79
80
|
verb = "Would copy" if self.dry_run else "Copied"
|
|
81
|
+
if not self.copied and not self.source.is_dir():
|
|
82
|
+
return f"Nothing to copy: {self.source} does not exist."
|
|
80
83
|
if not self.copied:
|
|
81
84
|
return f"Nothing to copy from {self.source} (Python's folder is up to date)."
|
|
82
85
|
return f"{verb} {', '.join(self.copied)} from {self.source} to {self.target}."
|
|
@@ -140,7 +143,10 @@ def global_settings() -> dict[str, Any]:
|
|
|
140
143
|
path = _global_settings_path()
|
|
141
144
|
if not path.is_file():
|
|
142
145
|
return {}
|
|
143
|
-
|
|
146
|
+
try:
|
|
147
|
+
data = _yaml.read(path)
|
|
148
|
+
except Exception as error:
|
|
149
|
+
raise SettingsError(f"Failed to parse global settings '{path}': {error}") from error
|
|
144
150
|
return data if isinstance(data, dict) else {}
|
|
145
151
|
|
|
146
152
|
|