framework-py 0.1.0.dev16__py3-none-any.whl → 0.1.0.dev18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +3 -0
- framework/_assets/schema/layer.v1.json +80 -0
- framework/_assets/schema/layers.v1.json +80 -0
- framework/_assets/schema/spec.v1.json +73 -0
- framework/_assets/sets/packages/easystats-r.json +54 -0
- framework/_pages.py +5 -1
- framework/cloud.py +16 -7
- framework/create.py +8 -4
- framework/data_registry.py +525 -0
- framework/hooks.py +6 -1
- framework/scaffold.py +3 -1
- framework/spec/check.py +75 -1
- framework/spec/merge.py +1 -0
- framework/spec/resolve.py +1 -0
- {framework_py-0.1.0.dev16.dist-info → framework_py-0.1.0.dev18.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev16.dist-info → framework_py-0.1.0.dev18.dist-info}/RECORD +18 -16
- {framework_py-0.1.0.dev16.dist-info → framework_py-0.1.0.dev18.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev16.dist-info → framework_py-0.1.0.dev18.dist-info}/licenses/LICENSE +0 -0
framework/__init__.py
CHANGED
|
@@ -63,6 +63,7 @@ from framework.create import (
|
|
|
63
63
|
project_create_from_layers,
|
|
64
64
|
)
|
|
65
65
|
from framework.data import data_add, data_read, data_save
|
|
66
|
+
from framework.data_registry import pull_data_registry, push_data_registry
|
|
66
67
|
from framework.display import TableDisplay, show_table
|
|
67
68
|
from framework.errors import (
|
|
68
69
|
CacheWarning,
|
|
@@ -249,8 +250,10 @@ __all__ = [
|
|
|
249
250
|
"publish_dir",
|
|
250
251
|
"publish_list",
|
|
251
252
|
"publish_notebook",
|
|
253
|
+
"pull_data_registry",
|
|
252
254
|
"pull_setup",
|
|
253
255
|
"pull_skills",
|
|
256
|
+
"push_data_registry",
|
|
254
257
|
"push_setup",
|
|
255
258
|
"push_skills",
|
|
256
259
|
"quarto_generate_all",
|
|
@@ -411,6 +411,23 @@
|
|
|
411
411
|
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
412
412
|
}
|
|
413
413
|
},
|
|
414
|
+
"data": {
|
|
415
|
+
"type": "object",
|
|
416
|
+
"additionalProperties": {
|
|
417
|
+
"anyOf": [
|
|
418
|
+
{
|
|
419
|
+
"$ref": "#/definitions/data_entry"
|
|
420
|
+
},
|
|
421
|
+
{
|
|
422
|
+
"type": "null"
|
|
423
|
+
}
|
|
424
|
+
]
|
|
425
|
+
},
|
|
426
|
+
"description": "The data catalog: dot-notation name -> entry. Each entry is replaced whole when a later layer sets it; project creation writes the names nested under data in the settings.",
|
|
427
|
+
"propertyNames": {
|
|
428
|
+
"pattern": "^[A-Za-z0-9_][A-Za-z0-9_-]*([.][A-Za-z0-9_][A-Za-z0-9_-]*)*$"
|
|
429
|
+
}
|
|
430
|
+
},
|
|
414
431
|
"data_hooks": {
|
|
415
432
|
"type": "object",
|
|
416
433
|
"additionalProperties": {
|
|
@@ -732,6 +749,69 @@
|
|
|
732
749
|
"required": [
|
|
733
750
|
"driver"
|
|
734
751
|
]
|
|
752
|
+
},
|
|
753
|
+
"data_entry": {
|
|
754
|
+
"type": "object",
|
|
755
|
+
"properties": {
|
|
756
|
+
"path": {
|
|
757
|
+
"type": "string",
|
|
758
|
+
"minLength": 1,
|
|
759
|
+
"pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))[^\\\\]+$"
|
|
760
|
+
},
|
|
761
|
+
"type": {
|
|
762
|
+
"type": "string",
|
|
763
|
+
"enum": [
|
|
764
|
+
"csv",
|
|
765
|
+
"tsv",
|
|
766
|
+
"rds",
|
|
767
|
+
"excel",
|
|
768
|
+
"xlsx",
|
|
769
|
+
"xls",
|
|
770
|
+
"parquet",
|
|
771
|
+
"stata",
|
|
772
|
+
"dta",
|
|
773
|
+
"spss",
|
|
774
|
+
"sav",
|
|
775
|
+
"zsav",
|
|
776
|
+
"spss_por",
|
|
777
|
+
"por",
|
|
778
|
+
"sas",
|
|
779
|
+
"sas7bdat",
|
|
780
|
+
"sas7bcat",
|
|
781
|
+
"sas_xpt",
|
|
782
|
+
"xpt"
|
|
783
|
+
]
|
|
784
|
+
},
|
|
785
|
+
"delimiter": {
|
|
786
|
+
"type": "string",
|
|
787
|
+
"enum": [
|
|
788
|
+
"comma",
|
|
789
|
+
"tab",
|
|
790
|
+
"semicolon",
|
|
791
|
+
"space"
|
|
792
|
+
]
|
|
793
|
+
},
|
|
794
|
+
"locked": {
|
|
795
|
+
"type": "boolean"
|
|
796
|
+
},
|
|
797
|
+
"private": {
|
|
798
|
+
"type": "boolean"
|
|
799
|
+
},
|
|
800
|
+
"description": {
|
|
801
|
+
"type": "string"
|
|
802
|
+
},
|
|
803
|
+
"stata_names": {
|
|
804
|
+
"type": "object",
|
|
805
|
+
"additionalProperties": {
|
|
806
|
+
"type": "string"
|
|
807
|
+
},
|
|
808
|
+
"description": "Column name in the file -> Stata variable name (fw_use)"
|
|
809
|
+
}
|
|
810
|
+
},
|
|
811
|
+
"additionalProperties": false,
|
|
812
|
+
"required": [
|
|
813
|
+
"path"
|
|
814
|
+
]
|
|
735
815
|
}
|
|
736
816
|
}
|
|
737
817
|
}
|
|
@@ -385,6 +385,69 @@
|
|
|
385
385
|
"driver"
|
|
386
386
|
]
|
|
387
387
|
},
|
|
388
|
+
"data_entry": {
|
|
389
|
+
"type": "object",
|
|
390
|
+
"properties": {
|
|
391
|
+
"path": {
|
|
392
|
+
"type": "string",
|
|
393
|
+
"minLength": 1,
|
|
394
|
+
"pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))[^\\\\]+$"
|
|
395
|
+
},
|
|
396
|
+
"type": {
|
|
397
|
+
"type": "string",
|
|
398
|
+
"enum": [
|
|
399
|
+
"csv",
|
|
400
|
+
"tsv",
|
|
401
|
+
"rds",
|
|
402
|
+
"excel",
|
|
403
|
+
"xlsx",
|
|
404
|
+
"xls",
|
|
405
|
+
"parquet",
|
|
406
|
+
"stata",
|
|
407
|
+
"dta",
|
|
408
|
+
"spss",
|
|
409
|
+
"sav",
|
|
410
|
+
"zsav",
|
|
411
|
+
"spss_por",
|
|
412
|
+
"por",
|
|
413
|
+
"sas",
|
|
414
|
+
"sas7bdat",
|
|
415
|
+
"sas7bcat",
|
|
416
|
+
"sas_xpt",
|
|
417
|
+
"xpt"
|
|
418
|
+
]
|
|
419
|
+
},
|
|
420
|
+
"delimiter": {
|
|
421
|
+
"type": "string",
|
|
422
|
+
"enum": [
|
|
423
|
+
"comma",
|
|
424
|
+
"tab",
|
|
425
|
+
"semicolon",
|
|
426
|
+
"space"
|
|
427
|
+
]
|
|
428
|
+
},
|
|
429
|
+
"locked": {
|
|
430
|
+
"type": "boolean"
|
|
431
|
+
},
|
|
432
|
+
"private": {
|
|
433
|
+
"type": "boolean"
|
|
434
|
+
},
|
|
435
|
+
"description": {
|
|
436
|
+
"type": "string"
|
|
437
|
+
},
|
|
438
|
+
"stata_names": {
|
|
439
|
+
"type": "object",
|
|
440
|
+
"additionalProperties": {
|
|
441
|
+
"type": "string"
|
|
442
|
+
},
|
|
443
|
+
"description": "Column name in the file -> Stata variable name (fw_use)"
|
|
444
|
+
}
|
|
445
|
+
},
|
|
446
|
+
"additionalProperties": false,
|
|
447
|
+
"required": [
|
|
448
|
+
"path"
|
|
449
|
+
]
|
|
450
|
+
},
|
|
388
451
|
"layer": {
|
|
389
452
|
"title": "Framework project spec layer",
|
|
390
453
|
"description": "One unmerged layer of a project spec. Every field is optional, and a null map entry removes that key when spec_resolve() merges layers.",
|
|
@@ -796,6 +859,23 @@
|
|
|
796
859
|
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
797
860
|
}
|
|
798
861
|
},
|
|
862
|
+
"data": {
|
|
863
|
+
"type": "object",
|
|
864
|
+
"additionalProperties": {
|
|
865
|
+
"anyOf": [
|
|
866
|
+
{
|
|
867
|
+
"$ref": "#/definitions/data_entry"
|
|
868
|
+
},
|
|
869
|
+
{
|
|
870
|
+
"type": "null"
|
|
871
|
+
}
|
|
872
|
+
]
|
|
873
|
+
},
|
|
874
|
+
"description": "The data catalog: dot-notation name -> entry. Each entry is replaced whole when a later layer sets it; project creation writes the names nested under data in the settings.",
|
|
875
|
+
"propertyNames": {
|
|
876
|
+
"pattern": "^[A-Za-z0-9_][A-Za-z0-9_-]*([.][A-Za-z0-9_][A-Za-z0-9_-]*)*$"
|
|
877
|
+
}
|
|
878
|
+
},
|
|
799
879
|
"data_hooks": {
|
|
800
880
|
"type": "object",
|
|
801
881
|
"additionalProperties": {
|
|
@@ -369,6 +369,16 @@
|
|
|
369
369
|
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
370
370
|
}
|
|
371
371
|
},
|
|
372
|
+
"data": {
|
|
373
|
+
"type": "object",
|
|
374
|
+
"additionalProperties": {
|
|
375
|
+
"$ref": "#/definitions/data_entry"
|
|
376
|
+
},
|
|
377
|
+
"description": "The data catalog: dot-notation name -> entry. Each entry is replaced whole when a later layer sets it; project creation writes the names nested under data in the settings.",
|
|
378
|
+
"propertyNames": {
|
|
379
|
+
"pattern": "^[A-Za-z0-9_][A-Za-z0-9_-]*([.][A-Za-z0-9_][A-Za-z0-9_-]*)*$"
|
|
380
|
+
}
|
|
381
|
+
},
|
|
372
382
|
"data_hooks": {
|
|
373
383
|
"type": "object",
|
|
374
384
|
"additionalProperties": {
|
|
@@ -720,6 +730,69 @@
|
|
|
720
730
|
"required": [
|
|
721
731
|
"driver"
|
|
722
732
|
]
|
|
733
|
+
},
|
|
734
|
+
"data_entry": {
|
|
735
|
+
"type": "object",
|
|
736
|
+
"properties": {
|
|
737
|
+
"path": {
|
|
738
|
+
"type": "string",
|
|
739
|
+
"minLength": 1,
|
|
740
|
+
"pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))[^\\\\]+$"
|
|
741
|
+
},
|
|
742
|
+
"type": {
|
|
743
|
+
"type": "string",
|
|
744
|
+
"enum": [
|
|
745
|
+
"csv",
|
|
746
|
+
"tsv",
|
|
747
|
+
"rds",
|
|
748
|
+
"excel",
|
|
749
|
+
"xlsx",
|
|
750
|
+
"xls",
|
|
751
|
+
"parquet",
|
|
752
|
+
"stata",
|
|
753
|
+
"dta",
|
|
754
|
+
"spss",
|
|
755
|
+
"sav",
|
|
756
|
+
"zsav",
|
|
757
|
+
"spss_por",
|
|
758
|
+
"por",
|
|
759
|
+
"sas",
|
|
760
|
+
"sas7bdat",
|
|
761
|
+
"sas7bcat",
|
|
762
|
+
"sas_xpt",
|
|
763
|
+
"xpt"
|
|
764
|
+
]
|
|
765
|
+
},
|
|
766
|
+
"delimiter": {
|
|
767
|
+
"type": "string",
|
|
768
|
+
"enum": [
|
|
769
|
+
"comma",
|
|
770
|
+
"tab",
|
|
771
|
+
"semicolon",
|
|
772
|
+
"space"
|
|
773
|
+
]
|
|
774
|
+
},
|
|
775
|
+
"locked": {
|
|
776
|
+
"type": "boolean"
|
|
777
|
+
},
|
|
778
|
+
"private": {
|
|
779
|
+
"type": "boolean"
|
|
780
|
+
},
|
|
781
|
+
"description": {
|
|
782
|
+
"type": "string"
|
|
783
|
+
},
|
|
784
|
+
"stata_names": {
|
|
785
|
+
"type": "object",
|
|
786
|
+
"additionalProperties": {
|
|
787
|
+
"type": "string"
|
|
788
|
+
},
|
|
789
|
+
"description": "Column name in the file -> Stata variable name (fw_use)"
|
|
790
|
+
}
|
|
791
|
+
},
|
|
792
|
+
"additionalProperties": false,
|
|
793
|
+
"required": [
|
|
794
|
+
"path"
|
|
795
|
+
]
|
|
723
796
|
}
|
|
724
797
|
}
|
|
725
798
|
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "easystats-r",
|
|
3
|
+
"name": "easystats",
|
|
4
|
+
"description": "The easystats packages: parameters, performance, effectsize, modelbased, correlation, bayestestR, and report to describe and check models, datawizard for data, and see for their plots.",
|
|
5
|
+
"language": [
|
|
6
|
+
"r"
|
|
7
|
+
],
|
|
8
|
+
"layer": {
|
|
9
|
+
"packages": {
|
|
10
|
+
"default_packages": {
|
|
11
|
+
"insight": {
|
|
12
|
+
"source": "cran",
|
|
13
|
+
"auto_attach": false
|
|
14
|
+
},
|
|
15
|
+
"datawizard": {
|
|
16
|
+
"source": "cran",
|
|
17
|
+
"auto_attach": false
|
|
18
|
+
},
|
|
19
|
+
"parameters": {
|
|
20
|
+
"source": "cran",
|
|
21
|
+
"auto_attach": false
|
|
22
|
+
},
|
|
23
|
+
"performance": {
|
|
24
|
+
"source": "cran",
|
|
25
|
+
"auto_attach": false
|
|
26
|
+
},
|
|
27
|
+
"effectsize": {
|
|
28
|
+
"source": "cran",
|
|
29
|
+
"auto_attach": false
|
|
30
|
+
},
|
|
31
|
+
"modelbased": {
|
|
32
|
+
"source": "cran",
|
|
33
|
+
"auto_attach": false
|
|
34
|
+
},
|
|
35
|
+
"correlation": {
|
|
36
|
+
"source": "cran",
|
|
37
|
+
"auto_attach": false
|
|
38
|
+
},
|
|
39
|
+
"bayestestR": {
|
|
40
|
+
"source": "cran",
|
|
41
|
+
"auto_attach": false
|
|
42
|
+
},
|
|
43
|
+
"report": {
|
|
44
|
+
"source": "cran",
|
|
45
|
+
"auto_attach": false
|
|
46
|
+
},
|
|
47
|
+
"see": {
|
|
48
|
+
"source": "cran",
|
|
49
|
+
"auto_attach": false
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
framework/_pages.py
CHANGED
|
@@ -36,7 +36,11 @@ class _Remote:
|
|
|
36
36
|
|
|
37
37
|
def _git(top: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
|
38
38
|
return subprocess.run(
|
|
39
|
-
["git", "-C", str(top), *args],
|
|
39
|
+
["git", "-C", str(top), *args],
|
|
40
|
+
capture_output=True,
|
|
41
|
+
encoding="utf-8",
|
|
42
|
+
errors="replace",
|
|
43
|
+
check=False,
|
|
40
44
|
)
|
|
41
45
|
|
|
42
46
|
|
framework/cloud.py
CHANGED
|
@@ -61,7 +61,12 @@ _TIMEOUT = 30.0
|
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
class CloudError(FrameworkError):
|
|
64
|
-
"""A Framework Cloud request failed.
|
|
64
|
+
"""A Framework Cloud request failed.
|
|
65
|
+
|
|
66
|
+
``status`` is the HTTP status of the reply, or None when there was no reply.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
status: int | None = None
|
|
65
70
|
|
|
66
71
|
|
|
67
72
|
def _version() -> str:
|
|
@@ -372,13 +377,17 @@ def _error(response: httpx.Response, what: str) -> CloudError:
|
|
|
372
377
|
]
|
|
373
378
|
if details:
|
|
374
379
|
message = "\n".join([str(message), *details])
|
|
380
|
+
error: CloudError
|
|
375
381
|
if response.status_code in (404, 405):
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
+
error = _MissingEndpointError(str(message))
|
|
383
|
+
elif response.status_code == 409:
|
|
384
|
+
error = CloudConflictError(str(message), body)
|
|
385
|
+
elif response.status_code >= 500:
|
|
386
|
+
error = CloudOfflineError(str(message))
|
|
387
|
+
else:
|
|
388
|
+
error = CloudError(str(message))
|
|
389
|
+
error.status = response.status_code
|
|
390
|
+
return error
|
|
382
391
|
|
|
383
392
|
|
|
384
393
|
def api(
|
framework/create.py
CHANGED
|
@@ -26,7 +26,7 @@ from framework.ai import write_ai_files
|
|
|
26
26
|
from framework.config_home import config_dir, global_settings, project_register
|
|
27
27
|
from framework.errors import FrameworkError
|
|
28
28
|
from framework.quarto import quarto_generate_all
|
|
29
|
-
from framework.spec.check import ASSISTANTS, spec_check
|
|
29
|
+
from framework.spec.check import ASSISTANTS, catalog_nest, spec_check
|
|
30
30
|
from framework.spec.merge import SpecError
|
|
31
31
|
from framework.spec.resolve import (
|
|
32
32
|
TYPES,
|
|
@@ -172,11 +172,11 @@ def _write_settings(root: Path, spec: Mapping[str, Any]) -> None:
|
|
|
172
172
|
main["connections"] = "settings/connections.yml"
|
|
173
173
|
_yaml.write(root / "settings/connections.yml", {"connections": sections["connections"]})
|
|
174
174
|
main["data"] = "settings/data.yml"
|
|
175
|
-
_yaml.write(root / "settings/data.yml", {"data":
|
|
175
|
+
_yaml.write(root / "settings/data.yml", {"data": catalog_nest(spec.get("data"))})
|
|
176
176
|
else:
|
|
177
177
|
for section in ("author", "quarto", "git", "ai", "scaffold"):
|
|
178
178
|
main[section] = sections[section]
|
|
179
|
-
main["data"] =
|
|
179
|
+
main["data"] = catalog_nest(spec.get("data"))
|
|
180
180
|
main["connections"] = sections["connections"]
|
|
181
181
|
|
|
182
182
|
if spec.get("publish") is not None:
|
|
@@ -403,7 +403,11 @@ def _git_init(root: Path) -> None:
|
|
|
403
403
|
|
|
404
404
|
def git(*args: str) -> subprocess.CompletedProcess[str]:
|
|
405
405
|
return subprocess.run(
|
|
406
|
-
["git", "-C", str(root), *args],
|
|
406
|
+
["git", "-C", str(root), *args],
|
|
407
|
+
capture_output=True,
|
|
408
|
+
encoding="utf-8",
|
|
409
|
+
errors="replace",
|
|
410
|
+
check=False,
|
|
407
411
|
)
|
|
408
412
|
|
|
409
413
|
from framework.hooks import git_hooks_install, outer_repository
|
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
"""A project's data registry (the catalog in the settings' ``data`` section), kept in step
|
|
2
|
+
with Framework Cloud.
|
|
3
|
+
|
|
4
|
+
``pull_data_registry()`` brings Cloud's entries into the settings; ``push_data_registry()``
|
|
5
|
+
sends the project's entries back. Only the definitions travel: never the data files, and
|
|
6
|
+
never their hashes, which stay in framework.db. R: ``pull_data_registry()``,
|
|
7
|
+
``push_data_registry()``; fw: ``fw data pull`` / ``fw data push``.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from datetime import UTC, datetime
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from framework import _yaml
|
|
18
|
+
from framework.catalog import _pathless_entry, catalog_locate
|
|
19
|
+
from framework.errors import FrameworkError
|
|
20
|
+
from framework.spec.check import DATA_FIELDS
|
|
21
|
+
|
|
22
|
+
_LOCK = ".framework/data.lock.json"
|
|
23
|
+
_PATH = "/api/v1/projects/data"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# Reading the local catalog ---------------------------------------------------------------
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _entries(catalog: Any, prefix: str = "") -> dict[str, dict[str, Any]]:
|
|
30
|
+
"""Every entry in a settings data section, by dot-notation name, as data_list() finds
|
|
31
|
+
them: nested, mixed, and flat keys, and path shorthands. Entries without a path are
|
|
32
|
+
left out."""
|
|
33
|
+
found: dict[str, dict[str, Any]] = {}
|
|
34
|
+
if not isinstance(catalog, dict):
|
|
35
|
+
return found
|
|
36
|
+
for key, item in catalog.items():
|
|
37
|
+
name = f"{prefix}.{key}" if prefix else str(key)
|
|
38
|
+
if isinstance(item, str):
|
|
39
|
+
found[name] = {"path": item}
|
|
40
|
+
elif isinstance(item, dict) and "path" in item:
|
|
41
|
+
found[name] = dict(item)
|
|
42
|
+
elif isinstance(item, dict) and not _pathless_entry(item):
|
|
43
|
+
found.update(_entries(item, name))
|
|
44
|
+
return found
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _row(name: str, entry: dict[str, Any]) -> dict[str, Any]:
|
|
48
|
+
"""An entry as a registry row: its name and the fields that travel."""
|
|
49
|
+
row: dict[str, Any] = {"name": name}
|
|
50
|
+
for field in DATA_FIELDS:
|
|
51
|
+
value: Any = entry.get(field)
|
|
52
|
+
if value is None:
|
|
53
|
+
continue
|
|
54
|
+
if field in ("locked", "private"):
|
|
55
|
+
value = bool(value)
|
|
56
|
+
if field == "stata_names":
|
|
57
|
+
if not value:
|
|
58
|
+
continue
|
|
59
|
+
value = {str(k): v for k, v in dict(value).items()}
|
|
60
|
+
row[field] = value
|
|
61
|
+
row.setdefault("locked", False)
|
|
62
|
+
return row
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _key(row: dict[str, Any]) -> tuple[Any, ...]:
|
|
66
|
+
"""What counts as a change: every field that travels."""
|
|
67
|
+
names = row.get("stata_names") or {}
|
|
68
|
+
return (
|
|
69
|
+
row.get("path") or "",
|
|
70
|
+
row.get("type") or "",
|
|
71
|
+
row.get("delimiter") or "",
|
|
72
|
+
bool(row.get("locked")),
|
|
73
|
+
None if row.get("private") is None else bool(row.get("private")),
|
|
74
|
+
row.get("description") or "",
|
|
75
|
+
tuple(sorted((str(k), str(v)) for k, v in dict(names).items())),
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _same(a: dict[str, Any], b: dict[str, Any]) -> bool:
|
|
80
|
+
return _key(a) == _key(b)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _diff(before: dict[str, Any], after: dict[str, Any]) -> str:
|
|
84
|
+
"""How ``after`` differs from ``before``, one item per field: "path a -> b" for the
|
|
85
|
+
short fields, "description changed" or "stata_names changed" for the long ones."""
|
|
86
|
+
|
|
87
|
+
def shown(value: Any) -> str:
|
|
88
|
+
if value is None or value == "":
|
|
89
|
+
return "none"
|
|
90
|
+
return str(value).lower() if isinstance(value, bool) else str(value)
|
|
91
|
+
|
|
92
|
+
parts = []
|
|
93
|
+
for field, a, b in zip(DATA_FIELDS, _key(before), _key(after), strict=True):
|
|
94
|
+
if a == b:
|
|
95
|
+
continue
|
|
96
|
+
if field in ("description", "stata_names"):
|
|
97
|
+
parts.append(f"{field} changed")
|
|
98
|
+
else:
|
|
99
|
+
parts.append(f"{field} {shown(a)} -> {shown(b)}")
|
|
100
|
+
return ", ".join(parts)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _names(rows: list[dict[str, Any]]) -> str:
|
|
104
|
+
return ", ".join(str(r.get("name")) for r in rows)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _location(root: Path) -> tuple[Path, list[str], Any]:
|
|
108
|
+
"""The file that holds the data section, the keys to it there, and the catalog.
|
|
109
|
+
|
|
110
|
+
Inline (under ``default:`` when the file has environments) or in a split file such as
|
|
111
|
+
``settings/data.yml``. Empty keys mean the split file holds the catalog itself.
|
|
112
|
+
"""
|
|
113
|
+
from framework.project import _settings_in
|
|
114
|
+
|
|
115
|
+
settings_file = _settings_in(root)
|
|
116
|
+
if settings_file is None:
|
|
117
|
+
raise FrameworkError(f"Not a Framework project (no settings.yml): {root}")
|
|
118
|
+
raw = _yaml.read(settings_file) or {}
|
|
119
|
+
raw = raw if isinstance(raw, dict) else {}
|
|
120
|
+
environments = isinstance(raw.get("default"), dict)
|
|
121
|
+
config = raw["default"] if environments else raw
|
|
122
|
+
source = config.get("data")
|
|
123
|
+
if isinstance(source, str):
|
|
124
|
+
path = Path(source).expanduser()
|
|
125
|
+
file = path if path.is_absolute() else root / path
|
|
126
|
+
content = _yaml.read(file) if file.is_file() else None
|
|
127
|
+
if (
|
|
128
|
+
isinstance(content, dict)
|
|
129
|
+
and isinstance(content.get("default"), dict)
|
|
130
|
+
and ("data" in content["default"])
|
|
131
|
+
):
|
|
132
|
+
return file, ["default", "data"], content["default"]["data"]
|
|
133
|
+
if isinstance(content, dict) and "data" in content:
|
|
134
|
+
return file, ["data"], content["data"]
|
|
135
|
+
if isinstance(content, dict) and content:
|
|
136
|
+
return file, [], content
|
|
137
|
+
return file, ["data"], {}
|
|
138
|
+
keys = ["default", "data"] if environments else ["data"]
|
|
139
|
+
return settings_file, keys, source if isinstance(source, dict) else {}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _local(root: Path) -> dict[str, dict[str, Any]]:
|
|
143
|
+
"""The project's entries as registry rows, by name."""
|
|
144
|
+
try:
|
|
145
|
+
catalog = _location(root)[2]
|
|
146
|
+
except Exception:
|
|
147
|
+
return {}
|
|
148
|
+
return {name: _row(name, entry) for name, entry in _entries(catalog).items()}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _write(root: Path, rows: list[dict[str, Any]]) -> None:
|
|
152
|
+
"""Write Cloud's rows into the settings' data section, each where its entry already is
|
|
153
|
+
(nested, mixed, or flat keys), else nested by name. Fields that don't travel stay. The
|
|
154
|
+
file is edited in place, so its comments and layout are kept."""
|
|
155
|
+
file, keys, _ = _location(root)
|
|
156
|
+
|
|
157
|
+
def apply(catalog: Any) -> None:
|
|
158
|
+
for row in rows:
|
|
159
|
+
found = catalog_locate(row["name"], catalog)
|
|
160
|
+
if found is not None:
|
|
161
|
+
holder, key = found
|
|
162
|
+
else:
|
|
163
|
+
holder = catalog
|
|
164
|
+
*groups, key = row["name"].split(".")
|
|
165
|
+
for part in groups:
|
|
166
|
+
if not isinstance(holder.get(part), dict):
|
|
167
|
+
holder[part] = {}
|
|
168
|
+
holder = holder[part]
|
|
169
|
+
before = holder.get(key)
|
|
170
|
+
if not isinstance(before, dict):
|
|
171
|
+
holder[key] = before = {}
|
|
172
|
+
for field in DATA_FIELDS:
|
|
173
|
+
value = row.get(field)
|
|
174
|
+
if field == "locked" and not value:
|
|
175
|
+
value = None
|
|
176
|
+
if value is None:
|
|
177
|
+
before.pop(field, None)
|
|
178
|
+
else:
|
|
179
|
+
before[field] = value
|
|
180
|
+
|
|
181
|
+
def change(content: Any) -> Any:
|
|
182
|
+
content = content if isinstance(content, dict) else {}
|
|
183
|
+
if not keys:
|
|
184
|
+
apply(content)
|
|
185
|
+
return {"data": content}
|
|
186
|
+
node = content
|
|
187
|
+
for key in keys[:-1]:
|
|
188
|
+
if not isinstance(node.get(key), dict):
|
|
189
|
+
node[key] = {}
|
|
190
|
+
node = node[key]
|
|
191
|
+
if not isinstance(node.get(keys[-1]), dict):
|
|
192
|
+
node[keys[-1]] = {}
|
|
193
|
+
apply(node[keys[-1]])
|
|
194
|
+
return content
|
|
195
|
+
|
|
196
|
+
if file.is_file():
|
|
197
|
+
_yaml.update(file, change)
|
|
198
|
+
else:
|
|
199
|
+
_yaml.write(file, change({}))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
# The lock and the connection -------------------------------------------------------------
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _lock_read(root: Path) -> dict[str, Any]:
|
|
206
|
+
try:
|
|
207
|
+
data = json.loads((root / _LOCK).read_text(encoding="utf-8"))
|
|
208
|
+
except (OSError, ValueError):
|
|
209
|
+
return {}
|
|
210
|
+
return data if isinstance(data, dict) else {}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _lock_write(root: Path, revision: str, server: str) -> None:
|
|
214
|
+
path = root / _LOCK
|
|
215
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
216
|
+
data = {
|
|
217
|
+
"schema": 1,
|
|
218
|
+
"revision": revision,
|
|
219
|
+
"server": server,
|
|
220
|
+
"updated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
221
|
+
}
|
|
222
|
+
path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _connection(root: Path) -> tuple[str, str]:
|
|
226
|
+
from framework.cloud import CloudError, cloud_server, project_token
|
|
227
|
+
|
|
228
|
+
token = project_token(root)
|
|
229
|
+
if token is None:
|
|
230
|
+
raise CloudError(
|
|
231
|
+
"Framework Cloud not reached (no project token in FW_PROJECT_TOKEN or the "
|
|
232
|
+
"project's .env). Nothing changed."
|
|
233
|
+
)
|
|
234
|
+
return cloud_server(scope="project", project_dir=root).url, token
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _call(
|
|
238
|
+
token: str, server: str, method: str = "GET", body: dict[str, Any] | None = None
|
|
239
|
+
) -> dict[str, Any]:
|
|
240
|
+
"""The registry endpoint, with its failures as data registry messages. A 409 is
|
|
241
|
+
raised as it is, for push to explain."""
|
|
242
|
+
from framework.cloud import (
|
|
243
|
+
CloudConflictError,
|
|
244
|
+
CloudError,
|
|
245
|
+
CloudOfflineError,
|
|
246
|
+
_MissingEndpointError,
|
|
247
|
+
api,
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
try:
|
|
251
|
+
reply = api(_PATH, token, server, method, body)
|
|
252
|
+
except CloudConflictError:
|
|
253
|
+
raise
|
|
254
|
+
except _MissingEndpointError as error:
|
|
255
|
+
raise CloudError(
|
|
256
|
+
"This Framework Cloud server has no data registry endpoint yet. Nothing changed."
|
|
257
|
+
) from error
|
|
258
|
+
except CloudError as error:
|
|
259
|
+
reason = str(error).rstrip(".")
|
|
260
|
+
if isinstance(error, CloudOfflineError) and error.status is None:
|
|
261
|
+
reason = reason or type(error).__name__
|
|
262
|
+
raise CloudError(f"Framework Cloud not reached ({reason}). Nothing changed.") from error
|
|
263
|
+
if error.status in (401, 403):
|
|
264
|
+
# A revoked or mistyped project token
|
|
265
|
+
raise CloudError(
|
|
266
|
+
f"Framework Cloud refused the project token ({reason or 'not accepted'}). "
|
|
267
|
+
"Check FW_PROJECT_TOKEN in the project's .env. Nothing changed."
|
|
268
|
+
) from error
|
|
269
|
+
if error.status == 422:
|
|
270
|
+
raise CloudError(
|
|
271
|
+
f"{reason or 'Framework Cloud refused the data registry'}. Nothing changed."
|
|
272
|
+
) from error
|
|
273
|
+
# Anything else is reported as Cloud answered it
|
|
274
|
+
status = f"HTTP {error.status}" if error.status else "error"
|
|
275
|
+
raise CloudError(
|
|
276
|
+
f"Framework Cloud answered with an error ({status}): "
|
|
277
|
+
f"{reason or 'no message'}. Nothing changed."
|
|
278
|
+
) from error
|
|
279
|
+
return reply if isinstance(reply, dict) else {}
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _project_name(root: Path) -> str:
|
|
283
|
+
from framework.setup_sync import _project_name as name
|
|
284
|
+
|
|
285
|
+
return name(root)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
# Pull and push -----------------------------------------------------------------------------
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def pull_data_registry(
|
|
292
|
+
project_path: str | Path = ".",
|
|
293
|
+
overwrite: bool = False,
|
|
294
|
+
confirm: bool | None = None,
|
|
295
|
+
dry_run: bool = False,
|
|
296
|
+
) -> dict[str, Any]:
|
|
297
|
+
"""Bring the project's data catalog up to date with its data registry on Framework Cloud.
|
|
298
|
+
|
|
299
|
+
Reads the registry with the project token (``GET /api/v1/projects/data``) and lists the
|
|
300
|
+
changes first: entries to add, entries whose path, type, or options differ, and entries
|
|
301
|
+
only in this project. Then asks "Apply these changes?".
|
|
302
|
+
|
|
303
|
+
Only the definitions move. Data files are never downloaded, and file hashes stay in the
|
|
304
|
+
project's framework.db. Applying writes the new entries into the settings' data section
|
|
305
|
+
(inline, or in its split file such as ``settings/data.yml``), keeping the file's comments.
|
|
306
|
+
Local entries are never removed, and an entry whose local definition differs keeps it
|
|
307
|
+
unless ``overwrite=True``. Fields Framework Cloud doesn't keep stay as they are. The
|
|
308
|
+
revision pulled is kept in ``.framework/data.lock.json`` for ``push_data_registry()``.
|
|
309
|
+
|
|
310
|
+
Args:
|
|
311
|
+
project_path: The project's root directory. Default: the current directory.
|
|
312
|
+
overwrite: Use Cloud's definitions for entries that differ.
|
|
313
|
+
confirm: True applies without asking; False cancels. None asks in a console.
|
|
314
|
+
dry_run: List the changes and write nothing.
|
|
315
|
+
|
|
316
|
+
Returns:
|
|
317
|
+
A dict: ``revision``, ``entries`` (``added``, ``changed``, ``differ_kept``, and
|
|
318
|
+
``kept_local``, as names), ``applied``, ``dry_run``, and ``message``.
|
|
319
|
+
|
|
320
|
+
Raises:
|
|
321
|
+
CloudError: Without a project token, offline, when the token is refused, or when
|
|
322
|
+
the server has no data registry endpoint. Nothing changes then.
|
|
323
|
+
FrameworkError: Without a console and without ``confirm=True`` or ``dry_run=True``.
|
|
324
|
+
"""
|
|
325
|
+
from framework._log import inform, show_messages
|
|
326
|
+
from framework.setup_sync import _ask
|
|
327
|
+
from framework.skill_pull import _root
|
|
328
|
+
|
|
329
|
+
show_messages()
|
|
330
|
+
root = _root(project_path)
|
|
331
|
+
server, token = _connection(root)
|
|
332
|
+
registry = _call(token, server)
|
|
333
|
+
revision = str(registry.get("revision") or "")
|
|
334
|
+
|
|
335
|
+
local = _local(root)
|
|
336
|
+
cloud = {
|
|
337
|
+
str(r["name"]): r
|
|
338
|
+
for r in registry.get("entries") or []
|
|
339
|
+
if isinstance(r, dict) and r.get("name")
|
|
340
|
+
}
|
|
341
|
+
added = [cloud[n] for n in cloud if n not in local]
|
|
342
|
+
differ = [n for n in cloud if n in local and not _same(local[n], cloud[n])]
|
|
343
|
+
kept_local = [n for n in local if n not in cloud]
|
|
344
|
+
|
|
345
|
+
lines = [f"Data registry changes from Framework Cloud (revision {revision}):"]
|
|
346
|
+
if added:
|
|
347
|
+
lines.append(f" Entries to add: {_names(added)}")
|
|
348
|
+
if differ:
|
|
349
|
+
details = "; ".join(f"{n} ({_diff(local[n], cloud[n])})" for n in differ)
|
|
350
|
+
tail = (
|
|
351
|
+
"Cloud's will be used"
|
|
352
|
+
if overwrite
|
|
353
|
+
else "kept; pull_data_registry(overwrite=True) uses Cloud's"
|
|
354
|
+
)
|
|
355
|
+
lines.append(f" Entries that differ (local -> Cloud): {details} — {tail}")
|
|
356
|
+
if kept_local:
|
|
357
|
+
lines.append(f" Only in this project (kept): {', '.join(kept_local)}")
|
|
358
|
+
changes = bool(added or (differ and overwrite))
|
|
359
|
+
if not added and not differ and not kept_local:
|
|
360
|
+
lines = [f"Data registry is up to date with Framework Cloud (revision {revision})."]
|
|
361
|
+
for line in lines:
|
|
362
|
+
inform(line)
|
|
363
|
+
|
|
364
|
+
result: dict[str, Any] = {
|
|
365
|
+
"revision": revision,
|
|
366
|
+
"entries": {
|
|
367
|
+
"added": [str(r["name"]) for r in added],
|
|
368
|
+
"changed": differ if overwrite else [],
|
|
369
|
+
"differ_kept": [] if overwrite else differ,
|
|
370
|
+
"kept_local": kept_local,
|
|
371
|
+
},
|
|
372
|
+
"applied": False,
|
|
373
|
+
"dry_run": dry_run,
|
|
374
|
+
"message": "\n".join(lines),
|
|
375
|
+
}
|
|
376
|
+
if dry_run:
|
|
377
|
+
return result
|
|
378
|
+
if not changes:
|
|
379
|
+
_lock_write(root, revision, server)
|
|
380
|
+
return result
|
|
381
|
+
if not _ask("Apply these changes?", confirm, "pull_data_registry"):
|
|
382
|
+
inform("Nothing changed.")
|
|
383
|
+
return result
|
|
384
|
+
|
|
385
|
+
rows = added + ([cloud[n] for n in differ] if overwrite else [])
|
|
386
|
+
_write(root, rows)
|
|
387
|
+
_lock_write(root, revision, server)
|
|
388
|
+
inform(
|
|
389
|
+
f"✓ Data registry: {len(rows)} {'entry' if len(rows) == 1 else 'entries'} added or updated"
|
|
390
|
+
)
|
|
391
|
+
result["applied"] = True
|
|
392
|
+
return result
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def push_data_registry(
|
|
396
|
+
project_path: str | Path = ".",
|
|
397
|
+
remove: bool = False,
|
|
398
|
+
force: bool = False,
|
|
399
|
+
confirm: bool | None = None,
|
|
400
|
+
dry_run: bool = False,
|
|
401
|
+
) -> dict[str, Any]:
|
|
402
|
+
"""Send the project's data catalog to its data registry on Framework Cloud.
|
|
403
|
+
|
|
404
|
+
The entries go to ``POST /api/v1/projects/data``, first as a dry run whose changes are
|
|
405
|
+
listed with a warning: new copies of the project, zips, and Update on GitHub use what is
|
|
406
|
+
pushed. Then asks "Push these changes?". Entries only on Cloud stay there unless
|
|
407
|
+
``remove=True``.
|
|
408
|
+
|
|
409
|
+
Only the definitions move: each entry's name, path, type, delimiter, locked and private
|
|
410
|
+
flags, description, and Stata names. Data files are never uploaded, and file hashes stay
|
|
411
|
+
in the project's framework.db.
|
|
412
|
+
|
|
413
|
+
When Cloud's registry changed since this project last pulled it (the revision in
|
|
414
|
+
``.framework/data.lock.json``), the push is refused with "Pull first"; ``force=True``
|
|
415
|
+
replaces Cloud's newer changes. A push resolves the project again from its blueprint's
|
|
416
|
+
latest version.
|
|
417
|
+
|
|
418
|
+
Args:
|
|
419
|
+
project_path: The project's root directory. Default: the current directory.
|
|
420
|
+
remove: Remove entries from Cloud's registry that this project doesn't have.
|
|
421
|
+
force: Push even when Cloud's registry changed since the last pull.
|
|
422
|
+
confirm: True pushes without asking; False cancels. None asks in a console.
|
|
423
|
+
dry_run: List the changes and send nothing.
|
|
424
|
+
|
|
425
|
+
Returns:
|
|
426
|
+
A dict: ``revision``, ``changes`` (Cloud's ``added``, ``changed``, ``removed``, and
|
|
427
|
+
``kept``), ``applied``, ``dry_run``, and ``message``.
|
|
428
|
+
|
|
429
|
+
Raises:
|
|
430
|
+
CloudError: Without a project token, offline, when the server has no data registry
|
|
431
|
+
endpoint, or when Cloud's registry changed and ``force`` is False.
|
|
432
|
+
FrameworkError: Without a console and without ``confirm=True`` or ``dry_run=True``.
|
|
433
|
+
"""
|
|
434
|
+
from framework._log import inform, show_messages
|
|
435
|
+
from framework.cloud import CloudConflictError, CloudError
|
|
436
|
+
from framework.setup_sync import _ask
|
|
437
|
+
from framework.skill_pull import _root
|
|
438
|
+
|
|
439
|
+
show_messages()
|
|
440
|
+
root = _root(project_path)
|
|
441
|
+
server, token = _connection(root)
|
|
442
|
+
local = _local(root)
|
|
443
|
+
# The base revision for a project that never pulled its registry
|
|
444
|
+
base = _lock_read(root).get("revision")
|
|
445
|
+
if base is None:
|
|
446
|
+
base = _call(token, server).get("revision")
|
|
447
|
+
|
|
448
|
+
def send(dry: bool) -> dict[str, Any]:
|
|
449
|
+
body = {
|
|
450
|
+
"base_revision": base,
|
|
451
|
+
"entries": list(local.values()),
|
|
452
|
+
"remove": remove,
|
|
453
|
+
"force": force,
|
|
454
|
+
"dry_run": dry,
|
|
455
|
+
}
|
|
456
|
+
try:
|
|
457
|
+
return _call(token, server, "POST", body)
|
|
458
|
+
except CloudConflictError as error:
|
|
459
|
+
latest = error.body.get("revision") or "unknown"
|
|
460
|
+
raise CloudError(
|
|
461
|
+
"Framework Cloud's data registry changed since this project last pulled it "
|
|
462
|
+
f"(revision {latest}). Pull first: pull_data_registry(). "
|
|
463
|
+
"push_data_registry(force=True) replaces Cloud's changes."
|
|
464
|
+
) from error
|
|
465
|
+
|
|
466
|
+
preview = send(True)
|
|
467
|
+
changes = preview.get("changes") or {}
|
|
468
|
+
added = changes.get("added") or []
|
|
469
|
+
changed = changes.get("changed") or []
|
|
470
|
+
removed = changes.get("removed") or []
|
|
471
|
+
kept = changes.get("kept") or []
|
|
472
|
+
|
|
473
|
+
lines = ["Data registry changes for Framework Cloud:"]
|
|
474
|
+
if added:
|
|
475
|
+
lines.append(f" Entries to add: {_names(added)}")
|
|
476
|
+
if changed:
|
|
477
|
+
details = "; ".join(f"{c['name']} ({_diff(c['from'], c['to'])})" for c in changed)
|
|
478
|
+
lines.append(f" Entries to change: {details}")
|
|
479
|
+
if removed:
|
|
480
|
+
lines.append(f" Entries to remove: {_names(removed)}")
|
|
481
|
+
if kept:
|
|
482
|
+
lines.append(f" On Cloud only (kept): {_names(kept)}")
|
|
483
|
+
pending = bool(added or changed or removed)
|
|
484
|
+
if not pending:
|
|
485
|
+
lines[0] = "Data registry on Framework Cloud already matches this project."
|
|
486
|
+
else:
|
|
487
|
+
lines.append(
|
|
488
|
+
f"This changes {_project_name(root)}'s data registry on Framework Cloud. "
|
|
489
|
+
"New copies of the project, zips, and Update on GitHub will use it. "
|
|
490
|
+
"Only definitions are sent: no data files and no hashes."
|
|
491
|
+
)
|
|
492
|
+
if force and preview.get("revision") is not None and base != preview.get("revision"):
|
|
493
|
+
lines.append(
|
|
494
|
+
"Cloud's newer data registry changes will be replaced by this project's entries."
|
|
495
|
+
)
|
|
496
|
+
for line in lines:
|
|
497
|
+
inform(line)
|
|
498
|
+
result: dict[str, Any] = {
|
|
499
|
+
"revision": preview.get("revision"),
|
|
500
|
+
"changes": {"added": added, "changed": changed, "removed": removed, "kept": kept},
|
|
501
|
+
"applied": False,
|
|
502
|
+
"dry_run": dry_run,
|
|
503
|
+
"message": "\n".join(lines),
|
|
504
|
+
}
|
|
505
|
+
if not pending or dry_run:
|
|
506
|
+
return result
|
|
507
|
+
|
|
508
|
+
if not _ask("Push these changes?", confirm, "push_data_registry"):
|
|
509
|
+
inform("Data registry not pushed.")
|
|
510
|
+
return result
|
|
511
|
+
reply = send(False)
|
|
512
|
+
revision = str(reply.get("revision") or "")
|
|
513
|
+
result.update(applied=bool(reply.get("applied")), revision=revision)
|
|
514
|
+
_lock_write(root, revision, server)
|
|
515
|
+
done = reply.get("changes") or changes
|
|
516
|
+
if reply.get("applied"):
|
|
517
|
+
parts = [
|
|
518
|
+
f"{len(done.get(key) or [])} {key}"
|
|
519
|
+
for key in ("added", "changed", "removed")
|
|
520
|
+
if done.get(key)
|
|
521
|
+
]
|
|
522
|
+
inform(f"Data registry pushed: revision {revision} ({', '.join(parts)})")
|
|
523
|
+
else:
|
|
524
|
+
inform("Already up to date.")
|
|
525
|
+
return result
|
framework/hooks.py
CHANGED
|
@@ -132,8 +132,13 @@ def _hook_name(name: str) -> str:
|
|
|
132
132
|
|
|
133
133
|
|
|
134
134
|
def _git(root: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
|
135
|
+
# git writes paths as UTF-8, whatever the locale (cp1252 on Windows)
|
|
135
136
|
return subprocess.run(
|
|
136
|
-
["git", "-C", str(root), *args],
|
|
137
|
+
["git", "-C", str(root), *args],
|
|
138
|
+
capture_output=True,
|
|
139
|
+
encoding="utf-8",
|
|
140
|
+
errors="replace",
|
|
141
|
+
check=False,
|
|
137
142
|
)
|
|
138
143
|
|
|
139
144
|
|
framework/scaffold.py
CHANGED
|
@@ -268,7 +268,9 @@ def _create_private_folders(root: Path, config: dict[str, Any]) -> list[str]:
|
|
|
268
268
|
def _git(*args: str) -> str | None:
|
|
269
269
|
if shutil.which("git") is None:
|
|
270
270
|
return None
|
|
271
|
-
result = subprocess.run(
|
|
271
|
+
result = subprocess.run(
|
|
272
|
+
["git", *args], capture_output=True, encoding="utf-8", errors="replace", check=False
|
|
273
|
+
)
|
|
272
274
|
return result.stdout.strip() if result.returncode == 0 else None
|
|
273
275
|
|
|
274
276
|
|
framework/spec/check.py
CHANGED
|
@@ -29,7 +29,17 @@ REQUIRED = (
|
|
|
29
29
|
"files",
|
|
30
30
|
"skills",
|
|
31
31
|
)
|
|
32
|
-
OPTIONAL = ("quarto", "source", "tools", "core_files", "settings_layout", "publish")
|
|
32
|
+
OPTIONAL = ("quarto", "source", "tools", "core_files", "settings_layout", "publish", "data")
|
|
33
|
+
|
|
34
|
+
# The data registry: the fields a catalog entry carries in a spec and to and from
|
|
35
|
+
# Framework Cloud, the types data_read() reads (with R's aliases), and its names.
|
|
36
|
+
DATA_FIELDS = ("path", "type", "delimiter", "locked", "private", "description", "stata_names")
|
|
37
|
+
DATA_TYPES = (
|
|
38
|
+
"csv", "tsv", "rds", "excel", "xlsx", "xls", "parquet", "stata", "dta",
|
|
39
|
+
"spss", "sav", "zsav", "spss_por", "por", "sas", "sas7bdat", "sas7bcat", "sas_xpt", "xpt",
|
|
40
|
+
) # fmt: skip
|
|
41
|
+
DATA_DELIMITERS = ("comma", "tab", "semicolon", "space")
|
|
42
|
+
_DATA_NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_-]*([.][A-Za-z0-9_][A-Za-z0-9_-]*)*$")
|
|
33
43
|
|
|
34
44
|
# Assistant -> (skills folder, stub file). Adding an assistant is one row.
|
|
35
45
|
ASSISTANTS: dict[str, tuple[str, str | None]] = {
|
|
@@ -267,8 +277,72 @@ def spec_check(spec: dict[str, Any]) -> list[str]:
|
|
|
267
277
|
add(f"env: invalid variable name '{name}'")
|
|
268
278
|
elif _SECRET_NAME.search(name) and value not in (None, ""):
|
|
269
279
|
add(f"env.{name}: secrets must not be stored in the spec; leave the value empty")
|
|
280
|
+
problems.extend(data_registry_problems(spec.get("data")))
|
|
270
281
|
for key, hook in _dict(spec.get("data_hooks")).items():
|
|
271
282
|
if not isinstance(hook, str) or not _HOOK.match(hook):
|
|
272
283
|
add(f"data_hooks.{key}: expected 'path/to/file.py::function'")
|
|
273
284
|
|
|
274
285
|
return problems
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def data_registry_problems(data: Any) -> list[str]:
|
|
289
|
+
"""Problems with a spec's ``data`` section (catalog name -> entry), as R's
|
|
290
|
+
spec_check() and the schema's data_entry see them."""
|
|
291
|
+
if data is None or data == {} or data == []:
|
|
292
|
+
return []
|
|
293
|
+
if not isinstance(data, dict):
|
|
294
|
+
return ["data must be an object keyed by catalog name, not a list"]
|
|
295
|
+
problems: list[str] = []
|
|
296
|
+
add = problems.append
|
|
297
|
+
for name, entry in data.items():
|
|
298
|
+
where = f"data.{name}"
|
|
299
|
+
if not _DATA_NAME.match(str(name)):
|
|
300
|
+
add(f"data: invalid catalog name '{name}'")
|
|
301
|
+
if not isinstance(entry, dict):
|
|
302
|
+
add(f"{where}: must be an entry with a path")
|
|
303
|
+
continue
|
|
304
|
+
for field in entry:
|
|
305
|
+
if field not in DATA_FIELDS:
|
|
306
|
+
add(f"{where}: unknown field '{field}'")
|
|
307
|
+
path = entry.get("path")
|
|
308
|
+
if not isinstance(path, str) or not path:
|
|
309
|
+
add(f"{where}: missing path")
|
|
310
|
+
elif not is_safe_path(path):
|
|
311
|
+
add(f"{where}: path '{path}' must be relative and stay inside the project")
|
|
312
|
+
if entry.get("type") is not None and entry["type"] not in DATA_TYPES:
|
|
313
|
+
add(f"{where}: type must be one of {', '.join(DATA_TYPES)}")
|
|
314
|
+
if entry.get("delimiter") is not None and entry["delimiter"] not in DATA_DELIMITERS:
|
|
315
|
+
add(f"{where}: delimiter must be comma, tab, semicolon, or space")
|
|
316
|
+
for flag in ("locked", "private"):
|
|
317
|
+
if entry.get(flag) is not None and not isinstance(entry[flag], bool):
|
|
318
|
+
add(f"{where}.{flag} must be true or false")
|
|
319
|
+
if entry.get("description") is not None and not isinstance(entry["description"], str):
|
|
320
|
+
add(f"{where}.description must be text")
|
|
321
|
+
names = entry.get("stata_names")
|
|
322
|
+
if names and (
|
|
323
|
+
not isinstance(names, dict) or not all(isinstance(v, str) for v in names.values())
|
|
324
|
+
):
|
|
325
|
+
add(f"{where}.stata_names must map column names to Stata names")
|
|
326
|
+
# A name that is also a group of other names can't be written nested
|
|
327
|
+
for name in data:
|
|
328
|
+
inside = [other for other in data if str(other).startswith(f"{name}.")]
|
|
329
|
+
if inside:
|
|
330
|
+
add(f"data: '{name}' is an entry, so '{inside[0]}' can't be inside it")
|
|
331
|
+
return problems
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def catalog_nest(data: Any) -> dict[str, Any]:
|
|
335
|
+
"""A spec's ``data`` section (dot-notation names) as the nested catalog the settings
|
|
336
|
+
hold: ``inputs.raw.survey`` becomes ``inputs: raw: survey:``."""
|
|
337
|
+
catalog: dict[str, Any] = {}
|
|
338
|
+
for name, entry in (data if isinstance(data, dict) else {}).items():
|
|
339
|
+
if entry is None:
|
|
340
|
+
continue
|
|
341
|
+
node = catalog
|
|
342
|
+
*groups, last = str(name).split(".")
|
|
343
|
+
for part in groups:
|
|
344
|
+
if not isinstance(node.get(part), dict):
|
|
345
|
+
node[part] = {}
|
|
346
|
+
node = node[part]
|
|
347
|
+
node[last] = entry
|
|
348
|
+
return catalog
|
framework/spec/merge.py
CHANGED
framework/spec/resolve.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: framework-py
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev18
|
|
4
4
|
Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
|
|
5
5
|
Project-URL: Homepage, https://framework.pub
|
|
6
6
|
Project-URL: Source, https://github.com/frameworktools/framework-py
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
framework/__init__.py,sha256=
|
|
1
|
+
framework/__init__.py,sha256=TFCs67qnMcZIKE-fpw2lYVCS6kAhAQI8y86Tz-6efH0,6710
|
|
2
2
|
framework/__main__.py,sha256=S0ChmRQnNa9W5_MPXhHnH61Kjr_pNthkeS41eJU1w0U,9093
|
|
3
3
|
framework/_agents_md.py,sha256=epVfhohNB2KIl6n3TB4xO4J6MgdSjn8Hxl8QyzioNTo,11517
|
|
4
4
|
framework/_artifacts.py,sha256=cOk2XLqctoqLaQSkksmc7cP4W0_AgqSxYNCTJxyZyik,36068
|
|
@@ -10,7 +10,7 @@ framework/_gitignore.py,sha256=U4bkdGwXzt-IfGtoH2K5-Yg43I51w45rflRxQLcWqPQ,6628
|
|
|
10
10
|
framework/_identifiers.py,sha256=1jarqO2hG9E-GAv8rCKfc-f27NKLJFtAligUj7beEak,5951
|
|
11
11
|
framework/_keychain.py,sha256=9QW8dFDhgy0knWCCu9blttIe_nCG0UhdY77wrVa1h6M,2798
|
|
12
12
|
framework/_log.py,sha256=ZgLaWbjiXK7Cybd_mWGURb4OOGaAAYsB6NkR15W3nnk,1160
|
|
13
|
-
framework/_pages.py,sha256=
|
|
13
|
+
framework/_pages.py,sha256=DFVXarSAYu7V_O3o1hjzYvI5Qa59KqG4GPnDvNTztrA,12561
|
|
14
14
|
framework/_paths.py,sha256=tek8DAVJBkWHcumkabwMTp-APadaTzHe8IgHi2l7xpw,3456
|
|
15
15
|
framework/_private.py,sha256=wc3W6WivkFsfGoR6AJFS4khy17nLzC9zQaPrW_kMVLE,14641
|
|
16
16
|
framework/_secretservice.py,sha256=zv5UvEekQic466qRmVpLG_hfEkisikHfab8e6kyzS2k,2450
|
|
@@ -25,19 +25,20 @@ framework/ai.py,sha256=hqBNXngw7VyQBcSt7kB20Zpa6N_5WS1hNhDi2zWJw0U,9218
|
|
|
25
25
|
framework/bootstrap.py,sha256=mYlD1BG9KaG9PAIHlw3l6677fJYwQdtih95W9Rqmnho,9687
|
|
26
26
|
framework/cache.py,sha256=jJxWIs6kQ7naMenC96pCPtqDCYiCe9c1KSITufV75q4,17160
|
|
27
27
|
framework/catalog.py,sha256=sl4V1judSCVoa2LYLXeQdDhWdsONTC_5s2_fpyPylto,13202
|
|
28
|
-
framework/cloud.py,sha256=
|
|
28
|
+
framework/cloud.py,sha256=U9oy1nrt_puGFPKvnC3CFqifzZEhHKhPxvY4iPtMAIc,64083
|
|
29
29
|
framework/config_home.py,sha256=iSb0VuENgNOSnsZ_v53xFqjRiGgrB7OzZbHbyDOqKzs,8265
|
|
30
30
|
framework/connections.py,sha256=EUkWtuQKW7X-BLJOKXzSbVTSNE6raxjV_GuIELjAOSY,32439
|
|
31
31
|
framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
|
|
32
|
-
framework/create.py,sha256=
|
|
32
|
+
framework/create.py,sha256=XTLUsMu2PpVhBB9k3spSNJZSDzshY-3JOcLJJost7ho,52576
|
|
33
33
|
framework/data.py,sha256=5EwVKFwXABIhbEuf2MsvS8v_yJJ27upG0itSYA6N5mo,31557
|
|
34
|
+
framework/data_registry.py,sha256=NdhJRiTDcP82JxzNq708sfK4f1FHyGE0XF_PcbZwt5M,20422
|
|
34
35
|
framework/db.py,sha256=Wruez2V352xA3gjk5RcUi_0zncFsDwTs9Gwa2hXBY3Y,8099
|
|
35
36
|
framework/display.py,sha256=0FuKUt67uf5h1WjD4bwzoruc2XAN4y_j3mz_f5FskhE,5334
|
|
36
37
|
framework/dotenv.py,sha256=6YQVR8d0CnHXHAYxvP3E4Am4Ei9mzqN2g63w_ft7qa0,4642
|
|
37
38
|
framework/errors.py,sha256=M0bQwnaBH_DE5n5QgHp_0S-SEwMeUxh9Kbx7n2wrVzw,1201
|
|
38
39
|
framework/features.py,sha256=abwC9XCdC0Mm-WFBN-Qw7ztBiFm_mWLqftl7MTNWjc8,7595
|
|
39
40
|
framework/global_config.py,sha256=fYLaxN_6Wc2xrIASn77dWkvGR3uuX-111bvfFPB82A4,11036
|
|
40
|
-
framework/hooks.py,sha256=
|
|
41
|
+
framework/hooks.py,sha256=7FbEWBeMaI0QG7lW1myc3KjVmHd92MqhNjuFoTv1OFQ,21204
|
|
41
42
|
framework/integrity.py,sha256=2hXk95tlNGs6HVTFA0AqcPWQg2AiO6Hy0APMIXHpCTU,586
|
|
42
43
|
framework/notebooks.py,sha256=nBPDgZOSPZzTHaYvvXgyC_P2KXo7Zv471h47cvYOirA,21008
|
|
43
44
|
framework/outputs.py,sha256=8CcKKedgR8rrgIgQP96l5x-geEGOoCnGIW3Ywq19oYc,27200
|
|
@@ -46,7 +47,7 @@ framework/project.py,sha256=1aMzfqLZVkxdImyG3ZQcI-ayfV7Rc3Lrqv1JCsY0GVI,5719
|
|
|
46
47
|
framework/publish_bundle.py,sha256=0vOrLZXSbdV2Fl05smq-2-04Fsts0na3rB_Y0mCOcuY,28802
|
|
47
48
|
framework/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
48
49
|
framework/quarto.py,sha256=FkzXmjd4gngH7CY3g6qs1UZOhU57_TFcn29tc-E6Aqo,17653
|
|
49
|
-
framework/scaffold.py,sha256=
|
|
50
|
+
framework/scaffold.py,sha256=tmNoe2SyGZxY20hTszymUzBo71Ht_cg9F_J8hbjicsY,14750
|
|
50
51
|
framework/scratch.py,sha256=idxthUR5BIMX7FIGS7KzIuhOrO3-HKLbBsV8It3EEE4,3890
|
|
51
52
|
framework/security.py,sha256=-x66bSIIE_xOzkTf5Ojt7xIECKuAIpKVJZ5TtIGrwys,51147
|
|
52
53
|
framework/settings.py,sha256=Sb2YyE53aYLWPj423P-pQVCvRpLsiHydgZh2K0luTQQ,23893
|
|
@@ -66,11 +67,12 @@ framework/_assets/features/targets.json,sha256=zvAJ909ZFydXLUK5xY30yjbn-uKxB7NZ9
|
|
|
66
67
|
framework/_assets/features/uv.json,sha256=TbIuWljGbZdPermU6C3OGbRzXL51iNoolqQ27FgMpRs,316
|
|
67
68
|
framework/_assets/hooks/pre-commit,sha256=R_RZworymj2V5TT0VG8xEEooRvkEm7rRThFI3hCrahk,2548
|
|
68
69
|
framework/_assets/schema/docs-reference.v1.json,sha256=-_nXHPleVjwtls9AheU-CiTlq1q5isIEohqj39Ief2o,5931
|
|
69
|
-
framework/_assets/schema/layer.v1.json,sha256=
|
|
70
|
-
framework/_assets/schema/layers.v1.json,sha256=
|
|
71
|
-
framework/_assets/schema/spec.v1.json,sha256=
|
|
70
|
+
framework/_assets/schema/layer.v1.json,sha256=qi_3Ybl0IGEq8jcx99AO3zJY5EaCqHBel6TtQolme68,19553
|
|
71
|
+
framework/_assets/schema/layers.v1.json,sha256=DDsfetRyQLtTtoxXjh94th6daIu2h-IdOm5FkuYwbd4,26218
|
|
72
|
+
framework/_assets/schema/spec.v1.json,sha256=lfbr0pdFRc-6emp33C2nmvaB-9ZEfoVI9gCxnzpuYkM,19109
|
|
72
73
|
framework/_assets/sets/packages/bayes-r.json,sha256=AoP3IiEg3KKJhZxU6cYEAqweNk9KuzGim9oSERzayzU,1258
|
|
73
74
|
framework/_assets/sets/packages/causal-inference-r.json,sha256=hcqkYaxO3e3OJvKU0AIc1OdSxZNPazHu65Yw_Mted_k,787
|
|
75
|
+
framework/_assets/sets/packages/easystats-r.json,sha256=YhbgOTBZzcl0FwVzZ0_RerMqO-yGS-I54jtz0CblwpY,1288
|
|
74
76
|
framework/_assets/sets/packages/fastapi-python.json,sha256=4aHUj2H09IxUN0umvIIS7TGB1y5OIs_O9H-oQf6V5ek,733
|
|
75
77
|
framework/_assets/sets/packages/large-data-r.json,sha256=mYXBcwMB4omeZRlU6JhmoTd6PQVi_-nThqFKuHZNK8o,633
|
|
76
78
|
framework/_assets/sets/packages/plumber-r.json,sha256=TiqI0ARuTW3bYa8ShzyReDK_wHUVcBRkANw2rIfR0Ls,621
|
|
@@ -101,9 +103,9 @@ framework/_assets/workflows/jupyter.json,sha256=npF0GrTx6fLWRHUxS83Wo8rqMnsYU4iQ
|
|
|
101
103
|
framework/_assets/workflows/quarto.json,sha256=afbSri5LLqtlYS6-RbAQMR-m-H45L5Go7jsSfkfSrwM,364
|
|
102
104
|
framework/_assets/workflows/scripts.json,sha256=-Lep7gb3kws4F3uIgCtaqpUuKJREM3erb-GV-1GDAVs,413
|
|
103
105
|
framework/spec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
104
|
-
framework/spec/check.py,sha256=
|
|
105
|
-
framework/spec/merge.py,sha256=
|
|
106
|
-
framework/spec/resolve.py,sha256=
|
|
106
|
+
framework/spec/check.py,sha256=v4TPrpu_if2QYoyJLn7RLRwvobnxf8nnbFtYTQlJXeg,14463
|
|
107
|
+
framework/spec/merge.py,sha256=gvauBNsFJYe_1J3sO9eF7PSt7mii8ONmnPe8qmyOaGg,6303
|
|
108
|
+
framework/spec/resolve.py,sha256=gyAewGbMcdTbh4feUp6DEj_D9wPBDzRiDC8PTOpyp6Q,12590
|
|
107
109
|
framework/templates/defaults.yml,sha256=TFxs3C4NXG7cGEaA0fbJMMPDFUDAwCcmwubv48-t6fY,3873
|
|
108
110
|
framework/templates/reader.html,sha256=SBw3I3rRMdAyoBosMVuZ9Yp177P11CoxIAaG0OOuvQg,19772
|
|
109
111
|
framework/templates/skill-history.json,sha256=65hjaifEWKR5dQH9OzxQ4k4qz8KHBHPu5BpB4zZpAk8,1350
|
|
@@ -126,7 +128,7 @@ framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIj
|
|
|
126
128
|
framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
|
|
127
129
|
framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
|
|
128
130
|
framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
|
|
129
|
-
framework_py-0.1.0.
|
|
130
|
-
framework_py-0.1.0.
|
|
131
|
-
framework_py-0.1.0.
|
|
132
|
-
framework_py-0.1.0.
|
|
131
|
+
framework_py-0.1.0.dev18.dist-info/METADATA,sha256=gUMUHROaskAq9O742M2RNeiK9G5YimLllH7S7WTxulQ,4281
|
|
132
|
+
framework_py-0.1.0.dev18.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
133
|
+
framework_py-0.1.0.dev18.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
|
|
134
|
+
framework_py-0.1.0.dev18.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|