framework-py 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/__init__.py +250 -0
- framework/__main__.py +89 -0
- framework/_assets/hooks/pre-commit +59 -0
- framework/_assets/init.sql +63 -0
- framework/_assets/schema/layer.v1.json +605 -0
- framework/_assets/schema/layers.v1.json +791 -0
- framework/_assets/schema/spec.v1.json +604 -0
- framework/_log.py +29 -0
- framework/_paths.py +45 -0
- framework/_secretservice.py +69 -0
- framework/_time.py +11 -0
- framework/_wincred.py +105 -0
- framework/_yaml.py +90 -0
- framework/ai.py +204 -0
- framework/bootstrap.py +206 -0
- framework/cache.py +335 -0
- framework/catalog.py +193 -0
- framework/cloud.py +899 -0
- framework/config_home.py +194 -0
- framework/connections.py +442 -0
- framework/console.py +41 -0
- framework/create.py +579 -0
- framework/data.py +519 -0
- framework/db.py +222 -0
- framework/dotenv.py +79 -0
- framework/errors.py +39 -0
- framework/global_config.py +238 -0
- framework/hooks.py +364 -0
- framework/integrity.py +22 -0
- framework/notebooks.py +384 -0
- framework/outputs.py +324 -0
- framework/packages.py +276 -0
- framework/project.py +52 -0
- framework/py.typed +0 -0
- framework/quarto.py +163 -0
- framework/scaffold.py +269 -0
- framework/scratch.py +108 -0
- framework/security.py +646 -0
- framework/settings.py +439 -0
- framework/spec/__init__.py +0 -0
- framework/spec/check.py +211 -0
- framework/spec/merge.py +195 -0
- framework/spec/resolve.py +254 -0
- framework/storage.py +516 -0
- framework/templates/defaults.yml +144 -0
- framework/templates/gitignore/course +120 -0
- framework/templates/gitignore/presentation +114 -0
- framework/templates/gitignore/project +108 -0
- framework/templates/gitignore/sensitive +139 -0
- framework/templates/skills/framework-data/SKILL.md +47 -0
- framework/templates/skills/framework-outputs/SKILL.md +68 -0
- framework/templates/skills/framework-packages/SKILL.md +33 -0
- framework/templates/skills/framework-sensitive-data/SKILL.md +38 -0
- framework/templates/skills/framework-workflow/SKILL.md +58 -0
- framework/templates/stubs/notebook-default.ipynb +34 -0
- framework/templates/stubs/notebook-default.qmd +43 -0
- framework/templates/stubs/notebook-minimal.qmd +15 -0
- framework/templates/stubs/notebook-revealjs.qmd +34 -0
- framework/templates/stubs/script-default.py +18 -0
- framework_py-0.1.0.dev0.dist-info/METADATA +111 -0
- framework_py-0.1.0.dev0.dist-info/RECORD +63 -0
- framework_py-0.1.0.dev0.dist-info/WHEEL +4 -0
- framework_py-0.1.0.dev0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Framework - Course Project
|
|
2
|
+
# ================================================================================
|
|
3
|
+
# Gitignore for course/educational materials with slides, assignments, and modules.
|
|
4
|
+
# ================================================================================
|
|
5
|
+
|
|
6
|
+
# ============================================================================
|
|
7
|
+
# PRIVATE DATA - NEVER COMMIT
|
|
8
|
+
# ============================================================================
|
|
9
|
+
|
|
10
|
+
# Private directories (anywhere in project tree)
|
|
11
|
+
**/private/**
|
|
12
|
+
**/*_private/**
|
|
13
|
+
|
|
14
|
+
# ============================================================================
|
|
15
|
+
# SECRETS AND CREDENTIALS - NEVER COMMIT
|
|
16
|
+
# ============================================================================
|
|
17
|
+
|
|
18
|
+
.env
|
|
19
|
+
.env.local
|
|
20
|
+
.env.*.local
|
|
21
|
+
.Renviron
|
|
22
|
+
.httr-oauth
|
|
23
|
+
**/credentials/**
|
|
24
|
+
**/secrets/**
|
|
25
|
+
*.key
|
|
26
|
+
*.pem
|
|
27
|
+
|
|
28
|
+
# ============================================================================
|
|
29
|
+
# UTILITY & CACHE - SAFE TO IGNORE
|
|
30
|
+
# ============================================================================
|
|
31
|
+
|
|
32
|
+
# Temporary processing directories
|
|
33
|
+
**/cache/**
|
|
34
|
+
**/scratch/**
|
|
35
|
+
**/tmp/**
|
|
36
|
+
**/temp/**
|
|
37
|
+
|
|
38
|
+
# Logs (run logs, cluster job output)
|
|
39
|
+
**/logs/**
|
|
40
|
+
slurm-*.out
|
|
41
|
+
|
|
42
|
+
# Rendered slide artifacts
|
|
43
|
+
/slides/_rendered/
|
|
44
|
+
|
|
45
|
+
# Allow structure documentation
|
|
46
|
+
!/slides/_rendered/**/README.md
|
|
47
|
+
|
|
48
|
+
# ============================================================================
|
|
49
|
+
# FRAMEWORK INTERNALS
|
|
50
|
+
# ============================================================================
|
|
51
|
+
|
|
52
|
+
framework.db
|
|
53
|
+
framework.db-wal
|
|
54
|
+
framework.db-shm
|
|
55
|
+
.initiated
|
|
56
|
+
.framework_scaffolded
|
|
57
|
+
|
|
58
|
+
# ============================================================================
|
|
59
|
+
# QUARTO ARTIFACTS
|
|
60
|
+
# ============================================================================
|
|
61
|
+
|
|
62
|
+
.quarto/
|
|
63
|
+
_freeze/
|
|
64
|
+
*_cache/
|
|
65
|
+
*_files/
|
|
66
|
+
|
|
67
|
+
# ============================================================================
|
|
68
|
+
# PYTHON ENVIRONMENT
|
|
69
|
+
# ============================================================================
|
|
70
|
+
|
|
71
|
+
__pycache__/
|
|
72
|
+
*.py[cod]
|
|
73
|
+
.venv/
|
|
74
|
+
venv/
|
|
75
|
+
.ipynb_checkpoints/
|
|
76
|
+
.pytest_cache/
|
|
77
|
+
.mypy_cache/
|
|
78
|
+
.ruff_cache/
|
|
79
|
+
*.egg-info/
|
|
80
|
+
build/
|
|
81
|
+
dist/
|
|
82
|
+
|
|
83
|
+
# ============================================================================
|
|
84
|
+
# IDE & SYSTEM FILES
|
|
85
|
+
# ============================================================================
|
|
86
|
+
|
|
87
|
+
# IDE - personal settings only (commit workspace settings if desired)
|
|
88
|
+
.idea/
|
|
89
|
+
.vscode/.history/
|
|
90
|
+
.positron/.history/
|
|
91
|
+
.history/
|
|
92
|
+
|
|
93
|
+
# macOS
|
|
94
|
+
.DS_Store
|
|
95
|
+
.AppleDouble
|
|
96
|
+
.LSOverride
|
|
97
|
+
|
|
98
|
+
# Windows
|
|
99
|
+
Thumbs.db
|
|
100
|
+
ehthumbs.db
|
|
101
|
+
|
|
102
|
+
# Linux
|
|
103
|
+
*~
|
|
104
|
+
|
|
105
|
+
# ============================================================================
|
|
106
|
+
# CLOUD/AWS CONFIG
|
|
107
|
+
# ============================================================================
|
|
108
|
+
|
|
109
|
+
.aws/
|
|
110
|
+
.config/
|
|
111
|
+
|
|
112
|
+
# ============================================================================
|
|
113
|
+
# TEMPORARY FILES
|
|
114
|
+
# ============================================================================
|
|
115
|
+
|
|
116
|
+
*.tmp
|
|
117
|
+
*.log
|
|
118
|
+
*.bak
|
|
119
|
+
*.swp
|
|
120
|
+
*~
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# Framework - Presentation Project
|
|
2
|
+
# ================================================================================
|
|
3
|
+
# Gitignore for single presentation projects (Quarto revealjs).
|
|
4
|
+
# ================================================================================
|
|
5
|
+
|
|
6
|
+
# ============================================================================
|
|
7
|
+
# PRIVATE DATA - NEVER COMMIT
|
|
8
|
+
# ============================================================================
|
|
9
|
+
|
|
10
|
+
# Private directories (anywhere in project tree)
|
|
11
|
+
**/private/**
|
|
12
|
+
**/*_private/**
|
|
13
|
+
|
|
14
|
+
# ============================================================================
|
|
15
|
+
# SECRETS AND CREDENTIALS - NEVER COMMIT
|
|
16
|
+
# ============================================================================
|
|
17
|
+
|
|
18
|
+
.env
|
|
19
|
+
.env.local
|
|
20
|
+
.env.*.local
|
|
21
|
+
.Renviron
|
|
22
|
+
.httr-oauth
|
|
23
|
+
**/credentials/**
|
|
24
|
+
**/secrets/**
|
|
25
|
+
*.key
|
|
26
|
+
*.pem
|
|
27
|
+
|
|
28
|
+
# ============================================================================
|
|
29
|
+
# UTILITY & CACHE - SAFE TO IGNORE
|
|
30
|
+
# ============================================================================
|
|
31
|
+
|
|
32
|
+
# Temporary processing directories
|
|
33
|
+
**/cache/**
|
|
34
|
+
**/scratch/**
|
|
35
|
+
**/tmp/**
|
|
36
|
+
**/temp/**
|
|
37
|
+
|
|
38
|
+
# Logs (run logs, cluster job output)
|
|
39
|
+
**/logs/**
|
|
40
|
+
slurm-*.out
|
|
41
|
+
|
|
42
|
+
# ============================================================================
|
|
43
|
+
# FRAMEWORK INTERNALS
|
|
44
|
+
# ============================================================================
|
|
45
|
+
|
|
46
|
+
framework.db
|
|
47
|
+
framework.db-wal
|
|
48
|
+
framework.db-shm
|
|
49
|
+
.initiated
|
|
50
|
+
.framework_scaffolded
|
|
51
|
+
|
|
52
|
+
# ============================================================================
|
|
53
|
+
# QUARTO ARTIFACTS
|
|
54
|
+
# ============================================================================
|
|
55
|
+
|
|
56
|
+
.quarto/
|
|
57
|
+
_freeze/
|
|
58
|
+
*_cache/
|
|
59
|
+
*_files/
|
|
60
|
+
|
|
61
|
+
# ============================================================================
|
|
62
|
+
# PYTHON ENVIRONMENT
|
|
63
|
+
# ============================================================================
|
|
64
|
+
|
|
65
|
+
__pycache__/
|
|
66
|
+
*.py[cod]
|
|
67
|
+
.venv/
|
|
68
|
+
venv/
|
|
69
|
+
.ipynb_checkpoints/
|
|
70
|
+
.pytest_cache/
|
|
71
|
+
.mypy_cache/
|
|
72
|
+
.ruff_cache/
|
|
73
|
+
*.egg-info/
|
|
74
|
+
build/
|
|
75
|
+
dist/
|
|
76
|
+
|
|
77
|
+
# ============================================================================
|
|
78
|
+
# IDE & SYSTEM FILES
|
|
79
|
+
# ============================================================================
|
|
80
|
+
|
|
81
|
+
# IDE - personal settings only (commit workspace settings if desired)
|
|
82
|
+
.idea/
|
|
83
|
+
.vscode/.history/
|
|
84
|
+
.positron/.history/
|
|
85
|
+
.history/
|
|
86
|
+
|
|
87
|
+
# macOS
|
|
88
|
+
.DS_Store
|
|
89
|
+
.AppleDouble
|
|
90
|
+
.LSOverride
|
|
91
|
+
|
|
92
|
+
# Windows
|
|
93
|
+
Thumbs.db
|
|
94
|
+
ehthumbs.db
|
|
95
|
+
|
|
96
|
+
# Linux
|
|
97
|
+
*~
|
|
98
|
+
|
|
99
|
+
# ============================================================================
|
|
100
|
+
# CLOUD/AWS CONFIG
|
|
101
|
+
# ============================================================================
|
|
102
|
+
|
|
103
|
+
.aws/
|
|
104
|
+
.config/
|
|
105
|
+
|
|
106
|
+
# ============================================================================
|
|
107
|
+
# TEMPORARY FILES
|
|
108
|
+
# ============================================================================
|
|
109
|
+
|
|
110
|
+
*.tmp
|
|
111
|
+
*.log
|
|
112
|
+
*.bak
|
|
113
|
+
*.swp
|
|
114
|
+
*~
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# Framework Research Project
|
|
2
|
+
# ================================================================================
|
|
3
|
+
# Pattern-based gitignore - works regardless of directory structure customization.
|
|
4
|
+
# This is the standard template for non-sensitive projects.
|
|
5
|
+
# ================================================================================
|
|
6
|
+
|
|
7
|
+
# ============================================================================
|
|
8
|
+
# PRIVATE DATA - NEVER COMMIT
|
|
9
|
+
# ============================================================================
|
|
10
|
+
|
|
11
|
+
# Private directories (anywhere in project tree)
|
|
12
|
+
**/private/**
|
|
13
|
+
**/*_private/**
|
|
14
|
+
|
|
15
|
+
# ============================================================================
|
|
16
|
+
# SECRETS AND CREDENTIALS - NEVER COMMIT
|
|
17
|
+
# ============================================================================
|
|
18
|
+
|
|
19
|
+
.env
|
|
20
|
+
.env.local
|
|
21
|
+
.env.*.local
|
|
22
|
+
.Renviron
|
|
23
|
+
.httr-oauth
|
|
24
|
+
**/credentials/**
|
|
25
|
+
**/secrets/**
|
|
26
|
+
*.key
|
|
27
|
+
*.pem
|
|
28
|
+
|
|
29
|
+
# ============================================================================
|
|
30
|
+
# UTILITY & CACHE - SAFE TO IGNORE
|
|
31
|
+
# ============================================================================
|
|
32
|
+
|
|
33
|
+
# Temporary processing directories
|
|
34
|
+
**/cache/**
|
|
35
|
+
**/scratch/**
|
|
36
|
+
**/tmp/**
|
|
37
|
+
**/temp/**
|
|
38
|
+
|
|
39
|
+
# Logs (run logs, cluster job output)
|
|
40
|
+
**/logs/**
|
|
41
|
+
slurm-*.out
|
|
42
|
+
|
|
43
|
+
# ============================================================================
|
|
44
|
+
# FRAMEWORK INTERNALS
|
|
45
|
+
# ============================================================================
|
|
46
|
+
|
|
47
|
+
framework.db
|
|
48
|
+
framework.db-wal
|
|
49
|
+
framework.db-shm
|
|
50
|
+
.initiated
|
|
51
|
+
.framework_scaffolded
|
|
52
|
+
|
|
53
|
+
# ============================================================================
|
|
54
|
+
# QUARTO ARTIFACTS
|
|
55
|
+
# ============================================================================
|
|
56
|
+
|
|
57
|
+
.quarto/
|
|
58
|
+
_freeze/
|
|
59
|
+
*_cache/
|
|
60
|
+
*_files/
|
|
61
|
+
|
|
62
|
+
# ============================================================================
|
|
63
|
+
# PYTHON ENVIRONMENT
|
|
64
|
+
# ============================================================================
|
|
65
|
+
|
|
66
|
+
__pycache__/
|
|
67
|
+
*.py[cod]
|
|
68
|
+
.venv/
|
|
69
|
+
venv/
|
|
70
|
+
.ipynb_checkpoints/
|
|
71
|
+
.pytest_cache/
|
|
72
|
+
.mypy_cache/
|
|
73
|
+
.ruff_cache/
|
|
74
|
+
*.egg-info/
|
|
75
|
+
build/
|
|
76
|
+
dist/
|
|
77
|
+
|
|
78
|
+
# ============================================================================
|
|
79
|
+
# IDE & SYSTEM FILES
|
|
80
|
+
# ============================================================================
|
|
81
|
+
|
|
82
|
+
# IDE - personal settings only (commit workspace settings if desired)
|
|
83
|
+
.idea/
|
|
84
|
+
.vscode/.history/
|
|
85
|
+
.positron/.history/
|
|
86
|
+
.history/
|
|
87
|
+
|
|
88
|
+
# macOS
|
|
89
|
+
.DS_Store
|
|
90
|
+
.AppleDouble
|
|
91
|
+
.LSOverride
|
|
92
|
+
|
|
93
|
+
# Windows
|
|
94
|
+
Thumbs.db
|
|
95
|
+
ehthumbs.db
|
|
96
|
+
|
|
97
|
+
# Linux
|
|
98
|
+
*~
|
|
99
|
+
|
|
100
|
+
# ============================================================================
|
|
101
|
+
# TEMPORARY FILES
|
|
102
|
+
# ============================================================================
|
|
103
|
+
|
|
104
|
+
*.tmp
|
|
105
|
+
*.log
|
|
106
|
+
*.bak
|
|
107
|
+
*.swp
|
|
108
|
+
*~
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# Framework (Privacy-First Mode)
|
|
2
|
+
# ================================================================================
|
|
3
|
+
# This project uses Framework's privacy-first structure.
|
|
4
|
+
# All */private/ directories are gitignored by default.
|
|
5
|
+
# Only */public/ directories can be committed after explicit review.
|
|
6
|
+
#
|
|
7
|
+
# WORKFLOW:
|
|
8
|
+
# 1. Sensitive/raw data goes in */private/ directories (NEVER committed)
|
|
9
|
+
# 2. De-identified/aggregated data goes in */public/ directories (safe to commit)
|
|
10
|
+
# 3. Analysis scripts are committed, but outputs are carefully controlled
|
|
11
|
+
#
|
|
12
|
+
# CRITICAL: Private directories are NEVER committed to version control.
|
|
13
|
+
# ================================================================================
|
|
14
|
+
|
|
15
|
+
# ============================================================================
|
|
16
|
+
# SENSITIVE DATA - NEVER COMMIT
|
|
17
|
+
# ============================================================================
|
|
18
|
+
|
|
19
|
+
# Private directories - explicit paths for this project type
|
|
20
|
+
/inputs/private/
|
|
21
|
+
/reference/private/
|
|
22
|
+
/outputs/private/
|
|
23
|
+
|
|
24
|
+
# Generic private directory patterns (defense-in-depth)
|
|
25
|
+
**/private/**
|
|
26
|
+
**/*_private/**
|
|
27
|
+
**/sensitive/**
|
|
28
|
+
|
|
29
|
+
# Allow structure documentation (README and .gitkeep files in private dirs)
|
|
30
|
+
!/inputs/**/README.md
|
|
31
|
+
!/inputs/**/.gitkeep
|
|
32
|
+
!/reference/**/README.md
|
|
33
|
+
!/reference/**/.gitkeep
|
|
34
|
+
!/outputs/**/README.md
|
|
35
|
+
!/outputs/**/.gitkeep
|
|
36
|
+
|
|
37
|
+
# ============================================================================
|
|
38
|
+
# SECRETS AND CREDENTIALS - NEVER COMMIT
|
|
39
|
+
# ============================================================================
|
|
40
|
+
|
|
41
|
+
.env
|
|
42
|
+
.env.local
|
|
43
|
+
.env.*.local
|
|
44
|
+
.Renviron
|
|
45
|
+
.httr-oauth
|
|
46
|
+
**/credentials/**
|
|
47
|
+
**/secrets/**
|
|
48
|
+
*.key
|
|
49
|
+
*.pem
|
|
50
|
+
*.p12
|
|
51
|
+
*.pfx
|
|
52
|
+
|
|
53
|
+
# ============================================================================
|
|
54
|
+
# UTILITY & CACHE - SAFE TO IGNORE
|
|
55
|
+
# ============================================================================
|
|
56
|
+
|
|
57
|
+
# Temporary processing directories
|
|
58
|
+
**/cache/**
|
|
59
|
+
**/scratch/**
|
|
60
|
+
**/tmp/**
|
|
61
|
+
**/temp/**
|
|
62
|
+
|
|
63
|
+
# Logs (run logs, cluster job output)
|
|
64
|
+
**/logs/**
|
|
65
|
+
slurm-*.out
|
|
66
|
+
|
|
67
|
+
# ============================================================================
|
|
68
|
+
# FRAMEWORK INTERNALS
|
|
69
|
+
# ============================================================================
|
|
70
|
+
|
|
71
|
+
framework.db
|
|
72
|
+
framework.db-wal
|
|
73
|
+
framework.db-shm
|
|
74
|
+
.initiated
|
|
75
|
+
.framework_scaffolded
|
|
76
|
+
|
|
77
|
+
# ============================================================================
|
|
78
|
+
# QUARTO ARTIFACTS
|
|
79
|
+
# ============================================================================
|
|
80
|
+
|
|
81
|
+
.quarto/
|
|
82
|
+
_freeze/
|
|
83
|
+
*_cache/
|
|
84
|
+
*_files/
|
|
85
|
+
|
|
86
|
+
# ============================================================================
|
|
87
|
+
# PYTHON ENVIRONMENT
|
|
88
|
+
# ============================================================================
|
|
89
|
+
|
|
90
|
+
__pycache__/
|
|
91
|
+
*.py[cod]
|
|
92
|
+
.venv/
|
|
93
|
+
venv/
|
|
94
|
+
.ipynb_checkpoints/
|
|
95
|
+
.pytest_cache/
|
|
96
|
+
.mypy_cache/
|
|
97
|
+
.ruff_cache/
|
|
98
|
+
*.egg-info/
|
|
99
|
+
build/
|
|
100
|
+
dist/
|
|
101
|
+
|
|
102
|
+
# ============================================================================
|
|
103
|
+
# IDE & SYSTEM FILES
|
|
104
|
+
# ============================================================================
|
|
105
|
+
|
|
106
|
+
# IDE - personal settings only (commit workspace settings if desired)
|
|
107
|
+
.idea/
|
|
108
|
+
.vscode/.history/
|
|
109
|
+
.positron/.history/
|
|
110
|
+
.history/
|
|
111
|
+
|
|
112
|
+
# macOS
|
|
113
|
+
.DS_Store
|
|
114
|
+
.AppleDouble
|
|
115
|
+
.LSOverride
|
|
116
|
+
|
|
117
|
+
# Windows
|
|
118
|
+
Thumbs.db
|
|
119
|
+
ehthumbs.db
|
|
120
|
+
|
|
121
|
+
# Linux
|
|
122
|
+
*~
|
|
123
|
+
|
|
124
|
+
# ============================================================================
|
|
125
|
+
# CLOUD/AWS CONFIG
|
|
126
|
+
# ============================================================================
|
|
127
|
+
|
|
128
|
+
.aws/
|
|
129
|
+
.config/
|
|
130
|
+
|
|
131
|
+
# ============================================================================
|
|
132
|
+
# TEMPORARY FILES
|
|
133
|
+
# ============================================================================
|
|
134
|
+
|
|
135
|
+
*.tmp
|
|
136
|
+
*.log
|
|
137
|
+
*.bak
|
|
138
|
+
*.swp
|
|
139
|
+
*~
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: framework-data
|
|
3
|
+
description: Use whenever reading or writing data files (CSV, Parquet, Excel, Stata, SPSS, SAS) in this Framework project. Read with fw.data_read() and write with fw.data_save(), which record a hash of each file in framework.db.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Framework Data
|
|
7
|
+
|
|
8
|
+
`fw.data_read()` and `fw.data_save()` record a SHA-256 hash of each data file
|
|
9
|
+
in the project's `framework.db`. Reading the file directly with pandas skips
|
|
10
|
+
that record, so a changed or replaced file would go unnoticed.
|
|
11
|
+
|
|
12
|
+
## Reading
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
survey = fw.data_read("inputs.raw.survey") # by catalog name (preferred)
|
|
16
|
+
customers = fw.data_read("inputs/raw/customers.csv") # by path
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
`data_read()` reads CSV, TSV, Parquet, Excel, Stata, SPSS, and SAS files.
|
|
20
|
+
Catalog files are checked against their recorded hash: a locked file that
|
|
21
|
+
changed raises an error, and an unlocked one warns. Use it instead of
|
|
22
|
+
`pd.read_csv()`, `pd.read_parquet()`, `pd.read_excel()`, `pd.read_stata()`,
|
|
23
|
+
or `pyreadstat`.
|
|
24
|
+
|
|
25
|
+
`.rds` files can only be read by R. Ask for a Parquet or CSV copy instead.
|
|
26
|
+
|
|
27
|
+
## Saving
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
fw.data_save(clean, "inputs.intermediate.survey_clean") # Parquet, added to the catalog
|
|
31
|
+
fw.data_save(final, "inputs/final/analysis_ready.csv", locked=True)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
`data_save()` writes Parquet or CSV, which R can read too. Use it instead of
|
|
35
|
+
`DataFrame.to_csv()` or `DataFrame.to_parquet()` for project data.
|
|
36
|
+
|
|
37
|
+
## The data catalog
|
|
38
|
+
|
|
39
|
+
Datasets are listed under `data` in the settings (often `settings/data.yml`)
|
|
40
|
+
and read by dot-notation name. Use the catalog name when an entry exists.
|
|
41
|
+
`fw.data_list()` shows the catalog.
|
|
42
|
+
|
|
43
|
+
## Folders
|
|
44
|
+
|
|
45
|
+
Data moves from raw to intermediate to final. The folders are listed under
|
|
46
|
+
`directories` in the settings. Raw inputs are never changed, and only
|
|
47
|
+
Framework writes go to the intermediate and final folders.
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: framework-outputs
|
|
3
|
+
description: Use when saving analysis outputs, caching expensive computations, running database queries, or publishing files in this Framework project. Covers save_table(), save_figure(), save_model(), cache_remember(), db_query(), and publish().
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Framework Outputs, Caching, Queries, and Publishing
|
|
7
|
+
|
|
8
|
+
## Saving outputs
|
|
9
|
+
|
|
10
|
+
Each `save_*()` function writes to the project's output folder for that kind
|
|
11
|
+
of result (created on first use) and records the file, so `result_list()`
|
|
12
|
+
finds it later.
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
fw.save_table(summary, "quarterly_summary") # outputs/tables/ (csv)
|
|
16
|
+
fw.save_table(report, "annual_report", format="xlsx")
|
|
17
|
+
fw.save_figure(fig, "enrollment_trend") # outputs/figures/ (png)
|
|
18
|
+
fw.save_model(model, "regression_model") # outputs/models/ (pickle)
|
|
19
|
+
fw.save_notebook("notebooks/analysis.qmd") # render with Quarto and save
|
|
20
|
+
|
|
21
|
+
fw.result_list() # everything saved so far
|
|
22
|
+
fw.result_list(type="model") # models only
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
In a sensitive project, outputs are private by default. Pass `public=True`
|
|
26
|
+
only for outputs that are safe to share.
|
|
27
|
+
|
|
28
|
+
## Caching
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
model = fw.cache_remember("fitted_model", lambda: fit(training), expire_after="7 days")
|
|
32
|
+
|
|
33
|
+
fw.cache("processed", frame) # write
|
|
34
|
+
frame = fw.cache_get("processed") # read (None if missing)
|
|
35
|
+
fw.cache_forget("processed") # remove
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Cache entries are tracked in `framework.db` with a hash and an expiry. Use
|
|
39
|
+
names that say what the value is.
|
|
40
|
+
|
|
41
|
+
## Database queries
|
|
42
|
+
|
|
43
|
+
Connections are defined in the settings (often `settings/connections.yml`).
|
|
44
|
+
Use Framework's helpers so credentials come from `.env`:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
users = fw.db_query("SELECT * FROM users WHERE active = 1", "main_db")
|
|
48
|
+
fw.db_execute("UPDATE runs SET done = 1 WHERE id = ?", "main_db", params=(42,))
|
|
49
|
+
|
|
50
|
+
with fw.db_connect("main_db") as conn:
|
|
51
|
+
...
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Publishing
|
|
55
|
+
|
|
56
|
+
`fw.publish()` and `fw.publish_notebook()` upload to the project's storage
|
|
57
|
+
bucket, or to Framework Cloud when the project was set up from it:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
fw.publish("outputs/report.html")
|
|
61
|
+
fw.publish_notebook("notebooks/analysis.qmd")
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The publish functions refuse files inside a `private` folder, because those
|
|
65
|
+
folders hold restricted data. They take `allow_private=True` to override
|
|
66
|
+
this. Never pass `allow_private=True` on your own: only the person who has
|
|
67
|
+
reviewed the file can decide that it is safe to publish. See the
|
|
68
|
+
`framework-sensitive-data` skill if this project has one.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: framework-packages
|
|
3
|
+
description: Use when adding, installing, or managing Python packages in this Framework project. Dependencies are recorded in pyproject.toml; add them with fw.package_add() or uv add instead of a bare pip install.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Framework Packages
|
|
7
|
+
|
|
8
|
+
The project's Python dependencies are listed in `pyproject.toml`, and
|
|
9
|
+
`uv.lock` records the exact versions when the project uses uv. A package
|
|
10
|
+
installed with a bare `pip install` is not recorded, so the next person (or
|
|
11
|
+
the next machine) will not have it.
|
|
12
|
+
|
|
13
|
+
## Adding packages
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
fw.package_add("statsmodels") # records it in pyproject.toml and installs it
|
|
17
|
+
fw.package_add("polars>=1.0") # with a version requirement
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
From a terminal, `uv add statsmodels` does the same.
|
|
21
|
+
|
|
22
|
+
## Checking and restoring
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
fw.packages_list() # what pyproject.toml declares
|
|
26
|
+
fw.packages_status() # missing or out-of-date packages
|
|
27
|
+
fw.packages_install() # install everything (for example after cloning)
|
|
28
|
+
fw.packages_snapshot() # update uv.lock
|
|
29
|
+
fw.packages_restore() # install exactly what uv.lock records
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Import packages normally in each notebook or script. Framework does not
|
|
33
|
+
import them for you.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: framework-sensitive-data
|
|
3
|
+
description: Use before any data operation, commit, output, or publish step in this privacy-sensitive Framework project. Covers private and public folders and the handling of identifiable data.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Sensitive Data
|
|
7
|
+
|
|
8
|
+
This project holds identifiable or restricted data. When it is unclear
|
|
9
|
+
whether something is safe, treat it as private.
|
|
10
|
+
|
|
11
|
+
## Rules
|
|
12
|
+
|
|
13
|
+
1. Files in `inputs/private/` and `outputs/private/` are never committed. They
|
|
14
|
+
can identify people, and the `.gitignore` keeps them out of git.
|
|
15
|
+
2. Raw data with identifiers goes in a `private/` folder.
|
|
16
|
+
3. Only de-identified, aggregated data goes in a `public/` folder.
|
|
17
|
+
4. Review every output before saving it to a public folder.
|
|
18
|
+
5. Never publish anything from a private folder, and never pass
|
|
19
|
+
`allow_private=True` to a publish function on your own.
|
|
20
|
+
6. Do not read or print restricted files. Values you print are sent to the
|
|
21
|
+
model provider. Work from the data dictionary and simulated data instead.
|
|
22
|
+
|
|
23
|
+
## Data flow
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
Raw identifiable data -> inputs/private/raw/
|
|
27
|
+
| clean, de-identify
|
|
28
|
+
Intermediate -> inputs/private/intermediate/
|
|
29
|
+
| aggregate, anonymize
|
|
30
|
+
Public-safe data -> inputs/public/final/
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Writing code
|
|
34
|
+
|
|
35
|
+
- Do not print rows with identifiers in notebook output that will be rendered.
|
|
36
|
+
- Do not put identifiers, names, or record-level values in code or comments.
|
|
37
|
+
- Public tables follow small-cell suppression rules where they apply.
|
|
38
|
+
- Save outputs with `public=False` (the default) until they have been reviewed.
|