dbt-tui 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. dbt_tui-0.1.0.dist-info/METADATA +271 -0
  2. dbt_tui-0.1.0.dist-info/RECORD +49 -0
  3. dbt_tui-0.1.0.dist-info/WHEEL +4 -0
  4. dbt_tui-0.1.0.dist-info/entry_points.txt +4 -0
  5. dbtui/__init__.py +0 -0
  6. dbtui/__main__.py +59 -0
  7. dbtui/backend/__init__.py +2 -0
  8. dbtui/backend/fetch.py +206 -0
  9. dbtui/backend/model.py +86 -0
  10. dbtui/backend/project.py +258 -0
  11. dbtui/backend/property_claim.py +191 -0
  12. dbtui/backend/property_discovery.py +461 -0
  13. dbtui/common/__init__.py +4 -0
  14. dbtui/common/cache.py +70 -0
  15. dbtui/common/exceptions.py +14 -0
  16. dbtui/common/model.py +25 -0
  17. dbtui/common/project.py +32 -0
  18. dbtui/frontend/__init__.py +1 -0
  19. dbtui/frontend/common/__init__.py +4 -0
  20. dbtui/frontend/common/dbtui_screen.py +50 -0
  21. dbtui/frontend/common/isolated.py +14 -0
  22. dbtui/frontend/common/model_list.py +41 -0
  23. dbtui/frontend/common/model_list_item.py +20 -0
  24. dbtui/frontend/lorem.txt +9 -0
  25. dbtui/frontend/main.py +138 -0
  26. dbtui/frontend/model_search/__init__.py +1 -0
  27. dbtui/frontend/model_search/model_search.py +52 -0
  28. dbtui/frontend/model_search/model_search_input.py +21 -0
  29. dbtui/frontend/model_search/model_search_list.py +14 -0
  30. dbtui/frontend/model_tree/__init__.py +1 -0
  31. dbtui/frontend/model_tree/children_list.py +28 -0
  32. dbtui/frontend/model_tree/constants.py +4 -0
  33. dbtui/frontend/model_tree/model_relatives_list.py +15 -0
  34. dbtui/frontend/model_tree/model_tree.py +98 -0
  35. dbtui/frontend/model_tree/parents_list.py +28 -0
  36. dbtui/frontend/model_view/__init__.py +2 -0
  37. dbtui/frontend/model_view/model_tree.py +162 -0
  38. dbtui/frontend/model_view/properties_panel.py +172 -0
  39. dbtui/frontend/new_model/__init__.py +1 -0
  40. dbtui/frontend/new_model/new_model.py +51 -0
  41. dbtui/frontend/new_model/select_filepath.py +30 -0
  42. dbtui/frontend/options/__init__.py +1 -0
  43. dbtui/frontend/options/options.py +37 -0
  44. dbtui/frontend/project_search/__init__.py +1 -0
  45. dbtui/frontend/project_search/project_search.py +109 -0
  46. dbtui/frontend/pseudo/__init__.py +2 -0
  47. dbtui/frontend/pseudo/dbt_model.py +50 -0
  48. dbtui/frontend/pseudo/dbt_project.py +37 -0
  49. dbtui/frontend/pseudo/utils.py +8 -0
@@ -0,0 +1,271 @@
1
+ Metadata-Version: 2.4
2
+ Name: dbt-tui
3
+ Version: 0.1.0
4
+ Summary: Terminal UI for exploring and managing dbt projects
5
+ License: MIT
6
+ Keywords: dbt,tui,terminal,data,analytics,sql
7
+ Author: Mike
8
+ Author-email: sortia@protonmail.com
9
+ Requires-Python: >=3.12, <4.0
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Topic :: Database
18
+ Classifier: Topic :: Software Development :: User Interfaces
19
+ Requires-Dist: jinja2 (>=3.0)
20
+ Requires-Dist: networkx (>=3.0)
21
+ Requires-Dist: platformdirs (>=4.5.0,<5.0.0)
22
+ Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
23
+ Requires-Dist: textual[syntax] (>=6.5.0,<7.0.0)
24
+ Project-URL: Homepage, https://github.com/sortia/dbt-tui
25
+ Project-URL: Repository, https://github.com/sortia/dbt-tui
26
+ Description-Content-Type: text/markdown
27
+
28
+ # dbtui
29
+
30
+ A Terminal User Interface (TUI) for navigating and managing [dbt (data build tool)](https://www.getdbt.com/) projects.
31
+
32
+ **dbtui** provides an interactive command-line interface for exploring dbt models, their relationships, and configurations without leaving your terminal.
33
+
34
+ ## Features
35
+
36
+ - **Model Explorer**: Browse all models in your dbt project with search capabilities
37
+ - **Dependency Visualization**: View parent and child relationships for any model
38
+ - **Model Preview**: Read model SQL and configuration inline
39
+ - **Quick Navigation**: Jump between related models with keyboard shortcuts
40
+ - **New Model Creation**: Create new models with automatic ref() generation
41
+ - **Property Discovery**: View effective configurations from all sources (dbt_project.yml, schema.yml, model config)
42
+ - **Session Persistence**: Automatically saves and restores your last viewed project and model
43
+ - **External Editor Integration**: Open models in your preferred editor
44
+
45
+ ## Installation
46
+
47
+ ### Requirements
48
+
49
+ - Python 3.12 or higher
50
+ - A dbt project to explore
51
+
52
+ ### Install from source
53
+
54
+ ```bash
55
+ # Clone the repository
56
+ git clone <repository-url>
57
+ cd dbtui
58
+
59
+ # Install dependencies using pip
60
+ pip install -e .
61
+
62
+ # Or using poetry
63
+ poetry install
64
+ ```
65
+
66
+ ## Usage
67
+
68
+ ### Basic Usage
69
+
70
+ Navigate to a dbt project directory and run:
71
+
72
+ ```bash
73
+ python -m dbtui
74
+ ```
75
+
76
+ Or if installed via poetry:
77
+
78
+ ```bash
79
+ poetry run python -m dbtui
80
+ ```
81
+
82
+ ### First Launch
83
+
84
+ On first launch, dbtui will:
85
+ 1. Load the test project (if no cached project exists)
86
+ 2. Open the model search screen
87
+ 3. Allow you to search for and select a model
88
+
89
+ ### Keyboard Shortcuts
90
+
91
+ #### Global Bindings
92
+
93
+ | Key | Action | Description |
94
+ |-----|--------|-------------|
95
+ | `o` | Options | Open options/settings screen |
96
+ | `f` | Find Model | Open model search screen |
97
+ | `p` | Project Search | Select a different dbt project |
98
+ | `q` | Quit | Exit the application |
99
+
100
+ #### Model View Screen
101
+
102
+ | Key | Action | Description |
103
+ |-----|--------|-------------|
104
+ | `E` | External Edit | Open current model in external editor |
105
+ | `n` | New Model | Create a new model that refs the current one |
106
+ | `Tab` | Switch Focus | Cycle between parents list, model content, and children list |
107
+ | `↑↓` | Navigate Lists | Move up/down in parent/child lists |
108
+ | `Enter` | Select Model | Navigate to the selected parent/child model |
109
+
110
+ #### Model Search Screen
111
+
112
+ | Key | Action | Description |
113
+ |-----|--------|-------------|
114
+ | Type to search | Live Search | Filter models by name as you type |
115
+ | `↑↓` | Navigate Results | Move through search results |
116
+ | `Enter` | Select Model | Open the selected model |
117
+
118
+ #### New Model Screen
119
+
120
+ | Key | Action | Description |
121
+ |-----|--------|-------------|
122
+ | Type path | Enter Path | Specify relative path for new model (e.g., `models/staging/stg_users.sql`) |
123
+ | `Enter` | Create Model | Create the model and open it |
124
+ | `Esc` | Cancel | Close modal without creating |
125
+
126
+ ## Configuration
127
+
128
+ ### Cache and Settings
129
+
130
+ dbtui stores its configuration in your system's cache directory:
131
+
132
+ - **Linux**: `~/.cache/dbtui/cache.json`
133
+ - **macOS**: `~/Library/Caches/dbtui/cache.json`
134
+ - **Windows**: `%LOCALAPPDATA%\dbtui\cache.json`
135
+
136
+ The cache stores:
137
+ - Last opened project path
138
+ - Last viewed model
139
+ - External editor command (default: `vi`)
140
+
141
+ ### External Editor
142
+
143
+ To configure your preferred external editor, open the options screen (`o` key) and set the editor command. Examples:
144
+
145
+ - `code` - Visual Studio Code
146
+ - `vim` or `vi` - Vim
147
+ - `nano` - Nano
148
+ - `subl` - Sublime Text
149
+
150
+ ## Project Structure
151
+
152
+ ```
153
+ dbtui/
154
+ ├── src/
155
+ │ ├── backend/ # Core dbt project logic
156
+ │ │ ├── project.py # DbtProject class
157
+ │ │ ├── model.py # DbtModel class
158
+ │ │ ├── fetch.py # Property fetching utilities
159
+ │ │ ├── property_claim.py # Property precedence logic
160
+ │ │ └── property_discovery.py # Property discovery from configs
161
+ │ ├── frontend/ # TUI screens and widgets
162
+ │ │ ├── main.py # Main application
163
+ │ │ ├── model_search/ # Model search interface
164
+ │ │ ├── model_tree/ # Model dependency tree view
165
+ │ │ ├── new_model/ # New model creation
166
+ │ │ └── options/ # Settings screen
167
+ │ └── common/ # Shared abstractions and utilities
168
+ │ ├── model.py # Abstract model interface
169
+ │ ├── project.py # Abstract project interface
170
+ │ └── cache.py # Configuration persistence
171
+ ├── tests/ # Test suite
172
+ │ └── testing/ # Test dbt project
173
+ └── pyproject.toml # Project dependencies
174
+ ```
175
+
176
+ ## Architecture
177
+
178
+ ### Backend
179
+
180
+ The backend implements the core dbt project parsing logic:
181
+
182
+ - **DbtProject**: Represents a dbt project, loads models from configured model-paths
183
+ - **DbtModel**: Represents a single model, parses Jinja2 templates to extract refs and configs
184
+ - **PropertyClaim**: Represents a property claim from any configuration source
185
+ - **Property Discovery**: Collects properties from dbt_project.yml, schema.yml, and model SQL
186
+
187
+ ### Frontend
188
+
189
+ Built with [Textual](https://textual.textualize.io/), the frontend provides:
190
+
191
+ - **Screen-based navigation**: Each feature is a separate screen
192
+ - **Reactive state management**: Changes to project/model automatically update UI
193
+ - **Keyboard-driven workflow**: All actions accessible via keyboard
194
+
195
+ ### Property Precedence
196
+
197
+ dbtui correctly implements dbt's configuration precedence rules:
198
+
199
+ 1. **Model-level** (highest): `{{ config(...) }}` in SQL files
200
+ 2. **Schema-level**: `config:` blocks in schema.yml
201
+ 3. **Project-level** (lowest): `models:` section in dbt_project.yml
202
+ - More specific paths override general ones
203
+ - Example: `models.project.staging.stg_users` > `models.project.staging` > `models.project`
204
+
205
+ ## Development
206
+
207
+ ### Running Tests
208
+
209
+ ```bash
210
+ # Run all tests
211
+ pytest
212
+
213
+ # Run specific test file
214
+ pytest tests/test_property_claims.py
215
+
216
+ # Run with coverage
217
+ pytest --cov=src
218
+
219
+ # Run with verbose output
220
+ pytest -v
221
+ ```
222
+
223
+ ### Test Project
224
+
225
+ The `tests/testing/` directory contains a sample dbt project used for testing:
226
+
227
+ - Multiple model directories (vanilla, complex_config, invalid)
228
+ - Models with various configurations
229
+ - Schema.yml files with properties and configs
230
+ - Project-level configurations in dbt_project.yml
231
+
232
+ ### Adding New Features
233
+
234
+ 1. **Backend changes**: Extend DbtProject or DbtModel in `src/backend/`
235
+ 2. **Frontend changes**: Create new screens/widgets in `src/frontend/`
236
+ 3. **Add tests**: Write tests in `tests/` following existing patterns
237
+ 4. **Update abstractions**: Modify `src/common/` interfaces if needed
238
+
239
+ ## Roadmap
240
+
241
+ - [ ] Enhanced model search (fuzzy matching, filters)
242
+ - [ ] Property viewer showing all effective configs with sources
243
+ - [ ] DAG visualization (ASCII graph)
244
+ - [ ] Model execution (dbt run, dbt test)
245
+ - [ ] Column-level lineage
246
+ - [ ] Documentation viewer
247
+ - [ ] Git integration
248
+ - [ ] Multiple project workspace
249
+
250
+ ## Contributing
251
+
252
+ Contributions are welcome! Please:
253
+
254
+ 1. Fork the repository
255
+ 2. Create a feature branch
256
+ 3. Add tests for new functionality
257
+ 4. Ensure all tests pass (`pytest`)
258
+ 5. Submit a pull request
259
+
260
+ ## License
261
+
262
+ [Add your license here]
263
+
264
+ ## Acknowledgments
265
+
266
+ Built with:
267
+ - [Textual](https://textual.textualize.io/) - TUI framework
268
+ - [Jinja2](https://jinja.palletsprojects.com/) - Template parsing
269
+ - [NetworkX](https://networkx.org/) - Graph operations
270
+ - [PyYAML](https://pyyaml.org/) - YAML parsing
271
+
@@ -0,0 +1,49 @@
1
+ dbtui/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ dbtui/__main__.py,sha256=wCZFzGy8hZ-lRt_wJzwUZkWLSSUWtRo6GsufIcBQroA,1602
3
+ dbtui/backend/__init__.py,sha256=ZACjlyjPvAXWDxBWEmSCrHxPO38sOBFTt0Zpm-wAMCU,87
4
+ dbtui/backend/fetch.py,sha256=-_MvLbP8ehpkF5jaEeuTfk92EpuSKmssx1XutIZ-Md8,5362
5
+ dbtui/backend/model.py,sha256=QC95HeyH4kspNG6_BRUFfHbyu-ZfB5ymJL0vJoZYT2o,3005
6
+ dbtui/backend/project.py,sha256=Acc1E90SenOepNSSBblAEA4EydMzV9lQzaXiIgqDpaI,9881
7
+ dbtui/backend/property_claim.py,sha256=V2FF4lYqNaywfG5G4F6uXNI0XQj6s7PDmC0RjoCS7yk,6883
8
+ dbtui/backend/property_discovery.py,sha256=8NF1DpempvMokVBG0H9bvBwPnPpiOKcB99RzLoMFF5M,15028
9
+ dbtui/common/__init__.py,sha256=z-QvzVhkV_9rDXFUswwrC_3dSVa40jSfid4myPdza_Q,155
10
+ dbtui/common/cache.py,sha256=aBUMofPYqGfm4KRd5xlJVIQXEhtnN-8rc6Bw2jPH8mI,2334
11
+ dbtui/common/exceptions.py,sha256=Xic3bVGjYrK4J9nChAwcHEbkhS4qovlvUoFH7W6zRaU,274
12
+ dbtui/common/model.py,sha256=qKQJYNIU8xydCIFMqd8f97tyJLiHsLTMe7efKLRU_FE,474
13
+ dbtui/common/project.py,sha256=6Vfj2EsSG4Tmszod-WYXX0ZYSvp_TzwZhQobRbadRlE,840
14
+ dbtui/frontend/__init__.py,sha256=hXVp-ZV0GJTIunfx2xqvSs4IUMcebSvUxTrLizeqhP8,44
15
+ dbtui/frontend/common/__init__.py,sha256=0B3YYs1RvFfQQEc4rSHZFyvjjFhAp8ML8ujoafdGRC0,157
16
+ dbtui/frontend/common/dbtui_screen.py,sha256=gajcZcSXNkOpdmtstb28szP57J4mykXlIcpT_7sr7EU,1650
17
+ dbtui/frontend/common/isolated.py,sha256=vRF1oSNQnt1M0c0yAgw92ibWPSgPMBi2UxYx34Ogolg,428
18
+ dbtui/frontend/common/model_list.py,sha256=Zf3ZJ9H0h7GKHX9QRrBAskiV2Eo0GGIHf_MH9hOBlUA,1165
19
+ dbtui/frontend/common/model_list_item.py,sha256=Alnqz2ypwdLqEKJhtqE2h-VdUOQ-supoGHbu5HL1HGo,480
20
+ dbtui/frontend/lorem.txt,sha256=HfuvMdyoE8hBebajImphe0gTGVHLMO6qbv9YunoCdLQ,3295
21
+ dbtui/frontend/main.py,sha256=TMz_VecJJp522-RiPwjd-MycAmOel6uRwpLwEvD64sE,4452
22
+ dbtui/frontend/model_search/__init__.py,sha256=OFwnctfsS-4l5q58Lf441gcMugoil0UykMWKuoyzreE,38
23
+ dbtui/frontend/model_search/model_search.py,sha256=13iayOB_V-Ub-3HHrsvZwJDhKRTmn_A6l0DWQEwEsrY,1430
24
+ dbtui/frontend/model_search/model_search_input.py,sha256=uD6yJ7QMiHhZ16fibXeqULelGWXSECS-PeIrTygcXDk,616
25
+ dbtui/frontend/model_search/model_search_list.py,sha256=cAV-9HnEbSBCTE1zNlJtzj9h_lrLuiJrCotOQc69Xtg,515
26
+ dbtui/frontend/model_tree/__init__.py,sha256=cjsjmAal7mH26AoOpoEd8uT1MrzZB97HF_5AgTa0URM,34
27
+ dbtui/frontend/model_tree/children_list.py,sha256=2bplKOXjg9u0cY9tZsRntT7Iep32AyHfI3bFiz62Icc,897
28
+ dbtui/frontend/model_tree/constants.py,sha256=1fO0isX7ObEJXSmKdQkRdKOn0bzbCchFe6tZwlwPMAM,117
29
+ dbtui/frontend/model_tree/model_relatives_list.py,sha256=tbvaILP-VWec8vITuL0MuGA3NNavITTLTmM4lzmvM5c,476
30
+ dbtui/frontend/model_tree/model_tree.py,sha256=820iKDA5jjZ20PqkGVj8XYoUuOija4EbIRfdH6csb7c,3268
31
+ dbtui/frontend/model_tree/parents_list.py,sha256=FhUK9zZw3j9SqmX7NqSS2sLLP8EXZ2HZxEmV9FLepIw,901
32
+ dbtui/frontend/model_view/__init__.py,sha256=iPWtYSByCYX_nE374ddtcSJhUBb2V5zTgQ355bTJdcE,80
33
+ dbtui/frontend/model_view/model_tree.py,sha256=jH6BDammNRrQ4v-SbEozzxurRlp_vUiGom2-YCTwZB4,5295
34
+ dbtui/frontend/model_view/properties_panel.py,sha256=4gAO4CKX0NtGwcWkctM0ufyW7C5I_xEJtUAYISbwOC0,5098
35
+ dbtui/frontend/new_model/__init__.py,sha256=EEQNjE0mlWlKC8BzB0F2RLEmOZgrhOPx3jDdV8o3Yu0,31
36
+ dbtui/frontend/new_model/new_model.py,sha256=eHi70emtt0v1QmzzNqG006npkJpWL4j96h5MMWVQ7wU,1564
37
+ dbtui/frontend/new_model/select_filepath.py,sha256=3Ai96GBjHNuCPmQvX87kdYpwgZeis390PBIRay_x4Uk,1001
38
+ dbtui/frontend/options/__init__.py,sha256=O0HOFS-oaXdpgFSjmUxFROj4LCizHENjggblfCluR78,29
39
+ dbtui/frontend/options/options.py,sha256=QJo8iuQKr3h1JcF8lPk71cT04UYxu_3jS8pviS7mmqM,837
40
+ dbtui/frontend/project_search/__init__.py,sha256=yxCMGHgd5O4rClMnLyD-X2CoX05M9kUWUe-T_5jnJ1U,42
41
+ dbtui/frontend/project_search/project_search.py,sha256=BvrprGVnpuh5myN27C2Z6wzySdXELIicbrlOackUTdI,3633
42
+ dbtui/frontend/pseudo/__init__.py,sha256=jTNtjd6Gcj7BpnUXCWAnLQL7AitByq05VPt5SoAyVrc,68
43
+ dbtui/frontend/pseudo/dbt_model.py,sha256=r8roTZnBpoAHkCLTBUOmLAryK-heHwACSwVkL1B9Rrk,1198
44
+ dbtui/frontend/pseudo/dbt_project.py,sha256=qRTqQjv2cV9nMSS92pob5mP22YiATIhSuoJWLX4YGZE,1127
45
+ dbtui/frontend/pseudo/utils.py,sha256=lDUc6q1vE7aegtb-rkjer4UOY_FSWADKJAnRG4ZObMA,132
46
+ dbt_tui-0.1.0.dist-info/METADATA,sha256=dcjV18nPbfxVh2Hl7fdFrREWe84U2V_8t6E3rGzd9XU,8504
47
+ dbt_tui-0.1.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
48
+ dbt_tui-0.1.0.dist-info/entry_points.txt,sha256=zEaIQ64eIxqYbrMCquQRHaeHpbVup2QXIfFQ1O08cCQ,73
49
+ dbt_tui-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: poetry-core 2.2.1
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,4 @@
1
+ [console_scripts]
2
+ dbt-tui=dbtui.__main__:main
3
+ dbtui=dbtui.__main__:main
4
+
dbtui/__init__.py ADDED
File without changes
dbtui/__main__.py ADDED
@@ -0,0 +1,59 @@
1
+ """
2
+ dbtui - Terminal UI for dbt projects
3
+
4
+ Usage:
5
+ python -m dbtui [project_dir]
6
+
7
+ If project_dir is not specified, launches with the last opened project.
8
+ """
9
+ import argparse
10
+ import sys
11
+ from pathlib import Path
12
+
13
+ from .frontend import frontend as dbtuiFrontend
14
+ from .common import load_cache, save_cache
15
+
16
+
17
+ def main():
18
+ parser = argparse.ArgumentParser(
19
+ prog='dbtui',
20
+ description='Terminal UI for exploring and managing dbt projects'
21
+ )
22
+ parser.add_argument(
23
+ 'project_dir',
24
+ nargs='?',
25
+ default=None,
26
+ help='Path to dbt project directory (optional, uses last project if not specified)'
27
+ )
28
+
29
+ args = parser.parse_args()
30
+
31
+ # If project_dir is provided, validate it and save to cache
32
+ if args.project_dir:
33
+ project_path = Path(args.project_dir).resolve()
34
+
35
+ if not project_path.exists():
36
+ print(f"Error: Directory not found: {project_path}", file=sys.stderr)
37
+ sys.exit(1)
38
+
39
+ if not (project_path / 'dbt_project.yml').exists():
40
+ print(f"Error: Not a dbt project (no dbt_project.yml found): {project_path}", file=sys.stderr)
41
+ sys.exit(1)
42
+
43
+ # Load existing cache to preserve external_editor_command
44
+ existing_cache = load_cache()
45
+
46
+ # Save the project path to cache so the app loads it
47
+ save_cache(
48
+ project_path=project_path,
49
+ model_name=None,
50
+ external_editor_command=existing_cache.external_editor_command
51
+ )
52
+
53
+ # Launch the app
54
+ app = dbtuiFrontend()
55
+ app.run()
56
+
57
+
58
+ if __name__ == '__main__':
59
+ main()
@@ -0,0 +1,2 @@
1
+ from .project import DbtProject, DbtModelNotFoundException
2
+ from .model import DbtModel
dbtui/backend/fetch.py ADDED
@@ -0,0 +1,206 @@
1
+ from dataclasses import dataclass
2
+ from pathlib import Path
3
+ from typing import Any, Literal
4
+
5
+
6
+ def get_model_path_parts(
7
+ model_path: Path,
8
+ project_path: Path,
9
+ ) -> list[str]:
10
+ """
11
+ models/foo/bar/my_model.sql -> ["foo", "bar", "my_model"]
12
+ """
13
+ models_dir = project_path / "models"
14
+ rel = model_path.resolve().relative_to(models_dir.resolve())
15
+ return list(rel.with_suffix("").parts)
16
+
17
+
18
+ def find_schema_files(
19
+ model_path: Path,
20
+ project_path: Path,
21
+ ) -> list[Path]:
22
+ schema_files: list[Path] = []
23
+
24
+ model_dir_path_abs = model_path.parent.resolve()
25
+ project_dir_path_abs = project_path.resolve()
26
+
27
+ yml_targets = [i for i in model_dir_path_abs.relative_to(project_dir_path_abs).parents if not i == Path('.')]
28
+
29
+ for target in yml_targets:
30
+ for ext in ("*.yml", "*.yaml"):
31
+ schema_files.extend(target.glob(ext))
32
+
33
+
34
+ return schema_files
35
+
36
+
37
+
38
+ @dataclass
39
+ class PropertyClaim:
40
+ source_type: Literal["dbt_project.yml", "schema.yml", "model.sql"]
41
+ source_path: Path
42
+ name: str
43
+ value: Any
44
+ kind: Literal["config", "property"]
45
+
46
+ import yaml
47
+ import re
48
+
49
+ def collect_project_configs(
50
+ model_path: Path,
51
+ project_path: Path,
52
+ ) -> list[PropertyClaim]:
53
+ claims: list[PropertyClaim] = []
54
+
55
+ project_file = project_path / "dbt_project.yml"
56
+ if not project_file.exists():
57
+ return claims
58
+
59
+ data = yaml.safe_load(project_file.read_text()) or {}
60
+ models = data.get("models", {})
61
+
62
+ model_parts = get_model_path_parts(model_path, project_path)
63
+
64
+ def walk(node, path_index: int):
65
+ """
66
+ node: current yaml dict
67
+ path_index: where we are in model_parts
68
+ """
69
+ if not isinstance(node, dict):
70
+ return
71
+
72
+ # collect configs at this level
73
+ for k, v in node.items():
74
+ if isinstance(k, str) and k.startswith("+"):
75
+ claims.append(
76
+ PropertyClaim(
77
+ source_type="dbt_project.yml",
78
+ source_path=project_file,
79
+ name=k[1:],
80
+ value=v,
81
+ kind="config",
82
+ )
83
+ )
84
+
85
+ # descend if possible
86
+ if path_index >= len(model_parts):
87
+ return
88
+
89
+ next_key = model_parts[path_index]
90
+ if next_key in node:
91
+ walk(node[next_key], path_index + 1)
92
+
93
+ # dbt_project.yml always has: models: <package_name>: ...
94
+ for package_node in models.values():
95
+ walk(package_node, 0)
96
+
97
+ return claims
98
+
99
+
100
+
101
+
102
+ def collect_schema_properties(
103
+ schema_file: Path,
104
+ model_name: str,
105
+ ) -> list[PropertyClaim]:
106
+ claims: list[PropertyClaim] = []
107
+
108
+ try:
109
+ data = yaml.safe_load(schema_file.read_text()) or {}
110
+ except Exception:
111
+ return claims # invalid yaml, dbt would scream; you quietly ignore
112
+
113
+ models = data.get("models", [])
114
+
115
+ for model in models:
116
+ if model.get("name") != model_name:
117
+ continue
118
+
119
+ for key, value in model.items():
120
+ if key == "config":
121
+ for ck, cv in (value or {}).items():
122
+ claims.append(
123
+ PropertyClaim(
124
+ source_type="schema.yml",
125
+ source_path=schema_file,
126
+ name=ck,
127
+ value=cv,
128
+ kind="config",
129
+ )
130
+ )
131
+ elif key != "name":
132
+ claims.append(
133
+ PropertyClaim(
134
+ source_type="schema.yml",
135
+ source_path=schema_file,
136
+ name=key,
137
+ value=value,
138
+ kind="property",
139
+ )
140
+ )
141
+
142
+ return claims
143
+
144
+
145
+ CONFIG_CALL_RE = re.compile(
146
+ r"config\s*\((.*?)\)",
147
+ re.DOTALL,
148
+ )
149
+
150
+
151
+ def collect_sql_configs(model_path: Path) -> list[PropertyClaim]:
152
+ claims: list[PropertyClaim] = []
153
+
154
+ text = model_path.read_text()
155
+ matches = CONFIG_CALL_RE.findall(text)
156
+
157
+ for match in matches:
158
+ # extremely naive kwarg parsing
159
+ parts = re.split(r",(?![^\[\]\(\)]*[\]\)])", match)
160
+
161
+ for part in parts:
162
+ if "=" not in part:
163
+ continue
164
+ key, value = part.split("=", 1)
165
+ claims.append(
166
+ PropertyClaim(
167
+ source_type="model.sql",
168
+ source_path=model_path,
169
+ name=key.strip(),
170
+ value=value.strip(),
171
+ kind="config",
172
+ )
173
+ )
174
+
175
+ return claims
176
+
177
+ def collect_model_claims(
178
+ model_path: Path,
179
+ project_path: Path,
180
+ ) -> list[PropertyClaim]:
181
+ claims: list[PropertyClaim] = []
182
+ model_name = model_path.stem
183
+
184
+ # dbt_project.yml
185
+ claims.extend(collect_project_configs(model_path, project_path))
186
+
187
+ # schema.yml files discovered automatically
188
+ schema_files = find_schema_files(model_path, project_path)
189
+ for schema_file in schema_files:
190
+ claims.extend(
191
+ collect_schema_properties(schema_file, model_name)
192
+ )
193
+
194
+ # model.sql
195
+ claims.extend(collect_sql_configs(model_path))
196
+
197
+ return claims
198
+
199
+
200
+ print(
201
+ collect_model_claims(
202
+ Path('models/mart/users_daily_stat.sql'),
203
+ Path('.'),
204
+ )
205
+ )
206
+
dbtui/backend/model.py ADDED
@@ -0,0 +1,86 @@
1
+ from pathlib import Path
2
+ from jinja2 import Environment, Template
3
+ from jinja2.nodes import Call, Const
4
+ import logging
5
+
6
+ from ..common import DbtModelAbstract, NotWithinSubdirectoryException
7
+
8
+ from typing import TYPE_CHECKING, Iterable, Self
9
+ if TYPE_CHECKING:
10
+ from .project import DbtProject
11
+ from .property_claim import PropertyClaimAggregate
12
+
13
+
14
+ class DbtModel(DbtModelAbstract):
15
+ file_name: str
16
+ file_path_full: Path
17
+ file_path_relative: Path
18
+ template: str
19
+ parsed_template: Template
20
+ project: 'DbtProject'
21
+ property_claims: 'PropertyClaimAggregate | None'
22
+
23
+ @property
24
+ def file_path_relative(self):
25
+ return self.file_path_full.relative_to(self.project.root_folder)
26
+
27
+ @property
28
+ def file_name(self):
29
+ return self.file_path_full.name
30
+
31
+ def __init__(self, file_path_full: Path, project: 'DbtProject'):
32
+ self.file_path_full = file_path_full
33
+ self.project = project
34
+ self.property_claims = None # Populated by DbtProject.collect_property_claims()
35
+ with open(file_path_full, 'r', encoding='utf-8') as f:
36
+ self.template = f.read()
37
+ self.parsed_template = Environment().parse(self.template)
38
+
39
+ def _find_calls(self, macro_name: str):
40
+ return [i for i in self.parsed_template.find_all(Call) if i.node.name == macro_name]
41
+
42
+
43
+ @property
44
+ def name(self) -> str:
45
+ # by default dbt sets model name as its filename without extension
46
+ default_name = self.file_name.rpartition('.')[0]
47
+ # we need to figure out if the model has been renamed with config() macro, so let's find this macro in the model file
48
+ calls = self._find_calls('config')
49
+
50
+ # double config would be invalid
51
+ if len(calls) > 1:
52
+ logging.warn("Duplicated invocation of config() in %s" % self.file_path_relative)
53
+ return default_name
54
+
55
+ if not calls:
56
+ return default_name
57
+ else:
58
+ config: Call = calls[0]
59
+ kwargs = {item.key: item.value for item in config.kwargs}
60
+ return kwargs.get('name', Const(default_name)).value
61
+
62
+ @property
63
+ def children(self) -> list[Self]:
64
+ return sorted(list(self.project.graph.successors(self)), key=lambda n: n.name)
65
+
66
+ @property
67
+ def parents(self) -> Iterable[Self]:
68
+ return sorted(list(self.project.graph.predecessors(self)), key=lambda n: n.name)
69
+
70
+ @property
71
+ def text(self) -> str:
72
+ with open(self.file_path_full, 'r', encoding='utf-8') as f:
73
+ self.template = f.read()
74
+ return self.template
75
+
76
+
77
+ @property
78
+ def refs(self) -> list[str]:
79
+ result = []
80
+ for call in self._find_calls('ref'):
81
+ if len(call.args) != 1:
82
+ logging.warn(f"Invalid number of args to ref() in model {self.name}: should be one, but it's {len(call.args)}: {[arg.value for arg in call.args]}")
83
+ continue
84
+ result.append(call.args[0].value)
85
+ return result
86
+