dbt-tui 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_tui-0.1.0.dist-info/METADATA +271 -0
- dbt_tui-0.1.0.dist-info/RECORD +49 -0
- dbt_tui-0.1.0.dist-info/WHEEL +4 -0
- dbt_tui-0.1.0.dist-info/entry_points.txt +4 -0
- dbtui/__init__.py +0 -0
- dbtui/__main__.py +59 -0
- dbtui/backend/__init__.py +2 -0
- dbtui/backend/fetch.py +206 -0
- dbtui/backend/model.py +86 -0
- dbtui/backend/project.py +258 -0
- dbtui/backend/property_claim.py +191 -0
- dbtui/backend/property_discovery.py +461 -0
- dbtui/common/__init__.py +4 -0
- dbtui/common/cache.py +70 -0
- dbtui/common/exceptions.py +14 -0
- dbtui/common/model.py +25 -0
- dbtui/common/project.py +32 -0
- dbtui/frontend/__init__.py +1 -0
- dbtui/frontend/common/__init__.py +4 -0
- dbtui/frontend/common/dbtui_screen.py +50 -0
- dbtui/frontend/common/isolated.py +14 -0
- dbtui/frontend/common/model_list.py +41 -0
- dbtui/frontend/common/model_list_item.py +20 -0
- dbtui/frontend/lorem.txt +9 -0
- dbtui/frontend/main.py +138 -0
- dbtui/frontend/model_search/__init__.py +1 -0
- dbtui/frontend/model_search/model_search.py +52 -0
- dbtui/frontend/model_search/model_search_input.py +21 -0
- dbtui/frontend/model_search/model_search_list.py +14 -0
- dbtui/frontend/model_tree/__init__.py +1 -0
- dbtui/frontend/model_tree/children_list.py +28 -0
- dbtui/frontend/model_tree/constants.py +4 -0
- dbtui/frontend/model_tree/model_relatives_list.py +15 -0
- dbtui/frontend/model_tree/model_tree.py +98 -0
- dbtui/frontend/model_tree/parents_list.py +28 -0
- dbtui/frontend/model_view/__init__.py +2 -0
- dbtui/frontend/model_view/model_tree.py +162 -0
- dbtui/frontend/model_view/properties_panel.py +172 -0
- dbtui/frontend/new_model/__init__.py +1 -0
- dbtui/frontend/new_model/new_model.py +51 -0
- dbtui/frontend/new_model/select_filepath.py +30 -0
- dbtui/frontend/options/__init__.py +1 -0
- dbtui/frontend/options/options.py +37 -0
- dbtui/frontend/project_search/__init__.py +1 -0
- dbtui/frontend/project_search/project_search.py +109 -0
- dbtui/frontend/pseudo/__init__.py +2 -0
- dbtui/frontend/pseudo/dbt_model.py +50 -0
- dbtui/frontend/pseudo/dbt_project.py +37 -0
- dbtui/frontend/pseudo/utils.py +8 -0
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dbt-tui
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Terminal UI for exploring and managing dbt projects
|
|
5
|
+
License: MIT
|
|
6
|
+
Keywords: dbt,tui,terminal,data,analytics,sql
|
|
7
|
+
Author: Mike
|
|
8
|
+
Author-email: sortia@protonmail.com
|
|
9
|
+
Requires-Python: >=3.12, <4.0
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Database
|
|
18
|
+
Classifier: Topic :: Software Development :: User Interfaces
|
|
19
|
+
Requires-Dist: jinja2 (>=3.0)
|
|
20
|
+
Requires-Dist: networkx (>=3.0)
|
|
21
|
+
Requires-Dist: platformdirs (>=4.5.0,<5.0.0)
|
|
22
|
+
Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
|
|
23
|
+
Requires-Dist: textual[syntax] (>=6.5.0,<7.0.0)
|
|
24
|
+
Project-URL: Homepage, https://github.com/sortia/dbt-tui
|
|
25
|
+
Project-URL: Repository, https://github.com/sortia/dbt-tui
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# dbtui
|
|
29
|
+
|
|
30
|
+
A Terminal User Interface (TUI) for navigating and managing [dbt (data build tool)](https://www.getdbt.com/) projects.
|
|
31
|
+
|
|
32
|
+
**dbtui** provides an interactive command-line interface for exploring dbt models, their relationships, and configurations without leaving your terminal.
|
|
33
|
+
|
|
34
|
+
## Features
|
|
35
|
+
|
|
36
|
+
- **Model Explorer**: Browse all models in your dbt project with search capabilities
|
|
37
|
+
- **Dependency Visualization**: View parent and child relationships for any model
|
|
38
|
+
- **Model Preview**: Read model SQL and configuration inline
|
|
39
|
+
- **Quick Navigation**: Jump between related models with keyboard shortcuts
|
|
40
|
+
- **New Model Creation**: Create new models with automatic ref() generation
|
|
41
|
+
- **Property Discovery**: View effective configurations from all sources (dbt_project.yml, schema.yml, model config)
|
|
42
|
+
- **Session Persistence**: Automatically saves and restores your last viewed project and model
|
|
43
|
+
- **External Editor Integration**: Open models in your preferred editor
|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
### Requirements
|
|
48
|
+
|
|
49
|
+
- Python 3.12 or higher
|
|
50
|
+
- A dbt project to explore
|
|
51
|
+
|
|
52
|
+
### Install from source
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
# Clone the repository
|
|
56
|
+
git clone <repository-url>
|
|
57
|
+
cd dbtui
|
|
58
|
+
|
|
59
|
+
# Install dependencies using pip
|
|
60
|
+
pip install -e .
|
|
61
|
+
|
|
62
|
+
# Or using poetry
|
|
63
|
+
poetry install
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Usage
|
|
67
|
+
|
|
68
|
+
### Basic Usage
|
|
69
|
+
|
|
70
|
+
Navigate to a dbt project directory and run:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
python -m dbtui
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Or if installed via poetry:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
poetry run python -m dbtui
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### First Launch
|
|
83
|
+
|
|
84
|
+
On first launch, dbtui will:
|
|
85
|
+
1. Load the test project (if no cached project exists)
|
|
86
|
+
2. Open the model search screen
|
|
87
|
+
3. Allow you to search for and select a model
|
|
88
|
+
|
|
89
|
+
### Keyboard Shortcuts
|
|
90
|
+
|
|
91
|
+
#### Global Bindings
|
|
92
|
+
|
|
93
|
+
| Key | Action | Description |
|
|
94
|
+
|-----|--------|-------------|
|
|
95
|
+
| `o` | Options | Open options/settings screen |
|
|
96
|
+
| `f` | Find Model | Open model search screen |
|
|
97
|
+
| `p` | Project Search | Select a different dbt project |
|
|
98
|
+
| `q` | Quit | Exit the application |
|
|
99
|
+
|
|
100
|
+
#### Model View Screen
|
|
101
|
+
|
|
102
|
+
| Key | Action | Description |
|
|
103
|
+
|-----|--------|-------------|
|
|
104
|
+
| `E` | External Edit | Open current model in external editor |
|
|
105
|
+
| `n` | New Model | Create a new model that refs the current one |
|
|
106
|
+
| `Tab` | Switch Focus | Cycle between parents list, model content, and children list |
|
|
107
|
+
| `↑↓` | Navigate Lists | Move up/down in parent/child lists |
|
|
108
|
+
| `Enter` | Select Model | Navigate to the selected parent/child model |
|
|
109
|
+
|
|
110
|
+
#### Model Search Screen
|
|
111
|
+
|
|
112
|
+
| Key | Action | Description |
|
|
113
|
+
|-----|--------|-------------|
|
|
114
|
+
| Type to search | Live Search | Filter models by name as you type |
|
|
115
|
+
| `↑↓` | Navigate Results | Move through search results |
|
|
116
|
+
| `Enter` | Select Model | Open the selected model |
|
|
117
|
+
|
|
118
|
+
#### New Model Screen
|
|
119
|
+
|
|
120
|
+
| Key | Action | Description |
|
|
121
|
+
|-----|--------|-------------|
|
|
122
|
+
| Type path | Enter Path | Specify relative path for new model (e.g., `models/staging/stg_users.sql`) |
|
|
123
|
+
| `Enter` | Create Model | Create the model and open it |
|
|
124
|
+
| `Esc` | Cancel | Close modal without creating |
|
|
125
|
+
|
|
126
|
+
## Configuration
|
|
127
|
+
|
|
128
|
+
### Cache and Settings
|
|
129
|
+
|
|
130
|
+
dbtui stores its configuration in your system's cache directory:
|
|
131
|
+
|
|
132
|
+
- **Linux**: `~/.cache/dbtui/cache.json`
|
|
133
|
+
- **macOS**: `~/Library/Caches/dbtui/cache.json`
|
|
134
|
+
- **Windows**: `%LOCALAPPDATA%\dbtui\cache.json`
|
|
135
|
+
|
|
136
|
+
The cache stores:
|
|
137
|
+
- Last opened project path
|
|
138
|
+
- Last viewed model
|
|
139
|
+
- External editor command (default: `vi`)
|
|
140
|
+
|
|
141
|
+
### External Editor
|
|
142
|
+
|
|
143
|
+
To configure your preferred external editor, open the options screen (`o` key) and set the editor command. Examples:
|
|
144
|
+
|
|
145
|
+
- `code` - Visual Studio Code
|
|
146
|
+
- `vim` or `vi` - Vim
|
|
147
|
+
- `nano` - Nano
|
|
148
|
+
- `subl` - Sublime Text
|
|
149
|
+
|
|
150
|
+
## Project Structure
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
dbtui/
|
|
154
|
+
├── src/
|
|
155
|
+
│ ├── backend/ # Core dbt project logic
|
|
156
|
+
│ │ ├── project.py # DbtProject class
|
|
157
|
+
│ │ ├── model.py # DbtModel class
|
|
158
|
+
│ │ ├── fetch.py # Property fetching utilities
|
|
159
|
+
│ │ ├── property_claim.py # Property precedence logic
|
|
160
|
+
│ │ └── property_discovery.py # Property discovery from configs
|
|
161
|
+
│ ├── frontend/ # TUI screens and widgets
|
|
162
|
+
│ │ ├── main.py # Main application
|
|
163
|
+
│ │ ├── model_search/ # Model search interface
|
|
164
|
+
│ │ ├── model_tree/ # Model dependency tree view
|
|
165
|
+
│ │ ├── new_model/ # New model creation
|
|
166
|
+
│ │ └── options/ # Settings screen
|
|
167
|
+
│ └── common/ # Shared abstractions and utilities
|
|
168
|
+
│ ├── model.py # Abstract model interface
|
|
169
|
+
│ ├── project.py # Abstract project interface
|
|
170
|
+
│ └── cache.py # Configuration persistence
|
|
171
|
+
├── tests/ # Test suite
|
|
172
|
+
│ └── testing/ # Test dbt project
|
|
173
|
+
└── pyproject.toml # Project dependencies
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
## Architecture
|
|
177
|
+
|
|
178
|
+
### Backend
|
|
179
|
+
|
|
180
|
+
The backend implements the core dbt project parsing logic:
|
|
181
|
+
|
|
182
|
+
- **DbtProject**: Represents a dbt project, loads models from configured model-paths
|
|
183
|
+
- **DbtModel**: Represents a single model, parses Jinja2 templates to extract refs and configs
|
|
184
|
+
- **PropertyClaim**: Represents a property claim from any configuration source
|
|
185
|
+
- **Property Discovery**: Collects properties from dbt_project.yml, schema.yml, and model SQL
|
|
186
|
+
|
|
187
|
+
### Frontend
|
|
188
|
+
|
|
189
|
+
Built with [Textual](https://textual.textualize.io/), the frontend provides:
|
|
190
|
+
|
|
191
|
+
- **Screen-based navigation**: Each feature is a separate screen
|
|
192
|
+
- **Reactive state management**: Changes to project/model automatically update UI
|
|
193
|
+
- **Keyboard-driven workflow**: All actions accessible via keyboard
|
|
194
|
+
|
|
195
|
+
### Property Precedence
|
|
196
|
+
|
|
197
|
+
dbtui correctly implements dbt's configuration precedence rules:
|
|
198
|
+
|
|
199
|
+
1. **Model-level** (highest): `{{ config(...) }}` in SQL files
|
|
200
|
+
2. **Schema-level**: `config:` blocks in schema.yml
|
|
201
|
+
3. **Project-level** (lowest): `models:` section in dbt_project.yml
|
|
202
|
+
- More specific paths override general ones
|
|
203
|
+
- Example: `models.project.staging.stg_users` > `models.project.staging` > `models.project`
|
|
204
|
+
|
|
205
|
+
## Development
|
|
206
|
+
|
|
207
|
+
### Running Tests
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
# Run all tests
|
|
211
|
+
pytest
|
|
212
|
+
|
|
213
|
+
# Run specific test file
|
|
214
|
+
pytest tests/test_property_claims.py
|
|
215
|
+
|
|
216
|
+
# Run with coverage
|
|
217
|
+
pytest --cov=src
|
|
218
|
+
|
|
219
|
+
# Run with verbose output
|
|
220
|
+
pytest -v
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
### Test Project
|
|
224
|
+
|
|
225
|
+
The `tests/testing/` directory contains a sample dbt project used for testing:
|
|
226
|
+
|
|
227
|
+
- Multiple model directories (vanilla, complex_config, invalid)
|
|
228
|
+
- Models with various configurations
|
|
229
|
+
- Schema.yml files with properties and configs
|
|
230
|
+
- Project-level configurations in dbt_project.yml
|
|
231
|
+
|
|
232
|
+
### Adding New Features
|
|
233
|
+
|
|
234
|
+
1. **Backend changes**: Extend DbtProject or DbtModel in `src/backend/`
|
|
235
|
+
2. **Frontend changes**: Create new screens/widgets in `src/frontend/`
|
|
236
|
+
3. **Add tests**: Write tests in `tests/` following existing patterns
|
|
237
|
+
4. **Update abstractions**: Modify `src/common/` interfaces if needed
|
|
238
|
+
|
|
239
|
+
## Roadmap
|
|
240
|
+
|
|
241
|
+
- [ ] Enhanced model search (fuzzy matching, filters)
|
|
242
|
+
- [ ] Property viewer showing all effective configs with sources
|
|
243
|
+
- [ ] DAG visualization (ASCII graph)
|
|
244
|
+
- [ ] Model execution (dbt run, dbt test)
|
|
245
|
+
- [ ] Column-level lineage
|
|
246
|
+
- [ ] Documentation viewer
|
|
247
|
+
- [ ] Git integration
|
|
248
|
+
- [ ] Multiple project workspace
|
|
249
|
+
|
|
250
|
+
## Contributing
|
|
251
|
+
|
|
252
|
+
Contributions are welcome! Please:
|
|
253
|
+
|
|
254
|
+
1. Fork the repository
|
|
255
|
+
2. Create a feature branch
|
|
256
|
+
3. Add tests for new functionality
|
|
257
|
+
4. Ensure all tests pass (`pytest`)
|
|
258
|
+
5. Submit a pull request
|
|
259
|
+
|
|
260
|
+
## License
|
|
261
|
+
|
|
262
|
+
[Add your license here]
|
|
263
|
+
|
|
264
|
+
## Acknowledgments
|
|
265
|
+
|
|
266
|
+
Built with:
|
|
267
|
+
- [Textual](https://textual.textualize.io/) - TUI framework
|
|
268
|
+
- [Jinja2](https://jinja.palletsprojects.com/) - Template parsing
|
|
269
|
+
- [NetworkX](https://networkx.org/) - Graph operations
|
|
270
|
+
- [PyYAML](https://pyyaml.org/) - YAML parsing
|
|
271
|
+
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
dbtui/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
dbtui/__main__.py,sha256=wCZFzGy8hZ-lRt_wJzwUZkWLSSUWtRo6GsufIcBQroA,1602
|
|
3
|
+
dbtui/backend/__init__.py,sha256=ZACjlyjPvAXWDxBWEmSCrHxPO38sOBFTt0Zpm-wAMCU,87
|
|
4
|
+
dbtui/backend/fetch.py,sha256=-_MvLbP8ehpkF5jaEeuTfk92EpuSKmssx1XutIZ-Md8,5362
|
|
5
|
+
dbtui/backend/model.py,sha256=QC95HeyH4kspNG6_BRUFfHbyu-ZfB5ymJL0vJoZYT2o,3005
|
|
6
|
+
dbtui/backend/project.py,sha256=Acc1E90SenOepNSSBblAEA4EydMzV9lQzaXiIgqDpaI,9881
|
|
7
|
+
dbtui/backend/property_claim.py,sha256=V2FF4lYqNaywfG5G4F6uXNI0XQj6s7PDmC0RjoCS7yk,6883
|
|
8
|
+
dbtui/backend/property_discovery.py,sha256=8NF1DpempvMokVBG0H9bvBwPnPpiOKcB99RzLoMFF5M,15028
|
|
9
|
+
dbtui/common/__init__.py,sha256=z-QvzVhkV_9rDXFUswwrC_3dSVa40jSfid4myPdza_Q,155
|
|
10
|
+
dbtui/common/cache.py,sha256=aBUMofPYqGfm4KRd5xlJVIQXEhtnN-8rc6Bw2jPH8mI,2334
|
|
11
|
+
dbtui/common/exceptions.py,sha256=Xic3bVGjYrK4J9nChAwcHEbkhS4qovlvUoFH7W6zRaU,274
|
|
12
|
+
dbtui/common/model.py,sha256=qKQJYNIU8xydCIFMqd8f97tyJLiHsLTMe7efKLRU_FE,474
|
|
13
|
+
dbtui/common/project.py,sha256=6Vfj2EsSG4Tmszod-WYXX0ZYSvp_TzwZhQobRbadRlE,840
|
|
14
|
+
dbtui/frontend/__init__.py,sha256=hXVp-ZV0GJTIunfx2xqvSs4IUMcebSvUxTrLizeqhP8,44
|
|
15
|
+
dbtui/frontend/common/__init__.py,sha256=0B3YYs1RvFfQQEc4rSHZFyvjjFhAp8ML8ujoafdGRC0,157
|
|
16
|
+
dbtui/frontend/common/dbtui_screen.py,sha256=gajcZcSXNkOpdmtstb28szP57J4mykXlIcpT_7sr7EU,1650
|
|
17
|
+
dbtui/frontend/common/isolated.py,sha256=vRF1oSNQnt1M0c0yAgw92ibWPSgPMBi2UxYx34Ogolg,428
|
|
18
|
+
dbtui/frontend/common/model_list.py,sha256=Zf3ZJ9H0h7GKHX9QRrBAskiV2Eo0GGIHf_MH9hOBlUA,1165
|
|
19
|
+
dbtui/frontend/common/model_list_item.py,sha256=Alnqz2ypwdLqEKJhtqE2h-VdUOQ-supoGHbu5HL1HGo,480
|
|
20
|
+
dbtui/frontend/lorem.txt,sha256=HfuvMdyoE8hBebajImphe0gTGVHLMO6qbv9YunoCdLQ,3295
|
|
21
|
+
dbtui/frontend/main.py,sha256=TMz_VecJJp522-RiPwjd-MycAmOel6uRwpLwEvD64sE,4452
|
|
22
|
+
dbtui/frontend/model_search/__init__.py,sha256=OFwnctfsS-4l5q58Lf441gcMugoil0UykMWKuoyzreE,38
|
|
23
|
+
dbtui/frontend/model_search/model_search.py,sha256=13iayOB_V-Ub-3HHrsvZwJDhKRTmn_A6l0DWQEwEsrY,1430
|
|
24
|
+
dbtui/frontend/model_search/model_search_input.py,sha256=uD6yJ7QMiHhZ16fibXeqULelGWXSECS-PeIrTygcXDk,616
|
|
25
|
+
dbtui/frontend/model_search/model_search_list.py,sha256=cAV-9HnEbSBCTE1zNlJtzj9h_lrLuiJrCotOQc69Xtg,515
|
|
26
|
+
dbtui/frontend/model_tree/__init__.py,sha256=cjsjmAal7mH26AoOpoEd8uT1MrzZB97HF_5AgTa0URM,34
|
|
27
|
+
dbtui/frontend/model_tree/children_list.py,sha256=2bplKOXjg9u0cY9tZsRntT7Iep32AyHfI3bFiz62Icc,897
|
|
28
|
+
dbtui/frontend/model_tree/constants.py,sha256=1fO0isX7ObEJXSmKdQkRdKOn0bzbCchFe6tZwlwPMAM,117
|
|
29
|
+
dbtui/frontend/model_tree/model_relatives_list.py,sha256=tbvaILP-VWec8vITuL0MuGA3NNavITTLTmM4lzmvM5c,476
|
|
30
|
+
dbtui/frontend/model_tree/model_tree.py,sha256=820iKDA5jjZ20PqkGVj8XYoUuOija4EbIRfdH6csb7c,3268
|
|
31
|
+
dbtui/frontend/model_tree/parents_list.py,sha256=FhUK9zZw3j9SqmX7NqSS2sLLP8EXZ2HZxEmV9FLepIw,901
|
|
32
|
+
dbtui/frontend/model_view/__init__.py,sha256=iPWtYSByCYX_nE374ddtcSJhUBb2V5zTgQ355bTJdcE,80
|
|
33
|
+
dbtui/frontend/model_view/model_tree.py,sha256=jH6BDammNRrQ4v-SbEozzxurRlp_vUiGom2-YCTwZB4,5295
|
|
34
|
+
dbtui/frontend/model_view/properties_panel.py,sha256=4gAO4CKX0NtGwcWkctM0ufyW7C5I_xEJtUAYISbwOC0,5098
|
|
35
|
+
dbtui/frontend/new_model/__init__.py,sha256=EEQNjE0mlWlKC8BzB0F2RLEmOZgrhOPx3jDdV8o3Yu0,31
|
|
36
|
+
dbtui/frontend/new_model/new_model.py,sha256=eHi70emtt0v1QmzzNqG006npkJpWL4j96h5MMWVQ7wU,1564
|
|
37
|
+
dbtui/frontend/new_model/select_filepath.py,sha256=3Ai96GBjHNuCPmQvX87kdYpwgZeis390PBIRay_x4Uk,1001
|
|
38
|
+
dbtui/frontend/options/__init__.py,sha256=O0HOFS-oaXdpgFSjmUxFROj4LCizHENjggblfCluR78,29
|
|
39
|
+
dbtui/frontend/options/options.py,sha256=QJo8iuQKr3h1JcF8lPk71cT04UYxu_3jS8pviS7mmqM,837
|
|
40
|
+
dbtui/frontend/project_search/__init__.py,sha256=yxCMGHgd5O4rClMnLyD-X2CoX05M9kUWUe-T_5jnJ1U,42
|
|
41
|
+
dbtui/frontend/project_search/project_search.py,sha256=BvrprGVnpuh5myN27C2Z6wzySdXELIicbrlOackUTdI,3633
|
|
42
|
+
dbtui/frontend/pseudo/__init__.py,sha256=jTNtjd6Gcj7BpnUXCWAnLQL7AitByq05VPt5SoAyVrc,68
|
|
43
|
+
dbtui/frontend/pseudo/dbt_model.py,sha256=r8roTZnBpoAHkCLTBUOmLAryK-heHwACSwVkL1B9Rrk,1198
|
|
44
|
+
dbtui/frontend/pseudo/dbt_project.py,sha256=qRTqQjv2cV9nMSS92pob5mP22YiATIhSuoJWLX4YGZE,1127
|
|
45
|
+
dbtui/frontend/pseudo/utils.py,sha256=lDUc6q1vE7aegtb-rkjer4UOY_FSWADKJAnRG4ZObMA,132
|
|
46
|
+
dbt_tui-0.1.0.dist-info/METADATA,sha256=dcjV18nPbfxVh2Hl7fdFrREWe84U2V_8t6E3rGzd9XU,8504
|
|
47
|
+
dbt_tui-0.1.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
48
|
+
dbt_tui-0.1.0.dist-info/entry_points.txt,sha256=zEaIQ64eIxqYbrMCquQRHaeHpbVup2QXIfFQ1O08cCQ,73
|
|
49
|
+
dbt_tui-0.1.0.dist-info/RECORD,,
|
dbtui/__init__.py
ADDED
|
File without changes
|
dbtui/__main__.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""
|
|
2
|
+
dbtui - Terminal UI for dbt projects
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
python -m dbtui [project_dir]
|
|
6
|
+
|
|
7
|
+
If project_dir is not specified, launches with the last opened project.
|
|
8
|
+
"""
|
|
9
|
+
import argparse
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from .frontend import frontend as dbtuiFrontend
|
|
14
|
+
from .common import load_cache, save_cache
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def main():
|
|
18
|
+
parser = argparse.ArgumentParser(
|
|
19
|
+
prog='dbtui',
|
|
20
|
+
description='Terminal UI for exploring and managing dbt projects'
|
|
21
|
+
)
|
|
22
|
+
parser.add_argument(
|
|
23
|
+
'project_dir',
|
|
24
|
+
nargs='?',
|
|
25
|
+
default=None,
|
|
26
|
+
help='Path to dbt project directory (optional, uses last project if not specified)'
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
args = parser.parse_args()
|
|
30
|
+
|
|
31
|
+
# If project_dir is provided, validate it and save to cache
|
|
32
|
+
if args.project_dir:
|
|
33
|
+
project_path = Path(args.project_dir).resolve()
|
|
34
|
+
|
|
35
|
+
if not project_path.exists():
|
|
36
|
+
print(f"Error: Directory not found: {project_path}", file=sys.stderr)
|
|
37
|
+
sys.exit(1)
|
|
38
|
+
|
|
39
|
+
if not (project_path / 'dbt_project.yml').exists():
|
|
40
|
+
print(f"Error: Not a dbt project (no dbt_project.yml found): {project_path}", file=sys.stderr)
|
|
41
|
+
sys.exit(1)
|
|
42
|
+
|
|
43
|
+
# Load existing cache to preserve external_editor_command
|
|
44
|
+
existing_cache = load_cache()
|
|
45
|
+
|
|
46
|
+
# Save the project path to cache so the app loads it
|
|
47
|
+
save_cache(
|
|
48
|
+
project_path=project_path,
|
|
49
|
+
model_name=None,
|
|
50
|
+
external_editor_command=existing_cache.external_editor_command
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
# Launch the app
|
|
54
|
+
app = dbtuiFrontend()
|
|
55
|
+
app.run()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == '__main__':
|
|
59
|
+
main()
|
dbtui/backend/fetch.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Any, Literal
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_model_path_parts(
|
|
7
|
+
model_path: Path,
|
|
8
|
+
project_path: Path,
|
|
9
|
+
) -> list[str]:
|
|
10
|
+
"""
|
|
11
|
+
models/foo/bar/my_model.sql -> ["foo", "bar", "my_model"]
|
|
12
|
+
"""
|
|
13
|
+
models_dir = project_path / "models"
|
|
14
|
+
rel = model_path.resolve().relative_to(models_dir.resolve())
|
|
15
|
+
return list(rel.with_suffix("").parts)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def find_schema_files(
|
|
19
|
+
model_path: Path,
|
|
20
|
+
project_path: Path,
|
|
21
|
+
) -> list[Path]:
|
|
22
|
+
schema_files: list[Path] = []
|
|
23
|
+
|
|
24
|
+
model_dir_path_abs = model_path.parent.resolve()
|
|
25
|
+
project_dir_path_abs = project_path.resolve()
|
|
26
|
+
|
|
27
|
+
yml_targets = [i for i in model_dir_path_abs.relative_to(project_dir_path_abs).parents if not i == Path('.')]
|
|
28
|
+
|
|
29
|
+
for target in yml_targets:
|
|
30
|
+
for ext in ("*.yml", "*.yaml"):
|
|
31
|
+
schema_files.extend(target.glob(ext))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
return schema_files
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class PropertyClaim:
|
|
40
|
+
source_type: Literal["dbt_project.yml", "schema.yml", "model.sql"]
|
|
41
|
+
source_path: Path
|
|
42
|
+
name: str
|
|
43
|
+
value: Any
|
|
44
|
+
kind: Literal["config", "property"]
|
|
45
|
+
|
|
46
|
+
import yaml
|
|
47
|
+
import re
|
|
48
|
+
|
|
49
|
+
def collect_project_configs(
|
|
50
|
+
model_path: Path,
|
|
51
|
+
project_path: Path,
|
|
52
|
+
) -> list[PropertyClaim]:
|
|
53
|
+
claims: list[PropertyClaim] = []
|
|
54
|
+
|
|
55
|
+
project_file = project_path / "dbt_project.yml"
|
|
56
|
+
if not project_file.exists():
|
|
57
|
+
return claims
|
|
58
|
+
|
|
59
|
+
data = yaml.safe_load(project_file.read_text()) or {}
|
|
60
|
+
models = data.get("models", {})
|
|
61
|
+
|
|
62
|
+
model_parts = get_model_path_parts(model_path, project_path)
|
|
63
|
+
|
|
64
|
+
def walk(node, path_index: int):
|
|
65
|
+
"""
|
|
66
|
+
node: current yaml dict
|
|
67
|
+
path_index: where we are in model_parts
|
|
68
|
+
"""
|
|
69
|
+
if not isinstance(node, dict):
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
# collect configs at this level
|
|
73
|
+
for k, v in node.items():
|
|
74
|
+
if isinstance(k, str) and k.startswith("+"):
|
|
75
|
+
claims.append(
|
|
76
|
+
PropertyClaim(
|
|
77
|
+
source_type="dbt_project.yml",
|
|
78
|
+
source_path=project_file,
|
|
79
|
+
name=k[1:],
|
|
80
|
+
value=v,
|
|
81
|
+
kind="config",
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# descend if possible
|
|
86
|
+
if path_index >= len(model_parts):
|
|
87
|
+
return
|
|
88
|
+
|
|
89
|
+
next_key = model_parts[path_index]
|
|
90
|
+
if next_key in node:
|
|
91
|
+
walk(node[next_key], path_index + 1)
|
|
92
|
+
|
|
93
|
+
# dbt_project.yml always has: models: <package_name>: ...
|
|
94
|
+
for package_node in models.values():
|
|
95
|
+
walk(package_node, 0)
|
|
96
|
+
|
|
97
|
+
return claims
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def collect_schema_properties(
|
|
103
|
+
schema_file: Path,
|
|
104
|
+
model_name: str,
|
|
105
|
+
) -> list[PropertyClaim]:
|
|
106
|
+
claims: list[PropertyClaim] = []
|
|
107
|
+
|
|
108
|
+
try:
|
|
109
|
+
data = yaml.safe_load(schema_file.read_text()) or {}
|
|
110
|
+
except Exception:
|
|
111
|
+
return claims # invalid yaml, dbt would scream; you quietly ignore
|
|
112
|
+
|
|
113
|
+
models = data.get("models", [])
|
|
114
|
+
|
|
115
|
+
for model in models:
|
|
116
|
+
if model.get("name") != model_name:
|
|
117
|
+
continue
|
|
118
|
+
|
|
119
|
+
for key, value in model.items():
|
|
120
|
+
if key == "config":
|
|
121
|
+
for ck, cv in (value or {}).items():
|
|
122
|
+
claims.append(
|
|
123
|
+
PropertyClaim(
|
|
124
|
+
source_type="schema.yml",
|
|
125
|
+
source_path=schema_file,
|
|
126
|
+
name=ck,
|
|
127
|
+
value=cv,
|
|
128
|
+
kind="config",
|
|
129
|
+
)
|
|
130
|
+
)
|
|
131
|
+
elif key != "name":
|
|
132
|
+
claims.append(
|
|
133
|
+
PropertyClaim(
|
|
134
|
+
source_type="schema.yml",
|
|
135
|
+
source_path=schema_file,
|
|
136
|
+
name=key,
|
|
137
|
+
value=value,
|
|
138
|
+
kind="property",
|
|
139
|
+
)
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
return claims
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
CONFIG_CALL_RE = re.compile(
|
|
146
|
+
r"config\s*\((.*?)\)",
|
|
147
|
+
re.DOTALL,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def collect_sql_configs(model_path: Path) -> list[PropertyClaim]:
|
|
152
|
+
claims: list[PropertyClaim] = []
|
|
153
|
+
|
|
154
|
+
text = model_path.read_text()
|
|
155
|
+
matches = CONFIG_CALL_RE.findall(text)
|
|
156
|
+
|
|
157
|
+
for match in matches:
|
|
158
|
+
# extremely naive kwarg parsing
|
|
159
|
+
parts = re.split(r",(?![^\[\]\(\)]*[\]\)])", match)
|
|
160
|
+
|
|
161
|
+
for part in parts:
|
|
162
|
+
if "=" not in part:
|
|
163
|
+
continue
|
|
164
|
+
key, value = part.split("=", 1)
|
|
165
|
+
claims.append(
|
|
166
|
+
PropertyClaim(
|
|
167
|
+
source_type="model.sql",
|
|
168
|
+
source_path=model_path,
|
|
169
|
+
name=key.strip(),
|
|
170
|
+
value=value.strip(),
|
|
171
|
+
kind="config",
|
|
172
|
+
)
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
return claims
|
|
176
|
+
|
|
177
|
+
def collect_model_claims(
|
|
178
|
+
model_path: Path,
|
|
179
|
+
project_path: Path,
|
|
180
|
+
) -> list[PropertyClaim]:
|
|
181
|
+
claims: list[PropertyClaim] = []
|
|
182
|
+
model_name = model_path.stem
|
|
183
|
+
|
|
184
|
+
# dbt_project.yml
|
|
185
|
+
claims.extend(collect_project_configs(model_path, project_path))
|
|
186
|
+
|
|
187
|
+
# schema.yml files discovered automatically
|
|
188
|
+
schema_files = find_schema_files(model_path, project_path)
|
|
189
|
+
for schema_file in schema_files:
|
|
190
|
+
claims.extend(
|
|
191
|
+
collect_schema_properties(schema_file, model_name)
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
# model.sql
|
|
195
|
+
claims.extend(collect_sql_configs(model_path))
|
|
196
|
+
|
|
197
|
+
return claims
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
print(
|
|
201
|
+
collect_model_claims(
|
|
202
|
+
Path('models/mart/users_daily_stat.sql'),
|
|
203
|
+
Path('.'),
|
|
204
|
+
)
|
|
205
|
+
)
|
|
206
|
+
|
dbtui/backend/model.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from jinja2 import Environment, Template
|
|
3
|
+
from jinja2.nodes import Call, Const
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
from ..common import DbtModelAbstract, NotWithinSubdirectoryException
|
|
7
|
+
|
|
8
|
+
from typing import TYPE_CHECKING, Iterable, Self
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from .project import DbtProject
|
|
11
|
+
from .property_claim import PropertyClaimAggregate
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class DbtModel(DbtModelAbstract):
|
|
15
|
+
file_name: str
|
|
16
|
+
file_path_full: Path
|
|
17
|
+
file_path_relative: Path
|
|
18
|
+
template: str
|
|
19
|
+
parsed_template: Template
|
|
20
|
+
project: 'DbtProject'
|
|
21
|
+
property_claims: 'PropertyClaimAggregate | None'
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def file_path_relative(self):
|
|
25
|
+
return self.file_path_full.relative_to(self.project.root_folder)
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def file_name(self):
|
|
29
|
+
return self.file_path_full.name
|
|
30
|
+
|
|
31
|
+
def __init__(self, file_path_full: Path, project: 'DbtProject'):
|
|
32
|
+
self.file_path_full = file_path_full
|
|
33
|
+
self.project = project
|
|
34
|
+
self.property_claims = None # Populated by DbtProject.collect_property_claims()
|
|
35
|
+
with open(file_path_full, 'r', encoding='utf-8') as f:
|
|
36
|
+
self.template = f.read()
|
|
37
|
+
self.parsed_template = Environment().parse(self.template)
|
|
38
|
+
|
|
39
|
+
def _find_calls(self, macro_name: str):
|
|
40
|
+
return [i for i in self.parsed_template.find_all(Call) if i.node.name == macro_name]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def name(self) -> str:
|
|
45
|
+
# by default dbt sets model name as its filename without extension
|
|
46
|
+
default_name = self.file_name.rpartition('.')[0]
|
|
47
|
+
# we need to figure out if the model has been renamed with config() macro, so let's find this macro in the model file
|
|
48
|
+
calls = self._find_calls('config')
|
|
49
|
+
|
|
50
|
+
# double config would be invalid
|
|
51
|
+
if len(calls) > 1:
|
|
52
|
+
logging.warn("Duplicated invocation of config() in %s" % self.file_path_relative)
|
|
53
|
+
return default_name
|
|
54
|
+
|
|
55
|
+
if not calls:
|
|
56
|
+
return default_name
|
|
57
|
+
else:
|
|
58
|
+
config: Call = calls[0]
|
|
59
|
+
kwargs = {item.key: item.value for item in config.kwargs}
|
|
60
|
+
return kwargs.get('name', Const(default_name)).value
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def children(self) -> list[Self]:
|
|
64
|
+
return sorted(list(self.project.graph.successors(self)), key=lambda n: n.name)
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def parents(self) -> Iterable[Self]:
|
|
68
|
+
return sorted(list(self.project.graph.predecessors(self)), key=lambda n: n.name)
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def text(self) -> str:
|
|
72
|
+
with open(self.file_path_full, 'r', encoding='utf-8') as f:
|
|
73
|
+
self.template = f.read()
|
|
74
|
+
return self.template
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def refs(self) -> list[str]:
|
|
79
|
+
result = []
|
|
80
|
+
for call in self._find_calls('ref'):
|
|
81
|
+
if len(call.args) != 1:
|
|
82
|
+
logging.warn(f"Invalid number of args to ref() in model {self.name}: should be one, but it's {len(call.args)}: {[arg.value for arg in call.args]}")
|
|
83
|
+
continue
|
|
84
|
+
result.append(call.args[0].value)
|
|
85
|
+
return result
|
|
86
|
+
|