trellis-datamodel 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trellis_datamodel-0.1.0/PKG-INFO +301 -0
- trellis_datamodel-0.1.0/README.md +265 -0
- trellis_datamodel-0.1.0/pyproject.toml +71 -0
- trellis_datamodel-0.1.0/setup.cfg +4 -0
- trellis_datamodel-0.1.0/trellis_datamodel/__init__.py +4 -0
- trellis_datamodel-0.1.0/trellis_datamodel/adapters/__init__.py +47 -0
- trellis_datamodel-0.1.0/trellis_datamodel/adapters/base.py +137 -0
- trellis_datamodel-0.1.0/trellis_datamodel/adapters/dbt_core.py +565 -0
- trellis_datamodel-0.1.0/trellis_datamodel/cli.py +107 -0
- trellis_datamodel-0.1.0/trellis_datamodel/config.py +193 -0
- trellis_datamodel-0.1.0/trellis_datamodel/models/__init__.py +13 -0
- trellis_datamodel-0.1.0/trellis_datamodel/models/schemas.py +27 -0
- trellis_datamodel-0.1.0/trellis_datamodel/routes/__init__.py +11 -0
- trellis_datamodel-0.1.0/trellis_datamodel/routes/data_model.py +222 -0
- trellis_datamodel-0.1.0/trellis_datamodel/routes/manifest.py +59 -0
- trellis_datamodel-0.1.0/trellis_datamodel/routes/schema.py +181 -0
- trellis_datamodel-0.1.0/trellis_datamodel/server.py +67 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/env.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/assets/0.x7WqvLKf.css +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/assets/2.DLAp_5AW.css +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/Bpg6QCuO.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/ByPwy4Kv.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/C-EkzuW7.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/C5F_HkED.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/Cp6tXdVM.js +2 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/DVImN82k.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/DfmIHBKM.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/G7wgDTPn.js +2 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/NHoXZvRd.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/entry/app.DzI2sypz.js +2 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/entry/start.dOhzzK6O.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/nodes/0.TGY9TRSm.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/nodes/1.vl5CW0xB.js +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/nodes/2.CgXmEV1Q.js +27 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/_app/version.json +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/index.html +40 -0
- trellis_datamodel-0.1.0/trellis_datamodel/static/robots.txt +3 -0
- trellis_datamodel-0.1.0/trellis_datamodel/tests/__init__.py +2 -0
- trellis_datamodel-0.1.0/trellis_datamodel/tests/conftest.py +131 -0
- trellis_datamodel-0.1.0/trellis_datamodel/tests/test_data_model.py +151 -0
- trellis_datamodel-0.1.0/trellis_datamodel/tests/test_dbt_schema.py +234 -0
- trellis_datamodel-0.1.0/trellis_datamodel/tests/test_manifest.py +72 -0
- trellis_datamodel-0.1.0/trellis_datamodel/tests/test_yaml_handler.py +228 -0
- trellis_datamodel-0.1.0/trellis_datamodel/utils/__init__.py +2 -0
- trellis_datamodel-0.1.0/trellis_datamodel/utils/yaml_handler.py +309 -0
- trellis_datamodel-0.1.0/trellis_datamodel.egg-info/PKG-INFO +301 -0
- trellis_datamodel-0.1.0/trellis_datamodel.egg-info/SOURCES.txt +49 -0
- trellis_datamodel-0.1.0/trellis_datamodel.egg-info/dependency_links.txt +1 -0
- trellis_datamodel-0.1.0/trellis_datamodel.egg-info/entry_points.txt +2 -0
- trellis_datamodel-0.1.0/trellis_datamodel.egg-info/requires.txt +19 -0
- trellis_datamodel-0.1.0/trellis_datamodel.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: trellis-datamodel
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Visual data model editor for dbt projects
|
|
5
|
+
Author: Tim Hiebenthal
|
|
6
|
+
Project-URL: Homepage, https://github.com/timhiebenthal/trellis-datamodel
|
|
7
|
+
Project-URL: Repository, https://github.com/timhiebenthal/trellis-datamodel
|
|
8
|
+
Project-URL: Issues, https://github.com/timhiebenthal/trellis-datamodel/issues
|
|
9
|
+
Keywords: dbt,data-modeling,erd,data-engineering,analytics-engineering,visualization,schema
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Database
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: dbt-core<2.0,>=1.6
|
|
20
|
+
Requires-Dist: fastapi>=0.121.3
|
|
21
|
+
Requires-Dist: python-dotenv>=1.2.1
|
|
22
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
23
|
+
Requires-Dist: ruamel.yaml>=0.18.0
|
|
24
|
+
Requires-Dist: typer>=0.9.0
|
|
25
|
+
Requires-Dist: uvicorn>=0.38.0
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
28
|
+
Requires-Dist: httpx>=0.27.0; extra == "dev"
|
|
29
|
+
Provides-Extra: examples
|
|
30
|
+
Requires-Dist: dbt-duckdb==1.10; extra == "examples"
|
|
31
|
+
Requires-Dist: duckdb>=1.4.2; extra == "examples"
|
|
32
|
+
Requires-Dist: marimo>=0.18.0; extra == "examples"
|
|
33
|
+
Requires-Dist: nba-api>=1.11.3; extra == "examples"
|
|
34
|
+
Requires-Dist: pandas>=2.3.3; extra == "examples"
|
|
35
|
+
Requires-Dist: tqdm>=4.67.1; extra == "examples"
|
|
36
|
+
|
|
37
|
+
# Trellis Data
|
|
38
|
+
|
|
39
|
+
A local-first tool to bridge Conceptual Data Modeling and Logical dbt Implementation.
|
|
40
|
+
|
|
41
|
+
## Motivation
|
|
42
|
+
|
|
43
|
+
**Current dbt workflow pains:**
|
|
44
|
+
- ERD diagrams live in separate tools (Lucidchart, draw.io) and quickly become stale
|
|
45
|
+
- Data transformations are done isolated from the conceptual data model.
|
|
46
|
+
- No single view connecting business concepts to logical schema
|
|
47
|
+
- Stakeholders can't easily understand model structure without technical context
|
|
48
|
+
|
|
49
|
+
**How Trellis helps:**
|
|
50
|
+
- Visual data model that stays in sync — reads directly from `manifest.json` / `catalog.json`
|
|
51
|
+
- Sketch entities and with their fields and auto-generate schema.yml's for dbt
|
|
52
|
+
- Draw relationships on canvas → auto-generates dbt `relationships` tests
|
|
53
|
+
- Two views: **Conceptual** (entity names, descriptions) and **Logical** (columns, types, materializations) to jump between high-level architect and execution-view.
|
|
54
|
+
|
|
55
|
+
**Two Ways of getting started**
|
|
56
|
+
- Greenfield: draft entities and fields before writing SQL, then sync to dbt YAML
|
|
57
|
+
- Brownfield: document your existing data model by loading existing dbt models and utilize relationship tests to infer links
|
|
58
|
+
|
|
59
|
+
## Prerequisites
|
|
60
|
+
- **Node.js 22+ (or 20.19+) & npm**
|
|
61
|
+
- Recommended: Use [nvm](https://github.com/nvm-sh/nvm) to install a compatible version (e.g., `nvm install 22`).
|
|
62
|
+
- Note: System packages (`apt-get`) may be too old for the frontend dependencies.
|
|
63
|
+
- A `.nvmrc` file is included; run `nvm use` to switch to the correct version automatically.
|
|
64
|
+
- **Python 3.12+ & [uv](https://github.com/astral-sh/uv)**
|
|
65
|
+
- Install uv via `curl -LsSf https://astral.sh/uv/install.sh | sh` and ensure it’s on your `$PATH`.
|
|
66
|
+
- **Make** (optional) for convenience targets defined in the `Makefile`.
|
|
67
|
+
|
|
68
|
+
## Installation
|
|
69
|
+
|
|
70
|
+
### Install from GitHub (for now)
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install git+https://github.com/yourorg/trellis-datamodel.git
|
|
74
|
+
# or with uv
|
|
75
|
+
uv pip install git+https://github.com/yourorg/trellis-datamodel.git
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### Install from local development
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
# Clone the repository
|
|
82
|
+
git clone https://github.com/yourorg/trellis-datamodel.git
|
|
83
|
+
cd trellis-datamodel
|
|
84
|
+
|
|
85
|
+
# Install in editable mode
|
|
86
|
+
pip install -e .
|
|
87
|
+
# or with uv
|
|
88
|
+
uv pip install -e .
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Quick Start
|
|
92
|
+
|
|
93
|
+
1. **Navigate to your dbt project directory**
|
|
94
|
+
```bash
|
|
95
|
+
cd /path/to/your/dbt-project
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
2. **Create a `trellis.yml` config file** (or use `config.yml` as fallback)
|
|
99
|
+
```yaml
|
|
100
|
+
framework: dbt-core
|
|
101
|
+
dbt_project_path: "."
|
|
102
|
+
dbt_manifest_path: "target/manifest.json"
|
|
103
|
+
dbt_catalog_path: "target/catalog.json"
|
|
104
|
+
data_model_file: "data_model.yml"
|
|
105
|
+
dbt_model_paths: [] # Empty = include all models
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
3. **Start the server**
|
|
109
|
+
```bash
|
|
110
|
+
trellis-datamodel serve
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
The server will start on **http://localhost:8089** and automatically open your browser.
|
|
114
|
+
|
|
115
|
+
## Development Setup
|
|
116
|
+
|
|
117
|
+
For local development with hot reload:
|
|
118
|
+
|
|
119
|
+
## Install Dependencies
|
|
120
|
+
Run these once per machine (or when dependencies change).
|
|
121
|
+
|
|
122
|
+
1. **Backend**
|
|
123
|
+
```bash
|
|
124
|
+
cd backend
|
|
125
|
+
uv sync
|
|
126
|
+
```
|
|
127
|
+
2. **Frontend**
|
|
128
|
+
```bash
|
|
129
|
+
cd frontend
|
|
130
|
+
npm install
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
**Terminal 1 – Backend**
|
|
134
|
+
```bash
|
|
135
|
+
make backend
|
|
136
|
+
# or
|
|
137
|
+
trellis-datamodel serve
|
|
138
|
+
```
|
|
139
|
+
Backend serves the API at http://localhost:8089.
|
|
140
|
+
|
|
141
|
+
**Terminal 2 – Frontend**
|
|
142
|
+
```bash
|
|
143
|
+
make frontend
|
|
144
|
+
# or
|
|
145
|
+
cd frontend && npm run dev
|
|
146
|
+
```
|
|
147
|
+
Frontend runs at http://localhost:5173 (for development with hot reload).
|
|
148
|
+
|
|
149
|
+
## Building for Distribution
|
|
150
|
+
|
|
151
|
+
To build the package with bundled frontend:
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
make build-package
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
This will:
|
|
158
|
+
1. Build the frontend (`npm run build`)
|
|
159
|
+
2. Copy static files to `trellis_datamodel/static/`
|
|
160
|
+
3. Build the Python wheel (`uv build`)
|
|
161
|
+
|
|
162
|
+
The wheel will be in `dist/` and can be installed with `pip install dist/trellis_datamodel-*.whl`.
|
|
163
|
+
|
|
164
|
+
## CLI Options
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
trellis-datamodel serve [OPTIONS]
|
|
168
|
+
|
|
169
|
+
Options:
|
|
170
|
+
--port, -p INTEGER Port to run the server on [default: 8089]
|
|
171
|
+
--config, -c TEXT Path to config file (trellis.yml or config.yml)
|
|
172
|
+
--no-browser Don't open browser automatically
|
|
173
|
+
--help Show help message
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
## dbt Metadata
|
|
177
|
+
- Generate `manifest.json` and `catalog.json` by running `dbt docs generate` in your dbt project.
|
|
178
|
+
- The UI reads these artifacts to power the ERD modeller.
|
|
179
|
+
- Without these artifacts, the UI loads but shows no dbt models.
|
|
180
|
+
|
|
181
|
+
## Configuration
|
|
182
|
+
|
|
183
|
+
Create a `trellis.yml` file (or `config.yml` as fallback) in your dbt project directory to configure paths:
|
|
184
|
+
|
|
185
|
+
- `framework`: Transformation framework to use. Currently supported: `dbt-core`. Future: `dbt-fusion`, `sqlmesh`, `bruin`, `pydantic`. Defaults to `dbt-core`.
|
|
186
|
+
- `dbt_project_path`: Path to your dbt project directory (relative to `config.yml` or absolute). **Required**.
|
|
187
|
+
- `dbt_manifest_path`: Path to `manifest.json` (relative to `dbt_project_path` or absolute). Defaults to `target/manifest.json`.
|
|
188
|
+
- `dbt_catalog_path`: Path to `catalog.json` (relative to `dbt_project_path` or absolute). Defaults to `target/catalog.json`.
|
|
189
|
+
- `data_model_file`: Path where the data model YAML will be saved (relative to `dbt_project_path` or absolute). Defaults to `data_model.yml`.
|
|
190
|
+
- `dbt_model_paths`: List of path patterns to filter which dbt models are shown (e.g., `["3_core"]`). If empty, all models are included.
|
|
191
|
+
|
|
192
|
+
**Example `trellis.yml`:**
|
|
193
|
+
```yaml
|
|
194
|
+
framework: dbt-core
|
|
195
|
+
dbt_project_path: "./dbt_built"
|
|
196
|
+
dbt_manifest_path: "target/manifest.json"
|
|
197
|
+
dbt_catalog_path: "target/catalog.json"
|
|
198
|
+
data_model_file: "data_model.yml"
|
|
199
|
+
dbt_model_paths:
|
|
200
|
+
- "3_core"
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
## Testing
|
|
205
|
+
|
|
206
|
+
### Frontend
|
|
207
|
+
**Testing Libraries:**
|
|
208
|
+
The following testing libraries are defined in `package.json` under `devDependencies` and are automatically installed when you run `npm install`:
|
|
209
|
+
- [Vitest](https://vitest.dev/) (Unit testing)
|
|
210
|
+
- [Playwright](https://playwright.dev/) (End-to-End testing)
|
|
211
|
+
- [Testing Library](https://testing-library.com/) (DOM & Svelte testing utilities)
|
|
212
|
+
- [jsdom](https://github.com/jsdom/jsdom) (DOM environment)
|
|
213
|
+
|
|
214
|
+
> **Playwright system dependencies (Ubuntu/WSL2)**
|
|
215
|
+
>
|
|
216
|
+
> The browsers downloaded by Playwright need a handful of native libraries. Install them before running `npm run test:e2e`:
|
|
217
|
+
>
|
|
218
|
+
> ```bash
|
|
219
|
+
> sudo apt-get update && sudo apt-get install -y \
|
|
220
|
+
> libxcursor1 libxdamage1 libgtk-3-0 libpangocairo-1.0-0 libpango-1.0-0 \
|
|
221
|
+
> libatk1.0-0 libcairo-gobject2 libcairo2 libgdk-pixbuf-2.0-0 libasound2 \
|
|
222
|
+
> libnspr4 libnss3 libgbm1 libgles2-mesa libgtk-4-1 libgraphene-1.0-0 \
|
|
223
|
+
> libxslt1.1 libwoff2dec0 libvpx7 libevent-2.1-7 libopus0 \
|
|
224
|
+
> libgstallocators-1.0-0 libgstapp-1.0-0 libgstpbutils-1.0-0 libgstaudio-1.0-0 \
|
|
225
|
+
> libgsttag-1.0-0 libgstvideo-1.0-0 libgstgl-1.0-0 libgstcodecparsers-1.0-0 \
|
|
226
|
+
> libgstfft-1.0-0 libflite1 libflite1-plugins libwebpdemux2 libavif13 \
|
|
227
|
+
> libharfbuzz-icu0 libwebpmux3 libenchant-2-2 libsecret-1-0 libhyphen0 \
|
|
228
|
+
> libwayland-server0 libmanette-0.2-0 libx264-163
|
|
229
|
+
> ```
|
|
230
|
+
|
|
231
|
+
**Running Tests:**
|
|
232
|
+
|
|
233
|
+
The test suite has multiple levels to catch different types of issues:
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
cd frontend
|
|
237
|
+
|
|
238
|
+
# Quick smoke test (catches 500 errors, runtime crashes, ESM issues)
|
|
239
|
+
# Fastest way to verify the app loads without errors
|
|
240
|
+
npm run test:smoke
|
|
241
|
+
|
|
242
|
+
# TypeScript/compilation check
|
|
243
|
+
npm run check
|
|
244
|
+
|
|
245
|
+
# Unit tests
|
|
246
|
+
npm run test:unit
|
|
247
|
+
|
|
248
|
+
# E2E tests (includes smoke test + full test suite)
|
|
249
|
+
# Note: Requires backend running with test data (see Test Data Isolation below)
|
|
250
|
+
npm run test:e2e
|
|
251
|
+
|
|
252
|
+
# Run all tests (check + smoke + unit + e2e)
|
|
253
|
+
npm run test
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
**Test Levels:**
|
|
257
|
+
1. **`npm run check`** - TypeScript compilation errors
|
|
258
|
+
2. **`npm run test:smoke`** - Runtime errors (500s, console errors, ESM issues) - **catches app crashes**
|
|
259
|
+
3. **`npm run test:unit`** - Unit tests with Vitest
|
|
260
|
+
4. **`npm run test:e2e`** - Full E2E tests with Playwright
|
|
261
|
+
|
|
262
|
+
**Using Makefile:**
|
|
263
|
+
```bash
|
|
264
|
+
# From project root
|
|
265
|
+
make test-smoke # Quick smoke test
|
|
266
|
+
make test-check # TypeScript check
|
|
267
|
+
make test-unit # Unit tests
|
|
268
|
+
make test-e2e # E2E tests (auto-starts backend with test data)
|
|
269
|
+
make test-all # All tests
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
**Test Data Isolation:**
|
|
273
|
+
E2E tests use a separate test data file (`frontend/tests/test_data_model.yml`) to avoid polluting your production data model. **Playwright automatically starts the backend** with the correct environment variable, so you don't need to manage it manually.
|
|
274
|
+
|
|
275
|
+
```bash
|
|
276
|
+
# Just run E2E tests - backend starts automatically with test data
|
|
277
|
+
make test-e2e
|
|
278
|
+
# OR:
|
|
279
|
+
# cd frontend && npm run test:e2e
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
The test data file is automatically cleaned before and after test runs via Playwright's `globalSetup` and `globalTeardown`. Your production `data_model.yml` remains untouched.
|
|
283
|
+
|
|
284
|
+
### Backend
|
|
285
|
+
**Testing Libraries:**
|
|
286
|
+
The following testing libraries are defined in `pyproject.toml` under `[project.optional-dependencies]` in the `dev` group:
|
|
287
|
+
- [pytest](https://docs.pytest.org/) (Testing framework)
|
|
288
|
+
- [httpx](https://www.python-httpx.org/) (Async HTTP client for API testing)
|
|
289
|
+
|
|
290
|
+
**Installation:**
|
|
291
|
+
Unlike `npm`, `uv sync` does not install optional dependencies by default. To include the testing libraries, run:
|
|
292
|
+
```bash
|
|
293
|
+
cd backend
|
|
294
|
+
uv sync --extra dev
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
**Running Tests:**
|
|
298
|
+
```bash
|
|
299
|
+
cd backend
|
|
300
|
+
uv run pytest
|
|
301
|
+
```
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
# Trellis Data
|
|
2
|
+
|
|
3
|
+
A local-first tool to bridge Conceptual Data Modeling and Logical dbt Implementation.
|
|
4
|
+
|
|
5
|
+
## Motivation
|
|
6
|
+
|
|
7
|
+
**Current dbt workflow pains:**
|
|
8
|
+
- ERD diagrams live in separate tools (Lucidchart, draw.io) and quickly become stale
|
|
9
|
+
- Data transformations are done isolated from the conceptual data model.
|
|
10
|
+
- No single view connecting business concepts to logical schema
|
|
11
|
+
- Stakeholders can't easily understand model structure without technical context
|
|
12
|
+
|
|
13
|
+
**How Trellis helps:**
|
|
14
|
+
- Visual data model that stays in sync — reads directly from `manifest.json` / `catalog.json`
|
|
15
|
+
- Sketch entities and with their fields and auto-generate schema.yml's for dbt
|
|
16
|
+
- Draw relationships on canvas → auto-generates dbt `relationships` tests
|
|
17
|
+
- Two views: **Conceptual** (entity names, descriptions) and **Logical** (columns, types, materializations) to jump between high-level architect and execution-view.
|
|
18
|
+
|
|
19
|
+
**Two Ways of getting started**
|
|
20
|
+
- Greenfield: draft entities and fields before writing SQL, then sync to dbt YAML
|
|
21
|
+
- Brownfield: document your existing data model by loading existing dbt models and utilize relationship tests to infer links
|
|
22
|
+
|
|
23
|
+
## Prerequisites
|
|
24
|
+
- **Node.js 22+ (or 20.19+) & npm**
|
|
25
|
+
- Recommended: Use [nvm](https://github.com/nvm-sh/nvm) to install a compatible version (e.g., `nvm install 22`).
|
|
26
|
+
- Note: System packages (`apt-get`) may be too old for the frontend dependencies.
|
|
27
|
+
- A `.nvmrc` file is included; run `nvm use` to switch to the correct version automatically.
|
|
28
|
+
- **Python 3.12+ & [uv](https://github.com/astral-sh/uv)**
|
|
29
|
+
- Install uv via `curl -LsSf https://astral.sh/uv/install.sh | sh` and ensure it’s on your `$PATH`.
|
|
30
|
+
- **Make** (optional) for convenience targets defined in the `Makefile`.
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
### Install from GitHub (for now)
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install git+https://github.com/yourorg/trellis-datamodel.git
|
|
38
|
+
# or with uv
|
|
39
|
+
uv pip install git+https://github.com/yourorg/trellis-datamodel.git
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### Install from local development
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
# Clone the repository
|
|
46
|
+
git clone https://github.com/yourorg/trellis-datamodel.git
|
|
47
|
+
cd trellis-datamodel
|
|
48
|
+
|
|
49
|
+
# Install in editable mode
|
|
50
|
+
pip install -e .
|
|
51
|
+
# or with uv
|
|
52
|
+
uv pip install -e .
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Quick Start
|
|
56
|
+
|
|
57
|
+
1. **Navigate to your dbt project directory**
|
|
58
|
+
```bash
|
|
59
|
+
cd /path/to/your/dbt-project
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
2. **Create a `trellis.yml` config file** (or use `config.yml` as fallback)
|
|
63
|
+
```yaml
|
|
64
|
+
framework: dbt-core
|
|
65
|
+
dbt_project_path: "."
|
|
66
|
+
dbt_manifest_path: "target/manifest.json"
|
|
67
|
+
dbt_catalog_path: "target/catalog.json"
|
|
68
|
+
data_model_file: "data_model.yml"
|
|
69
|
+
dbt_model_paths: [] # Empty = include all models
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
3. **Start the server**
|
|
73
|
+
```bash
|
|
74
|
+
trellis-datamodel serve
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
The server will start on **http://localhost:8089** and automatically open your browser.
|
|
78
|
+
|
|
79
|
+
## Development Setup
|
|
80
|
+
|
|
81
|
+
For local development with hot reload:
|
|
82
|
+
|
|
83
|
+
## Install Dependencies
|
|
84
|
+
Run these once per machine (or when dependencies change).
|
|
85
|
+
|
|
86
|
+
1. **Backend**
|
|
87
|
+
```bash
|
|
88
|
+
cd backend
|
|
89
|
+
uv sync
|
|
90
|
+
```
|
|
91
|
+
2. **Frontend**
|
|
92
|
+
```bash
|
|
93
|
+
cd frontend
|
|
94
|
+
npm install
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
**Terminal 1 – Backend**
|
|
98
|
+
```bash
|
|
99
|
+
make backend
|
|
100
|
+
# or
|
|
101
|
+
trellis-datamodel serve
|
|
102
|
+
```
|
|
103
|
+
Backend serves the API at http://localhost:8089.
|
|
104
|
+
|
|
105
|
+
**Terminal 2 – Frontend**
|
|
106
|
+
```bash
|
|
107
|
+
make frontend
|
|
108
|
+
# or
|
|
109
|
+
cd frontend && npm run dev
|
|
110
|
+
```
|
|
111
|
+
Frontend runs at http://localhost:5173 (for development with hot reload).
|
|
112
|
+
|
|
113
|
+
## Building for Distribution
|
|
114
|
+
|
|
115
|
+
To build the package with bundled frontend:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
make build-package
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
This will:
|
|
122
|
+
1. Build the frontend (`npm run build`)
|
|
123
|
+
2. Copy static files to `trellis_datamodel/static/`
|
|
124
|
+
3. Build the Python wheel (`uv build`)
|
|
125
|
+
|
|
126
|
+
The wheel will be in `dist/` and can be installed with `pip install dist/trellis_datamodel-*.whl`.
|
|
127
|
+
|
|
128
|
+
## CLI Options
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
trellis-datamodel serve [OPTIONS]
|
|
132
|
+
|
|
133
|
+
Options:
|
|
134
|
+
--port, -p INTEGER Port to run the server on [default: 8089]
|
|
135
|
+
--config, -c TEXT Path to config file (trellis.yml or config.yml)
|
|
136
|
+
--no-browser Don't open browser automatically
|
|
137
|
+
--help Show help message
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## dbt Metadata
|
|
141
|
+
- Generate `manifest.json` and `catalog.json` by running `dbt docs generate` in your dbt project.
|
|
142
|
+
- The UI reads these artifacts to power the ERD modeller.
|
|
143
|
+
- Without these artifacts, the UI loads but shows no dbt models.
|
|
144
|
+
|
|
145
|
+
## Configuration
|
|
146
|
+
|
|
147
|
+
Create a `trellis.yml` file (or `config.yml` as fallback) in your dbt project directory to configure paths:
|
|
148
|
+
|
|
149
|
+
- `framework`: Transformation framework to use. Currently supported: `dbt-core`. Future: `dbt-fusion`, `sqlmesh`, `bruin`, `pydantic`. Defaults to `dbt-core`.
|
|
150
|
+
- `dbt_project_path`: Path to your dbt project directory (relative to `config.yml` or absolute). **Required**.
|
|
151
|
+
- `dbt_manifest_path`: Path to `manifest.json` (relative to `dbt_project_path` or absolute). Defaults to `target/manifest.json`.
|
|
152
|
+
- `dbt_catalog_path`: Path to `catalog.json` (relative to `dbt_project_path` or absolute). Defaults to `target/catalog.json`.
|
|
153
|
+
- `data_model_file`: Path where the data model YAML will be saved (relative to `dbt_project_path` or absolute). Defaults to `data_model.yml`.
|
|
154
|
+
- `dbt_model_paths`: List of path patterns to filter which dbt models are shown (e.g., `["3_core"]`). If empty, all models are included.
|
|
155
|
+
|
|
156
|
+
**Example `trellis.yml`:**
|
|
157
|
+
```yaml
|
|
158
|
+
framework: dbt-core
|
|
159
|
+
dbt_project_path: "./dbt_built"
|
|
160
|
+
dbt_manifest_path: "target/manifest.json"
|
|
161
|
+
dbt_catalog_path: "target/catalog.json"
|
|
162
|
+
data_model_file: "data_model.yml"
|
|
163
|
+
dbt_model_paths:
|
|
164
|
+
- "3_core"
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
## Testing
|
|
169
|
+
|
|
170
|
+
### Frontend
|
|
171
|
+
**Testing Libraries:**
|
|
172
|
+
The following testing libraries are defined in `package.json` under `devDependencies` and are automatically installed when you run `npm install`:
|
|
173
|
+
- [Vitest](https://vitest.dev/) (Unit testing)
|
|
174
|
+
- [Playwright](https://playwright.dev/) (End-to-End testing)
|
|
175
|
+
- [Testing Library](https://testing-library.com/) (DOM & Svelte testing utilities)
|
|
176
|
+
- [jsdom](https://github.com/jsdom/jsdom) (DOM environment)
|
|
177
|
+
|
|
178
|
+
> **Playwright system dependencies (Ubuntu/WSL2)**
|
|
179
|
+
>
|
|
180
|
+
> The browsers downloaded by Playwright need a handful of native libraries. Install them before running `npm run test:e2e`:
|
|
181
|
+
>
|
|
182
|
+
> ```bash
|
|
183
|
+
> sudo apt-get update && sudo apt-get install -y \
|
|
184
|
+
> libxcursor1 libxdamage1 libgtk-3-0 libpangocairo-1.0-0 libpango-1.0-0 \
|
|
185
|
+
> libatk1.0-0 libcairo-gobject2 libcairo2 libgdk-pixbuf-2.0-0 libasound2 \
|
|
186
|
+
> libnspr4 libnss3 libgbm1 libgles2-mesa libgtk-4-1 libgraphene-1.0-0 \
|
|
187
|
+
> libxslt1.1 libwoff2dec0 libvpx7 libevent-2.1-7 libopus0 \
|
|
188
|
+
> libgstallocators-1.0-0 libgstapp-1.0-0 libgstpbutils-1.0-0 libgstaudio-1.0-0 \
|
|
189
|
+
> libgsttag-1.0-0 libgstvideo-1.0-0 libgstgl-1.0-0 libgstcodecparsers-1.0-0 \
|
|
190
|
+
> libgstfft-1.0-0 libflite1 libflite1-plugins libwebpdemux2 libavif13 \
|
|
191
|
+
> libharfbuzz-icu0 libwebpmux3 libenchant-2-2 libsecret-1-0 libhyphen0 \
|
|
192
|
+
> libwayland-server0 libmanette-0.2-0 libx264-163
|
|
193
|
+
> ```
|
|
194
|
+
|
|
195
|
+
**Running Tests:**
|
|
196
|
+
|
|
197
|
+
The test suite has multiple levels to catch different types of issues:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
cd frontend
|
|
201
|
+
|
|
202
|
+
# Quick smoke test (catches 500 errors, runtime crashes, ESM issues)
|
|
203
|
+
# Fastest way to verify the app loads without errors
|
|
204
|
+
npm run test:smoke
|
|
205
|
+
|
|
206
|
+
# TypeScript/compilation check
|
|
207
|
+
npm run check
|
|
208
|
+
|
|
209
|
+
# Unit tests
|
|
210
|
+
npm run test:unit
|
|
211
|
+
|
|
212
|
+
# E2E tests (includes smoke test + full test suite)
|
|
213
|
+
# Note: Requires backend running with test data (see Test Data Isolation below)
|
|
214
|
+
npm run test:e2e
|
|
215
|
+
|
|
216
|
+
# Run all tests (check + smoke + unit + e2e)
|
|
217
|
+
npm run test
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
**Test Levels:**
|
|
221
|
+
1. **`npm run check`** - TypeScript compilation errors
|
|
222
|
+
2. **`npm run test:smoke`** - Runtime errors (500s, console errors, ESM issues) - **catches app crashes**
|
|
223
|
+
3. **`npm run test:unit`** - Unit tests with Vitest
|
|
224
|
+
4. **`npm run test:e2e`** - Full E2E tests with Playwright
|
|
225
|
+
|
|
226
|
+
**Using Makefile:**
|
|
227
|
+
```bash
|
|
228
|
+
# From project root
|
|
229
|
+
make test-smoke # Quick smoke test
|
|
230
|
+
make test-check # TypeScript check
|
|
231
|
+
make test-unit # Unit tests
|
|
232
|
+
make test-e2e # E2E tests (auto-starts backend with test data)
|
|
233
|
+
make test-all # All tests
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
**Test Data Isolation:**
|
|
237
|
+
E2E tests use a separate test data file (`frontend/tests/test_data_model.yml`) to avoid polluting your production data model. **Playwright automatically starts the backend** with the correct environment variable, so you don't need to manage it manually.
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
# Just run E2E tests - backend starts automatically with test data
|
|
241
|
+
make test-e2e
|
|
242
|
+
# OR:
|
|
243
|
+
# cd frontend && npm run test:e2e
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
The test data file is automatically cleaned before and after test runs via Playwright's `globalSetup` and `globalTeardown`. Your production `data_model.yml` remains untouched.
|
|
247
|
+
|
|
248
|
+
### Backend
|
|
249
|
+
**Testing Libraries:**
|
|
250
|
+
The following testing libraries are defined in `pyproject.toml` under `[project.optional-dependencies]` in the `dev` group:
|
|
251
|
+
- [pytest](https://docs.pytest.org/) (Testing framework)
|
|
252
|
+
- [httpx](https://www.python-httpx.org/) (Async HTTP client for API testing)
|
|
253
|
+
|
|
254
|
+
**Installation:**
|
|
255
|
+
Unlike `npm`, `uv sync` does not install optional dependencies by default. To include the testing libraries, run:
|
|
256
|
+
```bash
|
|
257
|
+
cd backend
|
|
258
|
+
uv sync --extra dev
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
**Running Tests:**
|
|
262
|
+
```bash
|
|
263
|
+
cd backend
|
|
264
|
+
uv run pytest
|
|
265
|
+
```
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "trellis-datamodel"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Visual data model editor for dbt projects"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
authors = [
|
|
12
|
+
{name = "Tim Hiebenthal"},
|
|
13
|
+
]
|
|
14
|
+
keywords = ["dbt", "data-modeling", "erd", "data-engineering", "analytics-engineering", "visualization", "schema"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Topic :: Database",
|
|
22
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"dbt-core>=1.6,<2.0",
|
|
26
|
+
"fastapi>=0.121.3",
|
|
27
|
+
"python-dotenv>=1.2.1",
|
|
28
|
+
"pyyaml>=6.0.3",
|
|
29
|
+
"ruamel.yaml>=0.18.0",
|
|
30
|
+
"typer>=0.9.0",
|
|
31
|
+
"uvicorn>=0.38.0",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/timhiebenthal/trellis-datamodel"
|
|
36
|
+
Repository = "https://github.com/timhiebenthal/trellis-datamodel"
|
|
37
|
+
Issues = "https://github.com/timhiebenthal/trellis-datamodel/issues"
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
trellis = "trellis_datamodel.cli:app"
|
|
41
|
+
|
|
42
|
+
[project.optional-dependencies]
|
|
43
|
+
dev = [
|
|
44
|
+
"pytest>=8.0.0",
|
|
45
|
+
"httpx>=0.27.0",
|
|
46
|
+
]
|
|
47
|
+
examples = [
|
|
48
|
+
"dbt-duckdb==1.10",
|
|
49
|
+
"duckdb>=1.4.2",
|
|
50
|
+
"marimo>=0.18.0",
|
|
51
|
+
"nba-api>=1.11.3",
|
|
52
|
+
"pandas>=2.3.3",
|
|
53
|
+
"tqdm>=4.67.1",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
[tool.setuptools.packages.find]
|
|
57
|
+
where = ["."]
|
|
58
|
+
include = ["trellis_datamodel*"]
|
|
59
|
+
|
|
60
|
+
[tool.setuptools.package-data]
|
|
61
|
+
trellis_datamodel = ["static/**/*"]
|
|
62
|
+
|
|
63
|
+
[tool.pytest.ini_options]
|
|
64
|
+
testpaths = ["trellis_datamodel/tests"]
|
|
65
|
+
pythonpath = ["."]
|
|
66
|
+
|
|
67
|
+
[dependency-groups]
|
|
68
|
+
dev = [
|
|
69
|
+
"httpx>=0.28.1",
|
|
70
|
+
"pytest>=9.0.1",
|
|
71
|
+
]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Adapter factory for transformation framework integrations.
|
|
3
|
+
|
|
4
|
+
Provides get_adapter() to instantiate the appropriate adapter based on config.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from typing import Union
|
|
8
|
+
|
|
9
|
+
from trellis_datamodel.config import (
|
|
10
|
+
FRAMEWORK,
|
|
11
|
+
MANIFEST_PATH,
|
|
12
|
+
CATALOG_PATH,
|
|
13
|
+
DBT_PROJECT_PATH,
|
|
14
|
+
DATA_MODEL_PATH,
|
|
15
|
+
DBT_MODEL_PATHS,
|
|
16
|
+
)
|
|
17
|
+
from .base import TransformationAdapter
|
|
18
|
+
from .dbt_core import DbtCoreAdapter
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_adapter() -> Union[DbtCoreAdapter, TransformationAdapter]:
|
|
22
|
+
"""
|
|
23
|
+
Get the appropriate adapter based on the configured framework.
|
|
24
|
+
|
|
25
|
+
Returns:
|
|
26
|
+
An adapter instance implementing TransformationAdapter.
|
|
27
|
+
|
|
28
|
+
Raises:
|
|
29
|
+
ValueError: If the configured framework is not supported.
|
|
30
|
+
"""
|
|
31
|
+
if FRAMEWORK == "dbt-core":
|
|
32
|
+
return DbtCoreAdapter(
|
|
33
|
+
manifest_path=MANIFEST_PATH,
|
|
34
|
+
catalog_path=CATALOG_PATH,
|
|
35
|
+
project_path=DBT_PROJECT_PATH,
|
|
36
|
+
data_model_path=DATA_MODEL_PATH,
|
|
37
|
+
model_paths=DBT_MODEL_PATHS,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
raise ValueError(
|
|
41
|
+
f"Unknown framework: {FRAMEWORK}. "
|
|
42
|
+
f"Supported frameworks: dbt-core"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
__all__ = ["get_adapter", "TransformationAdapter", "DbtCoreAdapter"]
|
|
47
|
+
|