trellis-datamodel 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. trellis_datamodel-0.1.0/PKG-INFO +301 -0
  2. trellis_datamodel-0.1.0/README.md +265 -0
  3. trellis_datamodel-0.1.0/pyproject.toml +71 -0
  4. trellis_datamodel-0.1.0/setup.cfg +4 -0
  5. trellis_datamodel-0.1.0/trellis_datamodel/__init__.py +4 -0
  6. trellis_datamodel-0.1.0/trellis_datamodel/adapters/__init__.py +47 -0
  7. trellis_datamodel-0.1.0/trellis_datamodel/adapters/base.py +137 -0
  8. trellis_datamodel-0.1.0/trellis_datamodel/adapters/dbt_core.py +565 -0
  9. trellis_datamodel-0.1.0/trellis_datamodel/cli.py +107 -0
  10. trellis_datamodel-0.1.0/trellis_datamodel/config.py +193 -0
  11. trellis_datamodel-0.1.0/trellis_datamodel/models/__init__.py +13 -0
  12. trellis_datamodel-0.1.0/trellis_datamodel/models/schemas.py +27 -0
  13. trellis_datamodel-0.1.0/trellis_datamodel/routes/__init__.py +11 -0
  14. trellis_datamodel-0.1.0/trellis_datamodel/routes/data_model.py +222 -0
  15. trellis_datamodel-0.1.0/trellis_datamodel/routes/manifest.py +59 -0
  16. trellis_datamodel-0.1.0/trellis_datamodel/routes/schema.py +181 -0
  17. trellis_datamodel-0.1.0/trellis_datamodel/server.py +67 -0
  18. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/env.js +1 -0
  19. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/assets/0.x7WqvLKf.css +1 -0
  20. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/assets/2.DLAp_5AW.css +1 -0
  21. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/Bpg6QCuO.js +1 -0
  22. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/ByPwy4Kv.js +1 -0
  23. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/C-EkzuW7.js +1 -0
  24. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/C5F_HkED.js +1 -0
  25. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/Cp6tXdVM.js +2 -0
  26. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/DVImN82k.js +1 -0
  27. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/DfmIHBKM.js +1 -0
  28. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/G7wgDTPn.js +2 -0
  29. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/chunks/NHoXZvRd.js +1 -0
  30. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/entry/app.DzI2sypz.js +2 -0
  31. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/entry/start.dOhzzK6O.js +1 -0
  32. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/nodes/0.TGY9TRSm.js +1 -0
  33. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/nodes/1.vl5CW0xB.js +1 -0
  34. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/immutable/nodes/2.CgXmEV1Q.js +27 -0
  35. trellis_datamodel-0.1.0/trellis_datamodel/static/_app/version.json +1 -0
  36. trellis_datamodel-0.1.0/trellis_datamodel/static/index.html +40 -0
  37. trellis_datamodel-0.1.0/trellis_datamodel/static/robots.txt +3 -0
  38. trellis_datamodel-0.1.0/trellis_datamodel/tests/__init__.py +2 -0
  39. trellis_datamodel-0.1.0/trellis_datamodel/tests/conftest.py +131 -0
  40. trellis_datamodel-0.1.0/trellis_datamodel/tests/test_data_model.py +151 -0
  41. trellis_datamodel-0.1.0/trellis_datamodel/tests/test_dbt_schema.py +234 -0
  42. trellis_datamodel-0.1.0/trellis_datamodel/tests/test_manifest.py +72 -0
  43. trellis_datamodel-0.1.0/trellis_datamodel/tests/test_yaml_handler.py +228 -0
  44. trellis_datamodel-0.1.0/trellis_datamodel/utils/__init__.py +2 -0
  45. trellis_datamodel-0.1.0/trellis_datamodel/utils/yaml_handler.py +309 -0
  46. trellis_datamodel-0.1.0/trellis_datamodel.egg-info/PKG-INFO +301 -0
  47. trellis_datamodel-0.1.0/trellis_datamodel.egg-info/SOURCES.txt +49 -0
  48. trellis_datamodel-0.1.0/trellis_datamodel.egg-info/dependency_links.txt +1 -0
  49. trellis_datamodel-0.1.0/trellis_datamodel.egg-info/entry_points.txt +2 -0
  50. trellis_datamodel-0.1.0/trellis_datamodel.egg-info/requires.txt +19 -0
  51. trellis_datamodel-0.1.0/trellis_datamodel.egg-info/top_level.txt +1 -0
@@ -0,0 +1,301 @@
1
+ Metadata-Version: 2.4
2
+ Name: trellis-datamodel
3
+ Version: 0.1.0
4
+ Summary: Visual data model editor for dbt projects
5
+ Author: Tim Hiebenthal
6
+ Project-URL: Homepage, https://github.com/timhiebenthal/trellis-datamodel
7
+ Project-URL: Repository, https://github.com/timhiebenthal/trellis-datamodel
8
+ Project-URL: Issues, https://github.com/timhiebenthal/trellis-datamodel/issues
9
+ Keywords: dbt,data-modeling,erd,data-engineering,analytics-engineering,visualization,schema
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Topic :: Database
16
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
+ Requires-Python: >=3.11
18
+ Description-Content-Type: text/markdown
19
+ Requires-Dist: dbt-core<2.0,>=1.6
20
+ Requires-Dist: fastapi>=0.121.3
21
+ Requires-Dist: python-dotenv>=1.2.1
22
+ Requires-Dist: pyyaml>=6.0.3
23
+ Requires-Dist: ruamel.yaml>=0.18.0
24
+ Requires-Dist: typer>=0.9.0
25
+ Requires-Dist: uvicorn>=0.38.0
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
28
+ Requires-Dist: httpx>=0.27.0; extra == "dev"
29
+ Provides-Extra: examples
30
+ Requires-Dist: dbt-duckdb==1.10; extra == "examples"
31
+ Requires-Dist: duckdb>=1.4.2; extra == "examples"
32
+ Requires-Dist: marimo>=0.18.0; extra == "examples"
33
+ Requires-Dist: nba-api>=1.11.3; extra == "examples"
34
+ Requires-Dist: pandas>=2.3.3; extra == "examples"
35
+ Requires-Dist: tqdm>=4.67.1; extra == "examples"
36
+
37
+ # Trellis Data
38
+
39
+ A local-first tool to bridge Conceptual Data Modeling and Logical dbt Implementation.
40
+
41
+ ## Motivation
42
+
43
+ **Current dbt workflow pains:**
44
+ - ERD diagrams live in separate tools (Lucidchart, draw.io) and quickly become stale
45
+ - Data transformations are done isolated from the conceptual data model.
46
+ - No single view connecting business concepts to logical schema
47
+ - Stakeholders can't easily understand model structure without technical context
48
+
49
+ **How Trellis helps:**
50
+ - Visual data model that stays in sync — reads directly from `manifest.json` / `catalog.json`
51
+ - Sketch entities and with their fields and auto-generate schema.yml's for dbt
52
+ - Draw relationships on canvas → auto-generates dbt `relationships` tests
53
+ - Two views: **Conceptual** (entity names, descriptions) and **Logical** (columns, types, materializations) to jump between high-level architect and execution-view.
54
+
55
+ **Two Ways of getting started**
56
+ - Greenfield: draft entities and fields before writing SQL, then sync to dbt YAML
57
+ - Brownfield: document your existing data model by loading existing dbt models and utilize relationship tests to infer links
58
+
59
+ ## Prerequisites
60
+ - **Node.js 22+ (or 20.19+) & npm**
61
+ - Recommended: Use [nvm](https://github.com/nvm-sh/nvm) to install a compatible version (e.g., `nvm install 22`).
62
+ - Note: System packages (`apt-get`) may be too old for the frontend dependencies.
63
+ - A `.nvmrc` file is included; run `nvm use` to switch to the correct version automatically.
64
+ - **Python 3.12+ & [uv](https://github.com/astral-sh/uv)**
65
+ - Install uv via `curl -LsSf https://astral.sh/uv/install.sh | sh` and ensure it’s on your `$PATH`.
66
+ - **Make** (optional) for convenience targets defined in the `Makefile`.
67
+
68
+ ## Installation
69
+
70
+ ### Install from GitHub (for now)
71
+
72
+ ```bash
73
+ pip install git+https://github.com/yourorg/trellis-datamodel.git
74
+ # or with uv
75
+ uv pip install git+https://github.com/yourorg/trellis-datamodel.git
76
+ ```
77
+
78
+ ### Install from local development
79
+
80
+ ```bash
81
+ # Clone the repository
82
+ git clone https://github.com/yourorg/trellis-datamodel.git
83
+ cd trellis-datamodel
84
+
85
+ # Install in editable mode
86
+ pip install -e .
87
+ # or with uv
88
+ uv pip install -e .
89
+ ```
90
+
91
+ ## Quick Start
92
+
93
+ 1. **Navigate to your dbt project directory**
94
+ ```bash
95
+ cd /path/to/your/dbt-project
96
+ ```
97
+
98
+ 2. **Create a `trellis.yml` config file** (or use `config.yml` as fallback)
99
+ ```yaml
100
+ framework: dbt-core
101
+ dbt_project_path: "."
102
+ dbt_manifest_path: "target/manifest.json"
103
+ dbt_catalog_path: "target/catalog.json"
104
+ data_model_file: "data_model.yml"
105
+ dbt_model_paths: [] # Empty = include all models
106
+ ```
107
+
108
+ 3. **Start the server**
109
+ ```bash
110
+ trellis-datamodel serve
111
+ ```
112
+
113
+ The server will start on **http://localhost:8089** and automatically open your browser.
114
+
115
+ ## Development Setup
116
+
117
+ For local development with hot reload:
118
+
119
+ ## Install Dependencies
120
+ Run these once per machine (or when dependencies change).
121
+
122
+ 1. **Backend**
123
+ ```bash
124
+ cd backend
125
+ uv sync
126
+ ```
127
+ 2. **Frontend**
128
+ ```bash
129
+ cd frontend
130
+ npm install
131
+ ```
132
+
133
+ **Terminal 1 – Backend**
134
+ ```bash
135
+ make backend
136
+ # or
137
+ trellis-datamodel serve
138
+ ```
139
+ Backend serves the API at http://localhost:8089.
140
+
141
+ **Terminal 2 – Frontend**
142
+ ```bash
143
+ make frontend
144
+ # or
145
+ cd frontend && npm run dev
146
+ ```
147
+ Frontend runs at http://localhost:5173 (for development with hot reload).
148
+
149
+ ## Building for Distribution
150
+
151
+ To build the package with bundled frontend:
152
+
153
+ ```bash
154
+ make build-package
155
+ ```
156
+
157
+ This will:
158
+ 1. Build the frontend (`npm run build`)
159
+ 2. Copy static files to `trellis_datamodel/static/`
160
+ 3. Build the Python wheel (`uv build`)
161
+
162
+ The wheel will be in `dist/` and can be installed with `pip install dist/trellis_datamodel-*.whl`.
163
+
164
+ ## CLI Options
165
+
166
+ ```bash
167
+ trellis-datamodel serve [OPTIONS]
168
+
169
+ Options:
170
+ --port, -p INTEGER Port to run the server on [default: 8089]
171
+ --config, -c TEXT Path to config file (trellis.yml or config.yml)
172
+ --no-browser Don't open browser automatically
173
+ --help Show help message
174
+ ```
175
+
176
+ ## dbt Metadata
177
+ - Generate `manifest.json` and `catalog.json` by running `dbt docs generate` in your dbt project.
178
+ - The UI reads these artifacts to power the ERD modeller.
179
+ - Without these artifacts, the UI loads but shows no dbt models.
180
+
181
+ ## Configuration
182
+
183
+ Create a `trellis.yml` file (or `config.yml` as fallback) in your dbt project directory to configure paths:
184
+
185
+ - `framework`: Transformation framework to use. Currently supported: `dbt-core`. Future: `dbt-fusion`, `sqlmesh`, `bruin`, `pydantic`. Defaults to `dbt-core`.
186
+ - `dbt_project_path`: Path to your dbt project directory (relative to `config.yml` or absolute). **Required**.
187
+ - `dbt_manifest_path`: Path to `manifest.json` (relative to `dbt_project_path` or absolute). Defaults to `target/manifest.json`.
188
+ - `dbt_catalog_path`: Path to `catalog.json` (relative to `dbt_project_path` or absolute). Defaults to `target/catalog.json`.
189
+ - `data_model_file`: Path where the data model YAML will be saved (relative to `dbt_project_path` or absolute). Defaults to `data_model.yml`.
190
+ - `dbt_model_paths`: List of path patterns to filter which dbt models are shown (e.g., `["3_core"]`). If empty, all models are included.
191
+
192
+ **Example `trellis.yml`:**
193
+ ```yaml
194
+ framework: dbt-core
195
+ dbt_project_path: "./dbt_built"
196
+ dbt_manifest_path: "target/manifest.json"
197
+ dbt_catalog_path: "target/catalog.json"
198
+ data_model_file: "data_model.yml"
199
+ dbt_model_paths:
200
+ - "3_core"
201
+ ```
202
+
203
+
204
+ ## Testing
205
+
206
+ ### Frontend
207
+ **Testing Libraries:**
208
+ The following testing libraries are defined in `package.json` under `devDependencies` and are automatically installed when you run `npm install`:
209
+ - [Vitest](https://vitest.dev/) (Unit testing)
210
+ - [Playwright](https://playwright.dev/) (End-to-End testing)
211
+ - [Testing Library](https://testing-library.com/) (DOM & Svelte testing utilities)
212
+ - [jsdom](https://github.com/jsdom/jsdom) (DOM environment)
213
+
214
+ > **Playwright system dependencies (Ubuntu/WSL2)**
215
+ >
216
+ > The browsers downloaded by Playwright need a handful of native libraries. Install them before running `npm run test:e2e`:
217
+ >
218
+ > ```bash
219
+ > sudo apt-get update && sudo apt-get install -y \
220
+ > libxcursor1 libxdamage1 libgtk-3-0 libpangocairo-1.0-0 libpango-1.0-0 \
221
+ > libatk1.0-0 libcairo-gobject2 libcairo2 libgdk-pixbuf-2.0-0 libasound2 \
222
+ > libnspr4 libnss3 libgbm1 libgles2-mesa libgtk-4-1 libgraphene-1.0-0 \
223
+ > libxslt1.1 libwoff2dec0 libvpx7 libevent-2.1-7 libopus0 \
224
+ > libgstallocators-1.0-0 libgstapp-1.0-0 libgstpbutils-1.0-0 libgstaudio-1.0-0 \
225
+ > libgsttag-1.0-0 libgstvideo-1.0-0 libgstgl-1.0-0 libgstcodecparsers-1.0-0 \
226
+ > libgstfft-1.0-0 libflite1 libflite1-plugins libwebpdemux2 libavif13 \
227
+ > libharfbuzz-icu0 libwebpmux3 libenchant-2-2 libsecret-1-0 libhyphen0 \
228
+ > libwayland-server0 libmanette-0.2-0 libx264-163
229
+ > ```
230
+
231
+ **Running Tests:**
232
+
233
+ The test suite has multiple levels to catch different types of issues:
234
+
235
+ ```bash
236
+ cd frontend
237
+
238
+ # Quick smoke test (catches 500 errors, runtime crashes, ESM issues)
239
+ # Fastest way to verify the app loads without errors
240
+ npm run test:smoke
241
+
242
+ # TypeScript/compilation check
243
+ npm run check
244
+
245
+ # Unit tests
246
+ npm run test:unit
247
+
248
+ # E2E tests (includes smoke test + full test suite)
249
+ # Note: Requires backend running with test data (see Test Data Isolation below)
250
+ npm run test:e2e
251
+
252
+ # Run all tests (check + smoke + unit + e2e)
253
+ npm run test
254
+ ```
255
+
256
+ **Test Levels:**
257
+ 1. **`npm run check`** - TypeScript compilation errors
258
+ 2. **`npm run test:smoke`** - Runtime errors (500s, console errors, ESM issues) - **catches app crashes**
259
+ 3. **`npm run test:unit`** - Unit tests with Vitest
260
+ 4. **`npm run test:e2e`** - Full E2E tests with Playwright
261
+
262
+ **Using Makefile:**
263
+ ```bash
264
+ # From project root
265
+ make test-smoke # Quick smoke test
266
+ make test-check # TypeScript check
267
+ make test-unit # Unit tests
268
+ make test-e2e # E2E tests (auto-starts backend with test data)
269
+ make test-all # All tests
270
+ ```
271
+
272
+ **Test Data Isolation:**
273
+ E2E tests use a separate test data file (`frontend/tests/test_data_model.yml`) to avoid polluting your production data model. **Playwright automatically starts the backend** with the correct environment variable, so you don't need to manage it manually.
274
+
275
+ ```bash
276
+ # Just run E2E tests - backend starts automatically with test data
277
+ make test-e2e
278
+ # OR:
279
+ # cd frontend && npm run test:e2e
280
+ ```
281
+
282
+ The test data file is automatically cleaned before and after test runs via Playwright's `globalSetup` and `globalTeardown`. Your production `data_model.yml` remains untouched.
283
+
284
+ ### Backend
285
+ **Testing Libraries:**
286
+ The following testing libraries are defined in `pyproject.toml` under `[project.optional-dependencies]` in the `dev` group:
287
+ - [pytest](https://docs.pytest.org/) (Testing framework)
288
+ - [httpx](https://www.python-httpx.org/) (Async HTTP client for API testing)
289
+
290
+ **Installation:**
291
+ Unlike `npm`, `uv sync` does not install optional dependencies by default. To include the testing libraries, run:
292
+ ```bash
293
+ cd backend
294
+ uv sync --extra dev
295
+ ```
296
+
297
+ **Running Tests:**
298
+ ```bash
299
+ cd backend
300
+ uv run pytest
301
+ ```
@@ -0,0 +1,265 @@
1
+ # Trellis Data
2
+
3
+ A local-first tool to bridge Conceptual Data Modeling and Logical dbt Implementation.
4
+
5
+ ## Motivation
6
+
7
+ **Current dbt workflow pains:**
8
+ - ERD diagrams live in separate tools (Lucidchart, draw.io) and quickly become stale
9
+ - Data transformations are done isolated from the conceptual data model.
10
+ - No single view connecting business concepts to logical schema
11
+ - Stakeholders can't easily understand model structure without technical context
12
+
13
+ **How Trellis helps:**
14
+ - Visual data model that stays in sync — reads directly from `manifest.json` / `catalog.json`
15
+ - Sketch entities and with their fields and auto-generate schema.yml's for dbt
16
+ - Draw relationships on canvas → auto-generates dbt `relationships` tests
17
+ - Two views: **Conceptual** (entity names, descriptions) and **Logical** (columns, types, materializations) to jump between high-level architect and execution-view.
18
+
19
+ **Two Ways of getting started**
20
+ - Greenfield: draft entities and fields before writing SQL, then sync to dbt YAML
21
+ - Brownfield: document your existing data model by loading existing dbt models and utilize relationship tests to infer links
22
+
23
+ ## Prerequisites
24
+ - **Node.js 22+ (or 20.19+) & npm**
25
+ - Recommended: Use [nvm](https://github.com/nvm-sh/nvm) to install a compatible version (e.g., `nvm install 22`).
26
+ - Note: System packages (`apt-get`) may be too old for the frontend dependencies.
27
+ - A `.nvmrc` file is included; run `nvm use` to switch to the correct version automatically.
28
+ - **Python 3.12+ & [uv](https://github.com/astral-sh/uv)**
29
+ - Install uv via `curl -LsSf https://astral.sh/uv/install.sh | sh` and ensure it’s on your `$PATH`.
30
+ - **Make** (optional) for convenience targets defined in the `Makefile`.
31
+
32
+ ## Installation
33
+
34
+ ### Install from GitHub (for now)
35
+
36
+ ```bash
37
+ pip install git+https://github.com/yourorg/trellis-datamodel.git
38
+ # or with uv
39
+ uv pip install git+https://github.com/yourorg/trellis-datamodel.git
40
+ ```
41
+
42
+ ### Install from local development
43
+
44
+ ```bash
45
+ # Clone the repository
46
+ git clone https://github.com/yourorg/trellis-datamodel.git
47
+ cd trellis-datamodel
48
+
49
+ # Install in editable mode
50
+ pip install -e .
51
+ # or with uv
52
+ uv pip install -e .
53
+ ```
54
+
55
+ ## Quick Start
56
+
57
+ 1. **Navigate to your dbt project directory**
58
+ ```bash
59
+ cd /path/to/your/dbt-project
60
+ ```
61
+
62
+ 2. **Create a `trellis.yml` config file** (or use `config.yml` as fallback)
63
+ ```yaml
64
+ framework: dbt-core
65
+ dbt_project_path: "."
66
+ dbt_manifest_path: "target/manifest.json"
67
+ dbt_catalog_path: "target/catalog.json"
68
+ data_model_file: "data_model.yml"
69
+ dbt_model_paths: [] # Empty = include all models
70
+ ```
71
+
72
+ 3. **Start the server**
73
+ ```bash
74
+ trellis-datamodel serve
75
+ ```
76
+
77
+ The server will start on **http://localhost:8089** and automatically open your browser.
78
+
79
+ ## Development Setup
80
+
81
+ For local development with hot reload:
82
+
83
+ ## Install Dependencies
84
+ Run these once per machine (or when dependencies change).
85
+
86
+ 1. **Backend**
87
+ ```bash
88
+ cd backend
89
+ uv sync
90
+ ```
91
+ 2. **Frontend**
92
+ ```bash
93
+ cd frontend
94
+ npm install
95
+ ```
96
+
97
+ **Terminal 1 – Backend**
98
+ ```bash
99
+ make backend
100
+ # or
101
+ trellis-datamodel serve
102
+ ```
103
+ Backend serves the API at http://localhost:8089.
104
+
105
+ **Terminal 2 – Frontend**
106
+ ```bash
107
+ make frontend
108
+ # or
109
+ cd frontend && npm run dev
110
+ ```
111
+ Frontend runs at http://localhost:5173 (for development with hot reload).
112
+
113
+ ## Building for Distribution
114
+
115
+ To build the package with bundled frontend:
116
+
117
+ ```bash
118
+ make build-package
119
+ ```
120
+
121
+ This will:
122
+ 1. Build the frontend (`npm run build`)
123
+ 2. Copy static files to `trellis_datamodel/static/`
124
+ 3. Build the Python wheel (`uv build`)
125
+
126
+ The wheel will be in `dist/` and can be installed with `pip install dist/trellis_datamodel-*.whl`.
127
+
128
+ ## CLI Options
129
+
130
+ ```bash
131
+ trellis-datamodel serve [OPTIONS]
132
+
133
+ Options:
134
+ --port, -p INTEGER Port to run the server on [default: 8089]
135
+ --config, -c TEXT Path to config file (trellis.yml or config.yml)
136
+ --no-browser Don't open browser automatically
137
+ --help Show help message
138
+ ```
139
+
140
+ ## dbt Metadata
141
+ - Generate `manifest.json` and `catalog.json` by running `dbt docs generate` in your dbt project.
142
+ - The UI reads these artifacts to power the ERD modeller.
143
+ - Without these artifacts, the UI loads but shows no dbt models.
144
+
145
+ ## Configuration
146
+
147
+ Create a `trellis.yml` file (or `config.yml` as fallback) in your dbt project directory to configure paths:
148
+
149
+ - `framework`: Transformation framework to use. Currently supported: `dbt-core`. Future: `dbt-fusion`, `sqlmesh`, `bruin`, `pydantic`. Defaults to `dbt-core`.
150
+ - `dbt_project_path`: Path to your dbt project directory (relative to `config.yml` or absolute). **Required**.
151
+ - `dbt_manifest_path`: Path to `manifest.json` (relative to `dbt_project_path` or absolute). Defaults to `target/manifest.json`.
152
+ - `dbt_catalog_path`: Path to `catalog.json` (relative to `dbt_project_path` or absolute). Defaults to `target/catalog.json`.
153
+ - `data_model_file`: Path where the data model YAML will be saved (relative to `dbt_project_path` or absolute). Defaults to `data_model.yml`.
154
+ - `dbt_model_paths`: List of path patterns to filter which dbt models are shown (e.g., `["3_core"]`). If empty, all models are included.
155
+
156
+ **Example `trellis.yml`:**
157
+ ```yaml
158
+ framework: dbt-core
159
+ dbt_project_path: "./dbt_built"
160
+ dbt_manifest_path: "target/manifest.json"
161
+ dbt_catalog_path: "target/catalog.json"
162
+ data_model_file: "data_model.yml"
163
+ dbt_model_paths:
164
+ - "3_core"
165
+ ```
166
+
167
+
168
+ ## Testing
169
+
170
+ ### Frontend
171
+ **Testing Libraries:**
172
+ The following testing libraries are defined in `package.json` under `devDependencies` and are automatically installed when you run `npm install`:
173
+ - [Vitest](https://vitest.dev/) (Unit testing)
174
+ - [Playwright](https://playwright.dev/) (End-to-End testing)
175
+ - [Testing Library](https://testing-library.com/) (DOM & Svelte testing utilities)
176
+ - [jsdom](https://github.com/jsdom/jsdom) (DOM environment)
177
+
178
+ > **Playwright system dependencies (Ubuntu/WSL2)**
179
+ >
180
+ > The browsers downloaded by Playwright need a handful of native libraries. Install them before running `npm run test:e2e`:
181
+ >
182
+ > ```bash
183
+ > sudo apt-get update && sudo apt-get install -y \
184
+ > libxcursor1 libxdamage1 libgtk-3-0 libpangocairo-1.0-0 libpango-1.0-0 \
185
+ > libatk1.0-0 libcairo-gobject2 libcairo2 libgdk-pixbuf-2.0-0 libasound2 \
186
+ > libnspr4 libnss3 libgbm1 libgles2-mesa libgtk-4-1 libgraphene-1.0-0 \
187
+ > libxslt1.1 libwoff2dec0 libvpx7 libevent-2.1-7 libopus0 \
188
+ > libgstallocators-1.0-0 libgstapp-1.0-0 libgstpbutils-1.0-0 libgstaudio-1.0-0 \
189
+ > libgsttag-1.0-0 libgstvideo-1.0-0 libgstgl-1.0-0 libgstcodecparsers-1.0-0 \
190
+ > libgstfft-1.0-0 libflite1 libflite1-plugins libwebpdemux2 libavif13 \
191
+ > libharfbuzz-icu0 libwebpmux3 libenchant-2-2 libsecret-1-0 libhyphen0 \
192
+ > libwayland-server0 libmanette-0.2-0 libx264-163
193
+ > ```
194
+
195
+ **Running Tests:**
196
+
197
+ The test suite has multiple levels to catch different types of issues:
198
+
199
+ ```bash
200
+ cd frontend
201
+
202
+ # Quick smoke test (catches 500 errors, runtime crashes, ESM issues)
203
+ # Fastest way to verify the app loads without errors
204
+ npm run test:smoke
205
+
206
+ # TypeScript/compilation check
207
+ npm run check
208
+
209
+ # Unit tests
210
+ npm run test:unit
211
+
212
+ # E2E tests (includes smoke test + full test suite)
213
+ # Note: Requires backend running with test data (see Test Data Isolation below)
214
+ npm run test:e2e
215
+
216
+ # Run all tests (check + smoke + unit + e2e)
217
+ npm run test
218
+ ```
219
+
220
+ **Test Levels:**
221
+ 1. **`npm run check`** - TypeScript compilation errors
222
+ 2. **`npm run test:smoke`** - Runtime errors (500s, console errors, ESM issues) - **catches app crashes**
223
+ 3. **`npm run test:unit`** - Unit tests with Vitest
224
+ 4. **`npm run test:e2e`** - Full E2E tests with Playwright
225
+
226
+ **Using Makefile:**
227
+ ```bash
228
+ # From project root
229
+ make test-smoke # Quick smoke test
230
+ make test-check # TypeScript check
231
+ make test-unit # Unit tests
232
+ make test-e2e # E2E tests (auto-starts backend with test data)
233
+ make test-all # All tests
234
+ ```
235
+
236
+ **Test Data Isolation:**
237
+ E2E tests use a separate test data file (`frontend/tests/test_data_model.yml`) to avoid polluting your production data model. **Playwright automatically starts the backend** with the correct environment variable, so you don't need to manage it manually.
238
+
239
+ ```bash
240
+ # Just run E2E tests - backend starts automatically with test data
241
+ make test-e2e
242
+ # OR:
243
+ # cd frontend && npm run test:e2e
244
+ ```
245
+
246
+ The test data file is automatically cleaned before and after test runs via Playwright's `globalSetup` and `globalTeardown`. Your production `data_model.yml` remains untouched.
247
+
248
+ ### Backend
249
+ **Testing Libraries:**
250
+ The following testing libraries are defined in `pyproject.toml` under `[project.optional-dependencies]` in the `dev` group:
251
+ - [pytest](https://docs.pytest.org/) (Testing framework)
252
+ - [httpx](https://www.python-httpx.org/) (Async HTTP client for API testing)
253
+
254
+ **Installation:**
255
+ Unlike `npm`, `uv sync` does not install optional dependencies by default. To include the testing libraries, run:
256
+ ```bash
257
+ cd backend
258
+ uv sync --extra dev
259
+ ```
260
+
261
+ **Running Tests:**
262
+ ```bash
263
+ cd backend
264
+ uv run pytest
265
+ ```
@@ -0,0 +1,71 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "trellis-datamodel"
7
+ version = "0.1.0"
8
+ description = "Visual data model editor for dbt projects"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ authors = [
12
+ {name = "Tim Hiebenthal"},
13
+ ]
14
+ keywords = ["dbt", "data-modeling", "erd", "data-engineering", "analytics-engineering", "visualization", "schema"]
15
+ classifiers = [
16
+ "Development Status :: 4 - Beta",
17
+ "Intended Audience :: Developers",
18
+ "Programming Language :: Python :: 3",
19
+ "Programming Language :: Python :: 3.11",
20
+ "Programming Language :: Python :: 3.12",
21
+ "Topic :: Database",
22
+ "Topic :: Software Development :: Libraries :: Python Modules",
23
+ ]
24
+ dependencies = [
25
+ "dbt-core>=1.6,<2.0",
26
+ "fastapi>=0.121.3",
27
+ "python-dotenv>=1.2.1",
28
+ "pyyaml>=6.0.3",
29
+ "ruamel.yaml>=0.18.0",
30
+ "typer>=0.9.0",
31
+ "uvicorn>=0.38.0",
32
+ ]
33
+
34
+ [project.urls]
35
+ Homepage = "https://github.com/timhiebenthal/trellis-datamodel"
36
+ Repository = "https://github.com/timhiebenthal/trellis-datamodel"
37
+ Issues = "https://github.com/timhiebenthal/trellis-datamodel/issues"
38
+
39
+ [project.scripts]
40
+ trellis = "trellis_datamodel.cli:app"
41
+
42
+ [project.optional-dependencies]
43
+ dev = [
44
+ "pytest>=8.0.0",
45
+ "httpx>=0.27.0",
46
+ ]
47
+ examples = [
48
+ "dbt-duckdb==1.10",
49
+ "duckdb>=1.4.2",
50
+ "marimo>=0.18.0",
51
+ "nba-api>=1.11.3",
52
+ "pandas>=2.3.3",
53
+ "tqdm>=4.67.1",
54
+ ]
55
+
56
+ [tool.setuptools.packages.find]
57
+ where = ["."]
58
+ include = ["trellis_datamodel*"]
59
+
60
+ [tool.setuptools.package-data]
61
+ trellis_datamodel = ["static/**/*"]
62
+
63
+ [tool.pytest.ini_options]
64
+ testpaths = ["trellis_datamodel/tests"]
65
+ pythonpath = ["."]
66
+
67
+ [dependency-groups]
68
+ dev = [
69
+ "httpx>=0.28.1",
70
+ "pytest>=9.0.1",
71
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,4 @@
1
+ """Trellis Data - Visual data model editor for dbt projects."""
2
+
3
+ __version__ = "0.1.0"
4
+
@@ -0,0 +1,47 @@
1
+ """
2
+ Adapter factory for transformation framework integrations.
3
+
4
+ Provides get_adapter() to instantiate the appropriate adapter based on config.
5
+ """
6
+
7
+ from typing import Union
8
+
9
+ from trellis_datamodel.config import (
10
+ FRAMEWORK,
11
+ MANIFEST_PATH,
12
+ CATALOG_PATH,
13
+ DBT_PROJECT_PATH,
14
+ DATA_MODEL_PATH,
15
+ DBT_MODEL_PATHS,
16
+ )
17
+ from .base import TransformationAdapter
18
+ from .dbt_core import DbtCoreAdapter
19
+
20
+
21
+ def get_adapter() -> Union[DbtCoreAdapter, TransformationAdapter]:
22
+ """
23
+ Get the appropriate adapter based on the configured framework.
24
+
25
+ Returns:
26
+ An adapter instance implementing TransformationAdapter.
27
+
28
+ Raises:
29
+ ValueError: If the configured framework is not supported.
30
+ """
31
+ if FRAMEWORK == "dbt-core":
32
+ return DbtCoreAdapter(
33
+ manifest_path=MANIFEST_PATH,
34
+ catalog_path=CATALOG_PATH,
35
+ project_path=DBT_PROJECT_PATH,
36
+ data_model_path=DATA_MODEL_PATH,
37
+ model_paths=DBT_MODEL_PATHS,
38
+ )
39
+
40
+ raise ValueError(
41
+ f"Unknown framework: {FRAMEWORK}. "
42
+ f"Supported frameworks: dbt-core"
43
+ )
44
+
45
+
46
+ __all__ = ["get_adapter", "TransformationAdapter", "DbtCoreAdapter"]
47
+