trellis-datamodel 0.3.3__py3-none-any.whl

trellis_datamodel/utils/yaml_handler.py

@@ -0,0 +1,365 @@
+"""
+YAML Handler for round-trip editing of DBT schema files.
+Preserves comments, formatting, and structure while allowing updates.
+"""
+
+import os
+from typing import Dict, List, Optional, Any
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
+
+class YamlHandler:
+    """Handler for reading and writing DBT schema YAML files with round-trip capabilities."""
+
+    def __init__(self):
+        self.yaml = YAML()
+        self.yaml.preserve_quotes = True
+        self.yaml.default_flow_style = False
+        self.yaml.width = 4096  # Prevent unwanted line wrapping
+        # Match the two-space sequence indentation seen in existing dbt yml files
+        self.yaml.indent(mapping=2, sequence=2, offset=0)
+        # Disallow indentless sequences so list items are indented under their parent keys
+        self.yaml.indentless_sequences = False
+
+    def load_file(self, file_path: str) -> Optional[Dict]:
+        """
+        Load a YAML file with round-trip capabilities.
+
+        Args:
+            file_path: Path to the YAML file
+
+        Returns:
+            Parsed YAML content or None if file doesn't exist
+        """
+        if not os.path.exists(file_path):
+            return None
+
+        try:
+            with open(file_path, "r") as f:
+                return self.yaml.load(f)
+        except Exception as e:
+            print(f"Error loading YAML file {file_path}: {e}")
+            return None
+
+    def save_file(self, file_path: str, data: Dict) -> None:
+        """
+        Save YAML data to a file with round-trip preservation.
+
+        Args:
+            file_path: Path to the YAML file
+            data: YAML data to save
+        """
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+        # Write atomically using a temp file
+        temp_path = f"{file_path}.tmp"
+        try:
+            with open(temp_path, "w") as f:
+                self.yaml.dump(data, f)
+            os.replace(temp_path, file_path)
+        except Exception as e:
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+            raise e
+
+    def find_model(self, data: Dict, model_name: str) -> Optional[CommentedMap]:
+        """
+        Find a specific model entry in the YAML data.
+
+        Args:
+            data: Parsed YAML data
+            model_name: Name of the model to find
+
+        Returns:
+            Model entry or None if not found
+        """
+        if not data or "models" not in data:
+            return None
+
+        models = data.get("models", [])
+        for model in models:
+            if model.get("name") == model_name:
+                return model
+
+        return None
+
+    def ensure_model(self, data: Dict, model_name: str) -> CommentedMap:
+        """
+        Ensure a model entry exists in the YAML data, creating it if necessary.
+        Preserve existing tags if present.
+
+        Args:
+            data: Parsed YAML data
+            model_name: Name of the model
+
+        Returns:
+            Model entry (existing or newly created)
+        """
+        if "version" not in data:
+            data["version"] = 2
+
+        if "models" not in data:
+            data["models"] = CommentedSeq()
+
+        model = self.find_model(data, model_name)
+        if not model:
+            model = CommentedMap()
+            model["name"] = model_name
+            data["models"].append(model)
+        # Don't auto-create tags - let update_model_tags handle it when needed
+        return model
+
+    def set_latest_version(self, model: CommentedMap, version: int) -> None:
+        """Set or bump latest_version for a versioned model."""
+        existing = model.get("latest_version")
+        if existing is None or (isinstance(existing, int) and version > existing):
+            model["latest_version"] = version
+
+    def ensure_model_version(
+        self, model: CommentedMap, version: int
+    ) -> CommentedMap:
+        """
+        Ensure a version entry exists within a versioned model.
+
+        Args:
+            model: Model entry
+            version: Version number (e.g., 2)
+
+        Returns:
+            Version entry (existing or newly created)
+        """
+        if "versions" not in model or model.get("versions") is None:
+            model["versions"] = CommentedSeq()
+
+        for ver in model["versions"]:
+            if ver.get("v") == version:
+                return ver
+
+        ver_entry = CommentedMap()
+        ver_entry["v"] = version
+        model["versions"].append(ver_entry)
+        return ver_entry
+
+    def update_model_description(
+        self, model: CommentedMap, description: Optional[str]
+    ) -> None:
+        """
+        Update the description of a model.
+
+        Args:
+            model: Model entry
+            description: New description (or None to skip)
+        """
+        if description:
+            model["description"] = description
+
+    def get_model_tags(self, model: CommentedMap) -> List[str]:
+        """Return the list of tags for a model, combining top-level and config tags."""
+        top_level_tags = list(model.get("tags", []))
+        config = model.get("config", {})
+        config_tags = list(config.get("tags", [])) if config else []
+        # Combine and deduplicate, preserving order
+        seen = set()
+        combined = []
+        for tag in top_level_tags + config_tags:
+            if tag not in seen:
+                seen.add(tag)
+                combined.append(tag)
+        return combined
+
+    def update_version_tags(self, version: CommentedMap, tags: List[str]) -> None:
+        """
+        Replace the tags list for a model version using config.tags (dbt convention).
+        """
+        config = version.get("config")
+        if config is None:
+            version["config"] = CommentedMap()
+            config = version["config"]
+        config["tags"] = tags
+
+    def update_model_tags(self, model: CommentedMap, tags: List[str]) -> None:
+        """Replace the tags list for a model, preserving the original location.
+
+        Priority: 1) existing config.tags, 2) existing top-level tags, 3) config.tags (default)
+        Ensures tags are only in one location to avoid confusion.
+        """
+        config = model.get("config")
+        has_config_tags = config is not None and "tags" in config
+        has_top_level_tags = "tags" in model
+
+        if has_config_tags:
+            # Update in config block (original location)
+            config["tags"] = tags
+            # Remove top-level tags if present to avoid duplication
+            if has_top_level_tags:
+                del model["tags"]
+        elif has_top_level_tags:
+            # Update existing top-level tags
+            model["tags"] = tags
+        else:
+            # Default: use config.tags (dbt convention)
+            if config is None:
+                model["config"] = CommentedMap()
+                config = model["config"]
+            config["tags"] = tags
+
+    def find_column(
+        self, model: CommentedMap, column_name: str
+    ) -> Optional[CommentedMap]:
+        """
+        Find a specific column entry in the model.
+
+        Args:
+            model: Model entry
+            column_name: Name of the column to find
+
+        Returns:
+            Column entry or None if not found
+        """
+        columns = model.get("columns", [])
+        for col in columns:
+            if col.get("name") == column_name:
+                return col
+        return None
+
+    def ensure_column(self, model: CommentedMap, column_name: str) -> CommentedMap:
+        """
+        Ensure a column entry exists in the model, creating it if necessary.
+
+        Args:
+            model: Model entry
+            column_name: Name of the column
+
+        Returns:
+            Column entry (existing or newly created)
+        """
+        if "columns" not in model:
+            model["columns"] = CommentedSeq()
+
+        col = self.find_column(model, column_name)
+        if not col:
+            col = CommentedMap()
+            col["name"] = column_name
+            model["columns"].append(col)
+
+        return col
+
+    def update_column(
+        self,
+        column: CommentedMap,
+        data_type: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> None:
+        """
+        Update column properties.
+
+        Args:
+            column: Column entry
+            data_type: New data type (or None to skip)
+            description: New description (or None to skip)
+        """
+        if data_type:
+            column["data_type"] = data_type
+        if description:
+            column["description"] = description
+
+    def add_relationship_test(
+        self,
+        column: CommentedMap,
+        target_model: str,
+        target_field: str,
+    ) -> None:
+        """
+        Add or update a relationship test for a column.
+
+        Args:
+            column: Column entry
+            target_model: Target model name (will be wrapped in ref())
+            target_field: Target field name
+        """
+        existing_tests = CommentedSeq()
+
+        # Collect non-relationship tests from both tests and data_tests keys
+        for key in ("data_tests", "tests"):
+            if key in column:
+                for test in column.get(key, []):
+                    if isinstance(test, dict) and "relationships" in test:
+                        continue
+                    existing_tests.append(test)
+
+        # Build new relationships test using recommended arguments block
+        rel_test = CommentedMap()
+        rel_test["relationships"] = CommentedMap()
+        rel_test["relationships"]["arguments"] = CommentedMap()
+        rel_test["relationships"]["arguments"]["to"] = f"ref('{target_model}')"
+        rel_test["relationships"]["arguments"]["field"] = target_field
+
+        existing_tests.append(rel_test)
+
+        column["data_tests"] = existing_tests
+        # Drop tests key if present to avoid confusion
+        if "tests" in column:
+            del column["tests"]
+
+    def update_columns_batch(
+        self,
+        model: CommentedMap,
+        columns_data: List[Dict[str, Any]],
+    ) -> None:
+        """
+        Update multiple columns at once.
+
+        Args:
+            model: Model entry
+            columns_data: List of column dicts with name, data_type, description
+        """
+        for col_data in columns_data:
+            col_name = col_data.get("name")
+            if not col_name:
+                continue
+
+            col = self.ensure_column(model, col_name)
+            self.update_column(
+                col,
+                data_type=col_data.get("data_type"),
+                description=col_data.get("description"),
+            )
+
+    def get_columns(self, model: CommentedMap) -> List[Dict[str, Any]]:
+        """
+        Extract columns from a model as a list of dicts.
+
+        Args:
+            model: Model entry
+
+        Returns:
+            List of column dicts
+        """
+        columns = model.get("columns", [])
+        result = []
+
+        for col in columns:
+            col_dict = {
+                "name": col.get("name"),
+                "data_type": col.get("data_type"),
+                "description": col.get("description"),
+            }
+
+            # Extract tests (supports both dbt's tests and data_tests keys)
+            collected_tests: list[dict[str, Any]] = []
+            for key in ("data_tests", "tests"):
+                if key not in col:
+                    continue
+                for test in col.get(key, []):
+                    if isinstance(test, dict):
+                        collected_tests.append(dict(test))
+
+            if collected_tests:
+                col_dict["data_tests"] = collected_tests
+
+            result.append(col_dict)
+
+        return result
+

trellis_datamodel-0.3.3.dist-info/METADATA

@@ -0,0 +1,333 @@
+Metadata-Version: 2.4
+Name: trellis-datamodel
+Version: 0.3.3
+Summary: Visual data model editor for dbt projects
+Author: Tim Hiebenthal
+Project-URL: Homepage, https://github.com/timhiebenthal/trellis-datamodel
+Project-URL: Repository, https://github.com/timhiebenthal/trellis-datamodel
+Project-URL: Issues, https://github.com/timhiebenthal/trellis-datamodel/issues
+Keywords: dbt,data-modeling,erd,data-engineering,analytics-engineering,visualization,schema
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: dbt-core<2.0,>=1.10.5
+Requires-Dist: dbt-duckdb>=1.10.0
+Requires-Dist: fastapi>=0.121.3
+Requires-Dist: python-dotenv>=1.2.1
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: ruamel.yaml>=0.18.0
+Requires-Dist: typer>=0.9.0
+Requires-Dist: uvicorn>=0.38.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: httpx>=0.27.0; extra == "dev"
+Provides-Extra: dbt-example
+Requires-Dist: dbt-duckdb==1.10; extra == "dbt-example"
+Requires-Dist: duckdb>=1.4.2; extra == "dbt-example"
+Requires-Dist: faker>=24.0.0; extra == "dbt-example"
+Requires-Dist: marimo>=0.18.0; extra == "dbt-example"
+Requires-Dist: nba-api>=1.11.3; extra == "dbt-example"
+Requires-Dist: pandas>=2.3.3; extra == "dbt-example"
+Requires-Dist: tqdm>=4.67.1; extra == "dbt-example"
+Dynamic: license-file
+
+# Trellis Data
+
+![Trellis Logo](resources/trellis_with_text.png)
+
+A lightweight, local-first tool to bridge Conceptual Data Modeling, Logical Data Modeling and the Physical Implementation (currently with dbt-core).
+
+## Motivation
+
+**Current workflow pains:**
+- ERD diagrams live in separate tools (Lucidchart, draw.io) and quickly become stale or unreadable for large projects
+- Data transformations are done isolated from the conceptual data model.
+- No single view connecting business concepts to logical schema
+- Stakeholders can't easily understand model structure without technical context
+- Holistic Data Warehouse Automation Tools exists but do not integrate well with dbt and the Modern Data Stack
+
+**How Trellis helps:**
+- Visual data model that stays in sync — reads directly from `manifest.json` / `catalog.json`
+- Sketch entities and with their fields and auto-generate schema.yml's for dbt
+- Draw relationships on canvas → auto-generates dbt `relationships` tests
+- Two views: **Conceptual** (entity names, descriptions) and **Logical** (columns, types, materializations) to jump between high-level architect and execution-view.
+- Organize entities based on subdirectories and tags from your pyhsical implementation.
+- Write description or tags back to your dbt-project
+
+**Two Ways of getting started** 
+- Greenfield: draft entities and fields before writing SQL, then sync to dbt YAML  
+- Brownfield: document your existing data model by loading existing dbt models and utilize relationship tests to infer links
+
+## Tutorial
+
+Check out our [Full Tutorial](https://app.capacities.io/home/667ad256-ca68-4dfd-8231-e77d83127dcf) with video clips showing the core features in action.
+
+## Vision
+
+trellis is currently designed and tested specifically for **dbt-core**, but the vision is to be tool-agnostic. As the saying goes: *"tools evolve, concepts don't"* — data modeling concepts persist regardless of the transformation framework you use.
+
+If this project gains traction, we might explore support for:
+- **dbt-fusion** through adapter support
+- **Pydantic models** as a simple output format
+- Other frameworks like [SQLMesh](https://github.com/TobikoData/sqlmesh) or [Bruin](https://github.com/bruin-data/bruin) through adapter patterns, where compatibility allows
+
+This remains a vision for now — the current focus is on making Trellis work well with dbt-core.
+
+## Prerequisites
+- **Node.js 22+ (or 20.19+) & npm**  
+  - Recommended: Use [nvm](https://github.com/nvm-sh/nvm) to install a compatible version (e.g., `nvm install 22`).
+  - Note: System packages (`apt-get`) may be too old for the frontend dependencies.
+  - A `.nvmrc` file is included; run `nvm use` to switch to the correct version automatically.
+- **Python 3.11+ & [uv](https://github.com/astral-sh/uv)**  
+  - Install uv via `curl -LsSf https://astral.sh/uv/install.sh | sh` and ensure it's on your `$PATH`.
+- **Make** (optional) for convenience targets defined in the `Makefile`.
+
+## Installation
+
+### Install from PyPI
+
+```bash
+pip install trellis-datamodel
+# or with uv
+uv pip install trellis-datamodel
+```
+
+### Install from Source (Development)
+
+```bash
+# Clone the repository
+git clone https://github.com/timhiebenthal/trellis-datamodel.git
+cd trellis-datamodel
+
+# Install in editable mode
+pip install -e .
+# or with uv
+uv pip install -e .
+```
+
+## Quick Start
+
+1. **Navigate to your dbt project directory**
+   ```bash
+   cd /path/to/your/dbt-project
+   ```
+
+2. **Initialize configuration**
+   ```bash
+   trellis init
+   ```
+   This creates a `trellis.yml` file. Edit it to point to your dbt manifest and catalog locations.
+
+3. **Start the server**
+   ```bash
+   trellis run
+   ```
+
+   The server will start on **http://localhost:8089** and automatically open your browser.
+
+## Development Setup
+
+For local development with hot reload:
+
+### Install Dependencies
+Run these once per machine (or when dependencies change).
+
+1. **Backend**
+   ```bash
+   uv sync
+   ```
+2. **Frontend**
+   ```bash
+   cd frontend
+   npm install
+   ```
+
+**Terminal 1 – Backend**
+```bash
+make backend
+# or
+uv run trellis run
+```
+Backend serves the API at http://localhost:8089.
+
+**Terminal 2 – Frontend**
+```bash
+make frontend
+# or
+cd frontend && npm run dev
+```
+Frontend runs at http://localhost:5173 (for development with hot reload).
+
+## Building for Distribution
+
+To build the package with bundled frontend:
+
+```bash
+make build-package
+```
+
+This will:
+1. Build the frontend (`npm run build`)
+2. Copy static files to `trellis_datamodel/static/`
+3. Build the Python wheel (`uv build`)
+
+The wheel will be in `dist/` and can be installed with `pip install dist/trellis_datamodel-*.whl`.
+
+## CLI Options
+
+```bash
+trellis run [OPTIONS]
+
+Options:
+  --port, -p INTEGER    Port to run the server on [default: 8089]
+  --config, -c TEXT     Path to config file (trellis.yml or config.yml)
+  --no-browser          Don't open browser automatically
+  --help                Show help message
+```
+
+## dbt Metadata
+- Generate `manifest.json` and `catalog.json` by running `dbt docs generate` in your dbt project.
+- The UI reads these artifacts to power the ERD modeller.
+- Without these artifacts, the UI loads but shows no dbt models.
+
+## Configuration
+
+Run `trellis init` to create a starter `trellis.yml` file in your project.
+
+Options:
+
+- `framework`: Transformation framework to use. Currently supported: `dbt-core`. Future: `dbt-fusion`, `sqlmesh`, `bruin`, `pydantic`. Defaults to `dbt-core`.
+- `dbt_project_path`: Path to your dbt project directory (relative to `config.yml` or absolute). **Required**.
+- `dbt_manifest_path`: Path to `manifest.json` (relative to `dbt_project_path` or absolute). Defaults to `target/manifest.json`.
+- `dbt_catalog_path`: Path to `catalog.json` (relative to `dbt_project_path` or absolute). Defaults to `target/catalog.json`.
+- `data_model_file`: Path where the data model YAML will be saved (relative to `dbt_project_path` or absolute). Defaults to `data_model.yml`.
+- `dbt_model_paths`: List of path patterns to filter which dbt models are shown (e.g., `["3_core"]`). If empty, all models are included.
+
+**Example `trellis.yml`:**
+```yaml
+framework: dbt-core
+dbt_project_path: "./dbt_built"
+dbt_manifest_path: "target/manifest.json"
+dbt_catalog_path: "target/catalog.json"
+data_model_file: "data_model.yml"
+dbt_model_paths:
+  - "3_core"
+```
+
+
+## Testing
+
+### Frontend
+**Testing Libraries:**
+The following testing libraries are defined in `package.json` under `devDependencies` and are automatically installed when you run `npm install`:
+- [Vitest](https://vitest.dev/) (Unit testing)
+- [Playwright](https://playwright.dev/) (End-to-End testing)
+- [Testing Library](https://testing-library.com/) (DOM & Svelte testing utilities)
+- [jsdom](https://github.com/jsdom/jsdom) (DOM environment)
+
+> **Playwright system dependencies (Ubuntu/WSL2)**
+>
+> The browsers downloaded by Playwright need a handful of native libraries. Install them before running `npm run test:e2e`:
+>
+> ```bash
+> sudo apt-get update && sudo apt-get install -y \
+>   libxcursor1 libxdamage1 libgtk-3-0 libpangocairo-1.0-0 libpango-1.0-0 \
+>   libatk1.0-0 libcairo-gobject2 libcairo2 libgdk-pixbuf-2.0-0 libasound2 \
+>   libnspr4 libnss3 libgbm1 libgles2-mesa libgtk-4-1 libgraphene-1.0-0 \
+>   libxslt1.1 libwoff2dec0 libvpx7 libevent-2.1-7 libopus0 \
+>   libgstallocators-1.0-0 libgstapp-1.0-0 libgstpbutils-1.0-0 libgstaudio-1.0-0 \
+>   libgsttag-1.0-0 libgstvideo-1.0-0 libgstgl-1.0-0 libgstcodecparsers-1.0-0 \
+>   libgstfft-1.0-0 libflite1 libflite1-plugins libwebpdemux2 libavif13 \
+>   libharfbuzz-icu0 libwebpmux3 libenchant-2-2 libsecret-1-0 libhyphen0 \
+>   libwayland-server0 libmanette-0.2-0 libx264-163
+> ```
+
+**Running Tests:**
+
+The test suite has multiple levels to catch different types of issues:
+
+```bash
+cd frontend
+
+# Quick smoke test (catches 500 errors, runtime crashes, ESM issues)
+# Fastest way to verify the app loads without errors
+npm run test:smoke
+
+# TypeScript/compilation check
+npm run check
+
+# Unit tests
+npm run test:unit
+
+# E2E tests (includes smoke test + full test suite)
+# Note: Requires backend running with test data (see Test Data Isolation below)
+npm run test:e2e
+
+# Run all tests (check + smoke + unit + e2e)
+npm run test
+```
+
+**Test Levels:**
+1. **`npm run check`** - TypeScript compilation errors
+2. **`npm run test:smoke`** - Runtime errors (500s, console errors, ESM issues) - **catches app crashes**
+3. **`npm run test:unit`** - Unit tests with Vitest
+4. **`npm run test:e2e`** - Full E2E tests with Playwright
+
+**Using Makefile:**
+```bash
+# From project root
+make test-smoke     # Quick smoke test
+make test-check     # TypeScript check
+make test-unit      # Unit tests
+make test-e2e       # E2E tests (auto-starts backend with test data)
+make test-all       # All tests
+```
+
+**Test Data Isolation:**
+E2E tests use a separate test data file (`frontend/tests/test_data_model.yml`) to avoid polluting your production data model. **Playwright automatically starts the backend** with the correct environment variable, so you don't need to manage it manually.
+
+```bash
+# Just run E2E tests - backend starts automatically with test data
+make test-e2e
+# OR:
+# cd frontend && npm run test:e2e
+```
+
+The test data file is automatically cleaned before and after test runs via Playwright's `globalSetup` and `globalTeardown`. Your production `data_model.yml` remains untouched.
+
+### Backend
+**Testing Libraries:**
+The following testing libraries are defined in `pyproject.toml` under `[project.optional-dependencies]` in the `dev` group:
+- [pytest](https://docs.pytest.org/) (Testing framework)
+- [httpx](https://www.python-httpx.org/) (Async HTTP client for API testing)
+
+**Installation:**
+Unlike `npm`, `uv sync` does not install optional dependencies by default. To include the testing libraries, run:
+```bash
+uv sync --extra dev
+```
+
+**Running Tests:**
+```bash
+uv run pytest
+```
+
+## Collaboration
+
+If you want to collaborate, reach out!
+
+## Contributing and CLA
+- Contributions are welcome! Please read [`CONTRIBUTING.md`](CONTRIBUTING.md) for workflow, testing, and PR guidelines.
+- All contributors must sign the CLA once per GitHub account. The CLA bot on pull requests will guide you; see [`CLA.md`](CLA.md) for details.
+
+## License
+- Trellis Datamodel is licensed under the [GNU Affero General Public License v3.0](LICENSE).
+- See [`NOTICE`](NOTICE) for a summary of copyright and licensing information.