trellis-datamodel 0.3.3__py3-none-any.whl

trellis_datamodel/routes/data_model.py

@@ -0,0 +1,221 @@
+"""Routes for data model CRUD operations."""
+
+from fastapi import APIRouter, HTTPException
+import yaml
+import os
+from typing import Dict, Any, List, Tuple
+
+from trellis_datamodel import config as cfg
+from trellis_datamodel.models.schemas import DataModelUpdate
+
+router = APIRouter(prefix="/api", tags=["data-model"])
+
+
+def _load_canvas_layout() -> Dict[str, Any]:
+    """Load canvas layout file if it exists."""
+    if not os.path.exists(cfg.CANVAS_LAYOUT_PATH):
+        return {"version": 0.1, "entities": {}, "relationships": {}}
+
+    try:
+        with open(cfg.CANVAS_LAYOUT_PATH, "r") as f:
+            layout = yaml.safe_load(f) or {}
+        return {
+            "version": layout.get("version", 0.1),
+            "entities": layout.get("entities", {}),
+            "relationships": layout.get("relationships", {}),
+        }
+    except Exception as e:
+        print(f"Warning: Could not load canvas layout: {e}")
+        return {"version": 0.1, "entities": {}, "relationships": {}}
+
+
+def _merge_layout_into_model(
+    model_data: Dict[str, Any], layout_data: Dict[str, Any]
+) -> Dict[str, Any]:
+    """Merge canvas layout data into model data."""
+    entities_layout = layout_data.get("entities", {})
+    relationships_layout = layout_data.get("relationships", {})
+
+    # Merge entity visual properties
+    entities = model_data.get("entities", [])
+    for entity in entities:
+        entity_id = entity.get("id")
+        if entity_id and entity_id in entities_layout:
+            layout = entities_layout[entity_id]
+            if "position" in layout:
+                entity["position"] = layout["position"]
+            if "width" in layout:
+                entity["width"] = layout["width"]
+            if "panel_height" in layout:
+                entity["panel_height"] = layout["panel_height"]
+            if "collapsed" in layout:
+                entity["collapsed"] = layout["collapsed"]
+
+    # Merge relationship visual properties
+    relationships = model_data.get("relationships", [])
+    for idx, relationship in enumerate(relationships):
+        source = relationship.get("source")
+        target = relationship.get("target")
+        if source and target:
+            # Create key: source-target-index
+            rel_key = f"{source}-{target}-{idx}"
+            if rel_key in relationships_layout:
+                layout = relationships_layout[rel_key]
+                if "label_dx" in layout:
+                    relationship["label_dx"] = layout["label_dx"]
+                if "label_dy" in layout:
+                    relationship["label_dy"] = layout["label_dy"]
+
+    return model_data
+
+
+@router.get("/data-model")
+async def get_data_model():
+    """Return the current data model with layout merged in."""
+    if not os.path.exists(cfg.DATA_MODEL_PATH):
+        return {"version": 0.1, "entities": [], "relationships": []}
+
+    try:
+        # Load model data
+        with open(cfg.DATA_MODEL_PATH, "r") as f:
+            model_data = yaml.safe_load(f) or {}
+
+        if not model_data.get("entities"):
+            model_data["entities"] = []
+        if not model_data.get("relationships"):
+            model_data["relationships"] = []
+
+        # Load and merge layout data
+        layout_data = _load_canvas_layout()
+        merged_data = _merge_layout_into_model(model_data, layout_data)
+
+        return merged_data
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"Error reading data model: {str(e)}"
+        )
+
+
+def _split_model_and_layout(
+    content: Dict[str, Any],
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+    """Split incoming data into model-only and layout-only dictionaries."""
+    model_data = {
+        "version": content.get("version", 0.1),
+        "entities": [],
+        "relationships": [],
+    }
+
+    layout_data = {"version": 0.1, "entities": {}, "relationships": {}}
+
+    # Split entities
+    entities = content.get("entities", [])
+    for entity in entities:
+        entity_id = entity.get("id")
+        if not entity_id:
+            continue
+
+        # Model-only properties
+        model_entity = {
+            "id": entity_id,
+            "label": entity.get("label", ""),
+        }
+        if "description" in entity:
+            model_entity["description"] = entity["description"]
+        if "dbt_model" in entity:
+            model_entity["dbt_model"] = entity["dbt_model"]
+        if "additional_models" in entity:
+            model_entity["additional_models"] = entity["additional_models"]
+        if "drafted_fields" in entity:
+            model_entity["drafted_fields"] = entity["drafted_fields"]
+        if "tags" in entity:
+            model_entity["tags"] = entity["tags"]
+
+        model_data["entities"].append(model_entity)
+
+        # Layout-only properties
+        layout_entity = {}
+        if "position" in entity:
+            layout_entity["position"] = entity["position"]
+        if "width" in entity:
+            layout_entity["width"] = entity["width"]
+        if "panel_height" in entity:
+            layout_entity["panel_height"] = entity["panel_height"]
+        if "collapsed" in entity:
+            layout_entity["collapsed"] = entity["collapsed"]
+
+        if layout_entity:
+            layout_data["entities"][entity_id] = layout_entity
+
+    # Split relationships
+    relationships = content.get("relationships", [])
+    for idx, relationship in enumerate(relationships):
+        source = relationship.get("source")
+        target = relationship.get("target")
+        if not source or not target:
+            continue
+
+        # Model-only properties
+        model_rel = {
+            "source": source,
+            "target": target,
+        }
+        if "label" in relationship:
+            model_rel["label"] = relationship["label"]
+        if "type" in relationship:
+            model_rel["type"] = relationship["type"]
+        if "source_field" in relationship:
+            model_rel["source_field"] = relationship["source_field"]
+        if "target_field" in relationship:
+            model_rel["target_field"] = relationship["target_field"]
+
+        model_data["relationships"].append(model_rel)
+
+        # Layout-only properties
+        layout_rel = {}
+        if "label_dx" in relationship:
+            layout_rel["label_dx"] = relationship["label_dx"]
+        if "label_dy" in relationship:
+            layout_rel["label_dy"] = relationship["label_dy"]
+
+        if layout_rel:
+            # Use source-target-index as key
+            rel_key = f"{source}-{target}-{idx}"
+            layout_data["relationships"][rel_key] = layout_rel
+
+    return model_data, layout_data
+
+
+@router.post("/data-model")
+async def save_data_model(data: DataModelUpdate):
+    """Save the data model, splitting model and layout into separate files."""
+    try:
+        content = data.dict()  # Pydantic v1 (required by dbt-core==1.10)
+
+        # Split into model and layout
+        model_data, layout_data = _split_model_and_layout(content)
+
+        # Save model file
+        print(f"Saving data model to: {cfg.DATA_MODEL_PATH}")
+        os.makedirs(os.path.dirname(cfg.DATA_MODEL_PATH), exist_ok=True)
+        with open(cfg.DATA_MODEL_PATH, "w") as f:
+            yaml.dump(model_data, f, default_flow_style=False, sort_keys=False)
+            f.flush()
+            os.fsync(f.fileno())
+
+        # Save layout file
+        print(f"Saving canvas layout to: {cfg.CANVAS_LAYOUT_PATH}")
+        os.makedirs(os.path.dirname(cfg.CANVAS_LAYOUT_PATH), exist_ok=True)
+        with open(cfg.CANVAS_LAYOUT_PATH, "w") as f:
+            yaml.dump(layout_data, f, default_flow_style=False, sort_keys=False)
+            f.flush()
+            os.fsync(f.fileno())
+
+        return {"status": "success"}
+    except Exception as e:
+        import traceback
+
+        traceback.print_exc()
+        raise HTTPException(
+            status_code=500, detail=f"Error saving data model: {str(e)}"
+        )

trellis_datamodel/routes/manifest.py

@@ -0,0 +1,110 @@
+"""Routes for manifest and catalog operations."""
+
+from fastapi import APIRouter, HTTPException
+import os
+
+from trellis_datamodel import config as cfg
+from trellis_datamodel.config import (
+    find_config_file,
+)
+from trellis_datamodel.adapters import get_adapter
+
+router = APIRouter(prefix="/api", tags=["manifest"])
+
+
+def _resolve_config_path() -> str | None:
+    """Resolve config file path, preferring CONFIG_PATH from startup, falling back to search."""
+    if cfg.CONFIG_PATH and os.path.exists(cfg.CONFIG_PATH):
+        return cfg.CONFIG_PATH
+    return find_config_file()
+
+
+@router.get("/config-status")
+async def get_config_status():
+    """Return configuration status for the frontend."""
+    found_config = _resolve_config_path()
+    config_present = found_config is not None
+
+    # Determine expected config filename for display
+    if config_present:
+        config_filename = os.path.basename(found_config)
+    else:
+        # Default to trellis.yml (primary config file name)
+        config_filename = "trellis.yml"
+
+    manifest_exists = os.path.exists(cfg.MANIFEST_PATH) if cfg.MANIFEST_PATH else False
+    catalog_exists = os.path.exists(cfg.CATALOG_PATH) if cfg.CATALOG_PATH else False
+    data_model_exists = (
+        os.path.exists(cfg.DATA_MODEL_PATH) if cfg.DATA_MODEL_PATH else False
+    )
+
+    error = None
+    if not config_present:
+        error = "Config file not found."
+    elif not cfg.DBT_PROJECT_PATH:
+        error = "dbt_project_path not set in config."
+    elif not manifest_exists:
+        error = f"Manifest not found at {cfg.MANIFEST_PATH}"
+
+    return {
+        "config_present": config_present,
+        "config_filename": config_filename,
+        "framework": cfg.FRAMEWORK,
+        "dbt_project_path": cfg.DBT_PROJECT_PATH,
+        "manifest_path": cfg.MANIFEST_PATH,
+        "catalog_path": cfg.CATALOG_PATH,
+        "manifest_exists": manifest_exists,
+        "catalog_exists": catalog_exists,
+        "data_model_exists": data_model_exists,
+        "error": error,
+    }
+
+
+@router.get("/config-info")
+async def get_config_info():
+    """
+    Return resolved config paths and their existence for transparency/debugging.
+    """
+    config_path = _resolve_config_path()
+
+    adapter = get_adapter()
+    try:
+        model_dirs = adapter.get_model_dirs()  # type: ignore[attr-defined]
+    except Exception:
+        model_dirs = []
+
+    return {
+        "config_path": config_path,
+        "framework": cfg.FRAMEWORK,
+        "dbt_project_path": cfg.DBT_PROJECT_PATH,
+        "manifest_path": cfg.MANIFEST_PATH,
+        "manifest_exists": bool(
+            cfg.MANIFEST_PATH and os.path.exists(cfg.MANIFEST_PATH)
+        ),
+        "catalog_path": cfg.CATALOG_PATH,
+        "catalog_exists": bool(cfg.CATALOG_PATH and os.path.exists(cfg.CATALOG_PATH)),
+        "data_model_path": cfg.DATA_MODEL_PATH,
+        "data_model_exists": bool(
+            cfg.DATA_MODEL_PATH and os.path.exists(cfg.DATA_MODEL_PATH)
+        ),
+        "canvas_layout_path": cfg.CANVAS_LAYOUT_PATH,
+        "canvas_layout_exists": bool(
+            cfg.CANVAS_LAYOUT_PATH and os.path.exists(cfg.CANVAS_LAYOUT_PATH)
+        ),
+        "frontend_build_dir": cfg.FRONTEND_BUILD_DIR,
+        "model_paths_configured": cfg.DBT_MODEL_PATHS,
+        "model_paths_resolved": model_dirs,
+    }
+
+
+@router.get("/manifest")
+async def get_manifest():
+    """Return parsed models from the transformation framework."""
+    try:
+        adapter = get_adapter()
+        models = adapter.get_models()
+        return {"models": models}
+    except FileNotFoundError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error reading manifest: {str(e)}")

trellis_datamodel/routes/schema.py

@@ -0,0 +1,183 @@
+"""Routes for schema sync operations."""
+
+from fastapi import APIRouter, HTTPException
+import yaml
+import os
+
+from trellis_datamodel import config as cfg
+from trellis_datamodel.models.schemas import DbtSchemaRequest, ModelSchemaRequest
+from trellis_datamodel.adapters import get_adapter
+
+router = APIRouter(prefix="/api", tags=["schema"])
+
+
+@router.post("/dbt-schema")
+async def save_dbt_schema(request: DbtSchemaRequest):
+    """Generate and save a schema YAML file for the drafted fields."""
+    try:
+        if not cfg.DBT_PROJECT_PATH:
+            raise HTTPException(
+                status_code=400,
+                detail="dbt_project_path is not configured. Please set it in config.yml",
+            )
+
+        adapter = get_adapter()
+        output_path = adapter.save_dbt_schema(
+            entity_id=request.entity_id,
+            model_name=request.model_name,
+            fields=request.fields,
+            description=request.description,
+            tags=request.tags,
+        )
+
+        return {
+            "status": "success",
+            "file_path": str(output_path),
+            "message": f"Schema saved to {output_path}",
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error saving schema: {str(e)}")
+
+
+@router.post("/sync-dbt-tests")
+async def sync_dbt_tests():
+    """Sync relationship tests from data model to schema files."""
+    try:
+        if not cfg.DBT_PROJECT_PATH:
+            raise HTTPException(
+                status_code=400,
+                detail="dbt_project_path is not configured. Please set it in config.yml",
+            )
+
+        if not cfg.DATA_MODEL_PATH or not os.path.exists(cfg.DATA_MODEL_PATH):
+            raise HTTPException(status_code=404, detail="Data model file not found")
+
+        with open(cfg.DATA_MODEL_PATH, "r") as f:
+            data_model = yaml.safe_load(f) or {}
+
+        entities = data_model.get("entities", [])
+        relationships = data_model.get("relationships", [])
+
+        adapter = get_adapter()
+        updated_files = adapter.sync_relationships(entities, relationships)
+
+        return {
+            "status": "success",
+            "message": f"Updated {len(updated_files)} file(s)",
+            "files": [str(f) for f in updated_files],
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        import traceback
+
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=f"Error syncing tests: {str(e)}")
+
+
+@router.get("/models/{model_name}/schema")
+async def get_model_schema(model_name: str, version: int | None = None):
+    """Get the schema for a specific model from its YAML file."""
+    try:
+        if not cfg.DBT_PROJECT_PATH:
+            raise HTTPException(
+                status_code=400,
+                detail="dbt_project_path is not configured. Please set it in config.yml",
+            )
+
+        adapter = get_adapter()
+        schema = adapter.get_model_schema(model_name, version=version)
+
+        return {
+            "model_name": schema.get("model_name", model_name),
+            "description": schema.get("description", ""),
+            "columns": schema.get("columns", []),
+            "tags": schema.get("tags", []),
+            "file_path": schema.get("file_path", ""),
+        }
+
+    except FileNotFoundError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except ValueError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except HTTPException:
+        raise
+    except Exception as e:
+        import traceback
+
+        traceback.print_exc()
+        raise HTTPException(
+            status_code=500, detail=f"Error reading model schema: {str(e)}"
+        )
+
+
+@router.post("/models/{model_name}/schema")
+async def update_model_schema(model_name: str, request: ModelSchemaRequest):
+    """Update the schema for a specific model in its YAML file."""
+    try:
+        if not cfg.DBT_PROJECT_PATH:
+            raise HTTPException(
+                status_code=400,
+                detail="dbt_project_path is not configured. Please set it in config.yml",
+            )
+
+        adapter = get_adapter()
+        output_path = adapter.save_model_schema(
+            model_name=model_name,
+            columns=request.columns,
+            description=request.description,
+            tags=request.tags,
+            version=request.version,
+        )
+
+        return {
+            "status": "success",
+            "message": f"Schema updated for model '{model_name}'",
+            "file_path": str(output_path),
+        }
+
+    except FileNotFoundError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except ValueError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except HTTPException:
+        raise
+    except Exception as e:
+        import traceback
+
+        traceback.print_exc()
+        raise HTTPException(
+            status_code=500, detail=f"Error updating model schema: {str(e)}"
+        )
+
+
+@router.get("/infer-relationships")
+async def infer_relationships(include_unbound: bool = False):
+    """Scan schema files and infer entity relationships from relationship tests."""
+    try:
+        if not cfg.DBT_PROJECT_PATH:
+            raise HTTPException(
+                status_code=400,
+                detail="dbt_project_path is not configured. Please set it in config.yml",
+            )
+
+        adapter = get_adapter()
+        relationships = adapter.infer_relationships(include_unbound=include_unbound)
+
+        return {"relationships": relationships}
+
+    except FileNotFoundError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except HTTPException:
+        raise
+    except Exception as e:
+        import traceback
+
+        traceback.print_exc()
+        raise HTTPException(
+            status_code=500, detail=f"Error inferring relationships: {str(e)}"
+        )

trellis_datamodel/server.py

@@ -0,0 +1,101 @@
+"""
+Trellis Data - FastAPI Server
+
+This is the FastAPI application that serves the API and frontend.
+"""
+
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
+import os
+from importlib.resources import files
+
+from trellis_datamodel import config as cfg
+from trellis_datamodel.config import print_config
+from trellis_datamodel.routes import manifest_router, data_model_router, schema_router
+
+
+def create_app() -> FastAPI:
+    """Create and configure the FastAPI application."""
+    app = FastAPI(title="Trellis Data", version="0.1.0")
+
+    # CORS for development
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],  # Allow all for local dev
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Health check endpoint
+    @app.get("/health")
+    def health_check():
+        return {"status": "ok"}
+
+    # Find static files directory
+    # Priority: 1) configured FRONTEND_BUILD_DIR, 2) package static dir (if it has index.html)
+    static_dir_path = None
+
+    # Check configured build dir first
+    if cfg.FRONTEND_BUILD_DIR and os.path.exists(
+        os.path.join(cfg.FRONTEND_BUILD_DIR, "index.html")
+    ):
+        static_dir_path = cfg.FRONTEND_BUILD_DIR
+    else:
+        # Fall back to package static dir if it has index.html
+        try:
+            static_dir = files("trellis_datamodel") / "static"
+            if static_dir.is_dir():
+                pkg_index = str(static_dir / "index.html")
+                if os.path.exists(pkg_index):
+                    static_dir_path = str(static_dir)
+        except Exception:
+            pass
+
+    print(f"Serving frontend from: {static_dir_path}")
+
+    # Include API routers - these MUST be registered before mounting static files
+    app.include_router(manifest_router)
+    app.include_router(data_model_router)
+    app.include_router(schema_router)
+
+    # Mount static files AFTER API routes
+    # Important: app.mount() creates a sub-application, so we mount AFTER registering API routes
+    # However, mounted apps at "/" will intercept everything, so we need a different approach
+    if static_dir_path:
+        # Serve static assets at /assets
+        assets_path = os.path.join(static_dir_path, "assets")
+        if os.path.exists(assets_path):
+            app.mount("/assets", StaticFiles(directory=assets_path), name="assets")
+
+        # Serve other static files (like favicon, etc.) at /_app
+        # SvelteKit builds put immutable assets in /_app
+        app_path = os.path.join(static_dir_path, "_app")
+        if os.path.exists(app_path):
+            app.mount("/_app", StaticFiles(directory=app_path), name="app")
+
+        # Catch-all route for SPA - must be defined LAST
+        # FastAPI matches more specific routes first, so /api/* routes will match before this
+        @app.get("/{full_path:path}", include_in_schema=False)
+        async def serve_spa(request: Request, full_path: str):
+            """Serve SPA index.html for non-API routes."""
+            # Serve index.html for all routes (API routes are already matched above)
+            index_file = os.path.join(static_dir_path, "index.html")
+            if os.path.exists(index_file):
+                return FileResponse(index_file)
+            raise HTTPException(status_code=404, detail="Not found")
+
+    else:
+        print(
+            f"Warning: Frontend build not found. "
+            f"Bundled static files missing and {cfg.FRONTEND_BUILD_DIR} does not exist. "
+            f"Run 'npm run build' in frontend/ or install the package properly."
+        )
+
+    return app
+
+
+app = create_app()

trellis_datamodel/static/_app/env.js

@@ -0,0 +1 @@
+export const env={}