iflow-mcp_niclasolofsson-dbt-core-mcp 1.7.0__py3-none-any.whl

dbt_core_mcp/tools/get_lineage.py

@@ -0,0 +1,89 @@
+"""Get lineage (dependency tree) for any dbt resource.
+
+This module implements the get_lineage tool for dbt Core MCP.
+"""
+
+import logging
+from typing import Any
+
+from fastmcp.dependencies import Depends  # type: ignore[reportAttributeAccessIssue]
+from fastmcp.server.context import Context
+
+from ..context import DbtCoreServerContext
+from ..dependencies import get_state
+from . import dbtTool
+
+logger = logging.getLogger(__name__)
+
+
+async def _implementation(
+    ctx: Context | None,
+    name: str,
+    resource_type: str | None,
+    direction: str,
+    depth: int | None,
+    state: DbtCoreServerContext,
+    force_parse: bool = True,
+) -> dict[str, Any]:
+    """Implementation function for get_lineage tool.
+
+    Separated for testing purposes - tests call this directly with explicit state.
+    The @tool() decorated get_lineage() function calls this with injected dependencies.
+    """
+    # Initialize state if needed (metadata tool uses force_parse=True)
+    await state.ensure_initialized(ctx, force_parse)
+
+    # Delegate to manifest helper for lineage traversal
+    try:
+        return state.manifest.get_lineage(name, resource_type, direction, depth)  # type: ignore
+    except ValueError as e:
+        raise ValueError(f"Lineage error: {e}")
+
+
+@dbtTool()
+async def get_lineage(
+    ctx: Context,
+    name: str,
+    resource_type: str | None = None,
+    direction: str = "both",
+    depth: int | None = None,
+    state: DbtCoreServerContext = Depends(get_state),
+) -> dict[str, Any]:
+    """Get lineage (dependency tree) for any dbt resource with auto-detection.
+
+    This unified tool works across all resource types (models, sources, seeds, snapshots, etc.)
+    showing upstream and/or downstream dependencies with configurable depth.
+
+    Args:
+        name: Resource name. For sources, use "source_name.table_name" or just "table_name"
+            Examples: "customers", "jaffle_shop.orders", "raw_customers"
+        resource_type: Optional filter to narrow search:
+            - "model": Data transformation models
+            - "source": External data sources
+            - "seed": CSV reference data files
+            - "snapshot": SCD Type 2 historical tables
+            - "test": Data quality tests
+            - "analysis": Ad-hoc analysis queries
+            - None: Auto-detect (searches all types)
+        direction: Lineage direction:
+            - "upstream": Show where data comes from (parents)
+            - "downstream": Show what depends on this resource (children)
+            - "both": Show full lineage (default)
+        depth: Maximum levels to traverse (None for unlimited)
+            - depth=1: Immediate dependencies only
+            - depth=2: Dependencies + their dependencies
+            - None: Full dependency tree
+
+    Returns:
+        Lineage information with upstream/downstream nodes and statistics.
+        If multiple matches found, returns all matches for LLM to process.
+
+    Raises:
+        ValueError: If resource not found or invalid direction
+
+    Examples:
+        get_lineage("customers") -> auto-detect and show full lineage
+        get_lineage("customers", "model", "upstream") -> where customers model gets data
+        get_lineage("jaffle_shop.orders", "source", "downstream", 2) -> 2 levels of dependents
+    """
+    return await _implementation(ctx, name, resource_type, direction, depth, state)

dbt_core_mcp/tools/get_project_info.py

@@ -0,0 +1,96 @@
+"""Get information about the dbt project with optional diagnostics.
+
+This module implements the get_project_info tool for dbt Core MCP.
+"""
+
+import logging
+from typing import Any
+
+from fastmcp.dependencies import Depends  # type: ignore[reportAttributeAccessIssue]
+from fastmcp.server.context import Context
+
+from ..context import DbtCoreServerContext
+from ..dependencies import get_state
+from . import dbtTool
+
+logger = logging.getLogger(__name__)
+
+
+async def _implementation(
+    ctx: Context | None,
+    run_debug: bool,
+    state: DbtCoreServerContext,
+    force_parse: bool = True,
+) -> dict[str, Any]:
+    """Implementation function for get_project_info tool.
+
+    Separated for testing purposes - tests call this directly with explicit state.
+    The @tool() decorated get_project_info() function calls this with injected dependencies.
+    """
+    # Initialize state if needed (metadata tool uses force_parse=True)
+    await state.ensure_initialized(ctx, force_parse)
+
+    try:
+        # Collect manifest metadata for quick status check
+        info = state.manifest.get_project_info()  # type: ignore
+        info["project_dir"] = str(state.project_dir)
+        info["profiles_dir"] = state.profiles_dir
+        info["status"] = "ready"
+
+        # Add dbt-core-mcp version
+        from .. import __version__
+
+        info["dbt_core_mcp_version"] = __version__
+
+        # Optionally run full dbt debug for connectivity diagnostics
+        if run_debug:
+            runner = await state.get_runner()
+            debug_result_obj = await runner.invoke(["debug"])  # type: ignore
+
+            # Convert DbtRunnerResult to dictionary
+            debug_result = {
+                "success": debug_result_obj.success,
+                "output": debug_result_obj.stdout if debug_result_obj.stdout else "",
+            }
+
+            # Normalize debug output into structured diagnostics
+            diagnostics: dict[str, Any] = {
+                "command_run": "dbt debug",
+                "success": debug_result.get("success", False),
+                "output": debug_result.get("output", ""),
+            }
+
+            # Extract connection status from output
+            output = str(debug_result.get("output", ""))
+            if "Connection test: [OK connection ok]" in output or "Connection test: OK" in output:
+                diagnostics["connection_status"] = "ok"
+            elif "Connection test: [ERROR" in output or "Connection test: FAIL" in output:
+                diagnostics["connection_status"] = "failed"
+            else:
+                diagnostics["connection_status"] = "unknown"
+
+            info["diagnostics"] = diagnostics
+
+        return info
+
+    except Exception as e:
+        raise ValueError(f"Failed to get project info: {e}")
+
+
+@dbtTool()
+async def get_project_info(
+    ctx: Context,
+    run_debug: bool = True,
+    state: DbtCoreServerContext = Depends(get_state),
+) -> dict[str, Any]:
+    """Get information about the dbt project with optional diagnostics.
+
+    Args:
+        ctx: MCP context (provided by FastMCP)
+        run_debug: Run `dbt debug` to validate environment and test connection (default: True)
+        state: Shared state object injected by FastMCP
+
+    Returns:
+        Dictionary with project information and diagnostic results
+    """
+    return await _implementation(ctx, run_debug, state)

dbt_core_mcp/tools/get_resource_info.py

@@ -0,0 +1,134 @@
+"""Get detailed information about any dbt resource.
+
+This module implements the get_resource_info tool for dbt Core MCP.
+"""
+
+import logging
+from typing import Any
+
+from fastmcp.dependencies import Depends  # type: ignore[reportAttributeAccessIssue]
+from fastmcp.server.context import Context
+
+from ..context import DbtCoreServerContext
+from ..dependencies import get_state
+from . import dbtTool
+
+logger = logging.getLogger(__name__)
+
+
+async def _implementation(
+    ctx: Context | None,
+    name: str,
+    resource_type: str | None,
+    include_database_schema: bool,
+    include_compiled_sql: bool,
+    state: DbtCoreServerContext,
+    force_parse: bool = True,
+) -> dict[str, Any]:
+    """Implementation function for get_resource_info tool.
+
+    Separated for testing purposes - tests call this directly with explicit state.
+    The @tool() decorated get_resource_info() function calls this with injected dependencies.
+    """
+    # Initialize state if needed (metadata tool uses force_parse=True)
+    await state.ensure_initialized(ctx, force_parse)
+
+    try:
+        # Get resource info with manifest method (handles basic enrichment)
+        result = state.manifest.get_resource_info(  # type: ignore
+            name,
+            resource_type,
+            include_database_schema=False,  # We'll handle this below for database schema
+            include_compiled_sql=include_compiled_sql,
+        )
+
+        # Handle multiple matches case
+        if result.get("multiple_matches"):
+            # Enrich each match with database schema if requested
+            if include_database_schema:
+                matches = result.get("matches", [])
+                for match in matches:
+                    node_type = match.get("resource_type")
+                    if node_type in ("model", "seed", "snapshot", "source"):
+                        resource_name = match.get("name")
+                        source_name = match.get("source_name") if node_type == "source" else None
+                        schema = await state.get_table_schema_from_db(resource_name, source_name)
+                        if schema:
+                            match["database_columns"] = schema
+            return result
+
+        # Single match - check if we need to trigger compilation
+        node_type = result.get("resource_type")
+
+        if include_compiled_sql and node_type == "model":
+            # If compiled SQL requested but not available, trigger compilation
+            if result.get("compiled_sql") is None and not result.get("compiled_sql_cached"):
+                logger.info(f"Compiling model: {name}")
+                runner = await state.get_runner()
+                compile_result = await runner.invoke_compile(name, force=False)  # type: ignore
+
+                if compile_result.success:
+                    # Reload manifest to get compiled code
+                    await state.manifest.load()  # type: ignore
+                    # Re-fetch the resource to get updated compiled_code
+                    result = state.manifest.get_resource_info(  # type: ignore
+                        name,
+                        resource_type,
+                        include_database_schema=False,
+                        include_compiled_sql=True,
+                    )
+
+        # Query database schema for applicable resource types (ref/source aware)
+        if include_database_schema and node_type in ("model", "seed", "snapshot", "source"):
+            resource_name = result.get("name", name)
+            # For sources, pass source_name to use source() instead of ref()
+            source_name = result.get("source_name") if node_type == "source" else None
+            schema = await state.get_table_schema_from_db(resource_name, source_name)
+            if schema:
+                result["database_columns"] = schema
+
+        return result
+
+    except ValueError as e:
+        raise ValueError(f"Resource not found: {e}")
+
+
+@dbtTool()
+async def get_resource_info(
+    ctx: Context,
+    name: str,
+    resource_type: str | None = None,
+    include_database_schema: bool = True,
+    include_compiled_sql: bool = True,
+    state: DbtCoreServerContext = Depends(get_state),
+) -> dict[str, Any]:
+    """Get detailed information about any dbt resource (model, source, seed, snapshot, test, etc.).
+
+    This unified tool works across all resource types, auto-detecting the resource or filtering by type.
+    Designed for LLM consumption - returns complete data even when multiple matches exist.
+
+    Args:
+        name: Resource name. For sources, use "source_name.table_name" or just "table_name"
+        resource_type: Optional filter to narrow search:
+            - "model": Data transformation models
+            - "source": External data sources
+            - "seed": CSV reference data files
+            - "snapshot": SCD Type 2 historical tables
+            - "test": Data quality tests
+            - "analysis": Ad-hoc analysis queries
+            - None: Auto-detect (searches all types)
+        include_database_schema: If True (default), query actual database table schema
+            for models/seeds/snapshots/sources and add as 'database_columns' field
+        include_compiled_sql: If True (default), include compiled SQL with Jinja resolved
+            ({{ ref() }}, {{ source() }} → actual table names). Only applicable to models.
+            Will trigger dbt compile if not already compiled. Set to False to skip compilation.
+        state: Shared state object injected by FastMCP
+
+    Returns:
+        Resource information dictionary. If multiple matches found, returns:
+        {"multiple_matches": True, "matches": [...], "message": "..."}
+
+    Raises:
+        ValueError: If resource not found
+    """
+    return await _implementation(ctx, name, resource_type, include_database_schema, include_compiled_sql, state)

dbt_core_mcp/tools/install_deps.py

@@ -0,0 +1,102 @@
+"""Install dbt packages defined in packages.yml.
+
+This module implements the install_deps tool for dbt Core MCP.
+"""
+
+import logging
+from typing import Any
+
+from fastmcp.dependencies import Depends  # type: ignore[reportAttributeAccessIssue]
+from fastmcp.server.context import Context
+
+from ..context import DbtCoreServerContext
+from ..dependencies import get_state
+from . import dbtTool
+
+logger = logging.getLogger(__name__)
+
+
+async def _implementation(
+    ctx: Context | None,
+    state: DbtCoreServerContext,
+) -> dict[str, Any]:
+    """Implementation function for install_deps tool.
+
+    Separated for testing purposes - tests call this directly with explicit state.
+    The @tool() decorated install_deps() function calls this with injected dependencies.
+    """  # Ensure dbt components are initialized
+    await state.ensure_initialized(ctx, force_parse=False)
+    # Execute dbt deps
+    logger.info("Running dbt deps to install packages")
+
+    runner = await state.get_runner()
+    result = await runner.invoke(["deps"])
+
+    if not result.success:
+        # Bubble up installer failure with context
+        raise RuntimeError(f"dbt deps failed: {result.exception}")
+
+    # Parse installed packages from manifest
+    installed_packages: set[str] = set()
+
+    assert state.manifest is not None
+    manifest_dict = state.manifest.get_manifest_dict()
+    macros = manifest_dict.get("macros", {})
+    project_name = manifest_dict.get("metadata", {}).get("project_name", "")
+
+    for unique_id in macros:
+        # macro.package_name.macro_name format
+        if unique_id.startswith("macro."):
+            parts = unique_id.split(".")
+            if len(parts) >= 2:
+                package_name = parts[1]
+                # Exclude built-in dbt package and project package
+                if package_name != "dbt" and package_name != project_name:
+                    installed_packages.add(package_name)
+
+    return {
+        "status": "success",
+        "command": "dbt deps",
+        "installed_packages": sorted(installed_packages),
+        "message": f"Successfully installed {len(installed_packages)} package(s)",
+    }
+
+
+@dbtTool()
+async def install_deps(
+    ctx: Context,
+    state: DbtCoreServerContext = Depends(get_state),
+) -> dict[str, Any]:
+    """Install dbt packages defined in packages.yml.
+
+    This tool enables interactive workflow where an LLM can:
+    1. Suggest using a dbt package (e.g., dbt_utils)
+    2. Edit packages.yml to add the package
+    3. Run install_deps() to install it
+    4. Write code that uses the package's macros
+
+    This completes the recommendation workflow without breaking conversation flow.
+
+    **When to use**:
+    - After adding/modifying packages.yml
+    - Before using macros from external packages
+    - When setting up a new dbt project
+
+    **Package Discovery**:
+    After installation, use list_resources(resource_type="macro") to verify
+    installed packages and discover available macros.
+
+    Returns:
+        Installation results with status and installed packages
+
+    Example workflow:
+        User: "Create a date dimension table"
+        LLM: 1. Checks: list_resources(type="macro") -> no dbt_utils
+             2. Edits: packages.yml (adds dbt_utils package)
+             3. Runs: install_deps() (installs package)
+             4. Creates: models/date_dim.sql (uses dbt_utils.date_spine)
+
+    Note: This is an interactive development tool, not infrastructure automation.
+    It enables the LLM to act on its own recommendations mid-conversation.
+    """
+    return await _implementation(ctx, state)

dbt_core_mcp/tools/list_resources.py

@@ -0,0 +1,84 @@
+"""List all resources in the dbt project.
+
+This module implements the list_resources tool for dbt Core MCP.
+"""
+
+import logging
+from typing import Any
+
+from fastmcp.dependencies import Depends  # type: ignore[reportAttributeAccessIssue]
+from fastmcp.server.context import Context
+
+from ..context import DbtCoreServerContext
+from ..dependencies import get_state
+from . import dbtTool
+
+logger = logging.getLogger(__name__)
+
+
+async def _implementation(
+    ctx: Context | None,
+    resource_type: str | None,
+    state: DbtCoreServerContext,
+    force_parse: bool = True,
+) -> list[dict[str, Any]]:
+    """Implementation function for list_resources tool.
+
+    Separated for testing purposes - tests call this directly with explicit state.
+    The @tool() decorated list_resources() function calls this with injected dependencies.
+    """
+    # Initialize state if needed (metadata tool uses force_parse=True)
+    await state.ensure_initialized(ctx, force_parse)
+
+    # Return simplified manifest resources (LLM-friendly structure)
+    return state.manifest.get_resources(resource_type)  # type: ignore
+
+
+@dbtTool()
+async def list_resources(
+    ctx: Context,
+    resource_type: str | None = None,
+    state: DbtCoreServerContext = Depends(get_state),
+) -> list[dict[str, Any]]:
+    """List all resources in the dbt project with optional filtering by type.
+
+    This unified tool provides a consistent view across all dbt resource types.
+    Returns simplified resource information optimized for LLM consumption.
+
+    Args:
+        resource_type: Optional filter to narrow results:
+            - "model": Data transformation models
+            - "source": External data sources
+            - "seed": CSV reference data files
+            - "snapshot": SCD Type 2 historical tables
+            - "test": Data quality tests
+            - "analysis": Ad-hoc analysis queries
+            - "macro": Jinja macros (includes macros from installed packages)
+            - None: Return all resources (default)
+
+    Returns:
+        List of resource dictionaries with consistent structure across types.
+        Each resource includes: name, unique_id, resource_type, description, tags, etc.
+
+    Package Discovery:
+        Use resource_type="macro" to discover installed dbt packages.
+        Macros follow the naming pattern: macro.{package_name}.{macro_name}
+
+        Example - Check if dbt_utils is installed:
+            macros = list_resources("macro")
+            has_dbt_utils = any(m["unique_id"].startswith("macro.dbt_utils.") for m in macros)
+
+        Example - List all installed packages:
+            macros = list_resources("macro")
+            packages = {m["unique_id"].split(".")[1] for m in macros
+                       if m["unique_id"].startswith("macro.") and
+                       m["unique_id"].split(".")[1] != "dbt"}
+
+    Examples:
+        list_resources() -> all resources
+        list_resources("model") -> only models
+        list_resources("source") -> only sources
+        list_resources("test") -> only tests
+        list_resources("macro") -> all macros (discover installed packages)
+    """
+    return await _implementation(ctx, resource_type, state)

dbt_core_mcp/tools/load_seeds.py

@@ -0,0 +1,179 @@
+"""Load seed data (CSV files) from seeds/ directory into database tables.
+
+This module implements the load_seeds tool for dbt Core MCP.
+"""
+
+import logging
+from typing import Any
+
+from fastmcp.dependencies import Depends  # type: ignore[reportAttributeAccessIssue]
+from fastmcp.server.context import Context
+
+from ..context import DbtCoreServerContext
+from ..dependencies import get_state
+from . import dbtTool
+
+logger = logging.getLogger(__name__)
+
+
+async def _implementation(
+    ctx: Context | None,
+    select: str | None,
+    exclude: str | None,
+    select_state_modified: bool,
+    select_state_modified_plus_downstream: bool,
+    full_refresh: bool,
+    show: bool,
+    state: DbtCoreServerContext,
+) -> dict[str, Any]:
+    """Implementation function for load_seeds tool.
+
+    Separated for testing purposes - tests call this directly with explicit state.
+    The @tool() decorated load_seeds() function calls this with injected dependencies.
+    """
+    # Ensure dbt components are initialized
+    await state.ensure_initialized(ctx, force_parse=False)
+
+    # Build selector (state-based when available)
+    selector = await state.prepare_state_based_selection(select_state_modified, select_state_modified_plus_downstream, select)
+
+    if select_state_modified and not selector:
+        raise RuntimeError("No previous state found - cannot determine modifications. Run 'dbt seed' first to create baseline state.")
+
+    # Construct dbt CLI args for seed
+    args = ["seed"]
+
+    if selector:
+        target_path = state.get_project_paths().get("target-path", "target")
+        args.extend(["-s", selector, "--state", f"{target_path}/state_last_run"])
+    elif select:
+        args.extend(["-s", select])
+
+    if exclude:
+        args.extend(["--exclude", exclude])
+
+    if full_refresh:
+        args.append("--full-refresh")
+
+    if show:
+        args.append("--show")
+
+    logger.info(f"Running DBT seed with args: {args}")
+
+    # Stream progress back to MCP client (if provided)
+    async def progress_callback(current: int, total: int, message: str) -> None:
+        if ctx:
+            await ctx.report_progress(progress=current, total=total, message=message)
+
+    # Clear stale run_results so we parse only fresh output
+    state.clear_stale_run_results()
+
+    runner = await state.get_runner()
+    result = await runner.invoke(args, progress_callback=progress_callback if ctx else None)  # type: ignore
+
+    run_results = state.validate_and_parse_results(result, "seed")
+
+    if result.success:
+        await state.save_execution_state()
+
+    run_results = state.parse_run_results()
+
+    if ctx:
+        if run_results.get("results"):
+            results_list = run_results["results"]
+            total = len(results_list)
+            passed_count = sum(1 for r in results_list if r.get("status") == "success")
+            failed_count = sum(1 for r in results_list if r.get("status") in ("error", "fail"))
+
+            parts = []
+            if passed_count > 0:
+                parts.append(f"✅ {passed_count} passed" if failed_count > 0 else "✅ All passed")
+            if failed_count > 0:
+                parts.append(f"❌ {failed_count} failed")
+
+            summary = f"Seed: {total}/{total} seeds completed ({', '.join(parts)})"
+            await ctx.report_progress(progress=total, total=total, message=summary)
+        else:
+            await ctx.report_progress(progress=0, total=0, message="0 seeds matched selector")
+
+    if not run_results.get("results"):
+        raise RuntimeError(f"No seeds matched selector: {select or selector or 'all'}")
+
+    return {
+        "status": "success",
+        "command": " ".join(args),
+        "results": run_results.get("results", []),
+        "elapsed_time": run_results.get("elapsed_time"),
+    }
+
+
+@dbtTool()
+async def load_seeds(
+    ctx: Context,
+    select: str | None = None,
+    exclude: str | None = None,
+    select_state_modified: bool = False,
+    select_state_modified_plus_downstream: bool = False,
+    full_refresh: bool = False,
+    show: bool = False,
+    state: DbtCoreServerContext = Depends(get_state),
+) -> dict[str, Any]:
+    """Load seed data (CSV files) from seeds/ directory into database tables.
+
+    **When to use**: Run this before building models or tests that depend on reference data.
+    Seeds must be loaded before models that reference them can execute.
+
+    **What are seeds**: CSV files containing static reference data (country codes,
+    product categories, lookup tables, etc.). Unlike models (which are .sql files),
+    seeds are CSV files that are loaded directly into database tables.
+
+    State-based selection modes (detects changed CSV files):
+    - select_state_modified: Load only seeds modified since last successful run (state:modified)
+    - select_state_modified_plus_downstream: Load modified + downstream dependencies (state:modified+)
+      Note: Requires select_state_modified=True
+
+    Manual selection (alternative to state-based):
+    - select: dbt selector syntax (e.g., "raw_customers", "tag:lookup")
+    - exclude: Exclude specific seeds
+
+    Important: Change detection for seeds works via file hash comparison:
+    - Seeds < 1 MiB: Content hash is compared (recommended)
+    - Seeds >= 1 MiB: Only file path changes are detected (content changes ignored)
+    For large seeds, use manual selection or run all seeds.
+
+    Args:
+        select: Manual selector for seeds
+        exclude: Exclude selector
+        select_state_modified: Use state:modified selector (changed seeds only)
+        select_state_modified_plus_downstream: Extend to state:modified+ (changed + downstream)
+        full_refresh: Truncate and reload seed tables (default behavior)
+        show: Show preview of loaded data
+        state: Shared state object injected by FastMCP
+
+    Returns:
+        Seed results with status and loaded seed info
+
+    See also:
+        - run_models(): Execute .sql model files (not CSV seeds)
+        - build_models(): Runs both seeds and models together in DAG order
+        - test_models(): Run tests (requires seeds to be loaded first if tests reference them)
+
+    Examples:
+        # Before running tests that depend on reference data
+        load_seeds()
+        test_models(select="test_customer_country_code")
+
+        # After adding a new CSV lookup table
+        load_seeds(select="new_product_categories")
+
+        # Fix "relation does not exist" errors from models referencing seeds
+        load_seeds()  # Load missing seed tables first
+        run_models(select="stg_orders")
+
+        # Incremental workflow: only reload what changed
+        load_seeds(select_state_modified=True)
+
+        # Full refresh of a specific seed
+        load_seeds(select="country_codes", full_refresh=True)
+    """
+    return await _implementation(ctx, select, exclude, select_state_modified, select_state_modified_plus_downstream, full_refresh, show, state)