datamcp-ai 0.1.0__tar.gz

datamcp_ai-0.1.0/.claude/scheduled_tasks.lock

@@ -0,0 +1 @@
+{"sessionId":"c2f0e0a2-3135-4fbc-9e9b-a88b9dde2cce","pid":75083,"procStart":"Sat Oct  3 11:24:24 2026","acquiredAt":1791047745871}

datamcp_ai-0.1.0/.claude/settings.local.json

@@ -0,0 +1,41 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(read d *)",
+      "Bash(touch __TRACKED_VAR__/__init__.py)",
+      "Bash(python3 *)",
+      "Bash(uv --version)",
+      "Bash(rtk uv *)",
+      "Bash(uv run *)",
+      "Bash(rtk read *)",
+      "Bash(timeout 5 .venv/bin/python -m datamcp.server.mcp_server)",
+      "Bash(kill %1)",
+      "Bash(.venv/bin/python *)",
+      "Bash(brew list *)",
+      "Bash(brew services *)",
+      "Bash(rtk ls *)",
+      "Bash(xargs ls -la)",
+      "Bash(/Users/dev/Developer/DataMCP/.venv/bin/dbt compile *)",
+      "Bash(/Users/dev/Developer/DataMCP/.venv/bin/python --version)",
+      "Bash(claude mcp *)",
+      "mcp__plugin_supabase_supabase__list_projects",
+      "Bash(env)",
+      "mcp__datamcp__connect_database",
+      "Bash(rtk pytest *)",
+      "Bash(python -c \"import datamcp; print\\('import ok'\\)\")",
+      "Bash(rtk grep *)",
+      "Bash(rtk ps *)",
+      "Bash(rtk pip *)",
+      "Bash(rtk git *)",
+      "Bash(rtk find *)",
+      "Bash(/usr/bin/find /Users/dev/Developer/DataMCP/src -name \"*.pyc\" -delete)",
+      "Bash(rtk wc *)",
+      "mcp__datamcp__generate_mart_models",
+      "mcp__datamcp__scaffold_dlt"
+    ]
+  },
+  "enableAllProjectMcpServers": true,
+  "enabledMcpjsonServers": [
+    "datamcp"
+  ]
+}

datamcp_ai-0.1.0/.env.example

@@ -0,0 +1,40 @@
+# ── Required ────────────────────────────────────────────────
+# Postgres connection (stored in keychain after datamcp configure)
+# Also used by run_pipeline.py and scripts/validate_e2e.sh
+DATAMCP_DB_HOST=localhost
+DATAMCP_DB_PORT=5432
+DATAMCP_DB_NAME=postgres
+DATAMCP_DB_USER=myuser
+DATAMCP_DB_PASSWORD=
+DATAMCP_DB_SCHEMA=public
+DATAMCP_DB_SSL_MODE=prefer   # use "require" for Supabase / managed Postgres
+DATAMCP_PROJECT_NAME=datamcp
+
+# AI inference — user brings their own key (never stored by DataMCP)
+GEMINI_API_KEY=...
+
+# Only needed if DATAMCP_AI_PROVIDER=openai
+OPENAI_API_KEY=sk-...
+
+# ── Optional ─────────────────────────────────────────────────
+# Override AI model (default: gemini-2.5-flash; use a gpt-* model if
+# DATAMCP_AI_PROVIDER=openai)
+DATAMCP_AI_MODEL=gemini-2.5-flash
+
+# Override AI provider (default: gemini)
+DATAMCP_AI_PROVIDER=gemini   # gemini | openai | anthropic | ollama
+
+# Output paths
+DATAMCP_PROJECT_DIR=./datamcp_output
+DATAMCP_DUCKDB_PATH=./dev.duckdb
+
+# dbt settings
+DATAMCP_DBT_THREADS=4
+DATAMCP_DBT_DEFAULT_TARGET=duckdb  # Default to local DuckDB
+
+# Freshness thresholds
+DATAMCP_FRESHNESS_WARN_HOURS=24
+DATAMCP_FRESHNESS_ERROR_HOURS=48
+
+# Logging
+DATAMCP_LOG_LEVEL=INFO

datamcp_ai-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: CI
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main]
+
+jobs:
+  quality:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v2
+      - run: uv sync --all-extras
+
+      - name: Lint
+        run: uv run ruff check src/ tests/
+
+      - name: Format
+        run: uv run ruff format --check src/ tests/
+
+      - name: Type check
+        run: uv run mypy src/datamcp --strict
+
+      - name: Tests
+        run: uv run pytest --cov=datamcp --cov-fail-under=90
+        env:
+          DATAMCP_TEST_DB_URL: postgresql://postgres:postgres@localhost:5432/datamcp_test
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_TEST }}
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: datamcp_test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432

datamcp_ai-0.1.0/.gitignore

@@ -0,0 +1,55 @@
+# Environment and credentials — NEVER commit these
+.env
+.mcp.json
+.dlt/secrets.toml
+*.duckdb
+dev.duckdb
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+dist/
+*.egg-info/
+.eggs/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# uv
+.uv/
+
+# dbt
+target/
+dbt_packages/
+logs/
+
+# DataMCP knowledge base
+datamcp_output/.datamcp_knowledge.db
+*.db
+
+# Generated project output (regeneratable, not source)
+datamcp_output/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# macOS
+.DS_Store
+
+# Testing
+.coverage
+htmlcov/
+.pytest_cache/
+
+# mypy
+.mypy_cache/

datamcp_ai-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.10.0
+    hooks:
+      - id: mypy
+        args: [--strict]
+        additional_dependencies:
+          - pydantic>=2.0.0
+          - pydantic-settings>=2.0.0
+          - types-Jinja2
+
+  - repo: local
+    hooks:
+      - id: pytest
+        name: pytest
+        entry: uv run pytest --no-cov -x -q
+        language: system
+        pass_filenames: false
+        always_run: true

datamcp_ai-0.1.0/PKG-INFO

@@ -0,0 +1,205 @@
+Metadata-Version: 2.5
+Name: datamcp-ai
+Version: 0.1.0
+Summary: AI Data Engineer MCP — Postgres + dlt + dbt + Dagster
+License: Apache-2.0
+Keywords: dagster,data-engineering,dbt,dlt,mcp,postgres
+Requires-Python: >=3.11
+Requires-Dist: aiosqlite>=0.22.1
+Requires-Dist: asyncpg>=0.31.0
+Requires-Dist: dagster-dbt>=0.29.25
+Requires-Dist: dagster-webserver>=1.13.25
+Requires-Dist: dagster>=1.13.25
+Requires-Dist: dbt-core>=1.12.5
+Requires-Dist: dbt-duckdb>=1.11.0
+Requires-Dist: dbt-postgres>=1.11.0
+Requires-Dist: dlt>=1.30.0
+Requires-Dist: duckdb>=1.5.6
+Requires-Dist: google-genai>=2.27.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: jinja2>=3.1.0
+Requires-Dist: keyring>=25.7.0
+Requires-Dist: mcp>=2.2.0
+Requires-Dist: openai>=3.23.0
+Requires-Dist: pandas>=3.0.6
+Requires-Dist: plotly>=7.1.0
+Requires-Dist: psycopg2-binary>=2.9.13
+Requires-Dist: pydantic-settings>=2.15.0
+Requires-Dist: pydantic>=2.13.5
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: sqlite-vec>=0.1.9
+Requires-Dist: streamlit>=1.64.0
+Requires-Dist: typer>=0.27.2
+Provides-Extra: dev
+Requires-Dist: mypy>=2.4; extra == 'dev'
+Requires-Dist: pre-commit>=4.6.2; extra == 'dev'
+Requires-Dist: pytest-asyncio>=1.4.0; extra == 'dev'
+Requires-Dist: pytest-cov>=7.1.0; extra == 'dev'
+Requires-Dist: pytest>=9.1.1; extra == 'dev'
+Requires-Dist: ruff>=0.16; extra == 'dev'
+Provides-Extra: kafka
+Requires-Dist: kafka-python>=2.0.0; extra == 'kafka'
+Provides-Extra: typesafe
+Requires-Dist: typesafe-sdk>=0.7.2; extra == 'typesafe'
+Description-Content-Type: text/markdown
+
+# DataMCP — AI Data Engineer
+
+> Connect your Postgres database. Get a production dbt project in under 10 minutes.
+
+DataMCP is an MCP server that acts as an AI data engineer inside Claude Code, Cursor, and Codex. It connects to your Postgres database, reads and maps the schema, infers the business domain, and generates a complete, production-quality dbt project — automatically.
+
+## Quickstart
+
+```bash
+# Not yet on PyPI — install from source until the first release ships:
+git clone https://github.com/<org>/datamcp.git && cd datamcp
+pip install .
+
+datamcp init
+datamcp configure   # stores credentials in OS keychain, never on disk
+
+# Open Claude Code and type:
+# "Set up analytics for my project"
+```
+
+That's it. DataMCP calls the 10 core pipeline tools in sequence (of 30 MCP tools total — see below) and produces:
+- Staging models (one per source table, DuckDB-validated)
+- Fact and dimension mart models
+- 5–11 domain-specific KPI definitions
+- A dlt ingestion pipeline
+- A Dagster project with a daily schedule
+- 30+ data quality tests — all passing
+
+**Total time: under 10 minutes.**
+
+## Installation
+
+```bash
+# PyPI release is pending — install from source for now:
+git clone https://github.com/<org>/datamcp.git && cd datamcp
+pip install .
+```
+
+Requires Python 3.11+. Bring your own Gemini API key (OpenAI optional).
+
+## Configuration
+
+```bash
+datamcp configure
+# Prompts for: DB host, port, database, username, password, Gemini key
+# (OpenAI key optional — only needed if DATAMCP_AI_PROVIDER=openai)
+# All credentials stored in OS keychain — never on disk
+
+datamcp install --editor claude-code  # or cursor, codex
+```
+
+## MCP Tools (30)
+
+**Core pipeline** (steps 1–10 of the standard flow)
+
+| Tool | Description |
+|---|---|
+| `connect_database` | Establish and validate a Postgres connection |
+| `introspect_schema` | Read the complete Postgres schema: tables, columns, data types, PKs, FKs |
+| `infer_domain` | Classify the business domain from schema metadata only |
+| `generate_dbt_project` | Scaffold a complete dbt project with Postgres and DuckDB profiles |
+| `generate_staging_models` | Generate one staging model (stg_*.sql) per source table |
+| `generate_mart_models` | Generate fact and dimension mart models from the relationship graph |
+| `generate_kpis` | Generate domain-specific KPI definitions as dbt metrics |
+| `run_dbt` | Execute a dbt command (run, test, compile, build) |
+| `check_freshness` | Check when each source table last received new rows |
+| `get_pipeline_status` | Get an aggregated health dashboard across all pipeline components |
+
+**Explore & query**
+
+| Tool | Description |
+|---|---|
+| `query_knowledge_base` | Semantic search over the project knowledge base |
+| `explain_model` | Get a plain-English explanation of any dbt model |
+| `query_data` | Ask a natural language question about the data |
+| `execute_sql` | Execute a raw SELECT query against the connected database |
+
+**Schema analysis & scaffolding**
+
+| Tool | Description |
+|---|---|
+| `get_project_id` | Look up the internal project UUID for a project_name, creating it on first use |
+| `generate_er_diagram` | Generate a Mermaid erDiagram from the current schema snapshot |
+| `infer_model_candidates` | Classify tables as fact, dimension, bridge, lookup, or scd_snapshot candidates |
+| `generate_dbt_macros` | Generate dbt macros derived from schema column patterns |
+| `scaffold_dagster` | Generate a complete Dagster project with dbt asset definitions and a daily schedule |
+| `check_dlt_alignment` | Compare the dlt source schema against dbt staging models for drift |
+| `configure_postgres_sink` | Generate a Kafka Connect connector config for Postgres |
+| `scaffold_dlt` | Generate a complete dlt ingestion pipeline for the connected Postgres database |
+| `run_dlt` | Execute a generated dlt pipeline.py and persist the run result |
+
+**Orchestration, BI & streaming**
+
+| Tool | Description |
+|---|---|
+| `run_dagster` | Start, stop, check status of, or trigger a run on a generated Dagster dev server |
+| `connect_metabase` | Authenticate to Metabase, register the connected database, and trigger an initial sync |
+| `sync_metabase` | Re-sync Metabase's metadata and flag charts referencing changed dbt models |
+| `create_metabase_dashboard` | Create one Metabase question per project KPI and group them into a dashboard |
+| `create_metabase_chart` | Turn a natural-language question into a real Metabase chart |
+| `provision_kafka_topic` | Create a Kafka topic for streaming ingestion |
+| `check_consumer_lag` | Check Kafka consumer group lag per partition |
+
+## Security
+
+- Database credentials: OS keychain only (Mac Keychain / Linux Secret Service / Windows Credential Manager)
+- AI inference: schema metadata only — table names, column names, types, FK relationships. Never actual data values, credentials, or PII.
+- Knowledge base: schema metadata and model definitions only. No row data.
+
+See [SECURITY.md](docs/SECURITY.md) and [DATA_PRIVACY.md](docs/DATA_PRIVACY.md).
+
+## What gets generated
+
+```
+datamcp_output/
+├── dbt/
+│   └── my_project/
+│       ├── dbt_project.yml
+│       ├── profiles.yml          # Postgres target
+│       ├── profiles_duckdb.yml   # DuckDB local dev target
+│       └── models/
+│           ├── staging/
+│           │   ├── sources.yml
+│           │   ├── stg_users.sql
+│           │   ├── stg_orders.sql
+│           │   └── ...
+│           └── marts/
+│               ├── fct_orders.sql
+│               ├── dim_customers.sql
+│               └── _metrics.yml
+├── dlt/
+│   ├── pipeline.py
+│   └── sources/my_project.py
+└── dagster_datamcp/
+    ├── assets/dbt_assets.py
+    ├── schedules/daily_schedule.py
+    └── definitions.py
+```
+
+## Running locally
+
+```bash
+# Run dbt against DuckDB (no Postgres needed)
+cd datamcp_output/dbt/my_project
+dbt run --target duckdb
+dbt test --target duckdb
+
+# Or run the full Dagster pipeline
+cd datamcp_output/dagster_datamcp
+dagster dev
+```
+
+## Contributing
+
+See [CONTRIBUTING.md](docs/CONTRIBUTING.md).
+
+## License
+
+Apache 2.0