kelpmesh-core 1.0.0__py3-none-any.whl

kelpmesh/__init__.py

@@ -0,0 +1,6 @@
+"""
+kelpmesh — Code-native data transformation platform (SQL & Python models).
+Zero telemetry. Zero analytics. Zero phone-home.
+"""
+
+__phone_home__ = False

kelpmesh/__main__.py

@@ -0,0 +1,3 @@
+from kelpmesh.cli.main import main
+
+main()

kelpmesh/adapters/__init__.py

@@ -0,0 +1,51 @@
+from kelpmesh.adapters.base import WarehouseAdapter
+from kelpmesh.core.config import WarehouseConfig
+
+
+def get_adapter(config: WarehouseConfig, project_path: str | None = None) -> WarehouseAdapter:
+    match config.type:
+        case "duckdb":
+            from kelpmesh.adapters.duckdb import DuckDBAdapter
+            return DuckDBAdapter(config, project_path=project_path)
+        case "snowflake":
+            from kelpmesh.adapters.snowflake import SnowflakeAdapter
+            return SnowflakeAdapter(config)
+        case "bigquery":
+            from kelpmesh.adapters.bigquery import BigQueryAdapter
+            return BigQueryAdapter(config)
+        case "postgres":
+            from kelpmesh.adapters.postgres import PostgresAdapter
+            return PostgresAdapter(config)
+        case "redshift":
+            from kelpmesh.adapters.redshift import RedshiftAdapter
+            return RedshiftAdapter(config)
+        case "databricks":
+            from kelpmesh.adapters.databricks import DatabricksAdapter
+            return DatabricksAdapter(config)
+        case "fabric":
+            from kelpmesh.adapters.fabric import FabricAdapter
+            return FabricAdapter(config)
+        case "mysql" | "mariadb":
+            from kelpmesh.adapters.mysql import MySQLAdapter
+            return MySQLAdapter(config)
+        case "trino" | "presto":
+            from kelpmesh.adapters.trino import TrinoAdapter
+            return TrinoAdapter(config)
+        case "clickhouse":
+            from kelpmesh.adapters.clickhouse import ClickHouseAdapter
+            return ClickHouseAdapter(config)
+        case "spark":
+            from kelpmesh.adapters.spark import SparkAdapter
+            return SparkAdapter(config)
+        case "athena":
+            from kelpmesh.adapters.athena import AthenaAdapter
+            return AthenaAdapter(config)
+        case "hive":
+            from kelpmesh.adapters.hive import HiveAdapter
+            return HiveAdapter(config)
+        case "sqlserver" | "mssql" | "synapse" | "azuresynapse":
+            from kelpmesh.adapters.sqlserver import SQLServerAdapter
+            return SQLServerAdapter(config)
+        case _:
+            from kelpmesh.adapters.duckdb import DuckDBAdapter
+            return DuckDBAdapter(config, project_path=project_path)

kelpmesh/adapters/athena.py

@@ -0,0 +1,347 @@
+"""Amazon Athena adapter for KelpMesh.
+
+Install the driver:
+    pip install kelpmesh[athena]
+
+kelpmesh.yml:
+    warehouse:
+      type: athena
+      host: us-east-1                        # AWS region
+      database: my_glue_database
+      path: "s3://my-bucket/athena-results/" # S3 staging dir for query results
+      user: "{{ env_var('AWS_ACCESS_KEY_ID') }}"
+      password: "{{ env_var('AWS_SECRET_ACCESS_KEY') }}"
+
+Notes:
+  - `host`  maps to the AWS region (e.g. us-east-1).
+  - `path`  is the S3 staging directory used by Athena for result output.
+  - `user`  / `password` are the AWS access key id / secret access key.
+    Leave both unset to use the default credential chain (IAM role, env
+    vars, ~/.aws/credentials, etc.).
+  - Athena does not support INSERT INTO on CTAS tables; incremental runs
+    use a CTAS-then-rename workaround.
+  - External table creation requires `s3_location` to be passed as a
+    keyword argument to execute_model via the `extra` dict (not yet
+    exposed in the standard interface — extend as needed).
+"""
+
+from __future__ import annotations
+
+from kelpmesh.adapters.base import WarehouseAdapter, sanitize_name
+from kelpmesh.core.config import WarehouseConfig
+
+
+class AthenaAdapter(WarehouseAdapter):
+    def __init__(self, config: WarehouseConfig) -> None:
+        self.config = config
+        self.conn = None
+
+    # ------------------------------------------------------------------
+    # Connection helpers
+    # ------------------------------------------------------------------
+
+    def connect(self) -> None:
+        try:
+            import pyathena
+        except ImportError:
+            raise ImportError(
+                "PyAthena not installed. Run: pip install kelpmesh[athena]"
+            )
+
+        kwargs: dict = {
+            "s3_staging_dir": self.config.path or "",
+            "region_name": self.config.host or "us-east-1",
+        }
+        if self.config.user:
+            kwargs["aws_access_key_id"] = self.config.user
+        if self.config.password:
+            kwargs["aws_secret_access_key"] = self.config.password
+        if self.config.database:
+            kwargs["schema_name"] = self.config.database
+
+        self.conn = pyathena.connect(**kwargs)
+
+    def disconnect(self) -> None:
+        if self.conn:
+            self.conn.close()
+            self.conn = None
+
+    def _ensure_conn(self, conn=None):
+        c = conn or self.conn
+        if not c:
+            self.connect()
+            return self.conn
+        return c
+
+    # ------------------------------------------------------------------
+    # execute
+    # ------------------------------------------------------------------
+
+    def execute(self, sql: str, conn=None) -> list[dict]:
+        c = self._ensure_conn(conn)
+        cursor = c.cursor()
+        try:
+            cursor.execute(sql)
+            if cursor.description:
+                cols = [d[0] for d in cursor.description]
+                return [dict(zip(cols, row)) for row in cursor.fetchall()]
+            return []
+        finally:
+            cursor.close()
+
+    # ------------------------------------------------------------------
+    # execute_model
+    # ------------------------------------------------------------------
+
+    def execute_model(
+        self,
+        sql: str,
+        table_name: str,
+        materialized: str = "view",
+        conn=None,
+        unique_key: str | None = None,
+        incremental_strategy: str = "append",
+    ) -> None:
+        db = self.config.database or ""
+        safe = f"`{db}`.`{table_name}`" if db else f"`{table_name}`"
+        c = self._ensure_conn(conn)
+
+        if materialized == "incremental":
+            if self.table_exists(table_name, conn=c):
+                # Athena CTAS tables do not support INSERT INTO.
+                # Workaround: create a new CTAS table, then swap.
+                tmp = f"_km_inc_{table_name}"
+                safe_tmp = f"`{db}`.`{tmp}`" if db else f"`{tmp}`"
+                # Drop any leftover temp table
+                try:
+                    self.execute(f"DROP TABLE IF EXISTS {safe_tmp}", conn=c)
+                except Exception:
+                    pass
+                self.execute(f"CREATE TABLE {safe_tmp} AS {sql}", conn=c)
+                self.execute(f"DROP TABLE IF EXISTS {safe}", conn=c)
+                self.execute(
+                    f"ALTER TABLE {safe_tmp} RENAME TO `{table_name}`", conn=c
+                )
+            else:
+                self.execute(f"CREATE TABLE {safe} AS {sql}", conn=c)
+            return
+
+        self.drop_table(table_name, materialized, conn=c)
+        if materialized == "table":
+            self.execute(f"CREATE TABLE {safe} AS {sql}", conn=c)
+        elif materialized == "ephemeral":
+            pass
+        else:
+            self.execute(f"CREATE OR REPLACE VIEW {safe} AS {sql}", conn=c)
+
+    # ------------------------------------------------------------------
+    # table_exists
+    # ------------------------------------------------------------------
+
+    def table_exists(self, table_name: str, conn=None) -> bool:
+        db = self.config.database or ""
+        c = self._ensure_conn(conn)
+        try:
+            if db:
+                rows = self.execute(
+                    "SELECT COUNT(*) AS cnt FROM information_schema.tables "
+                    f"WHERE table_schema = '{db}' AND table_name = '{table_name}'",
+                    conn=c,
+                )
+            else:
+                rows = self.execute(
+                    "SELECT COUNT(*) AS cnt FROM information_schema.tables "
+                    f"WHERE table_name = '{table_name}'",
+                    conn=c,
+                )
+            return (rows[0].get("cnt") or 0) > 0 if rows else False
+        except Exception:
+            return False
+
+    # ------------------------------------------------------------------
+    # table_schema
+    # ------------------------------------------------------------------
+
+    def table_schema(self, table_name: str, conn=None) -> list[dict]:
+        db = self.config.database or ""
+        c = self._ensure_conn(conn)
+        if db:
+            rows = self.execute(
+                "SELECT column_name, data_type, is_nullable "
+                "FROM information_schema.columns "
+                f"WHERE table_schema = '{db}' AND table_name = '{table_name}' "
+                "ORDER BY ordinal_position",
+                conn=c,
+            )
+        else:
+            rows = self.execute(
+                "SELECT column_name, data_type, is_nullable "
+                "FROM information_schema.columns "
+                f"WHERE table_name = '{table_name}' "
+                "ORDER BY ordinal_position",
+                conn=c,
+            )
+        return rows or []
+
+    # ------------------------------------------------------------------
+    # drop_table
+    # ------------------------------------------------------------------
+
+    def drop_table(self, table_name: str, materialized: str = "view", conn=None) -> None:
+        db = self.config.database or ""
+        safe = f"`{db}`.`{table_name}`" if db else f"`{table_name}`"
+        c = self._ensure_conn(conn)
+        if materialized == "view":
+            self.execute(f"DROP VIEW IF EXISTS {safe}", conn=c)
+        else:
+            self.execute(f"DROP TABLE IF EXISTS {safe}", conn=c)
+
+    # ------------------------------------------------------------------
+    # execute_snapshot  (CTAS workaround — Athena has no MERGE INTO)
+    # ------------------------------------------------------------------
+
+    def execute_snapshot(
+        self,
+        sql: str,
+        table_name: str,
+        unique_key: str,
+        strategy: str = "timestamp",
+        updated_at: str = "updated_at",
+        conn=None,
+    ) -> None:
+        """SCD Type 2 snapshot for Athena.
+
+        Athena (Presto/Trino SQL engine) does not support MERGE INTO or UPDATE.
+        The implementation uses a full-rebuild CTAS pattern:
+
+        1. Read the existing snapshot table (if it exists).
+        2. Identify changed rows by joining with incoming data.
+        3. Expire changed rows by setting _valid_to / _is_current.
+        4. Union everything into a new CTAS table, then swap.
+        """
+        db = self.config.database or ""
+        safe = f"`{db}`.`{table_name}`" if db else f"`{table_name}`"
+        stage = f"_km_snap_{table_name}"
+        safe_stage = f"`{db}`.`{stage}`" if db else f"`{stage}`"
+        new_table = f"_km_snap_new_{table_name}"
+        safe_new = f"`{db}`.`{new_table}`" if db else f"`{new_table}`"
+        c = self._ensure_conn(conn)
+
+        try:
+            if not self.table_exists(table_name, conn=c):
+                if strategy == "timestamp":
+                    dbt_updated_expr = f"CAST(\"{updated_at}\" AS TIMESTAMP)"
+                else:
+                    dbt_updated_expr = "current_timestamp"
+
+                self.execute(f"""
+                    CREATE TABLE {safe} AS
+                    SELECT *,
+                        to_hex(md5(to_utf8(CAST("{unique_key}" AS VARCHAR)))) AS _scd_id,
+                        current_timestamp                                       AS _valid_from,
+                        CAST(NULL AS TIMESTAMP)                                 AS _valid_to,
+                        TRUE                                                    AS _is_current,
+                        {dbt_updated_expr}                                      AS _dbt_updated_at
+                    FROM ({sql}) _src
+                """, conn=c)
+                return
+
+            # Stage incoming data as a view (Athena supports CREATE OR REPLACE VIEW)
+            try:
+                self.execute(f"DROP VIEW IF EXISTS {safe_stage}", conn=c)
+            except Exception:
+                pass
+            self.execute(f"CREATE VIEW {safe_stage} AS {sql}", conn=c)
+
+            if strategy == "timestamp":
+                changed_cond = (
+                    f"CAST(n.\"{updated_at}\" AS TIMESTAMP) > s._dbt_updated_at"
+                )
+            else:
+                cols_info = self.table_schema(table_name, conn=c)
+                audit_cols = {
+                    "_scd_id", "_valid_from", "_valid_to",
+                    "_is_current", "_dbt_updated_at",
+                }
+                check_cols = [
+                    r["column_name"] for r in cols_info
+                    if r["column_name"] != unique_key
+                    and r["column_name"] not in audit_cols
+                ]
+                changed_cond = (
+                    " OR ".join(
+                        f"n.\"{col}\" IS DISTINCT FROM s.\"{col}\""
+                        for col in check_cols
+                    )
+                    if check_cols
+                    else "FALSE"
+                )
+
+            if strategy == "timestamp":
+                dbt_updated_insert = f"CAST(n.\"{updated_at}\" AS TIMESTAMP)"
+            else:
+                dbt_updated_insert = "current_timestamp"
+
+            # Build the full rebuilt snapshot in a new CTAS table
+            try:
+                self.execute(f"DROP TABLE IF EXISTS {safe_new}", conn=c)
+            except Exception:
+                pass
+
+            self.execute(f"""
+                CREATE TABLE {safe_new} AS
+                -- Existing rows: expire changed ones, keep unchanged
+                SELECT s.*,
+                    CASE
+                        WHEN n."{unique_key}" IS NOT NULL AND ({changed_cond})
+                        THEN current_timestamp
+                        ELSE s._valid_to
+                    END AS _valid_to_new,
+                    CASE
+                        WHEN n."{unique_key}" IS NOT NULL AND ({changed_cond})
+                        THEN FALSE
+                        ELSE s._is_current
+                    END AS _is_current_new
+                FROM {safe} s
+                LEFT JOIN {safe_stage} n ON n."{unique_key}" = s."{unique_key}"
+                    AND s._is_current = TRUE
+
+                UNION ALL
+
+                -- New rows for changed or brand-new keys
+                SELECT n.*,
+                    to_hex(md5(to_utf8(CAST(n."{unique_key}" AS VARCHAR)))) AS _scd_id,
+                    current_timestamp                                         AS _valid_from,
+                    CAST(NULL AS TIMESTAMP)                                   AS _valid_to_new,
+                    TRUE                                                      AS _is_current_new,
+                    {dbt_updated_insert}                                      AS _dbt_updated_at
+                FROM {safe_stage} n
+                WHERE NOT EXISTS (
+                    SELECT 1 FROM {safe} s
+                    WHERE s."{unique_key}" = n."{unique_key}" AND s._is_current = TRUE
+                )
+                   OR EXISTS (
+                    SELECT 1 FROM {safe} s
+                    WHERE s."{unique_key}" = n."{unique_key}"
+                      AND s._is_current = TRUE
+                      AND ({changed_cond})
+                )
+            """, conn=c)
+
+            # Swap: drop original, rename new
+            self.execute(f"DROP TABLE IF EXISTS {safe}", conn=c)
+            self.execute(
+                f"ALTER TABLE {safe_new} RENAME TO `{table_name}`", conn=c
+            )
+            self.execute(f"DROP VIEW IF EXISTS {safe_stage}", conn=c)
+
+        except Exception:
+            try:
+                self.execute(f"DROP VIEW IF EXISTS {safe_stage}", conn=c)
+            except Exception:
+                pass
+            try:
+                self.execute(f"DROP TABLE IF EXISTS {safe_new}", conn=c)
+            except Exception:
+                pass
+            raise

kelpmesh/adapters/base.py

@@ -0,0 +1,139 @@
+import re
+from abc import ABC, abstractmethod
+from typing import Any
+
+_IDENTIFIER_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+
+
+def sanitize_name(name: str) -> str:
+    """Validate and quote a SQL identifier to prevent injection."""
+    if not _IDENTIFIER_RE.match(name):
+        raise ValueError(f"Invalid SQL identifier: {name!r}")
+    return f'"{name}"'
+
+
+class WarehouseAdapter(ABC):
+    @abstractmethod
+    def connect(self) -> None:
+        ...
+
+    @abstractmethod
+    def disconnect(self) -> None:
+        ...
+
+    @abstractmethod
+    def execute(self, sql: str, conn=None) -> Any:
+        ...
+
+    @abstractmethod
+    def execute_model(
+        self, sql: str, table_name: str, materialized: str = "view",
+        conn=None, unique_key: str | None = None,
+        incremental_strategy: str = "append",
+    ) -> None:
+        ...
+
+    @abstractmethod
+    def table_exists(self, table_name: str, conn=None) -> bool:
+        ...
+
+    @abstractmethod
+    def table_schema(self, table_name: str, conn=None) -> list[dict]:
+        ...
+
+    @abstractmethod
+    def drop_table(self, table_name: str, materialized: str = "view", conn=None) -> None:
+        ...
+
+    def acquire_conn(self) -> Any:
+        return None
+
+    def release_conn(self, conn: Any) -> None:
+        pass
+
+    def preview(self, sql: str, limit: int = 100, conn=None) -> list[dict]:
+        wrapped = f"SELECT * FROM ({sql}) AS _km_preview LIMIT {limit}"
+        return self.execute(wrapped, conn=conn)
+
+    def fetch_row_count(self, table_name: str, conn=None) -> int:
+        result = self.execute(f"SELECT COUNT(*) AS cnt FROM {table_name}", conn=conn)
+        if result and len(result) > 0:
+            return result[0]["cnt"]
+        return 0
+
+    def load_csv(self, path: str, table_name: str, delimiter: str = ",") -> None:
+        """Load a CSV/TSV file into a table. Override for warehouse-native ingest."""
+        import pandas as pd
+        df = pd.read_csv(path, sep=delimiter)
+        self._write_df(df, table_name)
+
+    def execute_snapshot(
+        self,
+        sql: str,
+        table_name: str,
+        unique_key: str,
+        strategy: str = "timestamp",
+        updated_at: str = "updated_at",
+        conn=None,
+    ) -> None:
+        """SCD Type 2 snapshot. Override per adapter."""
+        raise NotImplementedError(
+            f"Snapshots are not yet implemented for {self.__class__.__name__}. "
+            f"Supported: DuckDB, Postgres, Snowflake, BigQuery, Databricks, Fabric, Redshift."
+        )
+
+    def execute_materialized_view(
+        self,
+        sql: str,
+        table_name: str,
+        conn=None,
+    ) -> None:
+        """Create or refresh a materialized view. Falls back to table if unsupported."""
+        # Default: fall back to regular table (DuckDB, MySQL, Hive don't support MV natively)
+        self.drop_table(table_name, materialized="table", conn=conn)
+        self.execute_model(sql, table_name, materialized="table", conn=conn)
+
+    def _write_df(self, df, table_name: str) -> None:
+        """Write a pandas DataFrame to the warehouse as a table."""
+        import pandas as pd
+        from io import StringIO
+        buf = StringIO()
+        df.to_csv(buf, index=False)
+        buf.seek(0)
+        lines = []
+        for _ in range(20):
+            line = buf.readline()
+            if not line:
+                break
+            lines.append(line.strip())
+        header = lines[0].split(",") if lines else []
+        sample = []
+        for line in lines[1:]:
+            vals = line.split(",")
+            if len(vals) == len(header):
+                sample.append(vals)
+        import re
+        clean = re.sub(r"[^a-zA-Z0-9_]", "_", table_name)
+        col_defs = ", ".join(
+            f'"{c}" VARCHAR' for c in header
+        )
+        stmt = f"CREATE TABLE IF NOT EXISTS \"{clean}\" ({col_defs})"
+        self.execute(stmt)
+        placeholders = ", ".join("?" for _ in header)
+        insert = f'INSERT INTO "{clean}" VALUES ({placeholders})'
+        for row in sample:
+            self.execute(insert, list(row))
+        # Rest via chunked INSERT from buffered CSV
+        buf.seek(0)
+        next(buf)  # skip header
+        chunk = []
+        for line in buf:
+            vals = line.strip().split(",")
+            if len(vals) == len(header):
+                chunk.append(vals)
+            if len(chunk) >= 500:
+                for row in chunk:
+                    self.execute(insert, row)
+                chunk = []
+        for row in chunk:
+            self.execute(insert, row)