dvt-core 0.59.0a51__py3-none-any.whl

dbt/include/data/build_registry.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+"""
+Build Script for DVT Adapters Registry
+
+This script reads CSV files from the csv/ directory and creates
+adapters_registry.duckdb with pre-populated type mappings, syntax rules,
+and adapter queries.
+
+Usage:
+    python build_registry.py
+
+The resulting adapters_registry.duckdb is shipped with the DVT package.
+"""
+
+import csv
+import os
+from pathlib import Path
+
+try:
+    import duckdb
+except ImportError:
+    print("Error: duckdb is required. Install with: pip install duckdb")
+    exit(1)
+
+
+def get_script_dir() -> Path:
+    """Get directory containing this script."""
+    return Path(__file__).parent
+
+
+def create_schema(conn: duckdb.DuckDBPyConnection) -> None:
+    """Create the database schema."""
+
+    # Table: datatype_mappings
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS datatype_mappings (
+            adapter_name VARCHAR NOT NULL,
+            adapter_type VARCHAR NOT NULL,
+            spark_type VARCHAR NOT NULL,
+            spark_version VARCHAR DEFAULT 'all',
+            is_complex BOOLEAN DEFAULT FALSE,
+            cast_expression VARCHAR,
+            notes VARCHAR,
+            UNIQUE (adapter_name, adapter_type, spark_version)
+        )
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS idx_datatype_lookup
+        ON datatype_mappings(adapter_name, adapter_type)
+    """)
+
+    # Table: syntax_registry
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS syntax_registry (
+            adapter_name VARCHAR NOT NULL PRIMARY KEY,
+            quote_start VARCHAR NOT NULL,
+            quote_end VARCHAR NOT NULL,
+            case_sensitivity VARCHAR NOT NULL,
+            reserved_keywords VARCHAR
+        )
+    """)
+
+    # Table: adapter_queries
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS adapter_queries (
+            adapter_name VARCHAR NOT NULL,
+            query_type VARCHAR NOT NULL,
+            query_template VARCHAR NOT NULL,
+            notes VARCHAR,
+            PRIMARY KEY (adapter_name, query_type)
+        )
+    """)
+
+    print("Schema created successfully")
+
+
+def load_type_mappings(conn: duckdb.DuckDBPyConnection, csv_dir: Path) -> int:
+    """Load type mappings from CSV files."""
+    total_rows = 0
+
+    # Find all type_mappings_*.csv files
+    for csv_file in sorted(csv_dir.glob("type_mappings_*.csv")):
+        adapter_name = csv_file.stem.replace("type_mappings_", "")
+        rows_loaded = 0
+
+        with open(csv_file, 'r', encoding='utf-8') as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                conn.execute("""
+                    INSERT OR REPLACE INTO datatype_mappings
+                    (adapter_name, adapter_type, spark_type, spark_version,
+                     is_complex, cast_expression, notes)
+                    VALUES (?, ?, ?, ?, ?, ?, ?)
+                """, [
+                    adapter_name,
+                    row['adapter_type'].strip(),
+                    row['spark_type'].strip(),
+                    row.get('spark_version', 'all').strip() or 'all',
+                    row.get('is_complex', 'false').strip().lower() == 'true',
+                    row.get('cast_expression', '').strip() or None,
+                    row.get('notes', '').strip() or None,
+                ])
+                rows_loaded += 1
+
+        print(f"  Loaded {rows_loaded} type mappings for {adapter_name}")
+        total_rows += rows_loaded
+
+    return total_rows
+
+
+def load_syntax_rules(conn: duckdb.DuckDBPyConnection, csv_dir: Path) -> int:
+    """Load syntax rules from CSV file."""
+    csv_file = csv_dir / "syntax_rules.csv"
+    if not csv_file.exists():
+        print("  Warning: syntax_rules.csv not found")
+        return 0
+
+    rows_loaded = 0
+    with open(csv_file, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            conn.execute("""
+                INSERT OR REPLACE INTO syntax_registry
+                (adapter_name, quote_start, quote_end, case_sensitivity, reserved_keywords)
+                VALUES (?, ?, ?, ?, ?)
+            """, [
+                row['adapter_name'].strip(),
+                row['quote_start'].strip(),
+                row['quote_end'].strip(),
+                row['case_sensitivity'].strip(),
+                row.get('reserved_keywords', '').strip() or None,
+            ])
+            rows_loaded += 1
+
+    print(f"  Loaded {rows_loaded} syntax rules")
+    return rows_loaded
+
+
+def load_adapter_queries(conn: duckdb.DuckDBPyConnection, csv_dir: Path) -> int:
+    """Load adapter queries from CSV file."""
+    csv_file = csv_dir / "adapter_queries.csv"
+    if not csv_file.exists():
+        print("  Warning: adapter_queries.csv not found")
+        return 0
+
+    rows_loaded = 0
+    with open(csv_file, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            conn.execute("""
+                INSERT OR REPLACE INTO adapter_queries
+                (adapter_name, query_type, query_template, notes)
+                VALUES (?, ?, ?, ?)
+            """, [
+                row['adapter_name'].strip(),
+                row['query_type'].strip(),
+                row['query_template'].strip(),
+                row.get('notes', '').strip() or None,
+            ])
+            rows_loaded += 1
+
+    print(f"  Loaded {rows_loaded} adapter queries")
+    return rows_loaded
+
+
+def print_stats(conn: duckdb.DuckDBPyConnection) -> None:
+    """Print statistics about the loaded data."""
+    print("\n=== Registry Statistics ===")
+
+    # Type mappings by adapter
+    result = conn.execute("""
+        SELECT adapter_name, COUNT(*) as count
+        FROM datatype_mappings
+        GROUP BY adapter_name
+        ORDER BY adapter_name
+    """).fetchall()
+    print("\nType mappings per adapter:")
+    for row in result:
+        print(f"  {row[0]}: {row[1]}")
+
+    # Syntax rules
+    result = conn.execute("SELECT COUNT(*) FROM syntax_registry").fetchone()
+    print(f"\nSyntax rules: {result[0]} adapters")
+
+    # Adapter queries
+    result = conn.execute("""
+        SELECT adapter_name, COUNT(*) as count
+        FROM adapter_queries
+        GROUP BY adapter_name
+        ORDER BY adapter_name
+    """).fetchall()
+    print("\nAdapter queries:")
+    for row in result:
+        print(f"  {row[0]}: {row[1]} queries")
+
+
+def main():
+    script_dir = get_script_dir()
+    csv_dir = script_dir / "csv"
+    db_path = script_dir / "adapters_registry.duckdb"
+
+    print(f"Building adapters_registry.duckdb")
+    print(f"CSV directory: {csv_dir}")
+    print(f"Output: {db_path}")
+    print()
+
+    # Remove existing database
+    if db_path.exists():
+        os.remove(db_path)
+        print("Removed existing database")
+
+    # Create new database
+    conn = duckdb.connect(str(db_path))
+
+    try:
+        # Create schema
+        print("\nCreating schema...")
+        create_schema(conn)
+
+        # Load data
+        print("\nLoading type mappings...")
+        type_count = load_type_mappings(conn, csv_dir)
+
+        print("\nLoading syntax rules...")
+        syntax_count = load_syntax_rules(conn, csv_dir)
+
+        print("\nLoading adapter queries...")
+        query_count = load_adapter_queries(conn, csv_dir)
+
+        # Print stats
+        print_stats(conn)
+
+        print(f"\n=== Build Complete ===")
+        print(f"Total: {type_count} type mappings, {syntax_count} syntax rules, {query_count} queries")
+        print(f"Database size: {db_path.stat().st_size / 1024:.1f} KB")
+
+    finally:
+        conn.close()
+
+
+if __name__ == "__main__":
+    main()

dbt/include/data/csv/adapter_queries.csv

@@ -0,0 +1,33 @@
+adapter_name,query_type,query_template,notes
+postgres,columns,"SELECT column_name, data_type, is_nullable, ordinal_position FROM information_schema.columns WHERE table_schema = '{schema}' AND table_name = '{table}' ORDER BY ordinal_position",Get column metadata
+postgres,tables,"SELECT table_name FROM information_schema.tables WHERE table_schema = '{schema}' AND table_type = 'BASE TABLE'",List tables in schema
+postgres,row_count,"SELECT COUNT(*) FROM ""{schema}"".""{table}""",Count rows in table
+postgres,primary_key,"SELECT kcu.column_name FROM information_schema.table_constraints tc JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name WHERE tc.table_schema = '{schema}' AND tc.table_name = '{table}' AND tc.constraint_type = 'PRIMARY KEY'",Get primary key columns
+snowflake,columns,"SELECT column_name, data_type, is_nullable, ordinal_position FROM information_schema.columns WHERE table_schema = '{schema}' AND table_name = '{table}' ORDER BY ordinal_position",Get column metadata
+snowflake,tables,"SELECT table_name FROM information_schema.tables WHERE table_schema = '{schema}' AND table_type = 'BASE TABLE'",List tables in schema
+snowflake,row_count,"SELECT COUNT(*) FROM ""{schema}"".""{table}""",Count rows in table
+snowflake,primary_key,"SELECT column_name FROM information_schema.table_constraints tc JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name WHERE tc.table_schema = '{schema}' AND tc.table_name = '{table}' AND tc.constraint_type = 'PRIMARY KEY'",Get primary key columns
+mysql,columns,"SELECT column_name, data_type, is_nullable, ordinal_position FROM information_schema.columns WHERE table_schema = '{schema}' AND table_name = '{table}' ORDER BY ordinal_position",Get column metadata
+mysql,tables,"SELECT table_name FROM information_schema.tables WHERE table_schema = '{schema}' AND table_type = 'BASE TABLE'",List tables in schema
+mysql,row_count,"SELECT COUNT(*) FROM `{schema}`.`{table}`",Count rows in table
+mysql,primary_key,"SELECT column_name FROM information_schema.key_column_usage WHERE table_schema = '{schema}' AND table_name = '{table}' AND constraint_name = 'PRIMARY'",Get primary key columns
+bigquery,columns,"SELECT column_name, data_type, is_nullable, ordinal_position FROM `{project}`.`{schema}`.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '{table}' ORDER BY ordinal_position",Get column metadata
+bigquery,tables,"SELECT table_name FROM `{project}`.`{schema}`.INFORMATION_SCHEMA.TABLES WHERE table_type = 'BASE TABLE'",List tables in dataset
+bigquery,row_count,"SELECT COUNT(*) FROM `{project}`.`{schema}`.`{table}`",Count rows in table
+bigquery,primary_key,"SELECT column_name FROM `{project}`.`{schema}`.INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE table_name = '{table}' AND constraint_name LIKE 'pk_%'",Get primary key columns
+redshift,columns,"SELECT column_name, data_type, is_nullable, ordinal_position FROM information_schema.columns WHERE table_schema = '{schema}' AND table_name = '{table}' ORDER BY ordinal_position",Get column metadata
+redshift,tables,"SELECT table_name FROM information_schema.tables WHERE table_schema = '{schema}' AND table_type = 'BASE TABLE'",List tables in schema
+redshift,row_count,"SELECT COUNT(*) FROM ""{schema}"".""{table}""",Count rows in table
+redshift,primary_key,"SELECT kcu.column_name FROM information_schema.table_constraints tc JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name WHERE tc.table_schema = '{schema}' AND tc.table_name = '{table}' AND tc.constraint_type = 'PRIMARY KEY'",Get primary key columns
+oracle,columns,"SELECT column_name, data_type, nullable as is_nullable, column_id as ordinal_position FROM all_tab_columns WHERE owner = UPPER('{schema}') AND table_name = UPPER('{table}') ORDER BY column_id",Get column metadata
+oracle,tables,"SELECT table_name FROM all_tables WHERE owner = UPPER('{schema}')",List tables in schema
+oracle,row_count,"SELECT COUNT(*) FROM ""{schema}"".""{table}""",Count rows in table
+oracle,primary_key,"SELECT cols.column_name FROM all_constraints cons JOIN all_cons_columns cols ON cons.constraint_name = cols.constraint_name WHERE cons.owner = UPPER('{schema}') AND cons.table_name = UPPER('{table}') AND cons.constraint_type = 'P'",Get primary key columns
+sqlserver,columns,"SELECT column_name, data_type, is_nullable, ordinal_position FROM information_schema.columns WHERE table_schema = '{schema}' AND table_name = '{table}' ORDER BY ordinal_position",Get column metadata
+sqlserver,tables,"SELECT table_name FROM information_schema.tables WHERE table_schema = '{schema}' AND table_type = 'BASE TABLE'",List tables in schema
+sqlserver,row_count,"SELECT COUNT(*) FROM [{schema}].[{table}]",Count rows in table
+sqlserver,primary_key,"SELECT kcu.column_name FROM information_schema.table_constraints tc JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name WHERE tc.table_schema = '{schema}' AND tc.table_name = '{table}' AND tc.constraint_type = 'PRIMARY KEY'",Get primary key columns
+databricks,columns,"SELECT column_name, data_type, is_nullable, ordinal_position FROM information_schema.columns WHERE table_schema = '{schema}' AND table_name = '{table}' ORDER BY ordinal_position",Get column metadata
+databricks,tables,"SELECT table_name FROM information_schema.tables WHERE table_schema = '{schema}' AND table_type IN ('BASE TABLE', 'MANAGED', 'EXTERNAL')",List tables in schema
+databricks,row_count,"SELECT COUNT(*) FROM `{schema}`.`{table}`",Count rows in table
+databricks,primary_key,"SELECT column_name FROM system.information_schema.table_constraints tc JOIN system.information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name WHERE tc.table_schema = '{schema}' AND tc.table_name = '{table}' AND tc.constraint_type = 'PRIMARY KEY'",Get primary key columns

dbt/include/data/csv/syntax_rules.csv

@@ -0,0 +1,9 @@
+adapter_name,quote_start,quote_end,case_sensitivity,reserved_keywords
+postgres,"""","""",lowercase,"SELECT,FROM,WHERE,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,TABLE,INDEX,VIEW,SCHEMA,DATABASE,USER,GRANT,REVOKE,AND,OR,NOT,NULL,TRUE,FALSE,IN,BETWEEN,LIKE,IS,AS,ON,JOIN,LEFT,RIGHT,INNER,OUTER,FULL,CROSS,UNION,EXCEPT,INTERSECT,ORDER,BY,GROUP,HAVING,LIMIT,OFFSET,DISTINCT,ALL,ANY,EXISTS,CASE,WHEN,THEN,ELSE,END,CAST,COALESCE,NULLIF"
+snowflake,"""","""",uppercase,"SELECT,FROM,WHERE,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,TABLE,INDEX,VIEW,SCHEMA,DATABASE,USER,GRANT,REVOKE,AND,OR,NOT,NULL,TRUE,FALSE,IN,BETWEEN,LIKE,ILIKE,IS,AS,ON,JOIN,LEFT,RIGHT,INNER,OUTER,FULL,CROSS,UNION,EXCEPT,INTERSECT,MINUS,ORDER,BY,GROUP,HAVING,LIMIT,OFFSET,FETCH,DISTINCT,ALL,ANY,EXISTS,CASE,WHEN,THEN,ELSE,END,CAST,COALESCE,NULLIF,IFF,QUALIFY,PIVOT,UNPIVOT,LATERAL,FLATTEN,VARIANT,OBJECT,ARRAY"
+mysql,`,`,lowercase,"SELECT,FROM,WHERE,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,TABLE,INDEX,VIEW,SCHEMA,DATABASE,USER,GRANT,REVOKE,AND,OR,NOT,NULL,TRUE,FALSE,IN,BETWEEN,LIKE,IS,AS,ON,JOIN,LEFT,RIGHT,INNER,OUTER,CROSS,UNION,ORDER,BY,GROUP,HAVING,LIMIT,OFFSET,DISTINCT,ALL,ANY,EXISTS,CASE,WHEN,THEN,ELSE,END,CAST,COALESCE,NULLIF,IF,IFNULL,DIV,MOD,XOR,REGEXP,RLIKE"
+bigquery,`,`,lowercase,"SELECT,FROM,WHERE,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,TABLE,VIEW,SCHEMA,DATASET,PROJECT,AND,OR,NOT,NULL,TRUE,FALSE,IN,BETWEEN,LIKE,IS,AS,ON,JOIN,LEFT,RIGHT,INNER,OUTER,FULL,CROSS,UNION,EXCEPT,INTERSECT,ORDER,BY,GROUP,HAVING,LIMIT,OFFSET,DISTINCT,ALL,ANY,EXISTS,CASE,WHEN,THEN,ELSE,END,CAST,COALESCE,NULLIF,IF,IFNULL,STRUCT,ARRAY,UNNEST,PARTITION,CLUSTER,TABLESAMPLE"
+redshift,"""","""",lowercase,"SELECT,FROM,WHERE,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,TABLE,INDEX,VIEW,SCHEMA,DATABASE,USER,GRANT,REVOKE,AND,OR,NOT,NULL,TRUE,FALSE,IN,BETWEEN,LIKE,ILIKE,SIMILAR,IS,AS,ON,JOIN,LEFT,RIGHT,INNER,OUTER,FULL,CROSS,UNION,EXCEPT,INTERSECT,ORDER,BY,GROUP,HAVING,LIMIT,OFFSET,DISTINCT,ALL,ANY,EXISTS,CASE,WHEN,THEN,ELSE,END,CAST,COALESCE,NULLIF,NVL,NVL2,DECODE,DISTKEY,SORTKEY,DISTSTYLE"
+oracle,"""","""",uppercase,"SELECT,FROM,WHERE,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,TABLE,INDEX,VIEW,SCHEMA,USER,GRANT,REVOKE,AND,OR,NOT,NULL,IN,BETWEEN,LIKE,IS,AS,ON,JOIN,LEFT,RIGHT,INNER,OUTER,FULL,CROSS,UNION,EXCEPT,INTERSECT,MINUS,ORDER,BY,GROUP,HAVING,FETCH,FIRST,NEXT,ROWS,ONLY,OFFSET,DISTINCT,ALL,ANY,EXISTS,CASE,WHEN,THEN,ELSE,END,CAST,COALESCE,NULLIF,NVL,NVL2,DECODE,CONNECT,START,WITH,PRIOR,LEVEL,ROWNUM,ROWID,SYSDATE,SYSTIMESTAMP,DUAL"
+sqlserver,[,],case_insensitive,"SELECT,FROM,WHERE,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,TABLE,INDEX,VIEW,SCHEMA,DATABASE,USER,GRANT,REVOKE,AND,OR,NOT,NULL,IN,BETWEEN,LIKE,IS,AS,ON,JOIN,LEFT,RIGHT,INNER,OUTER,FULL,CROSS,UNION,EXCEPT,INTERSECT,ORDER,BY,GROUP,HAVING,TOP,OFFSET,FETCH,DISTINCT,ALL,ANY,EXISTS,CASE,WHEN,THEN,ELSE,END,CAST,COALESCE,NULLIF,ISNULL,IIF,CHOOSE,PIVOT,UNPIVOT,APPLY,CROSS,OUTER,GO,USE,EXEC,EXECUTE,PRINT,DECLARE,SET,BEGIN,COMMIT,ROLLBACK"
+databricks,`,`,case_insensitive,"SELECT,FROM,WHERE,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,TABLE,INDEX,VIEW,SCHEMA,DATABASE,CATALOG,AND,OR,NOT,NULL,TRUE,FALSE,IN,BETWEEN,LIKE,ILIKE,RLIKE,IS,AS,ON,JOIN,LEFT,RIGHT,INNER,OUTER,FULL,CROSS,SEMI,ANTI,UNION,EXCEPT,INTERSECT,ORDER,BY,GROUP,HAVING,LIMIT,OFFSET,DISTINCT,ALL,ANY,EXISTS,CASE,WHEN,THEN,ELSE,END,CAST,COALESCE,NULLIF,IF,IFNULL,NVL,STRUCT,ARRAY,MAP,LATERAL,EXPLODE,POSEXPLODE,INLINE,STACK,TABLESAMPLE,PIVOT,UNPIVOT,DISTRIBUTE,CLUSTER,SORT"

dbt/include/data/csv/type_mappings_bigquery.csv

@@ -0,0 +1,28 @@
+adapter_type,spark_type,spark_version,is_complex,cast_expression,notes
+STRING,StringType,all,false,,Variable-length Unicode string
+BYTES,BinaryType,all,false,,Variable-length binary
+INT64,LongType,all,false,,64-bit signed integer
+INTEGER,LongType,all,false,,Alias for INT64
+INT,LongType,all,false,,Alias for INT64
+SMALLINT,LongType,all,false,,Alias for INT64
+BIGINT,LongType,all,false,,Alias for INT64
+TINYINT,LongType,all,false,,Alias for INT64
+BYTEINT,LongType,all,false,,Alias for INT64
+FLOAT64,DoubleType,all,false,,64-bit floating point
+FLOAT,DoubleType,all,false,,Alias for FLOAT64
+NUMERIC,DecimalType,all,false,,Exact numeric (precision 38 scale 9)
+DECIMAL,DecimalType,all,false,,Alias for NUMERIC
+BIGNUMERIC,DecimalType,all,false,,High precision numeric (76.76)
+BIGDECIMAL,DecimalType,all,false,,Alias for BIGNUMERIC
+BOOL,BooleanType,all,false,,Boolean true/false
+BOOLEAN,BooleanType,all,false,,Alias for BOOL
+DATE,DateType,all,false,,Calendar date
+TIME,StringType,all,false,,Time of day
+DATETIME,TimestampType,all,false,,Date and time without timezone
+TIMESTAMP,TimestampType,all,false,,Date and time with timezone
+INTERVAL,StringType,all,false,,Time interval
+GEOGRAPHY,StringType,all,false,,Geographic data (GeoJSON)
+JSON,StringType,all,true,TO_JSON_STRING({}),JSON document
+STRUCT,StringType,all,true,TO_JSON_STRING({}),Structured record
+RECORD,StringType,all,true,TO_JSON_STRING({}),Alias for STRUCT
+ARRAY,ArrayType,all,true,,Array of values

dbt/include/data/csv/type_mappings_databricks.csv

@@ -0,0 +1,30 @@
+adapter_type,spark_type,spark_version,is_complex,cast_expression,notes
+STRING,StringType,all,false,,Variable-length string
+VARCHAR,StringType,all,false,,Variable-length string (alias)
+CHAR,StringType,all,false,,Fixed-length string
+BINARY,BinaryType,all,false,,Binary data
+TINYINT,ByteType,all,false,,8-bit signed integer
+BYTE,ByteType,all,false,,Alias for TINYINT
+SMALLINT,ShortType,all,false,,16-bit signed integer
+SHORT,ShortType,all,false,,Alias for SMALLINT
+INT,IntegerType,all,false,,32-bit signed integer
+INTEGER,IntegerType,all,false,,Alias for INT
+BIGINT,LongType,all,false,,64-bit signed integer
+LONG,LongType,all,false,,Alias for BIGINT
+FLOAT,FloatType,all,false,,32-bit floating point
+REAL,FloatType,all,false,,Alias for FLOAT
+DOUBLE,DoubleType,all,false,,64-bit floating point
+DECIMAL,DecimalType,all,false,,Exact numeric with precision
+DEC,DecimalType,all,false,,Alias for DECIMAL
+NUMERIC,DecimalType,all,false,,Alias for DECIMAL
+BOOLEAN,BooleanType,all,false,,True/false value
+DATE,DateType,all,false,,Calendar date
+TIMESTAMP,TimestampType,all,false,,Timestamp without timezone
+TIMESTAMP_NTZ,TimestampType,all,false,,Timestamp no timezone (explicit)
+TIMESTAMP_LTZ,TimestampType,all,false,,Timestamp with local timezone
+INTERVAL,StringType,all,false,,Time interval
+ARRAY,ArrayType,all,true,,Array of elements
+MAP,MapType,all,true,TO_JSON({}),Key-value pairs
+STRUCT,StringType,all,true,TO_JSON({}),Structured record
+VARIANT,StringType,all,true,TO_JSON({}),Semi-structured (Unity Catalog)
+OBJECT,StringType,all,true,TO_JSON({}),Object type (Unity Catalog)

dbt/include/data/csv/type_mappings_mysql.csv

@@ -0,0 +1,40 @@
+adapter_type,spark_type,spark_version,is_complex,cast_expression,notes
+VARCHAR,StringType,all,false,,Variable-length string
+CHAR,StringType,all,false,,Fixed-length string
+TEXT,StringType,all,false,,Variable unlimited length text
+TINYTEXT,StringType,all,false,,Tiny text (255 chars)
+MEDIUMTEXT,StringType,all,false,,Medium text (16MB)
+LONGTEXT,StringType,all,false,,Long text (4GB)
+TINYINT,ByteType,all,false,,8-bit signed integer
+SMALLINT,ShortType,all,false,,16-bit signed integer
+MEDIUMINT,IntegerType,all,false,,24-bit signed integer
+INT,IntegerType,all,false,,32-bit signed integer
+INTEGER,IntegerType,all,false,,Alias for INT
+BIGINT,LongType,all,false,,64-bit signed integer
+FLOAT,FloatType,all,false,,32-bit floating point
+DOUBLE,DoubleType,all,false,,64-bit floating point
+DOUBLE PRECISION,DoubleType,all,false,,Alias for DOUBLE
+REAL,DoubleType,all,false,,Alias for DOUBLE
+DECIMAL,DecimalType,all,false,,Exact numeric with precision
+NUMERIC,DecimalType,all,false,,Alias for DECIMAL
+BIT,BooleanType,all,false,,Bit value (1 bit = boolean)
+BOOLEAN,BooleanType,all,false,,Alias for TINYINT(1)
+BOOL,BooleanType,all,false,,Alias for BOOLEAN
+DATE,DateType,all,false,,Calendar date
+TIME,StringType,all,false,,Time of day
+DATETIME,TimestampType,all,false,,Date and time
+TIMESTAMP,TimestampType,all,false,,Timestamp with auto-update
+YEAR,IntegerType,all,false,,Year in 4-digit format
+BINARY,BinaryType,all,false,,Fixed-length binary
+VARBINARY,BinaryType,all,false,,Variable-length binary
+TINYBLOB,BinaryType,all,false,,Tiny binary (255 bytes)
+BLOB,BinaryType,all,false,,Binary large object (64KB)
+MEDIUMBLOB,BinaryType,all,false,,Medium blob (16MB)
+LONGBLOB,BinaryType,all,false,,Long blob (4GB)
+ENUM,StringType,all,false,,Enumeration type
+SET,StringType,all,false,,Set of values
+JSON,StringType,all,true,CAST({} AS STRING),JSON document
+GEOMETRY,StringType,all,false,,Spatial geometry
+POINT,StringType,all,false,,Spatial point
+LINESTRING,StringType,all,false,,Spatial line
+POLYGON,StringType,all,false,,Spatial polygon

dbt/include/data/csv/type_mappings_oracle.csv

@@ -0,0 +1,30 @@
+adapter_type,spark_type,spark_version,is_complex,cast_expression,notes
+VARCHAR2,StringType,all,false,,Variable-length string (max 4000/32767)
+NVARCHAR2,StringType,all,false,,Unicode VARCHAR2
+CHAR,StringType,all,false,,Fixed-length string
+NCHAR,StringType,all,false,,Unicode fixed-length
+CLOB,StringType,all,false,,Character large object
+NCLOB,StringType,all,false,,Unicode CLOB
+LONG,StringType,all,false,,Legacy long text (deprecated)
+NUMBER,DecimalType,all,false,,Numeric with precision and scale
+FLOAT,DoubleType,all,false,,Floating point (binary precision)
+BINARY_FLOAT,FloatType,all,false,,32-bit IEEE floating point
+BINARY_DOUBLE,DoubleType,all,false,,64-bit IEEE floating point
+INTEGER,LongType,all,false,,Alias for NUMBER(38)
+INT,LongType,all,false,,Alias for NUMBER(38)
+SMALLINT,LongType,all,false,,Alias for NUMBER(38)
+DATE,TimestampType,all,false,,Date and time (second precision)
+TIMESTAMP,TimestampType,all,false,,Timestamp (fractional seconds)
+TIMESTAMP WITH TIME ZONE,TimestampType,all,false,,Timestamp with timezone
+TIMESTAMP WITH LOCAL TIME ZONE,TimestampType,all,false,,Timestamp with session timezone
+INTERVAL YEAR TO MONTH,StringType,all,false,,Year-month interval
+INTERVAL DAY TO SECOND,StringType,all,false,,Day-second interval
+RAW,BinaryType,all,false,,Raw binary data
+LONG RAW,BinaryType,all,false,,Legacy raw (deprecated)
+BLOB,BinaryType,all,false,,Binary large object
+BFILE,StringType,all,false,,External file pointer
+ROWID,StringType,all,false,,Physical row address
+UROWID,StringType,all,false,,Universal ROWID
+XMLTYPE,StringType,all,true,XMLSERIALIZE(CONTENT {} AS CLOB),XML document
+JSON,StringType,all,true,,JSON document (21c+)
+SDO_GEOMETRY,StringType,all,false,,Spatial geometry

dbt/include/data/csv/type_mappings_postgres.csv

@@ -0,0 +1,56 @@
+adapter_type,spark_type,spark_version,is_complex,cast_expression,notes
+VARCHAR,StringType,all,false,,Variable-length character string
+CHAR,StringType,all,false,,Fixed-length character string
+TEXT,StringType,all,false,,Variable unlimited length text
+BPCHAR,StringType,all,false,,Blank-padded character
+NAME,StringType,all,false,,Internal type for identifiers
+INTEGER,IntegerType,all,false,,32-bit signed integer
+INT,IntegerType,all,false,,Alias for INTEGER
+INT4,IntegerType,all,false,,Alias for INTEGER
+SMALLINT,ShortType,all,false,,16-bit signed integer
+INT2,ShortType,all,false,,Alias for SMALLINT
+BIGINT,LongType,all,false,,64-bit signed integer
+INT8,LongType,all,false,,Alias for BIGINT
+SERIAL,IntegerType,all,false,,Auto-incrementing integer
+BIGSERIAL,LongType,all,false,,Auto-incrementing bigint
+SMALLSERIAL,ShortType,all,false,,Auto-incrementing smallint
+REAL,FloatType,all,false,,32-bit floating point
+FLOAT4,FloatType,all,false,,Alias for REAL
+DOUBLE PRECISION,DoubleType,all,false,,64-bit floating point
+FLOAT8,DoubleType,all,false,,Alias for DOUBLE PRECISION
+FLOAT,DoubleType,all,false,,Floating point (defaults to double)
+NUMERIC,DecimalType,all,false,,Exact numeric with precision
+DECIMAL,DecimalType,all,false,,Alias for NUMERIC
+MONEY,DecimalType,all,false,,Currency amount
+BOOLEAN,BooleanType,all,false,,True/false value
+BOOL,BooleanType,all,false,,Alias for BOOLEAN
+DATE,DateType,all,false,,Calendar date
+TIME,StringType,all,false,,Time of day (no timezone)
+TIMETZ,StringType,all,false,,Time with timezone
+TIMESTAMP,TimestampType,all,false,,Date and time (no timezone)
+TIMESTAMPTZ,TimestampType,all,false,,Date and time with timezone
+INTERVAL,StringType,all,false,,Time interval
+BYTEA,BinaryType,all,false,,Binary data
+UUID,StringType,all,false,,Universally unique identifier
+JSON,StringType,all,true,CAST({} AS STRING),JSON data
+JSONB,StringType,all,true,CAST({} AS STRING),Binary JSON data
+XML,StringType,all,true,CAST({} AS STRING),XML data
+ARRAY,ArrayType,all,true,,PostgreSQL array type
+INET,StringType,all,false,,IPv4 or IPv6 host address
+CIDR,StringType,all,false,,IPv4 or IPv6 network address
+MACADDR,StringType,all,false,,MAC address
+MACADDR8,StringType,all,false,,MAC address (EUI-64 format)
+BIT,StringType,all,false,,Fixed-length bit string
+VARBIT,StringType,all,false,,Variable-length bit string
+POINT,StringType,all,false,,Geometric point
+LINE,StringType,all,false,,Infinite line
+LSEG,StringType,all,false,,Line segment
+BOX,StringType,all,false,,Rectangular box
+PATH,StringType,all,false,,Geometric path
+POLYGON,StringType,all,false,,Closed geometric path
+CIRCLE,StringType,all,false,,Circle
+TSVECTOR,StringType,all,false,,Text search document
+TSQUERY,StringType,all,false,,Text search query
+OID,LongType,all,false,,Object identifier
+REGCLASS,StringType,all,false,,Relation name
+REGTYPE,StringType,all,false,,Data type name

dbt/include/data/csv/type_mappings_redshift.csv

@@ -0,0 +1,33 @@
+adapter_type,spark_type,spark_version,is_complex,cast_expression,notes
+VARCHAR,StringType,all,false,,Variable-length string (max 65535)
+CHAR,StringType,all,false,,Fixed-length string
+BPCHAR,StringType,all,false,,Blank-padded character
+TEXT,StringType,all,false,,Variable unlimited length (alias)
+NVARCHAR,StringType,all,false,,Unicode VARCHAR
+NCHAR,StringType,all,false,,Unicode CHAR
+SMALLINT,ShortType,all,false,,16-bit signed integer
+INT2,ShortType,all,false,,Alias for SMALLINT
+INTEGER,IntegerType,all,false,,32-bit signed integer
+INT,IntegerType,all,false,,Alias for INTEGER
+INT4,IntegerType,all,false,,Alias for INTEGER
+BIGINT,LongType,all,false,,64-bit signed integer
+INT8,LongType,all,false,,Alias for BIGINT
+REAL,FloatType,all,false,,32-bit floating point
+FLOAT4,FloatType,all,false,,Alias for REAL
+DOUBLE PRECISION,DoubleType,all,false,,64-bit floating point
+FLOAT8,DoubleType,all,false,,Alias for DOUBLE PRECISION
+FLOAT,DoubleType,all,false,,Alias for DOUBLE PRECISION
+DECIMAL,DecimalType,all,false,,Exact numeric with precision
+NUMERIC,DecimalType,all,false,,Alias for DECIMAL
+BOOLEAN,BooleanType,all,false,,True/false value
+BOOL,BooleanType,all,false,,Alias for BOOLEAN
+DATE,DateType,all,false,,Calendar date
+TIME,StringType,all,false,,Time of day
+TIMETZ,StringType,all,false,,Time with timezone
+TIMESTAMP,TimestampType,all,false,,Timestamp without timezone
+TIMESTAMPTZ,TimestampType,all,false,,Timestamp with timezone
+GEOMETRY,StringType,all,false,,Spatial geometry (EWKB)
+GEOGRAPHY,StringType,all,false,,Geographic data
+HLLSKETCH,StringType,all,false,,HyperLogLog sketch
+SUPER,StringType,all,true,JSON_SERIALIZE({}),Semi-structured data
+VARBYTE,BinaryType,all,false,,Variable-length binary

dbt/include/data/csv/type_mappings_snowflake.csv

@@ -0,0 +1,38 @@
+adapter_type,spark_type,spark_version,is_complex,cast_expression,notes
+VARCHAR,StringType,all,false,,Variable-length string
+STRING,StringType,all,false,,Alias for VARCHAR
+TEXT,StringType,all,false,,Alias for VARCHAR
+CHAR,StringType,all,false,,Fixed-length string
+CHARACTER,StringType,all,false,,Alias for CHAR
+NVARCHAR,StringType,all,false,,Unicode VARCHAR
+NCHAR,StringType,all,false,,Unicode CHAR
+NUMBER,DecimalType,all,false,,Numeric with precision and scale
+DECIMAL,DecimalType,all,false,,Alias for NUMBER
+NUMERIC,DecimalType,all,false,,Alias for NUMBER
+INT,LongType,all,false,,Integer (38 digits precision)
+INTEGER,LongType,all,false,,Alias for INT
+BIGINT,LongType,all,false,,Alias for INT
+SMALLINT,LongType,all,false,,Alias for INT
+TINYINT,LongType,all,false,,Alias for INT
+BYTEINT,LongType,all,false,,Alias for INT
+FLOAT,DoubleType,all,false,,64-bit floating point
+FLOAT4,FloatType,all,false,,32-bit floating point
+FLOAT8,DoubleType,all,false,,64-bit floating point
+DOUBLE,DoubleType,all,false,,Alias for FLOAT
+DOUBLE PRECISION,DoubleType,all,false,,Alias for FLOAT
+REAL,FloatType,all,false,,32-bit floating point
+BOOLEAN,BooleanType,all,false,,True/false value
+DATE,DateType,all,false,,Calendar date
+TIME,StringType,all,false,,Time of day
+DATETIME,TimestampType,all,false,,Alias for TIMESTAMP
+TIMESTAMP,TimestampType,all,false,,Timestamp without timezone
+TIMESTAMP_LTZ,TimestampType,all,false,,Timestamp with local timezone
+TIMESTAMP_NTZ,TimestampType,all,false,,Timestamp no timezone
+TIMESTAMP_TZ,TimestampType,all,false,,Timestamp with timezone
+BINARY,BinaryType,all,false,,Binary data
+VARBINARY,BinaryType,all,false,,Variable binary
+VARIANT,StringType,all,true,TO_JSON({}),Semi-structured data
+OBJECT,StringType,all,true,TO_JSON({}),Key-value pairs
+ARRAY,StringType,all,true,TO_JSON({}),Ordered list of values
+GEOGRAPHY,StringType,all,false,,Geographic data (WKT)
+GEOMETRY,StringType,all,false,,Geometric data (WKT)

dbt/include/data/csv/type_mappings_sqlserver.csv

@@ -0,0 +1,35 @@
+adapter_type,spark_type,spark_version,is_complex,cast_expression,notes
+VARCHAR,StringType,all,false,,Variable-length string
+NVARCHAR,StringType,all,false,,Unicode variable-length
+CHAR,StringType,all,false,,Fixed-length string
+NCHAR,StringType,all,false,,Unicode fixed-length
+TEXT,StringType,all,false,,Variable text (deprecated)
+NTEXT,StringType,all,false,,Unicode text (deprecated)
+TINYINT,ShortType,all,false,,8-bit unsigned (0-255)
+SMALLINT,ShortType,all,false,,16-bit signed integer
+INT,IntegerType,all,false,,32-bit signed integer
+BIGINT,LongType,all,false,,64-bit signed integer
+REAL,FloatType,all,false,,32-bit floating point
+FLOAT,DoubleType,all,false,,64-bit floating point
+DECIMAL,DecimalType,all,false,,Exact numeric with precision
+NUMERIC,DecimalType,all,false,,Alias for DECIMAL
+MONEY,DecimalType,all,false,,Currency (8 bytes)
+SMALLMONEY,DecimalType,all,false,,Currency (4 bytes)
+BIT,BooleanType,all,false,,Boolean (0 or 1)
+DATE,DateType,all,false,,Calendar date
+TIME,StringType,all,false,,Time of day
+DATETIME,TimestampType,all,false,,Date and time (3.33ms precision)
+DATETIME2,TimestampType,all,false,,Date and time (100ns precision)
+SMALLDATETIME,TimestampType,all,false,,Date and time (minute precision)
+DATETIMEOFFSET,TimestampType,all,false,,Date time with timezone
+BINARY,BinaryType,all,false,,Fixed-length binary
+VARBINARY,BinaryType,all,false,,Variable-length binary
+IMAGE,BinaryType,all,false,,Binary data (deprecated)
+UNIQUEIDENTIFIER,StringType,all,false,,GUID/UUID
+XML,StringType,all,true,CAST({} AS NVARCHAR(MAX)),XML document
+SQL_VARIANT,StringType,all,false,,Mixed data types
+GEOGRAPHY,StringType,all,false,,Geographic data (CLR)
+GEOMETRY,StringType,all,false,,Geometric data (CLR)
+HIERARCHYID,StringType,all,false,,Hierarchy position
+TIMESTAMP,BinaryType,all,false,,Row version (not datetime!)
+ROWVERSION,BinaryType,all,false,,Alias for TIMESTAMP

dbt/include/dvt_starter_project/README.md

@@ -0,0 +1,15 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

dbt/include/dvt_starter_project/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

dbt/include/dvt_starter_project/dvt_project.yml

@@ -0,0 +1,39 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: '{project_name}'
+version: '1.0.0'
+
+# This setting configures which "profile" DVT uses for this project.
+profile: '{profile_name}'
+
+# These configurations specify where DVT should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+# DVT-specific: Path for flat files (CSV, Parquet, etc.) for local data ingestion
+flatfile-paths: ["flatfiles"]
+
+clean-targets:         # directories to be removed by `dvt clean`
+  - "target"
+  - "dbt_packages"
+
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell DVT to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{{{ config(...) }}}}` macro.
+models:
+  '{project_name}':
+    # Config indicated by + and applies to all files under models/example/
+    example:
+      +materialized: view