dvt-core 1.11.0b4__py3-none-any.whl

dvt/config/utils.py

@@ -0,0 +1,78 @@
+from typing import Any, Dict, Optional
+
+from dvt import deprecations
+from dvt.clients import yaml_helper
+from dvt.events.types import InvalidOptionYAML
+from dvt.exceptions import DbtExclusivePropertyUseError, OptionNotYamlDictError
+
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtValidationError
+
+
+def parse_cli_vars(var_string: str) -> Dict[str, Any]:
+    return parse_cli_yaml_string(var_string, "vars")
+
+
+def parse_cli_yaml_string(var_string: str, cli_option_name: str) -> Dict[str, Any]:
+    try:
+        cli_vars = yaml_helper.load_yaml_text(var_string)
+        var_type = type(cli_vars)
+        if cli_vars is not None and var_type is dict:
+            return cli_vars
+        else:
+            raise OptionNotYamlDictError(var_type, cli_option_name)
+    except (DbtValidationError, OptionNotYamlDictError):
+        fire_event(InvalidOptionYAML(option_name=cli_option_name))
+        raise
+
+
+def exclusive_primary_alt_value_setting(
+    dictionary: Optional[Dict[str, Any]],
+    primary: str,
+    alt: str,
+    parent_config: Optional[str] = None,
+) -> None:
+    """Munges in place under the primary the options for the primary and alt values
+
+    Sometimes we allow setting something via TWO keys, but not at the same time. If both the primary
+    key and alt key have values, an error gets raised. If the alt key has values, then we update
+    the dictionary to ensure the primary key contains the values. If neither are set, nothing happens.
+    """
+
+    if dictionary is None:
+        return
+
+    primary_options = dictionary.get(primary)
+    alt_options = dictionary.get(alt)
+
+    if primary_options and alt_options:
+        where = f" in `{parent_config}`" if parent_config is not None else ""
+        raise DbtExclusivePropertyUseError(
+            f"Only `{alt}` or `{primary}` can be specified{where}, not both"
+        )
+
+    if alt in dictionary:
+        alt_value = dictionary.pop(alt)
+        dictionary[primary] = alt_value
+
+
+def normalize_warn_error_options(warn_error_options: Dict[str, Any]) -> None:
+    has_include = "include" in warn_error_options
+    has_exclude = "exclude" in warn_error_options
+
+    if has_include or has_exclude:
+        deprecations.buffer(
+            "weo-include-exclude-deprecation",
+            found_include=has_include,
+            found_exclude=has_exclude,
+        )
+
+    exclusive_primary_alt_value_setting(
+        warn_error_options, "error", "include", "warn_error_options"
+    )
+    exclusive_primary_alt_value_setting(
+        warn_error_options, "warn", "exclude", "warn_error_options"
+    )
+    for key in ("error", "warn", "silence"):
+        if key in warn_error_options and warn_error_options[key] is None:
+            warn_error_options[key] = []

dvt/connectors/.gitignore

@@ -0,0 +1,6 @@
+# Ignore downloaded JAR files
+jars/
+*.jar
+
+# But keep the directory structure
+!.gitkeep

dvt/connectors/README.md

@@ -0,0 +1,306 @@
+# DVT Database Connectors
+
+This directory contains the connector catalog for DVT's compute engines (DuckDB and Spark).
+
+## Overview
+
+DVT uses **dbt adapters** for all database connections, both for reading from source databases and writing to target databases. This provides a unified, Python-based connection mechanism that works with any database supported by the dbt ecosystem.
+
+## No JARs Required
+
+Unlike traditional Spark solutions that require JDBC JAR files, DVT extracts data from databases using dbt adapters and transfers it to the compute layer via Apache Arrow format. This approach:
+
+- **Eliminates JAR dependencies** - Pure Python solution
+- **Works with any dbt adapter** - 30+ databases supported out of the box
+- **Provides consistent interface** - Same connection mechanism for all databases
+- **Reduces package size** - No 200+ MB of JARs to download
+- **Simplifies configuration** - Single `profiles.yml` for all connections
+
+## Architecture
+
+### Data Flow
+
+```
+Source DB → dbt adapter → Agate Table → Arrow Table →
+Compute Engine (DuckDB/Spark) → Arrow Table → Target dbt adapter → Target DB
+```
+
+### Example: Cross-Database Query
+
+```yaml
+# profiles.yml
+postgres_prod:
+  adapter: postgres
+  host: db.example.com
+  port: 5432
+  user: analytics
+  password: "{{ env_var('POSTGRES_PASSWORD') }}"
+  database: production
+  schema: public
+
+mysql_legacy:
+  adapter: mysql
+  host: legacy-db.example.com
+  port: 3306
+  user: readonly
+  password: "{{ env_var('MYSQL_PASSWORD') }}"
+  database: orders_db
+```
+
+```yaml
+# sources.yml
+sources:
+  - name: postgres_source
+    profile: postgres_prod
+    tables:
+      - name: customers
+
+  - name: mysql_source
+    profile: mysql_legacy
+    tables:
+      - name: orders
+```
+
+```sql
+-- models/cross_db_analysis.sql
+select
+    c.customer_id,
+    c.name,
+    count(o.order_id) as order_count
+from {{ source('postgres_source', 'customers') }} c
+left join {{ source('mysql_source', 'orders') }} o
+    on c.customer_id = o.customer_id
+group by c.customer_id, c.name
+```
+
+DVT will automatically:
+1. Detect heterogeneous sources (PostgreSQL + MySQL)
+2. Extract data via dbt adapters (`dbt-postgres`, `dbt-mysql`)
+3. Convert to Arrow format for efficient transfer
+4. Load into compute engine (DuckDB or Spark)
+5. Execute the query in the compute layer
+6. Return unified result set
+
+## Supported Databases
+
+DVT works with **any database that has a dbt adapter**. This includes:
+
+**Relational Databases:**
+- PostgreSQL (`dbt-postgres`)
+- MySQL (`dbt-mysql`)
+- SQL Server (`dbt-sqlserver`)
+- Oracle (`dbt-oracle`)
+
+**Cloud Data Warehouses:**
+- Snowflake (`dbt-snowflake`)
+- BigQuery (`dbt-bigquery`)
+- Redshift (`dbt-redshift`)
+- Databricks (`dbt-databricks`)
+
+**Analytics Databases:**
+- DuckDB (`dbt-duckdb`)
+- ClickHouse (`dbt-clickhouse`)
+- Trino (`dbt-trino`)
+
+**And many more...**
+
+See the [dbt adapter registry](https://docs.getdbt.com/docs/supported-data-platforms) for the complete list.
+
+## Installation
+
+Install DVT with the dbt adapters you need:
+
+```bash
+# Install DVT core
+pip install dvt-core
+
+# Install adapters for your databases
+pip install dbt-postgres dbt-mysql dbt-snowflake
+```
+
+## Configuration
+
+Configure profiles in `~/.dvt/profiles.yml` using the same format as dbt:
+
+```yaml
+# PostgreSQL
+postgres_prod:
+  adapter: postgres
+  host: db.example.com
+  port: 5432
+  user: analytics
+  password: "{{ env_var('POSTGRES_PASSWORD') }}"
+  database: production
+  schema: public
+
+# MySQL
+mysql_legacy:
+  adapter: mysql
+  host: legacy-db.example.com
+  port: 3306
+  user: readonly
+  password: "{{ env_var('MYSQL_PASSWORD') }}"
+  database: orders_db
+
+# Snowflake
+snowflake_analytics:
+  adapter: snowflake
+  account: mycompany
+  user: analytics
+  password: "{{ env_var('SNOWFLAKE_PASSWORD') }}"
+  database: analytics
+  warehouse: compute_wh
+  schema: public
+```
+
+Test your connections:
+
+```bash
+dvt profiles test --all
+```
+
+## Compute Engines
+
+DVT supports two compute engines for cross-database queries:
+
+### DuckDB (Default)
+- In-process analytical database
+- Fast for datasets < 10GB
+- Zero configuration required
+- Perfect for development and small-to-medium workloads
+
+### PySpark
+- Distributed compute engine
+- Scales to 100GB+ datasets
+- Local or cluster mode
+- No JDBC JARs required - uses dbt adapters
+
+Configure in `dvt_project.yml`:
+
+```yaml
+compute:
+  default_engine: duckdb  # or 'spark'
+
+  duckdb:
+    memory_limit: '4GB'
+    threads: 4
+
+  spark:
+    type: local
+    master: 'local[*]'
+    config:
+      spark.executor.memory: '4g'
+```
+
+## How It Works
+
+### Traditional Spark Approach (Old)
+```
+Spark → JDBC JARs (200+ MB) → Database
+```
+- Requires downloading JARs
+- Different JAR for each database
+- Version conflicts
+- Large package size
+
+### DVT Approach (New)
+```
+Database → dbt adapter → Arrow → Compute Engine
+```
+- Pure Python solution
+- Uses existing dbt adapters
+- No JARs needed
+- Small package size (~10 MB core)
+
+### Implementation Details
+
+When DVT executes a cross-database query:
+
+1. **Query Analysis**: DVT analyzes the SQL to identify all source databases
+2. **Data Extraction**: For each source:
+   - Get dbt adapter for the profile
+   - Execute `SELECT * FROM table` via adapter
+   - Receive results as Agate table
+3. **Arrow Conversion**: Convert Agate tables to Arrow format (zero-copy)
+4. **Compute Layer**:
+   - **DuckDB**: Register Arrow tables directly
+   - **Spark**: Convert Arrow → Pandas → Spark DataFrame
+5. **Query Execution**: Execute the original query in the compute engine
+6. **Results**: Return results as Arrow table
+
+## Catalog File
+
+The `catalog.yml` file is retained for documentation purposes and to map database types to dbt adapter names:
+
+```yaml
+postgres:
+  adapter_name: postgres
+  dbt_package: dbt-postgres
+  description: PostgreSQL database
+  connection_docs: https://docs.getdbt.com/reference/warehouse-setups/postgres-setup
+
+mysql:
+  adapter_name: mysql
+  dbt_package: dbt-mysql
+  description: MySQL database
+  connection_docs: https://docs.getdbt.com/reference/warehouse-setups/mysql-setup
+```
+
+## Troubleshooting
+
+### Missing Adapter Error
+
+If you see `adapter not found` errors:
+
+```bash
+# Check installed adapters
+pip list | grep dbt-
+
+# Install missing adapter
+pip install dbt-postgres
+```
+
+### Connection Failures
+
+If connections fail:
+
+```bash
+# Test individual profile
+dvt profiles test postgres_prod
+
+# Test all profiles
+dvt profiles test --all
+
+# Check profile configuration
+dvt profiles show postgres_prod
+```
+
+### Performance Issues
+
+For large datasets:
+
+1. Switch to Spark engine:
+   ```yaml
+   compute:
+     default_engine: spark
+   ```
+
+2. Increase memory limits:
+   ```yaml
+   spark:
+     config:
+       spark.executor.memory: '8g'
+       spark.driver.memory: '4g'
+   ```
+
+3. Use per-model configuration:
+   ```sql
+   {{ config(compute='spark') }}
+   ```
+
+## See Also
+
+- [Multi-Profile Guide](../../docs/multi-profile-guide.md)
+- [Compute Configuration](../../docs/compute-configuration.md)
+- [DVT Architecture](../../docs/DVT_ARCHITECTURE.md)
+- [dbt Adapter Documentation](https://docs.getdbt.com/docs/supported-data-platforms)

dvt/connectors/catalog.yml

@@ -0,0 +1,217 @@
+# DVT Spark JDBC Connector Catalog
+#
+# This file catalogs all JDBC connectors bundled with DVT for cross-database queries.
+# Each connector enables Spark to read from a specific database type.
+#
+# Format:
+#   connector_name:
+#     description: Human-readable description
+#     driver_class: JDBC driver class name
+#     maven_coordinates: Maven group:artifact:version
+#     url_pattern: JDBC URL pattern
+#     homepage: Official project homepage
+#     license: Software license
+
+version: "1.0.0"
+
+connectors:
+  # Relational Databases
+  postgresql:
+    description: "PostgreSQL - Open source relational database"
+    driver_class: "org.postgresql.Driver"
+    maven_coordinates: "org.postgresql:postgresql:42.7.1"
+    url_pattern: "jdbc:postgresql://<host>:<port>/<database>"
+    homepage: "https://www.postgresql.org/"
+    license: "PostgreSQL License"
+
+  mysql:
+    description: "MySQL - Popular open source relational database"
+    driver_class: "com.mysql.cj.jdbc.Driver"
+    maven_coordinates: "com.mysql:mysql-connector-j:8.2.0"
+    url_pattern: "jdbc:mysql://<host>:<port>/<database>"
+    homepage: "https://www.mysql.com/"
+    license: "GPL v2"
+
+  mariadb:
+    description: "MariaDB - MySQL-compatible open source database"
+    driver_class: "org.mariadb.jdbc.Driver"
+    maven_coordinates: "org.mariadb.jdbc:mariadb-java-client:3.3.1"
+    url_pattern: "jdbc:mariadb://<host>:<port>/<database>"
+    homepage: "https://mariadb.org/"
+    license: "LGPL v2.1"
+
+  oracle:
+    description: "Oracle Database - Enterprise relational database"
+    driver_class: "oracle.jdbc.OracleDriver"
+    maven_coordinates: "com.oracle.database.jdbc:ojdbc11:23.3.0.23.09"
+    url_pattern: "jdbc:oracle:thin:@<host>:<port>:<sid>"
+    homepage: "https://www.oracle.com/database/"
+    license: "Oracle Free Use Terms and Conditions"
+
+  sqlserver:
+    description: "Microsoft SQL Server - Enterprise database"
+    driver_class: "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+    maven_coordinates: "com.microsoft.sqlserver:mssql-jdbc:12.4.2.jre11"
+    url_pattern: "jdbc:sqlserver://<host>:<port>;databaseName=<database>"
+    homepage: "https://www.microsoft.com/sql-server/"
+    license: "MIT"
+
+  db2:
+    description: "IBM Db2 - Enterprise database"
+    driver_class: "com.ibm.db2.jcc.DB2Driver"
+    maven_coordinates: "com.ibm.db2:jcc:11.5.9.0"
+    url_pattern: "jdbc:db2://<host>:<port>/<database>"
+    homepage: "https://www.ibm.com/products/db2"
+    license: "IBM"
+
+  # Cloud Databases
+  snowflake:
+    description: "Snowflake - Cloud data warehouse"
+    driver_class: "net.snowflake.client.jdbc.SnowflakeDriver"
+    maven_coordinates: "net.snowflake:snowflake-jdbc:3.14.4"
+    url_pattern: "jdbc:snowflake://<account>.snowflakecomputing.com"
+    homepage: "https://www.snowflake.com/"
+    license: "Apache 2.0"
+
+  bigquery:
+    description: "Google BigQuery - Serverless data warehouse"
+    driver_class: "com.simba.googlebigquery.jdbc.Driver"
+    maven_coordinates: "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.35.1"
+    url_pattern: "jdbc:bigquery://<project>:<dataset>"
+    homepage: "https://cloud.google.com/bigquery"
+    license: "Apache 2.0"
+
+  redshift:
+    description: "Amazon Redshift - Cloud data warehouse"
+    driver_class: "com.amazon.redshift.jdbc.Driver"
+    maven_coordinates: "com.amazon.redshift:redshift-jdbc42:2.1.0.25"
+    url_pattern: "jdbc:redshift://<endpoint>:<port>/<database>"
+    homepage: "https://aws.amazon.com/redshift/"
+    license: "Apache 2.0"
+
+  athena:
+    description: "Amazon Athena - Serverless query service"
+    driver_class: "com.simba.athena.jdbc.Driver"
+    maven_coordinates: "com.amazonaws:aws-java-sdk-athena:1.12.565"
+    url_pattern: "jdbc:awsathena://<region>.amazonaws.com:443"
+    homepage: "https://aws.amazon.com/athena/"
+    license: "Apache 2.0"
+
+  # Analytics & MPP Databases
+  clickhouse:
+    description: "ClickHouse - Fast open-source column-oriented database"
+    driver_class: "com.clickhouse.jdbc.ClickHouseDriver"
+    maven_coordinates: "com.clickhouse:clickhouse-jdbc:0.6.0-patch1"
+    url_pattern: "jdbc:clickhouse://<host>:<port>/<database>"
+    homepage: "https://clickhouse.com/"
+    license: "Apache 2.0"
+
+  vertica:
+    description: "Vertica - Unified analytics platform"
+    driver_class: "com.vertica.jdbc.Driver"
+    maven_coordinates: "com.vertica.jdbc:vertica-jdbc:23.4.0-0"
+    url_pattern: "jdbc:vertica://<host>:<port>/<database>"
+    homepage: "https://www.vertica.com/"
+    license: "Apache 2.0"
+
+  presto:
+    description: "Presto - Distributed SQL query engine"
+    driver_class: "com.facebook.presto.jdbc.PrestoDriver"
+    maven_coordinates: "com.facebook.presto:presto-jdbc:0.285"
+    url_pattern: "jdbc:presto://<host>:<port>/<catalog>"
+    homepage: "https://prestodb.io/"
+    license: "Apache 2.0"
+
+  trino:
+    description: "Trino - Fast distributed SQL query engine"
+    driver_class: "io.trino.jdbc.TrinoDriver"
+    maven_coordinates: "io.trino:trino-jdbc:433"
+    url_pattern: "jdbc:trino://<host>:<port>/<catalog>"
+    homepage: "https://trino.io/"
+    license: "Apache 2.0"
+
+  # NoSQL & NewSQL
+  cassandra:
+    description: "Apache Cassandra - Distributed NoSQL database"
+    driver_class: "com.github.adejanovski.cassandra.jdbc.CassandraDriver"
+    maven_coordinates: "com.github.adejanovski:cassandra-jdbc-wrapper:4.10.2"
+    url_pattern: "jdbc:cassandra://<host>:<port>/<keyspace>"
+    homepage: "https://cassandra.apache.org/"
+    license: "Apache 2.0"
+
+  mongodb:
+    description: "MongoDB - Document database"
+    driver_class: "mongodb.jdbc.MongoDriver"
+    maven_coordinates: "org.mongodb.spark:mongo-spark-connector_2.12:10.2.1"
+    url_pattern: "mongodb://<host>:<port>/<database>"
+    homepage: "https://www.mongodb.com/"
+    license: "Server Side Public License"
+
+  # Other Databases
+  h2:
+    description: "H2 - Lightweight Java SQL database"
+    driver_class: "org.h2.Driver"
+    maven_coordinates: "com.h2database:h2:2.2.224"
+    url_pattern: "jdbc:h2:<file_path>"
+    homepage: "https://h2database.com/"
+    license: "MPL 2.0 or EPL 1.0"
+
+  sqlite:
+    description: "SQLite - Embedded relational database"
+    driver_class: "org.sqlite.JDBC"
+    maven_coordinates: "org.xerial:sqlite-jdbc:3.44.1.0"
+    url_pattern: "jdbc:sqlite:<file_path>"
+    homepage: "https://www.sqlite.org/"
+    license: "Public Domain"
+
+  derby:
+    description: "Apache Derby - Java-based relational database"
+    driver_class: "org.apache.derby.jdbc.EmbeddedDriver"
+    maven_coordinates: "org.apache.derby:derby:10.16.1.1"
+    url_pattern: "jdbc:derby:<database>"
+    homepage: "https://db.apache.org/derby/"
+    license: "Apache 2.0"
+
+  hive:
+    description: "Apache Hive - Data warehouse software"
+    driver_class: "org.apache.hive.jdbc.HiveDriver"
+    maven_coordinates: "org.apache.hive:hive-jdbc:3.1.3"
+    url_pattern: "jdbc:hive2://<host>:<port>/<database>"
+    homepage: "https://hive.apache.org/"
+    license: "Apache 2.0"
+
+  impala:
+    description: "Apache Impala - MPP SQL query engine"
+    driver_class: "com.cloudera.impala.jdbc.Driver"
+    maven_coordinates: "org.apache.impala:impala-jdbc:2.6.30"
+    url_pattern: "jdbc:impala://<host>:<port>/<database>"
+    homepage: "https://impala.apache.org/"
+    license: "Apache 2.0"
+
+  phoenix:
+    description: "Apache Phoenix - SQL skin over HBase"
+    driver_class: "org.apache.phoenix.jdbc.PhoenixDriver"
+    maven_coordinates: "org.apache.phoenix:phoenix-client-hbase-2.5:5.1.3"
+    url_pattern: "jdbc:phoenix:<zookeeper_quorum>"
+    homepage: "https://phoenix.apache.org/"
+    license: "Apache 2.0"
+
+  # Time Series & Specialized
+  timescaledb:
+    description: "TimescaleDB - Time-series extension for PostgreSQL"
+    driver_class: "org.postgresql.Driver"
+    maven_coordinates: "org.postgresql:postgresql:42.7.1"
+    url_pattern: "jdbc:postgresql://<host>:<port>/<database>"
+    homepage: "https://www.timescale.com/"
+    license: "Timescale License"
+
+  influxdb:
+    description: "InfluxDB - Time series database"
+    driver_class: "org.influxdb.jdbc.InfluxDBDriver"
+    maven_coordinates: "org.influxdb:influxdb-java:2.24"
+    url_pattern: "jdbc:influxdb://<host>:<port>/<database>"
+    homepage: "https://www.influxdata.com/"
+    license: "MIT"
+
+# Estimated total size: ~500MB - 1GB for all connectors
+# Individual connector sizes range from 1MB to 100MB