dvt-core 1.11.0b5__tar.gz

dvt_core-1.11.0b5/MANIFEST.in

@@ -0,0 +1,8 @@
+# Include DVT package data
+recursive-include dvt/include *.py *.sql *.yml *.html *.md .gitkeep .gitignore
+include dvt/py.typed
+recursive-include dvt/task/docs *.html
+recursive-include dvt/jsonschemas *.json
+
+# Exclude all JARs (no longer needed - using dbt adapters)
+global-exclude *.jar

dvt_core-1.11.0b5/PKG-INFO

@@ -0,0 +1,252 @@
+Metadata-Version: 2.4
+Name: dvt-core
+Version: 1.11.0b5
+Summary: DVT (Data Virtualization Tool) - An all-in-one ELT solution for data transformation and virtualization, built on dbt-core foundations.
+Author: DVT Core Contributors
+Maintainer: DVT Core Contributors
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/hex/dvt-core
+Project-URL: Repository, https://github.com/hex/dvt-core.git
+Project-URL: Issues, https://github.com/hex/dvt-core/issues
+Project-URL: Changelog, https://github.com/hex/dvt-core/blob/main/CHANGELOG.md
+Keywords: data,virtualization,ELT,transformation,analytics
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: agate<1.10,>=1.7.0
+Requires-Dist: Jinja2<4,>=3.1.3
+Requires-Dist: mashumaro[msgpack]<3.15,>=3.9
+Requires-Dist: click<9.0,>=8.0.2
+Requires-Dist: jsonschema<5.0,>=4.19.1
+Requires-Dist: networkx<4.0,>=2.3
+Requires-Dist: protobuf<7.0,>=6.0
+Requires-Dist: requests<3.0.0
+Requires-Dist: snowplow-tracker<2.0,>=1.0.2
+Requires-Dist: pathspec<0.13,>=0.9
+Requires-Dist: sqlparse<0.6.0,>=0.5.0
+Requires-Dist: dbt-extractor<=0.6,>=0.5.0
+Requires-Dist: dbt-semantic-interfaces<0.10,>=0.9.0
+Requires-Dist: dbt-common<2.0,>=1.27.0
+Requires-Dist: dbt-adapters<2.0,>=1.15.5
+Requires-Dist: dbt-protos<2.0,>=1.0.375
+Requires-Dist: pydantic<3
+Requires-Dist: duckdb<2.0,>=0.9.0
+Requires-Dist: pyspark<4.0,>=3.4.0
+Requires-Dist: pyarrow>=14.0.0
+Requires-Dist: packaging>20.9
+Requires-Dist: pytz>=2015.7
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: daff>=1.3.46
+Requires-Dist: typing-extensions>=4.4
+
+# DVT (Data Virtualization Tool)
+
+**DVT** is an all-in-one ELT solution that adds comprehensive data virtualization capabilities to dbt-core foundations.
+
+## Quick Start
+
+### Installation
+
+```bash
+# Install DVT with all features (includes PySpark, DuckDB, and 20 JDBC connectors)
+pip install dvt-core
+
+# Verify installation
+dvt --version
+```
+
+**What's included:**
+- ✅ Core DVT + all dbt functionality
+- ✅ DuckDB compute engine (~10MB)
+- ✅ PySpark compute engine (~250MB)
+- ✅ 20 JDBC connectors (~200MB) for Spark cross-database queries
+- ✅ Complete documentation and examples
+
+**Total install size:** ~450-500MB
+
+### Create Your First Project
+
+```bash
+# Initialize new project
+dvt init my_analytics_project
+cd my_analytics_project
+
+# Configure profiles in ~/.dvt/profiles.yml
+# Add multiple database connections
+
+# Test connections
+dvt profiles test --all
+
+# Run your models
+dvt run
+```
+
+## What is DVT?
+
+DVT enables you to:
+
+- **Query across databases**: Join PostgreSQL, MySQL, Snowflake, and 20+ other databases in a single SQL query
+- **Maintain dbt compatibility**: 100% backwards compatible with existing dbt projects
+- **Use intelligent query routing**: Automatic optimization between pushdown and compute layer execution
+- **Manage multiple connections**: Support multiple database profiles in a single project
+- **Choose your compute engine**: Use DuckDB for fast analytics or Spark for distributed processing
+
+## Key Features
+
+### Cross-Database Queries
+
+```sql
+-- Join PostgreSQL customers with MySQL orders
+select
+    c.customer_id,
+    c.name,
+    count(o.order_id) as order_count
+from {{ source('postgres_prod', 'customers') }} c
+left join {{ source('mysql_legacy', 'orders') }} o
+    on c.customer_id = o.customer_id
+group by c.customer_id, c.name
+```
+
+DVT automatically detects heterogeneous sources and routes execution to the appropriate compute engine.
+
+### Multi-Profile Support
+
+Configure multiple database connections:
+
+```yaml
+# profiles.yml
+postgres_prod:
+  adapter: postgres
+  host: prod-db.example.com
+  # ...
+
+mysql_legacy:
+  adapter: mysql
+  host: legacy-db.example.com
+  # ...
+```
+
+Reference profiles in source definitions:
+
+```yaml
+# sources.yml
+sources:
+  - name: postgres_prod
+    profile: postgres_prod
+    tables:
+      - name: customers
+
+  - name: mysql_legacy
+    profile: mysql_legacy
+    tables:
+      - name: orders
+```
+
+## Included JDBC Connectors
+
+DVT bundles 20 JDBC connectors (~200MB) for Spark:
+
+**Relational Databases:**
+- PostgreSQL, MySQL, MariaDB, Oracle, SQL Server, DB2
+
+**Cloud Data Warehouses:**
+- Snowflake, BigQuery, Redshift, Athena
+
+**Analytics & MPP:**
+- ClickHouse, Vertica, Presto, Trino
+
+**Other:**
+- H2, SQLite, Derby, Hive, InfluxDB, TimescaleDB
+
+**Note:** Phoenix connector (149MB) exceeds PyPI limits and can be downloaded separately:
+```bash
+dvt connectors download --connector=phoenix
+```
+
+## DVT Commands
+
+### Standard dbt Commands (100% Compatible)
+
+All dbt commands work identically:
+
+```bash
+dvt run                    # Run models
+dvt test                   # Run tests
+dvt build                  # Run + test models
+dvt compile                # Compile SQL
+dvt docs generate          # Generate documentation
+```
+
+### DVT Extensions
+
+**Profile Management:**
+```bash
+dvt profiles list           # List all configured profiles
+dvt profiles show postgres_prod
+dvt profiles test --all     # Test all profile connections
+```
+
+**Compute Layer:**
+```bash
+dvt compute show            # Show compute configuration
+dvt compute engines         # List available engines
+dvt compute test --all      # Test compute engines
+```
+
+**Enhanced Debug:**
+```bash
+dvt debug --all-profiles    # Test all profiles (not just target)
+```
+
+## Documentation
+
+For complete documentation, see:
+
+- **[Sample Project Guide](../docs/sample-project-guide.md)** - Complete walkthrough building a real DVT project
+- **[DVT Architecture](../docs/DVT_ARCHITECTURE.md)** - Detailed architecture overview
+- **[Migration Guide](../docs/migration-guide.md)** - Migrate from dbt to DVT
+- **[CLI Reference](../docs/cli-reference.md)** - Complete command reference
+- **[Connector Catalog](connectors/README.md)** - JDBC connector documentation
+
+## Migration from dbt
+
+DVT is 100% backwards compatible with dbt:
+
+```bash
+# Install DVT
+pip install dvt-core
+
+# Replace dbt commands with dvt
+dvt run  # instead of: dbt run
+```
+
+That's it! Your existing dbt project works as-is.
+
+## Requirements
+
+- Python 3.10, 3.11, 3.12, or 3.13
+- Database adapters for your databases (e.g., `dbt-postgres`, `dbt-snowflake`)
+
+## License
+
+Apache License 2.0 - Same as dbt-core
+
+## Support
+
+- **Repository**: https://github.com/hex/dvt-core
+- **Issues**: https://github.com/hex/dvt-core/issues
+- **Documentation**: See `/docs` directory in repository
+
+## Acknowledgments
+
+DVT is built on top of [dbt-core](https://github.com/dbt-labs/dbt-core) and maintains deep compatibility with the dbt ecosystem.

dvt_core-1.11.0b5/README.md

@@ -0,0 +1,201 @@
+# DVT (Data Virtualization Tool)
+
+**DVT** is an all-in-one ELT solution that adds comprehensive data virtualization capabilities to dbt-core foundations.
+
+## Quick Start
+
+### Installation
+
+```bash
+# Install DVT with all features (includes PySpark, DuckDB, and 20 JDBC connectors)
+pip install dvt-core
+
+# Verify installation
+dvt --version
+```
+
+**What's included:**
+- ✅ Core DVT + all dbt functionality
+- ✅ DuckDB compute engine (~10MB)
+- ✅ PySpark compute engine (~250MB)
+- ✅ 20 JDBC connectors (~200MB) for Spark cross-database queries
+- ✅ Complete documentation and examples
+
+**Total install size:** ~450-500MB
+
+### Create Your First Project
+
+```bash
+# Initialize new project
+dvt init my_analytics_project
+cd my_analytics_project
+
+# Configure profiles in ~/.dvt/profiles.yml
+# Add multiple database connections
+
+# Test connections
+dvt profiles test --all
+
+# Run your models
+dvt run
+```
+
+## What is DVT?
+
+DVT enables you to:
+
+- **Query across databases**: Join PostgreSQL, MySQL, Snowflake, and 20+ other databases in a single SQL query
+- **Maintain dbt compatibility**: 100% backwards compatible with existing dbt projects
+- **Use intelligent query routing**: Automatic optimization between pushdown and compute layer execution
+- **Manage multiple connections**: Support multiple database profiles in a single project
+- **Choose your compute engine**: Use DuckDB for fast analytics or Spark for distributed processing
+
+## Key Features
+
+### Cross-Database Queries
+
+```sql
+-- Join PostgreSQL customers with MySQL orders
+select
+    c.customer_id,
+    c.name,
+    count(o.order_id) as order_count
+from {{ source('postgres_prod', 'customers') }} c
+left join {{ source('mysql_legacy', 'orders') }} o
+    on c.customer_id = o.customer_id
+group by c.customer_id, c.name
+```
+
+DVT automatically detects heterogeneous sources and routes execution to the appropriate compute engine.
+
+### Multi-Profile Support
+
+Configure multiple database connections:
+
+```yaml
+# profiles.yml
+postgres_prod:
+  adapter: postgres
+  host: prod-db.example.com
+  # ...
+
+mysql_legacy:
+  adapter: mysql
+  host: legacy-db.example.com
+  # ...
+```
+
+Reference profiles in source definitions:
+
+```yaml
+# sources.yml
+sources:
+  - name: postgres_prod
+    profile: postgres_prod
+    tables:
+      - name: customers
+
+  - name: mysql_legacy
+    profile: mysql_legacy
+    tables:
+      - name: orders
+```
+
+## Included JDBC Connectors
+
+DVT bundles 20 JDBC connectors (~200MB) for Spark:
+
+**Relational Databases:**
+- PostgreSQL, MySQL, MariaDB, Oracle, SQL Server, DB2
+
+**Cloud Data Warehouses:**
+- Snowflake, BigQuery, Redshift, Athena
+
+**Analytics & MPP:**
+- ClickHouse, Vertica, Presto, Trino
+
+**Other:**
+- H2, SQLite, Derby, Hive, InfluxDB, TimescaleDB
+
+**Note:** Phoenix connector (149MB) exceeds PyPI limits and can be downloaded separately:
+```bash
+dvt connectors download --connector=phoenix
+```
+
+## DVT Commands
+
+### Standard dbt Commands (100% Compatible)
+
+All dbt commands work identically:
+
+```bash
+dvt run                    # Run models
+dvt test                   # Run tests
+dvt build                  # Run + test models
+dvt compile                # Compile SQL
+dvt docs generate          # Generate documentation
+```
+
+### DVT Extensions
+
+**Profile Management:**
+```bash
+dvt profiles list           # List all configured profiles
+dvt profiles show postgres_prod
+dvt profiles test --all     # Test all profile connections
+```
+
+**Compute Layer:**
+```bash
+dvt compute show            # Show compute configuration
+dvt compute engines         # List available engines
+dvt compute test --all      # Test compute engines
+```
+
+**Enhanced Debug:**
+```bash
+dvt debug --all-profiles    # Test all profiles (not just target)
+```
+
+## Documentation
+
+For complete documentation, see:
+
+- **[Sample Project Guide](../docs/sample-project-guide.md)** - Complete walkthrough building a real DVT project
+- **[DVT Architecture](../docs/DVT_ARCHITECTURE.md)** - Detailed architecture overview
+- **[Migration Guide](../docs/migration-guide.md)** - Migrate from dbt to DVT
+- **[CLI Reference](../docs/cli-reference.md)** - Complete command reference
+- **[Connector Catalog](connectors/README.md)** - JDBC connector documentation
+
+## Migration from dbt
+
+DVT is 100% backwards compatible with dbt:
+
+```bash
+# Install DVT
+pip install dvt-core
+
+# Replace dbt commands with dvt
+dvt run  # instead of: dbt run
+```
+
+That's it! Your existing dbt project works as-is.
+
+## Requirements
+
+- Python 3.10, 3.11, 3.12, or 3.13
+- Database adapters for your databases (e.g., `dbt-postgres`, `dbt-snowflake`)
+
+## License
+
+Apache License 2.0 - Same as dbt-core
+
+## Support
+
+- **Repository**: https://github.com/hex/dvt-core
+- **Issues**: https://github.com/hex/dvt-core/issues
+- **Documentation**: See `/docs` directory in repository
+
+## Acknowledgments
+
+DVT is built on top of [dbt-core](https://github.com/dbt-labs/dbt-core) and maintains deep compatibility with the dbt ecosystem.

dvt_core-1.11.0b5/dvt/__init__.py

@@ -0,0 +1,7 @@
+# N.B.
+# This will add to the package’s __path__ all subdirectories of directories on sys.path named after the package which effectively combines both modules into a single namespace (dbt.adapters)
+# The matching statement is in plugins/postgres/dbt/__init__.py
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

dvt_core-1.11.0b5/dvt/_pydantic_shim.py

@@ -0,0 +1,26 @@
+# type: ignore
+
+"""Shim to allow support for both Pydantic 1 and Pydantic 2.
+
+dbt-core must support both major versions of Pydantic because dbt-core users might be using an environment with
+either version, and we can't restrict them to one or the other. Here, we essentially import all Pydantic objects
+from version 1 that we use. Throughout the repo, we import these objects from this file instead of from Pydantic
+directly, meaning that we essentially only use Pydantic 1 in dbt-core currently, but without forcing that restriction
+on dbt users. The development environment for this repo should be pinned to Pydantic 1 to ensure devs get appropriate
+type hints.
+"""
+
+from importlib.metadata import version
+
+pydantic_version = version("pydantic")
+# Pydantic uses semantic versioning, i.e. <major>.<minor>.<patch>, and we need to know the major
+pydantic_major = pydantic_version.split(".")[0]
+
+if pydantic_major == "1":
+    from pydantic import BaseSettings  # noqa: F401
+elif pydantic_major == "2":
+    from pydantic.v1 import BaseSettings  # noqa: F401
+else:
+    raise RuntimeError(
+        f"Currently only pydantic 1 and 2 are supported, found pydantic {pydantic_version}"
+    )

dvt_core-1.11.0b5/dvt/adapters/__init__.py

@@ -0,0 +1,16 @@
+"""
+DVT adapter management extensions.
+
+This module extends dbt's adapter system to support multi-profile
+source connections and compute layer integration.
+"""
+
+from dvt.adapters.multi_adapter_manager import (
+    MultiAdapterManager,
+    create_multi_adapter_manager,
+)
+
+__all__ = [
+    "MultiAdapterManager",
+    "create_multi_adapter_manager",
+]