dbt-data-engineering-toolkit-compiler 3.0.1__py3-none-any.whl

dbt_data_engineering_toolkit_compiler/__init__.py

@@ -0,0 +1,5 @@
+"""Controlled data-product compiler for dbt_data_engineering_toolkit."""
+
+from .version import COMPILER_VERSION
+
+__version__ = COMPILER_VERSION

dbt_data_engineering_toolkit_compiler/__main__.py

@@ -0,0 +1,3 @@
+from .exposers.cli import main
+
+raise SystemExit(main())

dbt_data_engineering_toolkit_compiler/adapters.py

@@ -0,0 +1,352 @@
+"""Explicit adapter capabilities used by validation and project emission."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .operational import JsonValue
+
+
+def _env(name: str, default: str | None = None, *, integer: bool = False) -> str:
+    default_argument = f", '{default}'" if default is not None else ""
+    cast = " | int" if integer else ""
+    return "{{ env_var('" + name + "'" + default_argument + ")" + cast + " }}"
+
+
+@dataclass(frozen=True, slots=True)
+class AdapterProvider:
+    name: str
+    display_name: str
+    dbt_dependency: str
+    sqlfluff_dialect: str
+    logical_types: dict[str, str]
+    operational_types: dict[str, str]
+    supported_materializations: frozenset[str]
+    supports_incremental: bool
+    supports_enforced_contracts: bool
+    profile_output: dict[str, JsonValue]
+
+    def data_type(self, logical_type: str, physical_type: str | None) -> str:
+        return physical_type or self.logical_types.get(
+            logical_type.casefold(), logical_type.casefold()
+        )
+
+    def operational_type(self, field_name: str) -> str:
+        return self.operational_types.get(field_name, self.data_type("string", None))
+
+
+class AdapterRegistry:
+    """Single extension point for warehouse-specific compiler behavior."""
+
+    def __init__(self, providers: list[AdapterProvider]):
+        self._providers = {item.name: item for item in providers}
+
+    @classmethod
+    def default(cls) -> "AdapterRegistry":
+        common_materializations = frozenset({"view", "table", "ephemeral"})
+        portable = {
+            "string": "varchar",
+            "number": "numeric",
+            "numeric": "numeric",
+            "integer": "bigint",
+            "boolean": "boolean",
+            "date": "date",
+            "timestamp": "timestamp",
+            "time": "time",
+        }
+        return cls(
+            [
+                AdapterProvider(
+                    name="athena",
+                    display_name="Amazon Athena",
+                    dbt_dependency="dbt-athena-community>=1.10,<2",
+                    sqlfluff_dialect="athena",
+                    logical_types={
+                        **portable,
+                        "string": "varchar",
+                        "number": "double",
+                        "numeric": "decimal(38,6)",
+                        "timestamp": "timestamp",
+                    },
+                    operational_types={
+                        "_det_rejections": "array(varchar)",
+                        "_det_warnings": "array(varchar)",
+                        "_det_loaded_at": "timestamp",
+                        "_det_invocation_id": "varchar",
+                        "_det_source_system": "varchar",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=False,
+                    profile_output={
+                        "type": "athena",
+                        "database": _env("DET_ATHENA_DATABASE", "awsdatacatalog"),
+                        "schema": "__TARGET_SCHEMA__",
+                        "s3_staging_dir": _env("DET_ATHENA_S3_STAGING_DIR"),
+                        "region_name": _env("DET_ATHENA_REGION", "us-east-1"),
+                        "work_group": _env("DET_ATHENA_WORK_GROUP", "primary"),
+                        "threads": 4,
+                    },
+                ),
+                AdapterProvider(
+                    name="bigquery",
+                    display_name="Google BigQuery",
+                    dbt_dependency="dbt-bigquery>=1.10,<2",
+                    sqlfluff_dialect="bigquery",
+                    logical_types={
+                        **portable,
+                        "string": "string",
+                        "number": "float64",
+                        "numeric": "numeric",
+                        "integer": "int64",
+                        "boolean": "bool",
+                    },
+                    operational_types={
+                        "_det_rejections": "array<string>",
+                        "_det_warnings": "array<string>",
+                        "_det_loaded_at": "timestamp",
+                        "_det_invocation_id": "string",
+                        "_det_source_system": "string",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=True,
+                    profile_output={
+                        "type": "bigquery",
+                        "method": "oauth",
+                        "project": _env("DET_BIGQUERY_PROJECT"),
+                        "dataset": "__TARGET_SCHEMA__",
+                        "location": _env("DET_BIGQUERY_LOCATION", "US"),
+                        "threads": 4,
+                    },
+                ),
+                AdapterProvider(
+                    name="clickhouse",
+                    display_name="ClickHouse",
+                    dbt_dependency="dbt-clickhouse>=1.10,<2",
+                    sqlfluff_dialect="clickhouse",
+                    logical_types={
+                        **portable,
+                        "string": "String",
+                        "number": "Float64",
+                        "numeric": "Decimal(38,6)",
+                        "integer": "Int64",
+                        "boolean": "Bool",
+                        "date": "Date",
+                        "timestamp": "DateTime64(6)",
+                        "time": "String",
+                    },
+                    operational_types={
+                        "_det_rejections": "Array(String)",
+                        "_det_warnings": "Array(String)",
+                        "_det_loaded_at": "DateTime64(6)",
+                        "_det_invocation_id": "String",
+                        "_det_source_system": "String",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=False,
+                    profile_output={
+                        "type": "clickhouse",
+                        "host": _env("DET_CLICKHOUSE_HOST"),
+                        "port": _env("DET_CLICKHOUSE_PORT", "8443", integer=True),
+                        "user": _env("DET_CLICKHOUSE_USER", "default"),
+                        "password": _env("DBT_ENV_SECRET_DET_CLICKHOUSE_PASSWORD", ""),
+                        "schema": "__TARGET_SCHEMA__",
+                        "secure": True,
+                        "verify": True,
+                        "threads": 4,
+                    },
+                ),
+                AdapterProvider(
+                    name="databricks",
+                    display_name="Databricks",
+                    dbt_dependency="dbt-databricks>=1.10,<2",
+                    sqlfluff_dialect="databricks",
+                    logical_types={
+                        **portable,
+                        "string": "string",
+                        "number": "double",
+                        "numeric": "decimal(38,6)",
+                        "time": "string",
+                    },
+                    operational_types={
+                        "_det_rejections": "array<string>",
+                        "_det_warnings": "array<string>",
+                        "_det_loaded_at": "timestamp",
+                        "_det_invocation_id": "string",
+                        "_det_source_system": "string",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=True,
+                    profile_output={
+                        "type": "databricks",
+                        "host": _env("DET_DATABRICKS_HOST"),
+                        "http_path": _env("DET_DATABRICKS_HTTP_PATH"),
+                        "token": _env("DBT_ENV_SECRET_DET_DATABRICKS_TOKEN"),
+                        "catalog": _env("DET_DATABRICKS_CATALOG", "hive_metastore"),
+                        "schema": "__TARGET_SCHEMA__",
+                        "threads": 4,
+                    },
+                ),
+                AdapterProvider(
+                    name="duckdb",
+                    display_name="DuckDB",
+                    dbt_dependency="dbt-duckdb>=1.10,<2",
+                    sqlfluff_dialect="duckdb",
+                    logical_types={
+                        **portable,
+                        "string": "varchar",
+                        "number": "double",
+                        "numeric": "decimal(38,6)",
+                    },
+                    operational_types={
+                        "_det_rejections": "varchar[]",
+                        "_det_warnings": "varchar[]",
+                        "_det_loaded_at": "timestamp with time zone",
+                        "_det_invocation_id": "varchar",
+                        "_det_source_system": "varchar",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=True,
+                    profile_output={
+                        "type": "duckdb",
+                        "path": "target/data_product.duckdb",
+                        "schema": "__TARGET_SCHEMA__",
+                        "threads": 4,
+                    },
+                ),
+                AdapterProvider(
+                    name="postgres",
+                    display_name="Postgres",
+                    dbt_dependency="dbt-postgres>=1.10,<2",
+                    sqlfluff_dialect="postgres",
+                    logical_types={**portable, "string": "text"},
+                    operational_types={
+                        "_det_rejections": "text[]",
+                        "_det_warnings": "text[]",
+                        "_det_loaded_at": "timestamp with time zone",
+                        "_det_invocation_id": "text",
+                        "_det_source_system": "text",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=True,
+                    profile_output={
+                        "type": "postgres",
+                        "host": _env("DET_POSTGRES_HOST"),
+                        "port": _env("DET_POSTGRES_PORT", "5432", integer=True),
+                        "user": _env("DET_POSTGRES_USER"),
+                        "password": _env("DET_POSTGRES_PASSWORD"),
+                        "dbname": _env("DET_POSTGRES_DATABASE"),
+                        "schema": "__TARGET_SCHEMA__",
+                        "threads": 4,
+                    },
+                ),
+                AdapterProvider(
+                    name="redshift",
+                    display_name="Amazon Redshift",
+                    dbt_dependency="dbt-redshift>=1.10,<2",
+                    sqlfluff_dialect="redshift",
+                    logical_types={
+                        **portable,
+                        "string": "varchar",
+                        "number": "double precision",
+                        "numeric": "decimal(38,6)",
+                    },
+                    operational_types={
+                        "_det_rejections": "super",
+                        "_det_warnings": "super",
+                        "_det_loaded_at": "timestamp",
+                        "_det_invocation_id": "varchar",
+                        "_det_source_system": "varchar",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=True,
+                    profile_output={
+                        "type": "redshift",
+                        "host": _env("DET_REDSHIFT_HOST"),
+                        "port": _env("DET_REDSHIFT_PORT", "5439", integer=True),
+                        "user": _env("DET_REDSHIFT_USER"),
+                        "password": _env("DBT_ENV_SECRET_DET_REDSHIFT_PASSWORD"),
+                        "dbname": _env("DET_REDSHIFT_DATABASE"),
+                        "schema": "__TARGET_SCHEMA__",
+                        "threads": 4,
+                    },
+                ),
+                AdapterProvider(
+                    name="snowflake",
+                    display_name="Snowflake",
+                    dbt_dependency="dbt-snowflake>=1.10,<2",
+                    sqlfluff_dialect="snowflake",
+                    logical_types={
+                        **portable,
+                        "string": "varchar",
+                        "integer": "number(38,0)",
+                        "timestamp": "timestamp_tz",
+                    },
+                    operational_types={
+                        "_det_rejections": "array",
+                        "_det_warnings": "array",
+                        "_det_loaded_at": "timestamp_tz",
+                        "_det_invocation_id": "varchar",
+                        "_det_source_system": "varchar",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=True,
+                    profile_output={
+                        "type": "snowflake",
+                        "account": _env("DET_SNOWFLAKE_ACCOUNT"),
+                        "user": _env("DET_SNOWFLAKE_USER"),
+                        "password": _env("DBT_ENV_SECRET_DET_SNOWFLAKE_PASSWORD"),
+                        "role": _env("DET_SNOWFLAKE_ROLE"),
+                        "database": _env("DET_SNOWFLAKE_DATABASE"),
+                        "warehouse": _env("DET_SNOWFLAKE_WAREHOUSE"),
+                        "schema": "__TARGET_SCHEMA__",
+                        "threads": 4,
+                    },
+                ),
+                AdapterProvider(
+                    name="spark",
+                    display_name="Apache Spark",
+                    dbt_dependency="dbt-spark[PyHive]>=1.10,<2",
+                    sqlfluff_dialect="sparksql",
+                    logical_types={
+                        **portable,
+                        "string": "string",
+                        "number": "double",
+                        "numeric": "decimal(38,6)",
+                        "time": "string",
+                    },
+                    operational_types={
+                        "_det_rejections": "array<string>",
+                        "_det_warnings": "array<string>",
+                        "_det_loaded_at": "timestamp",
+                        "_det_invocation_id": "string",
+                        "_det_source_system": "string",
+                    },
+                    supported_materializations=common_materializations,
+                    supports_incremental=False,
+                    supports_enforced_contracts=False,
+                    profile_output={
+                        "type": "spark",
+                        "method": "thrift",
+                        "host": _env("DET_SPARK_HOST"),
+                        "port": _env("DET_SPARK_PORT", "10001", integer=True),
+                        "user": _env("DET_SPARK_USER", "dbt"),
+                        "schema": "__TARGET_SCHEMA__",
+                        "threads": 4,
+                    },
+                ),
+            ]
+        )
+
+    def get(self, name: str) -> AdapterProvider | None:
+        return self._providers.get(name.casefold())
+
+    def names(self) -> tuple[str, ...]:
+        return tuple(sorted(self._providers))

dbt_data_engineering_toolkit_compiler/brokers/__init__.py

@@ -0,0 +1,16 @@
+"""Replaceable boundaries for files, workbooks, and command-line dependencies."""
+
+from .commands import CommandBroker, SubprocessCommandBroker
+from .files import FileBroker, LocalFileBroker
+from .workbooks import OpenpyxlWorkbookBroker, SheetData, WorkbookBroker, WorkbookDocument
+
+__all__ = [
+    "CommandBroker",
+    "FileBroker",
+    "LocalFileBroker",
+    "OpenpyxlWorkbookBroker",
+    "SheetData",
+    "SubprocessCommandBroker",
+    "WorkbookBroker",
+    "WorkbookDocument",
+]

dbt_data_engineering_toolkit_compiler/brokers/commands.py

@@ -0,0 +1,46 @@
+"""Broker for executable discovery and subprocess execution."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Iterable, Protocol
+
+from ..operational import CommandResult
+
+
+class CommandBroker(Protocol):
+    """External command boundary consumed by dependency-specific brokers."""
+
+    def find(self, executable: str) -> str | None: ...
+
+    def run(self, command: Iterable[str], *, cwd: Path | None = None) -> CommandResult: ...
+
+
+class SubprocessCommandBroker:
+    """Production command broker backed by the operating system."""
+
+    def find(self, executable: str) -> str | None:
+        suffix = ".exe" if sys.platform == "win32" else ""
+        sibling = Path(sys.executable).parent / f"{executable}{suffix}"
+        if sibling.is_file():
+            return str(sibling)
+        return shutil.which(executable)
+
+    def run(self, command: Iterable[str], *, cwd: Path | None = None) -> CommandResult:
+        arguments = tuple(str(item) for item in command)
+        completed = subprocess.run(
+            arguments,
+            cwd=cwd,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        return CommandResult(
+            command=arguments,
+            return_code=completed.returncode,
+            stdout=completed.stdout,
+            stderr=completed.stderr,
+        )

dbt_data_engineering_toolkit_compiler/brokers/datacontracts.py

@@ -0,0 +1,95 @@
+"""Broker for the authoritative Data Contract CLI and Python API."""
+# pyright: reportMissingImports=false
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from ..errors import MissingDependencyError
+from ..operational import CommandResult
+from .commands import CommandBroker, SubprocessCommandBroker
+
+
+class DataContractBroker(Protocol):
+    def lint(self, contract: Path) -> CommandResult: ...
+
+    def dbt_sync(self, contract: Path, project: Path, *, dry_run: bool) -> CommandResult: ...
+
+    def import_excel(self, workbook: Path, output: Path) -> CommandResult: ...
+
+    def export_excel(self, contract: Path, output: Path) -> CommandResult: ...
+
+
+class CliDataContractBroker:
+    """Production Data Contract integration behind a local interface."""
+
+    def __init__(self, commands: CommandBroker | None = None) -> None:
+        self.commands = commands or SubprocessCommandBroker()
+
+    def lint(self, contract: Path) -> CommandResult:
+        try:
+            from datacontract.data_contract import DataContract
+        except ModuleNotFoundError as exc:
+            raise MissingDependencyError(
+                "datacontract",
+                "Data Contract CLI is missing; reinstall the compiler.",
+            ) from exc
+        result = DataContract(data_contract_file=str(contract)).lint()
+        return CommandResult(
+            command=("datacontract", "lint", str(contract)),
+            return_code=0 if result.has_passed() else 1,
+            stdout=str(result),
+        )
+
+    def dbt_sync(self, contract: Path, project: Path, *, dry_run: bool) -> CommandResult:
+        executable = self._executable()
+        command = [
+            executable,
+            "dbt",
+            "sync",
+            str(contract),
+            "--project-dir",
+            str(project),
+        ]
+        if dry_run:
+            command.append("--dry-run")
+        return self.commands.run(command, cwd=project)
+
+    def import_excel(self, workbook: Path, output: Path) -> CommandResult:
+        """Convert an official ODCS workbook with Data Contract CLI."""
+
+        output.parent.mkdir(parents=True, exist_ok=True)
+        command = [
+            self._executable(),
+            "import",
+            "excel",
+            "--source",
+            str(workbook),
+            "--output",
+            str(output),
+        ]
+        return self.commands.run(command, cwd=workbook.parent)
+
+    def export_excel(self, contract: Path, output: Path) -> CommandResult:
+        """Export canonical ODCS YAML using Data Contract CLI's official template."""
+
+        output.parent.mkdir(parents=True, exist_ok=True)
+        command = [
+            self._executable(),
+            "export",
+            "excel",
+            str(contract),
+            "--output",
+            str(output),
+        ]
+        return self.commands.run(command, cwd=contract.parent)
+
+    def _executable(self) -> str:
+        executable = self.commands.find("datacontract")
+        if executable:
+            return executable
+        raise MissingDependencyError(
+            "datacontract",
+            "Data Contract CLI is missing; reinstall the compiler.",
+        )

dbt_data_engineering_toolkit_compiler/brokers/dependencies.py

@@ -0,0 +1,97 @@
+"""Dependency-specific brokers built on the shared command boundary."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from ..errors import MissingDependencyError
+from ..operational import CommandResult
+from .commands import CommandBroker, SubprocessCommandBroker
+
+
+class DbtBroker(Protocol):
+    def parse(self, project: Path) -> CommandResult: ...
+
+    def seed(
+        self, project: Path, *, selector: str | None = None, exclude: str | None = None
+    ) -> CommandResult: ...
+
+    def build(self, project: Path, *, exclude: str | None = None) -> CommandResult: ...
+
+    def run(self, project: Path, *, selector: str | None = None) -> CommandResult: ...
+
+    def deps(self, project: Path) -> CommandResult: ...
+
+
+class CliDbtBroker:
+    def __init__(self, commands: CommandBroker | None = None) -> None:
+        self.commands = commands or SubprocessCommandBroker()
+
+    def _dbt(self) -> str:
+        executable = self.commands.find("dbt")
+        if not executable:
+            raise MissingDependencyError(
+                "dbt", "dbt is not installed; install the compiler and target adapter."
+            )
+        return executable
+
+    def _run(self, project: Path, operation: str, *arguments: str) -> CommandResult:
+        return self.commands.run(
+            (
+                self._dbt(),
+                operation,
+                "--project-dir",
+                str(project),
+                "--profiles-dir",
+                str(project),
+                *arguments,
+            ),
+            cwd=project,
+        )
+
+    def parse(self, project: Path) -> CommandResult:
+        return self._run(project, "parse")
+
+    def seed(
+        self, project: Path, *, selector: str | None = None, exclude: str | None = None
+    ) -> CommandResult:
+        arguments: list[str] = []
+        if selector:
+            arguments.extend(("--select", selector))
+        if exclude:
+            arguments.extend(("--exclude", exclude))
+        return self._run(project, "seed", *arguments)
+
+    def build(self, project: Path, *, exclude: str | None = None) -> CommandResult:
+        arguments = ("--exclude", exclude) if exclude else ()
+        return self._run(project, "build", *arguments)
+
+    def run(self, project: Path, *, selector: str | None = None) -> CommandResult:
+        arguments = ("--select", selector) if selector else ()
+        return self._run(project, "run", *arguments)
+
+    def deps(self, project: Path) -> CommandResult:
+        return self._run(project, "deps")
+
+
+class SqlFluffBroker(Protocol):
+    def lint(self, project: Path) -> CommandResult: ...
+
+
+class CliSqlFluffBroker:
+    def __init__(self, commands: CommandBroker | None = None) -> None:
+        self.commands = commands or SubprocessCommandBroker()
+
+    def lint(self, project: Path) -> CommandResult:
+        executable = self.commands.find("sqlfluff")
+        if not executable:
+            raise MissingDependencyError(
+                "sqlfluff", "sqlfluff is not installed; reinstall the compiler."
+            )
+        config = project / ".sqlfluff"
+        targets = tuple(name for name in ("models", "tests") if (project / name).exists())
+        return self.commands.run(
+            (executable, "lint", *targets, "--config", str(config)),
+            cwd=project,
+        )