omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/destinations.py

@@ -0,0 +1,982 @@
+import abc
+import base64
+import csv
+import datetime
+import json
+import os
+import shutil
+import struct
+import tempfile
+from urllib.parse import parse_qs, quote, urlparse
+
+import dlt
+import dlt.destinations.impl.filesystem.filesystem
+import requests
+from dlt.common.configuration.specs import AwsCredentials
+from dlt.common.destination.capabilities import DestinationCapabilitiesContext
+from dlt.common.schema import Schema
+from dlt.common.storages.configuration import FileSystemCredentials
+from dlt.destinations.impl.clickhouse.configuration import (
+    ClickHouseCredentials,
+)
+
+from omniload.src.elasticsearch.helpers import elasticsearch_insert
+from omniload.src.errors import MissingValueError
+from omniload.src.loader import load_dlt_file
+from omniload.src.mongodb import mongodb_insert
+
+
+class GenericSqlDestination:
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format <schema>.<table>")
+
+        res = {
+            "dataset_name": table_fields[-2],
+            "table_name": table_fields[-1],
+        }
+
+        return res
+
+    def post_load(self):
+        pass
+
+
+class BigQueryDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        source_fields = urlparse(uri)
+        source_params = parse_qs(source_fields.query)
+
+        cred_path = source_params.get("credentials_path")
+        credentials_base64 = source_params.get("credentials_base64")
+
+        location = None
+        if source_params.get("location"):
+            loc_params = source_params.get("location", [])
+            if len(loc_params) > 1:
+                raise ValueError("Only one location is allowed")
+            location = loc_params[0]
+
+        # Following dlt's pattern (like google_analytics), we let dlt's credential resolution
+        # handle defaults automatically. When credentials_path or credentials_base64 are not
+        # provided, dlt will use Application Default Credentials via GcpServiceAccountCredentials.
+        credentials = None
+        if cred_path:
+            with open(cred_path[0], "r") as f:
+                credentials = json.load(f)
+        elif credentials_base64:
+            credentials = json.loads(
+                base64.b64decode(credentials_base64[0]).decode("utf-8")
+            )
+
+        staging_bucket = kwargs.get("staging_bucket", None)
+        if staging_bucket:
+            if not staging_bucket.startswith("gs://"):
+                raise ValueError("Staging bucket must start with gs://")
+
+            os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = staging_bucket
+            if credentials:
+                os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__PROJECT_ID"] = (
+                    credentials.get("project_id", None)
+                )
+                os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__PRIVATE_KEY"] = (
+                    credentials.get("private_key", None)
+                )
+                os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__CLIENT_EMAIL"] = (
+                    credentials.get("client_email", None)
+                )
+
+        project_id = None
+        if source_fields.hostname:
+            project_id = source_fields.hostname
+
+        return dlt.destinations.bigquery(
+            credentials=credentials,  # type: ignore
+            location=location,  # type: ignore
+            project_id=project_id,
+            **kwargs,
+        )
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        table_fields = table.split(".")
+        if len(table_fields) != 2 and len(table_fields) != 3:
+            raise ValueError(
+                "Table name must be in the format <dataset>.<table> or <project>.<dataset>.<table>"
+            )
+
+        res = {
+            "dataset_name": table_fields[-2],
+            "table_name": table_fields[-1],
+        }
+
+        staging_bucket = kwargs.get("staging_bucket", None)
+        if staging_bucket:
+            res["staging"] = "filesystem"
+
+        return res
+
+    def post_load(self):
+        pass
+
+
+class CrateDBDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        uri = uri.replace("cratedb://", "postgres://")
+        import dlt_cratedb.impl.cratedb.factory
+
+        return dlt_cratedb.impl.cratedb.factory.cratedb(credentials=uri, **kwargs)
+
+
+class PostgresDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.postgres(credentials=uri, **kwargs)
+
+
+class SnowflakeDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.snowflake(credentials=uri, **kwargs)
+
+
+class RedshiftDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.redshift(
+            credentials=uri.replace("redshift://", "postgresql://"), **kwargs
+        )
+
+
+class DuckDBDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        kwargs.pop("dest_table", None)
+        kwargs.pop("staging_bucket", None)
+        return dlt.destinations.duckdb(uri, **kwargs)
+
+
+class MotherduckDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        from urllib.parse import parse_qs, urlparse
+
+        parsed = urlparse(uri)
+        query = parse_qs(parsed.query)
+        token = query.get("token", [None])[0]
+        from dlt.destinations.impl.motherduck.configuration import MotherDuckCredentials
+
+        creds = {
+            "password": token,
+        }
+        database = parsed.path.lstrip("/") or parsed.netloc
+        if database:
+            creds["database"] = database
+
+        return dlt.destinations.motherduck(MotherDuckCredentials(creds), **kwargs)
+
+
+def handle_datetimeoffset(dto_value: bytes) -> datetime.datetime:
+    # ref: https://github.com/mkleehammer/pyodbc/issues/134#issuecomment-281739794
+    tup = struct.unpack(
+        "<6hI2h", dto_value
+    )  # e.g., (2017, 3, 16, 10, 35, 18, 500000000, -6, 0)
+    return datetime.datetime(
+        tup[0],
+        tup[1],
+        tup[2],
+        tup[3],
+        tup[4],
+        tup[5],
+        tup[6] // 1000,
+        datetime.timezone(datetime.timedelta(hours=tup[7], minutes=tup[8])),
+    )
+
+
+# MSSQL_COPT_SS_ACCESS_TOKEN is a connection attribute used to pass
+# an Azure Active Directory access token to the SQL Server ODBC driver.
+MSSQL_COPT_SS_ACCESS_TOKEN = 1256
+
+
+def serialize_azure_token(token):
+    # https://github.com/mkleehammer/pyodbc/issues/228#issuecomment-494773723
+    encoded = token.encode("utf_16_le")
+    return struct.pack("<i", len(encoded)) + encoded
+
+
+def build_mssql_dest():
+    # https://github.com/bruin-data/ingestr/issues/293
+
+    from dlt.destinations.impl.mssql.configuration import MsSqlClientConfiguration
+    from dlt.destinations.impl.mssql.mssql import (
+        HINT_TO_MSSQL_ATTR,
+        MsSqlJobClient,
+    )
+    from dlt.destinations.impl.mssql.sql_client import (
+        PyOdbcMsSqlClient,
+    )
+
+    class OdbcMsSqlClient(PyOdbcMsSqlClient):
+        SKIP_CREDENTIALS = {"PWD", "AUTHENTICATION", "UID"}
+
+        def open_connection(self):
+            cfg = self.credentials.get_odbc_dsn_dict()
+            if (
+                cfg.get("AUTHENTICATION", "").strip().lower()
+                != "activedirectoryaccesstoken"
+            ):
+                return super().open_connection()
+
+            import pyodbc  # type: ignore
+
+            dsn = ";".join(
+                [f"{k}={v}" for k, v in cfg.items() if k not in self.SKIP_CREDENTIALS]
+            )
+
+            self._conn = pyodbc.connect(
+                dsn,
+                timeout=self.credentials.connect_timeout,
+                attrs_before={
+                    MSSQL_COPT_SS_ACCESS_TOKEN: serialize_azure_token(cfg["PWD"]),
+                },
+            )
+
+            # https://github.com/mkleehammer/pyodbc/wiki/Using-an-Output-Converter-function
+            self._conn.add_output_converter(-155, handle_datetimeoffset)
+            self._conn.autocommit = True
+            return self._conn
+
+    class MsSqlClient(MsSqlJobClient):
+        def __init__(
+            self,
+            schema: Schema,
+            config: MsSqlClientConfiguration,
+            capabilities: DestinationCapabilitiesContext,
+        ) -> None:
+            sql_client = OdbcMsSqlClient(
+                config.normalize_dataset_name(schema),
+                config.normalize_staging_dataset_name(schema),
+                config.credentials,
+                capabilities,
+            )
+            super(MsSqlJobClient, self).__init__(schema, config, sql_client)
+            self.config: MsSqlClientConfiguration = config
+            self.sql_client = sql_client
+            self.active_hints = HINT_TO_MSSQL_ATTR if self.config.create_indexes else {}
+            self.type_mapper = capabilities.get_type_mapper()
+
+    class MsSqlDestImpl(dlt.destinations.mssql):
+        @property
+        def client_class(self):
+            return MsSqlClient
+
+    return MsSqlDestImpl
+
+
+class MsSQLDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        cls = build_mssql_dest()
+        return cls(credentials=uri, **kwargs)
+
+
+def get_databricks_oauth_token(
+    server_hostname: str, client_id: str, client_secret: str
+) -> str:
+    """
+    Exchange Databricks OAuth M2M client credentials for an access token.
+
+    This implements the OAuth 2.0 client credentials grant flow for Databricks
+    service principal authentication.
+
+    Args:
+        server_hostname: The Databricks workspace hostname (e.g., dbc-xxx.cloud.databricks.com)
+        client_id: The service principal's client ID (application ID)
+        client_secret: The OAuth secret for the service principal
+
+    Returns:
+        The access token string
+
+    Raises:
+        ValueError: If inputs are invalid or the token request fails
+    """
+    if not server_hostname:
+        raise ValueError("server_hostname is required for OAuth token exchange")
+    if not client_id:
+        raise ValueError("client_id is required for OAuth token exchange")
+    if not client_secret:
+        raise ValueError("client_secret is required for OAuth token exchange")
+
+    token_url = f"https://{server_hostname}/oidc/v1/token"
+
+    try:
+        response = requests.post(
+            token_url,
+            data={
+                "grant_type": "client_credentials",
+                "scope": "all-apis",
+            },
+            auth=(client_id, client_secret),
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            timeout=30,
+        )
+    except requests.exceptions.RequestException as e:
+        raise ValueError(
+            f"Failed to connect to Databricks OAuth endpoint at {token_url}: {e}"
+        ) from e
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        raise ValueError(
+            f"Failed to obtain Databricks OAuth token: HTTP {response.status_code}"
+        ) from e
+
+    try:
+        token_data = response.json()
+    except (json.JSONDecodeError, ValueError) as e:
+        raise ValueError("Invalid JSON response from Databricks OAuth endpoint") from e
+
+    if "access_token" not in token_data:
+        raise ValueError("Databricks OAuth response missing 'access_token' field")
+
+    return token_data["access_token"]
+
+
+class DatabricksDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        p = urlparse(uri)
+        q = parse_qs(p.query)
+        server_hostname = p.hostname
+        http_path = q.get("http_path", [None])[0]
+        catalog = q.get("catalog", [None])[0]
+
+        if not server_hostname:
+            raise ValueError("Databricks URI must include a server hostname")
+
+        # Check for OAuth M2M credentials (client_id and client_secret)
+        client_id = q.get("client_id", [None])[0]
+        client_secret = q.get("client_secret", [None])[0]
+
+        access_token: str
+        if client_id and client_secret:
+            # OAuth M2M authentication: exchange client credentials for access token
+            access_token = get_databricks_oauth_token(
+                server_hostname, client_id, client_secret
+            )
+        else:
+            # Traditional token-based authentication
+            if not p.password:
+                raise ValueError(
+                    "Databricks URI must include an access token or client_id/client_secret"
+                )
+            access_token = p.password
+
+        creds = {
+            "access_token": access_token,
+            "server_hostname": server_hostname,
+            "http_path": http_path,
+            "catalog": catalog,
+        }
+
+        return dlt.destinations.databricks(
+            credentials=creds,
+            **kwargs,
+        )
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        p = urlparse(uri)
+        q = parse_qs(p.query)
+        uri_schema = q.get("schema", [None])[0]
+
+        table_fields = table.split(".")
+
+        # If table is in schema.table format, use that (overrides URI schema)
+        if len(table_fields) == 2:
+            return {
+                "dataset_name": table_fields[0],
+                "table_name": table_fields[1],
+            }
+
+        # If table is just a table name, use schema from URI
+        if len(table_fields) == 1 and uri_schema:
+            return {
+                "dataset_name": uri_schema,
+                "table_name": table_fields[0],
+            }
+
+        raise ValueError(
+            "Table name must be in the format <schema>.<table>, or specify schema in the URI"
+        )
+
+
+class SynapseDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.synapse(credentials=uri, **kwargs)
+
+
+class CustomCsvDestination(dlt.destinations.filesystem):
+    pass
+
+
+class CsvDestination(GenericSqlDestination):
+    temp_path: str
+    actual_path: str
+    uri: str
+    dataset_name: str
+    table_name: str
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format <schema>.<table>")
+
+        res = {
+            "dataset_name": table_fields[-2],
+            "table_name": table_fields[-1],
+        }
+
+        self.dataset_name = res["dataset_name"]
+        self.table_name = res["table_name"]
+        self.uri = uri
+
+        return res
+
+    def dlt_dest(self, uri: str, **kwargs):
+        if uri.startswith("csv://"):
+            uri = uri.replace("csv://", "file://")
+
+        temp_path = tempfile.mkdtemp()
+        self.actual_path = uri
+        self.temp_path = temp_path
+        return CustomCsvDestination(bucket_url=f"file://{temp_path}", **kwargs)
+
+    # I dislike this implementation quite a bit since it ties the implementation to some internal details on how dlt works
+    # I would prefer a custom destination that allows me to do this easily but dlt seems to have a lot of internal details that are not documented
+    # I tried to make it work with a nicer destination implementation but I couldn't, so I decided to go with this hack to experiment
+    # if anyone has a better idea on how to do this, I am open to contributions or suggestions
+    def post_load(self):
+        def find_first_file(path):
+            for entry in os.listdir(path):
+                full_path = os.path.join(path, entry)
+                if os.path.isfile(full_path):
+                    return full_path
+
+            return None
+
+        def filter_keys(dictionary):
+            return {
+                key: value
+                for key, value in dictionary.items()
+                if not key.startswith("_dlt_")
+            }
+
+        first_file_path = find_first_file(
+            f"{self.temp_path}/{self.dataset_name}/{self.table_name}"
+        )
+
+        output_path = self.uri.split("://")[1]
+        if output_path.count("/") > 1:
+            os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+        def _rewrite_csv_with_fieldnames(path, fieldnames):
+            tmp_fd, tmp_path = tempfile.mkstemp(
+                suffix=".csv", dir=os.path.dirname(path) or "."
+            )
+            try:
+                os.close(tmp_fd)
+                with (
+                    open(path, "r", newline="") as old,
+                    open(tmp_path, "w", newline="") as new,
+                ):
+                    reader = csv.DictReader(old)
+                    writer = csv.DictWriter(new, fieldnames=fieldnames, restval="")
+                    writer.writeheader()
+                    for r in reader:
+                        writer.writerow(r)
+                os.replace(tmp_path, path)
+            except BaseException:
+                os.unlink(tmp_path)
+                raise
+
+        fieldnames = {}
+        csv_writer = None
+        csv_file = None
+
+        try:
+            for row in load_dlt_file(first_file_path):
+                row = filter_keys(row)
+                new_fields = False
+                for key in row:
+                    if key not in fieldnames:
+                        fieldnames[key] = None
+                        new_fields = True
+
+                if csv_writer is None:
+                    csv_file = open(output_path, "w", newline="")
+                    csv_writer = csv.DictWriter(
+                        csv_file, fieldnames=fieldnames, restval=""
+                    )
+                    csv_writer.writeheader()
+                elif new_fields:
+                    csv_file.close()
+                    _rewrite_csv_with_fieldnames(output_path, list(fieldnames))
+                    csv_file = open(output_path, "a", newline="")
+                    csv_writer = csv.DictWriter(
+                        csv_file, fieldnames=fieldnames, restval=""
+                    )
+
+                csv_writer.writerow(row)
+        finally:
+            if csv_file:
+                csv_file.close()
+        shutil.rmtree(self.temp_path)
+
+
+class AthenaDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        encoded_uri = quote(uri, safe=":/?&=")
+        source_fields = urlparse(encoded_uri)
+        source_params = parse_qs(source_fields.query)
+
+        bucket = source_params.get("bucket", [None])[0]
+        if not bucket:
+            raise ValueError("A bucket is required to connect to Athena.")
+
+        if not bucket.startswith("s3://"):
+            bucket = f"s3://{bucket}"
+
+        bucket = bucket.rstrip("/")
+
+        dest_table = kwargs.get("dest_table", None)
+        if not dest_table:
+            raise ValueError("A destination table is required to connect to Athena.")
+
+        dest_table_fields = dest_table.split(".")
+        if len(dest_table_fields) != 2:
+            raise ValueError(
+                f"Table name must be in the format <schema>.<table>, given: {dest_table}"
+            )
+
+        query_result_path = f"{bucket}/{dest_table_fields[0]}_staging/metadata"
+
+        access_key_id = source_params.get("access_key_id", [None])[0]
+        secret_access_key = source_params.get("secret_access_key", [None])[0]
+        session_token = source_params.get("session_token", [None])[0]
+        profile_name = source_params.get("profile", ["default"])[0]
+        region_name = source_params.get("region_name", [None])[0]
+
+        if not access_key_id and not secret_access_key:
+            import botocore.session  # type: ignore
+
+            session = botocore.session.Session(profile=profile_name)
+            default = session.get_credentials()
+            if not profile_name:
+                raise ValueError(
+                    "You have to either provide access_key_id and secret_access_key pair or a valid AWS profile name."
+                )
+            access_key_id = default.access_key
+            secret_access_key = default.secret_key
+            session_token = default.token
+            if region_name is None:
+                region_name = session.get_config_variable("region")
+
+        if not region_name:
+            raise ValueError("The region_name is required to connect to Athena.")
+
+        os.environ["DESTINATION__BUCKET_URL"] = bucket
+        if access_key_id and secret_access_key:
+            os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
+            os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
+                secret_access_key
+            )
+        if session_token:
+            os.environ["DESTINATION__CREDENTIALS__AWS_SESSION_TOKEN"] = session_token
+
+        return dlt.destinations.athena(
+            query_result_bucket=query_result_path,
+            athena_work_group=source_params.get("workgroup", [None])[0],  # type: ignore
+            credentials=AwsCredentials(
+                aws_access_key_id=access_key_id,  # type: ignore
+                aws_secret_access_key=secret_access_key,  # type: ignore
+                aws_session_token=session_token,
+                region_name=region_name,
+            ),
+            destination_name=bucket,
+            force_iceberg=True,
+        )
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format <schema>.<table>")
+        return {
+            "table_format": "iceberg",
+            "dataset_name": table_fields[-2],
+            "table_name": table_fields[-1],
+        }
+
+    def post_load(self):
+        pass
+
+
+class ClickhouseDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        parsed_uri = urlparse(uri)
+
+        if "dest_table" in kwargs:
+            table = kwargs["dest_table"]
+            database = table.split(".")[0]
+        else:
+            database = parsed_uri.path.lstrip("/")
+
+        username = parsed_uri.username
+        if not username:
+            raise ValueError(
+                "A username is required to connect to the ClickHouse database."
+            )
+
+        password = parsed_uri.password
+        if not password:
+            raise ValueError(
+                "A password is required to authenticate with the ClickHouse database."
+            )
+
+        host = parsed_uri.hostname
+        if not host:
+            raise ValueError(
+                "The hostname or IP address of the ClickHouse server is required to establish a connection."
+            )
+
+        port = parsed_uri.port
+        if not port:
+            raise ValueError(
+                "The TCP port of the ClickHouse server is required to establish a connection."
+            )
+
+        query_params = parse_qs(parsed_uri.query)
+        secure = int(query_params["secure"][0]) if "secure" in query_params else 1
+
+        default_http_port = 8443 if secure == 1 else 8123
+        http_port = (
+            int(query_params["http_port"][0])
+            if "http_port" in query_params
+            else default_http_port
+        )
+
+        if secure not in (0, 1):
+            raise ValueError(
+                "Invalid value for secure. Set to `1` for a secure HTTPS connection or `0` for a non-secure HTTP connection."
+            )
+
+        credentials = ClickHouseCredentials(
+            {
+                "host": host,
+                "port": port,
+                "username": username,
+                "password": password,
+                "database": database,
+                "http_port": http_port,
+                "secure": secure,
+            }
+        )
+        return dlt.destinations.clickhouse(credentials=credentials)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format <schema>.<table>")
+        return {
+            "table_name": table_fields[-1],
+        }
+
+    def post_load(self):
+        pass
+
+    @staticmethod
+    def engine_settings(uri: str) -> dict[str, str]:
+        parsed_uri = urlparse(uri)
+        query_params = parse_qs(parsed_uri.query)
+        return {
+            key[len("engine.") :]: query_params[key][0]
+            for key in query_params
+            if key.startswith("engine.")
+        }
+
+    @staticmethod
+    def engine_type(uri: str) -> str | None:
+        parsed_uri = urlparse(uri)
+        query_params = parse_qs(parsed_uri.query)
+        values = query_params.get("engine")
+        if values:
+            return values[0]
+        return None
+
+
+class BlobFSClient(dlt.destinations.impl.filesystem.filesystem.FilesystemClient):
+    @property
+    def dataset_path(self):
+        # override to remove dataset path
+        return self.bucket_path
+
+
+class BlobFS(dlt.destinations.filesystem):
+    @property
+    def client_class(self):
+        return BlobFSClient
+
+
+class SqliteDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        return {
+            # https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
+            "dataset_name": "main",
+            "table_name": table,
+        }
+
+
+class MySqlDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        database = parsed.path.lstrip("/")
+        if not database:
+            raise ValueError("You need to specify a database")
+        return {
+            "dataset_name": database,
+            "table_name": table,
+        }
+
+
+class TrinoTypeMapper:
+    """Custom type mapper for Trino to handle unsupported types."""
+
+    @staticmethod
+    def create_type_mapper():
+        """Create a custom type mapper for Trino."""
+        from dlt.destinations.impl.sqlalchemy.type_mapper import SqlalchemyTypeMapper
+        from sqlalchemy import BigInteger, Text
+        from sqlalchemy.sql import sqltypes
+
+        class CustomTrinoTypeMapper(SqlalchemyTypeMapper):
+            """Custom type mapper that converts unsupported Trino types."""
+
+            def to_destination_type(self, column, table=None):
+                # Handle special cases before calling parent
+                data_type = column.get("data_type", "")
+
+                # Convert JSON to VARCHAR for Trino's Iceberg catalog
+                if data_type == "json":
+                    # Use TEXT (unlimited VARCHAR) for JSON data
+                    return Text()
+
+                # Convert BINARY to VARCHAR
+                if data_type == "binary":
+                    return Text()
+
+                # Handle integer types - always use BIGINT for Trino
+                # Note: dlt uses "bigint" internally, not "integer"
+                if data_type in ["bigint", "integer", "int"]:
+                    return BigInteger()
+
+                # For other types, try parent mapper
+                try:
+                    type_ = super().to_destination_type(column, table)
+                except Exception:
+                    # If parent can't handle it, default to TEXT
+                    return Text()
+
+                # Convert any INTEGER type to BIGINT
+                if isinstance(type_, sqltypes.Integer) and not isinstance(
+                    type_, sqltypes.BigInteger
+                ):
+                    return BigInteger()
+
+                # Ensure VARCHAR types don't have constraints that Trino doesn't support
+                if isinstance(type_, sqltypes.String):
+                    # Return TEXT for unlimited string
+                    return Text()
+
+                return type_
+
+        return CustomTrinoTypeMapper
+
+
+class TrinoDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        # Import required modules
+        from dlt.destinations.impl.sqlalchemy.factory import (
+            sqlalchemy as sqlalchemy_factory,
+        )
+
+        # Create the destination with custom type mapper
+        # We need to use the factory to properly configure the type mapper
+        dest = sqlalchemy_factory(
+            credentials=uri, type_mapper=TrinoTypeMapper.create_type_mapper(), **kwargs
+        )
+
+        return dest
+
+
+class BlobStorageDestination(abc.ABC):
+    @abc.abstractmethod
+    def credentials(self, params: dict) -> FileSystemCredentials:
+        """Build credentials for the blob storage destination."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def protocol(self) -> str:
+        """The protocol used for the blob storage destination."""
+        pass
+
+    def dlt_dest(self, uri: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        creds = self.credentials(params)
+
+        dest_table = kwargs["dest_table"]
+
+        # only validate if dest_table is not a full URI
+        if not parsed_uri.netloc:
+            dest_table = self.validate_table(dest_table)
+
+        table_parts = dest_table.split("/")
+
+        if parsed_uri.path.strip("/"):
+            path_parts = parsed_uri.path.strip("/ ").split("/")
+            table_parts = path_parts + table_parts
+
+        if parsed_uri.netloc:
+            table_parts.insert(0, parsed_uri.netloc.strip())
+
+        base_path = "/".join(table_parts[:-1])
+
+        opts = {
+            "bucket_url": f"{self.protocol}://{base_path}",
+            "credentials": creds,
+            # supresses dlt warnings about dataset name normalization.
+            # we don't use dataset names in S3 so it's fine to disable this.
+            "enable_dataset_name_normalization": False,
+        }
+        layout = params.get("layout", [None])[0]
+        if layout is not None:
+            opts["layout"] = layout
+
+        return BlobFS(**opts)  # type: ignore
+
+    def validate_table(self, table: str):
+        table = table.strip("/ ")
+        if len(table.split("/")) < 2:
+            raise ValueError("Table name must be in the format {bucket-name}/{path}")
+        return table
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        table_parts = table.split("/")
+        return {
+            "table_name": table_parts[-1].strip(),
+        }
+
+    def post_load(self) -> None:
+        pass
+
+
+class S3Destination(BlobStorageDestination):
+    @property
+    def protocol(self) -> str:
+        return "s3"
+
+    def credentials(self, params: dict) -> FileSystemCredentials:
+        access_key_id = params.get("access_key_id", [None])[0]
+        if access_key_id is None:
+            raise MissingValueError("access_key_id", "S3")
+
+        secret_access_key = params.get("secret_access_key", [None])[0]
+        if secret_access_key is None:
+            raise MissingValueError("secret_access_key", "S3")
+
+        endpoint_url = params.get("endpoint_url", [None])[0]
+        if endpoint_url is not None:
+            parsed_endpoint = urlparse(endpoint_url)
+            if not parsed_endpoint.scheme or not parsed_endpoint.netloc:
+                raise ValueError("Invalid endpoint_url. Must be a valid URL.")
+
+        return AwsCredentials(
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            endpoint_url=endpoint_url,
+        )
+
+
+class GCSDestination(BlobStorageDestination):
+    @property
+    def protocol(self) -> str:
+        return "gs"
+
+    def credentials(self, params: dict) -> FileSystemCredentials:
+        """Builds GCS credentials from the provided parameters."""
+        credentials_path = params.get("credentials_path")
+        credentials_base64 = params.get("credentials_base64")
+        credentials_available = any(
+            map(
+                lambda x: x is not None,
+                [credentials_path, credentials_base64],
+            )
+        )
+        if credentials_available is False:
+            raise MissingValueError("credentials_path or credentials_base64", "GCS")
+
+        credentials = None
+        if credentials_path:
+            with open(credentials_path[0], "r") as f:
+                credentials = json.load(f)
+        else:
+            credentials = json.loads(base64.b64decode(credentials_base64[0]).decode())  # type: ignore
+
+        return credentials
+
+
+class ElasticsearchDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        from urllib.parse import urlparse
+
+        parsed_uri = urlparse(uri)
+
+        # Extract connection details from URI
+        scheme = parsed_uri.scheme or "http"
+        host = parsed_uri.hostname or "localhost"
+        port = parsed_uri.port or 9200
+        username = parsed_uri.username
+        password = parsed_uri.password
+
+        # Build connection string
+        if username and password:
+            connection_string = f"{scheme}://{username}:{password}@{host}:{port}"
+        else:
+            connection_string = f"{scheme}://{host}:{port}"
+
+        # Add query parameters if any
+        if parsed_uri.query:
+            connection_string += f"?{parsed_uri.query}"
+
+        return elasticsearch_insert(connection_string=connection_string)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        return {
+            "table_name": table,
+        }
+
+    def post_load(self):
+        pass
+
+
+class MongoDBDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        return mongodb_insert(uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        return {
+            "table_name": table,
+        }
+
+    def post_load(self):
+        pass