ingestr 0.9.5__py3-none-any.whl → 0.10.0__py3-none-any.whl

ingestr/src/sql_database/__init__.py

@@ -1,206 +0,0 @@
-"""Source that loads tables form any SQLAlchemy supported database, supports batching requests and incremental loads."""
-
-from typing import Any, Callable, Dict, Iterable, List, Optional, Union
-
-import dlt
-from dlt.sources import DltResource
-from sqlalchemy import MetaData, Table
-from sqlalchemy.engine import Engine
-
-from .helpers import (
-    SqlDatabaseTableConfiguration,
-    SqlTableResourceConfiguration,
-    TableBackend,
-    _detect_precision_hints_deprecated,
-    engine_from_credentials,
-    table_rows,
-)
-from .override import IngestrConnectionStringCredentials as ConnectionStringCredentials
-from .schema_types import (
-    ReflectionLevel,
-    TTypeAdapter,
-    get_primary_key,
-    table_to_columns,
-)
-
-
-@dlt.source
-def sql_database(
-    credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value,
-    schema: Optional[str] = dlt.config.value,
-    metadata: Optional[MetaData] = None,
-    table_names: Optional[List[str]] = dlt.config.value,
-    chunk_size: int = 50000,
-    backend: TableBackend = "sqlalchemy",
-    detect_precision_hints: Optional[bool] = False,
-    reflection_level: Optional[ReflectionLevel] = "full",
-    defer_table_reflect: Optional[bool] = None,
-    table_adapter_callback: Callable[[Table], None] = None,
-    backend_kwargs: Dict[str, Any] = None,
-    include_views: bool = False,
-    type_adapter_callback: Optional[TTypeAdapter] = None,
-) -> Iterable[DltResource]:
-    """
-    A dlt source which loads data from an SQL database using SQLAlchemy.
-    Resources are automatically created for each table in the schema or from the given list of tables.
-
-    Args:
-        credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `sqlalchemy.Engine` instance.
-        schema (Optional[str]): Name of the database schema to load (if different from default).
-        metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. `schema` argument is ignored when this is used.
-        table_names (Optional[List[str]]): A list of table names to load. By default, all tables in the schema are loaded.
-        chunk_size (int): Number of rows yielded in one batch. SQL Alchemy will create additional internal rows buffer twice the chunk size.
-        backend (TableBackend): Type of backend to generate table data. One of: "sqlalchemy", "pyarrow", "pandas" and "connectorx".
-            "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames.
-            "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types,
-            "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself.
-        detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
-            This is disabled by default.
-        reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema.
-            "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data.
-            "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option.
-            "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types.
-        defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Requires table_names to be explicitly passed.
-            Enable this option when running on Airflow. Available on dlt 0.4.4 and later.
-        table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected.
-        backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx.
-        include_views (bool): Reflect views as well as tables. Note view names included in `table_names` are always included regardless of this setting.
-        type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns.
-            Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data)
-    Returns:
-
-        Iterable[DltResource]: A list of DLT resources for each table to be loaded.
-    """
-    # detect precision hints is deprecated
-    _detect_precision_hints_deprecated(detect_precision_hints)
-
-    if detect_precision_hints:
-        reflection_level = "full_with_precision"
-    else:
-        reflection_level = reflection_level or "minimal"
-
-    # set up alchemy engine
-    engine = engine_from_credentials(credentials)
-    engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size)
-    metadata = metadata or MetaData(schema=schema)
-
-    # use provided tables or all tables
-    if table_names:
-        tables = [
-            Table(name, metadata, autoload_with=None if defer_table_reflect else engine)
-            for name in table_names
-        ]
-    else:
-        if defer_table_reflect:
-            raise ValueError("You must pass table names to defer table reflection")
-        metadata.reflect(bind=engine, views=include_views)
-        tables = list(metadata.tables.values())
-
-    for table in tables:
-        if table_adapter_callback and not defer_table_reflect:
-            table_adapter_callback(table)
-
-        yield dlt.resource(
-            table_rows,
-            name=table.name,
-            primary_key=get_primary_key(table),
-            spec=SqlDatabaseTableConfiguration,
-            columns=table_to_columns(table, reflection_level, type_adapter_callback),
-        )(
-            engine,
-            table,
-            chunk_size,
-            backend,
-            reflection_level=reflection_level,
-            defer_table_reflect=defer_table_reflect,
-            table_adapter_callback=table_adapter_callback,
-            backend_kwargs=backend_kwargs,
-            type_adapter_callback=type_adapter_callback,
-        )
-
-
-@dlt.resource(
-    name=lambda args: args["table"], standalone=True, spec=SqlTableResourceConfiguration
-)
-def sql_table(
-    credentials: Union[ConnectionStringCredentials, Engine, str] = str,
-    table: str = dlt.config.value,
-    schema: Optional[str] = dlt.config.value,
-    metadata: Optional[MetaData] = None,
-    incremental: Optional[dlt.sources.incremental[Any]] = None,
-    chunk_size: int = 50000,
-    backend: TableBackend = "sqlalchemy",
-    detect_precision_hints: Optional[bool] = None,
-    reflection_level: Optional[ReflectionLevel] = "full_with_precision",
-    defer_table_reflect: Optional[bool] = None,
-    table_adapter_callback: Callable[[Table], None] = None,
-    backend_kwargs: Dict[str, Any] = None,
-    type_adapter_callback: Optional[TTypeAdapter] = None,
-    merge_key: Optional[str] = None,
-) -> DltResource:
-    """
-    A dlt resource which loads data from an SQL database table using SQLAlchemy.
-
-    Args:
-        credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `Engine` instance representing the database connection.
-        table (str): Name of the table or view to load.
-        schema (Optional[str]): Optional name of the schema the table belongs to.
-        metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. If provided, the `schema` argument is ignored.
-        incremental (Optional[dlt.sources.incremental[Any]]): Option to enable incremental loading for the table.
-            E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
-        chunk_size (int): Number of rows yielded in one batch. SQL Alchemy will create additional internal rows buffer twice the chunk size.
-        backend (TableBackend): Type of backend to generate table data. One of: "sqlalchemy", "pyarrow", "pandas" and "connectorx".
-            "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames.
-            "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types,
-            "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself.
-        reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema.
-            "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data.
-            "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option.
-            "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types.
-        detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
-            This is disabled by default.
-        defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Enable this option when running on Airflow. Available
-            on dlt 0.4.4 and later
-        table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected.
-        backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx.
-        type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns.
-            Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data)
-
-    Returns:
-        DltResource: The dlt resource for loading data from the SQL database table.
-    """
-    _detect_precision_hints_deprecated(detect_precision_hints)
-
-    if detect_precision_hints:
-        reflection_level = "full_with_precision"
-    else:
-        reflection_level = reflection_level or "minimal"
-
-    engine = engine_from_credentials(credentials)
-    engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size)
-    metadata = metadata or MetaData(schema=schema)
-
-    table_obj = Table(
-        table, metadata, autoload_with=None if defer_table_reflect else engine
-    )
-    if table_adapter_callback and not defer_table_reflect:
-        table_adapter_callback(table_obj)
-
-    return dlt.resource(
-        table_rows,
-        name=table_obj.name,
-        primary_key=get_primary_key(table_obj),
-        columns=table_to_columns(table_obj, reflection_level, type_adapter_callback),
-        merge_key=merge_key,
-    )(
-        engine,
-        table_obj,
-        chunk_size,
-        backend,
-        incremental=incremental,
-        reflection_level=reflection_level,
-        defer_table_reflect=defer_table_reflect,
-        table_adapter_callback=table_adapter_callback,
-        backend_kwargs=backend_kwargs,
-        type_adapter_callback=type_adapter_callback,
-    )

ingestr/src/sql_database/arrow_helpers.py

@@ -1,139 +0,0 @@
-from typing import Any, Optional, Sequence
-
-from dlt.common import logger
-from dlt.common.configuration import with_config
-from dlt.common.destination import DestinationCapabilitiesContext
-from dlt.common.json import custom_encode, map_nested_in_place
-from dlt.common.schema.typing import TTableSchemaColumns
-
-from .schema_types import RowAny
-
-
-@with_config
-def columns_to_arrow(
-    columns_schema: TTableSchemaColumns,
-    caps: DestinationCapabilitiesContext = None,
-    tz: str = "UTC",
-) -> Any:
-    """Converts `column_schema` to arrow schema using `caps` and `tz`. `caps` are injected from the container - which
-    is always the case if run within the pipeline. This will generate arrow schema compatible with the destination.
-    Otherwise generic capabilities are used
-    """
-    from dlt.common.destination.capabilities import DestinationCapabilitiesContext
-    from dlt.common.libs.pyarrow import get_py_arrow_datatype
-    from dlt.common.libs.pyarrow import pyarrow as pa
-
-    return pa.schema(
-        [
-            pa.field(
-                name,
-                get_py_arrow_datatype(
-                    schema_item,
-                    caps or DestinationCapabilitiesContext.generic_capabilities(),
-                    tz,
-                ),
-                nullable=schema_item.get("nullable", True),
-            )
-            for name, schema_item in columns_schema.items()
-            if schema_item.get("data_type") is not None
-        ]
-    )
-
-
-def row_tuples_to_arrow(
-    rows: Sequence[RowAny], columns: TTableSchemaColumns, tz: str
-) -> Any:
-    """Converts the rows to an arrow table using the columns schema.
-    Columns missing `data_type` will be inferred from the row data.
-    Columns with object types not supported by arrow are excluded from the resulting table.
-    """
-    import numpy as np
-    from dlt.common.libs.pyarrow import pyarrow as pa
-
-    try:
-        from pandas._libs import lib
-
-        pivoted_rows = lib.to_object_array_tuples(rows).T  # type: ignore[attr-defined]
-    except ImportError:
-        logger.info(
-            "Pandas not installed, reverting to numpy.asarray to create a table which is slower"
-        )
-        pivoted_rows = np.asarray(rows, dtype="object", order="k").T  # type: ignore[call-overload]
-
-    columnar = {
-        col: dat.ravel()
-        for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns)))
-    }
-    columnar_known_types = {
-        col["name"]: columnar[col["name"]]
-        for col in columns.values()
-        if col.get("data_type") is not None
-    }
-    columnar_unknown_types = {
-        col["name"]: columnar[col["name"]]
-        for col in columns.values()
-        if col.get("data_type") is None
-    }
-
-    arrow_schema = columns_to_arrow(columns, tz=tz)
-
-    for idx in range(0, len(arrow_schema.names)):
-        field = arrow_schema.field(idx)
-        py_type = type(rows[0][idx])
-        # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects
-        if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)):
-            logger.warning(
-                f"Field {field.name} was reflected as decimal type, but rows contains {py_type.__name__}. Additional cast is required which may slow down arrow table generation."
-            )
-            float_array = pa.array(columnar_known_types[field.name], type=pa.float64())
-            columnar_known_types[field.name] = float_array.cast(field.type, safe=False)
-
-    # If there are unknown type columns, first create a table to infer their types
-    if columnar_unknown_types:
-        new_schema_fields = []
-        for key in list(columnar_unknown_types):
-            arrow_col: Optional[pa.Array] = None
-            try:
-                arrow_col = pa.array(columnar_unknown_types[key])
-                if pa.types.is_null(arrow_col.type):
-                    logger.warning(
-                        f"Column {key} contains only NULL values and data type could not be inferred. This column is removed from a arrow table"
-                    )
-                    continue
-
-            except pa.ArrowInvalid as e:
-                # Try coercing types not supported by arrow to a json friendly format
-                # E.g. dataclasses -> dict, UUID -> str
-                try:
-                    arrow_col = pa.array(
-                        map_nested_in_place(
-                            custom_encode, list(columnar_unknown_types[key])
-                        )
-                    )
-                    logger.warning(
-                        f"Column {key} contains a data type which is not supported by pyarrow and got converted into {arrow_col.type}. This slows down arrow table generation."
-                    )
-                except (pa.ArrowInvalid, TypeError):
-                    logger.warning(
-                        f"Column {key} contains a data type which is not supported by pyarrow. This column will be ignored. Error: {e}"
-                    )
-            if arrow_col is not None:
-                columnar_known_types[key] = arrow_col
-                new_schema_fields.append(
-                    pa.field(
-                        key,
-                        arrow_col.type,
-                        nullable=columns[key]["nullable"],
-                    )
-                )
-
-        # New schema
-        column_order = {name: idx for idx, name in enumerate(columns)}
-        arrow_schema = pa.schema(
-            sorted(
-                list(arrow_schema) + new_schema_fields,
-                key=lambda x: column_order[x.name],
-            )
-        )
-
-    return pa.Table.from_pydict(columnar_known_types, schema=arrow_schema)

ingestr/src/sql_database/helpers.py

@@ -1,282 +0,0 @@
-"""SQL database source helpers"""
-
-import operator
-import warnings
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterator,
-    Literal,
-    Optional,
-    Union,
-)
-
-import dlt
-from dlt.common.configuration.specs import BaseConfiguration, configspec
-from dlt.common.exceptions import MissingDependencyException
-from dlt.common.schema import TTableSchemaColumns
-from dlt.common.typing import TDataItem, TSortOrder
-from sqlalchemy import create_engine
-from sqlalchemy.engine import Engine
-from sqlalchemy.exc import CompileError
-
-from .arrow_helpers import row_tuples_to_arrow
-from .override import IngestrConnectionStringCredentials as ConnectionStringCredentials
-from .schema_types import (
-    ReflectionLevel,
-    SelectAny,
-    Table,
-    TTypeAdapter,
-    get_primary_key,
-    table_to_columns,
-)
-
-TableBackend = Literal["sqlalchemy", "pyarrow", "pandas", "connectorx"]
-
-
-class TableLoader:
-    def __init__(
-        self,
-        engine: Engine,
-        backend: TableBackend,
-        table: Table,
-        columns: TTableSchemaColumns,
-        chunk_size: int = 1000,
-        incremental: Optional[dlt.sources.incremental[Any]] = None,
-    ) -> None:
-        self.engine = engine
-        self.backend = backend
-        self.table = table
-        self.columns = columns
-        self.chunk_size = chunk_size
-        self.incremental = incremental
-        if incremental:
-            try:
-                self.cursor_column = table.c[incremental.cursor_path]
-            except KeyError as e:
-                raise KeyError(
-                    f"Cursor column '{incremental.cursor_path}' does not exist in table '{table.name}'"
-                ) from e
-            self.last_value = incremental.last_value
-            self.end_value = incremental.end_value
-            self.row_order: TSortOrder = self.incremental.row_order
-        else:
-            self.cursor_column = None
-            self.last_value = None
-            self.end_value = None
-            self.row_order = None
-
-    def make_query(self) -> SelectAny:
-        table = self.table
-        query = table.select()
-        if not self.incremental:
-            return query
-
-        last_value_func = self.incremental.last_value_func
-
-        # generate where
-        if (
-            last_value_func is max
-        ):  # Query ordered and filtered according to last_value function
-            filter_op = operator.ge
-            filter_op_end = operator.lt
-        elif last_value_func is min:
-            filter_op = operator.le
-            filter_op_end = operator.gt
-        else:  # Custom last_value, load everything and let incremental handle filtering
-            return query
-
-        if self.last_value is not None:
-            query = query.where(filter_op(self.cursor_column, self.last_value))
-            if self.end_value is not None:
-                query = query.where(filter_op_end(self.cursor_column, self.end_value))
-
-        # generate order by from declared row order
-        order_by = None
-        if (self.row_order == "asc" and last_value_func is max) or (
-            self.row_order == "desc" and last_value_func is min
-        ):
-            order_by = self.cursor_column.asc()
-        elif (self.row_order == "asc" and last_value_func is min) or (
-            self.row_order == "desc" and last_value_func is max
-        ):
-            order_by = self.cursor_column.desc()
-        if order_by is not None:
-            query = query.order_by(order_by)
-
-        return query
-
-    def load_rows(self, backend_kwargs: Dict[str, Any] = None) -> Iterator[TDataItem]:
-        # make copy of kwargs
-        backend_kwargs = dict(backend_kwargs or {})
-        query = self.make_query()
-        if self.backend == "connectorx":
-            yield from self._load_rows_connectorx(query, backend_kwargs)
-        else:
-            yield from self._load_rows(query, backend_kwargs)
-
-    def _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:
-        with self.engine.connect() as conn:
-            result = conn.execution_options(yield_per=self.chunk_size).execute(query)
-            # NOTE: cursor returns not normalized column names! may be quite useful in case of Oracle dialect
-            # that normalizes columns
-            # columns = [c[0] for c in result.cursor.description]
-            columns = list(result.keys())
-            for partition in result.partitions(size=self.chunk_size):
-                if self.backend == "sqlalchemy":
-                    yield [dict(row._mapping) for row in partition]
-                elif self.backend == "pandas":
-                    from dlt.common.libs.pandas_sql import _wrap_result
-
-                    df = _wrap_result(
-                        partition,
-                        columns,
-                        **{"dtype_backend": "pyarrow", **backend_kwargs},
-                    )
-                    yield df
-                elif self.backend == "pyarrow":
-                    yield row_tuples_to_arrow(
-                        partition, self.columns, tz=backend_kwargs.get("tz", "UTC")
-                    )
-
-    def _load_rows_connectorx(
-        self, query: SelectAny, backend_kwargs: Dict[str, Any]
-    ) -> Iterator[TDataItem]:
-        try:
-            import connectorx as cx  # type: ignore
-        except ImportError:
-            raise MissingDependencyException(
-                "Connector X table backend", ["connectorx"]
-            )
-
-        # default settings
-        backend_kwargs = {
-            "return_type": "arrow2",
-            "protocol": "binary",
-            **backend_kwargs,
-        }
-        conn = backend_kwargs.pop(
-            "conn",
-            self.engine.url._replace(
-                drivername=self.engine.url.get_backend_name()
-            ).render_as_string(hide_password=False),
-        )
-        try:
-            query_str = str(
-                query.compile(self.engine, compile_kwargs={"literal_binds": True})
-            )
-        except CompileError as ex:
-            raise NotImplementedError(
-                f"Query for table {self.table.name} could not be compiled to string to execute it on ConnectorX. If you are on SQLAlchemy 1.4.x the causing exception is due to literals that cannot be rendered, upgrade to 2.x: {str(ex)}"
-            ) from ex
-        df = cx.read_sql(conn, query_str, **backend_kwargs)
-        yield df
-
-
-def table_rows(
-    engine: Engine,
-    table: Table,
-    chunk_size: int,
-    backend: TableBackend,
-    incremental: Optional[dlt.sources.incremental[Any]] = None,
-    defer_table_reflect: bool = False,
-    table_adapter_callback: Callable[[Table], None] = None,
-    reflection_level: ReflectionLevel = "minimal",
-    backend_kwargs: Dict[str, Any] = None,
-    type_adapter_callback: Optional[TTypeAdapter] = None,
-) -> Iterator[TDataItem]:
-    columns: TTableSchemaColumns = None
-    if defer_table_reflect:
-        table = Table(
-            table.name, table.metadata, autoload_with=engine, extend_existing=True
-        )
-        if table_adapter_callback:
-            table_adapter_callback(table)
-        columns = table_to_columns(table, reflection_level, type_adapter_callback)
-
-        # set the primary_key in the incremental
-        if incremental and incremental.primary_key is None:
-            primary_key = get_primary_key(table)
-            if primary_key is not None:
-                incremental.primary_key = primary_key
-        # yield empty record to set hints
-        yield dlt.mark.with_hints(
-            [],
-            dlt.mark.make_hints(
-                primary_key=get_primary_key(table),
-                columns=columns,
-            ),
-        )
-    else:
-        # table was already reflected
-        columns = table_to_columns(table, reflection_level, type_adapter_callback)
-
-    loader = TableLoader(
-        engine, backend, table, columns, incremental=incremental, chunk_size=chunk_size
-    )
-    yield from loader.load_rows(backend_kwargs)
-
-
-def engine_from_credentials(
-    credentials: Union[ConnectionStringCredentials, Engine, str], **backend_kwargs: Any
-) -> Engine:
-    if isinstance(credentials, Engine):
-        return credentials
-    if isinstance(credentials, ConnectionStringCredentials):
-        credentials = credentials.to_native_representation()
-    return create_engine(credentials, **backend_kwargs)
-
-
-def unwrap_json_connector_x(field: str) -> TDataItem:
-    """Creates a transform function to be added with `add_map` that will unwrap JSON columns
-    ingested via connectorx. Such columns are additionally quoted and translate SQL NULL to json "null"
-    """
-    import pyarrow as pa
-    import pyarrow.compute as pc
-
-    def _unwrap(table: TDataItem) -> TDataItem:
-        col_index = table.column_names.index(field)
-        # remove quotes
-        column = pc.replace_substring_regex(table[field], '"(.*)"', "\\1")
-        # convert json null to null
-        column = pc.replace_with_mask(
-            column,
-            pc.equal(column, "null").combine_chunks(),
-            pa.scalar(None, pa.large_string()),
-        )
-        return table.set_column(col_index, table.schema.field(col_index), column)
-
-    return _unwrap
-
-
-def _detect_precision_hints_deprecated(value: Optional[bool]) -> None:
-    if value is None:
-        return
-
-    msg = "`detect_precision_hints` argument is deprecated and will be removed in a future release. "
-    if value:
-        msg += "Use `reflection_level='full_with_precision'` which has the same effect instead."
-
-    warnings.warn(
-        msg,
-        DeprecationWarning,
-    )
-
-
-@configspec
-class SqlDatabaseTableConfiguration(BaseConfiguration):
-    incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]
-
-
-@configspec
-class SqlTableResourceConfiguration(BaseConfiguration):
-    credentials: Union[ConnectionStringCredentials, Engine, str] = None
-    table: str = None
-    schema: Optional[str] = None
-    incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]
-    chunk_size: int = 50000
-    backend: TableBackend = "sqlalchemy"
-    detect_precision_hints: Optional[bool] = None
-    defer_table_reflect: Optional[bool] = False
-    reflection_level: Optional[ReflectionLevel] = "full"

ingestr/src/sql_database/override.py

@@ -1,10 +0,0 @@
-from typing import Optional
-
-from dlt.common.configuration.specs.base_configuration import configspec
-from dlt.sources.credentials import ConnectionStringCredentials
-
-
-@configspec(init=False)
-class IngestrConnectionStringCredentials(ConnectionStringCredentials):
-    username: Optional[str] = None
-    database: Optional[str] = None