ingestr 0.6.6__py3-none-any.whl → 0.7.1__py3-none-any.whl

ingestr/src/sql_database/helpers.py

@@ -1,12 +1,12 @@
 """SQL database source helpers"""
 
 import operator
+import warnings
 from typing import (
     Any,
     Callable,
     Dict,
     Iterator,
-    List,
     Literal,
     Optional,
     Union,
@@ -16,16 +16,19 @@ import dlt
 from dlt.common.configuration.specs import BaseConfiguration, configspec
 from dlt.common.exceptions import MissingDependencyException
 from dlt.common.schema import TTableSchemaColumns
-from dlt.common.typing import TDataItem
-from dlt.sources.credentials import ConnectionStringCredentials
-from sqlalchemy import Table, create_engine
+from dlt.common.typing import TDataItem, TSortOrder
+from sqlalchemy import create_engine
 from sqlalchemy.engine import Engine
+from sqlalchemy.exc import CompileError
 
-from ingestr.src.sql_database.override import IngestrConnectionStringCredentials
-
+from .arrow_helpers import row_tuples_to_arrow
+from .override import IngestrConnectionStringCredentials as ConnectionStringCredentials
 from .schema_types import (
+    ReflectionLevel,
     SelectAny,
-    row_tuples_to_arrow,
+    Table,
+    TTypeAdapter,
+    get_primary_key,
     table_to_columns,
 )
 
@@ -57,7 +60,7 @@ class TableLoader:
                 ) from e
             self.last_value = incremental.last_value
             self.end_value = incremental.end_value
-            self.row_order = getattr(self.incremental, "row_order", None)
+            self.row_order: TSortOrder = self.incremental.row_order
         else:
             self.cursor_column = None
             self.last_value = None
@@ -69,6 +72,7 @@ class TableLoader:
         query = table.select()
         if not self.incremental:
             return query
+
         last_value_func = self.incremental.last_value_func
 
         # generate where
@@ -90,9 +94,13 @@ class TableLoader:
 
         # generate order by from declared row order
         order_by = None
-        if self.row_order == "asc":
+        if (self.row_order == "asc" and last_value_func is max) or (
+            self.row_order == "desc" and last_value_func is min
+        ):
             order_by = self.cursor_column.asc()
-        elif self.row_order == "desc":
+        elif (self.row_order == "asc" and last_value_func is min) or (
+            self.row_order == "desc" and last_value_func is max
+        ):
             order_by = self.cursor_column.desc()
         if order_by is not None:
             query = query.order_by(order_by)
@@ -121,14 +129,15 @@ class TableLoader:
                 elif self.backend == "pandas":
                     from dlt.common.libs.pandas_sql import _wrap_result
 
-                    yield _wrap_result(
+                    df = _wrap_result(
                         partition,
                         columns,
                         **{"dtype_backend": "pyarrow", **backend_kwargs},
                     )
+                    yield df
                 elif self.backend == "pyarrow":
                     yield row_tuples_to_arrow(
-                        partition, self.columns, tz=backend_kwargs.get("tz")
+                        partition, self.columns, tz=backend_kwargs.get("tz", "UTC")
                     )
 
     def _load_rows_connectorx(
@@ -153,11 +162,15 @@ class TableLoader:
                 drivername=self.engine.url.get_backend_name()
             ).render_as_string(hide_password=False),
         )
-        df = cx.read_sql(
-            conn,
-            str(query.compile(self.engine, compile_kwargs={"literal_binds": True})),
-            **backend_kwargs,
-        )
+        try:
+            query_str = str(
+                query.compile(self.engine, compile_kwargs={"literal_binds": True})
+            )
+        except CompileError as ex:
+            raise NotImplementedError(
+                f"Query for table {self.table.name} could not be compiled to string to execute it on ConnectorX. If you are on SQLAlchemy 1.4.x the causing exception is due to literals that cannot be rendered, upgrade to 2.x: {str(ex)}"
+            ) from ex
+        df = cx.read_sql(conn, query_str, **backend_kwargs)
         yield df
 
 
@@ -167,10 +180,11 @@ def table_rows(
     chunk_size: int,
     backend: TableBackend,
     incremental: Optional[dlt.sources.incremental[Any]] = None,
-    detect_precision_hints: bool = False,
     defer_table_reflect: bool = False,
     table_adapter_callback: Callable[[Table], None] = None,
+    reflection_level: ReflectionLevel = "minimal",
     backend_kwargs: Dict[str, Any] = None,
+    type_adapter_callback: Optional[TTypeAdapter] = None,
 ) -> Iterator[TDataItem]:
     columns: TTableSchemaColumns = None
     if defer_table_reflect:
@@ -179,7 +193,7 @@ def table_rows(
         )
         if table_adapter_callback:
             table_adapter_callback(table)
-        columns = table_to_columns(table, detect_precision_hints)
+        columns = table_to_columns(table, reflection_level, type_adapter_callback)
 
         # set the primary_key in the incremental
         if incremental and incremental.primary_key is None:
@@ -196,7 +210,7 @@ def table_rows(
         )
     else:
         # table was already reflected
-        columns = table_to_columns(table, detect_precision_hints)
+        columns = table_to_columns(table, reflection_level, type_adapter_callback)
 
     loader = TableLoader(
         engine, backend, table, columns, incremental=incremental, chunk_size=chunk_size
@@ -205,19 +219,13 @@ def table_rows(
 
 
 def engine_from_credentials(
-    credentials: Union[ConnectionStringCredentials, Engine, str],
+    credentials: Union[ConnectionStringCredentials, Engine, str], **backend_kwargs: Any
 ) -> Engine:
     if isinstance(credentials, Engine):
         return credentials
     if isinstance(credentials, ConnectionStringCredentials):
         credentials = credentials.to_native_representation()
-    return create_engine(credentials)
-
-
-def get_primary_key(table: Table) -> List[str]:
-    """Create primary key or return None if no key defined"""
-    primary_key = [c.name for c in table.primary_key]
-    return primary_key if len(primary_key) > 0 else None
+    return create_engine(credentials, **backend_kwargs)
 
 
 def unwrap_json_connector_x(field: str) -> TDataItem:
@@ -242,6 +250,20 @@ def unwrap_json_connector_x(field: str) -> TDataItem:
     return _unwrap
 
 
+def _detect_precision_hints_deprecated(value: Optional[bool]) -> None:
+    if value is None:
+        return
+
+    msg = "`detect_precision_hints` argument is deprecated and will be removed in a future release. "
+    if value:
+        msg += "Use `reflection_level='full_with_precision'` which has the same effect instead."
+
+    warnings.warn(
+        msg,
+        DeprecationWarning,
+    )
+
+
 @configspec
 class SqlDatabaseTableConfiguration(BaseConfiguration):
     incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]
@@ -249,10 +271,12 @@ class SqlDatabaseTableConfiguration(BaseConfiguration):
 
 @configspec
 class SqlTableResourceConfiguration(BaseConfiguration):
-    credentials: IngestrConnectionStringCredentials = None
+    credentials: Union[ConnectionStringCredentials, Engine, str] = None
     table: str = None
-    incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]
     schema: Optional[str] = None
-
-
-__source_name__ = "sql_database"
+    incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]
+    chunk_size: int = 50000
+    backend: TableBackend = "sqlalchemy"
+    detect_precision_hints: Optional[bool] = None
+    defer_table_reflect: Optional[bool] = False
+    reflection_level: Optional[ReflectionLevel] = "full"

ingestr/src/sql_database/schema_types.py

@@ -1,39 +1,73 @@
-from typing import TYPE_CHECKING, Any, Optional, Sequence, Type
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    List,
+    Literal,
+    Optional,
+    Type,
+    Union,
+)
 
 from dlt.common import logger
-from dlt.common.configuration import with_config
-from dlt.common.destination import DestinationCapabilitiesContext
 from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns
 from sqlalchemy import Column, Table
 from sqlalchemy.engine import Row
 from sqlalchemy.sql import Select, sqltypes
+from sqlalchemy.sql.sqltypes import TypeEngine
 from typing_extensions import TypeAlias
 
+ReflectionLevel = Literal["minimal", "full", "full_with_precision"]
+
+
 # optionally create generics with any so they can be imported by dlt importer
 if TYPE_CHECKING:
     SelectAny: TypeAlias = Select[Any]
     ColumnAny: TypeAlias = Column[Any]
     RowAny: TypeAlias = Row[Any]
+    TypeEngineAny = TypeEngine[Any]
 else:
     SelectAny: TypeAlias = Type[Any]
     ColumnAny: TypeAlias = Type[Any]
     RowAny: TypeAlias = Type[Any]
+    TypeEngineAny = Type[Any]
+
+
+TTypeAdapter = Callable[
+    [TypeEngineAny], Optional[Union[TypeEngineAny, Type[TypeEngineAny]]]
+]
 
 
 def sqla_col_to_column_schema(
-    sql_col: ColumnAny, add_precision: bool = False
+    sql_col: ColumnAny,
+    reflection_level: ReflectionLevel,
+    type_adapter_callback: Optional[TTypeAdapter] = None,
 ) -> Optional[TColumnSchema]:
     """Infer dlt schema column type from an sqlalchemy type.
 
     If `add_precision` is set, precision and scale is inferred from that types that support it,
     such as numeric, varchar, int, bigint. Numeric (decimal) types have always precision added.
     """
-    sql_t = sql_col.type
     col: TColumnSchema = {
         "name": sql_col.name,
-        "data_type": None,  # set that later
         "nullable": sql_col.nullable,
     }
+    if reflection_level == "minimal":
+        return col
+
+    sql_t = sql_col.type
+
+    if type_adapter_callback:
+        sql_t = type_adapter_callback(sql_t)  # type: ignore[assignment]
+        # Check if sqla type class rather than instance is returned
+        if sql_t is not None and isinstance(sql_t, type):
+            sql_t = sql_t()
+
+    if sql_t is None:
+        # Column ignored by callback
+        return col
+
+    add_precision = reflection_level == "full_with_precision"
 
     if isinstance(sql_t, sqltypes.SmallInteger):
         col["data_type"] = "bigint"
@@ -77,86 +111,29 @@ def sqla_col_to_column_schema(
         col["data_type"] = "bool"
     else:
         logger.warning(
-            f"A column with name {sql_col.name} contains unknown data type {sql_t} which cannot be mapped to `dlt` data type. When using sqlalchemy backend such data will be passed to the normalizer. In case of `pyarrow` backend such data will be ignored. In case of other backends, the behavior is backend-specific."
+            f"A column with name {sql_col.name} contains unknown data type {sql_t} which cannot be mapped to `dlt` data type. When using sqlalchemy backend such data will be passed to the normalizer. In case of `pyarrow` and `pandas` backend, data types are detected from numpy ndarrays. In case of other backends, the behavior is backend-specific."
         )
-        col = None
-    if col:
-        return {key: value for key, value in col.items() if value is not None}  # type: ignore[return-value]
-    return None
 
+    return {key: value for key, value in col.items() if value is not None}  # type: ignore[return-value]
 
-def table_to_columns(table: Table, add_precision: bool = False) -> TTableSchemaColumns:
-    """Convert an sqlalchemy table to a dlt table schema.
 
-    Adds precision to columns when `add_precision` is set.
-    """
-    return {
-        col["name"]: col
-        for col in (sqla_col_to_column_schema(c, add_precision) for c in table.columns)
-        if col is not None
-    }
+def get_primary_key(table: Table) -> Optional[List[str]]:
+    """Create primary key or return None if no key defined"""
+    primary_key = [c.name for c in table.primary_key]
+    return primary_key if len(primary_key) > 0 else None
 
 
-@with_config
-def columns_to_arrow(
-    columns_schema: TTableSchemaColumns,
-    caps: DestinationCapabilitiesContext = None,
-    tz: str = "UTC",
-) -> Any:
-    """Converts `column_schema` to arrow schema using `caps` and `tz`. `caps` are injected from the container - which
-    is always the case if run within the pipeline. This will generate arrow schema compatible with the destination.
-    Otherwise generic capabilities are used
-    """
-    from dlt.common.destination.capabilities import DestinationCapabilitiesContext
-    from dlt.common.libs.pyarrow import get_py_arrow_datatype
-    from dlt.common.libs.pyarrow import pyarrow as pa
-
-    return pa.schema(
-        [
-            pa.field(
-                name,
-                get_py_arrow_datatype(
-                    schema_item,
-                    caps or DestinationCapabilitiesContext.generic_capabilities(),
-                    tz,
-                ),
-                nullable=schema_item.get("nullable", True),
-            )
-            for name, schema_item in columns_schema.items()
-        ]
-    )
-
-
-def row_tuples_to_arrow(
-    rows: Sequence[RowAny], columns: TTableSchemaColumns, tz: str
-) -> Any:
-    import numpy as np
-    from dlt.common.libs.pyarrow import pyarrow as pa
-
-    arrow_schema = columns_to_arrow(columns, tz=tz)
-
-    try:
-        from pandas._libs import lib
-
-        pivoted_rows = lib.to_object_array_tuples(rows).T  # type: ignore[attr-defined]
-    except ImportError:
-        logger.info(
-            "Pandas not installed, reverting to numpy.asarray to create a table which is slower"
+def table_to_columns(
+    table: Table,
+    reflection_level: ReflectionLevel = "full",
+    type_conversion_fallback: Optional[TTypeAdapter] = None,
+) -> TTableSchemaColumns:
+    """Convert an sqlalchemy table to a dlt table schema."""
+    return {
+        col["name"]: col
+        for col in (
+            sqla_col_to_column_schema(c, reflection_level, type_conversion_fallback)
+            for c in table.columns
         )
-        pivoted_rows = np.asarray(rows, dtype="object", order="k").T  # type: ignore[call-overload]
-
-    columnar = {
-        col: dat.ravel()
-        for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns)))
+        if col is not None
     }
-    for idx in range(0, len(arrow_schema.names)):
-        field = arrow_schema.field(idx)
-        py_type = type(rows[0][idx])
-        # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects
-        if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)):
-            logger.warning(
-                f"Field {field.name} was reflected as decimal type, but rows contains {py_type.__name__}. Additional cast is required which may slow down arrow table generation."
-            )
-            float_array = pa.array(columnar[field.name], type=pa.float64())
-            columnar[field.name] = float_array.cast(field.type, safe=False)
-    return pa.Table.from_pydict(columnar, schema=arrow_schema)

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.6.6"
+__version__ = "0.7.1"

{ingestr-0.6.6.dist-info → ingestr-0.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.6
+Version: 0.7.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,9 +14,10 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
+Requires-Dist: connectorx==0.3.3
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==0.4.12
+Requires-Dist: dlt==0.5.1
 Requires-Dist: duckdb-engine==0.11.5
 Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-api-python-client==2.130.0

{ingestr-0.6.6.dist-info → ingestr-0.7.1.dist-info}/RECORD

@@ -1,13 +1,9 @@
-ingestr/main.py,sha256=PvZtqrlHO3aSFtdhIdLAhzsOLG1hP_ql24vof76SFaI,14862
-ingestr/main_test.py,sha256=MDV2Eo86W_CcxGgEkYYoBc6xIXjVMER4hMhgAdxXYMc,28464
+ingestr/main.py,sha256=8el31KbINLPqe_iwkLHAFoGPIX-j5gB2KqV2nuEXU1g,16544
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
-ingestr/src/factory.py,sha256=w6xb8scWa_uG4otoMjqvqcx99hofrN2YcLzaB7QSgYo,3545
-ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
-ingestr/src/sources.py,sha256=sLT8CDiLz6aTekm6QZCXatZAdTyzCabqjJesnEe88GU,10061
-ingestr/src/sources_test.py,sha256=wZopz4tFtFnOiCEO8pZW816Nj86pLlR5fEqfq0TtWMQ,3629
+ingestr/src/factory.py,sha256=XuT_8LvWd7gBxOjoD_NiG-jtPvHNQ9nqOeoCJzhRb6Y,3630
+ingestr/src/sources.py,sha256=QbSvECvGbHJKOpE9_dbq11343pA5ajsS9BPPPab1ivw,10007
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=I3h5MyD10PkOUQEBnR6L9ja7s4WeTEg8rRjRKTCWYWQ,22
+ingestr/src/version.py,sha256=2KJZDSMOG7KS82AxYOrZ4ZihYxX0wjfUjDsIZh3L024,22
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
@@ -15,7 +11,6 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
 ingestr/src/gorgias/__init__.py,sha256=BzX9X1Yc_1Mch6NP1pn26hjRIiaadErgHxkdJHw4P3o,21227
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/gorgias/helpers_test.py,sha256=kSR2nhB8U8HZ8pgDnd7HvXlzojmBnpOm8fTKHJvvKGY,1580
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
 ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
@@ -27,10 +22,11 @@ ingestr/src/shopify/__init__.py,sha256=EWjpvZz7K6Pms7uUoqqkM4Wj0XeE2NrDvVp4BNM8d
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
 ingestr/src/shopify/helpers.py,sha256=OO_Tw-HwVLnRhwT3vqUWEQEEcWIS9KWE6VDDe8BCC2w,4972
 ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
-ingestr/src/sql_database/__init__.py,sha256=S5MVJr8juPSs61C2D7pInsTwNEHetChK6RjjhPAD0Lg,8845
-ingestr/src/sql_database/helpers.py,sha256=tbn-GjjBIVu3hVVh5vrUSiZqQ32_Tp0oBP5Fvv_wY4E,8986
+ingestr/src/sql_database/__init__.py,sha256=HEqY6U-YzzbeZ8avIthj-Fatm2C3i3jqYs5DAIAu4Ss,11511
+ingestr/src/sql_database/arrow_helpers.py,sha256=yze1X3A9nUQA4HeuFDDWrfJVkCq8Uo5UyDo_zhJtI60,5699
+ingestr/src/sql_database/helpers.py,sha256=6o8e2_8MIuj3qlo40a2E6ns3gyK18ei1jCePONrMUjI,10191
 ingestr/src/sql_database/override.py,sha256=xbKGDztCzvrhJ5kJTXERal3LA56bEeVug4_rrTs8DgA,333
-ingestr/src/sql_database/schema_types.py,sha256=foGHh4iGagGLfS7nF3uGYhBjqgX0jlrjj0XYE1T3nSs,6592
+ingestr/src/sql_database/schema_types.py,sha256=qXTanvFPE8wMCSDzQWPDi5yqaO-llfrFXjiGJALI4NA,5013
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
@@ -41,8 +37,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.6.6.dist-info/METADATA,sha256=o9eKbLqKlQ-TkYoaLHFLeQAjuraP9UZhM1jsCL2v7Gg,5830
-ingestr-0.6.6.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.6.6.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.6.6.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.6.6.dist-info/RECORD,,
+ingestr-0.7.1.dist-info/METADATA,sha256=c5iTfQqmmQ0MHghgngJILHO8n5GuUq49OFe0TI8HG7Y,5862
+ingestr-0.7.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.7.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.7.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.7.1.dist-info/RECORD,,