ingestr 0.6.6__py3-none-any.whl → 0.7.0__py3-none-any.whl

ingestr/src/sql_database/schema_types.py

@@ -1,39 +1,73 @@
-from typing import TYPE_CHECKING, Any, Optional, Sequence, Type
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    List,
+    Literal,
+    Optional,
+    Type,
+    Union,
+)
 
 from dlt.common import logger
-from dlt.common.configuration import with_config
-from dlt.common.destination import DestinationCapabilitiesContext
 from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns
 from sqlalchemy import Column, Table
 from sqlalchemy.engine import Row
 from sqlalchemy.sql import Select, sqltypes
+from sqlalchemy.sql.sqltypes import TypeEngine
 from typing_extensions import TypeAlias
 
+ReflectionLevel = Literal["minimal", "full", "full_with_precision"]
+
+
 # optionally create generics with any so they can be imported by dlt importer
 if TYPE_CHECKING:
     SelectAny: TypeAlias = Select[Any]
     ColumnAny: TypeAlias = Column[Any]
     RowAny: TypeAlias = Row[Any]
+    TypeEngineAny = TypeEngine[Any]
 else:
     SelectAny: TypeAlias = Type[Any]
     ColumnAny: TypeAlias = Type[Any]
     RowAny: TypeAlias = Type[Any]
+    TypeEngineAny = Type[Any]
+
+
+TTypeAdapter = Callable[
+    [TypeEngineAny], Optional[Union[TypeEngineAny, Type[TypeEngineAny]]]
+]
 
 
 def sqla_col_to_column_schema(
-    sql_col: ColumnAny, add_precision: bool = False
+    sql_col: ColumnAny,
+    reflection_level: ReflectionLevel,
+    type_adapter_callback: Optional[TTypeAdapter] = None,
 ) -> Optional[TColumnSchema]:
     """Infer dlt schema column type from an sqlalchemy type.
 
     If `add_precision` is set, precision and scale is inferred from that types that support it,
     such as numeric, varchar, int, bigint. Numeric (decimal) types have always precision added.
     """
-    sql_t = sql_col.type
     col: TColumnSchema = {
         "name": sql_col.name,
-        "data_type": None,  # set that later
         "nullable": sql_col.nullable,
     }
+    if reflection_level == "minimal":
+        return col
+
+    sql_t = sql_col.type
+
+    if type_adapter_callback:
+        sql_t = type_adapter_callback(sql_t)  # type: ignore[assignment]
+        # Check if sqla type class rather than instance is returned
+        if sql_t is not None and isinstance(sql_t, type):
+            sql_t = sql_t()
+
+    if sql_t is None:
+        # Column ignored by callback
+        return col
+
+    add_precision = reflection_level == "full_with_precision"
 
     if isinstance(sql_t, sqltypes.SmallInteger):
         col["data_type"] = "bigint"
@@ -77,86 +111,29 @@ def sqla_col_to_column_schema(
         col["data_type"] = "bool"
     else:
         logger.warning(
-            f"A column with name {sql_col.name} contains unknown data type {sql_t} which cannot be mapped to `dlt` data type. When using sqlalchemy backend such data will be passed to the normalizer. In case of `pyarrow` backend such data will be ignored. In case of other backends, the behavior is backend-specific."
+            f"A column with name {sql_col.name} contains unknown data type {sql_t} which cannot be mapped to `dlt` data type. When using sqlalchemy backend such data will be passed to the normalizer. In case of `pyarrow` and `pandas` backend, data types are detected from numpy ndarrays. In case of other backends, the behavior is backend-specific."
         )
-        col = None
-    if col:
-        return {key: value for key, value in col.items() if value is not None}  # type: ignore[return-value]
-    return None
 
+    return {key: value for key, value in col.items() if value is not None}  # type: ignore[return-value]
 
-def table_to_columns(table: Table, add_precision: bool = False) -> TTableSchemaColumns:
-    """Convert an sqlalchemy table to a dlt table schema.
 
-    Adds precision to columns when `add_precision` is set.
-    """
-    return {
-        col["name"]: col
-        for col in (sqla_col_to_column_schema(c, add_precision) for c in table.columns)
-        if col is not None
-    }
+def get_primary_key(table: Table) -> Optional[List[str]]:
+    """Create primary key or return None if no key defined"""
+    primary_key = [c.name for c in table.primary_key]
+    return primary_key if len(primary_key) > 0 else None
 
 
-@with_config
-def columns_to_arrow(
-    columns_schema: TTableSchemaColumns,
-    caps: DestinationCapabilitiesContext = None,
-    tz: str = "UTC",
-) -> Any:
-    """Converts `column_schema` to arrow schema using `caps` and `tz`. `caps` are injected from the container - which
-    is always the case if run within the pipeline. This will generate arrow schema compatible with the destination.
-    Otherwise generic capabilities are used
-    """
-    from dlt.common.destination.capabilities import DestinationCapabilitiesContext
-    from dlt.common.libs.pyarrow import get_py_arrow_datatype
-    from dlt.common.libs.pyarrow import pyarrow as pa
-
-    return pa.schema(
-        [
-            pa.field(
-                name,
-                get_py_arrow_datatype(
-                    schema_item,
-                    caps or DestinationCapabilitiesContext.generic_capabilities(),
-                    tz,
-                ),
-                nullable=schema_item.get("nullable", True),
-            )
-            for name, schema_item in columns_schema.items()
-        ]
-    )
-
-
-def row_tuples_to_arrow(
-    rows: Sequence[RowAny], columns: TTableSchemaColumns, tz: str
-) -> Any:
-    import numpy as np
-    from dlt.common.libs.pyarrow import pyarrow as pa
-
-    arrow_schema = columns_to_arrow(columns, tz=tz)
-
-    try:
-        from pandas._libs import lib
-
-        pivoted_rows = lib.to_object_array_tuples(rows).T  # type: ignore[attr-defined]
-    except ImportError:
-        logger.info(
-            "Pandas not installed, reverting to numpy.asarray to create a table which is slower"
+def table_to_columns(
+    table: Table,
+    reflection_level: ReflectionLevel = "full",
+    type_conversion_fallback: Optional[TTypeAdapter] = None,
+) -> TTableSchemaColumns:
+    """Convert an sqlalchemy table to a dlt table schema."""
+    return {
+        col["name"]: col
+        for col in (
+            sqla_col_to_column_schema(c, reflection_level, type_conversion_fallback)
+            for c in table.columns
         )
-        pivoted_rows = np.asarray(rows, dtype="object", order="k").T  # type: ignore[call-overload]
-
-    columnar = {
-        col: dat.ravel()
-        for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns)))
+        if col is not None
     }
-    for idx in range(0, len(arrow_schema.names)):
-        field = arrow_schema.field(idx)
-        py_type = type(rows[0][idx])
-        # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects
-        if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)):
-            logger.warning(
-                f"Field {field.name} was reflected as decimal type, but rows contains {py_type.__name__}. Additional cast is required which may slow down arrow table generation."
-            )
-            float_array = pa.array(columnar[field.name], type=pa.float64())
-            columnar[field.name] = float_array.cast(field.type, safe=False)
-    return pa.Table.from_pydict(columnar, schema=arrow_schema)

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.6.6"
+__version__ = "0.7.0"

{ingestr-0.6.6.dist-info → ingestr-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.6
+Version: 0.7.0
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,7 +16,7 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==0.4.12
+Requires-Dist: dlt==0.5.1
 Requires-Dist: duckdb-engine==0.11.5
 Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-api-python-client==2.130.0

{ingestr-0.6.6.dist-info → ingestr-0.7.0.dist-info}/RECORD

@@ -1,13 +1,9 @@
-ingestr/main.py,sha256=PvZtqrlHO3aSFtdhIdLAhzsOLG1hP_ql24vof76SFaI,14862
-ingestr/main_test.py,sha256=MDV2Eo86W_CcxGgEkYYoBc6xIXjVMER4hMhgAdxXYMc,28464
+ingestr/main.py,sha256=JYgh3rZSO9n_Ko2D_9BLaF_cGQHrLSywK1WH6XioefQ,15961
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
-ingestr/src/factory.py,sha256=w6xb8scWa_uG4otoMjqvqcx99hofrN2YcLzaB7QSgYo,3545
-ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
-ingestr/src/sources.py,sha256=sLT8CDiLz6aTekm6QZCXatZAdTyzCabqjJesnEe88GU,10061
-ingestr/src/sources_test.py,sha256=wZopz4tFtFnOiCEO8pZW816Nj86pLlR5fEqfq0TtWMQ,3629
+ingestr/src/factory.py,sha256=XuT_8LvWd7gBxOjoD_NiG-jtPvHNQ9nqOeoCJzhRb6Y,3630
+ingestr/src/sources.py,sha256=QbSvECvGbHJKOpE9_dbq11343pA5ajsS9BPPPab1ivw,10007
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=I3h5MyD10PkOUQEBnR6L9ja7s4WeTEg8rRjRKTCWYWQ,22
+ingestr/src/version.py,sha256=RaANGbRu5e-vehwXI1-Qe2ggPPfs1TQaZj072JdbLk4,22
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
@@ -15,7 +11,6 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
 ingestr/src/gorgias/__init__.py,sha256=BzX9X1Yc_1Mch6NP1pn26hjRIiaadErgHxkdJHw4P3o,21227
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/gorgias/helpers_test.py,sha256=kSR2nhB8U8HZ8pgDnd7HvXlzojmBnpOm8fTKHJvvKGY,1580
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
 ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
@@ -27,10 +22,11 @@ ingestr/src/shopify/__init__.py,sha256=EWjpvZz7K6Pms7uUoqqkM4Wj0XeE2NrDvVp4BNM8d
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
 ingestr/src/shopify/helpers.py,sha256=OO_Tw-HwVLnRhwT3vqUWEQEEcWIS9KWE6VDDe8BCC2w,4972
 ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
-ingestr/src/sql_database/__init__.py,sha256=S5MVJr8juPSs61C2D7pInsTwNEHetChK6RjjhPAD0Lg,8845
-ingestr/src/sql_database/helpers.py,sha256=tbn-GjjBIVu3hVVh5vrUSiZqQ32_Tp0oBP5Fvv_wY4E,8986
+ingestr/src/sql_database/__init__.py,sha256=HEqY6U-YzzbeZ8avIthj-Fatm2C3i3jqYs5DAIAu4Ss,11511
+ingestr/src/sql_database/arrow_helpers.py,sha256=yze1X3A9nUQA4HeuFDDWrfJVkCq8Uo5UyDo_zhJtI60,5699
+ingestr/src/sql_database/helpers.py,sha256=6o8e2_8MIuj3qlo40a2E6ns3gyK18ei1jCePONrMUjI,10191
 ingestr/src/sql_database/override.py,sha256=xbKGDztCzvrhJ5kJTXERal3LA56bEeVug4_rrTs8DgA,333
-ingestr/src/sql_database/schema_types.py,sha256=foGHh4iGagGLfS7nF3uGYhBjqgX0jlrjj0XYE1T3nSs,6592
+ingestr/src/sql_database/schema_types.py,sha256=qXTanvFPE8wMCSDzQWPDi5yqaO-llfrFXjiGJALI4NA,5013
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
@@ -41,8 +37,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.6.6.dist-info/METADATA,sha256=o9eKbLqKlQ-TkYoaLHFLeQAjuraP9UZhM1jsCL2v7Gg,5830
-ingestr-0.6.6.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.6.6.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.6.6.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.6.6.dist-info/RECORD,,
+ingestr-0.7.0.dist-info/METADATA,sha256=MmQ_futv2ZZbVg4hdbCErSAgs2AobplIUr4vMErTXEI,5829
+ingestr-0.7.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.7.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.7.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.7.0.dist-info/RECORD,,