ingestr 0.9.4__py3-none-any.whl → 0.10.0rc0__py3-none-any.whl

ingestr/src/sources.py

@@ -10,6 +10,8 @@ import pendulum
 from dlt.common.configuration.specs import AwsCredentials
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TSecretStrValue
+from dlt.sources.credentials import ConnectionStringCredentials
+from dlt.sources.sql_database import sql_table
 
 from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
@@ -29,7 +31,6 @@ from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.shopify import shopify_source
 from ingestr.src.slack import slack_source
-from ingestr.src.sql_database import sql_table
 from ingestr.src.stripe_analytics import stripe_source
 from ingestr.src.table_definition import table_string_to_dataclass
 from ingestr.src.zendesk import zendesk_chat, zendesk_support, zendesk_talk
@@ -67,11 +68,10 @@ class SqlSource:
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
         table_instance = self.table_builder(
-            credentials=uri,
+            credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
             table=table_fields.table,
             incremental=incremental,
-            merge_key=kwargs.get("merge_key"),
             backend=kwargs.get("sql_backend", "sqlalchemy"),
             chunk_size=kwargs.get("page_size", None),
         )
@@ -759,7 +759,7 @@ class AdjustSource:
         metrics = None
         filters = []
         if table.startswith("custom:"):
-            fields = table.split(":")
+            fields = table.split(":", 3)
             if len(fields) != 3 and len(fields) != 4:
                 raise ValueError(
                     "Invalid Adjust custom table format. Expected format: custom:<dimensions>,<metrics> or custom:<dimensions>:<metrics>:<filters>"

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.9.4"
+__version__ = "0.10.0rc0"

ingestr/src/zendesk/__init__.py

@@ -280,8 +280,8 @@ def zendesk_support(
         primary_key="id",
         write_disposition="merge",
         columns={
-            "tags": {"data_type": "complex"},
-            "custom_fields": {"data_type": "complex"},
+            "tags": {"data_type": "json"},
+            "custom_fields": {"data_type": "json"},
         },
     )
     def ticket_table(

{ingestr-0.9.4.dist-info → ingestr-0.10.0rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.9.4
+Version: 0.10.0rc0
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -13,35 +13,35 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
-Requires-Dist: confluent-kafka>=2.3.0
+Requires-Dist: confluent-kafka>=2.6.1
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==0.5.1
-Requires-Dist: duckdb-engine==0.11.5
-Requires-Dist: duckdb==0.10.2
+Requires-Dist: dlt==1.4.0
+Requires-Dist: duckdb-engine==0.13.5
+Requires-Dist: duckdb==1.1.3
 Requires-Dist: facebook-business==20.0.0
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
-Requires-Dist: mysql-connector-python==9.0.0
+Requires-Dist: mysql-connector-python==9.1.0
 Requires-Dist: pendulum==3.0.0
-Requires-Dist: psycopg2-binary==2.9.9
-Requires-Dist: py-machineid==0.5.1
+Requires-Dist: psycopg2-binary==2.9.10
+Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3
-Requires-Dist: pymongo==4.6.3
-Requires-Dist: pymysql==1.1.0
-Requires-Dist: pyrate-limiter==3.6.1
-Requires-Dist: redshift-connector==2.1.0
-Requires-Dist: rich==13.7.1
-Requires-Dist: rudder-sdk-python==2.1.0
+Requires-Dist: pymongo==4.10.1
+Requires-Dist: pymysql==1.1.1
+Requires-Dist: pyrate-limiter==3.7.0
+Requires-Dist: redshift-connector==2.1.3
+Requires-Dist: rich==13.9.4
+Requires-Dist: rudder-sdk-python==2.1.4
 Requires-Dist: s3fs==2024.9.0
-Requires-Dist: snowflake-sqlalchemy==1.5.3
-Requires-Dist: sqlalchemy-bigquery==1.11.0
+Requires-Dist: snowflake-sqlalchemy==1.6.1
+Requires-Dist: sqlalchemy-bigquery==1.12.0
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: stripe==10.7.0
-Requires-Dist: tqdm==4.66.2
-Requires-Dist: typer==0.12.3
+Requires-Dist: tqdm==4.67.0
+Requires-Dist: typer==0.13.1
 Requires-Dist: types-requests==2.32.0.20240907
 Provides-Extra: odbc
 Requires-Dist: pyodbc==5.1.0; extra == 'odbc'

{ingestr-0.9.4.dist-info → ingestr-0.10.0rc0.dist-info}/RECORD

@@ -1,10 +1,10 @@
-ingestr/main.py,sha256=B8TAQotJoYSvmaQQm33o2lv99OVLYNz-1Aw_fgQahwE,17718
+ingestr/main.py,sha256=O0FnyiWBjWu-Inl56EVtdHVWOggTdeEgjKbfp7zT1Kc,20100
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
 ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
-ingestr/src/sources.py,sha256=0eLrkd3oVsK9bWG7rp2Asu5QNrAUP2ZeMIIkduEuLL8,33756
+ingestr/src/sources.py,sha256=t_t4xK-bC1dBf7w1axcopuzIERzPHIHZ_QLiEb7BOhQ,33805
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=e56AvHfJCtG2ZwwINqsxINVbehWdKxMYgIDbjd7P-II,22
+ingestr/src/version.py,sha256=xTnym0lQwATiKdBCgZ51qs6XA-BvYKGOI8vHCckrPzk,26
 ingestr/src/adjust/__init__.py,sha256=oTM7XozDcMuUiCZ0w4gWEBXuCCtMZ0iBfkKdd2pVa1E,3007
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -26,7 +26,7 @@ ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
 ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
-ingestr/src/gorgias/__init__.py,sha256=BzX9X1Yc_1Mch6NP1pn26hjRIiaadErgHxkdJHw4P3o,21227
+ingestr/src/gorgias/__init__.py,sha256=LZ3m6aGuhLVI3eNjvQE0rT4o_wbSPkY_SDKsM-g0V5U,21176
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
 ingestr/src/hubspot/__init__.py,sha256=eSD_lEIEd16YijAtUATFG8FGO8YGPm-MtAk94KKsx6o,9740
 ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
@@ -43,24 +43,19 @@ ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMO
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
-ingestr/src/shopify/__init__.py,sha256=LWO5hfPfwPrq-gZwSRHThDfLHbGo2wkm6CrN1BCm3CQ,62815
+ingestr/src/shopify/__init__.py,sha256=Hhv84zRfVsqAGP7pz-PmeopeX9CGu7TXSm3PSXHEwIA,62689
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
 ingestr/src/shopify/helpers.py,sha256=NfHD6lWXe88ybR0ri-FCQuh2Vf8l5WG0a0FVjmdoSC4,6296
 ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
-ingestr/src/slack/__init__.py,sha256=UfUhkS6FnCKJeXkkJ5QrmdT5nZm5czjtomsQu_x9WUM,9987
+ingestr/src/slack/__init__.py,sha256=UF-ficQ6K32u1EHytW3P35suACo9wuc6nMrAPViyZL8,9981
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
-ingestr/src/sql_database/__init__.py,sha256=HEqY6U-YzzbeZ8avIthj-Fatm2C3i3jqYs5DAIAu4Ss,11511
-ingestr/src/sql_database/arrow_helpers.py,sha256=yze1X3A9nUQA4HeuFDDWrfJVkCq8Uo5UyDo_zhJtI60,5699
-ingestr/src/sql_database/helpers.py,sha256=6o8e2_8MIuj3qlo40a2E6ns3gyK18ei1jCePONrMUjI,10191
-ingestr/src/sql_database/override.py,sha256=xbKGDztCzvrhJ5kJTXERal3LA56bEeVug4_rrTs8DgA,333
-ingestr/src/sql_database/schema_types.py,sha256=qXTanvFPE8wMCSDzQWPDi5yqaO-llfrFXjiGJALI4NA,5013
 ingestr/src/stripe_analytics/__init__.py,sha256=8yy6i4DAhUqY4ZForetQ0DWc_YQrY0FBH6yk0Z3m-Mw,4493
 ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
 ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
-ingestr/src/zendesk/__init__.py,sha256=FjsDhzKaK7RgFs9zIGjMHHl8TlfYRN3TKocbGiJklD4,17570
+ingestr/src/zendesk/__init__.py,sha256=C7HkN195DGdOHId2_Sa_kAlcBrUmnVYZUa_tPkiyf1Q,17564
 ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
 ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
 ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
@@ -74,8 +69,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.9.4.dist-info/METADATA,sha256=mQKVpJ_Z7Ur3CLMbieKRmUrTEV0KIUe3MSeH2A1AUoE,7058
-ingestr-0.9.4.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-ingestr-0.9.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.9.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.9.4.dist-info/RECORD,,
+ingestr-0.10.0rc0.dist-info/METADATA,sha256=cRvsEF34onQqsvTFbiaU9w3mb6digQL1iLG4BOi3HfQ,7063
+ingestr-0.10.0rc0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.10.0rc0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.10.0rc0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.10.0rc0.dist-info/RECORD,,

ingestr/src/sql_database/__init__.py

@@ -1,206 +0,0 @@
-"""Source that loads tables form any SQLAlchemy supported database, supports batching requests and incremental loads."""
-
-from typing import Any, Callable, Dict, Iterable, List, Optional, Union
-
-import dlt
-from dlt.sources import DltResource
-from sqlalchemy import MetaData, Table
-from sqlalchemy.engine import Engine
-
-from .helpers import (
-    SqlDatabaseTableConfiguration,
-    SqlTableResourceConfiguration,
-    TableBackend,
-    _detect_precision_hints_deprecated,
-    engine_from_credentials,
-    table_rows,
-)
-from .override import IngestrConnectionStringCredentials as ConnectionStringCredentials
-from .schema_types import (
-    ReflectionLevel,
-    TTypeAdapter,
-    get_primary_key,
-    table_to_columns,
-)
-
-
-@dlt.source
-def sql_database(
-    credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value,
-    schema: Optional[str] = dlt.config.value,
-    metadata: Optional[MetaData] = None,
-    table_names: Optional[List[str]] = dlt.config.value,
-    chunk_size: int = 50000,
-    backend: TableBackend = "sqlalchemy",
-    detect_precision_hints: Optional[bool] = False,
-    reflection_level: Optional[ReflectionLevel] = "full",
-    defer_table_reflect: Optional[bool] = None,
-    table_adapter_callback: Callable[[Table], None] = None,
-    backend_kwargs: Dict[str, Any] = None,
-    include_views: bool = False,
-    type_adapter_callback: Optional[TTypeAdapter] = None,
-) -> Iterable[DltResource]:
-    """
-    A dlt source which loads data from an SQL database using SQLAlchemy.
-    Resources are automatically created for each table in the schema or from the given list of tables.
-
-    Args:
-        credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `sqlalchemy.Engine` instance.
-        schema (Optional[str]): Name of the database schema to load (if different from default).
-        metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. `schema` argument is ignored when this is used.
-        table_names (Optional[List[str]]): A list of table names to load. By default, all tables in the schema are loaded.
-        chunk_size (int): Number of rows yielded in one batch. SQL Alchemy will create additional internal rows buffer twice the chunk size.
-        backend (TableBackend): Type of backend to generate table data. One of: "sqlalchemy", "pyarrow", "pandas" and "connectorx".
-            "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames.
-            "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types,
-            "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself.
-        detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
-            This is disabled by default.
-        reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema.
-            "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data.
-            "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option.
-            "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types.
-        defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Requires table_names to be explicitly passed.
-            Enable this option when running on Airflow. Available on dlt 0.4.4 and later.
-        table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected.
-        backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx.
-        include_views (bool): Reflect views as well as tables. Note view names included in `table_names` are always included regardless of this setting.
-        type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns.
-            Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data)
-    Returns:
-
-        Iterable[DltResource]: A list of DLT resources for each table to be loaded.
-    """
-    # detect precision hints is deprecated
-    _detect_precision_hints_deprecated(detect_precision_hints)
-
-    if detect_precision_hints:
-        reflection_level = "full_with_precision"
-    else:
-        reflection_level = reflection_level or "minimal"
-
-    # set up alchemy engine
-    engine = engine_from_credentials(credentials)
-    engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size)
-    metadata = metadata or MetaData(schema=schema)
-
-    # use provided tables or all tables
-    if table_names:
-        tables = [
-            Table(name, metadata, autoload_with=None if defer_table_reflect else engine)
-            for name in table_names
-        ]
-    else:
-        if defer_table_reflect:
-            raise ValueError("You must pass table names to defer table reflection")
-        metadata.reflect(bind=engine, views=include_views)
-        tables = list(metadata.tables.values())
-
-    for table in tables:
-        if table_adapter_callback and not defer_table_reflect:
-            table_adapter_callback(table)
-
-        yield dlt.resource(
-            table_rows,
-            name=table.name,
-            primary_key=get_primary_key(table),
-            spec=SqlDatabaseTableConfiguration,
-            columns=table_to_columns(table, reflection_level, type_adapter_callback),
-        )(
-            engine,
-            table,
-            chunk_size,
-            backend,
-            reflection_level=reflection_level,
-            defer_table_reflect=defer_table_reflect,
-            table_adapter_callback=table_adapter_callback,
-            backend_kwargs=backend_kwargs,
-            type_adapter_callback=type_adapter_callback,
-        )
-
-
-@dlt.resource(
-    name=lambda args: args["table"], standalone=True, spec=SqlTableResourceConfiguration
-)
-def sql_table(
-    credentials: Union[ConnectionStringCredentials, Engine, str] = str,
-    table: str = dlt.config.value,
-    schema: Optional[str] = dlt.config.value,
-    metadata: Optional[MetaData] = None,
-    incremental: Optional[dlt.sources.incremental[Any]] = None,
-    chunk_size: int = 50000,
-    backend: TableBackend = "sqlalchemy",
-    detect_precision_hints: Optional[bool] = None,
-    reflection_level: Optional[ReflectionLevel] = "full_with_precision",
-    defer_table_reflect: Optional[bool] = None,
-    table_adapter_callback: Callable[[Table], None] = None,
-    backend_kwargs: Dict[str, Any] = None,
-    type_adapter_callback: Optional[TTypeAdapter] = None,
-    merge_key: Optional[str] = None,
-) -> DltResource:
-    """
-    A dlt resource which loads data from an SQL database table using SQLAlchemy.
-
-    Args:
-        credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `Engine` instance representing the database connection.
-        table (str): Name of the table or view to load.
-        schema (Optional[str]): Optional name of the schema the table belongs to.
-        metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. If provided, the `schema` argument is ignored.
-        incremental (Optional[dlt.sources.incremental[Any]]): Option to enable incremental loading for the table.
-            E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
-        chunk_size (int): Number of rows yielded in one batch. SQL Alchemy will create additional internal rows buffer twice the chunk size.
-        backend (TableBackend): Type of backend to generate table data. One of: "sqlalchemy", "pyarrow", "pandas" and "connectorx".
-            "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames.
-            "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types,
-            "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself.
-        reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema.
-            "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data.
-            "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option.
-            "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types.
-        detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
-            This is disabled by default.
-        defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Enable this option when running on Airflow. Available
-            on dlt 0.4.4 and later
-        table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected.
-        backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx.
-        type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns.
-            Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data)
-
-    Returns:
-        DltResource: The dlt resource for loading data from the SQL database table.
-    """
-    _detect_precision_hints_deprecated(detect_precision_hints)
-
-    if detect_precision_hints:
-        reflection_level = "full_with_precision"
-    else:
-        reflection_level = reflection_level or "minimal"
-
-    engine = engine_from_credentials(credentials)
-    engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size)
-    metadata = metadata or MetaData(schema=schema)
-
-    table_obj = Table(
-        table, metadata, autoload_with=None if defer_table_reflect else engine
-    )
-    if table_adapter_callback and not defer_table_reflect:
-        table_adapter_callback(table_obj)
-
-    return dlt.resource(
-        table_rows,
-        name=table_obj.name,
-        primary_key=get_primary_key(table_obj),
-        columns=table_to_columns(table_obj, reflection_level, type_adapter_callback),
-        merge_key=merge_key,
-    )(
-        engine,
-        table_obj,
-        chunk_size,
-        backend,
-        incremental=incremental,
-        reflection_level=reflection_level,
-        defer_table_reflect=defer_table_reflect,
-        table_adapter_callback=table_adapter_callback,
-        backend_kwargs=backend_kwargs,
-        type_adapter_callback=type_adapter_callback,
-    )

ingestr/src/sql_database/arrow_helpers.py

@@ -1,139 +0,0 @@
-from typing import Any, Optional, Sequence
-
-from dlt.common import logger
-from dlt.common.configuration import with_config
-from dlt.common.destination import DestinationCapabilitiesContext
-from dlt.common.json import custom_encode, map_nested_in_place
-from dlt.common.schema.typing import TTableSchemaColumns
-
-from .schema_types import RowAny
-
-
-@with_config
-def columns_to_arrow(
-    columns_schema: TTableSchemaColumns,
-    caps: DestinationCapabilitiesContext = None,
-    tz: str = "UTC",
-) -> Any:
-    """Converts `column_schema` to arrow schema using `caps` and `tz`. `caps` are injected from the container - which
-    is always the case if run within the pipeline. This will generate arrow schema compatible with the destination.
-    Otherwise generic capabilities are used
-    """
-    from dlt.common.destination.capabilities import DestinationCapabilitiesContext
-    from dlt.common.libs.pyarrow import get_py_arrow_datatype
-    from dlt.common.libs.pyarrow import pyarrow as pa
-
-    return pa.schema(
-        [
-            pa.field(
-                name,
-                get_py_arrow_datatype(
-                    schema_item,
-                    caps or DestinationCapabilitiesContext.generic_capabilities(),
-                    tz,
-                ),
-                nullable=schema_item.get("nullable", True),
-            )
-            for name, schema_item in columns_schema.items()
-            if schema_item.get("data_type") is not None
-        ]
-    )
-
-
-def row_tuples_to_arrow(
-    rows: Sequence[RowAny], columns: TTableSchemaColumns, tz: str
-) -> Any:
-    """Converts the rows to an arrow table using the columns schema.
-    Columns missing `data_type` will be inferred from the row data.
-    Columns with object types not supported by arrow are excluded from the resulting table.
-    """
-    import numpy as np
-    from dlt.common.libs.pyarrow import pyarrow as pa
-
-    try:
-        from pandas._libs import lib
-
-        pivoted_rows = lib.to_object_array_tuples(rows).T  # type: ignore[attr-defined]
-    except ImportError:
-        logger.info(
-            "Pandas not installed, reverting to numpy.asarray to create a table which is slower"
-        )
-        pivoted_rows = np.asarray(rows, dtype="object", order="k").T  # type: ignore[call-overload]
-
-    columnar = {
-        col: dat.ravel()
-        for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns)))
-    }
-    columnar_known_types = {
-        col["name"]: columnar[col["name"]]
-        for col in columns.values()
-        if col.get("data_type") is not None
-    }
-    columnar_unknown_types = {
-        col["name"]: columnar[col["name"]]
-        for col in columns.values()
-        if col.get("data_type") is None
-    }
-
-    arrow_schema = columns_to_arrow(columns, tz=tz)
-
-    for idx in range(0, len(arrow_schema.names)):
-        field = arrow_schema.field(idx)
-        py_type = type(rows[0][idx])
-        # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects
-        if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)):
-            logger.warning(
-                f"Field {field.name} was reflected as decimal type, but rows contains {py_type.__name__}. Additional cast is required which may slow down arrow table generation."
-            )
-            float_array = pa.array(columnar_known_types[field.name], type=pa.float64())
-            columnar_known_types[field.name] = float_array.cast(field.type, safe=False)
-
-    # If there are unknown type columns, first create a table to infer their types
-    if columnar_unknown_types:
-        new_schema_fields = []
-        for key in list(columnar_unknown_types):
-            arrow_col: Optional[pa.Array] = None
-            try:
-                arrow_col = pa.array(columnar_unknown_types[key])
-                if pa.types.is_null(arrow_col.type):
-                    logger.warning(
-                        f"Column {key} contains only NULL values and data type could not be inferred. This column is removed from a arrow table"
-                    )
-                    continue
-
-            except pa.ArrowInvalid as e:
-                # Try coercing types not supported by arrow to a json friendly format
-                # E.g. dataclasses -> dict, UUID -> str
-                try:
-                    arrow_col = pa.array(
-                        map_nested_in_place(
-                            custom_encode, list(columnar_unknown_types[key])
-                        )
-                    )
-                    logger.warning(
-                        f"Column {key} contains a data type which is not supported by pyarrow and got converted into {arrow_col.type}. This slows down arrow table generation."
-                    )
-                except (pa.ArrowInvalid, TypeError):
-                    logger.warning(
-                        f"Column {key} contains a data type which is not supported by pyarrow. This column will be ignored. Error: {e}"
-                    )
-            if arrow_col is not None:
-                columnar_known_types[key] = arrow_col
-                new_schema_fields.append(
-                    pa.field(
-                        key,
-                        arrow_col.type,
-                        nullable=columns[key]["nullable"],
-                    )
-                )
-
-        # New schema
-        column_order = {name: idx for idx, name in enumerate(columns)}
-        arrow_schema = pa.schema(
-            sorted(
-                list(arrow_schema) + new_schema_fields,
-                key=lambda x: column_order[x.name],
-            )
-        )
-
-    return pa.Table.from_pydict(columnar_known_types, schema=arrow_schema)