ingestr 0.6.6__tar.gz → 0.7.0__tar.gz

ingestr-0.7.0/.github/workflows/tests.yml

@@ -0,0 +1,36 @@
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: install Microsoft ODBC
+        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        id: cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
+      - name: Install pip dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make deps-ci
+      - name: run tests
+        run: make test-ci
+      - name: check the formatting
+        run: make lint-ci

{ingestr-0.6.6 → ingestr-0.7.0}/Makefile

@@ -11,6 +11,9 @@ venv/touchfile: requirements-dev.txt requirements.txt
 deps:
 	uv pip install -r requirements-dev.txt
 
+deps-ci:
+	pip install -r requirements-dev.txt
+
 test-ci:
 	pytest -rP -vv --tb=short --cov=ingestr --no-cov-on-fail
 
@@ -33,4 +36,4 @@ build:
 	rm -rf dist && python3 -m build
 
 upload-release:
-	twine upload dist/*
+	twine upload --verbose dist/*

{ingestr-0.6.6 → ingestr-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.6
+Version: 0.7.0
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,7 +16,7 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==0.4.12
+Requires-Dist: dlt==0.5.1
 Requires-Dist: duckdb-engine==0.11.5
 Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-api-python-client==2.130.0

{ingestr-0.6.6 → ingestr-0.7.0}/ingestr/main.py

@@ -90,6 +90,7 @@ class IncrementalStrategy(str, Enum):
     append = "append"
     delete_insert = "delete+insert"
     merge = "merge"
+    scd2 = "scd2"
     none = "none"
 
 
@@ -208,6 +209,20 @@ def ingest(
             envvar="LOADER_FILE_FORMAT",
         ),
     ] = None,  # type: ignore
+    page_size: Annotated[
+        Optional[int],
+        typer.Option(
+            help="The page size to be used when fetching data from SQL sources",
+            envvar="PAGE_SIZE",
+        ),
+    ] = 50000,  # type: ignore
+    loader_file_size: Annotated[
+        Optional[int],
+        typer.Option(
+            help="The file size to be used by the loader to split the data into multiple files. This can be set independent of the page size, since page size is used for fetching the data from the sources whereas this is used for the processing/loading part.",
+            envvar="LOADER_FILE_SIZE",
+        ),
+    ] = 100000,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -216,6 +231,10 @@ def ingest(
         },
     )
 
+    dlt.config["normalize.parquet_normalizer.add_dlt_load_id"] = True
+    dlt.config["normalize.parquet_normalizer.add_dlt_id"] = True
+    dlt.config["data_writer.file_max_items"] = loader_file_size
+
     try:
         if not dest_table:
             if len(source_table.split(".")) != 2:
@@ -255,7 +274,7 @@ def ingest(
             ),
             progress=progressInstance,
             pipelines_dir="pipeline_data",
-            full_refresh=full_refresh,
+            refresh="drop_resources" if full_refresh else None,
         )
 
         if source.handles_incrementality():
@@ -282,6 +301,9 @@ def ingest(
         print(
             f"[bold yellow]  Incremental Key:[/bold yellow] {incremental_key if incremental_key else 'None'}"
         )
+        print(
+            f"[bold yellow]  Primary Key:[/bold yellow] {primary_key if primary_key else 'None'}"
+        )
         print()
 
         if not yes:
@@ -304,6 +326,7 @@ def ingest(
             interval_start=interval_start,
             interval_end=interval_end,
             sql_backend=sql_backend.value,
+            page_size=page_size,
         )
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
@@ -322,15 +345,17 @@ def ingest(
             ):
                 loader_file_format = None
 
+        write_disposition = None
+        if incremental_strategy != IncrementalStrategy.none:
+            write_disposition = incremental_strategy.value
+
         run_info: LoadInfo = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(
                 uri=dest_uri,
                 table=dest_table,
             ),
-            write_disposition=incremental_strategy.value
-            if incremental_strategy.value != IncrementalStrategy.none
-            else None,  # type: ignore
+            write_disposition=write_disposition,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
             loader_file_format=loader_file_format.value
             if loader_file_format is not None

{ingestr-0.6.6 → ingestr-0.7.0}/ingestr/src/factory.py

@@ -33,6 +33,7 @@ SQL_SOURCE_SCHEMES = [
     "mysql+mysqlconnector",
     "postgres",
     "postgresql",
+    "postgresql+psycopg2",
     "redshift",
     "redshift+psycopg2",
     "snowflake",
@@ -112,6 +113,7 @@ class SourceDestinationFactory:
             "mssql": MsSQLDestination(),
             "postgres": PostgresDestination(),
             "postgresql": PostgresDestination(),
+            "postgresql+psycopg2": PostgresDestination(),
             "redshift": RedshiftDestination(),
             "redshift+psycopg2": RedshiftDestination(),
             "redshift+redshift_connector": RedshiftDestination(),

{ingestr-0.6.6 → ingestr-0.7.0}/ingestr/src/sources.py

@@ -49,6 +49,7 @@ class SqlSource:
             incremental=incremental,
             merge_key=kwargs.get("merge_key"),
             backend=kwargs.get("sql_backend", "sqlalchemy"),
+            chunk_size=kwargs.get("page_size", None),
         )
 
         return table_instance
@@ -122,7 +123,6 @@ class LocalCsvSource:
                                 f"incremental_key '{incremental_key}' not found in the CSV file"
                             )
 
-                        print("BURAYA GELLDIII")
                         if inc_value < incremental.start_value:
                             continue
 
@@ -183,8 +183,6 @@ class ShopifySource:
                 "Shopify takes care of incrementality on its own, you should not provide incremental_key"
             )
 
-        # shopify://shop_url?api_key=private_app_password
-
         source_fields = urlparse(uri)
         source_params = parse_qs(source_fields.query)
         api_key = source_params.get("api_key")

{ingestr-0.6.6 → ingestr-0.7.0}/ingestr/src/sql_database/__init__.py

@@ -3,7 +3,6 @@
 from typing import Any, Callable, Dict, Iterable, List, Optional, Union
 
 import dlt
-from dlt.common.configuration.specs.config_section_context import ConfigSectionContext
 from dlt.sources import DltResource
 from sqlalchemy import MetaData, Table
 from sqlalchemy.engine import Engine
@@ -12,35 +11,41 @@ from .helpers import (
     SqlDatabaseTableConfiguration,
     SqlTableResourceConfiguration,
     TableBackend,
+    _detect_precision_hints_deprecated,
     engine_from_credentials,
-    get_primary_key,
     table_rows,
 )
-from .override import IngestrConnectionStringCredentials
-from .schema_types import table_to_columns
+from .override import IngestrConnectionStringCredentials as ConnectionStringCredentials
+from .schema_types import (
+    ReflectionLevel,
+    TTypeAdapter,
+    get_primary_key,
+    table_to_columns,
+)
 
 
 @dlt.source
 def sql_database(
-    credentials: Union[
-        IngestrConnectionStringCredentials, Engine, str
-    ] = dlt.secrets.value,
+    credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value,
     schema: Optional[str] = dlt.config.value,
     metadata: Optional[MetaData] = None,
     table_names: Optional[List[str]] = dlt.config.value,
     chunk_size: int = 50000,
     backend: TableBackend = "sqlalchemy",
-    detect_precision_hints: Optional[bool] = dlt.config.value,
-    defer_table_reflect: Optional[bool] = dlt.config.value,
+    detect_precision_hints: Optional[bool] = False,
+    reflection_level: Optional[ReflectionLevel] = "full",
+    defer_table_reflect: Optional[bool] = None,
     table_adapter_callback: Callable[[Table], None] = None,
     backend_kwargs: Dict[str, Any] = None,
+    include_views: bool = False,
+    type_adapter_callback: Optional[TTypeAdapter] = None,
 ) -> Iterable[DltResource]:
     """
     A dlt source which loads data from an SQL database using SQLAlchemy.
     Resources are automatically created for each table in the schema or from the given list of tables.
 
     Args:
-        credentials (Union[IngestrConnectionStringCredentials, Engine, str]): Database credentials or an `sqlalchemy.Engine` instance.
+        credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `sqlalchemy.Engine` instance.
         schema (Optional[str]): Name of the database schema to load (if different from default).
         metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. `schema` argument is ignored when this is used.
         table_names (Optional[List[str]]): A list of table names to load. By default, all tables in the schema are loaded.
@@ -49,15 +54,30 @@ def sql_database(
             "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames.
             "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types,
             "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself.
-        detect_precision_hints (bool): Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
+        detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
             This is disabled by default.
+        reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema.
+            "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data.
+            "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option.
+            "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types.
         defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Requires table_names to be explicitly passed.
             Enable this option when running on Airflow. Available on dlt 0.4.4 and later.
         table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected.
         backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx.
+        include_views (bool): Reflect views as well as tables. Note view names included in `table_names` are always included regardless of this setting.
+        type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns.
+            Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data)
     Returns:
+
         Iterable[DltResource]: A list of DLT resources for each table to be loaded.
     """
+    # detect precision hints is deprecated
+    _detect_precision_hints_deprecated(detect_precision_hints)
+
+    if detect_precision_hints:
+        reflection_level = "full_with_precision"
+    else:
+        reflection_level = reflection_level or "minimal"
 
     # set up alchemy engine
     engine = engine_from_credentials(credentials)
@@ -73,49 +93,49 @@ def sql_database(
     else:
         if defer_table_reflect:
             raise ValueError("You must pass table names to defer table reflection")
-        metadata.reflect(bind=engine)
+        metadata.reflect(bind=engine, views=include_views)
         tables = list(metadata.tables.values())
 
     for table in tables:
         if table_adapter_callback and not defer_table_reflect:
             table_adapter_callback(table)
+
         yield dlt.resource(
             table_rows,
             name=table.name,
             primary_key=get_primary_key(table),
             spec=SqlDatabaseTableConfiguration,
-            columns=table_to_columns(table, detect_precision_hints),
+            columns=table_to_columns(table, reflection_level, type_adapter_callback),
         )(
             engine,
             table,
             chunk_size,
             backend,
-            detect_precision_hints=detect_precision_hints,
+            reflection_level=reflection_level,
             defer_table_reflect=defer_table_reflect,
             table_adapter_callback=table_adapter_callback,
             backend_kwargs=backend_kwargs,
+            type_adapter_callback=type_adapter_callback,
         )
 
 
-@dlt.sources.config.with_config(
-    sections=("sources", "sql_database"),
-    spec=SqlTableResourceConfiguration,
-    sections_merge_style=ConfigSectionContext.resource_merge_style,
+@dlt.resource(
+    name=lambda args: args["table"], standalone=True, spec=SqlTableResourceConfiguration
 )
 def sql_table(
-    credentials: Union[
-        IngestrConnectionStringCredentials, Engine, str
-    ] = dlt.secrets.value,
+    credentials: Union[ConnectionStringCredentials, Engine, str] = str,
     table: str = dlt.config.value,
     schema: Optional[str] = dlt.config.value,
     metadata: Optional[MetaData] = None,
     incremental: Optional[dlt.sources.incremental[Any]] = None,
-    chunk_size: int = 1000,
+    chunk_size: int = 50000,
     backend: TableBackend = "sqlalchemy",
-    detect_precision_hints: Optional[bool] = dlt.config.value,
-    defer_table_reflect: Optional[bool] = dlt.config.value,
+    detect_precision_hints: Optional[bool] = None,
+    reflection_level: Optional[ReflectionLevel] = "full_with_precision",
+    defer_table_reflect: Optional[bool] = None,
     table_adapter_callback: Callable[[Table], None] = None,
     backend_kwargs: Dict[str, Any] = None,
+    type_adapter_callback: Optional[TTypeAdapter] = None,
     merge_key: Optional[str] = None,
 ) -> DltResource:
     """
@@ -123,7 +143,7 @@ def sql_table(
 
     Args:
         credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `Engine` instance representing the database connection.
-        table (str): Name of the table to load.
+        table (str): Name of the table or view to load.
         schema (Optional[str]): Optional name of the schema the table belongs to.
         metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. If provided, the `schema` argument is ignored.
         incremental (Optional[dlt.sources.incremental[Any]]): Option to enable incremental loading for the table.
@@ -133,16 +153,29 @@ def sql_table(
             "sqlalchemy" yields batches as lists of Python dictionaries, "pyarrow" and "connectorx" yield batches as arrow tables, "pandas" yields panda frames.
             "sqlalchemy" is the default and does not require additional dependencies, "pyarrow" creates stable destination schemas with correct data types,
             "connectorx" is typically the fastest but ignores the "chunk_size" so you must deal with large tables yourself.
-        detect_precision_hints (bool): Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
+        reflection_level: (ReflectionLevel): Specifies how much information should be reflected from the source database schema.
+            "minimal": Only table names, nullability and primary keys are reflected. Data types are inferred from the data.
+            "full": Data types will be reflected on top of "minimal". `dlt` will coerce the data into reflected types if necessary. This is the default option.
+            "full_with_precision": Sets precision and scale on supported data types (ie. decimal, text, binary). Creates big and regular integer types.
+        detect_precision_hints (bool): Deprecated. Use `reflection_level`. Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
             This is disabled by default.
         defer_table_reflect (bool): Will connect and reflect table schema only when yielding data. Enable this option when running on Airflow. Available
             on dlt 0.4.4 and later
         table_adapter_callback: (Callable): Receives each reflected table. May be used to modify the list of columns that will be selected.
         backend_kwargs (**kwargs): kwargs passed to table backend ie. "conn" is used to pass specialized connection string to connectorx.
+        type_adapter_callback(Optional[Callable]): Callable to override type inference when reflecting columns.
+            Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data)
 
     Returns:
         DltResource: The dlt resource for loading data from the SQL database table.
     """
+    _detect_precision_hints_deprecated(detect_precision_hints)
+
+    if detect_precision_hints:
+        reflection_level = "full_with_precision"
+    else:
+        reflection_level = reflection_level or "minimal"
+
     engine = engine_from_credentials(credentials)
     engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size)
     metadata = metadata or MetaData(schema=schema)
@@ -157,7 +190,7 @@ def sql_table(
         table_rows,
         name=table_obj.name,
         primary_key=get_primary_key(table_obj),
-        columns=table_to_columns(table_obj, detect_precision_hints),
+        columns=table_to_columns(table_obj, reflection_level, type_adapter_callback),
         merge_key=merge_key,
     )(
         engine,
@@ -165,8 +198,9 @@ def sql_table(
         chunk_size,
         backend,
         incremental=incremental,
-        detect_precision_hints=detect_precision_hints,
+        reflection_level=reflection_level,
         defer_table_reflect=defer_table_reflect,
         table_adapter_callback=table_adapter_callback,
         backend_kwargs=backend_kwargs,
+        type_adapter_callback=type_adapter_callback,
     )

ingestr-0.7.0/ingestr/src/sql_database/arrow_helpers.py

@@ -0,0 +1,139 @@
+from typing import Any, Optional, Sequence
+
+from dlt.common import logger
+from dlt.common.configuration import with_config
+from dlt.common.destination import DestinationCapabilitiesContext
+from dlt.common.json import custom_encode, map_nested_in_place
+from dlt.common.schema.typing import TTableSchemaColumns
+
+from .schema_types import RowAny
+
+
+@with_config
+def columns_to_arrow(
+    columns_schema: TTableSchemaColumns,
+    caps: DestinationCapabilitiesContext = None,
+    tz: str = "UTC",
+) -> Any:
+    """Converts `column_schema` to arrow schema using `caps` and `tz`. `caps` are injected from the container - which
+    is always the case if run within the pipeline. This will generate arrow schema compatible with the destination.
+    Otherwise generic capabilities are used
+    """
+    from dlt.common.destination.capabilities import DestinationCapabilitiesContext
+    from dlt.common.libs.pyarrow import get_py_arrow_datatype
+    from dlt.common.libs.pyarrow import pyarrow as pa
+
+    return pa.schema(
+        [
+            pa.field(
+                name,
+                get_py_arrow_datatype(
+                    schema_item,
+                    caps or DestinationCapabilitiesContext.generic_capabilities(),
+                    tz,
+                ),
+                nullable=schema_item.get("nullable", True),
+            )
+            for name, schema_item in columns_schema.items()
+            if schema_item.get("data_type") is not None
+        ]
+    )
+
+
+def row_tuples_to_arrow(
+    rows: Sequence[RowAny], columns: TTableSchemaColumns, tz: str
+) -> Any:
+    """Converts the rows to an arrow table using the columns schema.
+    Columns missing `data_type` will be inferred from the row data.
+    Columns with object types not supported by arrow are excluded from the resulting table.
+    """
+    import numpy as np
+    from dlt.common.libs.pyarrow import pyarrow as pa
+
+    try:
+        from pandas._libs import lib
+
+        pivoted_rows = lib.to_object_array_tuples(rows).T  # type: ignore[attr-defined]
+    except ImportError:
+        logger.info(
+            "Pandas not installed, reverting to numpy.asarray to create a table which is slower"
+        )
+        pivoted_rows = np.asarray(rows, dtype="object", order="k").T  # type: ignore[call-overload]
+
+    columnar = {
+        col: dat.ravel()
+        for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns)))
+    }
+    columnar_known_types = {
+        col["name"]: columnar[col["name"]]
+        for col in columns.values()
+        if col.get("data_type") is not None
+    }
+    columnar_unknown_types = {
+        col["name"]: columnar[col["name"]]
+        for col in columns.values()
+        if col.get("data_type") is None
+    }
+
+    arrow_schema = columns_to_arrow(columns, tz=tz)
+
+    for idx in range(0, len(arrow_schema.names)):
+        field = arrow_schema.field(idx)
+        py_type = type(rows[0][idx])
+        # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects
+        if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)):
+            logger.warning(
+                f"Field {field.name} was reflected as decimal type, but rows contains {py_type.__name__}. Additional cast is required which may slow down arrow table generation."
+            )
+            float_array = pa.array(columnar_known_types[field.name], type=pa.float64())
+            columnar_known_types[field.name] = float_array.cast(field.type, safe=False)
+
+    # If there are unknown type columns, first create a table to infer their types
+    if columnar_unknown_types:
+        new_schema_fields = []
+        for key in list(columnar_unknown_types):
+            arrow_col: Optional[pa.Array] = None
+            try:
+                arrow_col = pa.array(columnar_unknown_types[key])
+                if pa.types.is_null(arrow_col.type):
+                    logger.warning(
+                        f"Column {key} contains only NULL values and data type could not be inferred. This column is removed from a arrow table"
+                    )
+                    continue
+
+            except pa.ArrowInvalid as e:
+                # Try coercing types not supported by arrow to a json friendly format
+                # E.g. dataclasses -> dict, UUID -> str
+                try:
+                    arrow_col = pa.array(
+                        map_nested_in_place(
+                            custom_encode, list(columnar_unknown_types[key])
+                        )
+                    )
+                    logger.warning(
+                        f"Column {key} contains a data type which is not supported by pyarrow and got converted into {arrow_col.type}. This slows down arrow table generation."
+                    )
+                except (pa.ArrowInvalid, TypeError):
+                    logger.warning(
+                        f"Column {key} contains a data type which is not supported by pyarrow. This column will be ignored. Error: {e}"
+                    )
+            if arrow_col is not None:
+                columnar_known_types[key] = arrow_col
+                new_schema_fields.append(
+                    pa.field(
+                        key,
+                        arrow_col.type,
+                        nullable=columns[key]["nullable"],
+                    )
+                )
+
+        # New schema
+        column_order = {name: idx for idx, name in enumerate(columns)}
+        arrow_schema = pa.schema(
+            sorted(
+                list(arrow_schema) + new_schema_fields,
+                key=lambda x: column_order[x.name],
+            )
+        )
+
+    return pa.Table.from_pydict(columnar_known_types, schema=arrow_schema)